<filename>scripts-history/posecam-async-udp-threads.py
from argparse import ArgumentParser
import sys
import cv2
import json
import numpy as np
from modules.myconst import *
from modules.draw import Plotter3d, draw_poses
import modules.myutils as mu
from modules.thread_video_streaming import *
from modules.thread_video_processing import *

import modules.UdpComms as U
import time

# python demo_unitycam.py --model human-pose-estimation-3d.xml --device CPU --use_openvino --height_size 256 --cam_id 2 --cam_width 640 --cam_height 480
# python demo_unitycam.py --model human-pose-estimation-3d.pth --height_size 256 --cam_id 2 --cam_width 640 --cam_height 480

if __name__ == '__main__':
    parser = ArgumentParser(description='Pose Camera Demo. '
                                        'Press esc to exit, "p" to (un)pause video or process next image.')
    parser.add_argument('--model',
                        help='Required. Path to checkpoint with a trained model '
                             '(or an .xml file in case of OpenVINO inference).',
                        type=str, required=True)
    parser.add_argument('--video', help='Optional. Path to video file.', type=str, default='')
    parser.add_argument('--cam_id', type=int, default=1)
    parser.add_argument('--cam_width', help='Optional. camera width.', type=int, default=640)
    parser.add_argument('--cam_height', help='Optional. camera height.', type=int, default=480)    
    parser.add_argument('--mirror_flip', help='Optional. Do mirror flipping on the image frame.',
                        action='store_true')
    parser.add_argument('--skip_frames', type=int, default=1, help='Optional. Skip frames to speed up the detection.')
    parser.add_argument('--show_frames', help='Optional. Show frames with processed poses.',
                        action='store_true')    
    parser.add_argument('--height_size', help='Optional. Network input layer height size.', type=int, default=256)
    parser.add_argument('--frame_scaling_option', help='Optional. Frame scaling option: 0--padding, 1--cropping, 2--frame shape.', type=int, default=0)    
    parser.add_argument('--device', help='Optional. The target device to infer on: CPU or GPU (default).',
                        type=str, default='GPU')
    parser.add_argument('--use_openvino',
                        help='Optional. Run network with OpenVINO as inference engine, support CPU, GPU, FPGA, HDDL or MYRIAD.',
                        action='store_true')
    parser.add_argument('--use_tensorrt', help='Optional. Run network with TensorRT as inference engine.',
                        action='store_true')
    parser.add_argument('--extrinsics_path',
                        help='Optional. Path to file with camera extrinsics.',
                        type=str, default=None)
    parser.add_argument('--fx', type=np.float32, default=-1, help='Optional. Camera focal length.')
    args = parser.parse_args()

    # Create UDP socket to use for sending (and receiving)
    sock = U.UdpComms(udpIP="127.0.0.1", portTX=8000, portRX=8001, enableRX=True, suppressWarnings=True)

    video_param = []
    pose_param = []

    while True:
        
        #sock.SendData('Sent from Python: ' + str(frame_count)) # Send this string to other application
        #data = sock.ReadReceivedData() # read data
        #if data != None: # if NEW data has been received since last ReadReceivedData function call
        #   print(data) # print new received data

        data = sock.ReadReceivedData() # read data
        if data == None:
            continue
        else:
            msg_from_client = data

        if msg_from_client == "UNITY_CLIENT_ALIVE":
            greeting = f"PY_SERVER_ALIVE"
            try:             
                sock.SendData(greeting) # Send this string to other application
                print(f"{TextColor.fg.red}-> {greeting}{TextColor.fg.black}")
            except:
                print(f"{TextColor.fg.red}-> In greeting, encountered {Exception}@@@ {TextColor.fg.black}")
                break

        elif (msg_from_client[0:10] == "{\"cam_id\":"):
            
            ucam_param = json.loads(msg_from_client)
            args.cam_id = ucam_param["cam_id"]
            args.cam_width = ucam_param["cam_width"]
            args.cam_height = ucam_param["cam_height"]            
            args.mirror_flip = ucam_param["mirror_flip"]            
            args.skip_frames = ucam_param["skip_frames"]
            args.height_size = ucam_param["height_size"]            

            video_param, pose_param = mu.check_input_args(args)

            ucam_args = json.dumps({
                "cam_id": args.cam_id,
                "cam_width": args.cam_width, 
                "cam_height": args.cam_height, 
                "mirror_flip": args.mirror_flip, 
                "skip_frames": args.skip_frames, 
                "height_size": args.height_size
                })
            try: 
                sock.SendData(ucam_args)
                print(f"{TextColor.fg.red}-> ucam_args sent... {ucam_args}{TextColor.fg.black}")                        
            except:
                print(f"{TextColor.fg.red}-> In sending ucam_args, encountered {Exception}@@@ {TextColor.fg.black}")
                break

        elif (msg_from_client == "UNITY_CLIENT_READY"):

            #setup the video capture stream with multiple threads
            if video_param.video_path != '':
                cap = ThreadVideoStreaming(video_param.video_path, video_param.mirror_flip).start()  
            else:
                cap = ThreadVideoStreaming(video_param.cam_id, video_param.mirror_flip, \
                    video_param.cam_width, video_param.cam_height).start()  
            frame_width = cap.width
            frame_height = cap.height

            proc = ThreadVideoProcessing(pose_param).start()  

            #setup 3D skeleton plot window
            if video_param.show_frames:
                #setup the canvas_3d window to show skeletons
                canvas_3d = np.zeros((frame_height, frame_width, 3), dtype=np.uint8)
                plotter = Plotter3d(canvas_3d.shape[:2])
                canvas_3d_window_name = 'PoseCam 3D'
                cv2.namedWindow(canvas_3d_window_name)
                cv2.setMouseCallback(canvas_3d_window_name, Plotter3d.mouse_callback)

            #prepare for loop rolling
            frame_width = cap.width
            frame_height = cap.height
            delay = 1
            frame_count = 0
            tic_start = cv2.getTickCount()    
            while (True):
                tic0 = cv2.getTickCount()                            
                grabbed, frame = cap.read()
                if not grabbed: 
                    if cap: cap.stop()
                    if proc: proc.stop()
                    break
                fps.video = cv2.getTickFrequency() / (cv2.getTickCount() - tic0)            

                tic1 = cv2.getTickCount()  
                posedata = proc.read()
                num_poses = len(posedata.poses_2d)                  
                if (len(posedata.poses_id) == num_poses and len(posedata.poses_id_last) == num_poses):
                    pid = np.minimum(posedata.poses_id, posedata.poses_id_last)
                else:
                    pid = np.linspace(0, num_poses, num_poses, dtype=np.int32)            
                fps.openpose = cv2.getTickFrequency() / (cv2.getTickCount() - tic1)                                        

                frame_count += 1
                fps.processing = frame_count / ((cv2.getTickCount() - tic_start) / cv2.getTickFrequency())
                #print(f"frame# {frame_count} || fps.video = {int(fps.video)} || fps.openpose = {int(fps.openpose)} || fps.processing = {int(fps.processing)}", end="\r")
                #total_elapsed_time = fps.t_scale + fps.t_infer + fps.t_parse + fps.t_calc3d + fps.t_json + fps.t_send       
                #print(f"elapsed time={total_elapsed_time:.4f}: scale={fps.t_scale:.4f}, infer={fps.t_infer:.4f}, parse={fps.t_parse:.4f}, calc3d={fps.t_calc3d:.4f}, json={fps.t_json:.4f}, send={fps.t_send:.4f}")

                if video_param.show_frames:
                    #update skeleton plot
                    edges = []
                    if num_poses > 0:
                        edges = (Plotter3d.SKELETON_EDGES + NUMKPTS_PANOPTIC * np.arange(posedata.poses_3d.shape[0]).reshape((-1, 1, 1))).reshape((-1, 2))
                        plotter.plot(canvas_3d, posedata.poses_3d, edges)
                        cv2.imshow(canvas_3d_window_name, canvas_3d)
                    #update pose plot
                    if num_poses > 0: draw_poses(frame, posedata.poses_2d)                
                    #update pose_id
                    if pose_param.is_video:
                        for pose_idx in range(num_poses):        
                            cv2.putText(frame, 'id:{0:3d}'.format(int(pid[pose_idx])), \
                                (max(int(posedata.poses_2d[pose_idx, 3]), 20), max(20,int(posedata.poses_2d[pose_idx, 4])-100)), \
                                cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0), 2)
                    #update frame rates
                    x0, y0, dx, dy = 20, 20, 0, 25
                    cv2.putText(frame, 'FPS-video: {0:d}'.format(int(fps.video)), (x0, y0), cv2.FONT_HERSHEY_COMPLEX, 0.6, (0, 0, 255), 1)
                    cv2.putText(frame, 'FPS-openpose: {0:d}'.format(int(fps.openpose)), (x0, y0+dy), cv2.FONT_HERSHEY_COMPLEX, 0.6, (0, 0, 255), 1)
                    cv2.putText(frame, 'FPS-processing: {0:d}'.format(int(fps.processing)), (x0, y0+2*dy), cv2.FONT_HERSHEY_COMPLEX, 0.6, (0, 0, 255), 1)        
                    #update pose view
                    cv2.imshow('PoseCam View', frame)
        
                key = cv2.waitKey(delay)
                if key == ESC_CODE: 
                    if cap: cap.stop()
                    if proc: proc.stop()             
                    break
                if key == P_CODE:
                    if delay == 1: delay = 0
                    else:          delay = 1
                if delay == 0 or not pose_param.is_video:  # allow to rotate 3D canvas while on pause
                    key = 0
                    while (key != P_CODE and key != ESC_CODE and key != SPACE_CODE):
                        plotter.plot(canvas_3d, posedata.poses_3d, edges)
                        cv2.imshow(canvas_3d_window_name, canvas_3d)
                        key = cv2.waitKey(33)
                    if key == ESC_CODE: 
                        if cap: cap.stop()
                        if proc: proc.stop()                             
                        break
                    else:               
                        delay = 1

                all_poses = []
                num_poses = len(posedata.poses_2d)
                tic = cv2.getTickCount()
                if (num_poses > 0):
                    kpt2d_pos, kpt2d_score, kpt_last = mu.get_keypoints2d(posedata.poses_2d)
                    kpt3d_pos = mu.get_keypoints3d(posedata.poses_3d)            
                    pid = np.minimum(posedata.poses_id, posedata.poses_id_last)

                    for pose_idx in range(num_poses):
                        ks = kpt2d_score[pose_idx * NUMKPTS_PANOPTIC : (pose_idx + 1) * NUMKPTS_PANOPTIC :1]
                        high_score_idx = np.where(ks > MIN_CONFIDENCE)
                        pose_keypoints = []
                        for kpt_idx in range(NUMKPTS_PANOPTIC):
                            jj = pose_idx * NUMKPTS_PANOPTIC + kpt_idx
                            this_keypoint = {
                                "part": KPTS_PANOPTIC_NAME[kpt_idx],
                                "sc": int(ks[kpt_idx]*100),
                                "pt2": { 
                                    "x": int(kpt2d_pos[jj, 0] - frame_width / 2), 
                                    "y": int(frame_height / 2 - kpt2d_pos[jj, 1]) 
                                    },
                                "pt3": { 
                                    "x": int(kpt3d_pos[jj, 0] - frame_width / 2),
                                    "y": int(frame_height / 2 - kpt3d_pos[jj, 1]),
                                    "z": int(kpt3d_pos[jj, 2])
                                    }
                            }
                            pose_keypoints.append(this_keypoint)
                        this_pose = {
                            "id": int(pid[pose_idx]),
                            "sc": int(100*min(1, np.multiply(KPTS_PANOPTIC_SCORE_WEIGHT[high_score_idx], ks[high_score_idx]).sum())),
                            "kpts": pose_keypoints
                            }
                        all_poses.append(this_pose)

                    pose_json = json.dumps({"poses": all_poses})
                    fps.t_json = (cv2.getTickCount() - tic) / cv2.getTickFrequency()

                    tic = cv2.getTickCount()
                    try: 
                        sock.SendData(pose_json)
                        print(f"{TextColor.fg.red}-> frame {frame_count} sent || fps.openpose = {int(fps.openpose)} || fps.processing = {int(fps.processing)} {TextColor.fg.black}") #, end="\r")                        
                    except:
                        print(f"{TextColor.fg.red}-> In sending frame {frame_count}, encountered {Exception}@@@ {TextColor.fg.black}")
                        break
                    fps.t_send = (cv2.getTickCount() - tic) / cv2.getTickFrequency()                    

                    #total_elapsed_time = fps.t_scale + fps.t_infer + fps.t_parse + fps.t_calc3d + fps.t_json + fps.t_send       
                    #print(f"elapsed time={total_elapsed_time:.4f}: scale={fps.t_scale:.4f}, infer={fps.t_infer:.4f}, parse={fps.t_parse:.4f}, calc3d={fps.t_calc3d:.4f}, json={fps.t_json:.4f}, send={fps.t_send:.4f}")

            # do a bit of cleanup
            if cap: cap.stop()
            if proc: proc.stop()
            if video_param.show_frames: cv2.destroyAllWindows()

        else:
            greeting = f"Echoing {msg_from_client}"
            try:             
                sock.SendData(greeting)
                print(f"{TextColor.fg.red}-> {greeting}{TextColor.fg.black}")
            except:
                print(f"{TextColor.fg.red}-> In echoing, encountered {Exception}@@@ {TextColor.fg.black}")
                break

    sys.exit()

<filename>scripts-history/posecam-async-udp-threads.py
from argparse import ArgumentParser
import sys
import cv2
import json
import numpy as np
from modules.myconst import *
from modules.draw import Plotter3d, draw_poses
import modules.myutils as mu
from modules.thread_video_streaming import *
from modules.thread_video_processing import *

import modules.UdpComms as U
import time

# python demo_unitycam.py --model human-pose-estimation-3d.xml --device CPU --use_openvino --height_size 256 --cam_id 2 --cam_width 640 --cam_height 480
# python demo_unitycam.py --model human-pose-estimation-3d.pth --height_size 256 --cam_id 2 --cam_width 640 --cam_height 480

if __name__ == '__main__':
    parser = ArgumentParser(description='Pose Camera Demo. '
                                        'Press esc to exit, "p" to (un)pause video or process next image.')
    parser.add_argument('--model',
                        help='Required. Path to checkpoint with a trained model '
                             '(or an .xml file in case of OpenVINO inference).',
                        type=str, required=True)
    parser.add_argument('--video', help='Optional. Path to video file.', type=str, default='')
    parser.add_argument('--cam_id', type=int, default=1)
    parser.add_argument('--cam_width', help='Optional. camera width.', type=int, default=640)
    parser.add_argument('--cam_height', help='Optional. camera height.', type=int, default=480)    
    parser.add_argument('--mirror_flip', help='Optional. Do mirror flipping on the image frame.',
                        action='store_true')
    parser.add_argument('--skip_frames', type=int, default=1, help='Optional. Skip frames to speed up the detection.')
    parser.add_argument('--show_frames', help='Optional. Show frames with processed poses.',
                        action='store_true')    
    parser.add_argument('--height_size', help='Optional. Network input layer height size.', type=int, default=256)
    parser.add_argument('--frame_scaling_option', help='Optional. Frame scaling option: 0--padding, 1--cropping, 2--frame shape.', type=int, default=0)    
    parser.add_argument('--device', help='Optional. The target device to infer on: CPU or GPU (default).',
                        type=str, default='GPU')
    parser.add_argument('--use_openvino',
                        help='Optional. Run network with OpenVINO as inference engine, support CPU, GPU, FPGA, HDDL or MYRIAD.',
                        action='store_true')
    parser.add_argument('--use_tensorrt', help='Optional. Run network with TensorRT as inference engine.',
                        action='store_true')
    parser.add_argument('--extrinsics_path',
                        help='Optional. Path to file with camera extrinsics.',
                        type=str, default=None)
    parser.add_argument('--fx', type=np.float32, default=-1, help='Optional. Camera focal length.')
    args = parser.parse_args()

    # Create UDP socket to use for sending (and receiving)
    sock = U.UdpComms(udpIP="127.0.0.1", portTX=8000, portRX=8001, enableRX=True, suppressWarnings=True)

    video_param = []
    pose_param = []

    while True:
        
        #sock.SendData('Sent from Python: ' + str(frame_count)) # Send this string to other application
        #data = sock.ReadReceivedData() # read data
        #if data != None: # if NEW data has been received since last ReadReceivedData function call
        #   print(data) # print new received data

        data = sock.ReadReceivedData() # read data
        if data == None:
            continue
        else:
            msg_from_client = data

        if msg_from_client == "UNITY_CLIENT_ALIVE":
            greeting = f"PY_SERVER_ALIVE"
            try:             
                sock.SendData(greeting) # Send this string to other application
                print(f"{TextColor.fg.red}-> {greeting}{TextColor.fg.black}")
            except:
                print(f"{TextColor.fg.red}-> In greeting, encountered {Exception}@@@ {TextColor.fg.black}")
                break

        elif (msg_from_client[0:10] == "{\"cam_id\":"):
            
            ucam_param = json.loads(msg_from_client)
            args.cam_id = ucam_param["cam_id"]
            args.cam_width = ucam_param["cam_width"]
            args.cam_height = ucam_param["cam_height"]            
            args.mirror_flip = ucam_param["mirror_flip"]            
            args.skip_frames = ucam_param["skip_frames"]
            args.height_size = ucam_param["height_size"]            

            video_param, pose_param = mu.check_input_args(args)

            ucam_args = json.dumps({
                "cam_id": args.cam_id,
                "cam_width": args.cam_width, 
                "cam_height": args.cam_height, 
                "mirror_flip": args.mirror_flip, 
                "skip_frames": args.skip_frames, 
                "height_size": args.height_size
                })
            try: 
                sock.SendData(ucam_args)
                print(f"{TextColor.fg.red}-> ucam_args sent... {ucam_args}{TextColor.fg.black}")                        
            except:
                print(f"{TextColor.fg.red}-> In sending ucam_args, encountered {Exception}@@@ {TextColor.fg.black}")
                break

        elif (msg_from_client == "UNITY_CLIENT_READY"):

            #setup the video capture stream with multiple threads
            if video_param.video_path != '':
                cap = ThreadVideoStreaming(video_param.video_path, video_param.mirror_flip).start()  
            else:
                cap = ThreadVideoStreaming(video_param.cam_id, video_param.mirror_flip, \
                    video_param.cam_width, video_param.cam_height).start()  
            frame_width = cap.width
            frame_height = cap.height

            proc = ThreadVideoProcessing(pose_param).start()  

            #setup 3D skeleton plot window
            if video_param.show_frames:
                #setup the canvas_3d window to show skeletons
                canvas_3d = np.zeros((frame_height, frame_width, 3), dtype=np.uint8)
                plotter = Plotter3d(canvas_3d.shape[:2])
                canvas_3d_window_name = 'PoseCam 3D'
                cv2.namedWindow(canvas_3d_window_name)
                cv2.setMouseCallback(canvas_3d_window_name, Plotter3d.mouse_callback)

            #prepare for loop rolling
            frame_width = cap.width
            frame_height = cap.height
            delay = 1
            frame_count = 0
            tic_start = cv2.getTickCount()    
            while (True):
                tic0 = cv2.getTickCount()                            
                grabbed, frame = cap.read()
                if not grabbed: 
                    if cap: cap.stop()
                    if proc: proc.stop()
                    break
                fps.video = cv2.getTickFrequency() / (cv2.getTickCount() - tic0)            

                tic1 = cv2.getTickCount()  
                posedata = proc.read()
                num_poses = len(posedata.poses_2d)                  
                if (len(posedata.poses_id) == num_poses and len(posedata.poses_id_last) == num_poses):
                    pid = np.minimum(posedata.poses_id, posedata.poses_id_last)
                else:
                    pid = np.linspace(0, num_poses, num_poses, dtype=np.int32)            
                fps.openpose = cv2.getTickFrequency() / (cv2.getTickCount() - tic1)                                        

                frame_count += 1
                fps.processing = frame_count / ((cv2.getTickCount() - tic_start) / cv2.getTickFrequency())
                #print(f"frame# {frame_count} || fps.video = {int(fps.video)} || fps.openpose = {int(fps.openpose)} || fps.processing = {int(fps.processing)}", end="\r")
                #total_elapsed_time = fps.t_scale + fps.t_infer + fps.t_parse + fps.t_calc3d + fps.t_json + fps.t_send       
                #print(f"elapsed time={total_elapsed_time:.4f}: scale={fps.t_scale:.4f}, infer={fps.t_infer:.4f}, parse={fps.t_parse:.4f}, calc3d={fps.t_calc3d:.4f}, json={fps.t_json:.4f}, send={fps.t_send:.4f}")

                if video_param.show_frames:
                    #update skeleton plot
                    edges = []
                    if num_poses > 0:
                        edges = (Plotter3d.SKELETON_EDGES + NUMKPTS_PANOPTIC * np.arange(posedata.poses_3d.shape[0]).reshape((-1, 1, 1))).reshape((-1, 2))
                        plotter.plot(canvas_3d, posedata.poses_3d, edges)
                        cv2.imshow(canvas_3d_window_name, canvas_3d)
                    #update pose plot
                    if num_poses > 0: draw_poses(frame, posedata.poses_2d)                
                    #update pose_id
                    if pose_param.is_video:
                        for pose_idx in range(num_poses):        
                            cv2.putText(frame, 'id:{0:3d}'.format(int(pid[pose_idx])), \
                                (max(int(posedata.poses_2d[pose_idx, 3]), 20), max(20,int(posedata.poses_2d[pose_idx, 4])-100)), \
                                cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0), 2)
                    #update frame rates
                    x0, y0, dx, dy = 20, 20, 0, 25
                    cv2.putText(frame, 'FPS-video: {0:d}'.format(int(fps.video)), (x0, y0), cv2.FONT_HERSHEY_COMPLEX, 0.6, (0, 0, 255), 1)
                    cv2.putText(frame, 'FPS-openpose: {0:d}'.format(int(fps.openpose)), (x0, y0+dy), cv2.FONT_HERSHEY_COMPLEX, 0.6, (0, 0, 255), 1)
                    cv2.putText(frame, 'FPS-processing: {0:d}'.format(int(fps.processing)), (x0, y0+2*dy), cv2.FONT_HERSHEY_COMPLEX, 0.6, (0, 0, 255), 1)        
                    #update pose view
                    cv2.imshow('PoseCam View', frame)
        
                key = cv2.waitKey(delay)
                if key == ESC_CODE: 
                    if cap: cap.stop()
                    if proc: proc.stop()             
                    break
                if key == P_CODE:
                    if delay == 1: delay = 0
                    else:          delay = 1
                if delay == 0 or not pose_param.is_video:  # allow to rotate 3D canvas while on pause
                    key = 0
                    while (key != P_CODE and key != ESC_CODE and key != SPACE_CODE):
                        plotter.plot(canvas_3d, posedata.poses_3d, edges)
                        cv2.imshow(canvas_3d_window_name, canvas_3d)
                        key = cv2.waitKey(33)
                    if key == ESC_CODE: 
                        if cap: cap.stop()
                        if proc: proc.stop()                             
                        break
                    else:               
                        delay = 1

                all_poses = []
                num_poses = len(posedata.poses_2d)
                tic = cv2.getTickCount()
                if (num_poses > 0):
                    kpt2d_pos, kpt2d_score, kpt_last = mu.get_keypoints2d(posedata.poses_2d)
                    kpt3d_pos = mu.get_keypoints3d(posedata.poses_3d)            
                    pid = np.minimum(posedata.poses_id, posedata.poses_id_last)

                    for pose_idx in range(num_poses):
                        ks = kpt2d_score[pose_idx * NUMKPTS_PANOPTIC : (pose_idx + 1) * NUMKPTS_PANOPTIC :1]
                        high_score_idx = np.where(ks > MIN_CONFIDENCE)
                        pose_keypoints = []
                        for kpt_idx in range(NUMKPTS_PANOPTIC):
                            jj = pose_idx * NUMKPTS_PANOPTIC + kpt_idx
                            this_keypoint = {
                                "part": KPTS_PANOPTIC_NAME[kpt_idx],
                                "sc": int(ks[kpt_idx]*100),
                                "pt2": { 
                                    "x": int(kpt2d_pos[jj, 0] - frame_width / 2), 
                                    "y": int(frame_height / 2 - kpt2d_pos[jj, 1]) 
                                    },
                                "pt3": { 
                                    "x": int(kpt3d_pos[jj, 0] - frame_width / 2),
                                    "y": int(frame_height / 2 - kpt3d_pos[jj, 1]),
                                    "z": int(kpt3d_pos[jj, 2])
                                    }
                            }
                            pose_keypoints.append(this_keypoint)
                        this_pose = {
                            "id": int(pid[pose_idx]),
                            "sc": int(100*min(1, np.multiply(KPTS_PANOPTIC_SCORE_WEIGHT[high_score_idx], ks[high_score_idx]).sum())),
                            "kpts": pose_keypoints
                            }
                        all_poses.append(this_pose)

                    pose_json = json.dumps({"poses": all_poses})
                    fps.t_json = (cv2.getTickCount() - tic) / cv2.getTickFrequency()

                    tic = cv2.getTickCount()
                    try: 
                        sock.SendData(pose_json)
                        print(f"{TextColor.fg.red}-> frame {frame_count} sent || fps.openpose = {int(fps.openpose)} || fps.processing = {int(fps.processing)} {TextColor.fg.black}") #, end="\r")                        
                    except:
                        print(f"{TextColor.fg.red}-> In sending frame {frame_count}, encountered {Exception}@@@ {TextColor.fg.black}")
                        break
                    fps.t_send = (cv2.getTickCount() - tic) / cv2.getTickFrequency()                    

                    #total_elapsed_time = fps.t_scale + fps.t_infer + fps.t_parse + fps.t_calc3d + fps.t_json + fps.t_send       
                    #print(f"elapsed time={total_elapsed_time:.4f}: scale={fps.t_scale:.4f}, infer={fps.t_infer:.4f}, parse={fps.t_parse:.4f}, calc3d={fps.t_calc3d:.4f}, json={fps.t_json:.4f}, send={fps.t_send:.4f}")

            # do a bit of cleanup
            if cap: cap.stop()
            if proc: proc.stop()
            if video_param.show_frames: cv2.destroyAllWindows()

        else:
            greeting = f"Echoing {msg_from_client}"
            try:             
                sock.SendData(greeting)
                print(f"{TextColor.fg.red}-> {greeting}{TextColor.fg.black}")
            except:
                print(f"{TextColor.fg.red}-> In echoing, encountered {Exception}@@@ {TextColor.fg.black}")
                break

    sys.exit()

# python demo_unitycam.py --model human-pose-estimation-3d.xml --device CPU --use_openvino --height_size 256 --cam_id 2 --cam_width 640 --cam_height 480 # python demo_unitycam.py --model human-pose-estimation-3d.pth --height_size 256 --cam_id 2 --cam_width 640 --cam_height 480 # Create UDP socket to use for sending (and receiving) #sock.SendData('Sent from Python: ' + str(frame_count)) # Send this string to other application #data = sock.ReadReceivedData() # read data #if data != None: # if NEW data has been received since last ReadReceivedData function call #   print(data) # print new received data # read data # Send this string to other application #setup the video capture stream with multiple threads #setup 3D skeleton plot window #setup the canvas_3d window to show skeletons #prepare for loop rolling #print(f"frame# {frame_count} || fps.video = {int(fps.video)} || fps.openpose = {int(fps.openpose)} || fps.processing = {int(fps.processing)}", end="\r") #total_elapsed_time = fps.t_scale + fps.t_infer + fps.t_parse + fps.t_calc3d + fps.t_json + fps.t_send #print(f"elapsed time={total_elapsed_time:.4f}: scale={fps.t_scale:.4f}, infer={fps.t_infer:.4f}, parse={fps.t_parse:.4f}, calc3d={fps.t_calc3d:.4f}, json={fps.t_json:.4f}, send={fps.t_send:.4f}") #update skeleton plot #update pose plot #update pose_id #update frame rates #update pose view # allow to rotate 3D canvas while on pause #, end="\r") #total_elapsed_time = fps.t_scale + fps.t_infer + fps.t_parse + fps.t_calc3d + fps.t_json + fps.t_send #print(f"elapsed time={total_elapsed_time:.4f}: scale={fps.t_scale:.4f}, infer={fps.t_infer:.4f}, parse={fps.t_parse:.4f}, calc3d={fps.t_calc3d:.4f}, json={fps.t_json:.4f}, send={fps.t_send:.4f}") # do a bit of cleanup