title: (UDP socket with packet loss management, cv2, Python) echo camera image, stop working after some time?
body: <p>I haven't found any streaming library to echo a camera image with a packet loss management, so I'm trying to make one.</p>
  <p>The strategy is : dividing the image into segments of 2^16 (maximum size that a UDP socket can send), keep on sending one segment until the other side (client or server) sends the right informations, I think the issue is here, the buffer gets overcrowded and stop receiving. Maybe it is possible to empty that buffer or accept multiple income segments....</p>
  <p>The header used is :</p>
  <ul>
  <li>total length of the picture (bytes),</li>
  <li>number of the segment to send,</li>
  <li>total number of segments,</li>
  <li>the id of the image.</li>
  </ul>
  <p>It works well on local, but stop working after some time when using a server address like one of google platform.</p>
  <p>Class :</p>
  <pre><code>import socket
  #import cv2
  import time
  import struct
  import numpy as np
  import math
  import select
  
  class Link:
      def __init__(self, server_or_client, connection_type, addr_server, port_udp_data, 
                  port_tcp_check, encoding_key=&quot;&quot;, reverse=True):
          &quot;&quot;&quot;
              server_or_client (str): &quot;server&quot;, or &quot;client&quot;
  
              connection_type (str): &quot;UDP&quot;, &quot;TCP&quot;, &quot;Bluetooth&quot;
                  only UDP is implemented for now
              
              addr_server (str): &quot;192.68.0.98&quot; for example
                      addr_server = &quot;0.0.0.0&quot;
                      addr_server = &quot;127.0.0.1&quot;
  
              port_udp_data (int): 6000 - 65000
                      port_server = 65432
  
              port_tcp_check (int)
  
              encoding_key (str): to encode your link, not implemented
          
          &quot;&quot;&quot;
          
          ### Connection to the other system
          if server_or_client == &quot;server&quot;:
              print(&quot;Waiting for connection....&quot;)
  
              ### Data UDP socket
              self.s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
              self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # to avoid socket already in use avec ctrl + C
              self.s.bind((addr_server, port_udp_data))
              d, self.addr_UDP = self.s.recvfrom(1024) # get the address of the client
              print(&quot;data : &quot;, d, &quot; from : &quot;, self.addr_UDP)
              self.s.sendto(b'ok !', self.addr_UDP)
  
              ### Check TCP socket
              self.s_tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
              self.s_tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
              self.s_tcp.bind((addr_server, port_tcp_check))
              self.s_tcp.listen(1)
              self.s_check, self.addr_TCP = self.s_tcp.accept()
              
              print(&quot;connected! conn_check = &quot;, self.s_check)
              #self.s_check.sendall(b&quot;cucu&quot;)
          
          elif server_or_client == &quot;client&quot;:
              print(&quot;Connecting....&quot;)
              ### Data UDP socket
              self.s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
              self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
              self.addr_UDP = (addr_server, port_udp_data)
              self.s.sendto(b'c mwa!', self.addr_UDP)
              ready1 = select.select([self.s], [], [])
              if ready1[0]:
                  dat = self.s.recvfrom(1024) # address is the same as self.addr_UDP for client
              else:
                  print(&quot;bug&quot;)
              print(dat)
  
              ### Check TCP socket
              self.addr_TCP = (addr_server, port_tcp_check)
              self.s_check = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
              self.s_check.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
              self.s_check.connect(self.addr_TCP)
              print(&quot;connected!&quot;)
          
          self.max_length_r = 2**16
          self.max_length_s = 2**16 - 2**7
          self.count_img = -1
          self.img = b''
          self.dat_length_actual = -1
          self.num_seg_new = -1
          self.id_img_old = -1
          self.t1 = time.time()
          self.struct_types = {int: &quot;Q&quot;, float: &quot;d&quot;}
          self.id_img = 0
          self.tutuyutu = 0
          self.id_img_actual = -2
  
      def send(self, elements, image, n_tries=10, t_wait=1):
          &quot;&quot;&quot;
              elements (list): float elements that will be converted
                  into double 8 bytes, struct.pack(&quot;d&quot;, elements[0])
              
              image (bytes): image to be sent.
  
              n_tries (int): if no check message are sent back from
                  the receiver, the program will try again. After 
                  n_tries, the program sends b&quot;tutuyutu&quot; and returns
                  &quot;break&quot;.
  
              t_wait (int): seconds to wait when the program is waiting for
                  the s_check to be readable.
  
              This function is gonna divide the elements to be sent into
              segments, then send one, check its reception, then send another,
              check its reception, and so on ...
          
          &quot;&quot;&quot;
          print(&quot;sending.... &quot;)
          elements_size, dat = self.create_packet(elements, image)
  
          t1 = time.time()
  
          self.id_img += 1
  
          #frame = cv2.imread(&quot;./frame.jpg&quot;)
          #compress_img = cv2.imencode('.jpg', frame)[1]
          #dat = compress_img.tostring()
          #print(type(dat))
  
          #size = len(dat)
          size = len(image)
          print(&quot;len(dat) : &quot;, len(dat))
          #print(size)
          packets = math.ceil(size/self.max_length_s)
  
          left = 0
          right = self.max_length_s
  
          i = 0
          tries = 0
  
          #print(packets)
          while i &lt; packets:
              #print(i+1, &quot;/&quot;, packets)
              print(&quot;size : &quot;, size)
              print(&quot;left : &quot;, left, &quot; right : &quot;, right)
              print(&quot;len(dat)[left:right] : &quot;, len(dat[left:right]))
              packet = struct.pack(&quot;Q&quot;, size) + struct.pack(&quot;Q&quot;, i) + struct.pack(&quot;Q&quot;, packets-1) + struct.pack(&quot;Q&quot;, self.id_img) + struct.pack(&quot;Q&quot;, len(elements)) + dat[left:right]
              #print(&quot;len(packet) : &quot;, len(packet))
              #print(&quot;sleeping....&quot;)
              #time.sleep(0.05)
              #print(&quot;sending to : &quot;, (addr_server, port))
  
              #print(&quot;id_img : &quot;, id_img)
              print(&quot;##### Sending!&quot;)
              print(&quot;len(packet) : &quot;, len(packet))
              self.s.sendto(packet, self.addr_UDP)
  
              print(&quot;waiting for s_check to be readable....&quot;)
              ready = select.select([self.s_check], [], [], t_wait) # 0.0000001
              print(&quot;ready : &quot;, ready)
              if ready[0]:
                  check = self.s_check.recv(1024)
                  ids = []
                  num_segs = []
                  print(&quot;check msgs length = &quot;, len(check)/16)
                  print(&quot;len(check) : &quot;, len(check))
                  for k in range(int(len(check)/16)): # 2 octes per check msg (id and num_seg=i)
                      beg_id = k*8
                      end_id = (k+1)*8
                      beg_num_seg = (k+1)*8
                      end_num_seg = (k+2)*8
                      info_ids_r = check[beg_id:end_id]
                      info_num_segs = check[beg_num_seg:end_num_seg]
                      ids.append(struct.unpack(&quot;Q&quot;, info_ids_r)[0])
                      num_segs.append(struct.unpack(&quot;Q&quot;, info_num_segs)[0])
                      if ids[k] == self.id_img and num_segs[k] == i:
                          i += 1
                          left = right
                          right += self.max_length_s
                          tries = 0
              else:
                  print(&quot;Trying to send again ! id : &quot;, self.id_img, &quot;len(dat[left:right]) : &quot;, len(dat[left:right]), &quot; size : &quot;, size)                
                  tries += 1
                  tries_old = tries
                  print(&quot;############ Tries = &quot;, tries)
                  if tries == n_tries:
                      print(&quot;########### &quot;, n_tries, &quot; tries !&quot;)
                      packet = b'tutuyutu'
                      self.tutuyutu += 1
                      print(&quot;len(packet) : &quot;, len(packet))
                      self.s.sendto(packet, self.addr_UDP)
                      #i = packets
                      break_man = &quot;break&quot;
                      #if tutuyutu == 10:
  
                      print(&quot;break&quot;)
                      return(&quot;break&quot;)
          return &quot;Sent! &lt;3&quot;
              #break
  
          #if break_man == &quot;break&quot;:
          #    break_man = &quot;yoyo&quot;
          #    return(&quot;break&quot;)
              
          &quot;&quot;&quot;
          #if packet == b'tutuyutu':
          #    img_sent = False
          #else:
          #    img_sent = True
          #img_received = True
          img_received = False
          print(&quot;##### IMAGE SENT!&quot;)
          text = str(int(1/(time.time() - t1))) + &quot; fps&quot; + str(tries_old) + &quot; tries&quot; + str(tutuyutu) + &quot; tutuyutu&quot;
          tries_old = 0
          frame = put_text(frame, text)
          #cv2.imshow(&quot;frame&quot;, frame)
          cv2.waitKey(1)
          #break
          &quot;&quot;&quot;
      
      def receive(self, elements_types):
          &quot;&quot;&quot;
              elements_types (list): list of types to be received.
              
              Use packet loss management and receive each segment.
              When everything is received, it returns back 
              received = True, elements, the image.
  
              return : (True, self.elements, self.img_received) when
              image is received, or (False, data1, data2) if not, 
              with data1 and data2 being 0, or &quot;tutuyutu&quot;, or
              [dat_length, num_seg, num_seg_tot, id_img].
          
          &quot;&quot;&quot;
          self.elements_types = elements_types
          self.received = False
          data = None
  
          print(&quot;##### receiving....&quot;)
          ready1 = select.select([self.s], [], [])
          if ready1[0]:
              data, self.addr_UDP = self.s.recvfrom(self.max_length_r)
          else:
              print(&quot;bug&quot;)
              return(&quot;break&quot;)
          print(&quot;received ! &quot;, len(data))
          
          if data == b'tutuyutu':
              print(&quot;Tutuyutu&quot;)
              self.count_img = -1
              self.img = b''
              return(False, 0, &quot;tutuyutu&quot;)
          else:
              # Get all the elements like this
              info_dat_length = data[0:8]
              info_num_seg = data[8:16]
              info_num_seg_tot = data[16:24] # nbr of the image's segment
              info_id_image = data[24:32]      # id of the image
              info_nbr_elements = data[32:40]
              data_img = data[40:]
              
  
              dat_length = struct.unpack(self.struct_types[int], info_dat_length)[0]
              num_seg = struct.unpack(self.struct_types[int], info_num_seg)[0]
              num_seg_tot = struct.unpack(self.struct_types[int], info_num_seg_tot)[0]
              id_img = struct.unpack(self.struct_types[int], info_id_image)[0]
              print(&quot;- - - - - - - - - - -&quot;)
              print(&quot;dat_length : &quot;, dat_length)
              print(&quot;num_seg : &quot;, num_seg)
              print(&quot;num_seg_tot : &quot;, num_seg_tot)
              print(&quot;id_img : &quot;, id_img)
              print(&quot;- - - - - - - - - - -&quot;)
              number_elements = struct.unpack(self.struct_types[int], info_nbr_elements)[0]
              
              if number_elements != len(elements_types):
                  return &quot;False&quot;, 0, 0
              
  
              self.num_seg_new = num_seg
              id_img_new = id_img
  
              print(&quot;id_img_old : &quot;, self.id_img_old, &quot;id_img_new : &quot;, id_img_new)
              if self.id_img_old == id_img_new and self.count_img == -1: # to implement at the end, 
                  print(&quot;##### id_img_old == id_img_new and count_img == -1&quot;)
                  pass                                     # add id_img_old when image is complete
              elif (self.id_img_old &lt; id_img_new or (id_img_new == 0 and self.id_img_old == 99)) and self.count_img == -1 and len(elements_types) == number_elements:
                  print(&quot;##### id_img_old &lt; id_img_new and count_img == -1&quot;)
  
                  # Unpack elements
                  self.elements, data_img = self.unpack_packet(data_img, elements_types, number_elements)
  
                  # Start getting the image
                  self.img_L = [&quot;&quot; for i in range(num_seg_tot+1)]
                  self.img_L[num_seg] = data_img
                  print(&quot;waiting for select to be writable....&quot;)
                  ready = select.select([], [self.s_check], [])
                  if ready[1]:
                      msg = struct.pack(&quot;Q&quot;, id_img) + struct.pack(&quot;Q&quot;, num_seg)
                      self.s_check.sendall(msg)
                  print(&quot;Sent check msg to conn_check!&quot;)
                  self.count_img += 1
                  self.id_img_actual = id_img
                  self.dat_length_actual = dat_length
                  self.nums_img_taken = []
                  self.nums_img_taken.append(num_seg)
              elif id_img != self.id_img_actual and self.count_img &gt; -1:
                  print(&quot;##### id_img != id_img_actual and count_img &gt; -1&quot;)
                  pass
              elif id_img == self.id_img_actual and num_seg in self.nums_img_taken and self.count_img &gt; -1:
                  print(&quot;##### id_img == id_img_actual and num_seg in nums_img_taken and count_img &gt; -1&quot;)
                  if self.count_img == -1:
                      print(&quot;PROBLEEEMES&quot;)
                      print(&quot;id_img == id_img_actual : &quot;, id_img == self.id_img_actual,
                       &quot; num_seg in nums_img_taken : &quot;, num_seg in self.nums_img_taken)
                      return(&quot;break&quot;)
                  pass
              elif id_img == self.id_img_actual and num_seg not in self.nums_img_taken and self.count_img &gt; -1:
                  print(&quot;##### id_img == id_img_actual and num_seg not in nums_img_taken and count_img &gt; -1&quot;)
                  self.img_L[num_seg] = data_img
                  self.count_img += 1
                  self.nums_img_taken.append(num_seg)
                  print(&quot;waiting for select to be writable....&quot;)
                  ready = select.select([], [self.s_check], [])
                  if ready[1]:
                      msg = struct.pack(&quot;Q&quot;, id_img) + struct.pack(&quot;Q&quot;, num_seg)
                      self.s_check.sendall(msg)
                  print(&quot;Sent check msg to conn_check!&quot;)
              
              print(&quot;count_img : &quot;, self.count_img, &quot; num_seg_tot : &quot;, num_seg_tot)
              if self.count_img == num_seg_tot:
                  print(&quot;##### count_img == num_seg_tot&quot;)
                  self.img = b&quot;&quot;.join(self.img_L)
                  print(&quot;len(self.img) : &quot;, len(self.img), &quot; self.dat_length_actual : &quot;, self.dat_length_actual)
                  print(&quot;len(img) == dat_length_actual is : &quot;, len(self.img) == self.dat_length_actual)
                  #if not len(self.img) == self.dat_length_actual:
                  #    return(&quot;break&quot;)
                  self.count_img = -1
              
              print(&quot;len(self.img) == self.dat_length_actual : &quot;, len(self.img), &quot;, &quot;, self.dat_length_actual)
              if len(self.img) == self.dat_length_actual or (len(self.img) &gt; 0 and len(self.img) &lt; 30) :
                  print(&quot;##### len(img) == dat_length_actual&quot;)
                  self.img_received = self.img
                  self.img = b&quot;&quot;
                  
  
                  #&quot;&quot;&quot;
                  
                  #print(&quot;len(img) : &quot;, len(img))
                  #img = cv2.imdecode(np.frombuffer(self.img, dtype=np.uint8), 1)
                  #cv2.imwrite(&quot;./frame.jpg&quot;, img)
                  #self.received = True
                  #cv2.imshow(&quot;image&quot;, img)
                  #cv2.waitKey(1)
                  #self.img = b''
  
                  print(&quot;##### IMAGE WELL RECEIVED!&quot;)
                  #self.received = True
                  self.id_img_old = self.id_img_actual
                  #t2 = time.time()
                  #print(&quot;########################################### fps : &quot;, int(1/(t2 - self.t1)))
                  #t1 = time.time()
                  #break
                  #&quot;&quot;&quot;
                  return True, self.elements, self.img_received
                  
              else:
                  
                  return(False, [dat_length, num_seg, num_seg_tot, id_img], 0)
      
      def create_packet(self, elements, image):
          &quot;&quot;&quot;
              Create the packet before send() sends it by segment
              (part of the packet).
          
          &quot;&quot;&quot;
          #packet = struct.pack(self.struct_types[int], len(elements))
          packet = b''
          types = [type(element) for element in elements]
          #str_types = []
          #for t in types:
          #    if t == int:
          #        str_types.append(&quot;int&quot;)
          #    elif t == float:
          #        str_types.append(&quot;float&quot;)
          #print(&quot;str_types : &quot;, str_types, &quot; types : &quot; , types)
          #print(types, &quot; &quot;, elements, &quot; &quot;, self.struct_types)
          for i in range(len(types)):
              print(&quot;self.struct_types[types[i]] : &quot;, self.struct_types[types[i]])
              packet += struct.pack(self.struct_types[types[i]], elements[i])
          elements_size = len(packet)
          packet += image
          return elements_size, packet
  
      def unpack_packet(self, packet, elements_types, number_elements):
          print(&quot;elements_types : &quot;, elements_types)
          elements = []
          print(&quot;len(packet)&quot;, len(packet))
          
          print(&quot;number_elements : &quot;, number_elements)
          for i in range(number_elements):
              print(&quot;self.elements_types : &quot;, elements_types, &quot; i : &quot;, i)
              #print(&quot;self.struct_types : &quot;, self.struct_types, &quot; self.elements_types[i] : &quot;, self.elements_types[i])
              #print(&quot;self.struct_types[self.elements_types[i]] : &quot;, self.struct_types[self.elements_types[i]])
              value = struct.unpack(self.struct_types[elements_types[i]], packet[8*i:8*(i+1)])[0]
              elements.append(value)
          binary_data = packet[8*number_elements:]
          print(&quot;elements are : &quot;, elements)
          return elements, binary_data
  
  </code></pre>
  <p>The demo code :
  client.py</p>
  <pre><code>import Link # Need to add the current Link file into the Link_demo directory
  import cv2
  import numpy as np
  
  #addr_server = &quot;0.0.0.0&quot;
  addr_server = &quot;127.0.0.1&quot;
  
  link = Link.Link(&quot;client&quot;, &quot;UDP&quot;, addr_server, 65432, 65431)
  
  cam = cv2.VideoCapture(0)
  
  sent = False
  elements_types = []
  
  while True:
      if not sent:
          _, frame = cam.read()
          compress_img = cv2.imencode('.jpg', frame)[1]
          data = compress_img.tostring()
          stat = &quot;break&quot;
          while stat == &quot;break&quot;:
              stat = link.send([], data)
          sent = True
  
      
      dat = link.receive(elements_types)
      if dat[0] == True:
          image = dat[2]
          image = cv2.imdecode(np.frombuffer(image, dtype=np.uint8), 1)
          cv2.imshow(&quot;image&quot;, image)
          cv2.waitKey(1)
          sent = False
  </code></pre>
  <p>Server.py</p>
  <pre><code>import Link # Need to add the current Link file into the Link_demo directory
  import cv2
  import numpy as np
  
  #addr_server = &quot;127.0.0.1&quot;
  addr_server = &quot;0.0.0.0&quot;
  link = Link.Link(&quot;server&quot;, &quot;UDP&quot;, addr_server, 65432, 65431)
  dat = [False, 0, 0]
  
  received = False
  elements_types = []
  
  while True:
      dat = link.receive(elements_types)
      if dat[0] == True:
          image = dat[2]
          image = cv2.imdecode(np.frombuffer(image, dtype=np.uint8), 1)
          cv2.imwrite(&quot;./frame.jpg&quot;, image)
          #image = cv2.imdecode(np.frombuffer(image, dtype=np.uint8), 1)
          #cv2.imshow(&quot;image&quot;, image)
          #cv2.waitKey(1)
          received = True
      
      if received :
          #print(&quot;type(dat) : &quot;, type(dat))
          frame = cv2.imread(&quot;./frame.jpg&quot;)
          compress_img = cv2.imencode('.jpg', frame)[1]
          dat = compress_img.tostring()
          stat = &quot;break&quot;
          while stat == &quot;break&quot;:
              stat = link.send([], dat)
          received = False
  </code></pre>