title: Synchronizing audio and video with OpenCV and PyAudio
body: <p>I have gotten both OpenCV and PyAudio working however I am not sure how I would sync them together. I am unable to get a framerate from OpenCV and measuring the call time for a frame changes from moment to moment. However with PyAudio it's basis is grabbing a certain sample rate. How would I sync them to be at the same rate. I assume there is some standard or some way codecs do it. (I've tried google all I got was information on lip syncing :/). </p>
  
  <p>OpenCV Frame rate</p>
  
  <pre><code>from __future__ import division
  import time
  import math
  import cv2, cv
  
  vc = cv2.VideoCapture(0)
  # get the frame
  while True:
  
      before_read = time.time()
      rval, frame = vc.read()
      after_read  = time.time()
      if frame is not None:
          print len(frame)
          print math.ceil((1.0 / (after_read - before_read)))
          cv2.imshow("preview", frame)
  
          if cv2.waitKey(1) &amp; 0xFF == ord('q'):
              break
  
      else:
          print "None..."
          cv2.waitKey(1)
  
  # display the frame
  
  while True:
      cv2.imshow("preview", frame)
  
      if cv2.waitKey(1) &amp; 0xFF == ord('q'):
          break
  </code></pre>
  
  <p>Grabbing and saving audio</p>
  
  <pre><code>from sys import byteorder
  from array import array
  from struct import pack
  
  import pyaudio
  import wave
  
  THRESHOLD = 500
  CHUNK_SIZE = 1024
  FORMAT = pyaudio.paInt16
  RATE = 44100
  
  def is_silent(snd_data):
      "Returns 'True' if below the 'silent' threshold"
      print "\n\n\n\n\n\n\n\n"
      print max(snd_data)
      print "\n\n\n\n\n\n\n\n"
      return max(snd_data) &lt; THRESHOLD
  
  def normalize(snd_data):
      "Average the volume out"
      MAXIMUM = 16384
      times = float(MAXIMUM)/max(abs(i) for i in snd_data)
  
      r = array('h')
      for i in snd_data:
          r.append(int(i*times))
      return r
  
  def trim(snd_data):
      "Trim the blank spots at the start and end"
      def _trim(snd_data):
          snd_started = False
          r = array('h')
  
          for i in snd_data:
              if not snd_started and abs(i)&gt;THRESHOLD:
                  snd_started = True
                  r.append(i)
  
              elif snd_started:
                  r.append(i)
          return r
  
      # Trim to the left
      snd_data = _trim(snd_data)
  
      # Trim to the right
      snd_data.reverse()
      snd_data = _trim(snd_data)
      snd_data.reverse()
      return snd_data
  
  def add_silence(snd_data, seconds):
      "Add silence to the start and end of 'snd_data' of length 'seconds' (float)"
      r = array('h', [0 for i in xrange(int(seconds*RATE))])
      r.extend(snd_data)
      r.extend([0 for i in xrange(int(seconds*RATE))])
      return r
  
  def record():
      """
      Record a word or words from the microphone and 
      return the data as an array of signed shorts.
  
      Normalizes the audio, trims silence from the 
      start and end, and pads with 0.5 seconds of 
      blank sound to make sure VLC et al can play 
      it without getting chopped off.
      """
      p = pyaudio.PyAudio()
      stream = p.open(format=FORMAT, channels=1, rate=RATE,
          input=True, output=True,
          frames_per_buffer=CHUNK_SIZE)
  
      num_silent = 0
      snd_started = False
  
      r = array('h')
  
      while 1:
          # little endian, signed short
          snd_data = array('h', stream.read(1024))
          if byteorder == 'big':
              snd_data.byteswap()
  
          print "\n\n\n\n\n\n"
          print len(snd_data)
          print snd_data
  
          r.extend(snd_data)
  
          silent = is_silent(snd_data)
  
          if silent and snd_started:
              num_silent += 1
          elif not silent and not snd_started:
              snd_started = True
  
          if snd_started and num_silent &gt; 1:
              break
  
      sample_width = p.get_sample_size(FORMAT)
      stream.stop_stream()
      stream.close()
      p.terminate()
  
      r = normalize(r)
      r = trim(r)
      r = add_silence(r, 0.5)
      return sample_width, r
  
  def record_to_file(path):
      "Records from the microphone and outputs the resulting data to 'path'"
      sample_width, data = record()
      data = pack('&lt;' + ('h'*len(data)), *data)
  
      wf = wave.open(path, 'wb')
      wf.setnchannels(1)
      wf.setsampwidth(sample_width)
      wf.setframerate(RATE)
      wf.writeframes(data)
      wf.close()
  
  if __name__ == '__main__':
      print("please speak a word into the microphone")
      record_to_file('demo.wav')
      print("done - result written to demo.wav")
  </code></pre>