title: Encoding video and audio with humble video
body: <p>I'm trying to create a video file from a remote stream, to store it on the harddrive. I'm using Humble Video for the encoding. Recording and creating the video file works fine. But now I want to add audio to the video.</p>
  
  <p>I am receiving JPG images and audio data as byte arrays from the remote source. Audio data has a sample rate of 11025 and is 1 channel 16 bit values.</p>
  
  <p>I think I am still missing something regarding the timebase and timestamp of the audio data I try to insert into the video file. Most of the time all recorded audio is very scrambled and only hearable in the first few seconds of the video.</p>
  
  <p>Here is my code:</p>
  
  <pre><code>import java.awt.image.BufferedImage;
  import java.io.ByteArrayInputStream;
  import java.io.IOException;
  import java.io.InputStream;
  import java.util.concurrent.ExecutorService;
  import java.util.concurrent.Executors;
  import java.util.function.Consumer;
  
  import javax.imageio.ImageIO;
  
  import io.humble.video.AudioChannel.Layout;
  import io.humble.video.AudioFormat.Type;
  import io.humble.video.Codec;
  import io.humble.video.Encoder;
  import io.humble.video.MediaAudio;
  import io.humble.video.MediaPacket;
  import io.humble.video.MediaPicture;
  import io.humble.video.Muxer;
  import io.humble.video.MuxerFormat;
  import io.humble.video.PixelFormat;
  import io.humble.video.Rational;
  import io.humble.video.awt.MediaPictureConverter;
  import io.humble.video.awt.MediaPictureConverterFactory;
  
  public class VideoStream {
      private static final int FPS = 30;
      private static final int AUDIO_SAMPLE_RATE = 11025;
      private static final int AUDIO_SAMPLE_SIZE = 11025;
  
      private final String filename;
      private final long startTimestampVideo;
      private final long startTimestampAudio;
      private final Muxer muxer;
      private final Encoder videoEncoder;
      private final Encoder audioEncoder;
      private final Rational timebase;
  
      private MediaPicture picture;
      private MediaPictureConverter converter;
      private MediaPacket packet;
      private MediaAudio sound;
  
      private boolean finished = false;
  
      private static class EncoderTask implements Runnable {
          private final byte[] data;
          private final long timestamp;
          private final Consumer&lt;EncoderTask&gt; encoder;
  
          public EncoderTask(byte[] data, long timestamp, Consumer&lt;EncoderTask&gt; encoder) {
              if (encoder == null) {
                  throw new IllegalArgumentException("Encoder must not be null.");
              }
  
              this.data = data;
              this.timestamp = timestamp;
              this.encoder = encoder;
          }
  
          @Override
          public void run() {
              encoder.accept(this);
          }
  
          public byte[] getData() {
              return data;
          }
  
          public long getTimestamp() {
              return timestamp;
          }
  
          public Consumer&lt;EncoderTask&gt; getEncoder() {
              return encoder;
          }
      }
  
      private ExecutorService threadPool = Executors.newSingleThreadExecutor();
  
      public VideoStream(String filename, int width, int height, long startTimestampVideo, long startTimestampAudio)
              throws IOException, InterruptedException {
          this.filename = filename;
          this.startTimestampVideo = startTimestampVideo;
          this.startTimestampAudio = startTimestampAudio;
  
          this.timebase = Rational.make(1, FPS);
  
          this.muxer = Muxer.make(filename, null, null);
  
          PixelFormat.Type pixelFormat = PixelFormat.Type.PIX_FMT_YUV420P;
          Codec videoCodec = Codec.findEncodingCodec(muxer.getFormat().getDefaultVideoCodecId());
          Codec audioCodec = Codec.findEncodingCodec(muxer.getFormat().getDefaultAudioCodecId());
  
          this.videoEncoder = createVideoEncoder(videoCodec, width, height, pixelFormat);
          this.audioEncoder = createAudioEncoder(audioCodec);
  
          videoEncoder.open(null, null);
          audioEncoder.open(null, null);
          muxer.addNewStream(videoEncoder);
          muxer.addNewStream(audioEncoder);
          muxer.open(null, null);
  
          picture = MediaPicture.make(videoEncoder.getWidth(), videoEncoder.getHeight(), pixelFormat);
          picture.setTimeBase(timebase);
  
          sound = MediaAudio.make(AUDIO_SAMPLE_SIZE, AUDIO_SAMPLE_RATE, 1, Layout.CH_LAYOUT_MONO, Type.SAMPLE_FMT_S16);
          sound.setTimeBase(timebase);
  
          packet = MediaPacket.make();
      }
  
      private Encoder createVideoEncoder(Codec codec, int width, int height, PixelFormat.Type pixelFormat) {
          Encoder encoder = Encoder.make(codec);
  
          encoder.setWidth(width);
          encoder.setHeight(height);
          encoder.setPixelFormat(pixelFormat);
          encoder.setTimeBase(timebase);
  
          if (muxer.getFormat().getFlag(MuxerFormat.Flag.GLOBAL_HEADER)) {
              encoder.setFlag(Encoder.Flag.FLAG_GLOBAL_HEADER, true);
          }
  
          return encoder;
      }
  
      private Encoder createAudioEncoder(Codec codec) {
          Encoder encoder = Encoder.make(codec);
  
          encoder.setSampleRate(AUDIO_SAMPLE_RATE);
          encoder.setChannels(1);
          encoder.setChannelLayout(Layout.CH_LAYOUT_MONO);
          encoder.setSampleFormat(Type.SAMPLE_FMT_S16);
  
          if (muxer.getFormat().getFlag(MuxerFormat.Flag.GLOBAL_HEADER)) {
              encoder.setFlag(Encoder.Flag.FLAG_GLOBAL_HEADER, true);
          }
  
          return encoder;
      }
  
      public String getFilename() {
          return filename;
      }
  
      public synchronized void addImage(byte[] imageData, long timestamp) {
          if (!finished) {
              System.out.println("Adding image: " + (timestamp - startTimestampVideo));
              threadPool.execute(new EncoderTask(imageData, timestamp, this::encodeImage));
          }
      }
  
      public synchronized void addAudio(byte[] audioData, long timestamp) {
          if (!finished) {
              System.out.println("Adding audio: " + (timestamp - startTimestampAudio));
              threadPool.execute(new EncoderTask(audioData, timestamp, this::encodeAudio));
          }
      }
  
      public synchronized void finish() {
          if (!finished) {
              threadPool.execute(new EncoderTask(null, 0, this::finish));
          }
      }
  
      private synchronized void encodeImage(EncoderTask task) {
          BufferedImage jpegImage = convertImageDataToBufferedImage(task.getData());
          if (jpegImage == null) {
              return;
          }
  
          BufferedImage image = convertToType(jpegImage, BufferedImage.TYPE_3BYTE_BGR);
  
          if (converter == null) {
              converter = MediaPictureConverterFactory.createConverter(image, picture);
          }
  
          long t = Math.round((task.getTimestamp() - startTimestampVideo) * timebase.getDouble());
          converter.toPicture(picture, image, t);
  
          System.out.println("Encoding video: " + t);
          do {
              videoEncoder.encode(packet, picture);
              if (packet.isComplete()) {
                  muxer.write(packet, false);
              }
          } while (packet.isComplete());
      }
  
      private synchronized void encodeAudio(EncoderTask task) {
          System.out.println(
                  "Audio delta: " + (task.getTimestamp() - startTimestampAudio) + " timebase: " + timebase.getDouble());
          long t = Math.round((task.getTimestamp() - startTimestampAudio) * timebase.getDouble() * AUDIO_SAMPLE_RATE);
  
          sound.getData(0).put(task.data, 0, 0, task.data.length);
          sound.setNumSamples(task.data.length);
          sound.setTimeStamp(t);
          sound.setComplete(true);
  
          System.out.println("Encoding audio: " + t);
          do {
              audioEncoder.encode(packet, sound);
              if (packet.isComplete()) {
                  muxer.write(packet, false);
              }
          } while (packet.isComplete());
      }
  
      private synchronized void finish(EncoderTask task) {
          do {
              videoEncoder.encode(packet, null);
              if (packet.isComplete()) {
                  muxer.write(packet, false);
              }
          } while (packet.isComplete());
          do {
              audioEncoder.encode(packet, null);
              if (packet.isComplete()) {
                  muxer.write(packet, false);
              }
          } while (packet.isComplete());
  
          muxer.close();
          finished = true;
  
          threadPool.shutdown();
      }
  
      private static BufferedImage convertImageDataToBufferedImage(byte[] imageData) {
          try (InputStream in = new ByteArrayInputStream(imageData)) {
              return ImageIO.read(in);
          } catch (IOException e) {
              return null;
          }
      }
  
      private static BufferedImage convertToType(BufferedImage sourceImage, int targetType) {
          BufferedImage image;
  
          if (sourceImage.getType() == targetType) {
              // if the source image is already the target type, return the source image
              image = sourceImage;
          } else {
              // otherwise create a new image of the target type and draw the new
              // image
              image = new BufferedImage(sourceImage.getWidth(), sourceImage.getHeight(), targetType);
              image.getGraphics().drawImage(sourceImage, 0, 0, null);
          }
  
          return image;
      }
  }
  </code></pre>
  
  <p>It would be great if anyone has a sample on how to encode audio and video in the same file. The demos in the Humble Video git are only for encoding video data.</p>