title: Encoding of raw frames (D3D11Texture2D) to an rtsp stream using libav*
body: <p>I have managed to create a rtsp stream using libav* and directX texture (which I am obtaining from <a href="https://docs.microsoft.com/en-us/archive/blogs/dsui_team/ways-to-capture-the-screen" rel="nofollow noreferrer">GDI API using Bitblit</a> method). Here's my approach for creating live rtsp stream:</p>
  <ol>
  <li><p>Create output context and stream (skipping the checks here)</p>
  <ul>
  <li>avformat_alloc_output_context2(&amp;ofmt_ctx, NULL, &quot;rtsp&quot;, rtsp_url); //RTSP</li>
  <li>vid_codec = avcodec_find_encoder(ofmt_ctx-&gt;oformat-&gt;video_codec);</li>
  <li>vid_stream = avformat_new_stream(ofmt_ctx,vid_codec);</li>
  <li>vid_codec_ctx = avcodec_alloc_context3(vid_codec);</li>
  </ul>
  </li>
  <li><p>Set codec params</p>
  <pre><code>codec_ctx-&gt;codec_tag = 0;
  codec_ctx-&gt;codec_id = ofmt_ctx-&gt;oformat-&gt;video_codec;
  //codec_ctx-&gt;codec_type = AVMEDIA_TYPE_VIDEO;
  codec_ctx-&gt;width = width;   codec_ctx-&gt;height = height;
  codec_ctx-&gt;gop_size = 12;
   //codec_ctx-&gt;gop_size = 40;
   //codec_ctx-&gt;max_b_frames = 3;
  codec_ctx-&gt;pix_fmt = target_pix_fmt; // AV_PIX_FMT_YUV420P
  codec_ctx-&gt;framerate = { stream_fps, 1 };
  codec_ctx-&gt;time_base = { 1, stream_fps};
  if (fctx-&gt;oformat-&gt;flags &amp; AVFMT_GLOBALHEADER)
   {
       codec_ctx-&gt;flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
   }
  </code></pre>
  </li>
  <li><p>Initialize video stream</p>
  <pre><code>if (avcodec_parameters_from_context(stream-&gt;codecpar, codec_ctx) &lt; 0)
  {
   Debug::Error(&quot;Could not initialize stream codec parameters!&quot;);
   return false;
  }
  
  AVDictionary* codec_options = nullptr;
  if (codec-&gt;id == AV_CODEC_ID_H264) {
   av_dict_set(&amp;codec_options, &quot;profile&quot;, &quot;high&quot;, 0);
   av_dict_set(&amp;codec_options, &quot;preset&quot;, &quot;fast&quot;, 0);
   av_dict_set(&amp;codec_options, &quot;tune&quot;, &quot;zerolatency&quot;, 0);
  }
  // open video encoder
  int ret = avcodec_open2(codec_ctx, codec, &amp;codec_options);
  if (ret&lt;0) {
   Debug::Error(&quot;Could not open video encoder: &quot;, avcodec_get_name(codec-&gt;id), &quot; error ret: &quot;, AVERROR(ret));
   return false;
  }
  
  stream-&gt;codecpar-&gt;extradata = codec_ctx-&gt;extradata;
  stream-&gt;codecpar-&gt;extradata_size = codec_ctx-&gt;extradata_size;
  </code></pre>
  </li>
  <li><p>Start streaming</p>
  <pre><code>// Create new frame and allocate buffer
  AVFrame* AllocateFrameBuffer(AVCodecContext* codec_ctx, double width, double height)
  {
   AVFrame* frame = av_frame_alloc();
   std::vector&lt;uint8_t&gt; framebuf(av_image_get_buffer_size(codec_ctx-&gt;pix_fmt, width, height, 1));
   av_image_fill_arrays(frame-&gt;data, frame-&gt;linesize, framebuf.data(), codec_ctx-&gt;pix_fmt, width, height, 1);
   frame-&gt;width = width;
   frame-&gt;height = height;
   frame-&gt;format = static_cast&lt;int&gt;(codec_ctx-&gt;pix_fmt);
   //Debug::Log(&quot;framebuf size: &quot;, framebuf.size(), &quot;  frame format: &quot;, frame-&gt;format);
   return frame;
  }
  
  void RtspStream(AVFormatContext* ofmt_ctx, AVStream* vid_stream, AVCodecContext* vid_codec_ctx, char* rtsp_url)
  {
   printf(&quot;Output stream info:\n&quot;);
   av_dump_format(ofmt_ctx, 0, rtsp_url, 1);
  
   const int width = WindowManager::Get().GetWindow(RtspStreaming::WindowId())-&gt;GetTextureWidth();
   const int height = WindowManager::Get().GetWindow(RtspStreaming::WindowId())-&gt;GetTextureHeight();
  
   //DirectX BGRA to h264 YUV420p
   SwsContext* conversion_ctx = sws_getContext(width, height, src_pix_fmt,
       vid_stream-&gt;codecpar-&gt;width, vid_stream-&gt;codecpar-&gt;height, target_pix_fmt, 
       SWS_BICUBIC | SWS_BITEXACT, nullptr, nullptr, nullptr);
  if (!conversion_ctx)
  {
       Debug::Error(&quot;Could not initialize sample scaler!&quot;);
       return;
  }
  
   AVFrame* frame = AllocateFrameBuffer(vid_codec_ctx,vid_codec_ctx-&gt;width,vid_codec_ctx-&gt;height);
   if (!frame) {
       Debug::Error(&quot;Could not allocate video frame\n&quot;);
       return;
   }
  
  
   if (avformat_write_header(ofmt_ctx, NULL) &lt; 0) {
       Debug::Error(&quot;Error occurred when writing header&quot;);
       return;
   }
   if (av_frame_get_buffer(frame, 0) &lt; 0) {
       Debug::Error(&quot;Could not allocate the video frame data\n&quot;);
       return;
   }
  
   int frame_cnt = 0;
   //av start time in microseconds
   int64_t start_time_av = av_gettime();
   AVRational time_base = vid_stream-&gt;time_base;
   AVRational time_base_q = { 1, AV_TIME_BASE };
  
   // frame pixel data info
   int data_size = width * height * 4;
   uint8_t* data = new uint8_t[data_size];
  //    AVPacket* pkt = av_packet_alloc();
  
   while (RtspStreaming::IsStreaming())
   {
       /* make sure the frame data is writable */
       if (av_frame_make_writable(frame) &lt; 0)
       {
           Debug::Error(&quot;Can't make frame writable&quot;);
           break;
       }
  
       //get copy/ref of the texture
       //uint8_t* data = WindowManager::Get().GetWindow(RtspStreaming::WindowId())-&gt;GetBuffer();
       if (!WindowManager::Get().GetWindow(RtspStreaming::WindowId())-&gt;GetPixels(data, 0, 0, width, height))
       {
           Debug::Error(&quot;Failed to get frame buffer. ID: &quot;, RtspStreaming::WindowId());
           std::this_thread::sleep_for (std::chrono::seconds(2));
           continue;
       }
       //printf(&quot;got pixels data\n&quot;);
       // convert BGRA to yuv420 pixel format
       int srcStrides[1] = { 4 * width };
       if (sws_scale(conversion_ctx, &amp;data, srcStrides, 0, height, frame-&gt;data, frame-&gt;linesize) &lt; 0)
       {
           Debug::Error(&quot;Unable to scale d3d11 texture to frame. &quot;, frame_cnt);
           break;
       }
       //Debug::Log(&quot;frame pts: &quot;, frame-&gt;pts, &quot;  time_base:&quot;, av_rescale_q(1, vid_codec_ctx-&gt;time_base, vid_stream-&gt;time_base));
       frame-&gt;pts = frame_cnt++; 
       //frame_cnt++;
       //printf(&quot;scale conversion done\n&quot;);
  
       //encode to the video stream
       int ret = avcodec_send_frame(vid_codec_ctx, frame);
       if (ret &lt; 0)
       {
           Debug::Error(&quot;Error sending frame to codec context! &quot;,frame_cnt);
           break;
       }
  
       AVPacket* pkt = av_packet_alloc();
       //av_init_packet(pkt);
       ret = avcodec_receive_packet(vid_codec_ctx, pkt);
       if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
       {
           //av_packet_unref(pkt);
           av_packet_free(&amp;pkt);
           continue;
       }
       else if (ret &lt; 0)
       {
           Debug::Error(&quot;Error during receiving packet: &quot;,AVERROR(ret));
           //av_packet_unref(pkt);
           av_packet_free(&amp;pkt);
           break;
       }
  
       if (pkt-&gt;pts == AV_NOPTS_VALUE)
       {
           //Write PTS
           //Duration between 2 frames (us)
           int64_t calc_duration = (double)AV_TIME_BASE / av_q2d(vid_stream-&gt;r_frame_rate);
           //Parameters
           pkt-&gt;pts = (double)(frame_cnt * calc_duration) / (double)(av_q2d(time_base) * AV_TIME_BASE);
           pkt-&gt;dts = pkt-&gt;pts;
           pkt-&gt;duration = (double)calc_duration / (double)(av_q2d(time_base) * AV_TIME_BASE);
       }
       int64_t pts_time = av_rescale_q(pkt-&gt;dts, time_base, time_base_q);
       int64_t now_time = av_gettime() - start_time_av;
  
       if (pts_time &gt; now_time)
           av_usleep(pts_time - now_time);
  
       //pkt.pts = av_rescale_q_rnd(pkt.pts, in_stream-&gt;time_base, out_stream-&gt;time_base, (AVRounding)(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
       //pkt.dts = av_rescale_q_rnd(pkt.dts, in_stream-&gt;time_base, out_stream-&gt;time_base, (AVRounding)(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
       //pkt.duration = av_rescale_q(pkt.duration, in_stream-&gt;time_base, out_stream-&gt;time_base);
       //pkt-&gt;pos = -1;
  
       //write frame and send
       if (av_interleaved_write_frame(ofmt_ctx, pkt)&lt;0)
       {
           Debug::Error(&quot;Error muxing packet, frame number:&quot;,frame_cnt);
           break;
       }
  
       //Debug::Log(&quot;RTSP streaming...&quot;);
       //sstd::this_thread::sleep_for(std::chrono::milliseconds(1000/20));
       //av_packet_unref(pkt);
       av_packet_free(&amp;pkt);
   }
  
   //av_free_packet(pkt);
   delete[] data;
  
   /* Write the trailer, if any. The trailer must be written before you
    * close the CodecContexts open when you wrote the header; otherwise
    * av_write_trailer() may try to use memory that was freed on
    * av_codec_close(). */
   av_write_trailer(ofmt_ctx);
   av_frame_unref(frame);
   av_frame_free(&amp;frame);
   printf(&quot;streaming thread CLOSED!\n&quot;);
  }
  </code></pre>
  </li>
  </ol>
  <p>Now, this allows me to connect to my rtsp server and maintain the connection. However, on the rtsp client side I am getting either gray or single static frame as shown below:</p>
  <p><a href="https://i.stack.imgur.com/oz04l.png" rel="nofollow noreferrer"><img src="https://i.stack.imgur.com/oz04l.png" alt="static frame on client side" /></a></p>
  <p>Would appreciate if you can help with following questions:</p>
  <ol>
  <li>Firstly, why the stream is not working in spite of continued connection to the server and updating frames?</li>
  <li>Video codec. By default rtsp format uses Mpeg4 codec, is it possible to use h264? When I manually set it to AV_CODEC_ID_H264 the program fails at avcodec_open2 with return value of -22.</li>
  <li>Do I need to create and allocate new &quot;AVFrame&quot; and &quot;AVPacket&quot; for every frame? Or can I just reuse global variable for this?</li>
  <li>Do I need to explicitly define some code for real-time streaming? (Like in ffmpeg we use &quot;-re&quot; flag).</li>
  </ol>
  <p>Would be great if you can point out some example code for creating livestream. I have checked following resources:</p>
  <ul>
  <li><a href="https://github.com/FFmpeg/FFmpeg/blob/master/doc/examples/encode_video.c" rel="nofollow noreferrer">https://github.com/FFmpeg/FFmpeg/blob/master/doc/examples/encode_video.c</a></li>
  <li><a href="https://stackoverflow.com/questions/48578088/streaming-flv-to-rtmp-with-ffmpeg-using-h264-codec-and-c-api-to-flv-js">streaming FLV to RTMP with FFMpeg using H264 codec and C++ API to flv.js</a></li>
  <li><a href="https://medium.com/swlh/streaming-video-with-ffmpeg-and-directx-11-7395fcb372c4" rel="nofollow noreferrer">https://medium.com/swlh/streaming-video-with-ffmpeg-and-directx-11-7395fcb372c4</a></li>
  </ul>
  <p><strong>Update</strong></p>
  <p>While test I found that I am able to play the stream using ffplay, while it's getting stuck on VLC player. Here is snapshot on the ffplay log</p>
  <p><a href="https://i.stack.imgur.com/TQ735.png" rel="nofollow noreferrer"><img src="https://i.stack.imgur.com/TQ735.png" alt="ffplay log" /></a></p>