[Libav-user] Receiving packets from RTMP server

[Denis]

Hello

I have a strange problem... I have a RTMP server, which is nginx, built so:

wget https://github.com/arut/nginx-rtmp-module/archive/master.zip

./configure --with-http_ssl_module --add-module=../nginx-rtmp-module-master

Its config:

#user nginx;
worker_processes  1;

error_log  logs/nginx.log debug;

events {
     worker_connections  1024;
}

http {
     include       mime.types;
     default_type  application/octet-stream;

     sendfile        on;

     keepalive_timeout  65;

     server {
         listen       80;
     server_name cloud.wtport.ru;
     location / {
         fastcgi_pass 127.0.0.1:9000;
         include fastcgi_params;
     }

         error_page   500 502 503 504  /50x.html;
         location = /50x.html {
             root   html;
         }
     }
}

rtmp {
     server {
         listen 1935;
             chunk_size 10000;
         application webteleportstreamingrtmp {
             allow publish all;
             allow play all;
             live on;
             #wait_video on;
             meta copy;
             #wait_key on;
             on_publish
http://localhost/webteleport/webnotify/on_publish_rtmp;
             on_play http://localhost/webteleport/webnotify/on_play_rtmp;
             on_play_done
http://localhost/webteleport/webnotify/on_stop_rtmp;
             on_update
http://localhost/webteleport/webnotify/on_update_rtmp;
             notify_method get;
          }
         application webteleportarchivertmp {
             allow publish all;
             allow play all;
              play /usr/local/videoscope/archive/motion;
              on_play
http://localhost/webteleport/webnotify/on_playarchive_rtmp;
             on_play_done
http://localhost/webteleport/webnotify/on_stoparchive_rtmp;
             notify_method get;
         }
     }
}

It's running on Ubuntu, based on ffmpeg and translates a video from a
camera.

I want to receive the video on Windows 7, this is the main circle of the
client:

void FramesThread::run()
{
     av_register_all();

     AVFormatContext* format_context = 0;
     int err = avformat_open_input(&format_context, url.toUtf8().data(),
NULL, NULL); //открыть url
     if (err < 0)
     {
         WebTeleportCommon_Logprintf_error(QString("Unable to open url:
%1").arg(url));
         return;
     }

     //format_context->probesize = 10000000;
     //format_context->max_analyze_duration = 2 * AV_TIME_BASE;

     err = avformat_find_stream_info(format_context, NULL);
     if (err < 0)
     {
         WebTeleportCommon_Logprintf_error(QString("Unable to find
stream info"));
         return;
     }

     AVStream* video_stream = 0;
     AVStream* audio_stream = 0;
     for (int i = 0; i < format_context->nb_streams; ++i)
     {
         if (format_context->streams[i]->codec->codec_type ==
AVMEDIA_TYPE_VIDEO)
             video_stream = format_context->streams[i];
         else if (format_context->streams[i]->codec->codec_type ==
AVMEDIA_TYPE_AUDIO)
             audio_stream = format_context->streams[i];
     }

     if (video_stream == 0)
     {
         WebTeleportCommon_Logprintf_error("Unable to find video stream");
         return;
     }

     AVCodecContext* video_codec_context = 0;
     {
         QMutexLocker m(&player->frames_mutex);
         video_codec_context =
format_context->streams[video_stream->index]->codec;
         player->video_codec_context = video_codec_context;
     }

     AVCodec* codec = avcodec_find_decoder(video_codec_context->codec_id);
     err = avcodec_open2(video_codec_context, codec, NULL);
     if (err < 0)
     {
         WebTeleportCommon_Logprintf_error("Unable to open codec");
         return;
     }

     AVCodecContext* audio_codec_context = 0;
     if (audio_stream != 0)
     {
         QMutexLocker m(&player->frames_mutex);

         AVCodecContext* orig_context =
format_context->streams[audio_stream->index]->codec;
         codec = avcodec_find_decoder(orig_context->codec_id);
         audio_codec_context = avcodec_alloc_context3(codec);
         avcodec_copy_context(audio_codec_context, orig_context);

         err = avcodec_open2(audio_codec_context, codec, NULL);
         if (err < 0)
         {
             WebTeleportCommon_Logprintf_error("Unable to open audio
codec");
             return;
         }

         player->audio_codec_context = audio_codec_context;
     }

     struct SwsContext* sws_ctx =
sws_getContext(video_codec_context->width, video_codec_context->height,
         video_codec_context->pix_fmt, video_codec_context->width,
video_codec_context->height,
         AV_PIX_FMT_YUV420P, SWS_BILINEAR, NULL, NULL, NULL);

     int y_plane_size = video_codec_context->width *
video_codec_context->height;
     int uv_plane_size = video_codec_context->width *
video_codec_context->height / 4;

     uint8_t* y_plane = (uint8_t*)malloc(y_plane_size);
     uint8_t* u_plane = (uint8_t*)malloc(uv_plane_size);
     uint8_t* v_plane = (uint8_t*)malloc(uv_plane_size);

     {
         QMutexLocker m(&player->frames_mutex);
         player->y_plane = (uint8_t*)malloc(y_plane_size);
         player->u_plane = (uint8_t*)malloc(uv_plane_size);
         player->v_plane = (uint8_t*)malloc(uv_plane_size);
     }

     SwrContext* audio_resample_context = 0;

     if (audio_codec_context)
     {
         audio_resample_context = swr_alloc_set_opts(NULL,
AV_CH_LAYOUT_MONO, AV_SAMPLE_FMT_S16,
             audio_codec_context->sample_rate,
audio_codec_context->channel_layout, audio_codec_context->sample_fmt,
             audio_codec_context->sample_rate, 0, NULL);

         if (audio_resample_context == NULL)
         {
             WebTeleportCommon_Logprintf_error("Unable to open audio
context");
             return;
         }

         if (swr_init(audio_resample_context) != 0)
         {
             WebTeleportCommon_Logprintf_error("Unable to init audio
context");
             return;
         }
     }

     AVPacket packet;
     int uv_pitch = video_codec_context->width / 2;
     int frame_finished = 0;
     AVFrame* frame = av_frame_alloc();

     //поток чтения фреймов
     while (av_read_frame(format_context, &packet) >= 0 && !exit)
     {
         av_init_packet(&packet);

         if (packet.stream_index == video_stream->index)
         {
             WebTeleportCommon_Logprintf_debug("Video frame");
             avcodec_decode_video2(video_codec_context, frame,
&frame_finished, &packet);

             if (frame_finished)
             {
                 AVPicture pict;
                 pict.data[0] = y_plane;
                 pict.data[1] = u_plane;
                 pict.data[2] = v_plane;
                 pict.linesize[0] = video_codec_context->width;
                 pict.linesize[1] = uv_pitch;
                 pict.linesize[2] = uv_pitch;

                 //сконвертировать изображение в формат YUV который
использует SDL
                 sws_scale(sws_ctx, (uint8_t const * const *)frame->data,
                     frame->linesize, 0, video_codec_context->height,
pict.data,
                     pict.linesize);

                 //предоставить для отображения
                 QMutexLocker m(&player->frames_mutex);
                 memcpy(player->y_plane, pict.data[0], y_plane_size);
                 memcpy(player->u_plane, pict.data[1], uv_plane_size);
                 memcpy(player->v_plane, pict.data[2], uv_plane_size);
                 player->new_video_frame = true;
             }
         }
         else if (packet.stream_index == audio_stream->index)
         {
             WebTeleportCommon_Logprintf_debug("Audio frame");
             int res = 0;
             if ((res = avcodec_send_packet(audio_codec_context,
&packet)) != 0)
             {
                 av_packet_unref(&packet);
                 av_free_packet(&packet);
                 av_frame_unref(frame);
                 continue;
             }

             if (avcodec_receive_frame(audio_codec_context, frame) != 0)
             {
                 av_packet_unref(&packet);
                 av_free_packet(&packet);
                 av_frame_unref(frame);
                 continue;
             }

             uint8_t* resampled_data;
             int resampled_data_size = 0;

             if (!ResampleAudioStream(audio_resample_context, frame,
frame->nb_samples,
                 &resampled_data, &resampled_data_size))
             {
                 av_packet_unref(&packet);
                 av_free_packet(&packet);
                 av_frame_unref(frame);
                 av_freep(&resampled_data);
                 continue;
             }

             QMutexLocker m(&player->frames_mutex);

             player->audio_data.clear();

             std::vector<uint8_t> d(resampled_data, resampled_data +
resampled_data_size);
             player->audio_data.push_back(d);

             av_freep(&resampled_data);
             av_packet_unref(&packet);

             while (swr_get_delay(audio_resample_context, 1) > 0)
             {
                 uint8_t* flushed_data;
                 int flushed_data_size;

                 frame->extended_data = NULL;
                 frame->nb_samples = 0;

                 if (!ResampleAudioStream(audio_resample_context, frame,
1024,
                     &flushed_data, &flushed_data_size))
                 {
                     av_freep(&flushed_data);
                     break;
                 }

                 d.assign(resampled_data, resampled_data +
resampled_data_size);
                 player->audio_data.push_back(d);

                 av_freep(&flushed_data);
             }

             player->new_audio_frame = true;
         }

         av_free_packet(&packet);

         //Sleep(10);
     }

     av_frame_free(&frame);
     frame = 0;

     {
         QMutexLocker m(&player->frames_mutex);
         free(player->y_plane);
         player->y_plane = 0;
         free(player->u_plane);
         player->u_plane = 0;
         free(player->v_plane);
         player->v_plane = 0;
     }

     free(y_plane);
     free(u_plane);
     free(v_plane);

     avcodec_close(video_codec_context);
     if (audio_codec_context)
         avcodec_close(audio_codec_context);
     avformat_close_input(&format_context);
}

The problem is: if the server emits both video and audio packets, while
(av_read_frame()) circle gives only audio packets. If the server emits
only video packets, the circle gives them. I cannot give the both video
and audio packets. The code above the while works well, I have both
video_stream and audio_stream, I open both codecs. Wireshark shows me
the both types of RTMP packets come. Windows 7 running in VirtualBox if
it has meaning.