音视频

通过学习跟着写了一个音视频播放器。内容有点多，所以我就不打算在博客都展示出来了，这里就展示一下每个类的核心逻辑。思路很清晰，这里就简单讲解一下吧((～o￣3￣)～)

音视频队列

  • packet队列
    avpacketqueue 存储解复用线程中读取到的packet数据，因为队列的实现比较简单，我就展示一下该类的结构吧
1
2
3
4
5
6
7
8
9
10
11
12
13
class AVPacketQueue
{
public:
    AVPacketQueue();
    ~AVPacketQueue();
    void Abort();
    int Size();
    int Push(AVPacket *val);
    AVPacket *Pop(const int timeout);
private:
    void Release();
    Queue<AVPacket *>queue_;
};
  • frame视频队列
    avframequeue 存储解码线程中解码后的frame数据，类结构如下
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
class AVFrameQueue
{
public:
    AVFrameQueue();
    ~AVFrameQueue();
    void Abort();
    int Push(AVFrame *val);
    AVFrame *Pop(const int timeout);
    AVFrame *Front();
    int Size();

private:
    void release();
    Queue<AVFrame *>queue_;
};

线程分工

  • 主线程
    主要完成线程的创建，让子线程完成具体的工作。因为主线程对程序的理解挺重要的，所以你可以在文章的最后直接看到主线程的源码

  • 解复用线程
    demuxthread 完成音频的解复用，并将pkt数据写入packet队列，同时音视频的相关参数也由该线程提供。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
while(abort_ != 1){
    ret = av_read_frame(ifmt_ctx_, &pkt);
    if(ret < 0){
        av_strerror(ret, err2str, sizeof(err2str));
        av_log(NULL, AV_LOG_ERROR, "av_read_frame(ifmt_ctx_, &pkt) failed, ret: %s\n", err2str);
        break;
    }
    if(pkt.stream_index == audio_index_){
        audio_queue_->Push(&pkt);
        av_log(NULL, AV_LOG_INFO, "audio pkt queue size:%d\n", audio_queue_->Size());
    }else if(pkt.stream_index == video_index_){
        video_queue_->Push(&pkt);
        av_log(NULL, AV_LOG_INFO, "video pkt queue size:%d\n", video_queue_->Size());
    }else {
        av_packet_unref(&pkt);
    }
}
  • 解码线程
    decodethread 完成音视频的解码，并将解码后的数据写入frame队列。
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
while(abort_ != 1){
    // if(frame_queue_->Size() > 10){
    //     std::this_thread::sleep_for(std::chrono::milliseconds(10));
    //     continue;
    // }
    AVPacket *pkt = packet_queue_->Pop(10);
    if(pkt){
        int ret = avcodec_send_packet(codec_ctx_, pkt);
        av_packet_free(&pkt);
        if(ret < 0){
            av_strerror(ret, err2str, sizeof(err2str));
            av_log(NULL, AV_LOG_ERROR, "avcodec_send_packet(codec_ctx_, pkt) failed, ret: %s\n", err2str);
            break;
        }
        // 读取解码后的数据frame
        while(true){
            ret = avcodec_receive_frame(codec_ctx_, frame);
            if(ret == 0){
                frame_queue_->Push(frame);
                av_log(NULL, AV_LOG_INFO, "%s frame queue size:%d\n",codec_ctx_->codec->name, frame_queue_->Size());
                continue;
            }else if(ret == AVERROR(EAGAIN) || ret == AVERROR_EOF){
                break;
            }else {
                abort_ = 1;
                av_strerror(ret, err2str, sizeof(err2str));
                av_log(NULL, AV_LOG_ERROR, "avcodec_receive_frame(codec_ctx_, frame) failed, ret: %s\n", err2str);
                break;
            }
        }
    }else {
        //av_log(NULL, AV_LOG_INFO, "not get packet\n");
    }
}

输出渲染

  • 音频输出
    audiooutput 这里涉及到SDL对播放音频的处理。实际上SDL会自己建立线程完成对音频数据的获取，即通过回调的方式，所以最主要的就是要设计好回调函数，下面是回调函数的设计
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
void fill_audio_pcm(void *udata, Uint8 *stream, int len){
    //1. 从frame queue读取解码后的PCM数据，填充到stream
    AudioOutput *is = (AudioOutput *)udata;
    int len1 = 0;
    int audio_size = 0;
    while(len > 0){
        if(is->audio_buf_index == is->audio_buf_size){
            is->audio_buf_index = 0;
            AVFrame *frame = is->frame_queue_->Pop(10);
            if(frame){
                is->pts_ = frame->pts;
                //判断要不要重采样
                if((frame->format != is->dst_tgt_.fmt)
                    || (frame->sample_rate != is->dst_tgt_.freq)
                    || (frame->ch_layout.nb_channels != is->dst_tgt_.channle_layout.nb_channels)
                    && (!is->swr_ctx_)){
                    int ret = swr_alloc_set_opts2(&is->swr_ctx_,
                                        &is->dst_tgt_.channle_layout,
                                        (enum AVSampleFormat)is->dst_tgt_.fmt,
                                        is->dst_tgt_.freq,
                                        &frame->ch_layout,
                                        (enum AVSampleFormat)frame->format,
                                        frame->sample_rate,
                                        0, NULL);
                    if(ret != 0 || swr_init(is->swr_ctx_) < 0){
                        av_log(NULL, AV_LOG_ERROR, "swr_alloc_set_opts2 or swr_init(is->swr_ctx_) failed!\n");
                        swr_free(&is->swr_ctx_);
                        return ;
                    }
                }
                if(is->swr_ctx_){
                    const uint8_t **in = (const uint8_t **)frame->extended_data;
                    uint8_t **out = &is->audio_buf1_;
                    int out_samples = frame->nb_samples * is->dst_tgt_.freq / frame->sample_rate + 256;
                    int out_bytes = av_samples_get_buffer_size(NULL, is->dst_tgt_.channles, out_samples, is->dst_tgt_.fmt, 0);
                    if(out_bytes < 0){
                        av_log(NULL, AV_LOG_ERROR, "av_samples_get_buffer_size failed!\n");
                        return ;
                    }
                    av_fast_malloc(&is->audio_buf1_, &is->audio_buf1_size, out_bytes);
                    int len2 = swr_convert(is->swr_ctx_, out, out_samples, in, frame->nb_samples); //返回样本数
                    if(len2 < 0){
                        av_log(NULL, AV_LOG_ERROR, "swr_convert failed!\n");
                        return ;
                    }
                    is->audio_buf_ = is->audio_buf1_;
                    is->audio_buf_size = av_samples_get_buffer_size(NULL, is->dst_tgt_.channles, len2, is->dst_tgt_.fmt, 1);
                }else {
                    audio_size = av_samples_get_buffer_size(NULL, frame->ch_layout.nb_channels, frame->nb_samples, (enum AVSampleFormat)frame->format, 1);
                    av_fast_malloc(&is->audio_buf1_, &is->audio_buf1_size, audio_size);
                    is->audio_buf_ = is->audio_buf1_;
                    is->audio_buf_size = audio_size;
                    memcpy(is->audio_buf_, frame->data[0], audio_size);
                }
                av_frame_free(&frame);
            }else{
                is->audio_buf_ = NULL;
                is->audio_buf_size = 512;
            }
        }
        len1 = is->audio_buf_size - is->audio_buf_index;
        if(len1 > len){
            len1 = len;
        }
        if(!is->audio_buf_){
            memset(stream, 0, len1);
        }else {
            memcpy(stream, is->audio_buf_ + is->audio_buf_index, len1);
        }
        len -= len1;
        stream += len1;
        is->audio_buf_index += len1;
    }
    //设置时钟
    if(is->pts_ != AV_NOPTS_VALUE){
        double pts = is->pts_ * av_q2d(is->time_base_);
        av_log(NULL, AV_LOG_INFO, "audio pts:%0.3lf\n", pts);
        is->avsync_->SetClock(pts);
    }
}
  • 视频输出
    videooutput 在主线成里调用videooutput的一个循环输出的方法，实现视频的渲染，但这样的方式肯定是不好的，总之就先这样吧(ㄟ( ▔, ▔ )ㄏ)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
int VideoOutput::MainLoop()
{
    SDL_Event event;
    while(true){
        RefreshLoopWaitEvent(&event);
        switch(event.type){
        case SDL_KEYDOWN:
            if(event.key.keysym.sym == SDLK_ESCAPE){
                av_log(NULL, AV_LOG_INFO, "esc key down");
                return 0;
            }
            break;
        case SDL_QUIT:
            av_log(NULL, AV_LOG_INFO, "SDL_QUIT");
            return 0;
        default:
            break;
        }
    }
    return 0;
}

RefreshLoopWaitEvent()这个函数的设计主要是在等待事件的同时完成音视频的对齐，由于是视频对齐音频，在视频质量很高的情况下，会出现视频跟不上音频的情况，所以我在主线程里设置了一个等待时间，目的是让解码线程先运行一段时间，让视频播放时提前有一些数据，避免视频解码过慢造成的问题。但这样肯定是不行的，我想到用开多个视频解码线程的方式解决，但这样又会引出许多问题(＞﹏＜)但我相信会在以后的学习中，这将不会是问题(o(￣︶￣)o)

音视频同步

avsync 音视频的同步有三种方式，这里采用的是视频基于音频的播放，对时间的获取和设置都是通过该类实现的，内容很少我就直接贴出来了

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
AVSync(){}

void InitClock(){
    SetClock(NAN); //数学对比是一个无效值
}
void SetClockAt(double pts, double time){
    pts_ = pts;
    pts_drift_ = pts_ - time;
}

double GetClock(){
    double time = GetMicroseconds() / 1000000.0;
    return pts_drift_ + time;
}

void SetClock(double pts){
    double time = GetMicroseconds() / 1000000.0; //ui->s
    SetClockAt(pts, time);
}

time_t GetMicroseconds(){
    system_clock::time_point time_point_new = system_clock::now();
    system_clock::duration duration = time_point_new.time_since_epoch();

    time_t us = duration_cast<microseconds>(duration).count();
    return us;
}

double pts_ = 0;
double pts_drift_ = 0;

主函数

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
#define SDL_MAIN_HANDLED
#include "demuxthread.h"
#include "decodethread.h"
#include "audiooutput.h"
#include "videooutput.h"

using namespace std;

char err2str[256] = {0};

int main(int argc, char *argv[])
{
    int ret = 0;
    av_log_set_level(AV_LOG_INFO);
    if(argc < 2){
        av_log(NULL, AV_LOG_ERROR, "argc < 2!\n");
        return -1;
    }
    //创建音视频队列
    AVPacketQueue audio_packet_queue;
    AVPacketQueue video_packet_queue;

    AVFrameQueue audio_frame_queue;
    AVFrameQueue video_frame_queue;

    AVSync avsync;
    avsync.InitClock();

    //1. 解复用线程
    DemuxThread *demux_thread = new DemuxThread(&audio_packet_queue, &video_packet_queue);
    ret = demux_thread->Init(argv[1]);
    if(ret < 0){
        av_log(NULL, AV_LOG_ERROR, "demux_thread.Init(argv[1]) failed!\n");
        return -1;
    }

    ret = demux_thread->Start();
    if(ret < 0){
        av_log(NULL, AV_LOG_ERROR, "demux_thread.Start() failed!\n");
        return -1;
    }

    //2. 解码线程
    DecodeThread *video_decode_thread = new DecodeThread(&video_packet_queue, &video_frame_queue);
    ret = video_decode_thread->Init(demux_thread->VideoCodecParameters());
    if(ret < 0){
        av_log(NULL, AV_LOG_ERROR, "video_decode_thread->Init() failed!\n");
        return -1;
    }
    ret = video_decode_thread->Start();
    if(ret < 0){
        av_log(NULL, AV_LOG_ERROR, "video_decode_thread->Start() failed!\n");
        return -1;
    }

    DecodeThread *audio_decode_thread = new DecodeThread(&audio_packet_queue, &audio_frame_queue);
    ret = audio_decode_thread->Init(demux_thread->AudioCodecParameters());
    if(ret < 0){
        av_log(NULL, AV_LOG_ERROR, "audio_decode_thread->Init() failed!\n");
        return -1;
    }
    ret = audio_decode_thread->Start();
    if(ret < 0){
        av_log(NULL, AV_LOG_ERROR, "audio_decode_thread->Start() failed!\n");
        return -1;
    }

    //休眠 这里预先解码一些音视频的数据，尽量防止音频的播放速度大于视频的播放速度导致视频播放跟不上音频播放的问题 但这样的解决方式肯定是不够的，可能需要多线程解码的方式
    std::this_thread::sleep_for(std::chrono::milliseconds(3 * 1000));

    // 初始化audio输出
    AudioParams audio_params = {0};
    memset(&audio_params, 0, sizeof(AudioParams));
    audio_params.channles = demux_thread->AudioCodecParameters()->ch_layout.nb_channels;
    audio_params.channle_layout = demux_thread->AudioCodecParameters()->ch_layout;
    audio_params.fmt = (enum AVSampleFormat)demux_thread->AudioCodecParameters()->format;
    audio_params.freq = demux_thread->AudioCodecParameters()->sample_rate;
    audio_params.frame_size = demux_thread->AudioCodecParameters()->frame_size;

    AudioOutput *audio_output = new AudioOutput(&avsync, demux_thread->AudioStreamTime(), audio_params, &audio_frame_queue);
    ret = audio_output->Init();
    if(ret < 0){
        av_log(NULL, AV_LOG_ERROR, "audio_output->Init() failed!\n");
        return -1;
    }

    // 初始化视频输出
    VideoOutput *video_output = new VideoOutput(&avsync, demux_thread->VideoStreamTime(),
                                                &video_frame_queue,
                                                demux_thread->VideoCodecParameters()->width,
                                                demux_thread->VideoCodecParameters()->height,
                                                640, 480);
    ret = video_output->Init();
    if(ret < 0){
        av_log(NULL, AV_LOG_ERROR, "video_output->Init() failed!\n");
        return -1;
    }
    video_output->MainLoop(); //程序会停在这里直到退出

    //休眠
    //std::this_thread::sleep_for(std::chrono::milliseconds(120 * 1000));

__END:
    demux_thread->Stop();
    delete demux_thread;

    audio_decode_thread->Stop();
    delete audio_decode_thread;
    video_decode_thread->Stop();
    delete video_decode_thread;

    audio_output->DeInit();
    delete audio_output;

    av_log(NULL, AV_LOG_INFO, "main finished\n");

    return 0;
}

说些什么吧！

valine