开发环境：Windows 10， Qt 5.13.1, ffmpeg 4.2.2
上几篇介绍分别介绍了ffmpeg解码视频显示在界面上，解码音频用SDL播放。
本篇整合两个功能，使用音视频同步。
这里主要讲下声音和视频同步的步骤。
首先刚开始播放的时候通过av_gettime()获取系统主时钟，记录下来。
以后便不断调用av_gettime()获取系统时钟 减去之前记录下的差值，便得到了一个视频播放了多久的实际时间。
对于视频的同步我们这样做：
从视频读取出的数据中包含一个pts的信息(每一帧图像都会带有pts的信息，pts就是播放视频的时候此图像应该显示的时间)。 这样只需要使用pts和前面获取的时间进行对比，pts比实际时间大，就调用sleep函数等一等，否则就直接播放出来。这样就达到了某种意义上的同步了。
而对于音频：
从前面使用SDL的例子，其实就能够发现一个现象：我们读取音频的线程差不多就是瞬间读完放入队列的，但是音频播放速度却是正常的，并不是一下子播放完毕。因此可以看出，在音频播放上，SDL已经帮我们做了处理了，只需要将数据直接交给SDL就行了。
创建Qt的工程与前面一样，完整代码如下：
CVideoPlayer.h
#ifndef CVIDEOPLAYER_H
#define CVIDEOPLAYER_H
#include <QThread>
#include <QImage>
class CVideoPlayer: public QThread
{
Q_OBJECT
public:
CVideoPlayer();
void videoDecode();
protected:
void run();
signals:
void signalGetOneFrame(QImage image);
void signalDecodeError(int error);
};
#endif // CVIDEOPLAYER_HCVideoPlayer.cpp
#include "CVideoPlayer.h"
#include <stdio.h>
#include <QDebug>
extern "C"
{
#include "libavcodec/avcodec.h"
#include "libavformat/avformat.h"
#include "libswscale/swscale.h"
#include "libavdevice/avdevice.h"
#include <libavutil/time.h>
#include "libavutil/pixfmt.h"
#include "libswresample/swresample.h"
#include <SDL.h>//这个不要放在头文件包含中，否则报main函数冲突
//#include <SDL_audio.h>
//#include <SDL_types.h>
//#include <SDL_name.h>
//#include <SDL_main.h>
//#include <SDL_config.h>
}
typedef struct PacketQueue {
AVPacketList *first_pkt, *last_pkt;
int nb_packets;
int size;
SDL_mutex *mutex;
SDL_cond *cond;
} PacketQueue;
#define VIDEO_PICTURE_QUEUE_SIZE 1
#define AVCODEC_MAX_AUDIO_FRAME_SIZE 192000 // 1 second of 48khz 32bit audio
typedef struct VideoState {
AVCodecContext *aCodecCtx; //音频解码器
AVFrame *audioFrame;// 解码音频过程中的使用缓存
PacketQueue *audioq;
double video_clock; ///<pts of last decoded frame / predicted pts of next decoded frame
AVStream *video_st;
} VideoState;
#define SDL_AUDIO_BUFFER_SIZE 1024
VideoState mVideoState; //用来 传递给 SDL音频回调函数的数据
void packet_queue_init(PacketQueue *q) {
memset(q, 0, sizeof(PacketQueue));
q->mutex = SDL_CreateMutex();
q->cond = SDL_CreateCond();
}
int packet_queue_put(PacketQueue *q, AVPacket *pkt) {
AVPacketList *pkt1;
if (av_dup_packet(pkt) < 0) {
return -1;
}
pkt1 = (AVPacketList*)av_malloc(sizeof(AVPacketList));
if (!pkt1)
return -1;
pkt1->pkt = *pkt;
pkt1->next = NULL;
SDL_LockMutex(q->mutex);
if (!q->last_pkt)
q->first_pkt = pkt1;
else
q->last_pkt->next = pkt1;
q->last_pkt = pkt1;
q->nb_packets++;
q->size += pkt1->pkt.size;
SDL_CondSignal(q->cond);
SDL_UnlockMutex(q->mutex);
return 0;
}
//http://blog.yundiantech.com/?log=blog&id=9
static int packet_queue_get(PacketQueue *q, AVPacket *pkt, int block) {
AVPacketList *pkt1;
int ret;
SDL_LockMutex(q->mutex);
for (;;) {
pkt1 = q->first_pkt;
if (pkt1) {
q->first_pkt = pkt1->next;
if (!q->first_pkt)
q->last_pkt = NULL;
q->nb_packets--;
q->size -= pkt1->pkt.size;
*pkt = pkt1->pkt;
av_free(pkt1);
ret = 1;
break;
} else if (!block) {
ret = 0;
break;
} else {
SDL_CondWait(q->cond, q->mutex);
}
}
SDL_UnlockMutex(q->mutex);
return ret;
}
int audio_decode_frame(VideoState *is, uint8_t *audio_buf, int buf_size)
{
static AVPacket pkt;
static uint8_t *audio_pkt_data = NULL;
static int audio_pkt_size = 0;
int len1, data_size;
AVCodecContext *aCodecCtx = is->aCodecCtx;
AVFrame *audioFrame = is->audioFrame;
PacketQueue *audioq = is->audioq;
for(;;)
{
if(packet_queue_get(audioq, &pkt, 1) < 0)
{
return -1;
}
audio_pkt_data = pkt.data;
audio_pkt_size = pkt.size;
while(audio_pkt_size > 0)
{
int got_picture;
int ret = avcodec_decode_audio4( aCodecCtx, audioFrame, &got_picture, &pkt);
if( ret < 0 ) {
printf("Error in decoding audio frame.\n");
exit(0);
}
if( got_picture ) {
int in_samples = audioFrame->nb_samples;
short *sample_buffer = (short*)malloc(audioFrame->nb_samples * 2 * 2);
memset(sample_buffer, 0, audioFrame->nb_samples * 4);
int i=0;
float *inputChannel0 = (float*)(audioFrame->extended_data[0]);
// Mono
if( audioFrame->channels == 1 ) {
for( i=0; i<in_samples; i++ ) {
float sample = *inputChannel0++;
if( sample < -1.0f ) {
sample = -1.0f;
} else if( sample > 1.0f ) {
sample = 1.0f;
}
sample_buffer[i] = (int16_t)(sample * 32767.0f);
}
} else { // Stereo
float* inputChannel1 = (float*)(audioFrame->extended_data[1]);
for( i=0; i<in_samples; i++) {
sample_buffer[i*2] = (int16_t)((*inputChannel0++) * 32767.0f);
sample_buffer[i*2+1] = (int16_t)((*inputChannel1++) * 32767.0f);
}
}
// fwrite(sample_buffer, 2, in_samples*2, pcmOutFp);
memcpy(audio_buf,sample_buffer,in_samples*4);
free(sample_buffer);
}
audio_pkt_size -= ret;
if (audioFrame->nb_samples <= 0)
{
continue;
}
data_size = audioFrame->nb_samples * 4;
return data_size;
}
if(pkt.data)
av_free_packet(&pkt);
}
}
void audio_callback(void *userdata, Uint8 *stream, int len)
{
// AVCodecContext *aCodecCtx = (AVCodecContext *) userdata;
VideoState *is = (VideoState *) userdata;
int len1, audio_data_size;
static uint8_t audio_buf[(AVCODEC_MAX_AUDIO_FRAME_SIZE * 3) / 2];
static unsigned int audio_buf_size = 0;
static unsigned int audio_buf_index = 0;
/* len是由SDL传入的SDL缓冲区的大小，如果这个缓冲未满，我们就一直往里填充数据 */
while (len > 0) {
/* audio_buf_index 和 audio_buf_size 标示我们自己用来放置解码出来的数据的缓冲区，*/
/* 这些数据待copy到SDL缓冲区， 当audio_buf_index >= audio_buf_size的时候意味着我*/
/* 们的缓冲为空，没有数据可供copy，这时候需要调用audio_decode_frame来解码出更
/* 多的桢数据 */
if (audio_buf_index >= audio_buf_size) {
audio_data_size = audio_decode_frame(is, audio_buf,sizeof(audio_buf));
/* audio_data_size < 0 标示没能解码出数据，我们默认播放静音 */
if (audio_data_size < 0) {
/* silence */
audio_buf_size = 1024;
/* 清零，静音 */
memset(audio_buf, 0, audio_buf_size);
} else {
audio_buf_size = audio_data_size;
}
audio_buf_index = 0;
}
/* 查看stream可用空间，决定一次copy多少数据，剩下的下次继续copy */
len1 = audio_buf_size - audio_buf_index;
if (len1 > len) {
len1 = len;
}
memcpy(stream, (uint8_t *) audio_buf + audio_buf_index, len1);
len -= len1;
stream += len1;
audio_buf_index += len1;
}
}
static double synchronize_video(VideoState *is, AVFrame *src_frame, double pts) {
double frame_delay;
if (pts != 0) {
/* if we have pts, set video clock to it */
is->video_clock = pts;
} else {
/* if we aren't given a pts, set it to the clock */
pts = is->video_clock;
}
/* update the video clock */
frame_delay = av_q2d(is->video_st->codec->time_base);
/* if we are repeating a frame, adjust clock accordingly */
frame_delay += src_frame->repeat_pict * (frame_delay * 0.5);
is->video_clock += frame_delay;
return pts;
}
CVideoPlayer::CVideoPlayer()
{
}
void CVideoPlayer::run()
{
videoDecode();
}
void CVideoPlayer::videoDecode()
{
char *filePath = (char*)"F:\\github\\QtPlayLearn\\win\\mp4\\lasa.mp4";
AVFormatContext *pFormatCtx;
AVCodecContext *pCodecCtx;
AVCodec *pCodec;
AVFrame *pFrame, *pFrameRGB;
AVPacket *packet;
uint8_t *outBuffer;
AVCodecContext *aCodecCtx;
AVCodec *aCodec;
static struct SwsContext *img_convert_ctx;
unsigned int i;
int audioStream ,videoStream, numBytes;
int ret, got_picture;
av_register_all();//初始化ffmpeg 调用了这个才能正常适用编码器和解码器
if (SDL_Init(SDL_INIT_AUDIO)) {
fprintf(stderr,"Could not initialize SDL - %s. \n", SDL_GetError());
exit(1);
}
//Allocate an AVFormatContext.
pFormatCtx = avformat_alloc_context();
if(0 != avformat_open_input(&pFormatCtx, filePath, nullptr, nullptr))
{
emit signalDecodeError(-1);
return;
}
if(avformat_find_stream_info(pFormatCtx, nullptr))
{
emit signalDecodeError(-2);
return;
}
videoStream = -1;
audioStream = -1;
//循环查找视频中包含的流信息，直到找到视频类型的流
//便将其记录下来 保存到videoStream变量中
//这里我们现在只处理视频流 音频流先不管他
for(i = 0; i < pFormatCtx->nb_streams; ++i)
{
qDebug() << "pFormatCtx->streams[" << i << "]->codec->codec_type = " << pFormatCtx->streams[i]->codec->codec_type << endl;
//0:视频类型 1:音频类型
if(pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO)
{
videoStream = i;
}
if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO && audioStream < 0)
{
audioStream = i;
}
}
qDebug() << "videoStream===========" << videoStream << " pFormatCtx->nb_streams==" << pFormatCtx->nb_streams << endl;
//如果videoStream为-1 说明没有找到视频流
if (videoStream == -1) {
printf("Didn't find a video stream.\n");
return;
}
if (audioStream == -1) {
printf("Didn't find a audio stream.\n");
return;
}
///查找音频解码器
aCodecCtx = pFormatCtx->streams[audioStream]->codec;
aCodec = avcodec_find_decoder(aCodecCtx->codec_id);
if (aCodec == NULL) {
printf("ACodec not found.\n");
return;
}
///打开音频解码器
if (avcodec_open2(aCodecCtx, aCodec, NULL) < 0) {
printf("Could not open audio codec.\n");
return;
}
//初始化音频队列
PacketQueue *audioq = new PacketQueue;
packet_queue_init(audioq);
// 分配解码过程中的使用缓存
//AVFrame* audioFrame = avcodec_alloc_frame();
AVFrame* audioFrame = av_frame_alloc(); //ffmpeg v4.2.2
mVideoState.aCodecCtx = aCodecCtx;
mVideoState.audioq = audioq;
mVideoState.audioFrame = audioFrame;
/// 打开SDL播放设备 - 开始
SDL_LockAudio();
SDL_AudioSpec spec;
SDL_AudioSpec wanted_spec;
wanted_spec.freq = aCodecCtx->sample_rate;
wanted_spec.format = AUDIO_S16SYS;
wanted_spec.channels = aCodecCtx->channels;
wanted_spec.silence = 0;
wanted_spec.samples = SDL_AUDIO_BUFFER_SIZE;
wanted_spec.callback = audio_callback;
wanted_spec.userdata = &mVideoState;
if(SDL_OpenAudio(&wanted_spec, &spec) < 0)
{
fprintf(stderr, "SDL_OpenAudio: %s\n", SDL_GetError());
return;
}
SDL_UnlockAudio();
SDL_PauseAudio(0);
/// 打开SDL播放设备 - 结束
///查找视频解码器
pCodecCtx = pFormatCtx->streams[videoStream]->codec;
qDebug() << "pCodecCtx->codec_id===========" << pCodecCtx->codec_id << endl;
//测试时这个值为27，查到枚举值对应的是AV_CODEC_ID_H264 ，即是H264压缩格式的文件。
pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
if(nullptr == pCodec)
{
printf("PCodec not found.\n");
emit signalDecodeError(-4);
return;
}
//打开视频解码器s
if(avcodec_open2(pCodecCtx, pCodec, nullptr) < 0)
{
printf("Could not open video codec.\n");
emit signalDecodeError(-5);
return;
}
mVideoState.video_st = pFormatCtx->streams[videoStream];
pFrame = av_frame_alloc();
pFrameRGB = av_frame_alloc();
img_convert_ctx = sws_getContext(pCodecCtx->width, pCodecCtx->height, \
pCodecCtx->pix_fmt, pCodecCtx->width, pCodecCtx->height, \
AV_PIX_FMT_RGB32, SWS_BICUBIC, nullptr, nullptr, nullptr);
numBytes = avpicture_get_size(AV_PIX_FMT_RGB32, pCodecCtx->width, pCodecCtx->height);
//av_image_get_buffer_size();
outBuffer = (uint8_t *)av_malloc(numBytes * sizeof(uint8_t));
avpicture_fill((AVPicture *)pFrameRGB, outBuffer, AV_PIX_FMT_RGB32, pCodecCtx->width, pCodecCtx->height);
int y_size = pCodecCtx->width * pCodecCtx->height;
packet = (AVPacket *)malloc(sizeof(AVPacket));//分配一个packet
av_new_packet(packet, y_size);//分配packet的数据
av_dump_format(pFormatCtx, 0, filePath, 0);//输出视频信息
int64_t start_time = av_gettime();
int64_t pts = 0; //当前视频的pts
int index = 0;
while (1)
{
if(av_read_frame(pFormatCtx, packet) < 0)
{
qDebug() << "index===============" << index;
break;//这里认为视频读取完了
}
int64_t realTime = av_gettime() - start_time; //主时钟时间
while(pts > realTime)
{
SDL_Delay(10);
realTime = av_gettime() - start_time; //主时钟时间
}
if(packet->stream_index == videoStream)
{
ret = avcodec_decode_video2(pCodecCtx, pFrame, &got_picture, packet);
if(ret < 0)
{
emit signalDecodeError(-6);
return;
}
//add
if (packet->dts == AV_NOPTS_VALUE && pFrame->opaque&& *(uint64_t*) pFrame->opaque != AV_NOPTS_VALUE)
{
pts = *(uint64_t *) pFrame->opaque;
}
else if (packet->dts != AV_NOPTS_VALUE)
{
pts = packet->dts;
}
else
{
pts = 0;
}
pts *= 1000000 * av_q2d(mVideoState.video_st->time_base);
pts = synchronize_video(&mVideoState, pFrame, pts);
//--------
if(got_picture)
{
sws_scale(img_convert_ctx, (uint8_t const * const *)pFrame->data,
pFrame->linesize, 0, pCodecCtx->height, pFrameRGB->data, pFrameRGB->linesize);
++index;
//把这个RGB数据 用QImage加载
QImage tempImage((uchar*)outBuffer, pCodecCtx->width, pCodecCtx->height, QImage::Format_RGB32);
QImage image = tempImage.copy();//把图像复制一份 传递给界面显示
qDebug() << "image.width==" << image.width() << "image.height==" << image.height();
emit signalGetOneFrame(image);
}
av_free_packet(packet);
}
else if( packet->stream_index == audioStream )
{
packet_queue_put(mVideoState.audioq, packet);
//这里我们将数据存入队列 因此不调用 av_free_packet 释放
}
else
{
// Free the packet that was allocated by av_read_frame
av_free_packet(packet);
}
}
av_free(outBuffer);
av_free(pFrameRGB);
avcodec_close(pCodecCtx);
avformat_close_input(&pFormatCtx);
}mainwindow.h
#ifndef MAINWINDOW_H
#define MAINWINDOW_H
#include <QMainWindow>
#include "CVideoPlayer.h"
#include <QImage>
QT_BEGIN_NAMESPACE
namespace Ui { class MainWindow; }
QT_END_NAMESPACE
class MainWindow : public QMainWindow
{
Q_OBJECT
public:
MainWindow(QWidget *parent = nullptr);
~MainWindow();
void initView();
void initData();
void showVideo();
public slots:
void slotDecodeError(int error);
void slotGetOneFrame(QImage image);
protected:
void paintEvent(QPaintEvent *event);
private:
Ui::MainWindow *ui;
CVideoPlayer *m_pVideoPlayer;
QImage m_Image;
};
#endif // MAINWINDOW_Hmainwindow.cpp
#include "mainwindow.h"
#include "ui_mainwindow.h"
#include <QDebug>
#include <QPainter>
#include <QFileDialog>
MainWindow::MainWindow(QWidget *parent)
: QMainWindow(parent)
, ui(new Ui::MainWindow)
{
ui->setupUi(this);
initView();
initData();
showVideo();
}
MainWindow::~MainWindow()
{
delete ui;
}
void MainWindow::initView()
{
//this->setStyleSheet(QString::fromUtf8("background-color: rgb(255, 255, 255);"));
}
void MainWindow::initData()
{
//av_register_all();//这个函数已被弃用
}
void MainWindow::showVideo()
{
m_pVideoPlayer = new CVideoPlayer();
connect(m_pVideoPlayer, SIGNAL(signalDecodeError(int)), this, SLOT(slotDecodeError(int)));
connect(m_pVideoPlayer, SIGNAL(signalGetOneFrame(QImage)), this, SLOT(slotGetOneFrame(QImage)));
m_pVideoPlayer->start();
}
void MainWindow::slotDecodeError(int error)
{
qDebug() << "slotDecodeError======error====" << error;
}
void MainWindow::slotGetOneFrame(QImage image)
{
m_Image = image;
update();//调用update将执行 paintEvent函数
}
void MainWindow::paintEvent(QPaintEvent *)
{
QPainter painter(this);
painter.setBrush(Qt::black);
painter.drawRect(0, 0, this->width(), this->height());//先画成黑色
if(m_Image.size().width() <= 0)
return;
//将图像按比例缩放成和窗口一样大小
QImage img = m_Image.scaled(this->size(), Qt::KeepAspectRatio);
int x = this->width() - img.width();
int y = this->height() - img.height();
x /= 2;
y /= 2;
painter.drawImage(QPoint(x, y), img);
}main.cpp
#include "mainwindow.h"
#include <QApplication>
int main(int argc, char *argv[])
{
QApplication a(argc, argv);
MainWindow w;
w.show();
return a.exec();
}运行效果如下：
声音有点杂音，后续优化
本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。
