项目场景:
最近学习使用ffmpeg在做一个视频缩略图的例子。在这里做一个记录。 在网上找了很多资料,加边框都是更改图片外围数据,这样图片会缺失一部分,不太符合我的需求。 按照这个思路,下面是一种实现在原始图片的基础上,在外部添加一个框。
需求描述:
- 需要获取指定时间点处的视频缩略图。
- 缩略图需要保持原有的长宽比列,缩放至480x360,缺失的部分使用纯色填充。
实现计划:
- 根据视频的分辨率和要求的480x360计算处,缩放后的视频分辨率。
- 视频需要解复用并且解码视频,使用ffmpeg直接软解。
- 视频需要缩放,考虑过avfilter过滤器,但是部分图片缩放后会花屏【还没有找到问题原因】。暂时先使用swscale来实现。
- 拿到缩放后的数据,分辨率可能不完全等于480x360,这个时候需要在数据外部填充一些纯色。
具体实现:
1.计算缩放后的视频分辨率
//视频原始大小
m_VideoWidth = pCodecCtx->width;
m_VideoHeight = pCodecCtx->height;
//calculate
//m_width 和 m_height 是用户规定的大小,这里设置为480x360
//m_SwsWidth 和 m_SwsHeight是视频缩放后的大小
if (m_VideoWidth <= m_width && m_VideoHeight <= m_height)
{
m_SwsWidth = m_VideoWidth;
m_SwsHeight = m_VideoHeight;
}else if(m_VideoWidth <= m_width && m_VideoHeight > m_height)
{
float cc_h = (double)m_VideoHeight / m_height;
m_SwsHeight = m_height;
m_SwsWidth = m_VideoWidth / cc_h;
}else if(m_VideoWidth > m_width && m_VideoHeight <= m_height)
{
float cc_w = (double)m_VideoWidth / m_width;
m_SwsWidth = m_width;
m_SwsHeight = m_VideoHeight / cc_w;
}else{//m_VideoWidth > m_width && m_VideoHeight > m_height
float cc_w = (double)m_VideoWidth / m_width;
float cc_h = (double)m_VideoHeight / m_height;
if(cc_w >= cc_h)
{
m_SwsWidth = m_width;
m_SwsHeight = m_VideoHeight / cc_w;
}else
{
m_SwsHeight = m_height;
m_SwsWidth = m_VideoWidth / cc_h;
}
}
printf("video:width = %d, height = %d.\n", m_VideoWidth, m_VideoHeight);
printf("user:width = %d, height = %d.\n", m_width, m_height);
printf("SWS:width = %d, height = %d.\n", m_SwsWidth, m_SwsHeight);
2.由于我使用的是ffmpeg 4.3.2版本,所以解码过程如下:
AVPacket packet;
AVFrame *frame;
frame = av_frame_alloc();
/* read all packets */
while (1) {
if ((ret = av_read_frame(fmt_ctx, &packet)) < 0)
break;
if (packet.stream_index == video_stream_index) {
ret = avcodec_send_packet(dec_ctx, &packet);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Error while sending a packet to the decoder\n");
break;
}
while (ret >= 0) {
ret = avcodec_receive_frame(dec_ctx, frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
break;
} else if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Error while receiving a frame from the decoder\n");
goto end;
}
//这里我们拿到了一包解码后的数据frame.
av_frame_unref(frame);
}
}
av_packet_unref(&packet);
}
3.使用swscale进行格式转换和缩放。
AVFrame *pRgba = av_frame_alloc();
int video_size = av_image_get_buffer_size(AV_PIX_FMT_RGBA, m_SwsWidth, m_SwsHeight, 1);
uint8_t * picture_buf = (uint8_t *)av_malloc(video_size);
//picture_buf 指向pRgba的data
av_image_fill_arrays(pRgba->data, pRgba->linesize, picture_buf, AV_PIX_FMT_RGBA, m_SwsWidth, m_SwsHeight, 1);
//视频像素数据格式转换上下文
SwsContext *sws_ctx = sws_getContext(dec_ctx->width, dec_ctx->height, dec_ctx->pix_fmt, /*INPUT W H FMT*/
m_SwsWidth, m_SwsHeight, AV_PIX_FMT_RGBA, /*OUTPUT W H FMT*/
SWS_BICUBIC, /*拉伸算法*/
NULL, NULL, NULL); /*其他参数*/
//将AVFrame转成视频像素数YUV420P格式
sws_scale(sws_ctx, (const uint8_t * const *)frame->data, frame->linesize,
0, dec_ctx->height,
pRgba->data, pRgba->linesize);
sws_freeContext(sws_ctx);
av_frame_free(&pRgba);
4.拿到picture_buf后,需要进行判断并填充。测试视频为3840x2160
了解rgba数据格式,参考:[FFmpeg] RGBA 和 YUV 存储方式
- m_width == m_SwsWidth && m_height == m_SwsHeight:
这个是最简单的,不需要任何处理。 - m_width == m_SwsWidth && m_height > m_SwsHeight:
宽度相同,但是高度不匹配,需要对上下分别填充 m_width * (m_height - m_SwsHeight)/2 边框。
/**
* -------------
* | top |
* -------------
* -------------
* | pic |
* -------------
* -------------
* | buttom |
* -------------
*/
int diff_h = m_height - m_SwsHeight;
int diff_top = diff_h/2;
int diff_buttom = diff_h - diff_top; //这里是防止diff_h 是一个奇数。
int pos = 0; //用来标记当前行的位置
//create top
for(int i = 0; i < diff_top; i++)
{
pos = i * m_width * 4;
for(int j = 0; j < m_width; j++)
{
m_picture_buf2[pos + j * 4 + 0] = 235;//R
m_picture_buf2[pos + j * 4 + 1] = 142;//G
m_picture_buf2[pos + j * 4 + 2] = 85;//B
m_picture_buf2[pos + j * 4 + 3] = 0;//A 0:完全透明 ===>> 1:完全不透明
}
}
//original buf
memcpy(picture_buf2 + diff_top * m_width * 4, picture_buf, m_SwsWidth * m_SwsHeight * 4);
//create buttom
for(int i = diff_top + m_SwsHeight; i < m_height; i++)
{
pos = i * m_width * 4;
for(int j = 0; j < m_width; j++)
{
m_picture_buf2[pos + j * 4 + 0] = 235;//R
m_picture_buf2[pos + j * 4 + 1] = 142;//G
m_picture_buf2[pos + j * 4 + 2] = 85;//B
m_picture_buf2[pos + j * 4 + 3] = 0;//A 0:完全透明 ===>> 1:完全不透明
}
}
原图缩放后:
加上边框后:
- m_width > m_SwsWidth && m_height == m_SwsHeight:
高度相同,但是宽度不匹配,需要对左右分别填充 (m_width - m_SwsWidth )/2 * m_height 边框。
/**
* |--left--||--pic--||--right--|
*/
int diff_w = m_width - m_SwsWidth;
int diff_left = diff_w/2;
int diff_right = diff_w - diff_left;
int pos = 0;//原图的位置
int pos_line = 0;//行位置
for(int i = 0; i < m_height; i++)
{
pos_line = i * m_width * 4;
for(int j = 0; j < m_width; j++)
{
if(j < diff_left || j >= (diff_left + m_SwsWidth))
{
m_picture_buf2[pos_line + j * 4 + 0] = 235; //R
m_picture_buf2[pos_line + j * 4 + 1] = 142; //G
m_picture_buf2[pos_line + j * 4 + 2] = 85; //B
m_picture_buf2[pos_line + j * 4 + 3] = 0; //A
}else
{
m_picture_buf2[pos_line + j * 4 + 0] = picture_buf[pos ++];//R
m_picture_buf2[pos_line + j * 4 + 1] = picture_buf[pos ++];//G
m_picture_buf2[pos_line + j * 4 + 2] = picture_buf[pos ++];//B
m_picture_buf2[pos_line + j * 4 + 3] = picture_buf[pos ++];//A
}
}
}
- m_width > m_SwsWidth && m_height > m_SwsHeight:
宽度和高度不匹配,需要对上下m_width * (m_height - m_SwsHeight)/2 边框,左右 (m_width - m_SwsWidth )/2 * m_SwsHeight。
/**
* |----------------------top-----------------------|
* |--left--||------------pic------------||--right--|
* |-------------------buttom-----------------------|
*/
//top & buttom 's height
int diff_h = m_height - m_SwsHeight;
int diff_top = diff_h/2;
int diff_buttom = diff_h - diff_top;
//left & right 's width
int diff_w = m_width - m_SwsWidth;
int diff_left = diff_w/2;
int diff_right = diff_w - diff_left;
int pos = 0;
int pos_line = 0;
for(int i = 0; i < m_height; i++)
{
pos_line = i * m_width * 4;
for(int j = 0; j < m_width; j++)
{
if(j < diff_left || j >= (diff_left + m_SwsWidth) || i < diff_top || i >= (diff_top + m_SwsHeight))
{
m_picture_buf2[pos_line + j * 4 + 0] = 235; //R
m_picture_buf2[pos_line + j * 4 + 1] = 142; //G
m_picture_buf2[pos_line + j * 4 + 2] = 85; //B
m_picture_buf2[pos_line + j * 4 + 3] = 0; //A
}else
{
m_picture_buf2[pos_line + j * 4 + 0] = picture_buf[pos ++];//R
m_picture_buf2[pos_line + j * 4 + 1] = picture_buf[pos ++];//G
m_picture_buf2[pos_line + j * 4 + 2] = picture_buf[pos ++];//B
m_picture_buf2[pos_line + j * 4 + 3] = picture_buf[pos ++];//A
}
}
}