项目场景:


最近学习使用ffmpeg在做一个视频缩略图的例子。在这里做一个记录。 在网上找了很多资料,加边框都是更改图片外围数据,这样图片会缺失一部分,不太符合我的需求。 按照这个思路,下面是一种实现在原始图片的基础上,在外部添加一个框。


需求描述:

  • 需要获取指定时间点处的视频缩略图。
  • 缩略图需要保持原有的长宽比列,缩放至480x360,缺失的部分使用纯色填充。

实现计划:



  1. 根据视频的分辨率和要求的480x360计算处,缩放后的视频分辨率。
  2. 视频需要解复用并且解码视频,使用ffmpeg直接软解。
  3. 视频需要缩放,考虑过avfilter过滤器,但是部分图片缩放后会花屏【还没有找到问题原因】。暂时先使用swscale来实现。
  4. 拿到缩放后的数据,分辨率可能不完全等于480x360,这个时候需要在数据外部填充一些纯色。

具体实现:



1.计算缩放后的视频分辨率

//视频原始大小
m_VideoWidth = pCodecCtx->width;
m_VideoHeight = pCodecCtx->height;

//calculate
//m_width 和 m_height 是用户规定的大小,这里设置为480x360
//m_SwsWidth 和 m_SwsHeight是视频缩放后的大小
if (m_VideoWidth <= m_width && m_VideoHeight <= m_height)
{
    m_SwsWidth  = m_VideoWidth;
    m_SwsHeight = m_VideoHeight;
}else if(m_VideoWidth <= m_width && m_VideoHeight > m_height)
{
    float cc_h = (double)m_VideoHeight / m_height;
    m_SwsHeight = m_height;
    m_SwsWidth  = m_VideoWidth / cc_h;
}else if(m_VideoWidth > m_width && m_VideoHeight <= m_height)
{
    float cc_w = (double)m_VideoWidth / m_width;
    m_SwsWidth  = m_width;
    m_SwsHeight = m_VideoHeight / cc_w;
}else{//m_VideoWidth > m_width && m_VideoHeight > m_height
    float cc_w = (double)m_VideoWidth / m_width;
    float cc_h = (double)m_VideoHeight / m_height;
    if(cc_w >= cc_h)
    {
        m_SwsWidth  = m_width;
        m_SwsHeight = m_VideoHeight / cc_w;
    }else
    {
        m_SwsHeight = m_height;
        m_SwsWidth  = m_VideoWidth / cc_h;
    }
}

printf("video:width = %d, height = %d.\n", m_VideoWidth, m_VideoHeight);
printf("user:width = %d, height = %d.\n", m_width, m_height);
printf("SWS:width = %d, height = %d.\n", m_SwsWidth, m_SwsHeight);

2.由于我使用的是ffmpeg 4.3.2版本,所以解码过程如下:

AVPacket packet;
AVFrame *frame;

frame = av_frame_alloc();

/* read all packets */
  while (1) {
      if ((ret = av_read_frame(fmt_ctx, &packet)) < 0)
          break;

      if (packet.stream_index == video_stream_index) {
          ret = avcodec_send_packet(dec_ctx, &packet);
          if (ret < 0) {
              av_log(NULL, AV_LOG_ERROR, "Error while sending a packet to the decoder\n");
              break;
          }

          while (ret >= 0) {
              ret = avcodec_receive_frame(dec_ctx, frame);
              if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
                  break;
              } else if (ret < 0) {
                  av_log(NULL, AV_LOG_ERROR, "Error while receiving a frame from the decoder\n");
                  goto end;
              }
              //这里我们拿到了一包解码后的数据frame.
              av_frame_unref(frame);
          }
      }
      av_packet_unref(&packet);
  }

3.使用swscale进行格式转换和缩放。

AVFrame *pRgba = av_frame_alloc();

int video_size = av_image_get_buffer_size(AV_PIX_FMT_RGBA, m_SwsWidth, m_SwsHeight, 1);
uint8_t * picture_buf = (uint8_t *)av_malloc(video_size);
//picture_buf 指向pRgba的data
av_image_fill_arrays(pRgba->data, pRgba->linesize, picture_buf, AV_PIX_FMT_RGBA, m_SwsWidth, m_SwsHeight, 1);
//视频像素数据格式转换上下文
SwsContext *sws_ctx = sws_getContext(dec_ctx->width, dec_ctx->height, dec_ctx->pix_fmt, /*INPUT  W H FMT*/
                                     m_SwsWidth, m_SwsHeight, AV_PIX_FMT_RGBA,                 /*OUTPUT W H FMT*/
                                     SWS_BICUBIC,                                              /*拉伸算法*/
                                     NULL, NULL, NULL);                                        /*其他参数*/
//将AVFrame转成视频像素数YUV420P格式
sws_scale(sws_ctx, (const uint8_t * const *)frame->data, frame->linesize,
                                        0, dec_ctx->height, 
                                        pRgba->data, pRgba->linesize);
sws_freeContext(sws_ctx);
av_frame_free(&pRgba);

4.拿到picture_buf后,需要进行判断并填充。测试视频为3840x2160
了解rgba数据格式,参考:[FFmpeg] RGBA 和 YUV 存储方式

  • m_width == m_SwsWidth && m_height == m_SwsHeight:
    这个是最简单的,不需要任何处理。
  • m_width == m_SwsWidth && m_height > m_SwsHeight:
    宽度相同,但是高度不匹配,需要对上下分别填充 m_width * (m_height - m_SwsHeight)/2 边框。
/**
 * -------------
 * |    top    |
 * -------------
 * -------------
 * |    pic    |
 * -------------
 * -------------
 * |   buttom  |
 * -------------
 */ 
int diff_h       = m_height - m_SwsHeight;
int diff_top     = diff_h/2;
int diff_buttom  = diff_h - diff_top; //这里是防止diff_h 是一个奇数。

int pos = 0; //用来标记当前行的位置
//create top
for(int i = 0; i < diff_top; i++)
{
    pos = i * m_width * 4;
    for(int j  = 0; j < m_width; j++)
    {
        m_picture_buf2[pos + j * 4 + 0] = 235;//R
        m_picture_buf2[pos + j * 4 + 1] = 142;//G
        m_picture_buf2[pos + j * 4 + 2] = 85;//B
        m_picture_buf2[pos + j * 4 + 3] = 0;//A 0:完全透明  ===>>  1:完全不透明
    }
}
//original buf
memcpy(picture_buf2 + diff_top * m_width * 4, picture_buf, m_SwsWidth * m_SwsHeight * 4);

//create buttom
for(int i = diff_top + m_SwsHeight; i < m_height; i++)
{
    pos = i * m_width * 4;
    for(int j = 0; j < m_width; j++)
    {
        m_picture_buf2[pos + j * 4 + 0] = 235;//R
        m_picture_buf2[pos + j * 4 + 1] = 142;//G
        m_picture_buf2[pos + j * 4 + 2] = 85;//B
        m_picture_buf2[pos + j * 4 + 3] = 0;//A 0:完全透明  ===>>  1:完全不透明
    }
}

原图缩放后:

idesign软件设置rgb格式_ide


加上边框后:

idesign软件设置rgb格式_ffmpeg_02

  • m_width > m_SwsWidth && m_height == m_SwsHeight:
    高度相同,但是宽度不匹配,需要对左右分别填充 (m_width - m_SwsWidth )/2 * m_height 边框。
/**
 * |--left--||--pic--||--right--|
 */ 
int diff_w      = m_width - m_SwsWidth;
int diff_left   = diff_w/2;
int diff_right  = diff_w - diff_left;
int pos         = 0;//原图的位置
int pos_line    = 0;//行位置

for(int i = 0; i < m_height; i++)
{
    pos_line = i * m_width * 4;
    for(int j = 0; j < m_width; j++)
    {
        if(j < diff_left || j >= (diff_left + m_SwsWidth))
        {
            m_picture_buf2[pos_line + j * 4 + 0] = 235; //R
            m_picture_buf2[pos_line + j * 4 + 1] = 142; //G
            m_picture_buf2[pos_line + j * 4 + 2] = 85;  //B
            m_picture_buf2[pos_line + j * 4 + 3] = 0;   //A
        }else
        {
            m_picture_buf2[pos_line + j * 4 + 0] = picture_buf[pos ++];//R
            m_picture_buf2[pos_line + j * 4 + 1] = picture_buf[pos ++];//G
            m_picture_buf2[pos_line + j * 4 + 2] = picture_buf[pos ++];//B
            m_picture_buf2[pos_line + j * 4 + 3] = picture_buf[pos ++];//A
        }
    }
}
  • m_width > m_SwsWidth && m_height > m_SwsHeight:
    宽度和高度不匹配,需要对上下m_width * (m_height - m_SwsHeight)/2 边框,左右 (m_width - m_SwsWidth )/2 * m_SwsHeight。
/**
 * |----------------------top-----------------------|
 * |--left--||------------pic------------||--right--|
 * |-------------------buttom-----------------------|
 */ 
//top & buttom 's height
int diff_h      = m_height - m_SwsHeight;
int diff_top    = diff_h/2;
int diff_buttom = diff_h - diff_top;
//left & right 's width
int diff_w      = m_width - m_SwsWidth;
int diff_left   = diff_w/2;
int diff_right  = diff_w - diff_left;
int pos         = 0;
int pos_line    = 0;

for(int i = 0; i < m_height; i++)
{
    pos_line = i * m_width * 4;
    for(int j = 0; j < m_width; j++)
    {
        if(j < diff_left || j >= (diff_left + m_SwsWidth) || i < diff_top || i >= (diff_top + m_SwsHeight))
        {
            m_picture_buf2[pos_line + j * 4 + 0] = 235; //R
            m_picture_buf2[pos_line + j * 4 + 1] = 142; //G
            m_picture_buf2[pos_line + j * 4 + 2] = 85;  //B
            m_picture_buf2[pos_line + j * 4 + 3] = 0;   //A
        }else
        {
            m_picture_buf2[pos_line + j * 4 + 0] = picture_buf[pos ++];//R
            m_picture_buf2[pos_line + j * 4 + 1] = picture_buf[pos ++];//G
            m_picture_buf2[pos_line + j * 4 + 2] = picture_buf[pos ++];//B
            m_picture_buf2[pos_line + j * 4 + 3] = picture_buf[pos ++];//A
        }
    }
}