android ffmpeg 多线程 android ffmpeg opengl

前言

在我的博文 中，我们在Android平台上，实现了通过FFmpeg在native（C/C++）层进行视频解码，并通过SurfaceView（基于ANativeWindow）实现了图像的显示。 实际上，我们也可以通过OpenGL来进行硬件的渲染工作，减少CPU的消耗，提高整个视频播放的性能。

实现需求

只演示视频解码和显示，不进行音频解码，也不做音视频同步，每两帧视频之间通过固定延时来间隔，所以视频播放时存在偏慢和偏快的问题；

基于FFmpeg来进行解码，而不是基于Android自带的MediaPlayer播放器，也不基于Android的mediacodec硬件解码；

视频显示层，在JAVA层基于SurfaceView，在原生（本地C/C++）层通过OpenGL来实现渲染（通过EGL与ANativeWindow进行连接）；

解码后的YUV视频数据，需要转换成rgb565le格式，这也是通过FFmpeg函数实现的格式转换，所以性能会偏低，后期我们再考虑OpenGL ES的转换方法；

实现架构

我们在整体上仍然沿用博文《基于FFmpeg和SurfaceView实现Android原生窗口（ANativeWindow）的视频播放》 中的播放流程设计，所以这里不再进行累述，差异主要体现在如下两点：

1. 需要将视频最后显示的步骤，由“通过ANativeWindow窗口显示视频”变换成“通过OpenGL显示视频”。这点很好理解，因为我们要借用OpenGL来进行图像渲染；

2. 由于OpenGL的渲染，最终都依赖于平台的适配层与本地显示窗口进行连接，所以我们需要对Android平台的OpenGL适配层（EGL）进行一些初始化管理。有关于EGL的详细介绍，可以参考博文：

。

下面是本文要实现的视频播放器主要架构图：

代码结构

这里增加了shader.cpp文件，主要用于OpenGL渲染的GLSL语言编程。有关OpenGL ES 2.0的基本知识，需要大家参考相关资料，了解GPU工作原理，以及着色语言编程方法。

其他代码结构和实现思路和博文《基于FFmpeg和SurfaceView实现Android原生窗口（ANativeWindow）的视频播放》 一致。

EGL相关代码

对比博文《基于FFmpeg和SurfaceView实现Android原生窗口（ANativeWindow）的视频播放》 ，在surface.cpp文件中，添加了EGL相关的代码处理，用于连接OpenGL和Android本地窗口ANativeWindow，函数eglOpen（）代码如下：

int eglOpen() {
    EGLDisplay eglDisplay = eglGetDisplay(EGL_DEFAULT_DISPLAY );
    if (eglDisplay == EGL_NO_DISPLAY ) {
        LOGV("eglGetDisplay failure.");
        return -1;
    }
    global_context.eglDisplay = eglDisplay;
    LOGV("eglGetDisplay ok");

    EGLint majorVersion;
    EGLint minorVersion;
    EGLBoolean success = eglInitialize(eglDisplay, &majorVersion,
            &minorVersion);
    if (!success) {
        LOGV("eglInitialize failure.");
        return -1;
    }
    LOGV("eglInitialize ok");

    GLint numConfigs;
    EGLConfig config;
    static const EGLint CONFIG_ATTRIBS[] = { EGL_BUFFER_SIZE, EGL_DONT_CARE,
            EGL_RED_SIZE, 5, EGL_GREEN_SIZE, 6, EGL_BLUE_SIZE, 5,
            EGL_DEPTH_SIZE, 16, EGL_ALPHA_SIZE, EGL_DONT_CARE, EGL_STENCIL_SIZE,
            EGL_DONT_CARE, EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
            EGL_SURFACE_TYPE, EGL_WINDOW_BIT, EGL_NONE // the end
            };
    success = eglChooseConfig(eglDisplay, CONFIG_ATTRIBS, &config, 1,
            &numConfigs);
    if (!success) {
        LOGV("eglChooseConfig failure.");
        return -1;
    }
    LOGV("eglChooseConfig ok");

    const EGLint attribs[] = { EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE };
    EGLContext elgContext = eglCreateContext(eglDisplay, config, EGL_NO_CONTEXT,
            attribs);
    if (elgContext == EGL_NO_CONTEXT ) {
        LOGV("eglCreateContext failure, error is %d", eglGetError());
        return -1;
    }
    global_context.eglContext = elgContext;
    LOGV("eglCreateContext ok");

    EGLint eglFormat;
    success = eglGetConfigAttrib(eglDisplay, config, EGL_NATIVE_VISUAL_ID,
            &eglFormat);
    if (!success) {
        LOGV("eglGetConfigAttrib failure.");
        return -1;
    }
    global_context.eglFormat = eglFormat;
    LOGV("eglGetConfigAttrib ok");

    EGLSurface eglSurface = eglCreateWindowSurface(eglDisplay, config,
            mANativeWindow, 0);
    if (NULL == eglSurface) {
        LOGV("eglCreateWindowSurface failure.");
        return -1;
    }
    global_context.eglSurface = eglSurface;
    LOGV("eglCreateWindowSurface ok");
    return 0;
}

按照eglGetDisplay->eglInitialize->eglChooseConfig->eglCreateContext ->eglGetConfigAttrib ->eglCreateWindowSurface的过程逐步完成egl和本地窗口的绑定。其中，配置参数如下：

static const EGLint CONFIG_ATTRIBS[] = { EGL_BUFFER_SIZE, EGL_DONT_CARE,
            EGL_RED_SIZE, 5, EGL_GREEN_SIZE, 6, EGL_BLUE_SIZE, 5,
            EGL_DEPTH_SIZE, 16, EGL_ALPHA_SIZE, EGL_DONT_CARE, EGL_STENCIL_SIZE,
            EGL_DONT_CARE, EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
            EGL_SURFACE_TYPE, EGL_WINDOW_BIT, EGL_NONE // the end
            };

EGL_RED_SIZE, 5, EGL_GREEN_SIZE, 6, EGL_BLUE_SIZE, 5表示我们最终渲染的图形是RGB565格式； EGL_DEPTH_SIZE, 16表示数据RGB565是16位的； EGL_ALPHA_SIZE, EGL_DONT_CARE，不考虑透明度alpha；

EGLSurface eglSurface = eglCreateWindowSurface(eglDisplay, config,
            mANativeWindow, 0);
    if (NULL == eglSurface) {
        LOGV("eglCreateWindowSurface failure.");
        return -1;
    }

OpenGL显示层和本地窗口ANativeWindow的绑定，就是通过上述函数的完成的。 下面是eglClose（）函数的代码：

int eglClose() {
    EGLBoolean success = eglDestroySurface(global_context.eglDisplay,
            global_context.eglSurface);
    if (!success) {
        LOGV("eglDestroySurface failure.");
    }

    success = eglDestroyContext(global_context.eglDisplay,
            global_context.eglContext);
    if (!success) {
        LOGV("eglDestroySurface failure.");
    }

    success = eglTerminate(global_context.eglDisplay);
    if (!success) {
        LOGV("eglDestroySurface failure.");
    }

    global_context.eglSurface = NULL;
    global_context.eglContext = NULL;
    global_context.eglDisplay = NULL;

    return 0;
}

基本上，和egl_open（）相比是逆向的调用。 egp_open（）和egl_close（）函数调用的上下文如下：

// obtain a native window from a Java surface
    mANativeWindow = ANativeWindow_fromSurface(env, surface);
    LOGV("mANativeWindow ok");

    if ((global_context.eglSurface != NULL)
            || (global_context.eglContext != NULL)
            || (global_context.eglDisplay != NULL)) {
        eglClose();
    }
    eglOpen();

    pthread_create(&thread_1, NULL, open_media, NULL);

当开启新的egl资源时，一定要记得关闭上一个资源。

OpenGL相关代码

这部分代码主要位于shader.cpp中，如下：

#include "player.h"

#define BYTES_PER_FLOAT 4
#define POSITION_COMPONENT_COUNT 2
#define TEXTURE_COORDINATES_COMPONENT_COUNT 2
#define STRIDE ((POSITION_COMPONENT_COUNT + TEXTURE_COORDINATES_COMPONENT_COUNT)*BYTES_PER_FLOAT)

/* type specifies the Shader type: GL_VERTEX_SHADER or GL_FRAGMENT_SHADER */
GLuint LoadShader(GLenum type, const char *shaderSrc) {
    GLuint shader;
    GLint compiled;

    // Create an empty shader object, which maintain the source code strings that define a shader
    shader = glCreateShader(type);

    if (shader == 0)
        return 0;

    // Replaces the source code in a shader object
    glShaderSource(shader, 1, &shaderSrc, NULL);

    // Compile the shader object
    glCompileShader(shader);

    // Check the shader object compile status
    glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled);

    if (!compiled) {
        GLint infoLen = 0;

        glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &infoLen);

        if (infoLen > 1) {
            GLchar* infoLog = (GLchar*) malloc(sizeof(GLchar) * infoLen);

            glGetShaderInfoLog(shader, infoLen, NULL, infoLog);
            LOGV("Error compiling shader:\n%s\n", infoLog);

            free(infoLog);
        }

        glDeleteShader(shader);
        return 0;
    }

    return shader;
}

GLuint LoadProgram(const char *vShaderStr, const char *fShaderStr) {
    GLuint vertexShader;
    GLuint fragmentShader;
    GLuint programObject;
    GLint linked;

    // Load the vertex/fragment shaders
    vertexShader = LoadShader(GL_VERTEX_SHADER, vShaderStr);
    fragmentShader = LoadShader(GL_FRAGMENT_SHADER, fShaderStr);

    // Create the program object
    programObject = glCreateProgram();
    if (programObject == 0)
        return 0;

    // Attaches a shader object to a program object
    glAttachShader(programObject, vertexShader);
    glAttachShader(programObject, fragmentShader);
    // Bind vPosition to attribute 0
    glBindAttribLocation(programObject, 0, "vPosition");
    // Link the program object
    glLinkProgram(programObject);

    // Check the link status
    glGetProgramiv(programObject, GL_LINK_STATUS, &linked);

    if (!linked) {
        GLint infoLen = 0;

        glGetProgramiv(programObject, GL_INFO_LOG_LENGTH, &infoLen);

        if (infoLen > 1) {
            GLchar* infoLog = (GLchar*) malloc(sizeof(GLchar) * infoLen);

            glGetProgramInfoLog(programObject, infoLen, NULL, infoLog);
            LOGV("Error linking program:\n%s\n", infoLog);

            free(infoLog);
        }

        glDeleteProgram(programObject);
        return GL_FALSE;
    }

    // Free no longer needed shader resources
    glDeleteShader(vertexShader);
    glDeleteShader(fragmentShader);

    return programObject;
}

int CreateProgram() {
    GLuint programObject;

    GLbyte vShaderStr[] = "attribute vec4 a_Position;           \n"
            "attribute vec2 a_TextureCoordinates;               \n"
            "varying vec2 v_TextureCoordinates;                 \n"
            "void main()                                        \n"
            "{                                                  \n"
            "    v_TextureCoordinates = a_TextureCoordinates;   \n"
            "    gl_Position = a_Position;                      \n"
            "}                                                  \n";

    GLbyte fShaderStr[] =
            "precision mediump float; \n"
                    "uniform sampler2D u_TextureUnit;                   \n"
                    "varying vec2 v_TextureCoordinates;                 \n"
                    "void main()                                        \n"
                    "{                                                  \n"
                    "    gl_FragColor = texture2D(u_TextureUnit, v_TextureCoordinates);  \n"
                    "}                                                  \n";

    // Load the shaders and get a linked program object
    programObject = LoadProgram((const char*) vShaderStr,
            (const char*) fShaderStr);
    if (programObject == 0) {
        return GL_FALSE;
    }

    // Store the program object
    global_context.glProgram = programObject;

    // Get the attribute locations
    global_context.positionLoc = glGetAttribLocation(programObject,
            "v_position");
    glClearColor(0.0f, 0.0f, 0.0f, 1.0f);

    glGenTextures(1, &global_context.mTextureID);
    glBindTexture(GL_TEXTURE_2D, global_context.mTextureID);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
    glEnable(GL_TEXTURE_2D);

    return 0;
}

void setUniforms(int uTextureUnitLocation, int textureId) {
    // Pass the matrix into the shader program.
    //glUniformMatrix4fv(uMatrixLocation, 1, false, matrix);

    // Set the active texture unit to texture unit 0.
    glActiveTexture(GL_TEXTURE0);

    // Bind the texture to this unit.
    glBindTexture(GL_TEXTURE_2D, textureId);

    // Tell the texture uniform sampler to use this texture in the shader by
    // telling it to read from texture unit 0.
    glUniform1i(uTextureUnitLocation, 0);
}

void Render(uint8_t *pixel) {
    GLfloat vVertices[] = { 0.0f, 0.5f, 0.0f, -0.5f, -0.5f, 0.0f, 0.5f, -0.5f,
            0.0f };

    // Set the viewport
    glViewport(0, 0, global_context.vcodec_ctx->width,
            global_context.vcodec_ctx->height);

    // Clear the color buffer
    //glClear(GL_COLOR_BUFFER_BIT);

    // Use the program object
    glUseProgram(global_context.glProgram);

    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, global_context.vcodec_ctx->width,
            global_context.vcodec_ctx->height, 0, GL_RGB,
            GL_UNSIGNED_SHORT_5_6_5, pixel);

    // Retrieve uniform locations for the shader program.
    GLint uTextureUnitLocation = glGetUniformLocation(global_context.glProgram,
            "u_TextureUnit");
    setUniforms(uTextureUnitLocation, global_context.mTextureID);

    // Retrieve attribute locations for the shader program.
    GLint aPositionLocation = glGetAttribLocation(global_context.glProgram,
            "a_Position");
    GLint aTextureCoordinatesLocation = glGetAttribLocation(
            global_context.glProgram, "a_TextureCoordinates");

    // Order of coordinates: X, Y, S, T
    // Triangle Fan
    GLfloat VERTEX_DATA[] = { 0.0f, 0.0f, 0.5f, 0.5f, -1.0f, -1.0f, 0.0f, 1.0f,
            1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 0.0f, -1.0f, 1.0f, 0.0f,
            0.0f, -1.0f, -1.0f, 0.0f, 1.0f };

    glVertexAttribPointer(aPositionLocation, POSITION_COMPONENT_COUNT, GL_FLOAT,
            false, STRIDE, VERTEX_DATA);
    glEnableVertexAttribArray(aPositionLocation);

    glVertexAttribPointer(aTextureCoordinatesLocation, POSITION_COMPONENT_COUNT,
            GL_FLOAT, false, STRIDE, &VERTEX_DATA[POSITION_COMPONENT_COUNT]);
    glEnableVertexAttribArray(aTextureCoordinatesLocation);

    glDrawArrays(GL_TRIANGLE_FAN, 0, 6);

    eglSwapBuffers(global_context.eglDisplay, global_context.eglSurface);
}

从上述代码可知，CreateProgram（）函数通过调用LoadProgram（）函数，继而调用LoadShader（）函数，完成GLSL源码的加载和编译，最终生成对应的程序program。通过glGenTextures和glBindTexture函数产生和绑定对应的纹理。 Render（）函数最终通过纹理，完成OpenGL的渲染。其中，VERTEX_DAT[ ]数组中存储的是纹理以及屏幕的坐标的顶点，在这里要注意纹理坐标和屏幕坐标的定义，以保证视频可以全屏显示，关于坐标系的介绍，可以参考博文 。 如下代码完成纹理坐标以及屏幕坐标的顶点设置：

glVertexAttribPointer(aPositionLocation, POSITION_COMPONENT_COUNT, GL_FLOAT,
            false, STRIDE, VERTEX_DATA);
    glEnableVertexAttribArray(aPositionLocation);

    glVertexAttribPointer(aTextureCoordinatesLocation, POSITION_COMPONENT_COUNT,
            GL_FLOAT, false, STRIDE, &VERTEX_DATA[POSITION_COMPONENT_COUNT]);
    glEnableVertexAttribArray(aTextureCoordinatesLocation);

最后，通过glDrawArrays函数，完成绘制。这里是从矩形的中心点开始，6个坐标点，完成扇形的4个三角形绘制。

glDrawArrays(GL_TRIANGLE_FAN, 0, 6);

由于egl采用双缓冲的方式绘制，所以最后要交换front/background画布。

eglSwapBuffers(global_context.eglDisplay, global_context.eglSurface);

最后，需要说明一下，CreateProgram（）函数和Render（）函数，都是在视频解码线程video_thread里调用的，需要通过如下函数，保证上下文的统一。

EGLBoolean success = eglMakeCurrent(global_context.eglDisplay,
            global_context.eglSurface, global_context.eglSurface,
            global_context.eglContext);

gitbub源码

请参考完整的源码路径： https://github.com/ericbars/FFmpegOpenGL