视频画中画实际就是两张纹理叠加，其中之一为前景，另外一个为背景。前景通常是一个小窗口叠加在背景上。
在 Android 应用中，通常视频源来自 Camera 或者其他经过硬解码（MediaCodec）而来的视频流。我们知道不论是 Camera 还是硬解码后的视频流都是可以直接送显到 Surface 上的（存在相应的 API 直接调用），所以可以通过 OpenGL ES 生成对应的纹理，然后包装成 Surface 交给对应的视频源去输出。需要注意的是，片段着色器中需要声明为 GL_OES_EGL_image_external，也就是外部扩展纹理。
有了以上方案基础就不难实现相应的代码了。首先来开发前景和背景纹理使用 Drawer。前景可经过一定的矩阵变换（缩小、平移），可以改变其位置和大小。
/**
* author : liuhongwei
* e-mail :
* date : 2021/7/22 16:28
* desc : OES绘制器
* version: 1.0
*/
public class OesDrawer {
private static final String TAG = "OesDrawer";
private final float[] mVertexCoors = new float[]{
-1.0F, -1.0F,
1.0F, -1.0F,
-1.0F, 1.0F,
1.0F, 1.0F};
private final float[] mTextureCoors = new float[]{
0.0F, 1.0F,
1.0F, 1.0F,
0.0F, 0.0F,
1.0F, 0.0F};
private int mTextureId = -1;
private SurfaceTexture mSurfaceTexture;
private int mProgram = -1;
private int mVertexMatrixHandler = -1;
private int mVertexPosHandler = -1;
private int mTexturePosHandler = -1;
private int mTextureHandler = -1;
private int mAlphaHandler = -1;
private FloatBuffer mVertexBuffer;
private FloatBuffer mTextureBuffer;
private final float[] mMatrix = new float[16];
private float mAlpha = 1.0F;
public OesDrawer(){
initPos();
Matrix.setIdentityM(mMatrix, 0);
}
public void scale(float sx, float sy) {
Matrix.scaleM(mMatrix, 0, sx, sy, 1f);
}
public void translate(float x, float y, float z) {
Matrix.translateM(mMatrix, 0, x, y, z);
}
private void initPos() {
ByteBuffer bb = ByteBuffer.allocateDirect(mVertexCoors.length * 4);
bb.order(ByteOrder.nativeOrder());
//将坐标数据转换为FloatBuffer，用以传入给OpenGL ES程序
mVertexBuffer = bb.asFloatBuffer();
mVertexBuffer.put(mVertexCoors);
mVertexBuffer.position(0);
ByteBuffer cc = ByteBuffer.allocateDirect(mTextureCoors.length * 4);
cc.order(ByteOrder.nativeOrder());
mTextureBuffer = cc.asFloatBuffer();
mTextureBuffer.put(mTextureCoors);
mTextureBuffer.position(0);
}
public void setAlpha(float alpha) {
this.mAlpha = alpha;
}
public void setTextureID(int id) {
mTextureId = id;
mSurfaceTexture = new SurfaceTexture(id);
}
public SurfaceTexture getSurfaceTexture() {
return mSurfaceTexture;
}
private void createGLPrg() {
if (mProgram == -1) {
int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, getVertexShader());
int fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, getFragmentShader());
//创建OpenGL ES程序，注意：需要在OpenGL渲染线程中创建，否则无法渲染
mProgram = GLES20.glCreateProgram();
//将顶点着色器加入到程序
GLES20.glAttachShader(mProgram, vertexShader);
//将片元着色器加入到程序中
GLES20.glAttachShader(mProgram, fragmentShader);
//连接到着色器程序
GLES20.glLinkProgram(mProgram);
mVertexMatrixHandler = GLES20.glGetUniformLocation(mProgram, "uMatrix");
mVertexPosHandler = GLES20.glGetAttribLocation(mProgram, "aPosition");
mTextureHandler = GLES20.glGetUniformLocation(mProgram, "uTexture");
mTexturePosHandler = GLES20.glGetAttribLocation(mProgram, "aCoordinate");
mAlphaHandler = GLES20.glGetAttribLocation(mProgram, "alpha");
}
//使用OpenGL程序
GLES20.glUseProgram(mProgram);
}
private void activateTexture() {
//激活指定纹理单元
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
//绑定纹理ID到纹理单元
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureId);
//将激活的纹理单元传递到着色器里面
GLES20.glUniform1i(mTextureHandler, 0);
//配置边缘过渡参数
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR * 1.0f);
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR * 1.0f);
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
}
public void updateTexture() {
if (null != mSurfaceTexture) {
mSurfaceTexture.updateTexImage();
}
}
public void doDraw() {
if (mTextureId != -1) {
createGLPrg();
activateTexture();
//启用顶点的句柄
GLES20.glEnableVertexAttribArray(mVertexPosHandler);
GLES20.glEnableVertexAttribArray(mTexturePosHandler);
GLES20.glUniformMatrix4fv(mVertexMatrixHandler, 1, false, mMatrix, 0);
//设置着色器参数， 第二个参数表示一个顶点包含的数据数量，这里为xy，所以为2
GLES20.glVertexAttribPointer(mVertexPosHandler, 2, GLES20.GL_FLOAT, false, 0, mVertexBuffer);
GLES20.glVertexAttribPointer(mTexturePosHandler, 2, GLES20.GL_FLOAT, false, 0, mTextureBuffer);
GLES20.glVertexAttrib1f(mAlphaHandler, mAlpha);
//开始绘制
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
}
}
public void release() {
GLES20.glDisableVertexAttribArray(mVertexPosHandler);
GLES20.glDisableVertexAttribArray(mTexturePosHandler);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);
GLES20.glDeleteTextures(1, new int[]{mTextureId}, 0);
GLES20.glDeleteProgram(mProgram);
}
private String getVertexShader() {
return "attribute vec4 aPosition;" +
"precision mediump float;" +
"uniform mat4 uMatrix;" +
"attribute vec2 aCoordinate;" +
"varying vec2 vCoordinate;" +
"attribute float alpha;" +
"varying float inAlpha;" +
"void main() {" +
" gl_Position = uMatrix*aPosition;" +
" vCoordinate = aCoordinate;" +
" inAlpha = alpha;" +
"}";
}
private String getFragmentShader() {
//一定要加换行"\n"，否则会和下一行的precision混在一起，导致编译出错
return "#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;" +
"varying vec2 vCoordinate;" +
"varying float inAlpha;" +
"uniform samplerExternalOES uTexture;" +
"void main() {" +
" vec4 color = texture2D(uTexture, vCoordinate);" +
" gl_FragColor = vec4(color.r, color.g, color.b, inAlpha);" +
"}";
}
private int loadShader(int type, String shaderCode) {
//根据type创建顶点着色器或者片元着色器
int shader = GLES20.glCreateShader(type);
//将资源加入到着色器中，并编译
GLES20.glShaderSource(shader, shaderCode);
GLES20.glCompileShader(shader);
return shader;
}
}前景和背景使用的 Drawer 开发好之后，需要更进一步，也就是将两张纹理直接绘制到 FBO，如此可以更加灵活的定制，比如还要贴一个水印 logo 都可以在 FBO Drawer 中处理。
/**
* author : liuhongwei
* e-mail :
* date : 2021/7/26 14:59
* desc : FBO绘制器
* version: 1.0
*/
public class FBODrawer {
private static final String TAG = "FBODrawer";
//单画面或PIP模式
private Mode mMode;
private int[] mFrameBuffers;
private int[] mFBOTextures;
private int mFrameWidth;
private int mFrameHeight;
private OesDrawer mBgOesDrawer;
private OesDrawer mFgOesDrawer;
public FBODrawer(){
mBgOesDrawer = new OesDrawer();
mFgOesDrawer = new OesDrawer();
}
public SurfaceTexture getBgSurfaceTexture() {
return mBgOesDrawer.getSurfaceTexture();
}
public SurfaceTexture getFgSurfaceTexture() {
return mFgOesDrawer.getSurfaceTexture();
}
private void loadFBO(int frameWidth, int frameHeight) {
mFrameWidth = frameWidth;
mFrameHeight = frameHeight;
if (mFrameBuffers != null) {
destroyFrameBuffers();
}
//创建FrameBuffer
mFrameBuffers = new int[1];
GLES20.glGenFramebuffers(mFrameBuffers.length, mFrameBuffers, 0);
//创建FBO中的纹理
mFBOTextures = new int[1];
GlesUtils.genTextures(mFBOTextures);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mFBOTextures[0]);
Log.d(TAG, "mFBOTextures[0]=" + mFBOTextures[0]);
//指定FBO纹理的输出图像的格式 RGBA
GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGBA, frameWidth, frameHeight,
0, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, null);
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, mFrameBuffers[0]);
//将fbo绑定到2d的纹理上
GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,
GLES20.GL_TEXTURE_2D, mFBOTextures[0], 0);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
}
private void destroyFrameBuffers() {
//删除fbo的纹理
if (mFBOTextures != null) {
GLES20.glDeleteTextures(1, mFBOTextures, 0);
mFBOTextures = null;
}
//删除fbo
if (mFrameBuffers != null) {
GLES20.glDeleteFramebuffers(1, mFrameBuffers, 0);
mFrameBuffers = null;
}
}
/**
* 前置纹理应用矩阵转换，右上角
* @param w 纹理宽
* @param h 纹理高
*/
private void applyMatrix2Fg(int w, int h){
// 先缩放，后平移
float scale = 0.4f;
mFgOesDrawer.scale(scale, scale);
float scaleW = w * scale;
float scaleH = h * scale;
float startX = w / 2.0f + scaleW / 2.0f;
float startY = h / 2.0f + scaleH / 2.0f;
float deltaX = w - startX;
float deltaY = h - startY;
mFgOesDrawer.translate(deltaX / scaleW * 2, deltaY / scaleH * 2, 0);
mFgOesDrawer.setAlpha(1.0f);
}
public void prepareDraw(int frameWidth, int frameHeight) {
applyMatrix2Fg(frameWidth, frameHeight);
loadFBO(frameWidth, frameHeight);
int[] textureIds = GlesUtils.createTextureIds(2);
mBgOesDrawer.setTextureID(textureIds[0]);
mFgOesDrawer.setTextureID(textureIds[1]);
Log.d(TAG, "textureIds[0]=" + textureIds[0]);
Log.d(TAG, "textureIds[1]=" + textureIds[1]);
//开启混合，即半透明
GLES20.glEnable(GLES20.GL_BLEND);
GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA);
}
public void destroyDraw() {
mBgOesDrawer.release();
mFgOesDrawer.release();
if (mFrameBuffers != null) {
destroyFrameBuffers();
}
}
public int doDraw() {
//锁定绘制的区域 绘制是从左下角开始的
GLES20.glViewport(0, 0, mFrameWidth, mFrameHeight);
//绑定FBO，在FBO上操作
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, mFrameBuffers[0]);
GLES20.glClearColor(0f, 0f, 0f, 0f);
mBgOesDrawer.doDraw();
if (mMode == Mode.PIP) {
mFgOesDrawer.doDraw();
}
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0);
return mFBOTextures[0];
}
public void setMode(Mode mode) {
this.mMode = mode;
}
public Mode getMode() {
return mMode;
}
public enum Mode {
SINGLE, PIP
}
}GlesUtils 工具类实现。
/**
* author : liuhongwei
* e-mail :
* date : 2021/7/22 13:46
* desc : openGL ES 工具类
* version: 1.0
*/
public class GlesUtils {
private final static String TAG = "GlesUtils";
public static int[] createTextureIds(int count) {
int[] texture = new int[count];
GLES20.glGenTextures(count, texture, 0); //生成纹理
return texture;
}
public static void genTextures(int[] textures) {
GLES20.glGenTextures(textures.length, textures, 0);
for (int texture : textures) {
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, texture);
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR);
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);
}
}
}最后，直接将 FBO 绘制到需要显示的 EGL 上，整个画中画预览就实现了。
本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。
