相机处理是OpenGL一个重要的应用场景，因为OpenGL的主要工作是处理图像，而相机每秒生成几十帧图像，用GPU来处理再合适不过了。
至于Android CameraX和OpenGL的结合使用，网上有不少教程了，然而它们都有一个特点，就是给两者增加了不必要的耦合。由于两者本身架构都设计得非常好，实际上它们只需要一点耦合：就是OpenGL给Camera提供一个Surface。
如果分别实现了CameraX和OpenGL的Texture纹理图片，那么只需要改动极少的代码就能把两者结合起来。
1 CameraX使用
详见教程https://www.jianshu.com/p/f79855586ee2
在一个MainActivity中就能写完全部代码
typealias LumaListener = (luma: Double) -> Unit
class MainActivity : AppCompatActivity() {
private var imageCapture: ImageCapture? = null
private lateinit var outputDirectory: File
private lateinit var cameraExecutor: ExecutorService
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
setContentView(R.layout.activity_main)
if (allPermissionsGranted()) {
startCamera()
} else {
ActivityCompat.requestPermissions(this, REQUIRED_PERMISSIONS, REQUEST_CODE_PERMISSIONS)
}
camera_capture_button.setOnClickListener { takePhoto() }
outputDirectory = getOutputDirectory()
cameraExecutor = Executors.newSingleThreadExecutor()
}
private class LuminosityAnalyzer(private val listener : LumaListener) : ImageAnalysis.Analyzer {
private fun ByteBuffer.toByteArray() : ByteArray {
rewind()
val data = ByteArray(remaining())
get(data)
return data
}
override fun analyze(image: ImageProxy) {
val buffer = image.planes[0].buffer
val data = buffer.toByteArray()
val pixels = data.map { it.toInt() and 0xFF }
val luma = pixels.average()
listener(luma)
image.close()
}
}
private fun takePhoto() {
val imageCapture = imageCapture ?: return
val photoFile = File(
outputDirectory, SimpleDateFormat(FILENAME_FORMAT, Locale.US).format(System.currentTimeMillis()) + ".jpg"
)
val outputOptions = ImageCapture.OutputFileOptions.Builder(photoFile).build()
imageCapture.takePicture(
outputOptions, ContextCompat.getMainExecutor(this), object : ImageCapture.OnImageSavedCallback {
override fun onImageSaved(outputFileResults: ImageCapture.OutputFileResults) {
val savedUri = Uri.fromFile(photoFile)
val msg = "Photo captured: $savedUri"
Toast.makeText(baseContext, msg, Toast.LENGTH_SHORT).show()
Log.d("chao", msg)
}
override fun onError(exception: ImageCaptureException) {
exception.printStackTrace()
}
})
}
private fun startCamera() {
val cameraProviderFuture = ProcessCameraProvider.getInstance(this)
cameraProviderFuture.addListener(Runnable {
// 1. 选择摄像头，相机预览
val cameraSelector = CameraSelector.DEFAULT_BACK_CAMERA
val cameraProvider: ProcessCameraProvider = cameraProviderFuture.get()
val preview = Preview.Builder()
.setTargetResolution(Size(480, 640))
.build().also {
it.setSurfaceProvider(viewFinder.createSurfaceProvider())
}
// 2. 相机拍照
imageCapture = ImageCapture.Builder().build()
// 3. 数据处理
val imageAnalyzer = ImageAnalysis.Builder().build().also {
it.setAnalyzer(cameraExecutor, LuminosityAnalyzer {luma ->
Log.d("chao", "Average luminosity: $luma")
})
}
try {
cameraProvider.unbindAll()
cameraProvider.bindToLifecycle(this, cameraSelector, preview, imageCapture, imageAnalyzer)
} catch (e: Exception) {
e.printStackTrace()
}
}, ContextCompat.getMainExecutor(this))
}
private fun allPermissionsGranted() = REQUIRED_PERMISSIONS.all {
ContextCompat.checkSelfPermission(baseContext, it) == PackageManager.PERMISSION_GRANTED
}
private fun getOutputDirectory(): File {
val mediaDir = externalMediaDirs.firstOrNull()?.let {
File(it, resources.getString(R.string.app_name)).apply { mkdirs() }
}
return if (mediaDir != null && mediaDir.exists()) mediaDir else filesDir
}
override fun onDestroy() {
cameraExecutor.shutdown()
super.onDestroy()
}
override fun onRequestPermissionsResult(requestCode: Int, permissions: Array<out String>, grantResults: IntArray) {
if (requestCode == REQUEST_CODE_PERMISSIONS) {
if (allPermissionsGranted()) {
startCamera()
} else {
Toast.makeText(this,"Permissions not granted by the user.",
Toast.LENGTH_SHORT).show()
finish()
}
}
}
companion object {
private const val TAG = "CameraXBasic"
private const val FILENAME_FORMAT = "yyyy-MM-dd-HH-mm-ss-SSS"
private const val REQUEST_CODE_PERMISSIONS = 10
private val REQUIRED_PERMISSIONS = arrayOf(Manifest.permission.CAMERA)
}2 OpenGL绘制Texture图片
顶点数据里添加坐标、颜色和纹理坐标；
创建VAO、VBO和EBO；
创建GL_TEXTURE_2D纹理，用glUniform1i传给Shader；
绘制时用glUniformMatrix4fv传入变化矩阵给Shader，因为不需要变化，所以暂时用单元矩阵。
package com.example.opengles3camera
import android.graphics.Bitmap
import android.graphics.SurfaceTexture
import android.opengl.*
import android.opengl.GLES20.*
import androidx.camera.core.Preview
import androidx.camera.core.SurfaceRequest
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.nio.FloatBuffer
import java.nio.IntBuffer
import javax.microedition.khronos.egl.EGLConfig
import javax.microedition.khronos.opengles.GL10
class CameraRender: GLSurfaceView.Renderer {
var vertices = floatArrayOf( // ---- 位置 ---- ---- 颜色 ---- - 纹理坐标 -
0.5f, 0.5f, 0.0f, 1.0f, 0.0f, 0.0f, 1.0f, 1.0f, // 右上
0.5f, -0.5f, 0.0f, 0.0f, 1.0f, 0.0f, 1.0f, 0.0f, // 右下
-0.5f, -0.5f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, // 左下
-0.5f, 0.5f, 0.0f, 1.0f, 1.0f, 0.0f, 0.0f, 1.0f // 左上
)
val indices = intArrayOf( // 注意索引从0开始!
0, 1, 3, // 第一个三角形
1, 2, 3 // 第二个三角形
)
var program = 0
var vertexBuffer: FloatBuffer? = null
var intBuffer: IntBuffer? = null
var vao: IntArray = IntArray(1)
var tex: IntArray = IntArray(1)
override fun onSurfaceCreated(gl: GL10?, config: EGLConfig?) {
program = ShaderUtils.loadProgram()
//分配内存空间,每个浮点型占4字节空间
vertexBuffer = ByteBuffer.allocateDirect(vertices.size * 4)
.order(ByteOrder.nativeOrder())
.asFloatBuffer()
//传入指定的坐标数据
vertexBuffer!!.put(vertices)
vertexBuffer!!.position(0)
vao = IntArray(1)
GLES30.glGenVertexArrays(1, vao, 0)
GLES30.glBindVertexArray(vao[0])
val vbo = IntArray(1)
glGenBuffers(1, vbo, 0)
glBindBuffer(GL_ARRAY_BUFFER, vbo[0])
glBufferData(GL_ARRAY_BUFFER, vertices.size * 4, vertexBuffer, GL_STATIC_DRAW)
intBuffer = IntBuffer.allocate(indices.size * 4)
intBuffer!!.put(indices)
intBuffer!!.position(0)
val ebo = IntArray(1)
glGenBuffers(1, ebo, 0)
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo[0])
glBufferData(GL_ELEMENT_ARRAY_BUFFER, indices.size * 4, intBuffer, GL_STATIC_DRAW)
glGenTextures(1, tex, 0)
glActiveTexture(GL_TEXTURE0)
glBindTexture(GL_TEXTURE_2D, tex[0])
// 为当前绑定的纹理对象设置环绕、过滤方式
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT)
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT)
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR)
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)
val bitmap: Bitmap = ShaderUtils.loadImageAssets("face.png")
GLUtils.texImage2D(GL_TEXTURE_2D, 0, bitmap, 0)
glGenerateMipmap(GL_TEXTURE_2D)
glUseProgram(program)
val loc0 = glGetUniformLocation(program, "texture1")
glUniform1i(loc0, 0)
// Load the vertex data
glVertexAttribPointer(0, 3, GL_FLOAT, false, 8 * 4, 0)
glEnableVertexAttribArray(0)
glVertexAttribPointer(1, 3, GL_FLOAT, false, 8 * 4, 3 * 4)
glEnableVertexAttribArray(1)
glVertexAttribPointer(2, 2, GL_FLOAT, false, 8 * 4, 6 * 4)
glEnableVertexAttribArray(2)
glBindBuffer(GL_ARRAY_BUFFER, 0)
GLES30.glBindVertexArray(0)
glClearColor(0.5f, 0.5f, 0.5f, 0.5f)
}
override fun onSurfaceChanged(gl: GL10?, width: Int, height: Int) {
glViewport(0, 0, width, height)
}
var transform = FloatArray(16)
override fun onDrawFrame(gl: GL10?) {
// Clear the color buffer
glClear(GL_COLOR_BUFFER_BIT)
// Use the program object
glUseProgram(program)
glBindTexture(GL_TEXTURE_2D, tex[0])
Matrix.setIdentityM(transform, 0)
// Matrix.translateM(transform, 0, 0, 0, 0);
val loc = glGetUniformLocation(program, "transform")
glUniformMatrix4fv(loc, 1, false, transform, 0)
GLES30.glBindVertexArray(vao[0])
// glDrawArrays ( GL_TRIANGLES, 0, vertices.length );
glDrawElements(GL_TRIANGLES, vertices.size, GL_UNSIGNED_INT, 0)
}
}在MainActivity中做一点修改，创建GLSurfaceView添加到Activity中。
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
glSurfaceView = GLSurfaceView(this)
val screenW = resources.displayMetrics.widthPixels
glSurfaceView.layoutParams = FrameLayout.LayoutParams(screenW, screenW * 4 / 3)
setContentView(glSurfaceView)
glSurfaceView.setEGLContextClientVersion(3)
cameraRender = CameraRender()
glSurfaceView.setRenderer(cameraRender)
glSurfaceView.renderMode = GLSurfaceView.RENDERMODE_CONTINUOUSLY
if (allPermissionsGranted()) {
startCamera()
} else {
ActivityCompat.requestPermissions(this, REQUIRED_PERMISSIONS, REQUEST_CODE_PERMISSIONS)
}
// camera_capture_button.setOnClickListener { takePhoto() }
outputDirectory = getOutputDirectory()
cameraExecutor = Executors.newSingleThreadExecutor()
}3 用OpenGL预览相机
修改相机预览目标
将startCamera方法中setSurfaceProvider参数改成cameraRender
private fun startCamera() {
val cameraProviderFuture = ProcessCameraProvider.getInstance(this)
cameraProviderFuture.addListener(Runnable {
// 1. 选择摄像头，相机预览
val cameraSelector = CameraSelector.DEFAULT_BACK_CAMERA
val cameraProvider: ProcessCameraProvider = cameraProviderFuture.get()
val preview = Preview.Builder()
.setTargetResolution(Size(480, 640))
.build().also {
// it.setSurfaceProvider(viewFinder.createSurfaceProvider())
it.setSurfaceProvider(cameraRender)
}cameraRender创建OESTexture，实现Preview.SurfaceProvider接口
将原来创建GL_TEXTURE_2D的方法改成创建GL_TEXTURE_EXTERNAL_OES，并创建一个SurfaceTexture对象；
var surfaceTexture: SurfaceTexture? = null
private val executor = Executors.newSingleThreadExecutor()
override fun onSurfaceCreated(gl: GL10?, config: EGLConfig?) {
...
// glGenTextures(1, tex, 0)
// glActiveTexture(GL_TEXTURE0)
// glBindTexture(GL_TEXTURE_2D, tex[0])
// // 为当前绑定的纹理对象设置环绕、过滤方式
// glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT)
// glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT)
// glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR)
// glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)
// val bitmap: Bitmap = ShaderUtils.loadImageAssets("face.png")
// GLUtils.texImage2D(GL_TEXTURE_2D, 0, bitmap, 0)
// glGenerateMipmap(GL_TEXTURE_2D)
tex = createOESTexture()
surfaceTexture = SurfaceTexture(tex[0])
surfaceTexture?.setOnFrameAvailableListener {
// requestRender()
}
}
fun createOESTexture(): IntArray {
val arr = IntArray(1)
glGenTextures(1, arr, 0)
glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, arr[0])
glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR.toFloat())
glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR.toFloat())
glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE.toFloat())
glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE.toFloat())
return arr
}实现Preview.SurfaceProvider接口的onSurfaceRequested方法，根据方法提供的参数设置surfaceTexture的BufferSize，然后用surfaceTexture创建一个Surface，将它提供给SurfaceRequest。
class CameraRender: GLSurfaceView.Renderer, Preview.SurfaceProvider {
...
override fun onSurfaceRequested(request: SurfaceRequest) {
val size = request.resolution
surfaceTexture?.setDefaultBufferSize(size.width, size.height)
val surface = Surface(surfaceTexture)
request.provideSurface(surface, executor, Consumer {
surfaceTexture?.release()
surface.release()
})
}
}绘制
在绘制前调用surfaceTexture?.updateTexImage()更新画面，然后获取坐标变换矩阵给transform。
override fun onDrawFrame(gl: GL10?) {
// Clear the color buffer
glClear(GL_COLOR_BUFFER_BIT)
surfaceTexture?.updateTexImage()
surfaceTexture?.getTransformMatrix(transform)
// Use the program object
glUseProgram(program)
glBindTexture(GL_TEXTURE_2D, tex[0])
// Matrix.setIdentityM(transform, 0)
// Matrix.translateM(transform, 1, 1f, 0f, 0f);
var loc = glGetUniformLocation(program, "transform")
glUniformMatrix4fv(loc, 1, false, transform, 0)
GLES30.glBindVertexArray(vao[0])
// glDrawArrays ( GL_TRIANGLES, 0, vertices.length );
glDrawElements(GL_TRIANGLES, vertices.size, GL_UNSIGNED_INT, 0)
}Shader
在顶点着色器中用transform对纹理坐标进行转换。
#version 300 es
layout (location = 0) in vec3 vPosition;
layout (location = 1) in vec3 vColor;
layout (location = 2) in vec2 vTexCoord;
out vec3 aColor;
out vec2 aTexCoord;
uniform mat4 transform;
void main() {
gl_Position = vec4(vPosition, 1.0f);
aColor = vColor;
aTexCoord = (transform * vec4(vTexCoord, 1.0f, 1.0f)).xy;
}在片段着色器中将sampler2D改成samplerExternalOES，添加必要的extension。
#version 300 es
#extension GL_OES_EGL_image_external : require
#extension GL_OES_EGL_image_external_essl3 : require
precision mediump float;
in vec3 aColor;
in vec2 aTexCoord;
out vec4 fragColor;
uniform samplerExternalOES texture1;
void main() {
// 正常画面
fragColor = texture(texture1, aTexCoord);
}这样就能看到相机画面了。
4 实现简单滤镜
在片段着色器中稍作处理就能实现一些滤镜效果。
// 彩色滤镜效果
fragColor = mix(texture(texture1, aTexCoord), vec4(aColor, 1.0f), 0.5f);
// 四分屏效果
float s = aTexCoord.s;
float t = aTexCoord.t;
if (s > 0.5f) {
s = s - 0.5f;
}
if (t > 0.5f) {
t = t - 0.5f;
}
fragColor = texture(texture1, vec2(s, t));
// 马赛克效果
float s = aTexCoord.s;
float t = aTexCoord.t;
float msk = 50.0f;
s = float(int(s * msk)) / msk;
t = float(int(t * msk)) / msk;
fragColor = texture(texture1, vec2(s, t));
5 Github地址
完整项目在SurfacePaint项目下的opengles3camera模块里。
本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。
