在Blender中使用代码控制人物模型的头部姿态 - 代码实践Dlib版本
flyfish
在文章的最后，贴上完整的代码
需要一个普通的摄像头，可以像虚拟UP主一样做头部动作
环境：
Ubuntu18.04
Blender 版本2.82
Dlib：用于人脸关键点检测
OpenCV版本3.4.16
solvePnP：从3D-2D点对应关系中找到一个目标的姿态，在这里用于通过人脸关键点计算出头部姿态
目标：输入二维的人脸关键点，输出三维的头部姿态
这里使用的dlib模型检测出68个关键点，在计算头部姿态时使用其中6个
关键点的索引分别是：代码中索引从0开始计算
下巴：8
鼻尖：30
左眼角：36
右眼角：45
左嘴角：48
右嘴角：54
3D头部模型（3D Head Model）
构建一个关键点的3D头部模型
所以会看到这样的代码
image_points = np.array([shape[30], # Nose tip - 31
shape[8], # Chin - 9
shape[36], # Left eye left corner - 37
shape[45], # Right eye right corne - 46
shape[48], # Left Mouth corner - 49
shape[54] # Right mouth corner - 55
], dtype = np.float32)
# 3D model points.
model_points = np.array([
(0.0, 0.0, 0.0), # Nose tip
(0.0, -330.0, -65.0), # Chin
(-225.0, 170.0, -135.0), # Left eye left corner
(225.0, 170.0, -135.0), # Right eye right corne
(-150.0, -150.0, -125.0), # Left Mouth corner
(150.0, -150.0, -125.0) # Right mouth corner
], dtype = np.float32)例如下巴：(0.0，-330.0，-65.0)这个叫世界坐标（World Coordinates），在OpenCV中叫模型坐标（Model Coordinates）
Blender中的目标存在一个世界中，摄像头拍摄的目标存在另一个世界中
在代码中，需要转换Blender和OpenCV和之间的坐标
通过对比两者的关系
X轴相同
Blender的Y轴是OpenCV的负Z轴
Blender的Z轴是OpenCV的Y轴
总结就是
Blender's red X axis = OpenCV 's X axis
Blender's green Y axis = OpenCV's -Z axis
Blender's blue Z axis = OpenCV's Y axis所以会看到这样的代码
bones["head_fk"].rotation_euler[0] = x
bones["head_fk"].rotation_euler[1] = -z
bones["head_fk"].rotation_euler[2] = y
solvePnP函数说明
输入
objectPoints - 世界坐标系下的关键点的坐标，即代码中的model_points
imagePoints - 在图像坐标系下对应的关键点的坐标。即代码中的image_points，与model_points的关键点顺序一致
cameraMatrix - 相机的内参矩阵
distCoeffs - 相机的畸变系数
输出
旋转向量(roatation vector)也就是上图中R
平移向量(translation vector)上图中的t
w：目标在世界坐标系下的坐标
c：目标在相机坐标系下的坐标
solvePnP求出w和c之间的映射关系
solvePnP函数返回结果包括旋转向量(roatation vector)和平移向量(translation vector)。这里只用了rotation_vector，赋值给bones[“head_fk”].rotation_euler
函数说明参考
https://docs.opencv.org/4.x/dc/d2c/tutorial_real_time_pose.html
solvePnP() 对异常点敏感,当相机拍摄目标时如果出现一些异常点，导致位姿估计的不准，可以尝试使用solvePnPRansac()
其他解决方案参考
https://docs.opencv.org/4.x/d9/d0c/group__calib3d.html#ga357634492a94efe8858d0ce1509da869 完整代码
import bpy
from imutils import face_utils
import dlib
import cv2
import time
import numpy as np
class DlibAnimOperator(bpy.types.Operator):
"""Operator which runs its self from a timer"""
bl_idname = "wm.dlib_operator"
bl_label = "Dlib Animation Operator"
p = "/media/ubuntu/data/tool/blender-2.82-linux64/face/shape_predictor_68_face_landmarks.dat"
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor(p)
# rig_name - take it from your scene collection tree
rig_name = "RIG-Vincent"
_timer = None
_cap = None
width = 800
height = 600
stop :bpy.props.BoolProperty()
# 3D model points.
model_points = np.array([
(0.0, 0.0, 0.0), # Nose tip
(0.0, -330.0, -65.0), # Chin
(-225.0, 170.0, -135.0), # Left eye left corner
(225.0, 170.0, -135.0), # Right eye right corne
(-150.0, -150.0, -125.0), # Left Mouth corner
(150.0, -150.0, -125.0) # Right mouth corner
], dtype = np.float32)
# Camera internals
camera_matrix = np.array(
[[height, 0.0, width/2],
[0.0, height, height/2],
[0.0, 0.0, 1.0]], dtype = np.float32
)
# Keeps a moving average of given length
def smooth_value(self, name, length, value):
if not hasattr(self, 'smooth'):
self.smooth = {}
if not name in self.smooth:
self.smooth[name] = np.array([value])
else:
self.smooth[name] = np.insert(arr=self.smooth[name], obj=0, values=value)
if self.smooth[name].size > length:
self.smooth[name] = np.delete(self.smooth[name], self.smooth[name].size-1, 0)
sum = 0
for val in self.smooth[name]:
sum += val
return sum / self.smooth[name].size
# Keeps min and max values, then returns the value in a ranve 0 - 1
def get_range(self, name, value):
if not hasattr(self, 'range'):
self.range = {}
if not name in self.range:
self.range[name] = np.array([value, value])
else:
self.range[name] = np.array([min(value, self.range[name][0]), max(value, self.range[name][1])] )
val_range = self.range[name][1] - self.range[name][0]
if val_range != 0:
return (value - self.range[name][0]) / val_range
else:
return 0
def modal(self, context, event):
if (event.type in {'RIGHTMOUSE', 'ESC'}) or self.stop == True:
self.cancel(context)
return {'CANCELLED'}
if event.type == 'TIMER':
self.init_camera()
_, image = self._cap.read()
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
rects = self.detector(gray, 0)
# bpy.context.scene.frame_set(frame_num)
# For each detected face, find the landmark.
for (i, rect) in enumerate(rects):
shape = self.predictor(gray, rect)
shape = face_utils.shape_to_np(shape)
#2D image points. If you change the image, you need to change vector
image_points = np.array([shape[30], # Nose tip - 31
shape[8], # Chin - 9
shape[36], # Left eye left corner - 37
shape[45], # Right eye right corne - 46
shape[48], # Left Mouth corner - 49
shape[54] # Right mouth corner - 55
], dtype = np.float32)
dist_coeffs = np.zeros((4,1)) # Assuming no lens distortion
if hasattr(self, 'rotation_vector'):
(success, self.rotation_vector, self.translation_vector) = cv2.solvePnP(self.model_points,
image_points, self.camera_matrix, dist_coeffs, flags=cv2.SOLVEPNP_ITERATIVE,
rvec=self.rotation_vector, tvec=self.translation_vector,
useExtrinsicGuess=True)
else:
(success, self.rotation_vector, self.translation_vector) = cv2.solvePnP(self.model_points,
image_points, self.camera_matrix, dist_coeffs, flags=cv2.SOLVEPNP_ITERATIVE,
useExtrinsicGuess=False)
if not hasattr(self, 'first_angle'):
self.first_angle = np.copy(self.rotation_vector)
bones = bpy.data.objects[self.rig_name].pose.bones
x=self.smooth_value("h_x", 3, (self.rotation_vector[0] - self.first_angle[0]))
y=self.smooth_value("h_y", 3, (self.rotation_vector[1] - self.first_angle[1]))
z=self.smooth_value("h_z", 3, (self.rotation_vector[2] - self.first_angle[2]))
bones["head_fk"].rotation_euler[0] = x
bones["head_fk"].rotation_euler[1] = -z
bones["head_fk"].rotation_euler[2] = y
bones["head_fk"].keyframe_insert(data_path="rotation_euler", index=-1)
for (x, y) in shape:
cv2.circle(image, (x, y), 2, (0, 255, 255), -1)
cv2.imshow("Output", image)
cv2.waitKey(1)
return {'PASS_THROUGH'}
def init_camera(self):
if self._cap == None:
self._cap = cv2.VideoCapture(0)
#self._cap = cv2.VideoCapture("/media/ubuntu/data/sign_videos/HabenSieSchmerzen0.mp4")
self._cap.set(cv2.CAP_PROP_FRAME_WIDTH, self.width)
self._cap.set(cv2.CAP_PROP_FRAME_HEIGHT, self.height)
self._cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)
time.sleep(0.5)
def stop_playback(self, scene):
print(format(scene.frame_current) + " / " + format(scene.frame_end))
if scene.frame_current == scene.frame_end:
bpy.ops.screen.animation_cancel(restore_frame=False)
def execute(self, context):
bpy.app.handlers.frame_change_pre.append(self.stop_playback)
wm = context.window_manager
self._timer = wm.event_timer_add(0.02, window=context.window)
wm.modal_handler_add(self)
return {'RUNNING_MODAL'}
def cancel(self, context):
wm = context.window_manager
wm.event_timer_remove(self._timer)
cv2.destroyAllWindows()
self._cap.release()
self._cap = None
def register():
bpy.utils.register_class(DlibAnimOperator)
def unregister():
bpy.utils.unregister_class(DlibAnimOperator)
if __name__ == "__main__":
register()
# test call
bpy.ops.wm.dlib_operator()恢复原始位置执行的代码
import bpy
import math
import mathutils
def Original():
ob = bpy.data.objects['RIG-Vincent']
head=ob.pose.bones['head_fk']
head.rotation_mode = 'XYZ'
head.rotation_euler = mathutils.Euler((0.0, 0.0, 0.0), 'XYZ')
head.rotation_mode = 'QUATERNION'
head.keyframe_insert(data_path="rotation_euler" ,frame=-1)
Original()
本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。
