deep-motion-editing
deep-motion-editing copied to clipboard
DeepMotionEditing
更换数据集（human3.6m）去重新训练retargeting 模型
我将human3.6m部分bvh数据集作为groupA , 然后您提供的Mixamo数据集中的***_m.bvh作为训练集中的groupB，训练结束后使用human3.6m中的另一部分作为测试集去测试，结果是正确的。但是我再使用项目VideoTo3dPoseAndBvh将视频中提取出来的bvh （观察过该bvh结构和human3.6m的相同）作为输入时，retarget的结果就是错乱的动作，可否有可以提供的思路去解决的？
我不太了解你提到的VideoTo3dPoseAndBvh项目，也许可以检查一下video提取出来的bvh的rest pose和训练集是否相同。在rest pose不同的情况下，即使表示同一个动作也会用到非常不一样的旋转。
另一方面，根据我们的实验以及一些反馈，我们的方法有可能会对某个数据集过拟合，也就是说如果用数据集A进行训练，而在数据集B上进行测试可能是行不通的。这里的问题可能就是视频中提取的bvh和human3.6m数据集差异很大。
我检查了测试时候video提取出来的bvh的rest pose和训练集（human3.6的bvh）的相同欸。 这里有个疑问的地方，我的训练集是通过human3.6的3D坐标提取出来的bvh,一开始提取的时候发现提取出来的bvh用blender打开预览的时候人物动作是背对着，这个时候我强制给他转身（下面的代码）让它是面对着我们。这样子处理会不会影响每个关节的旋转角度？ for frame in prediction3dpoint: for point3d in #frame: point3d[0] = point3d[0] / 100 point3d[1] = point3d[1] / 100 point3d[2] = point3d[2] / 100 ## 交换Y和Z的坐标 X = point3d[0] Y = point3d[1] Z = point3d[2] ## point3d[0] = -X ## original point3d[0] = X point3d[1] = Z ## point3d[2] = Y ## original point3d[2] = -Y
@xinzi2018 , hi, how did you convert the human3.6m motion to bvh, could you please make your code public available? thanks very much
import json
import os
import numpy as np
from bvh_skeleton import h36m_skeleton,smartbody_skeleton
def write_smartbody_bvh(outbvhfilepath,prediction3dpoint):
# '''
# :param outbvhfilepath: 输出bvh动作文件路径
# :param prediction3dpoint: 预测的三维关节点
# :return:
# '''
# 将预测的点放大100倍
for frame in prediction3dpoint:
for point3d in frame:
# point3d[0] *= 100
# point3d[1] *= 100
# point3d[2] *= 100
# my add
point3d[0] = point3d[0] / 100
point3d[1] = point3d[1] / 100
point3d[2] = point3d[2] / 100
# 交换Y和Z的坐标
X = point3d[0]
Y = point3d[1]
Z = point3d[2]
# point3d[0] = -X # original
point3d[0] = X
point3d[1] = Z
point3d[2] = -Y
dir_name = os.path.dirname(outbvhfilepath)
if not os.path.exists(dir_name):
os.makedirs(dir_name)
# dir_name = os.path.dirname(outbvhfilepath)
# basename = os.path.basename(outbvhfilepath)
# video_name = basename[:basename.rfind('.')]
# bvhfileDirectory = os.path.join(dir_name, video_name, "bvh")
# if not os.path.exists(bvhfileDirectory):
# os.makedirs(bvhfileDirectory)
# bvhfileName = os.path.join(dir_name, video_name, "bvh", "{}.bvh".format(video_name))
SmartBody_skeleton = smartbody_skeleton.SmartBodySkeleton()
SmartBody_skeleton.poses2bvh(prediction3dpoint, output_file=outbvhfilepath)
def write_standard_bvh(outbvhfilepath,prediction3dpoint):
# '''
# :param outbvhfilepath: 输出bvh动作文件路径
# :param prediction3dpoint: 预测的三维关节点
# :return:
# '''
# 将预测的点放大100倍
for frame in prediction3dpoint:
for point3d in frame:
# my add
point3d[0] = point3d[0] / 100
point3d[1] = point3d[1] / 100
point3d[2] = point3d[2] / 100
# 交换Y和Z的坐标
X = point3d[0]
Y = point3d[1]
Z = point3d[2]
point3d[0] = -X
point3d[1] = Z
point3d[2] = Y
dir_name = os.path.dirname(outbvhfilepath)
if not os.path.exists(dir_name):
os.makedirs(dir_name)
# dir_name = os.path.dirname(outbvhfilepath)
# basename = os.path.basename(outbvhfilepath)
# video_name = basename[:basename.rfind('.')]
# bvhfileDirectory = os.path.join(dir_name, video_name, "bvh")
# if not os.path.exists(bvhfileDirectory):
# os.makedirs(bvhfileDirectory)
# bvhfileName = os.path.join(dir_name, video_name, "bvh", "{}.bvh".format(video_name))
human36m_skeleton = h36m_skeleton.H36mSkeleton()
human36m_skeleton.poses2bvh(prediction3dpoint, output_file=outbvhfilepath)
def read_json(file):
with open(file, 'r') as f:
temp = json.loads(f.read())
return temp
def read_points_3d(content):
flag = 0
for value in content.values():
if flag == 0:
points_3d = np.array(value)[np.newaxis, :, :]
flag = 1
else:
temp = np.array(value)[np.newaxis, :, :]
points_3d = np.concatenate((points_3d, temp), axis=0)
return points_3d
if __name__ == '__main__':
index = [1, 5, 6, 7, 8, 9, 11]
# index = [5]
kind = ['camera', 'data', 'joint_3d']
annotations_dir = '../annotations'
filelists = os.listdir(annotations_dir)
for i in index:
# # camera
# filename_camera = 'Human36M_subject{}_{}.json'.format(i,kind[0])
# file_path = os.path.join(annotations_dir, filename_camera)
# content_camera = read_json(file_path)
#
# # data
# filename_data = 'Human36M_subject{}_{}.json'.format(i, kind[1])
# file_path = os.path.join(annotations_dir, filename_data)
# content_data = read_json(file_path)
# joint_3d
filename_joint_3d = 'Human36M_subject{}_{}.json'.format(i, kind[2])
file_path = os.path.join(annotations_dir, filename_joint_3d)
content_joint_3d = read_json(file_path)
for content1_key in content_joint_3d.keys():
for content2_key in content_joint_3d[content1_key].keys():
content3 = content_joint_3d[content1_key][content2_key]
points_3d = read_points_3d(content3)
save_path = './human3.6m/{}/{}_{}.bvh'.format(i, content1_key,content2_key)
write_smartbody_bvh(save_path, points_3d)
# write_standard_bvh(save_path, points_3d)
print(save_path)
a = 0
其中头文件中提到的bvh_skeleton 来自VideoTo3dPoseAndBvh
我将human3.6m部分bvh数据集作为groupA , 然后您提供的Mixamo数据集中的***_m.bvh作为训练集中的groupB，训练结束后使用human3.6m中的另一部分作为测试集去测试，结果是正确的。但是我再使用项目VideoTo3dPoseAndBvh将视频中提取出来的bvh （观察过该bvh结构和human3.6m的相同）作为输入时，retarget的结果就是错乱的动作，可否有可以提供的思路去解决的？
@xinzi2018 ，大佬解决了这个问题了吗？
我将human3.6m部分bvh数据集作为groupA , 然后您提供的Mixamo数据集中的***_m.bvh作为训练集中的groupB，训练结束后使用human3.6m中的另一部分作为测试集去测试，结果是正确的。但是我再使用项目VideoTo3dPoseAndBvh将视频中提取出来的bvh （观察过该bvh结构和human3.6m的相同）作为输入时，retarget的结果就是错乱的动作，可否有可以提供的思路去解决的？
@xinzi2018 ，大佬解决了这个问题了吗？
这是因为视频中提取出来的bvh的人物没有训练的原因吧，把motion粘贴到human3.6m中的一个人身上调整一下motion里的offset应该就可以了？
