K0lb3
malloc error processing AnimationClip
Code
def extract_clip(animation_clip: UnityPy.classes.AnimationClip):
motion = {}
# read meta data
motion["Name"] = animation_clip.m_Name
motion["SampleRate"] = animation_clip.m_SampleRate
motion["Duration"] = format_float(animation_clip.m_MuscleClip.m_StopTime)
motion["TrackList"] = []
motion["Events"] = []
if is_verbose:
print(
f'[D] processing clip {motion["Name"]} in {list(animation_clip.assets_file.container.items())[0][0]}'
)
muscle_clip: UnityPy.classes.Clip = animation_clip.m_MuscleClip.m_Clip.data
streamed_frames = process_streamed_clip(muscle_clip.m_StreamedClip.data)
clip_binding_constant = animation_clip.m_ClipBindingConstant
for frame in streamed_frames:
time = frame["time"]
for curve_key in frame["keyList"]:
read_streamed_data(motion, clip_binding_constant, time, curve_key)
dense_clip = muscle_clip.m_DenseClip
stream_count = muscle_clip.m_StreamedClip.curveCount
for frame_idx in range(dense_clip.m_FrameCount):
time = dense_clip.m_BeginTime + frame_idx / dense_clip.m_SampleRate
for curve_idx in range(dense_clip.m_CurveCount):
idx = stream_count + curve_idx
read_curve_data(motion, clip_binding_constant, idx, time,
dense_clip.m_SampleArray, curve_idx)
constant_clip = muscle_clip.m_ConstantClip
dense_count = dense_clip.m_CurveCount
time2 = 0.0
for _ in range(2):
for curve_idx in range(len(constant_clip.data)):
idx = stream_count + dense_count + curve_idx
read_curve_data(motion, clip_binding_constant, idx, time2,
constant_clip.data, curve_idx)
time2 = animation_clip.m_MuscleClip.m_StopTime
for ev in animation_clip.m_Events:
motion["Events"].append({"time": ev.time, "value": ev.data})
def process_motion(env: UnityPy.Environment):
container_items = env.container.items()
# find BuildMotionData
mono_behav_obj: Any = next(
(i[1].read() for i in container_items
if i[1].type.name == "MonoBehaviour"
and "buildmotiondata" in i[0].lower()), None)
if mono_behav_obj is None:
print("MonoBehaviour 'BuildMotionData' not found.")
exit(1)
# find all AnimationClip
anim_clip_list: list[tuple[str, UnityPy.classes.AnimationClip]] = [
(i[0], i[1].deref().read()) for i in container_items # type: ignore
if i[1].type.name == "AnimationClip"
] # type: ignore
# get facial list
is_facial_from_behav = len(mono_behav_obj.Facials) != 0
if not is_facial_from_behav:
facial_list = [i[1] for i in anim_clip_list if "facial" in i[0].split('/')[-2].lower()]
else:
facial_list = mono_behav_obj.Facials
# extract facial clips
facial_map = {}
for facial in facial_list:
if isinstance(facial, UnityPy.classes.AnimationClip):
facial_clip = facial
facial_clip_name = facial_clip.m_Name
else:
facial_clip = facial.Clip.deref()
facial_clip_name = facial.ClipAssetName
if not facial_clip:
facial_clip = next(
(i[1] for i in anim_clip_list
if facial.ClipAssetName in i[0]), None)
if not facial_clip:
print(f'[W] unable to find facial clip for {facial.ClipAssetName}, skipping',
file=sys.stderr)
continue
else:
print(f'[W] found facial clip in animation clip list for {facial.ClipAssetName}',
file=sys.stderr)
else:
facial_clip = facial_clip.read()
facial_map[facial_clip_name] = extract_clip(facial_clip)
Error
Python(1029,0x20551cf40) malloc: *** error for object 0x10134a910: pointer being freed was not allocated
Python(1029,0x20551cf40) malloc: *** set a breakpoint in malloc_error_break to debug
Bug The cpp implementation of AnimationClip seems not correct.
To Reproduce
- a copy of the file that causes the problem 03honami_motion_base.zip
- following data:
- Python version: 3.10.14
- UnityPy version: 1.20.4
The unpack_floats and unpack_ints are left-over functions from pre UnityPy 1.20.
Currently they shouldn't be used, as the PackedBitVector contains a list of ints instead of bytes.
Converting that to bytes and then using the C function to parse it is not that much faster than just unpacking the data in python.
So instead of using these two C functions, it would be better to use the new unpack functions in UnityPy.helpers.PackedBitVector.
The error you encountered might either be caused due to an invalid input or due to a weird behavior of free. In theory free should never error for a given pointer returned by malloc. Instead malloc should either already error if one tries to reserve an invalid size, or simply return a non-null pointer that can be freed.
I didn't use the unpack-floats and unpack_ints functions.
The code of other three functions used above, some code maybe pre-1.20 because I've revert all changes:
def format_float(num):
if isinstance(num, float) and int(num) == num:
return int(num)
elif isinstance(num, float):
return float("{:.3f}".format(num))
return num
class StreamedCurveKey(object):
def __init__(self, bs):
super().__init__()
self.index = bs.readUInt32()
self.coeff = [bs.readFloat() for i in range(3)]
self.outSlope = self.coeff[2]
self.value = bs.readFloat()
self.inSlope = 0.0
def __repr__(self) -> str:
return str({
"index": self.index,
"coeff": self.coeff,
"inSlope": self.inSlope,
"outSlope": self.outSlope,
"value": self.value
})
def calc_next_in_slope(self, dx, rhs):
if self.coeff[0] == 0 and self.coeff[1] == 0 and self.coeff[2] == 0:
return float('Inf')
dx = max(dx, 0.0001)
dy = rhs.value - self.value
length = 1.0 / (dx * dx)
d1 = self.outSlope * dx
d2 = dy + dy + dy - d1 - d1 - self.coeff[1] / length
return d2 / dx
def process_streamed_clip(streamed_clip):
_b = struct.pack('I' * len(streamed_clip), *streamed_clip)
bs = BinaryStream(BytesIO(_b))
ret = []
# key_list = []
while bs.base_stream.tell() < len(_b):
time = bs.readFloat()
num_keys = bs.readUInt32()
key_list = []
for i in range(num_keys):
key_list.append(StreamedCurveKey(bs))
assert (len(key_list) == num_keys)
if time < 0:
continue
ret.append({"time": time, "keyList": key_list})
# if is_verbose:
# print(ret)
for k, v in enumerate(ret):
if k < 2 or k == len(ret) - 1: continue
for ck in v["keyList"]:
for fI in range(k - 1, 0, -1):
pre_frame = ret[fI]
pre_curve_key = next(
(x for x in pre_frame["keyList"] if x.index == ck.index),
None)
if pre_curve_key:
ck.inSlope = pre_curve_key.calc_next_in_slope(
v["time"] - pre_frame["time"], ck)
break
return ret
def read_streamed_data(motion, clip_binding_constant, time, curve_key):
idx = curve_key.index
binding_constant = find_binding(clip_binding_constant, idx)
if binding_constant is None:
print(f'binding constant not found for {idx}', file=sys.stderr)
return
mono_script = binding_constant.script.deref().read()
target, bone_name = live2d_target_map[mono_script.name]
if not bone_name:
bone_name = str(binding_constant.path)
if bone_name:
track = next(
(x for x in motion["TrackList"] if x["Name"] == bone_name), None)
if not track:
track = {
"Name":
bone_name,
"Target":
target,
"Curve": [{
"time": time,
"value": curve_key.value,
"inSlope": curve_key.inSlope,
"outSlope": curve_key.outSlope,
"coeff": curve_key.coeff
}]
}
motion["TrackList"].append(track)
else:
# track["Target"] = target
track["Curve"].append({
"time": time,
"value": curve_key.value,
"inSlope": curve_key.inSlope,
"outSlope": curve_key.outSlope,
"coeff": curve_key.coeff
})
def read_curve_data(motion, clip_binding_constant, idx, time, sample_list,
curve_idx):
binding_constant = find_binding(clip_binding_constant, idx)
if binding_constant is None:
print(f'binding constant not found for {idx}', file=sys.stderr)
return
mono_script = binding_constant.script.deref().read()
target, bone_name = live2d_target_map[mono_script.name]
if not bone_name:
bone_name = str(binding_constant.path)
if bone_name:
track = next(
(x for x in motion["TrackList"] if x["Name"] == bone_name), None)
if not track:
track = {
"Name":
bone_name,
"Target":
target,
"Curve": [{
"time": time,
"value": sample_list[curve_idx],
"inSlope": 0,
"outSlope": 0,
"coeff": None
}]
}
motion["TrackList"].append(track)
else:
# track["Target"] = target
track["Curve"].append({
"time": time,
"value": sample_list[curve_idx],
"inSlope": 0,
"outSlope": 0,
"coeff": None
})
None of this errors or causes a problem on my side. The "find_binding" and "live2d_target_map" are missing btw.
UnityPy itself doesn't throw any error and can read all objects fine, and as it doesn't do any post processing automatically anymore, I doubt that UnityPy is at fault.
Please pinpoint the problem to a section and prove that the error is caused there, and then explain how UnityPy is connected.
I tried, the problem happens after processing of that file, and UnityPy is the only module I used here with cpp integration. all my other parts are pure Python and should not cause any malloc and free faults. I can provide you my full script and test samples if you want.
You can try applying the following settings.
UnityPy.helpers.MeshHelper.UnityPyBoost = None
UnityPy.helpers.TypeTreeHelper.read_typetree_boost = None
I doubt that it should be the TypeTreeHelper, and the MeshHelper should in theory at most leak memory.
Thanks for the hints! It seems that the crash is caused by UnityPy.helpers.TypeTreeHelper.read_typetree_boost. My script does not crash if I set it to None.
I fixed the C typetree reader during the last days, so you should be able to use it again for the typetree parsing without having to fear segfaults.