sagemaker-debugger
sagemaker-debugger copied to clipboard
MXNet hook saving more tensors than specified
def test_save_shapes(out_dir, hook=None):
hook_created = False
if hook is None:
hook_created = True
global_reduce_config = ReductionConfig(save_raw_tensor=True)
global_save_config = SaveConfig(save_steps=[0, 1])
hook = t_hook(
out_dir=out_dir,
save_config=global_save_config,
include_collections=[
"weights",
"biases",
"gradients",
"default",
"ReluActivation",
"flatten",
],
reduction_config=global_reduce_config,
)
hook.get_collection("ReluActivation").include(["relu*"])
hook.get_collection("ReluActivation").save_config = SaveConfig(save_steps=[1])
hook.get_collection("flatten").include(["flatten*"])
hook.get_collection("ReluActivation").save_config = SaveConfig(save_steps=[1])
run_mnist_gluon_model(hook=hook, num_steps_train=10, num_steps_eval=10)
tr = create_trial(out_dir)
print(0, len(tr.tensor_names(step=0)))
print(1, len(tr.tensor_names(step=1)))
if hook_created:
shutil.rmtree(out_dir)
In step 0 it should only save 21 tensors, and 31 in step 1. But both steps save 31 tensors.