how to modify example.py if use openshape-pointbert-vitg14-rgb

[MathamPollard]

[eliphatfs]

Did you meet any problems by just changing the model name?

[MathamPollard]

yes,I change

"def load_model(config, model_path="OpenShape/openshape-spconv-all"): model = models.make(config).cuda()

if config.model.name.startswith('Mink'):
    model = ME.MinkowskiSyncBatchNorm.convert_sync_batchnorm(model) # minkowski only
else:
    model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model)

checkpoint = torch.load(f"{model_path}/model.pt", weights_only=True)
model_dict = OrderedDict()
pattern = re.compile('module.')
for k,v in checkpoint['state_dict'].items():
    if re.search("module", k):
        model_dict[re.sub(pattern, '', k)] = v
model.load_state_dict(model_dict)
return model"

to

"def load_model(config, model_path="OpenShape/openshape-pointbert-vitg14-rgb"): model = models.make(config).cuda()

if config.model.name.startswith('Mink'):
    model = ME.MinkowskiSyncBatchNorm.convert_sync_batchnorm(model) # minkowski only
else:
    model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model)

checkpoint = torch.load(f"{model_path}/model.pt", weights_only=True)
model_dict = OrderedDict()
pattern = re.compile('module.')
for k,v in checkpoint['state_dict'].items():
    if re.search("module", k):
        model_dict[re.sub(pattern, '', k)] = v
model.load_state_dict(model_dict)
return model"

the error occurred.It seems the model is not instantiated properly

Traceback (most recent call last): File "/home/mdisk1/tanjunwen/anaconda3/envs/OpenShape/lib/python3.9/runpy.py", line 197, in _run_module_as_main return _run_code(code, main_globals, None, File "/home/mdisk1/tanjunwen/anaconda3/envs/OpenShape/lib/python3.9/runpy.py", line 87, in _run_code exec(code, run_globals) File "/home/junwentan/.vscode-server/extensions/ms-python.debugpy-2024.1.10371006-linux-x64/bundled/libs/debugpy/adapter/../../debugpy/launcher/../../debugpy/main.py", line 39, in cli.main() File "/home/junwentan/.vscode-server/extensions/ms-python.debugpy-2024.1.10371006-linux-x64/bundled/libs/debugpy/adapter/../../debugpy/launcher/../../debugpy/../debugpy/server/cli.py", line 430, in main run() File "/home/junwentan/.vscode-server/extensions/ms-python.debugpy-2024.1.10371006-linux-x64/bundled/libs/debugpy/adapter/../../debugpy/launcher/../../debugpy/../debugpy/server/cli.py", line 284, in run_file runpy.run_path(target, run_name="main") File "/home/junwentan/.vscode-server/extensions/ms-python.debugpy-2024.1.10371006-linux-x64/bundled/libs/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_runpy.py", line 321, in run_path return _run_module_code(code, init_globals, run_name, File "/home/junwentan/.vscode-server/extensions/ms-python.debugpy-2024.1.10371006-linux-x64/bundled/libs/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_runpy.py", line 135, in _run_module_code _run_code(code, mod_globals, init_globals, File "/home/junwentan/.vscode-server/extensions/ms-python.debugpy-2024.1.10371006-linux-x64/bundled/libs/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_runpy.py", line 124, in _run_code exec(code, run_globals) File "/home/mdisk1/tanjunwen/gitprj/3D/OpenShape_code/src/example.py", line 98, in model = load_model(config) File "/home/mdisk1/tanjunwen/gitprj/3D/OpenShape_code/src/example.py", line 77, in load_model model.load_state_dict(model_dict) File "/home/mdisk1/tanjunwen/anaconda3/envs/OpenShape/lib/python3.9/site-packages/torch/nn/modules/module.py", line 2215, in load_state_dict raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format( RuntimeError: Error(s) in loading state_dict for MinkResNet34: Missing key(s) in state_dict: "conv0p1s1.kernel", "bn0.bn.weight", "bn0.bn.bias", "bn0.bn.running_mean", "bn0.bn.running_var", "conv1p1s2.kernel", "bn1.bn.weight", "bn1.bn.bias", "bn1.bn.running_mean", "bn1.bn.running_var", "block1.0.conv1.kernel", "block1.0.norm1.bn.weight", "block1.0.norm1.bn.bias", "block1.0.norm1.bn.running_mean", "block1.0.norm1.bn.running_var", "block1.0.conv2.kernel", "block1.0.norm2.bn.weight", "block1.0.norm2.bn.bias", "block1.0.norm2.bn.running_mean", "block1.0.norm2.bn.running_var", "block1.1.conv1.kernel", "block1.1.norm1.bn.weight", "block1.1.norm1.bn.bias", "block1.1.norm1.bn.running_mean", "block1.1.norm1.bn.running_var", "block1.1.conv2.kernel", "block1.1.norm2.bn.weight", "block1.1.norm2.bn.bias", "block1.1.norm2.bn.running_mean", "block1.1.norm2.bn.running_var", "block1.2.conv1.kernel", "block1.2.norm1.bn.weight", "block1.2.norm1.bn.bias", "block1.2.norm1.bn.running_mean", "block1.2.norm1.bn.running_var", "block1.2.conv2.kernel", "block1.2.norm2.bn.weight", "block1.2.norm2.bn.bias", "block1.2.norm2.bn.running_mean", "block1.2.norm2.bn.running_var", "conv2p2s2.kernel", "bn2.bn.weight", "bn2.bn.bias", "bn2.bn.running_mean", "bn2.bn.running_var", "block2.0.conv1.kernel", "block2.0.norm1.bn.weight", "block2.0.norm1.bn.bias", "block2.0.norm1.bn.running_mean", "block2.0.norm1.bn.running_var", "block2.0.conv2.kernel", "block2.0.norm2.bn.weight", "block2.0.norm2.bn.bias", "block2.0.norm2.bn.running_mean", "block2.0.norm2.bn.running_var", "block2.0.downsample.0.kernel", "block2.0.downsample.1.bn.weight", "block2.0.downsample.1.bn.bias", "block2.0.downsample.1.bn.running_mean", "block2.0.downsample.1.bn.running_var", "block2.1.conv1.kernel", "block2.1.norm1.bn.weight", "block2.1.norm1.bn.bias", "block2.1.norm1.bn.running_mean", "block2.1.norm1.bn.running_var", "block2.1.conv2.kernel", "block2.1.norm2.bn.weight", "block2.1.norm2.bn.bias", "block2.1.norm2.bn.running_mean", "block2.1.norm2.bn.running_var", "block2.2.conv1.kernel", "block2.2.norm1.bn.weight", "block2.2.norm1.bn.bias", "block2.2.norm1.bn.running_mean", "block2.2.norm1.bn.running_var", "block2.2.conv2.kernel", "block2.2.norm2.bn.weight", "block2.2.norm2.bn.bias", "block2.2.norm2.bn.running_mean", "block2.2.norm2.bn.running_var", "block2.3.conv1.kernel", "block2.3.norm1.bn.weight", "block2.3.norm1.bn.bias", "block2.3.norm1.bn.running_mean", "block2.3.norm1.bn.running_var", "block2.3.conv2.kernel", "block2.3.norm2.bn.weight", "block2.3.norm2.bn.bias", "block2.3.norm2.bn.running_mean", "block2.3.norm2.bn.running_var", "conv3p4s2.kernel", "bn3.bn.weight", "bn3.bn.bias", "bn3.bn.running_mean", "bn3.bn.running_var", "block3.0.conv1.kernel", "block3.0.norm1.bn.weight", "block3.0.norm1.bn.bias", "block3.0.norm1.bn.running_mean", "block3.0.norm1.bn.running_var", "block3.0.conv2.kernel", "block3.0.norm2.bn.weight", "block3.0.norm2.bn.bias", "block3.0.norm2.bn.running_mean", "block3.0.norm2.bn.running_var", "block3.0.downsample.0.kernel", "block3.0.downsample.1.bn.weight", "block3.0.downsample.1.bn.bias", "block3.0.downsample.1.bn.running_mean", "block3.0.downsample.1.bn.running_var", "block3.1.conv1.kernel", "block3.1.norm1.bn.weight", "block3.1.norm1.bn.bias", "block3.1.norm1.bn.running_mean", "block3.1.norm1.bn.running_var", "block3.1.conv2.kernel", "block3.1.norm2.bn.weight", "block3.1.norm2.bn.bias", "block3.1.norm2.bn.running_mean", "block3.1.norm2.bn.running_var", "block3.2.conv1.kernel", "block3.2.norm1.bn.weight", "block3.2.norm1.bn.bias", "block3.2.norm1.bn.running_mean", "block3.2.norm1.bn.running_var", "block3.2.conv2.kernel", "block3.2.norm2.bn.weight", "block3.2.norm2.bn.bias", "block3.2.norm2.bn.running_mean", "block3.2.norm2.bn.running_var", "block3.3.conv1.kernel", "block3.3.norm1.bn.weight", "block3.3.norm1.bn.bias", "block3.3.norm1.bn.running_mean", "block3.3.norm1.bn.running_var", "block3.3.conv2.kernel", "block3.3.norm2.bn.weight", "block3.3.norm2.bn.bias", "block3.3.norm2.bn.running_mean", "block3.3.norm2.bn.running_var", "block3.4.conv1.kernel", "block3.4.norm1.bn.weight", "block3.4.norm1.bn.bias", "block3.4.norm1.bn.running_mean", "block3.4.norm1.bn.running_var", "block3.4.conv2.kernel", "block3.4.norm2.bn.weight", "block3.4.norm2.bn.bias", "block3.4.norm2.bn.running_mean", "block3.4.norm2.bn.running_var", "block3.5.conv1.kernel", "block3.5.norm1.bn.weight", "block3.5.norm1.bn.bias", "block3.5.norm1.bn.running_mean", "block3.5.norm1.bn.running_var", "block3.5.conv2.kernel", "block3.5.norm2.bn.weight", "block3.5.norm2.bn.bias", "block3.5.norm2.bn.running_mean", "block3.5.norm2.bn.running_var", "conv4p8s2.kernel", "bn4.bn.weight", "bn4.bn.bias", "bn4.bn.running_mean", "bn4.bn.running_var", "block4.0.conv1.kernel", "block4.0.norm1.bn.weight", "block4.0.norm1.bn.bias", "block4.0.norm1.bn.running_mean", "block4.0.norm1.bn.running_var", "block4.0.conv2.kernel", "block4.0.norm2.bn.weight", "block4.0.norm2.bn.bias", "block4.0.norm2.bn.running_mean", "block4.0.norm2.bn.running_var", "block4.0.downsample.0.kernel", "block4.0.downsample.1.bn.weight", "block4.0.downsample.1.bn.bias", "block4.0.downsample.1.bn.running_mean", "block4.0.downsample.1.bn.running_var", "block4.1.conv1.kernel", "block4.1.norm1.bn.weight", "block4.1.norm1.bn.bias", "block4.1.norm1.bn.running_mean", "block4.1.norm1.bn.running_var", "block4.1.conv2.kernel", "block4.1.norm2.bn.weight", "block4.1.norm2.bn.bias", "block4.1.norm2.bn.running_mean", "block4.1.norm2.bn.running_var", "block4.2.conv1.kernel", "block4.2.norm1.bn.weight", "block4.2.norm1.bn.bias", "block4.2.norm1.bn.running_mean", "block4.2.norm1.bn.running_var", "block4.2.conv2.kernel", "block4.2.norm2.bn.weight", "block4.2.norm2.bn.bias", "block4.2.norm2.bn.running_mean", "block4.2.norm2.bn.running_var", "conv5.0.0.kernel", "conv5.0.1.bn.weight", "conv5.0.1.bn.bias", "conv5.0.1.bn.running_mean", "conv5.0.1.bn.running_var", "conv5.1.0.kernel", "conv5.1.1.bn.weight", "conv5.1.1.bn.bias", "conv5.1.1.bn.running_mean", "conv5.1.1.bn.running_var", "final.0.0.linear.weight", "final.0.1.bn.weight", "final.0.1.bn.bias", "final.0.1.bn.running_mean", "final.0.1.bn.running_var", "final.2.0.linear.weight", "final.2.1.bn.weight", "final.2.1.bn.bias", "final.2.1.bn.running_mean", "final.2.1.bn.running_var", "final.3.linear.weight", "final.3.linear.bias". Unexpected key(s) in state_dict: "ppat.cls_token", "ppat.sa.mlp_convs.0.weight", "ppat.sa.mlp_convs.0.bias", "ppat.sa.mlp_convs.1.weight", "ppat.sa.mlp_convs.1.bias", "ppat.sa.mlp_convs.2.weight", "ppat.sa.mlp_convs.2.bias", "ppat.sa.mlp_bns.0.weight", "ppat.sa.mlp_bns.0.bias", "ppat.sa.mlp_bns.0.running_mean", "ppat.sa.mlp_bns.0.running_var", "ppat.sa.mlp_bns.0.num_batches_tracked", "ppat.sa.mlp_bns.1.weight", "ppat.sa.mlp_bns.1.bias", "ppat.sa.mlp_bns.1.running_mean", "ppat.sa.mlp_bns.1.running_var", "ppat.sa.mlp_bns.1.num_batches_tracked", "ppat.sa.mlp_bns.2.weight", "ppat.sa.mlp_bns.2.bias", "ppat.sa.mlp_bns.2.running_mean", "ppat.sa.mlp_bns.2.running_var", "ppat.sa.mlp_bns.2.num_batches_tracked", "ppat.lift.0.weight", "ppat.lift.0.bias", "ppat.lift.2.weight", "ppat.lift.2.bias", "ppat.transformer.layers.0.0.norm.weight", "ppat.transformer.layers.0.0.norm.bias", "ppat.transformer.layers.0.0.fn.to_qkv.weight", "ppat.transformer.layers.0.0.fn.to_out.0.weight", "ppat.transformer.layers.0.0.fn.to_out.0.bias", "ppat.transformer.layers.0.1.norm.weight", "ppat.transformer.layers.0.1.norm.bias", "ppat.transformer.layers.0.1.fn.net.0.weight", "ppat.transformer.layers.0.1.fn.net.0.bias", "ppat.transformer.layers.0.1.fn.net.3.weight", "ppat.transformer.layers.0.1.fn.net.3.bias", "ppat.transformer.layers.1.0.norm.weight", "ppat.transformer.layers.1.0.norm.bias", "ppat.transformer.layers.1.0.fn.to_qkv.weight", "ppat.transformer.layers.1.0.fn.to_out.0.weight", "ppat.transformer.layers.1.0.fn.to_out.0.bias", "ppat.transformer.layers.1.1.norm.weight", "ppat.transformer.layers.1.1.norm.bias", "ppat.transformer.layers.1.1.fn.net.0.weight", "ppat.transformer.layers.1.1.fn.net.0.bias", "ppat.transformer.layers.1.1.fn.net.3.weight", "ppat.transformer.layers.1.1.fn.net.3.bias", "ppat.transformer.layers.2.0.norm.weight", "ppat.transformer.layers.2.0.norm.bias", "ppat.transformer.layers.2.0.fn.to_qkv.weight", "ppat.transformer.layers.2.0.fn.to_out.0.weight", "ppat.transformer.layers.2.0.fn.to_out.0.bias", "ppat.transformer.layers.2.1.norm.weight", "ppat.transformer.layers.2.1.norm.bias", "ppat.transformer.layers.2.1.fn.net.0.weight", "ppat.transformer.layers.2.1.fn.net.0.bias", "ppat.transformer.layers.2.1.fn.net.3.weight", "ppat.transformer.layers.2.1.fn.net.3.bias", "ppat.transformer.layers.3.0.norm.weight", "ppat.transformer.layers.3.0.norm.bias", "ppat.transformer.layers.3.0.fn.to_qkv.weight", "ppat.transformer.layers.3.0.fn.to_out.0.weight", "ppat.transformer.layers.3.0.fn.to_out.0.bias", "ppat.transformer.layers.3.1.norm.weight", "ppat.transformer.layers.3.1.norm.bias", "ppat.transformer.layers.3.1.fn.net.0.weight", "ppat.transformer.layers.3.1.fn.net.0.bias", "ppat.transformer.layers.3.1.fn.net.3.weight", "ppat.transformer.layers.3.1.fn.net.3.bias", "ppat.transformer.layers.4.0.norm.weight", "ppat.transformer.layers.4.0.norm.bias", "ppat.transformer.layers.4.0.fn.to_qkv.weight", "ppat.transformer.layers.4.0.fn.to_out.0.weight", "ppat.transformer.layers.4.0.fn.to_out.0.bias", "ppat.transformer.layers.4.1.norm.weight", "ppat.transformer.layers.4.1.norm.bias", "ppat.transformer.layers.4.1.fn.net.0.weight", "ppat.transformer.layers.4.1.fn.net.0.bias", "ppat.transformer.layers.4.1.fn.net.3.weight", "ppat.transformer.layers.4.1.fn.net.3.bias", "ppat.transformer.layers.5.0.norm.weight", "ppat.transformer.layers.5.0.norm.bias", "ppat.transformer.layers.5.0.fn.to_qkv.weight", "ppat.transformer.layers.5.0.fn.to_out.0.weight", "ppat.transformer.layers.5.0.fn.to_out.0.bias", "ppat.transformer.layers.5.1.norm.weight", "ppat.transformer.layers.5.1.norm.bias", "ppat.transformer.layers.5.1.fn.net.0.weight", "ppat.transformer.layers.5.1.fn.net.0.bias", "ppat.transformer.layers.5.1.fn.net.3.weight", "ppat.transformer.layers.5.1.fn.net.3.bias", "ppat.transformer.layers.6.0.norm.weight", "ppat.transformer.layers.6.0.norm.bias", "ppat.transformer.layers.6.0.fn.to_qkv.weight", "ppat.transformer.layers.6.0.fn.to_out.0.weight", "ppat.transformer.layers.6.0.fn.to_out.0.bias", "ppat.transformer.layers.6.1.norm.weight", "ppat.transformer.layers.6.1.norm.bias", "ppat.transformer.layers.6.1.fn.net.0.weight", "ppat.transformer.layers.6.1.fn.net.0.bias", "ppat.transformer.layers.6.1.fn.net.3.weight", "ppat.transformer.layers.6.1.fn.net.3.bias", "ppat.transformer.layers.7.0.norm.weight", "ppat.transformer.layers.7.0.norm.bias", "ppat.transformer.layers.7.0.fn.to_qkv.weight", "ppat.transformer.layers.7.0.fn.to_out.0.weight", "ppat.transformer.layers.7.0.fn.to_out.0.bias", "ppat.transformer.layers.7.1.norm.weight", "ppat.transformer.layers.7.1.norm.bias", "ppat.transformer.layers.7.1.fn.net.0.weight", "ppat.transformer.layers.7.1.fn.net.0.bias", "ppat.transformer.layers.7.1.fn.net.3.weight", "ppat.transformer.layers.7.1.fn.net.3.bias", "ppat.transformer.layers.8.0.norm.weight", "ppat.transformer.layers.8.0.norm.bias", "ppat.transformer.layers.8.0.fn.to_qkv.weight", "ppat.transformer.layers.8.0.fn.to_out.0.weight", "ppat.transformer.layers.8.0.fn.to_out.0.bias", "ppat.transformer.layers.8.1.norm.weight", "ppat.transformer.layers.8.1.norm.bias", "ppat.transformer.layers.8.1.fn.net.0.weight", "ppat.transformer.layers.8.1.fn.net.0.bias", "ppat.transformer.layers.8.1.fn.net.3.weight", "ppat.transformer.layers.8.1.fn.net.3.bias", "ppat.transformer.layers.9.0.norm.weight", "ppat.transformer.layers.9.0.norm.bias", "ppat.transformer.layers.9.0.fn.to_qkv.weight", "ppat.transformer.layers.9.0.fn.to_out.0.weight", "ppat.transformer.layers.9.0.fn.to_out.0.bias", "ppat.transformer.layers.9.1.norm.weight", "ppat.transformer.layers.9.1.norm.bias", "ppat.transformer.layers.9.1.fn.net.0.weight", "ppat.transformer.layers.9.1.fn.net.0.bias", "ppat.transformer.layers.9.1.fn.net.3.weight", "ppat.transformer.layers.9.1.fn.net.3.bias", "ppat.transformer.layers.10.0.norm.weight", "ppat.transformer.layers.10.0.norm.bias", "ppat.transformer.layers.10.0.fn.to_qkv.weight", "ppat.transformer.layers.10.0.fn.to_out.0.weight", "ppat.transformer.layers.10.0.fn.to_out.0.bias", "ppat.transformer.layers.10.1.norm.weight", "ppat.transformer.layers.10.1.norm.bias", "ppat.transformer.layers.10.1.fn.net.0.weight", "ppat.transformer.layers.10.1.fn.net.0.bias", "ppat.transformer.layers.10.1.fn.net.3.weight", "ppat.transformer.layers.10.1.fn.net.3.bias", "ppat.transformer.layers.11.0.norm.weight", "ppat.transformer.layers.11.0.norm.bias", "ppat.transformer.layers.11.0.fn.to_qkv.weight", "ppat.transformer.layers.11.0.fn.to_out.0.weight", "ppat.transformer.layers.11.0.fn.to_out.0.bias", "ppat.transformer.layers.11.1.norm.weight", "ppat.transformer.layers.11.1.norm.bias", "ppat.transformer.layers.11.1.fn.net.0.weight", "ppat.transformer.layers.11.1.fn.net.0.bias", "ppat.transformer.layers.11.1.fn.net.3.weight", "ppat.transformer.layers.11.1.fn.net.3.bias", "proj.weight", "proj.bias".

[eliphatfs]

you need to add some arguments to the command line; see the readme: replace main.py with example.py for what you need

python3 src/main.py --trial_name pointbert_all model.name=PointBERT model.scaling=4 model.use_dense=True training.lr=0.0005 training.lr_decay_rate=0.967 

[chenyenru]

I was able to run example.py with the following command and modification of src/example.py:

python3 src/example.py --trial_name pointbert_all model.name=PointBERT model.scaling=4 model.use_dense=True training.lr=0.0005 training.lr_decay_rate=0.967

xyz, feat = load_ply("demo/owl.ply")
# add batch dimension
xyz = xyz[None, ...].to(device)
feat = feat[None, ...].to(device)
# keep only xyz and drop depth (B, N, 4) -> (B, N, 3)
xyz = xyz[:, :, :3]
shape_feat = model(xyz, feat, device='cuda', quantization_size=config.model.voxel_size) 

Specifically I added batch dimension and kept only xyz and dropped depth.

@eliphatfs Does this sound correct? Not sure if I'm approaching correctly because from my understanding this means spconv-all requires rgbd while pointbert-vitg14-rgb only needs rgb.

Thank you!

[eliphatfs]

depth means point map which is the xyz part, rgb should be the feat part.

[chenyenru]

ah i see, i guess the fourth dimension would be the intensity value as in reading point cloud with XYZI format, thanks!