How to change the camera view

[zwbx]

Thanks you for the great work. I am trying to change the camera view to see if Octo are robust to view changing. I ran task "widowx_spoon_on_towel" and modify the CameraConfig file, Then I encounter some bugs.

This is my script.

import simpler_env
from simpler_env.utils.env.observation_utils import get_image_from_maniskill2_obs_dict
import mediapy
import sapien.core as sapien
import numpy as np  
from simpler_env.policies.octo.octo_server_model import OctoServerInference
try:
    from simpler_env.policies.octo.octo_model import OctoInference
except ImportError as e:
    print("Octo is not correctly imported.")
    print(e)


model = OctoInference(model_type="octo-small", policy_setup="widowx_bridge", action_scale=1.0,)
task_name = "widowx_spoon_on_towel"  # @param ["google_robot_pick_coke_can", "google_robot_move_near", "google_robot_open_drawer", "google_robot_close_drawer", "widowx_spoon_on_towel", "widowx_carrot_on_plate", "widowx_stack_cube", "widowx_put_eggplant_in_basket"]

if 'env' in locals():
  print("Closing existing env")
  env.close()
  del env
env = simpler_env.make(task_name)
# Colab GPU does not supoort denoiser
sapien.render_config.rt_use_denoiser = False
obs, reset_info = env.reset()
instruction = env.get_language_instruction()
print("Reset info", reset_info)
print("Instruction", instruction)

frames = []
done, truncated = False, False
while not (done or truncated):
   # action[:3]: delta xyz; action[3:6]: delta rotation in axis-angle representation;
   # action[6:7]: gripper (the meaning of open / close depends on robot URDF)
  image = get_image_from_maniskill2_obs_dict(env, obs)
  raw_action, action = model.step(image, instruction)
  print(action["world_vector"])
  obs, reward, done, truncated, info = env.step(np.concatenate([action["world_vector"], action["rot_axangle"], action["gripper"]]),)
#  print(truncated)
  frames.append(image)

episode_stats = info.get('episode_stats', {})
print("Episode stats", episode_stats)
mediapy.show_video(frames, fps=10)

to change the camera view, I modify the 'p' and 'q' in file SimplerEnv/ManiSkill2_real2sim/mani_skill2_real2sim/agents/configs/widowx/defaults.py I have checked that in this env, '3rd_view_camera' will be used.

@property
def cameras(self):
    # Table width: about 36cm

    return [
        CameraConfig(
            uid="3rd_view_camera",  # the camera used for real evaluation
            p=[0.0, -0.16, 0.36],
            # this rotation allows simulation proxy table to align almost perfectly with real table for bridge_real_eval_1.png
            # when calling env.reset(options={'robot_init_options': {'init_xy': [0.147, 0.028], 'init_rot_quat': [0, 0, 0, 1]}})
            q=look_at([0, 0, 0], [1, 0.553, -1.085]).q,
            width=640,
            height=480,
            actor_uid="base_link",
            intrinsic=np.array(
                [[623.588, 0, 319.501], [0, 623.588, 239.545], [0, 0, 1]]
            ),  # logitech C920
        ),
    ]

But the output video show that the camera view does not change, the object's position change. So how to change camera view in a right way?