IsaacLab
IsaacLab copied to clipboard
isaac-sim
[Question] Help: Camera outputting same image to files despite moving in IsaacSim
Question
As my "rod" agent moves across the scene, so too does the affixed camera, meaning the saved image every 200 timesteps should change. However, it remains the same as the image provided upon reset. I'm not sure if this is a bug or something I've done, please help:
import argparse
from omni.isaac.lab.app import AppLauncher
# create argparser
parser = argparse.ArgumentParser(description="Tutorial on running IsaacSim via the AppLauncher.")
parser.add_argument("--size", type=float, default=1.0, help="Side-length of cuboid")
# SimulationApp arguments https://docs.omniverse.nvidia.com/py/isaacsim/source/extensions/omni.isaac.kit/docs/index.html?highlight=simulationapp#omni.isaac.kit.SimulationApp
parser.add_argument(
"--width", type=int, default=1280, help="Width of the viewport and generated images. Defaults to 1280"
)
parser.add_argument(
"--height", type=int, default=720, help="Height of the viewport and generated images. Defaults to 720"
)
# append AppLauncher cli args
AppLauncher.add_app_launcher_args(parser)
# parse the arguments
args_cli = parser.parse_args()
# launch omniverse app
app_launcher = AppLauncher(args_cli)
simulation_app = app_launcher.app
"""Rest everything follows."""
import omni.isaac.lab.sim as sim_utils
from omni.isaac.lab.sensors import Camera, CameraCfg
import omni.isaac.core.utils.prims as prim_utils
import gym
import cv2
import numpy as np
import torch
import os
import time
import random
from omni.isaac.lab.scene import InteractiveScene, InteractiveSceneCfg
from omni.isaac.lab.assets import AssetBaseCfg
from omni.isaac.lab.utils import configclass
from PIL import Image
from omni.isaac.lab.sim.spawners import RigidObjectSpawnerCfg
from omni.isaac.lab.sim import schemas
@configclass
class SimpleSceneCfg(InteractiveSceneCfg):
"""Design the scene with a rod and camera."""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# Ground plane and lights (unchanged)
self.ground = AssetBaseCfg(prim_path="/World/defaultGroundPlane", spawn=sim_utils.GroundPlaneCfg())
self.dome_light = AssetBaseCfg(
prim_path="/World/Light", spawn=sim_utils.DomeLightCfg(intensity=3000.0, color=(0.75, 0.75, 0.75))
)
# Rod (unchanged)
self.rod = AssetBaseCfg(
prim_path="{ENV_REGEX_NS}/Rod",
spawn=sim_utils.CuboidCfg(
size=[0.1,0.1, 3],
visual_material=sim_utils.PreviewSurfaceCfg(
diffuse_color=(0.0, 0.1, 0.4) # Set color to light blue
),
),
)
colors = {
"red": (0.9, 0.0, 0.0),
"green": (0.0, 0.9, 0.0),
"blue": (0.0, 0.0, 0.9),
"black": (0.0, 0.0, 0.0),
"yellow": (0.9, 0.9, 0.0)
}
# Randomly select a position for x and y within the range [0, 10]
x_position = random.randint(-5, 5)
y_position = random.randint(-5, 5)
print("position")
print(x_position)
print(y_position)
# Spawn the block at the random position
object_name = "object_1"
setattr(self, object_name, AssetBaseCfg(
prim_path=f"{{ENV_REGEX_NS}}/Object_1",
spawn=sim_utils.CuboidCfg(size=[1, 1, 1], visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=(0.9, 0.0, 0.0))),
init_state=AssetBaseCfg.InitialStateCfg(pos=[x_position+0.5, y_position+0.5, 0.6])))
# Camera (unchanged)
self.camera = CameraCfg(
prim_path="{ENV_REGEX_NS}/Rod/Camera",
update_period=0.1,
height=300,
width=300,
data_types=["rgb", "distance_to_image_plane"],
spawn=sim_utils.PinholeCameraCfg(focal_length=24.0, focus_distance=100.0, horizontal_aperture=20.955, clipping_range=(0.1, 1.0e5)),
offset=CameraCfg.OffsetCfg(pos=(0.2, 0.0, 1), rot= (0,0,-0.7071,0.7071), convention="ros"),
)
self.wall_1 = AssetBaseCfg(
prim_path="{ENV_REGEX_NS}/Wall_1",
spawn=sim_utils.CuboidCfg(
size=[12,1, 2],
visual_material=sim_utils.PreviewSurfaceCfg(
diffuse_color=(0.1, 0.1, 0.1) # Set color to light blue
),
),
init_state=AssetBaseCfg.InitialStateCfg(
pos=([0.0, 5.5, 1])
)
)
self.wall_2 = AssetBaseCfg(
prim_path="{ENV_REGEX_NS}/Wall_2",
spawn=sim_utils.CuboidCfg(
size=[12,1, 2],
visual_material=sim_utils.PreviewSurfaceCfg(
diffuse_color=(0.1, 0.1, 0.1) # Set color to light blue
),
),
init_state=AssetBaseCfg.InitialStateCfg(
pos=([0.0, -5.5, 1])
)
)
self.wall_3 = AssetBaseCfg(
prim_path="{ENV_REGEX_NS}/Wall_3",
spawn=sim_utils.CuboidCfg(
size=[1,10, 2],
visual_material=sim_utils.PreviewSurfaceCfg(
diffuse_color=(0.1, 0.1, 0.1) # Set color to light blue
),
),
init_state=AssetBaseCfg.InitialStateCfg(
pos=([5.5, 0.0, 1])
)
)
self.wall_4 = AssetBaseCfg(
prim_path="{ENV_REGEX_NS}/Wall_4",
spawn=sim_utils.CuboidCfg(
size=[1,10, 2],
visual_material=sim_utils.PreviewSurfaceCfg(
diffuse_color=(0.1, 0.1, 0.1) # Set color to light blue
),
),
init_state=AssetBaseCfg.InitialStateCfg(
pos=([-5.5, 0.0, 1])
)
)
class CameraAgentEnv(gym.Env):
def __init__(self):
super(CameraAgentEnv, self).__init__()
self.sim = self.create_simulation_context()
self.scene = self.create_scene()
self.image_counter = 0
self.image_dir = "line_images"
self.last_image_time = time.time()
self.sim_time = 0
# Define observation and action spaces
self.observation_space = gym.spaces.Dict({
"image": gym.spaces.Box(low=0, high=255, shape=(args_cli.height, args_cli.width, 3), dtype=np.uint8),
})
self.action_space = gym.spaces.Box(low=-1.0, high=1.0, shape=(3,), dtype=np.float32)
def create_simulation_context(self):
# Initialize the simulation context
sim_cfg = sim_utils.SimulationCfg(dt=0.01)
sim = sim_utils.SimulationContext(sim_cfg)
# Set main camera view - zoomed out to see the entire scene
sim.set_camera_view([5.0, 5.0, 5.0], [0.0, 0.0, 0.0]) # Adjust these values as needed for your scene
return sim
def create_scene(self):
scene_cfg = SimpleSceneCfg(num_envs=1, env_spacing=2.0)
scene = InteractiveScene(scene_cfg)
return scene
def reset(self):
self.sim.reset()
self.init_rod_position()
return self.get_observations()
def step(self, action):
# Apply action to move the rod and camera
self.apply_action(action)
self.sim.step(render=True)
observations = self.get_observations()
self.sim_time += 1
#(self.sim_time)
# Save the image once per second
if self.sim_time % 200 == 0 and "image" in observations:
# input()
image = observations["image"]
image = image[0]
print("imaging")
#image = image[:,:,:]
image = Image.fromarray(image)
image_filename = os.path.join(self.image_dir, f"image_{self.image_counter:04d}.png")
print(image_filename)
image.save(image_filename)
self.image_counter += 1
reward = self.compute_reward()
done = self.check_done()
return observations, reward, done, {}
def init_rod_position(self):
# Initialize rod position to be slightly above the ground
initial_position = torch.tensor([[0, 0, 0.5]], dtype=torch.float32, device=self.sim.device)
self.scene["rod"].set_world_poses(positions=initial_position)
def apply_action(self, action):
# Apply the action to move the rod
current_position = self.scene["rod"].get_world_poses()[0]
new_position = current_position + torch.tensor(action, dtype=torch.float32, device=self.sim.device) * 0.01 # scale action for movement
new_position = new_position.unsqueeze(0) # Ensure correct shape
self.scene["rod"].set_world_poses(positions=new_position)
def get_observations(self):
# Get the camera image
image = self.scene['camera'].data.output["rgb"]
image = image.cpu().numpy() # Ensure it's a numpy array
return {"image": image}
def compute_reward(self):
# Define reward calculation
return 1.0
def check_done(self):
# Define termination conditions
return False
def main():
"""Main function."""
# Initialize the simulation context
sim_cfg = sim_utils.SimulationCfg(dt=0.01)
sim = sim_utils.SimulationContext(sim_cfg)
# Set main camera
sim.set_camera_view([5.0, 5.0, 5.0], [0.0, 0.0, 0.0]) # Zoomed out view
# Design scene by adding assets to it
env = CameraAgentEnv()
env.reset()
# Play the simulator
sim.play() # Start the simulation
# Now we are ready!
print("[INFO]: Setup complete...")
# Simulate physics
while simulation_app.is_running():
# Generate random action
action = env.action_space.sample() # Random action
action[0] = 1
action[1] = 0
action[2] = 0
for i in range(5000):
observations, _, _, _ = env.step(action)
image = observations["image"]
if __name__ == "__main__":
# run the main function
main()
# close sim app
simulation_app.close()
Hi, I assume that you placed your camera under the wrong prim. "{ENV_REGEX_NS}/Rod" will be the Xform under which the robot is spawned, but the actual articulation will be something like "{ENV_REGEX_NS}/Rod/base". For the camera to move with the robot, the camera would have to be placed under base.
As a reference, you can check the example here
Hi,
I've a similar issue with the RayCasterCamera where I attach it to a needle which is then attached to a holder at the last link. Once the simulation loop is running I get my first set of captured images,but after I update the root pose of the robot, my camera is not moving with the link and this then results in gray images rather than the expected output.
Camera config in InteractiveSceneCfg:
raycast_camera = RayCasterCameraCfg( prim_path="{ENV_REGEX_NS}/Robot/robot/tooltip", mesh_prim_paths=["{ENV_REGEX_NS}/cube1", "{ENV_REGEX_NS}/cube2"], update_period=0.1, data_types=["distance_to_image_plane", "normals", "distance_to_camera"], debug_vis=True, pattern_cfg=patterns.PinholeCameraPatternCfg( focal_length=24.0, horizontal_aperture=20.955, height=240, width=320, ), )
So to summarise:
- I spawn my robot in Pose1 and attach camera to eef - I get the expected output
- My robot root pose changes to Pose2 and my camera output is only gray images (no output)
EDIT:
I solved this issue, after spawing the robot and resetting its root pose the camera also moves with it however the direction of rays were different and my camera was seeing in the exact opposite direction of my target items. Learning of the day, always check orientations!!
