From a558cac1049f7d5ad54ea9f2f4ca879f2dad4d61 Mon Sep 17 00:00:00 2001 From: alex Date: Mon, 15 Jun 2026 11:23:12 +0200 Subject: [PATCH 1/3] Change the pick and place task, such that it can be composed. --- isaaclab_arena/tasks/pick_and_place_task.py | 28 ++--- .../tests/test_object_on_termination.py | 2 +- isaaclab_arena/tests/test_object_set.py | 6 +- ...ace_two_objects_maple_table_environment.py | 118 ++++++++++++++++++ 4 files changed, 136 insertions(+), 18 deletions(-) create mode 100644 isaaclab_arena_environments/ilab_pick_and_place_two_objects_maple_table_environment.py diff --git a/isaaclab_arena/tasks/pick_and_place_task.py b/isaaclab_arena/tasks/pick_and_place_task.py index ac93679d16..e2de312956 100644 --- a/isaaclab_arena/tasks/pick_and_place_task.py +++ b/isaaclab_arena/tasks/pick_and_place_task.py @@ -11,7 +11,6 @@ from isaaclab.envs.common import ViewerCfg from isaaclab.envs.mimic_env_cfg import MimicEnvCfg, SubTaskConfig from isaaclab.managers import SceneEntityCfg, TerminationTermCfg -from isaaclab.sensors.contact_sensor.contact_sensor_cfg import ContactSensorCfg from isaaclab.utils import configclass from isaaclab_arena.assets.asset import Asset @@ -26,6 +25,7 @@ from isaaclab_arena.tasks.task_transition import Relocate, TaskTransition from isaaclab_arena.tasks.terminations import object_on_destination from isaaclab_arena.utils.cameras import get_viewer_cfg_look_at_object +from isaaclab_arena.utils.configclass import make_configclass @agent_ready @@ -64,11 +64,8 @@ def __init__( self.destination_object = destination_object self.background_scene = background_scene self.destination_location = destination_location - self.scene_config = SceneCfg( - pick_up_object_contact_sensor=self.pick_up_object.get_contact_sensor_cfg( - contact_against_object=self.destination_location, - ), - ) + self.contact_sensor_name = f"{pick_up_object.name}_contact_sensor" + self.scene_config = self.make_scene_cfg() self.force_threshold = force_threshold self.velocity_threshold = velocity_threshold self.mimic_env_cfg_factory = mimic_env_cfg_factory @@ -80,6 +77,16 @@ def __init__( else task_description ) + def make_scene_cfg(self): + contact_sensor_cfg = self.pick_up_object.get_contact_sensor_cfg( + contact_against_object=self.destination_location, + ) + SceneCfg = make_configclass( + "SceneCfg", + [(self.contact_sensor_name, type(contact_sensor_cfg), contact_sensor_cfg)], + ) + return SceneCfg() + def get_scene_cfg(self): return self.scene_config @@ -91,7 +98,7 @@ def make_termination_cfg(self): func=object_on_destination, params={ "object_cfg": SceneEntityCfg(self.pick_up_object.name), - "contact_sensor_cfg": SceneEntityCfg("pick_up_object_contact_sensor"), + "contact_sensor_cfg": SceneEntityCfg(self.contact_sensor_name), "force_threshold": self.force_threshold, "velocity_threshold": self.velocity_threshold, }, @@ -148,13 +155,6 @@ def success_state_transition(cls, pick_up_object: str, destination_location: str ) -@configclass -class SceneCfg: - """Scene configuration for the pick and place task.""" - - pick_up_object_contact_sensor: ContactSensorCfg = MISSING - - @configclass class TerminationsCfg: """Termination terms for the MDP.""" diff --git a/isaaclab_arena/tests/test_object_on_termination.py b/isaaclab_arena/tests/test_object_on_termination.py index af987dff18..04ad53898b 100644 --- a/isaaclab_arena/tests/test_object_on_termination.py +++ b/isaaclab_arena/tests/test_object_on_termination.py @@ -69,7 +69,7 @@ def _test_object_on_destination_termination(simulation_app) -> bool: velocities_vec = [] success_vec = [] terminated_vec = [] - sensor = env.unwrapped.scene.sensors["pick_up_object_contact_sensor"] + sensor = env.unwrapped.scene.sensors[f"{cracker_box.name}_contact_sensor"] for _ in tqdm.tqdm(range(NUM_STEPS)): with torch.inference_mode(): actions = torch.zeros(env.action_space.shape, device=env.unwrapped.device) diff --git a/isaaclab_arena/tests/test_object_set.py b/isaaclab_arena/tests/test_object_set.py index 5014c66c56..e2c4a2d689 100644 --- a/isaaclab_arena/tests/test_object_set.py +++ b/isaaclab_arena/tests/test_object_set.py @@ -216,7 +216,7 @@ def _run_pick_and_place_object_set_test( assert obj_set.get_initial_pose() is not None, "Initial pose is None" assert env.unwrapped.scene[obj_set.name].data.root_pose_w is not None, "Root pose is None" assert ( - env.unwrapped.scene.sensors["pick_up_object_contact_sensor"].data.force_matrix_w is not None + env.unwrapped.scene.sensors[f"{obj_set.name}_contact_sensor"].data.force_matrix_w is not None ), "Contact sensor data is None" return True except Exception as e: @@ -303,7 +303,7 @@ def _test_single_object_in_one_object_set(simulation_app): assert env.unwrapped.scene[obj_set.name].data.root_pose_w is not None, "Root pose is None" assert ( - env.unwrapped.scene.sensors["pick_up_object_contact_sensor"].data.force_matrix_w is not None + env.unwrapped.scene.sensors[f"{obj_set.name}_contact_sensor"].data.force_matrix_w is not None ), "Contact sensor data is None" except Exception as e: print(f"Error: {e}") @@ -359,7 +359,7 @@ def _test_multi_objects_in_one_object_set(simulation_app): assert env.unwrapped.scene[obj_set.name].data.root_pose_w is not None, "Root pose is None" assert ( - env.unwrapped.scene.sensors["pick_up_object_contact_sensor"].data.force_matrix_w is not None + env.unwrapped.scene.sensors[f"{obj_set.name}_contact_sensor"].data.force_matrix_w is not None ), "Contact sensor data is None" # replace * in OBJECT_SET_PRIM_PATH with env_index diff --git a/isaaclab_arena_environments/ilab_pick_and_place_two_objects_maple_table_environment.py b/isaaclab_arena_environments/ilab_pick_and_place_two_objects_maple_table_environment.py new file mode 100644 index 0000000000..5be70a34b2 --- /dev/null +++ b/isaaclab_arena_environments/ilab_pick_and_place_two_objects_maple_table_environment.py @@ -0,0 +1,118 @@ +# Copyright (c) 2025-2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import argparse +from typing import TYPE_CHECKING + +from isaaclab_arena.assets.register import register_environment +from isaaclab_arena_environments.example_environment_base import ExampleEnvironmentBase + +if TYPE_CHECKING: + from isaaclab_arena.environments.isaaclab_arena_environment import IsaacLabArenaEnvironment + + +@register_environment +class IlabPickAndPlaceTwoObjectsMapleTableEnvironment(ExampleEnvironmentBase): + """Pick-and-place of two objects into a destination on the maple table. + + Completion order does not matter; the composite task succeeds once both + objects rest on the destination. + """ + + name: str = "ilab_pick_and_place_two_objects_maple_table" + + def get_env(self, args_cli: argparse.Namespace) -> IsaacLabArenaEnvironment: + import isaaclab.sim as sim_utils + from isaaclab.envs.common import ViewerCfg + + from isaaclab_arena.assets.object_base import ObjectType + from isaaclab_arena.assets.object_reference import ObjectReference + from isaaclab_arena.environments.isaaclab_arena_environment import IsaacLabArenaEnvironment + from isaaclab_arena.relations.relations import IsAnchor, On + from isaaclab_arena.scene.scene import Scene + from isaaclab_arena.tasks.composite_task_base import CompositeTaskBase + from isaaclab_arena.tasks.pick_and_place_task import PickAndPlaceTask + + # Step 1: Retrieve assets from the registry + background = self.asset_registry.get_asset_by_name("maple_table_robolab")() + pick_up_object_1 = self.asset_registry.get_asset_by_name(args_cli.pick_up_object_1)() + pick_up_object_2 = self.asset_registry.get_asset_by_name(args_cli.pick_up_object_2)() + destination_location = self.asset_registry.get_asset_by_name(args_cli.destination_location)() + + # Step 2: Describe spatial relationships + table_reference = ObjectReference( + name="table", + prim_path="{ENV_REGEX_NS}/maple_table_robolab/table", + parent_asset=background, + object_type=ObjectType.RIGID, + ) + table_reference.add_relation(IsAnchor()) + + pick_up_object_1.add_relation(On(table_reference)) + pick_up_object_2.add_relation(On(table_reference)) + destination_location.add_relation(On(table_reference)) + + # Step 3: Configure lighting + light = self.asset_registry.get_asset_by_name("light")( + spawner_cfg=sim_utils.DomeLightCfg(intensity=args_cli.light_intensity), + ) + if args_cli.hdr is not None: + light.add_hdr(self.hdr_registry.get_hdr_by_name(args_cli.hdr)()) + + # Step 4: Select the embodiment + embodiment = self.asset_registry.get_asset_by_name(args_cli.embodiment)( + enable_cameras=args_cli.enable_cameras, + ) + + # Step 5: Compose the scene + scene = Scene( + assets=[background, light, pick_up_object_1, pick_up_object_2, destination_location, table_reference] + ) + + # Step 6: Define the task as a composite of two pick-and-place subtasks. Each subtask's contact + # sensor is named after its pick-up object, so the combined scene config does not collide. + pick_and_place_task_1 = PickAndPlaceTask( + pick_up_object=pick_up_object_1, + destination_location=destination_location, + destination_object=destination_location, + background_scene=background, + ) + pick_and_place_task_2 = PickAndPlaceTask( + pick_up_object=pick_up_object_2, + destination_location=destination_location, + destination_object=destination_location, + background_scene=background, + ) + task = CompositeTaskBase( + subtasks=[pick_and_place_task_1, pick_and_place_task_2], + episode_length_s=20.0, + ) + + # Set viewport camera to match the robolab droid view + def _set_viewer_cfg(env_cfg): + env_cfg.viewer = ViewerCfg(eye=(1.5, 0.0, 1.0), lookat=(0.2, 0.0, 0.0)) + return env_cfg + + # Step 7: Assemble the environment + isaaclab_arena_environment = IsaacLabArenaEnvironment( + name=self.name, + embodiment=embodiment, + scene=scene, + task=task, + env_cfg_callback=_set_viewer_cfg, + ) + return isaaclab_arena_environment + + @staticmethod + def add_cli_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--embodiment", type=str, default="droid_abs_joint_pos") + parser.add_argument("--teleop_device", type=str, default=None) + parser.add_argument("--hdr", type=str, default=None) + parser.add_argument("--light_intensity", type=float, default=500.0) + parser.add_argument("--pick_up_object_1", type=str, default="rubiks_cube_hot3d_robolab") + parser.add_argument("--pick_up_object_2", type=str, default="mug_ycb_robolab") + parser.add_argument("--destination_location", type=str, default="bowl_ycb_robolab") From e90e78ddafa938fe79e1d3e837200b23347f162e Mon Sep 17 00:00:00 2001 From: alex Date: Mon, 15 Jun 2026 13:25:46 +0200 Subject: [PATCH 2/3] Add first ilab environment. Two objects in a bowl. --- .../eval_jobs_configs/ilab_jobs_config.json | 23 +++++++++++++++++++ ...ace_two_objects_maple_table_environment.py | 9 ++------ .../pick_and_place_maple_table_environment.py | 1 - 3 files changed, 25 insertions(+), 8 deletions(-) create mode 100644 isaaclab_arena_environments/eval_jobs_configs/ilab_jobs_config.json diff --git a/isaaclab_arena_environments/eval_jobs_configs/ilab_jobs_config.json b/isaaclab_arena_environments/eval_jobs_configs/ilab_jobs_config.json new file mode 100644 index 0000000000..14a53208a4 --- /dev/null +++ b/isaaclab_arena_environments/eval_jobs_configs/ilab_jobs_config.json @@ -0,0 +1,23 @@ +{ + "jobs": [ + { + "name": "ilab_pick_and_place_two_objects_maple_table_billiard_hall", + "arena_env_args": { + "enable_cameras": true, + "environment": "ilab_pick_and_place_two_objects_maple_table", + "embodiment": "droid_abs_joint_pos", + "hdr": "billiard_hall_robolab" + }, + "num_episodes": 10, + "language_instruction": "Pick up both objects and place them both in the bowl.", + "policy_type": "isaaclab_arena_openpi.policy.pi0_remote_policy.Pi0RemotePolicy", + "policy_config_dict": { + "policy_variant": "pi05", + "policy_device": "cuda:0", + "remote_host": "127.0.0.1", + "remote_port": 8000, + "openpi_embodiment_adapter": "droid" + } + } + ] +} diff --git a/isaaclab_arena_environments/ilab_pick_and_place_two_objects_maple_table_environment.py b/isaaclab_arena_environments/ilab_pick_and_place_two_objects_maple_table_environment.py index 5be70a34b2..a052b84870 100644 --- a/isaaclab_arena_environments/ilab_pick_and_place_two_objects_maple_table_environment.py +++ b/isaaclab_arena_environments/ilab_pick_and_place_two_objects_maple_table_environment.py @@ -26,7 +26,6 @@ class IlabPickAndPlaceTwoObjectsMapleTableEnvironment(ExampleEnvironmentBase): name: str = "ilab_pick_and_place_two_objects_maple_table" def get_env(self, args_cli: argparse.Namespace) -> IsaacLabArenaEnvironment: - import isaaclab.sim as sim_utils from isaaclab.envs.common import ViewerCfg from isaaclab_arena.assets.object_base import ObjectType @@ -57,9 +56,7 @@ def get_env(self, args_cli: argparse.Namespace) -> IsaacLabArenaEnvironment: destination_location.add_relation(On(table_reference)) # Step 3: Configure lighting - light = self.asset_registry.get_asset_by_name("light")( - spawner_cfg=sim_utils.DomeLightCfg(intensity=args_cli.light_intensity), - ) + light = self.asset_registry.get_asset_by_name("light")() if args_cli.hdr is not None: light.add_hdr(self.hdr_registry.get_hdr_by_name(args_cli.hdr)()) @@ -110,9 +107,7 @@ def _set_viewer_cfg(env_cfg): @staticmethod def add_cli_args(parser: argparse.ArgumentParser) -> None: parser.add_argument("--embodiment", type=str, default="droid_abs_joint_pos") - parser.add_argument("--teleop_device", type=str, default=None) - parser.add_argument("--hdr", type=str, default=None) - parser.add_argument("--light_intensity", type=float, default=500.0) parser.add_argument("--pick_up_object_1", type=str, default="rubiks_cube_hot3d_robolab") parser.add_argument("--pick_up_object_2", type=str, default="mug_ycb_robolab") parser.add_argument("--destination_location", type=str, default="bowl_ycb_robolab") + parser.add_argument("--hdr", type=str, default=None) diff --git a/isaaclab_arena_environments/pick_and_place_maple_table_environment.py b/isaaclab_arena_environments/pick_and_place_maple_table_environment.py index 37fddb83e9..84c565eaa7 100644 --- a/isaaclab_arena_environments/pick_and_place_maple_table_environment.py +++ b/isaaclab_arena_environments/pick_and_place_maple_table_environment.py @@ -97,7 +97,6 @@ def _set_viewer_cfg(env_cfg): @staticmethod def add_cli_args(parser: argparse.ArgumentParser) -> None: parser.add_argument("--embodiment", type=str, default="droid_abs_joint_pos") - parser.add_argument("--teleop_device", type=str, default=None) parser.add_argument("--hdr", type=str, default=None) parser.add_argument("--light_intensity", type=float, default=500.0) parser.add_argument("--pick_up_object", type=str, default="rubiks_cube_hot3d_robolab") From 488bf88330ebc36a97910a965e6f750a9cfb109f Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 17 Jun 2026 15:03:49 +0200 Subject: [PATCH 3/3] Add microwave task to ilab. --- .../eval_jobs_configs/ilab_jobs_config.json | 19 ++++ ..._open_microwave_maple_table_environment.py | 101 ++++++++++++++++++ ...ace_two_objects_maple_table_environment.py | 2 +- 3 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 isaaclab_arena_environments/ilab_open_microwave_maple_table_environment.py diff --git a/isaaclab_arena_environments/eval_jobs_configs/ilab_jobs_config.json b/isaaclab_arena_environments/eval_jobs_configs/ilab_jobs_config.json index 14a53208a4..7fb799d1de 100644 --- a/isaaclab_arena_environments/eval_jobs_configs/ilab_jobs_config.json +++ b/isaaclab_arena_environments/eval_jobs_configs/ilab_jobs_config.json @@ -18,6 +18,25 @@ "remote_port": 8000, "openpi_embodiment_adapter": "droid" } + }, + { + "name": "ilab_open_microwave_maple_table_billiard_hall", + "arena_env_args": { + "enable_cameras": true, + "environment": "ilab_open_microwave_maple_table", + "embodiment": "droid_abs_joint_pos", + "hdr": "billiard_hall_robolab" + }, + "num_episodes": 10, + "language_instruction": "Open the microwave door.", + "policy_type": "isaaclab_arena_openpi.policy.pi0_remote_policy.Pi0RemotePolicy", + "policy_config_dict": { + "policy_variant": "pi05", + "policy_device": "cuda:0", + "remote_host": "127.0.0.1", + "remote_port": 8000, + "openpi_embodiment_adapter": "droid" + } } ] } diff --git a/isaaclab_arena_environments/ilab_open_microwave_maple_table_environment.py b/isaaclab_arena_environments/ilab_open_microwave_maple_table_environment.py new file mode 100644 index 0000000000..06760cdb3c --- /dev/null +++ b/isaaclab_arena_environments/ilab_open_microwave_maple_table_environment.py @@ -0,0 +1,101 @@ +# Copyright (c) 2025-2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import argparse +import math +from typing import TYPE_CHECKING + +from isaaclab_arena.assets.register import register_environment +from isaaclab_arena_environments.example_environment_base import ExampleEnvironmentBase + +if TYPE_CHECKING: + from isaaclab_arena.environments.isaaclab_arena_environment import IsaacLabArenaEnvironment + + +@register_environment +class IlabOpenMicrowaveMapleTableEnvironment(ExampleEnvironmentBase): + """Open the door of a microwave resting on the maple table. + + Shares the maple-table background used by the first ilab environment; the + microwave is placed on the table and the task succeeds once its door swings + past the openness threshold. + """ + + name: str = "ilab_open_microwave_maple_table" + + def get_env(self, args_cli: argparse.Namespace) -> IsaacLabArenaEnvironment: + from isaaclab.envs.common import ViewerCfg + + from isaaclab_arena.assets.object_base import ObjectType + from isaaclab_arena.assets.object_reference import ObjectReference + from isaaclab_arena.environments.isaaclab_arena_environment import IsaacLabArenaEnvironment + from isaaclab_arena.relations.relations import IsAnchor, On, RotateAroundSolution + from isaaclab_arena.scene.scene import Scene + from isaaclab_arena.tasks.open_door_task import OpenDoorTask + from isaaclab_arena.utils.pose import Pose + + # Step 1: Retrieve assets from the registry + background = self.asset_registry.get_asset_by_name("maple_table_robolab")() + microwave = self.asset_registry.get_asset_by_name("microwave")() + + # Step 2: Describe spatial relationships. The microwave sits on the same + # table the first ilab environment uses; the yaw rotation faces its door + # toward the robot. + table_reference = ObjectReference( + name="table", + prim_path="{ENV_REGEX_NS}/maple_table_robolab/table", + parent_asset=background, + object_type=ObjectType.RIGID, + ) + table_reference.add_relation(IsAnchor()) + + microwave.add_relation(On(table_reference)) + microwave.add_relation(RotateAroundSolution(yaw_rad=-math.pi / 2)) + + # Step 3: Configure lighting + light = self.asset_registry.get_asset_by_name("light")() + if args_cli.hdr is not None: + light.add_hdr(self.hdr_registry.get_hdr_by_name(args_cli.hdr)()) + light.set_intensity(2000.0) + + # Step 4: Select the embodiment + embodiment = self.asset_registry.get_asset_by_name(args_cli.embodiment)( + enable_cameras=args_cli.enable_cameras, + ) + embodiment.set_initial_pose(Pose(position_xyz=(-0.3, 0.0, 0.0), rotation_xyzw=(0.0, 0.0, 0.0, 1.0))) + + # Step 5: Compose the scene + scene = Scene(assets=[background, light, microwave, table_reference]) + + # Step 6: Define the task as opening the microwave door + task = OpenDoorTask( + openable_object=microwave, + openness_threshold=args_cli.openness_threshold, + reset_openness=0.0, + episode_length_s=20.0, + ) + + # Set viewport camera to match the robolab droid view + def _set_viewer_cfg(env_cfg): + env_cfg.viewer = ViewerCfg(eye=(1.5, 0.0, 1.0), lookat=(0.2, 0.0, 0.0)) + return env_cfg + + # Step 7: Assemble the environment + isaaclab_arena_environment = IsaacLabArenaEnvironment( + name=self.name, + embodiment=embodiment, + scene=scene, + task=task, + env_cfg_callback=_set_viewer_cfg, + ) + return isaaclab_arena_environment + + @staticmethod + def add_cli_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--embodiment", type=str, default="droid_abs_joint_pos") + parser.add_argument("--openness_threshold", type=float, default=0.8) + parser.add_argument("--hdr", type=str, default="home_office_robolab") diff --git a/isaaclab_arena_environments/ilab_pick_and_place_two_objects_maple_table_environment.py b/isaaclab_arena_environments/ilab_pick_and_place_two_objects_maple_table_environment.py index a052b84870..b2c6bded94 100644 --- a/isaaclab_arena_environments/ilab_pick_and_place_two_objects_maple_table_environment.py +++ b/isaaclab_arena_environments/ilab_pick_and_place_two_objects_maple_table_environment.py @@ -110,4 +110,4 @@ def add_cli_args(parser: argparse.ArgumentParser) -> None: parser.add_argument("--pick_up_object_1", type=str, default="rubiks_cube_hot3d_robolab") parser.add_argument("--pick_up_object_2", type=str, default="mug_ycb_robolab") parser.add_argument("--destination_location", type=str, default="bowl_ycb_robolab") - parser.add_argument("--hdr", type=str, default=None) + parser.add_argument("--hdr", type=str, default="home_office_robolab")