diff --git a/docs/pages/quickstart/first_experiments/running_a_real_policy/gr00t.rst b/docs/pages/quickstart/first_experiments/running_a_real_policy/gr00t.rst index cfa3c5715..dd7e980d1 100644 --- a/docs/pages/quickstart/first_experiments/running_a_real_policy/gr00t.rst +++ b/docs/pages/quickstart/first_experiments/running_a_real_policy/gr00t.rst @@ -153,7 +153,7 @@ episode; the runner writes an ``index.html`` which is then served over HTTP. python isaaclab_arena/evaluation/eval_runner.py \ --viz kit \ --eval_jobs_config isaaclab_arena_environments/eval_jobs_configs/droid_pnp_srl_gr00t_jobs_config.json \ - --video_base_dir ./output \ + --output_base_dir ./output \ --record_camera_video --serve_evaluation_report You can also (re)build and serve a report later by pointing the standalone tool at the output diff --git a/docs/pages/quickstart/first_experiments/running_a_real_policy/openpi.rst b/docs/pages/quickstart/first_experiments/running_a_real_policy/openpi.rst index a4e201884..928360313 100644 --- a/docs/pages/quickstart/first_experiments/running_a_real_policy/openpi.rst +++ b/docs/pages/quickstart/first_experiments/running_a_real_policy/openpi.rst @@ -162,7 +162,7 @@ episode; the runner writes an ``index.html`` which is then served over HTTP. python isaaclab_arena/evaluation/eval_runner.py \ --viz kit \ --eval_jobs_config isaaclab_arena_environments/eval_jobs_configs/droid_pnp_srl_openpi_jobs_config.json \ - --video_base_dir ./output \ + --output_base_dir ./output \ --record_camera_video --serve_evaluation_report You can also (re)build and serve a report later by pointing the standalone tool at the output diff --git a/isaaclab_arena/environments/arena_env_builder.py b/isaaclab_arena/environments/arena_env_builder.py index fb8b6b493..6aa286fe8 100644 --- a/isaaclab_arena/environments/arena_env_builder.py +++ b/isaaclab_arena/environments/arena_env_builder.py @@ -34,6 +34,8 @@ make_progress_tracking_events_cfg, make_progress_tracking_recorder_cfg, ) +from isaaclab_arena.recording.common_terms import CoreEpisodeRecorderTermCfg, VariationEpisodeRecorderTermCfg +from isaaclab_arena.recording.episode_recorder_manager import EpisodeRecorderTermCfg from isaaclab_arena.relations.placement_events import PLACEMENT_RESET_EVENT_NAME from isaaclab_arena.tasks.no_task import NoTask from isaaclab_arena.utils.configclass import combine_configclass_instances, make_configclass @@ -152,14 +154,30 @@ def _modify_recorder_cfg_dataset_filename(self, recorder_cfg: RecorderManagerBas ) return recorder_cfg - @staticmethod - def _metrics_to_metrics_cfg(metrics: list[MetricBase] | None) -> object | None: + def _compose_metrics_cfg(self, metrics: list[MetricBase] | None) -> object | None: """Build a configclass container with one ``MetricTermCfg`` field per metric.""" if not metrics: return None fields = [(m.name, MetricTermCfg, m.get_metric_term_cfg()) for m in metrics] return make_configclass("MetricsCfg", fields)() + def _compose_episode_recorders_cfg(self, extra_terms: dict[str, EpisodeRecorderTermCfg] | None = None) -> object: + """Build a configclass container with one EpisodeRecorderTermCfg field per episode recorder term. + + Note that this function automatically adds the core and variations terms. + """ + fields = [ + ("core", EpisodeRecorderTermCfg, CoreEpisodeRecorderTermCfg()), + ("variations", EpisodeRecorderTermCfg, VariationEpisodeRecorderTermCfg()), + ] + for name, term_cfg in (extra_terms or {}).items(): + assert name not in ( + "core", + "variations", + ), f"Episode recorder term name '{name}' collides with a built-in term." + fields.append((name, EpisodeRecorderTermCfg, term_cfg)) + return make_configclass("EpisodeRecorderManagerCfg", fields)() + def compose_manager_cfg(self) -> tuple[IsaacLabArenaManagerBasedRLEnvCfg, dict[str, Any]]: """Return the base ManagerBased cfg and the env kwargs (no registration). @@ -240,7 +258,7 @@ def compose_manager_cfg(self) -> tuple[IsaacLabArenaManagerBasedRLEnvCfg, dict[s elif isinstance(device_cfg, DeviceCfg): teleop_devices_cfg = DevicesCfg(devices={self.arena_env.teleop_device.name: device_cfg}) metrics = task.get_metrics() - metrics_cfg = self._metrics_to_metrics_cfg(metrics) + metrics_cfg = self._compose_metrics_cfg(metrics) metrics_recorder_manager_cfg = metrics_to_recorder_manager_cfg(metrics) progress_tracking_recorder_cfg: Any = ( make_progress_tracking_recorder_cfg(progress_objectives) if progress_objectives else None @@ -278,11 +296,15 @@ def compose_manager_cfg(self) -> tuple[IsaacLabArenaManagerBasedRLEnvCfg, dict[s task.get_commands_cfg(), ) + episode_recorders_cfg = self._compose_episode_recorders_cfg(self.arena_env.episode_recorder_terms) + viewer_cfg = task.get_viewer_cfg() episode_length_s = task.get_episode_length_s() - task_description = task.get_task_description() + # Language instruction is optionally overridden on the CLI. + language_instruction = getattr(self.args, "language_instruction", None) + task_description = language_instruction or task.get_task_description() # Build the environment configuration if not self.args.mimic: @@ -300,6 +322,7 @@ def compose_manager_cfg(self) -> tuple[IsaacLabArenaManagerBasedRLEnvCfg, dict[s teleop_devices=teleop_devices_cfg, recorders=recorder_manager_cfg, metrics=metrics_cfg, + episode_recorders=episode_recorders_cfg, task_description=task_description, viewer=viewer_cfg, ) diff --git a/isaaclab_arena/environments/isaaclab_arena_environment.py b/isaaclab_arena/environments/isaaclab_arena_environment.py index 985f0b461..7ef8be9fa 100644 --- a/isaaclab_arena/environments/isaaclab_arena_environment.py +++ b/isaaclab_arena/environments/isaaclab_arena_environment.py @@ -12,6 +12,7 @@ from isaaclab_arena.assets.teleop_device_base import TeleopDeviceBase from isaaclab_arena.embodiments.embodiment_base import EmbodimentBase from isaaclab_arena.environments.isaaclab_arena_manager_based_env_cfg import IsaacLabArenaManagerBasedRLEnvCfg + from isaaclab_arena.recording.episode_recorder_manager import EpisodeRecorderTermCfg from isaaclab_arena.scene.scene import Scene from isaaclab_arena.tasks.task_base import TaskBase @@ -29,6 +30,7 @@ def __init__( env_cfg_callback: Callable[IsaacLabArenaManagerBasedRLEnvCfg] | None = None, rl_framework_entry_point: str | None = None, rl_policy_cfg: str | None = None, + episode_recorder_terms: dict[str, EpisodeRecorderTermCfg] | None = None, ): """ Args: @@ -46,6 +48,8 @@ def __init__( ``rl_policy_cfg`` is set. rl_policy_cfg: Import path to the RL policy config class, e.g. ``"my_module:RLPolicyCfg"``. + episode_recorder_terms: Additional per-episode recorder terms to record alongside the + built-in ones, keyed by name. """ self.name = name self.scene = scene @@ -57,3 +61,4 @@ def __init__( raise ValueError("rl_framework_entry_point and rl_policy_cfg must both be set or both be None.") self.rl_framework_entry_point = rl_framework_entry_point self.rl_policy_cfg = rl_policy_cfg + self.episode_recorder_terms = episode_recorder_terms or {} diff --git a/isaaclab_arena/environments/isaaclab_arena_manager_based_env.py b/isaaclab_arena/environments/isaaclab_arena_manager_based_env.py index d5f97e8cb..e29937b6d 100644 --- a/isaaclab_arena/environments/isaaclab_arena_manager_based_env.py +++ b/isaaclab_arena/environments/isaaclab_arena_manager_based_env.py @@ -5,11 +5,14 @@ from __future__ import annotations +from collections.abc import Sequence + from isaaclab.envs import ManagerBasedRLEnv from isaaclab_arena.environments.isaaclab_arena_manager_based_env_cfg import IsaacLabArenaManagerBasedRLEnvCfg from isaaclab_arena.metrics.metric_data import MetricsDataCollection from isaaclab_arena.metrics.metrics_manager import MetricsManager +from isaaclab_arena.recording.episode_recorder_manager import EpisodeRecorderManager from isaaclab_arena.variations.variation_recorder import VariationRecorder @@ -26,6 +29,13 @@ def __init__( **kwargs, ): self._variation_recorder = variation_recorder + if variation_recorder is not None: + # Bind so run-time variation draws can be attributed to the current episode index. + variation_recorder.bind_env(self) + # Per-env count of completed episodes; advanced in ``_reset_idx``. + self._episode_counts: dict[int, int] = {} + # The initial reset touches every env before any episode has run; skip it. + self._first_reset = True super().__init__(cfg=cfg, render_mode=render_mode, **kwargs) @property @@ -38,9 +48,41 @@ def variation_recorder(self) -> VariationRecorder | None: ) return self._variation_recorder + @property + def episode_recorder(self) -> EpisodeRecorderManager: + """The per-episode recorder.""" + return self.episode_recorder_manager + def load_managers(self) -> None: super().load_managers() self.metrics_manager = MetricsManager(self.cfg.metrics, self) + self.episode_recorder_manager = EpisodeRecorderManager(self.cfg.episode_recorders, self) + + def get_language_instruction(self) -> str | None: + """Return the language instruction that is passed to the policy.""" + return self.cfg.task_description + + def get_episode_index(self, env_id: int) -> int: + """Return the index of the current episode in ``env_id``.""" + return self._episode_counts.get(env_id, 0) + + def _advance_episode_indices(self, env_ids: Sequence[int]) -> None: + """Advance the per-env episode counter for each episode in ``env_ids``.""" + for env_id in env_ids: + env_id = int(env_id) + self._episode_counts[env_id] = self._episode_counts.get(env_id, 0) + 1 + + def _reset_idx(self, env_ids: Sequence[int]) -> None: + # The initial reset touches every env before any episode has run; nothing to record or count. + if self._first_reset: + self._first_reset = False + super()._reset_idx(env_ids) + return + # Runs recorder before super() so the just-finished episode is still intact. + self.episode_recorder_manager.record_pre_reset(env_ids) + # Advance before super() so reset-mode variation draws are tagged with the episode they begin. + self._advance_episode_indices(env_ids) + super()._reset_idx(env_ids) def compute_metrics(self) -> MetricsDataCollection: """Compute all registered metrics. diff --git a/isaaclab_arena/environments/isaaclab_arena_manager_based_env_cfg.py b/isaaclab_arena/environments/isaaclab_arena_manager_based_env_cfg.py index f16b60312..ba9bfb620 100644 --- a/isaaclab_arena/environments/isaaclab_arena_manager_based_env_cfg.py +++ b/isaaclab_arena/environments/isaaclab_arena_manager_based_env_cfg.py @@ -64,6 +64,8 @@ class IsaacLabArenaManagerBasedRLEnvCfg(ManagerBasedRLEnvCfg): metrics: object | None = None + episode_recorders: object | None = None + # Task language description task_description: str | None = None diff --git a/isaaclab_arena/evaluation/eval_runner.py b/isaaclab_arena/evaluation/eval_runner.py index 3f82a502c..8aea72741 100644 --- a/isaaclab_arena/evaluation/eval_runner.py +++ b/isaaclab_arena/evaluation/eval_runner.py @@ -38,17 +38,20 @@ def load_env( job_name: str, variations: list[str] | None = None, render_mode: str | None = None, + language_instruction: str | None = None, ): args_parser = get_isaaclab_arena_environments_cli_parser() arena_env_args_cli = args_parser.parse_args(arena_env_args) + # Optionally override the language instruction. + arena_env_args_cli.language_instruction = language_instruction arena_builder = get_arena_builder_from_cli(arena_env_args_cli, hydra_overrides=variations) - env_name, env_cfg, env_kwargs = arena_builder.build_registered() + _, env_cfg, env_kwargs = arena_builder.build_registered() # Set unique dataset filename for this job to avoid file locking conflicts - if hasattr(env_cfg, "recorders") and env_cfg.recorders is not None: + if env_cfg.recorders is not None: env_cfg.recorders.dataset_filename = f"dataset_{job_name}" env = arena_builder.make_registered(env_cfg, env_kwargs, render_mode=render_mode) @@ -262,11 +265,11 @@ def main(): # Always dated so every run produces its own report dir, recording or not. # TODO(alexmillane): Currently each chunk produces its own output directory. # We should use the same output directory for all chunks in the future. - run_video_dir = timestamped_run_dir(args_cli.video_base_dir) + run_output_dir = timestamped_run_dir(args_cli.output_base_dir) if args_cli.record_viewport_video: - os.makedirs(run_video_dir, exist_ok=True) - print(f"[INFO] Video recording enabled. Videos will be saved to: {run_video_dir}") + os.makedirs(run_output_dir, exist_ok=True) + print(f"[INFO] Video recording enabled. Videos will be saved to: {run_output_dir}") for job in job_manager: if job is None: @@ -283,17 +286,30 @@ def main(): # aggregate the metrics across rebuilds into a single result. for rebuild_idx in range(job.num_rebuilds): try: + job_output_dir = os.path.join(run_output_dir, job.name) + # Per-job video output directory; cameras are tagged with the rebuild index. video_cfg = VideoRecordingCfg( record_viewport_video=args_cli.record_viewport_video, record_camera_video=args_cli.record_camera_video, - video_base_dir=os.path.join(run_video_dir, job.name), + video_base_dir=job_output_dir, camera_name_prefix=f"robot-cam-rebuild{rebuild_idx}", ) env = load_env( - job.arena_env_args, job.name, variations=job.variations, render_mode=video_cfg.render_mode + job.arena_env_args, + job.name, + variations=job.variations, + render_mode=video_cfg.render_mode, + language_instruction=job.language_instruction, ) + # Write per-episode results to disk. + # TODO: Aggregate the per-episode records across rebuilds into a single file, + # as is done for the metrics below. + results_path = os.path.join(job_output_dir, f"episode_results_rebuild{rebuild_idx}.jsonl") + env.unwrapped.episode_recorder.set_job_name(job.name) + env.unwrapped.episode_recorder.set_output_path(results_path) + policy = get_policy_from_job(job) # Episodes allotted to this rebuild (None when the job is length-driven by steps). @@ -314,7 +330,6 @@ def main(): policy, num_steps=job.num_steps, num_episodes=num_episodes_this_rebuild, - language_instruction=job.language_instruction, ) job_manager.complete_job(job, metrics=metrics, status=Status.COMPLETED) @@ -347,7 +362,7 @@ def main(): metrics_logger.print_metrics() # Write HTML report. - report_path = build_report(run_video_dir) + report_path = build_report(run_output_dir) if args_cli.serve_evaluation_report: serve_until_ctrl_c(report_path.parent, args_cli.evaluation_report_port, report_path.name) diff --git a/isaaclab_arena/evaluation/eval_runner_cli.py b/isaaclab_arena/evaluation/eval_runner_cli.py index 3d08c44f9..235b4302f 100644 --- a/isaaclab_arena/evaluation/eval_runner_cli.py +++ b/isaaclab_arena/evaluation/eval_runner_cli.py @@ -27,10 +27,13 @@ def add_eval_runner_arguments(parser: argparse.ArgumentParser) -> None: help="Record one mp4 per (env, camera, episode) from obs['camera_obs'] for each eval job.", ) parser.add_argument( - "--video_base_dir", + "--output_base_dir", type=str, - default="/eval/videos", - help="Base directory for recorded videos; a reverse-dated run subdirectory and per-job subdirectory are added.", + default="/eval/output", + help=( + "Base directory for evaluation outputs (videos, per-episode results, report); a" + " reverse-dated run subdirectory and per-job subdirectory are added." + ), ) parser.add_argument( "--serve_evaluation_report", diff --git a/isaaclab_arena/evaluation/policy_runner.py b/isaaclab_arena/evaluation/policy_runner.py index 41a097f23..3015d3eed 100644 --- a/isaaclab_arena/evaluation/policy_runner.py +++ b/isaaclab_arena/evaluation/policy_runner.py @@ -6,6 +6,7 @@ from __future__ import annotations import argparse +import os import torch import tqdm from importlib import import_module @@ -64,7 +65,6 @@ def rollout_policy( policy: PolicyBase, num_steps: int | None, num_episodes: int | None, - language_instruction: str | None = None, ) -> MetricsDataCollection | None: assert num_steps is not None or num_episodes is not None, "Either num_steps or num_episodes must be provided" assert num_steps is None or num_episodes is None, "Only one of num_steps or num_episodes must be provided" @@ -73,10 +73,7 @@ def rollout_policy( try: obs, _ = env.reset() policy.reset() - # Determine language instruction: CLI/job-level override takes precedence over the task's own - # description. Use unwrapped to reach the base env through any gym wrappers (e.g. OrderEnforcing). - task_description = language_instruction or env.unwrapped.cfg.task_description - policy.set_task_description(task_description) + policy.set_task_description(env.unwrapped.get_language_instruction()) # Setup progress bar based on num_steps or num_episodes if num_steps is not None: @@ -192,12 +189,18 @@ def main(): print(arena_builder.get_variations_catalogue_as_string()) return + output_dir = timestamped_run_dir(args_cli.output_base_dir) video_cfg = VideoRecordingCfg( record_viewport_video=args_cli.record_viewport_video, record_camera_video=args_cli.record_camera_video, - video_base_dir=timestamped_run_dir(args_cli.video_base_dir), + video_base_dir=output_dir, ) - env, cfg = arena_builder.make_registered_and_return_cfg(render_mode=video_cfg.render_mode) + env = arena_builder.make_registered(render_mode=video_cfg.render_mode) + + # Write per-episode results to disk. + results_path = os.path.join(output_dir, f"episode_results_rank{local_rank}.jsonl") + env.unwrapped.episode_recorder.set_job_name("policy_runner") + env.unwrapped.episode_recorder.set_output_path(results_path) # Create the policy from the arguments policy = policy_cls.from_args(args_cli) @@ -223,7 +226,7 @@ def main(): steps_str = f"{num_steps} steps" if num_steps is not None else f"{num_episodes} episodes" print(f"[Rank {local_rank}/{world_size}] Starting rollout ({steps_str})") - metrics = rollout_policy(env, policy, num_steps, num_episodes, args_cli.language_instruction) + metrics = rollout_policy(env, policy, num_steps, num_episodes) if metrics is not None: print(f"[Rank {local_rank}/{world_size}] Metrics: {metrics_to_plain_python_types(metrics)}") @@ -241,7 +244,7 @@ def main(): # Write and serve the evaluation report. # Only the local rank 0 writes/serves it, to avoid races on a shared output dir. if get_local_rank() == 0: - report_path = build_report(video_cfg.video_base_dir) + report_path = build_report(output_dir) if args_cli.serve_evaluation_report: serve_until_ctrl_c(report_path.parent, args_cli.evaluation_report_port, report_path.name) diff --git a/isaaclab_arena/evaluation/policy_runner_cli.py b/isaaclab_arena/evaluation/policy_runner_cli.py index 62bbc5851..9f78d6245 100644 --- a/isaaclab_arena/evaluation/policy_runner_cli.py +++ b/isaaclab_arena/evaluation/policy_runner_cli.py @@ -39,12 +39,12 @@ def add_policy_runner_arguments(parser: argparse.ArgumentParser) -> None: help="Record an mp4 video of the rollout viewport (uses gymnasium.wrappers.RecordVideo).", ) parser.add_argument( - "--video_base_dir", + "--output_base_dir", type=str, - default="/eval/videos", + default="/eval/output", help=( - "Base directory for recorded videos; a reverse-dated run subdirectory is added per run." - " Used with --record_viewport_video and/or --record_camera_video." + "Base directory for evaluation outputs (videos, per-episode results, report); a" + " reverse-dated run subdirectory is added per run." ), ) parser.add_argument( diff --git a/isaaclab_arena/recording/__init__.py b/isaaclab_arena/recording/__init__.py new file mode 100644 index 000000000..16ea4c218 --- /dev/null +++ b/isaaclab_arena/recording/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 diff --git a/isaaclab_arena/recording/common_terms.py b/isaaclab_arena/recording/common_terms.py new file mode 100644 index 000000000..ecd189ef0 --- /dev/null +++ b/isaaclab_arena/recording/common_terms.py @@ -0,0 +1,60 @@ +# Copyright (c) 2025-2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import datetime +import torch +from collections.abc import Callable +from typing import Any + +from isaaclab.utils import configclass + +from isaaclab_arena.recording.episode_recorder_manager import EpisodeRecorderTermCfg + + +def record_core_episode_results(env, env_id: int) -> dict[str, Any]: + """Record the core per-episode fields for ``env_id``.""" + success = None + if "success" in env.termination_manager.active_terms: + success = bool(env.termination_manager.get_term("success")[env_id].item()) + return { + "env_id": env_id, + "episode_in_env": env.get_episode_index(env_id), + "seed": env.cfg.seed, + "success": success, + "episode_length": int(env.episode_length_buf[env_id].item()), + "language_instruction": env.get_language_instruction(), + "timestamp": datetime.datetime.now().isoformat(), + } + + +def record_variation_samples(env, env_id: int) -> dict[str, Any]: + """Record the variation value drawn for ``env_id``'s finished episode under ``variations``.""" + recorder = env.variation_recorder + if recorder is None or not recorder.records: + return {} + episode_idx = env.get_episode_index(env_id) + samples: dict[str, Any] = {} + for key, record in recorder.records.items(): + value = record.sample_for_episode(env_id, episode_idx) + if value is None: + continue + samples[key] = value.tolist() if isinstance(value, torch.Tensor) else value + return {"variations": samples} if samples else {} + + +@configclass +class CoreEpisodeRecorderTermCfg(EpisodeRecorderTermCfg): + """Term recording the core per-episode metadata (env id, indices, success, seed, timing).""" + + func: Callable[..., dict[str, Any]] = record_core_episode_results + + +@configclass +class VariationEpisodeRecorderTermCfg(EpisodeRecorderTermCfg): + """Term recording each variation's per-env sampled value for the episode.""" + + func: Callable[..., dict[str, Any]] = record_variation_samples diff --git a/isaaclab_arena/recording/episode_recorder_manager.py b/isaaclab_arena/recording/episode_recorder_manager.py new file mode 100644 index 000000000..cb0001986 --- /dev/null +++ b/isaaclab_arena/recording/episode_recorder_manager.py @@ -0,0 +1,142 @@ +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +import torch +from collections.abc import Callable, Sequence +from dataclasses import MISSING +from pathlib import Path +from prettytable import PrettyTable +from typing import Any + +from isaaclab.managers import ManagerBase, ManagerTermBaseCfg +from isaaclab.utils import configclass + + +@configclass +class EpisodeRecorderTermCfg(ManagerTermBaseCfg): + """Configuration for an episode recorder term.""" + + func: Callable[..., dict[str, Any]] = MISSING + """The callable that records this term's fields for one finishing episode. + + Invoked as ``func(env, env_id, **params)`` for the env whose episode just finished, and must + return a flat, JSON-serializable dict that is merged into the episode's record. It may be a plain + function or a callable class inheriting from ManagerTermBase (built once by the manager). + """ + + +class EpisodeRecorderManager(ManagerBase): + """Records per-episode data, described by terms. Written out as JSONL on request.""" + + def __init__(self, cfg: object, env) -> None: + """Initialize the manager and its episode-recording state. + + Args: + cfg: The episode recorder manager cfg. + env: The environment instance. + """ + self._term_names: list[str] = [] + self._term_cfgs: list[EpisodeRecorderTermCfg] = [] + self._job_name: str = "default" + self._output_path: Path | None = None + super().__init__(cfg, env) + + def __str__(self) -> str: + """Returns: A string representation for the episode recorder manager.""" + table = PrettyTable() + table.title = "Active Episode Recorder Terms" + table.field_names = ["Index", "Name"] + table.align["Name"] = "l" + for index, name in enumerate(self._term_names): + table.add_row([index, name]) + return f" contains {len(self._term_names)} active terms.\n{table.get_string()}\n" + + @property + def active_terms(self) -> list[str]: + """Name of active episode recorder terms.""" + return self._term_names + + def set_job_name(self, job_name: str) -> None: + """Set the job name stamped onto subsequently recorded episodes.""" + self._job_name = job_name + + def set_output_path(self, output_path: str | Path) -> None: + """Set the path of the JSONL file that records are appended to as episodes finish. + + Must be called before recording to persist results; without it, finished episodes are not + written anywhere. + """ + path = Path(output_path) + path.parent.mkdir(parents=True, exist_ok=True) + # Delete the contents of the file if it already exists by writing an empty string. + path.write_text("", encoding="utf-8") + self._output_path = path + + def record_pre_reset(self, env_ids: Sequence[int] | torch.Tensor | None) -> None: + """Record one record per finished episode. + + This function fires each recording terms' function and merges the results into a single record. + + Args: + env_ids: The env ids being reset (tensor, sequence, or ``None`` for all envs). + """ + for env_id in self._normalize_env_ids(env_ids): + # The manager stamps the job name; terms add the per-episode fields. + record: dict[str, Any] = { + "job_name": self._job_name, + } + # Fire each recording term's function. + for term_name, term_cfg in zip(self._term_names, self._term_cfgs): + fields = term_cfg.func(self._env, env_id, **term_cfg.params) + collisions = record.keys() & fields.keys() + assert not collisions, ( + f"Episode recorder term '{term_name}' redefines fields {collisions} already set" + " by the manager or an earlier term." + ) + self._assert_json_serializable(term_name, fields) + record.update(fields) + self._append_record(record) + + def _append_record(self, record: dict[str, Any]) -> None: + """Append one record to the output JSONL (one object per line); no-op if no path was set.""" + if self._output_path is None: + return + with open(self._output_path, "a", encoding="utf-8") as f: + f.write(json.dumps(record) + "\n") + + def _normalize_env_ids(self, env_ids: Sequence[int] | torch.Tensor | None) -> list[int]: + """Normalize ``env_ids`` (tensor, sequence, or ``None`` for all envs) to a list of ints.""" + if env_ids is None: + return list(range(self._env.num_envs)) + if isinstance(env_ids, torch.Tensor): + env_ids = env_ids.tolist() + return [int(env_id) for env_id in env_ids] + + def _prepare_terms(self) -> None: + """Build the term callables from the configuration object.""" + for term_name, term_cfg in self.cfg.__dict__.items(): + if term_cfg is None: + continue + # Validate the term's func/params. + self._resolve_common_term_cfg(term_name, term_cfg, min_argc=2) + self._term_names.append(term_name) + self._term_cfgs.append(term_cfg) + + @staticmethod + def _assert_json_serializable(term_name: str, fields: dict[str, Any]) -> None: + """Check ``term_name``'s recorded ``fields`` are JSON-serializable, failing fast if not. + + This points at the offending term rather than surfacing a cryptic error later at write() time when + the whole record is serialized. + """ + try: + json.dumps(fields) + except TypeError as e: + raise TypeError( + f"Episode recorder term '{term_name}' returned non-JSON-serializable fields ({fields!r}): {e}" + ) from e diff --git a/isaaclab_arena/tests/test_camera_observation_video_recorder.py b/isaaclab_arena/tests/test_camera_observation_video_recorder.py index c22ac1a5d..815f56ffd 100644 --- a/isaaclab_arena/tests/test_camera_observation_video_recorder.py +++ b/isaaclab_arena/tests/test_camera_observation_video_recorder.py @@ -35,13 +35,24 @@ class _StubEnv(gym.Env): def __init__(self): super().__init__() self._step_return = ({}, None, torch.zeros(1, dtype=torch.bool), torch.zeros(1, dtype=torch.bool), None) + # Per-env completed-episode counts, mirroring the Arena env's centralized episode index. + self._episode_counts: dict[int, int] = {} def reset(self, **kwargs): return {}, {} def step(self, action): + # Mirror the Arena env: advance the per-env episode index for each env that resets this + # step (the real env does this within _reset_idx, before step() returns). + _, _, terminated, truncated, _ = self._step_return + for env_id in (terminated | truncated).nonzero().flatten().tolist(): + self._episode_counts[env_id] = self._episode_counts.get(env_id, 0) + 1 return self._step_return + def get_episode_index(self, env_id: int) -> int: + """The current episode index for ``env_id`` (its count of completed episodes).""" + return self._episode_counts.get(env_id, 0) + def render(self): pass @@ -85,7 +96,7 @@ def test_video_files_written_on_termination(tmp_path): def test_episode_counter_increments_per_env(tmp_path): - """Each env tracks its own episode count independently.""" + """Each env tracks its own episode count independently via the env's centralized index.""" env = _make_env() with patch("isaaclab_arena.video.camera_observation_video_recorder.ImageSequenceClip"): recorder = CameraObsVideoRecorder(env, video_folder=str(tmp_path)) @@ -105,8 +116,8 @@ def test_episode_counter_increments_per_env(tmp_path): _configure_step(env, done_envs=[1]) recorder.step(None) # env 1: episode 0 done - assert recorder.episode_counts[0] == 2 - assert recorder.episode_counts[1] == 1 + assert env.get_episode_index(0) == 2 + assert env.get_episode_index(1) == 1 def test_multiple_episodes_produce_sequential_filenames(tmp_path): @@ -143,17 +154,20 @@ def test_partial_episode_dropped_on_close(tmp_path): mock_clip_cls.return_value.write_videofile.assert_not_called() -def test_episode_counter_not_incremented_for_empty_buffer(tmp_path): - """Episode count stays at 0 if the env terminates before any frame is appended.""" +def test_no_video_written_for_empty_episode(tmp_path): + """An env terminating with no buffered frames writes no video; its episode index still advances.""" env = _make_env() - with patch("isaaclab_arena.video.camera_observation_video_recorder.ImageSequenceClip"): + with patch("isaaclab_arena.video.camera_observation_video_recorder.ImageSequenceClip") as mock_clip_cls: recorder = CameraObsVideoRecorder(env, video_folder=str(tmp_path)) # Terminate on the very first step — no prior frames were recorded. _configure_step(env, done_envs=[0]) recorder.step(None) - assert recorder.episode_counts[0] == 0 + # No video for the empty episode, but the env's centralized index still advanced past it + # (so a later episode's video number stays in lockstep with the per-episode results record). + mock_clip_cls.return_value.write_videofile.assert_not_called() + assert env.get_episode_index(0) == 1 def test_post_reset_frame_not_appended(tmp_path): diff --git a/isaaclab_arena/tests/test_episode_recorder.py b/isaaclab_arena/tests/test_episode_recorder.py new file mode 100644 index 000000000..367883fd6 --- /dev/null +++ b/isaaclab_arena/tests/test_episode_recorder.py @@ -0,0 +1,267 @@ +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +import json +import tempfile +import torch +import tqdm +from dataclasses import field +from pathlib import Path + +from isaaclab.managers import EventTermCfg, SceneEntityCfg +from isaaclab.utils import configclass + +from isaaclab_arena.tests.utils.subprocess import run_simulation_app_function +from isaaclab_arena.variations.uniform_sampler import UniformSamplerCfg +from isaaclab_arena.variations.variation_base import RunTimeVariationBase, VariationBaseCfg + +NUM_STEPS = 200 +NUM_ENVS = 2 +HEADLESS = True + +JOB_NAME = "unit_test" +LANGUAGE_INSTRUCTION = "put the box in the drawer" + +# Fields stamped by the manager (metadata) plus those from the default core term. +CORE_KEYS = { + "job_name", + "episode_in_env", + "env_id", + "seed", + "success", + "episode_length", + "language_instruction", + "timestamp", +} + +# Field contributed by the custom term registered in the custom-term test. +CUSTOM_KEY = "step_bucket" + +# Deterministic, single-valued (low == high) sample for the variation test, so each draw is known. +VARIATION_NAME = "record_test_variation" +VARIATION_SAMPLE = [0.25, 0.5] + + +def record_step_bucket(env, env_id): + """Custom recorder term: records the finished episode's length bucketed into tens.""" + return {CUSTOM_KEY: int(env.episode_length_buf[env_id].item()) // 10} + + +def draw_record_test_variation(env, env_ids, asset_cfg, sampler): # noqa: ARG001 + """Reset event that only draws a sample, so the variation recorder attributes it to the episode.""" + sampler.sample(num_samples=len(env_ids), env_ids=env_ids) + + +@configclass +class RecordTestVariationCfg(VariationBaseCfg): + """Cfg for ``RecordTestVariation`` with a degenerate (constant) sampler for deterministic draws.""" + + sampler_cfg: UniformSamplerCfg = field( + default_factory=lambda: UniformSamplerCfg(low=VARIATION_SAMPLE, high=VARIATION_SAMPLE), + ) + + +class RecordTestVariation(RunTimeVariationBase): + """Minimal run-time variation that samples on each reset without mutating the scene.""" + + cfg: RecordTestVariationCfg + + def __init__(self, asset_name: str, name: str = VARIATION_NAME): + super().__init__(cfg=RecordTestVariationCfg(), name=name) + self.asset_name = asset_name + + def build_event_cfg(self) -> tuple[str, EventTermCfg]: + event_cfg = EventTermCfg( + func=draw_record_test_variation, + mode="reset", + params={"asset_cfg": SceneEntityCfg(self.asset_name), "sampler": self._sampler}, + ) + return f"{self.asset_name}_{VARIATION_NAME}", event_cfg + + +def create_recorder_env( + output_dir, *, episode_recorder_terms: dict[str, object] | None = None, enable_variation: bool = False +): + """Build a registered two-env pick-and-place env wired for per-episode recording. + + env 0's box lands in the drawer (success) while env 1's box lands outside it (failure). + + Args: + output_dir: Directory the JSONL records are written into. + episode_recorder_terms: Extra per-episode recorder terms (i.e. EpisodeRecorderTermCfg. + enable_variation: When True, attach an enabled run-time variation to the cracker box. + + Returns: + An ``(env, output_path)`` tuple: the registered env and the JSONL path to write the records to. + """ + from isaaclab_arena.assets.object_reference import ObjectReference + from isaaclab_arena.assets.registries import AssetRegistry + from isaaclab_arena.cli.isaaclab_arena_cli import get_isaaclab_arena_cli_parser + from isaaclab_arena.environments.arena_env_builder import ArenaEnvBuilder + from isaaclab_arena.environments.isaaclab_arena_environment import IsaacLabArenaEnvironment + from isaaclab_arena.scene.scene import Scene + from isaaclab_arena.tasks.pick_and_place_task import PickAndPlaceTask + from isaaclab_arena.terms.events import set_object_pose_per_env + from isaaclab_arena.utils.pose import Pose + + asset_registry = AssetRegistry() + background = asset_registry.get_asset_by_name("kitchen_with_open_drawer")() + embodiment = asset_registry.get_asset_by_name("franka_ik")() + cracker_box = asset_registry.get_asset_by_name("cracker_box")() + destination_location = ObjectReference( + name="destination_location", + prim_path="{ENV_REGEX_NS}/kitchen_with_open_drawer/Cabinet_B_02", + parent_asset=background, + ) + + if enable_variation: + variation = RecordTestVariation(cracker_box.name) + variation.enable() + cracker_box.add_variation(variation) + + scene = Scene(assets=[background, cracker_box]) + isaaclab_arena_environment = IsaacLabArenaEnvironment( + name="episode_recorder", + embodiment=embodiment, + scene=scene, + task=PickAndPlaceTask(cracker_box, destination_location, background), + teleop_device=None, + episode_recorder_terms=episode_recorder_terms or {}, + ) + + args_cli = get_isaaclab_arena_cli_parser().parse_args([]) + args_cli.num_envs = NUM_ENVS + # The builder applies the language-instruction override onto the env cfg's task_description, which the + # core recorder then records. + args_cli.language_instruction = LANGUAGE_INSTRUCTION + env_builder = ArenaEnvBuilder(isaaclab_arena_environment, args_cli) + env_cfg, env_kwargs = env_builder.compose_manager_cfg() + + # Per-env reset poses: env 0 lands in the drawer (success), env 1 lands outside (failure). + pose_list = [ + Pose(position_xyz=(0.0, -0.5, 0.2), rotation_xyzw=(0.0, 0.0, 0.0, 1.0)), + Pose(position_xyz=(-0.5, -0.5, 0.2), rotation_xyzw=(0.0, 0.0, 0.0, 1.0)), + ] + env_cfg.events.reset_pick_up_object_pose = EventTermCfg( + func=set_object_pose_per_env, + mode="reset", + params={ + "pose_list": pose_list, + "asset_cfg": SceneEntityCfg(cracker_box.name), + }, + ) + + output_path = Path(output_dir) / "episode_results.jsonl" + + env = env_builder.make_registered(env_cfg, env_kwargs) + env.unwrapped.episode_recorder.set_job_name(JOB_NAME) + env.unwrapped.episode_recorder.set_output_path(output_path) + env.reset() + return env, output_path + + +def _roll_out_and_read_episode_record(env, output_path) -> list[dict]: + """Step the env for ``NUM_STEPS`` (records stream to disk as episodes finish), then parse them.""" + for _ in tqdm.tqdm(range(NUM_STEPS)): + with torch.inference_mode(): + actions = torch.zeros(env.action_space.shape, device=env.unwrapped.device) + env.step(actions) + + assert output_path.exists(), f"Expected JSONL at {output_path}" + with open(output_path, encoding="utf-8") as f: + records = [json.loads(line) for line in f if line.strip()] + print(f"Recorded {len(records)} episode(s)") + return records + + +def _test_core_terms(simulation_app, output_dir): # noqa: ARG001 + env, output_path = create_recorder_env(output_dir) + try: + records = _roll_out_and_read_episode_record(env, output_path) + assert len(records) >= NUM_ENVS, f"Expected at least {NUM_ENVS} episodes, got {len(records)}" + + # episode_in_env must increment from 0 per env, and the deterministic poses fix success. + per_env_counter: dict[int, int] = {} + for record in records: + # With no variation drawn and no custom term, every record is exactly the core schema. + assert set(record.keys()) == CORE_KEYS, f"Unexpected keys: {set(record.keys()) - CORE_KEYS}" + assert record["job_name"] == JOB_NAME + assert record["language_instruction"] == LANGUAGE_INSTRUCTION + assert isinstance(record["episode_length"], int) + + env_id = record["env_id"] + assert env_id in (0, 1) + assert record["episode_in_env"] == per_env_counter.get(env_id, 0) + per_env_counter[env_id] = per_env_counter.get(env_id, 0) + 1 + expected_success = env_id == 0 + assert ( + record["success"] is expected_success + ), f"env {env_id} episode {record['episode_in_env']}: expected success={expected_success}" + + # Both envs must have completed at least one episode. + assert set(per_env_counter.keys()) == {0, 1} + finally: + env.close() + return True + + +def _test_variations_recorded(simulation_app, output_dir): # noqa: ARG001 + env, output_path = create_recorder_env(output_dir, enable_variation=True) + try: + records = _roll_out_and_read_episode_record(env, output_path) + + # The enabled variation must be registered with the recorder and recorded on every episode. + recorded_keys = set(env.unwrapped.variation_recorder.records.keys()) + assert recorded_keys, "Expected the enabled variation to be attached to the variation recorder" + for record in records: + assert "variations" in record, f"Missing 'variations' field: {set(record.keys())}" + assert set(record["variations"].keys()) == recorded_keys + for value in record["variations"].values(): + assert value == VARIATION_SAMPLE, f"Expected sample {VARIATION_SAMPLE}, got {value}" + finally: + env.close() + return True + + +def _test_custom_term(simulation_app, output_dir): # noqa: ARG001 + from isaaclab_arena.recording.episode_recorder_manager import EpisodeRecorderTermCfg + + custom_terms = {"step_bucket": EpisodeRecorderTermCfg(func=record_step_bucket)} + env, output_path = create_recorder_env(output_dir, episode_recorder_terms=custom_terms) + try: + records = _roll_out_and_read_episode_record(env, output_path) + + # The custom term's field is present and derived from the same intact episode-length buffer. + for record in records: + assert set(record.keys()) == CORE_KEYS | {CUSTOM_KEY}, f"Unexpected keys: {set(record.keys())}" + assert record[CUSTOM_KEY] == record["episode_length"] // 10 + finally: + env.close() + return True + + +def test_core_terms(tmp_path): + assert run_simulation_app_function( + _test_core_terms, headless=HEADLESS, output_dir=tmp_path + ), "core recorder terms test failed" + + +def test_variations_recorded(tmp_path): + assert run_simulation_app_function( + _test_variations_recorded, headless=HEADLESS, output_dir=tmp_path + ), "variation recording test failed" + + +def test_custom_term(tmp_path): + assert run_simulation_app_function( + _test_custom_term, headless=HEADLESS, output_dir=tmp_path + ), "custom recorder term test failed" + + +if __name__ == "__main__": + with tempfile.TemporaryDirectory(prefix="episode_recorder_") as _tmp_dir: + test_core_terms(Path(_tmp_dir)) + test_variations_recorded(Path(_tmp_dir)) + test_custom_term(Path(_tmp_dir)) diff --git a/isaaclab_arena/tests/test_variation_recorder.py b/isaaclab_arena/tests/test_variation_recorder.py index cd07ff91f..b588df7e4 100644 --- a/isaaclab_arena/tests/test_variation_recorder.py +++ b/isaaclab_arena/tests/test_variation_recorder.py @@ -10,6 +10,7 @@ light via the registry pulls in ``isaaclab.sim``. """ +import torch from dataclasses import field import pytest @@ -19,7 +20,7 @@ from isaaclab_arena.variations.choice_sampler import ChoiceSampler from isaaclab_arena.variations.uniform_sampler import UniformSamplerCfg from isaaclab_arena.variations.variation_base import BuildTimeVariationBase, VariationBaseCfg -from isaaclab_arena.variations.variation_recorder import VariationRecorder +from isaaclab_arena.variations.variation_recorder import VariationRecord, VariationRecorder HEADLESS = True @@ -53,27 +54,32 @@ def apply(self) -> None: def test_uniform_sampler_notifies_listeners(): sampler = UniformSamplerCfg(low=[0.0], high=[1.0]).build() seen: list = [] - sampler.add_listener(lambda s: seen.append(s)) - result = sampler.sample(num_samples=4) + sampler.add_listener(lambda s, env_ids: seen.append((s, env_ids))) + result = sampler.sample(num_samples=4, env_ids=[0, 1, 2, 3]) assert len(seen) == 1 - assert seen[0] is result - assert tuple(seen[0].shape) == (4, 1) + sample, env_ids = seen[0] + assert sample is result + assert tuple(sample.shape) == (4, 1) + # env_ids are forwarded so listeners can attribute each row to its env. + assert env_ids == [0, 1, 2, 3] def test_choice_sampler_notifies_listeners(): sampler = ChoiceSampler() seen: list = [] - sampler.add_listener(lambda s: seen.append(s)) - sampler.sample(num_samples=3, choices=["x", "y", "z"]) + sampler.add_listener(lambda s, env_ids: seen.append((s, env_ids))) + sampler.sample(num_samples=3, choices=["x", "y", "z"], env_ids=[4, 5, 6]) assert len(seen) == 1 - assert isinstance(seen[0], list) and len(seen[0]) == 3 + sample, env_ids = seen[0] + assert isinstance(sample, list) and len(sample) == 3 + assert env_ids == [4, 5, 6] def test_variation_listener_survives_sampler_swap(): """A listener added via the variation must re-bind to the sampler rebuilt by apply_cfg.""" variation = _RecorderTestVariation() seen: list = [] - variation.add_sample_listener(lambda s: seen.append(s)) + variation.add_sample_listener(lambda s, _env_ids: seen.append(s)) # Swap in a new cfg (rebuilds the underlying sampler), then draw a sample. variation.apply_cfg(_RecorderTestVariationCfg(sampler_cfg=UniformSamplerCfg(low=[2.0], high=[2.0]))) @@ -83,21 +89,79 @@ def test_variation_listener_survives_sampler_swap(): assert sample.item() == pytest.approx(2.0, abs=1e-6) -def test_recorder_records_samples_from_attached_variation(): +def test_recorder_records_build_time_sample_for_every_episode(): + """A build-time draw (env_ids=None) is recorded and surfaced for every env/episode.""" variation = _RecorderTestVariation() variation.enable() recorder = VariationRecorder() recorder.attach({"asset": [variation]}) record = recorder["asset.recorder_test"] - assert len(record.samples) == 0 + assert record.sample_for_episode(0, 0) is None - variation.apply() variation.apply() - assert len(record.samples) == 2 + value = record.sample_for_episode(0, 0) # Tensor samples are stored detached on CPU. - assert record.samples[0].device.type == "cpu" + assert value.device.type == "cpu" + # A build-time draw applies to every env and episode, not just (0, 0). + assert record.sample_for_episode(7, 3).tolist() == value.tolist() + + +def test_variation_record_tracks_per_env_episode_values(): + """Each (env, episode) draw is stored and surfaced separately across envs and episodes.""" + record = VariationRecord(name="asset.var", cfg=_RecorderTestVariationCfg()) + + # Runtime-style draw: rows map to the given env ids, all in episode 0. + record.record_runtime_sample(torch.tensor([[1.0], [2.0]]), env_ids=[2, 5], episode_indices=[0, 0]) + assert record.sample_for_episode(2, 0).tolist() == [1.0] + assert record.sample_for_episode(5, 0).tolist() == [2.0] + # An env/episode not drawn for has no recorded value. + assert record.sample_for_episode(0, 0) is None + assert record.sample_for_episode(2, 1) is None + + # A draw for env 2 in the next episode lands under its own key. + record.record_runtime_sample(torch.tensor([[4.0]]), env_ids=[2], episode_indices=[1]) + assert record.sample_for_episode(2, 1).tolist() == [4.0] + assert record.sample_for_episode(2, 0).tolist() == [1.0] + + # Each (env, episode) is expected to be drawn for at most once. + with pytest.raises(AssertionError): + record.record_runtime_sample(torch.tensor([[3.0]]), env_ids=[2], episode_indices=[0]) + + +class _FakeEnv: + """Minimal stand-in exposing the attributes ``record_variation_samples`` reads.""" + + def __init__(self, recorder: VariationRecorder, episode_index: int = 0) -> None: + self.variation_recorder = recorder + self._episode_index = episode_index + + def get_episode_index(self, env_id: int) -> int: # noqa: ARG002 + return self._episode_index + + +def test_record_variation_samples_emits_the_per_episode_draw(): + """The episode term emits the draw made for the finishing episode, one value per variation.""" + from isaaclab_arena.recording.common_terms import record_variation_samples + + variation = _RecorderTestVariation() + variation.enable() + recorder = VariationRecorder() + recorder.attach({"asset": [variation]}) + env = _FakeEnv(recorder, episode_index=0) + recorder.bind_env(env) + + # One run-time draw for env 0 during episode 0. + variation.sampler.sample(num_samples=1, env_ids=torch.tensor([0])) + + fields = record_variation_samples(env, env_id=0) + sample = fields["variations"]["asset.recorder_test"] + # The single (1,)-shaped draw is recorded as a flat list, not a list of draws. + assert isinstance(sample, list) and len(sample) == 1 + + # A different env in the same episode has nothing recorded, so no variations field is emitted. + assert record_variation_samples(env, env_id=1) == {} def test_recorder_skips_disabled_variations(): @@ -137,15 +201,14 @@ def _test_hdr_variation_recorder_captures_chosen_hdr_name(simulation_app): recorder.attach(scene.get_asset_variations()) record = recorder["light.hdr_image"] - assert len(record.samples) == 0 + assert record.sample_for_episode(0, 0) is None variation.apply() - assert len(record.samples) == 1 - sample = record.samples[0] + # The HDR draw is build-time (env_ids=None), so it applies to every env/episode. + value = record.sample_for_episode(0, 0) # The recorder must capture the chosen HDR *name*, not an index. - assert isinstance(sample, list) and len(sample) == 1 - assert sample[0] in pool + assert value in pool return True diff --git a/isaaclab_arena/variations/camera_extrinsics_variation.py b/isaaclab_arena/variations/camera_extrinsics_variation.py index 668f06c64..3364747f3 100644 --- a/isaaclab_arena/variations/camera_extrinsics_variation.py +++ b/isaaclab_arena/variations/camera_extrinsics_variation.py @@ -148,8 +148,9 @@ def __call__( assert self._t_parent_C_in_parent is not None assert self._q_parent_C_xyzw is not None - # Sample a decalibration vector in the camera's ROS-style optical frame. - sample = sampler.sample(num_samples=len(env_ids)) + # Sample a decalibration vector in the camera's ROS-style optical frame. Pass env_ids so + # sample listeners (e.g. the variation recorder) can attribute each row to its env. + sample = sampler.sample(num_samples=len(env_ids), env_ids=env_ids) t_C_Cnew_in_Cros = sample.to(device=self._t_parent_C_in_parent.device, dtype=self._t_parent_C_in_parent.dtype) # Isaac Lab tensors use xyzw. 180 deg about +X maps ROS optical axes to OpenGL camera axes. diff --git a/isaaclab_arena/variations/choice_sampler.py b/isaaclab_arena/variations/choice_sampler.py index 8f3c95262..c516d5d44 100644 --- a/isaaclab_arena/variations/choice_sampler.py +++ b/isaaclab_arena/variations/choice_sampler.py @@ -27,19 +27,21 @@ def build(self) -> ChoiceSampler: class ChoiceSampler(SamplerBase, Generic[T]): """Uniform sampler returning items drawn from a per-call ``choices`` sequence.""" - def sample(self, num_samples: int, choices: Sequence[T]) -> list[T]: + def sample(self, num_samples: int, choices: Sequence[T], env_ids: torch.Tensor | None = None) -> list[T]: """Draw ``num_samples`` items from ``choices``. Args: num_samples: Number of independent samples to draw, typically the number of environments we're drawing a sample for. choices: Pool of items to draw from. Must be non-empty. + env_ids: The env ids the drawn items correspond to, forwarded to sample listeners so + they can attribute values per env. ``None`` when the draw applies to all envs. Returns: A ``list`` of length ``num_samples`` of items drawn from ``choices``. """ result = self._sample(num_samples, choices) - self._notify(result) + self._notify(result, env_ids) return result def _sample(self, num_samples: int, choices: Sequence[T]) -> list[T]: diff --git a/isaaclab_arena/variations/continuous_sampler.py b/isaaclab_arena/variations/continuous_sampler.py index 2796cc832..48787db0e 100644 --- a/isaaclab_arena/variations/continuous_sampler.py +++ b/isaaclab_arena/variations/continuous_sampler.py @@ -32,18 +32,20 @@ class ContinuousSampler(SamplerBase): ``(num_samples, *shape_per_sample)``. """ - def sample(self, num_samples: int) -> torch.Tensor: + def sample(self, num_samples: int, env_ids: torch.Tensor | None = None) -> torch.Tensor: """Draw ``num_samples`` values from this distribution. Args: num_samples: Number of independent samples to draw, typically the number of environments we're drawing a sample for. + env_ids: The env ids the drawn rows correspond to, forwarded to sample listeners so + they can attribute values per env. ``None`` when the draw applies to all envs. Returns: A tensor of shape ``(num_samples, *shape_per_sample)``. """ result = self._sample(num_samples) - self._notify(result) + self._notify(result, env_ids) return result @abstractmethod diff --git a/isaaclab_arena/variations/sampler_base.py b/isaaclab_arena/variations/sampler_base.py index 1b94078bd..b1e80ece9 100644 --- a/isaaclab_arena/variations/sampler_base.py +++ b/isaaclab_arena/variations/sampler_base.py @@ -7,10 +7,13 @@ from abc import ABC from collections.abc import Callable -from typing import Any +from typing import TYPE_CHECKING, Any from isaaclab.utils import configclass +if TYPE_CHECKING: + import torch + @configclass class SamplerBaseCfg: @@ -31,13 +34,17 @@ class SamplerBase(ABC): """Base class shared by every sampler family.""" def __init__(self) -> None: - self._listeners: list[Callable[[Any], None]] = [] + self._listeners: list[Callable[[Any, torch.Tensor | None], None]] = [] - def add_listener(self, listener: Callable[[Any], None]) -> None: - """Register ``listener`` to be called with every sample drawn from this sampler.""" + def add_listener(self, listener: Callable[[Any, torch.Tensor | None], None]) -> None: + """Register ``listener``, called as ``listener(sample, env_ids)`` for every sample drawn.""" self._listeners.append(listener) - def _notify(self, sample: Any) -> None: - """Forward ``sample`` to every registered listener, in registration order.""" + def _notify(self, sample: Any, env_ids: torch.Tensor | None = None) -> None: + """Forward ``sample`` (and the ``env_ids`` it was drawn for) to every registered listener. + + ``env_ids`` is the per-env id tensor/sequence the sample's rows correspond to, or ``None`` + when the single sample applies to all envs (e.g. a build-time draw). + """ for listener in self._listeners: - listener(sample) + listener(sample, env_ids) diff --git a/isaaclab_arena/variations/variation_base.py b/isaaclab_arena/variations/variation_base.py index 9d35f2fb4..7738a56c5 100644 --- a/isaaclab_arena/variations/variation_base.py +++ b/isaaclab_arena/variations/variation_base.py @@ -21,13 +21,16 @@ from abc import ABC, abstractmethod from collections.abc import Callable from dataclasses import field -from typing import Any +from typing import TYPE_CHECKING, Any from isaaclab.managers import EventTermCfg from isaaclab.utils import configclass from isaaclab_arena.variations.sampler_base import SamplerBase, SamplerBaseCfg +if TYPE_CHECKING: + import torch + @configclass class VariationBaseCfg: @@ -57,7 +60,7 @@ class VariationBase(ABC): def __init__(self, cfg: VariationBaseCfg, name: str): self.name = name self._sampler: SamplerBase | None = None - self._sample_listeners: list[Callable[[Any], None]] = [] + self._sample_listeners: list[Callable[[Any, Any], None]] = [] self.apply_cfg(cfg) @property @@ -78,8 +81,8 @@ def sampler(self) -> SamplerBase | None: """The sampler driving this variation, or ``None`` if not yet set.""" return self._sampler - def add_sample_listener(self, listener: Callable[[Any], None]) -> None: - """Subscribe ``listener`` to every sample drawn by this variation's sampler. + def add_sample_listener(self, listener: Callable[[Any, torch.Tensor | None], None]) -> None: + """Subscribe ``listener`` (called as ``listener(sample, env_ids)``) to this variation's samples. Listeners are stored on the variation, so ``apply_cfg`` re-binds them onto the rebuilt sampler and they survive cfg/sampler swaps. diff --git a/isaaclab_arena/variations/variation_recorder.py b/isaaclab_arena/variations/variation_recorder.py index bae237c28..0c5922dcf 100644 --- a/isaaclab_arena/variations/variation_recorder.py +++ b/isaaclab_arena/variations/variation_recorder.py @@ -6,60 +6,67 @@ from __future__ import annotations import torch -from collections.abc import Callable +from collections.abc import Sequence +from dataclasses import dataclass from typing import TYPE_CHECKING, Any -from omegaconf import OmegaConf - if TYPE_CHECKING: from isaaclab_arena.variations.variation_base import VariationBase, VariationBaseCfg +@dataclass(frozen=True) +class EnvEpisodeKey: + """Hashable key identifying one env's draws during one episode.""" + + env_id: int + episode_idx: int + + class VariationRecord: - """Per-variation record configuration and samples.""" + """Per-variation record of the values drawn for it.""" def __init__(self, name: str, cfg: VariationBaseCfg) -> None: self.name = name self.cfg = cfg - self.samples: list[Any] = [] - - def _header_lines(self) -> list[str]: - """Return the shared preamble (identity, cfg, sample-call count) for renderers.""" - # Add the title for this variation - lines = [f"--- {self.name} ---", "cfg:"] - # Print the Cfg - lines.append(OmegaConf.to_yaml(OmegaConf.structured(self.cfg)).rstrip()) - # Print basic information about the samples - lines.append(f"sample calls: {len(self.samples)}") - if self.samples and isinstance(self.samples[0], torch.Tensor): - stacked_shape = (len(self.samples), *tuple(self.samples[0].shape)) - lines.append(f"stacked shape: {stacked_shape}") - return lines - - @staticmethod - def _format_sample(sample: Any) -> str: - """Render a single sample value (tensors as nested lists, others via ``repr``).""" - return f"{sample.tolist()}" if isinstance(sample, torch.Tensor) else f"{sample!r}" - - def summary(self) -> str: - """Return a multi-line human-readable summary of this record (first/last sample only).""" - lines = self._header_lines() - if self.samples: - # Print the first and last sample - lines.append(f"first call: {self._format_sample(self.samples[0])}") - lines.append(f"last call: {self._format_sample(self.samples[-1])}") - return "\n".join(lines) - - def details(self) -> str: - """Return a multi-line human-readable view of this record, listing every sample.""" - lines = self._header_lines() - # Print every sample - for i, sample in enumerate(self.samples): - lines.append(f"call {i}: {self._format_sample(sample)}") - return "\n".join(lines) - - def __str__(self) -> str: - return self.summary() + # Run-time draw, one per (env id, episode index). + self._samples_by_env_episode: dict[EnvEpisodeKey, Any] = {} + # Build-time (all-envs) draw; applies to every episode of every env. + self._build_time_sample: Any = None + + def record_runtime_sample(self, sample: Any, env_ids: Sequence[int], episode_indices: Sequence[int]) -> None: + """Record each row of ``sample`` against the (env id, episode index) it was drawn for. + + Each (env id, episode index) is expected to be drawn for at most once. + + Args: + sample: The drawn sample; row ``i`` is the value for the ``i``-th env in ``env_ids``. + env_ids: The env ids the sample's rows correspond to. + episode_indices: The episode index each row was drawn during, aligned with ``env_ids``. + """ + for row, (env_id, episode_idx) in enumerate(zip(env_ids, episode_indices)): + key = EnvEpisodeKey(env_id, episode_idx) + assert ( + key not in self._samples_by_env_episode + ), f"Variation '{self.name}' already recorded a sample for env {env_id}, episode {episode_idx}." + self._samples_by_env_episode[key] = sample[row] + + def record_buildtime_sample(self, sample: Any) -> None: + """Record the all-envs (build-time) ``sample``; it applies to every episode of every env.""" + assert ( + len(sample) == 1 + ), f"Variation '{self.name}' build-time draw expected a single sample for all envs; got {len(sample)}." + self._build_time_sample = sample[0] + + def sample_for_episode(self, env_id: int, episode_idx: int) -> Any: + """Return the value drawn for ``env_id``'s ``episode_idx``, or ``None`` if none was drawn. + + A build-time (all-envs) draw applies to every episode; otherwise the run-time draw made + during that episode is returned. + """ + key = EnvEpisodeKey(env_id, episode_idx) + if key in self._samples_by_env_episode: + return self._samples_by_env_episode[key] + return self._build_time_sample class VariationRecorder: @@ -68,6 +75,12 @@ class VariationRecorder: def __init__(self) -> None: # Records are keyed by: "{asset_name}.{variation_name}" self.records: dict[str, VariationRecord] = {} + # Bound after env construction; supplies the episode index for per-env run-time draws. + self._env: Any = None + + def bind_env(self, env: Any) -> None: + """Bind the env so run-time draws can be attributed to its current episode index.""" + self._env = env def __getitem__(self, key: str) -> VariationRecord: """Return the record stored under "{asset_name}.{variation_name}".""" @@ -92,26 +105,18 @@ def attach(self, variations: dict[str, list[VariationBase]]) -> None: record = VariationRecord(name=variation_key, cfg=variation.cfg) self.records[variation_key] = record - def on_sample(sample: Any, record: VariationRecord = record) -> None: + def on_sample( + sample: Any, env_ids: torch.Tensor | None = None, record: VariationRecord = record + ) -> None: if isinstance(sample, torch.Tensor): - record.samples.append(sample.detach().cpu()) + sample = sample.detach().cpu() + if env_ids is None: + # Build-time / all-envs draw: applies to every episode of every env. + record.record_buildtime_sample(sample) else: - record.samples.append(sample) + assert self._env is not None, "VariationRecorder needs bind_env() before per-env draws." + env_id_list = env_ids.tolist() + episode_indices = [self._env.get_episode_index(env_id) for env_id in env_id_list] + record.record_runtime_sample(sample, env_id_list, episode_indices) variation.add_sample_listener(on_sample) - - def _render(self, render_record: Callable[[VariationRecord], str]) -> str: - """Join ``render_record`` applied to every attached record under a shared header.""" - parts = [f"VariationRecorder: {len(self.records)} record(s)"] - for record in self.records.values(): - parts.append("") - parts.append(render_record(record)) - return "\n".join(parts) - - def summary(self) -> str: - """Return a multi-line human-readable summary of every attached record (first/last sample).""" - return self._render(VariationRecord.summary) - - def details(self) -> str: - """Return a multi-line human-readable view of every attached record, listing all samples.""" - return self._render(VariationRecord.details) diff --git a/isaaclab_arena/video/camera_observation_video_recorder.py b/isaaclab_arena/video/camera_observation_video_recorder.py index a0ec33e00..842775ad1 100644 --- a/isaaclab_arena/video/camera_observation_video_recorder.py +++ b/isaaclab_arena/video/camera_observation_video_recorder.py @@ -115,9 +115,6 @@ def __init__( # camera_name -> list of per-env frame lists: buffers[camera_name][env_idx] = [frame, ...] self.buffers: dict[str, list[list[np.ndarray]]] = {} - # How many episodes have been flushed for each env. - self.episode_counts: list[int] = [] - self._n_envs: int | None = None def step(self, action): result = self.env.step(action) @@ -127,10 +124,6 @@ def step(self, action): if cam_obs: n_envs = next(iter(cam_obs.values())).shape[0] - if self._n_envs is None: - self._n_envs = n_envs - self.episode_counts = [0] * n_envs - # Determine done envs before appending frames. Isaac Lab auto-resets on # termination, so the obs returned for a done env is the post-reset first # frame of the new episode — discard it so it doesn't contaminate the @@ -153,8 +146,11 @@ def step(self, action): def _flush_envs(self, env_ids: list[int]) -> None: for env_idx in env_ids: - episode_num = self.episode_counts[env_idx] - wrote_any = False + # The Arena env has already advanced its per-env episode counter for this reset (within + # env.step, before it returned), so the just-finished episode's index is one behind the + # current count. Sharing the env's index keeps the filename's episode number in lockstep + # with the per-episode results record's ``episode_in_env``. + episode_num = self.unwrapped.get_episode_index(env_idx) - 1 for camera_name, env_frame_lists in self.buffers.items(): frames = env_frame_lists[env_idx] if not frames: @@ -167,9 +163,6 @@ def _flush_envs(self, env_ids: list[int]) -> None: clip.write_videofile(path, logger=None, audio=False) del clip env_frame_lists[env_idx] = [] - wrote_any = True - if wrote_any: - self.episode_counts[env_idx] += 1 def close(self) -> None: # Partial episodes (cut off by num_steps rather than a real reset) are discarded. diff --git a/isaaclab_arena/visualization/report.py b/isaaclab_arena/visualization/report.py index 10115f2eb..329c670c2 100644 --- a/isaaclab_arena/visualization/report.py +++ b/isaaclab_arena/visualization/report.py @@ -11,14 +11,24 @@ import functools import html import http.server +import json import pathlib import re import socketserver import string -from dataclasses import dataclass +from dataclasses import dataclass, field from isaaclab_arena.video.camera_observation_video_recorder import parse_episode_video_filename +# Matches the per-episode results filename written by EpisodeRecorderManager.write. The eval runner +# writes one file per rebuild (``episode_results_rebuild.jsonl``); the policy runner writes one +# per rank (``episode_results_rank.jsonl``, which carries no rebuild and so maps to rebuild 0). +_RESULTS_FILENAME_PATTERN = re.compile(r"^episode_results(?:_rebuild(?P\d+))?(?:_rank\d+)?\.jsonl$") + +# Record fields rendered explicitly elsewhere (status badge / row label), so excluded from the +# trailing metadata column to avoid duplication. +_METADATA_EXCLUDED_FIELDS = frozenset({"env_id", "episode_in_env", "success", "job_name"}) + # Reverse-dated run directory written by ``timestamped_run_dir`` (e.g. ``2026-06-17_14-42-54``). _RUN_DIR_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}$") @@ -41,6 +51,9 @@ class EpisodeVideos: video_by_camera: dict[str, str] """Camera name -> mp4 path, relative to the scanned root (and so to the report's index.html).""" + record: dict = field(default_factory=dict) + """The matching per-episode results record (success, seed, timing, ...); empty when unavailable.""" + @dataclass class JobReport: @@ -64,6 +77,33 @@ class EvaluationReport: jobs: list[JobReport] +def _scan_results(root: pathlib.Path) -> dict[str, dict[tuple[int, int, int], dict]]: + """Scan ``root`` for the recorder's JSONL files, indexed per job by ``(env, rebuild, episode)``. + + The key matches how ``_scan_jobs`` identifies a video: ``episode_in_env`` lines up with the video + filename's ``-episode-`` number, and the rebuild comes from the results filename. + + Args: + root: Directory of evaluation results to scan. + """ + results: dict[str, dict[tuple[int, int, int], dict]] = {} + for path in sorted(root.rglob("*.jsonl")): + match = _RESULTS_FILENAME_PATTERN.match(path.name) + if match is None: + continue + relative = path.relative_to(root) + job = "" if relative.parent == pathlib.Path(".") else str(relative.parent) + rebuild = int(match.group("rebuild")) if match.group("rebuild") is not None else 0 + job_results = results.setdefault(job, {}) + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + record = json.loads(line) + job_results[(int(record["env_id"]), rebuild, int(record["episode_in_env"]))] = record + return results + + def _scan_jobs(root: pathlib.Path) -> list[JobReport]: """Recursively scan ``root`` for recorder mp4s and group them into per-job reports. @@ -79,6 +119,7 @@ def _scan_jobs(root: pathlib.Path) -> list[JobReport]: # job -> env -> {(rebuild, recorder_episode): {camera: relative_path}} raw: dict[str, dict[int, dict[tuple[int, int], dict[str, str]]]] = {} cameras_by_job: dict[str, list[str]] = {} + results = _scan_results(root) for path in sorted(root.rglob("*.mp4")): parsed = parse_episode_video_filename(path.name) @@ -105,11 +146,14 @@ def _scan_jobs(root: pathlib.Path) -> list[JobReport]: for env_index in sorted(raw[job]): # Renumber (rebuild, recorder_episode) pairs into a contiguous, rebuild-agnostic index. for episode_index, recording_key in enumerate(sorted(raw[job][env_index])): + rebuild, recorder_episode = recording_key + record = results.get(job, {}).get((env_index, rebuild, recorder_episode), {}) episodes.append( EpisodeVideos( env_index=env_index, episode_index=episode_index, video_by_camera=raw[job][env_index][recording_key], + record=record, ) ) jobs.append(JobReport(name=job, cameras=sorted(cameras_by_job[job]), episodes=episodes)) @@ -127,12 +171,59 @@ def _render_video_cell(src: str) -> str: ) +def _render_row_label(episode: EpisodeVideos) -> str: + """Render the sticky first cell: env/episode label plus a success/failure status badge. + + The cell is tinted green for success and red for failure (neutral when the task records no + ``success`` term or no record was found). + """ + success = episode.record.get("success") + if success is True: + status_class, status_text = " success", "success" + elif success is False: + status_class, status_text = " failure", "failure" + else: + status_class, status_text = "", "n/a" + return ( + f'' + f"env {episode.env_index}
episode {episode.episode_index}" + f'
{status_text}' + ) + + +def _render_metadata_entry(key: str, value: object) -> str: + """Render one metadata field as a labelled block. + + Dict values (e.g. the per-episode ``variations``) are split one indented sub-line per item so + they don't render as a single long line. + """ + if isinstance(value, dict): + sub_rows = "".join( + f'
{html.escape(str(sub_key))}' + f" {html.escape(str(sub_value))}
" + for sub_key, sub_value in value.items() + ) + return f'
{html.escape(key)}{sub_rows}
' + return f'
{html.escape(key)} {html.escape(str(value))}
' + + +def _render_metadata_cell(record: dict) -> str: + """Render the trailing cell holding the remaining per-episode metadata as a key/value list.""" + rows = [ + _render_metadata_entry(key, value) + for key, value in record.items() + if key not in _METADATA_EXCLUDED_FIELDS and value is not None + ] + return '—' if not rows else f'{"".join(rows)}' + + def _render_row(episode: EpisodeVideos, cameras: list[str]) -> str: - """Render one table row: the env/episode label followed by one cell per camera column.""" - cells = [f'env {episode.env_index}
episode {episode.episode_index}'] + """Render one table row: status label, one cell per camera column, then a metadata cell.""" + cells = [_render_row_label(episode)] for camera in cameras: src = episode.video_by_camera.get(camera) cells.append('—' if src is None else _render_video_cell(src)) + cells.append(_render_metadata_cell(episode.record)) return "" + "".join(cells) + "" @@ -143,7 +234,7 @@ def _render_job_section(job: JobReport) -> str: body_rows = "\n".join(_render_row(episode, job.cameras) for episode in job.episodes) return ( f"
{heading}" - f'{header_cells}' + f'{header_cells}' f"\n{body_rows}\n
env / episode
env / episodedetails
" ) @@ -191,7 +282,7 @@ def serve_until_ctrl_c(directory: pathlib.Path, port: int, filename: str) -> Non """ handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory=str(directory)) url = f"http://localhost:{port}/{filename}" - # Avoid "Address already in use". + # Avoid "Address already in use" when a previous server's socket is still in TIME_WAIT. socketserver.TCPServer.allow_reuse_address = True try: server = socketserver.TCPServer(("0.0.0.0", port), handler) diff --git a/isaaclab_arena/visualization/report_template.html b/isaaclab_arena/visualization/report_template.html index 3e8a6bf9d..0ff9edb4f 100644 --- a/isaaclab_arena/visualization/report_template.html +++ b/isaaclab_arena/visualization/report_template.html @@ -16,8 +16,14 @@ thead th { position: sticky; top: 0; background: #f0f0f0; z-index: 2; font-size: 0.85rem; } .rowlabel { position: sticky; left: 0; background: #f0f0f0; font-family: ui-monospace, monospace; font-size: 0.8rem; white-space: nowrap; text-align: left; z-index: 1; } + .rowlabel.success { background: #e6f4ea; color: #137333; } + .rowlabel.failure { background: #fce8e6; color: #c5221f; } + .rowlabel .status { font-weight: 700; text-transform: uppercase; font-size: 0.7rem; } video { width: 320px; height: auto; display: block; background: #000; } .missing { color: #bbb; text-align: center; } + .meta { font-size: 0.75rem; color: #444; white-space: nowrap; } + .meta .k { color: #888; } + .meta .subitem { margin-left: 0.75rem; } diff --git a/isaaclab_arena_environments/robolab/bagel_plate_banana_bowl_linked.yaml b/isaaclab_arena_environments/robolab/bagel_plate_banana_bowl_linked.yaml index 62f01bbfb..f8ee43399 100644 --- a/isaaclab_arena_environments/robolab/bagel_plate_banana_bowl_linked.yaml +++ b/isaaclab_arena_environments/robolab/bagel_plate_banana_bowl_linked.yaml @@ -39,6 +39,7 @@ tasks: pick_up_object: banana destination_location: plate background_scene: maple_table_robolab + episode_length_s: 20.0 description: Pick up the banana and place it on the plate. id: task_0_PickAndPlaceTask initial_state_spec_id: state_initial diff --git a/isaaclab_arena_environments/robolab/bin_mug_marker_bowl_linked.yaml b/isaaclab_arena_environments/robolab/bin_mug_marker_bowl_linked.yaml index 72cda0789..a4c00ee0f 100644 --- a/isaaclab_arena_environments/robolab/bin_mug_marker_bowl_linked.yaml +++ b/isaaclab_arena_environments/robolab/bin_mug_marker_bowl_linked.yaml @@ -39,6 +39,7 @@ tasks: pick_up_object: bowl destination_location: grey_bin background_scene: maple_table_robolab + episode_length_s: 20.0 description: Pick up the bowl and place it in the grey bin. id: task_0_PickAndPlaceTask initial_state_spec_id: state_initial diff --git a/isaaclab_arena_environments/robolab/butter_raisin_box_grey_bin_linked.yaml b/isaaclab_arena_environments/robolab/butter_raisin_box_grey_bin_linked.yaml index 7c0017573..d79b9829a 100644 --- a/isaaclab_arena_environments/robolab/butter_raisin_box_grey_bin_linked.yaml +++ b/isaaclab_arena_environments/robolab/butter_raisin_box_grey_bin_linked.yaml @@ -31,8 +31,8 @@ tasks: pick_up_object: raisin_box destination_location: grey_bin background_scene: maple_table_robolab - episode_length_s: 20.0 max_separation: [0.1, 0.1, 0.1] + episode_length_s: 20.0 description: Pick up the red raisin box and place it into the grey bin on the maple table. id: task_0_PickAndPlaceTask diff --git a/isaaclab_arena_environments/robolab/mustard_raisin_box_linked.yaml b/isaaclab_arena_environments/robolab/mustard_raisin_box_linked.yaml index 9eb99cbd5..824fb950a 100644 --- a/isaaclab_arena_environments/robolab/mustard_raisin_box_linked.yaml +++ b/isaaclab_arena_environments/robolab/mustard_raisin_box_linked.yaml @@ -27,6 +27,7 @@ tasks: pick_up_object: mustard_bottle destination_location: raisin_box background_scene: maple_table_robolab + episode_length_s: 20.0 description: Pick up the mustard bottle and place it on the raisin box. id: task_0_PickAndPlaceTask initial_state_spec_id: state_initial