Start improving high-throughput multi-camera recording path and diagnostics

.gitignore

@@ -113,3 +113,8 @@ venv.bak/
 !dlclivegui/config.py
 # uv package files
 uv.lock
+
+# profiling
+profile*.svg
+scalene*.json
+scalene*.html

dlclivegui/cameras/backends/basler_backend.py

@@ -30,6 +30,8 @@
 except Exception:  # pragma: no cover - optional dependency
     pylon = None  # type: ignore
 
+DEBUG_TRIGGER_LOGS = False
+
 
 @register_backend("basler")
 class BaslerCameraBackend(CameraBackend):
@@ -627,7 +629,8 @@ def open(self) -> None:
                 pass
 
             self._camera.StartGrabbing(
-                pylon.GrabStrategy_LatestImageOnly,
+                # pylon.GrabStrategy_LatestImageOnly,
+                pylon.GrabStrategy_OneByOne,
             )
             LOG.info(
                 "[Basler] grabbing=%s max_buffers=%s",
@@ -650,7 +653,7 @@ def open(self) -> None:
         )
 
         # ----------------------------
-        # Persist stable identity into namespace (migration-safe)
+        # Persist stable identity into namespace
         # ----------------------------
         try:
             serial = device.GetSerialNumber()
@@ -947,7 +950,7 @@ def _set_numeric_feature(self, name: str, value, *, strict: bool = False) -> boo
             return False
 
     def _debug_trigger_nodes(self, *, context: str = "") -> None:
-        if not LOG.isEnabledFor(logging.DEBUG):
+        if not LOG.isEnabledFor(logging.DEBUG) or not DEBUG_TRIGGER_LOGS:
             return
 
         names = (

dlclivegui/config.py

@@ -25,11 +25,11 @@
 ## Debug
 ### Timing logs
 SINGLE_CAMERA_WORKER_DO_LOG_TIMING: bool = False
-MULTI_CAMERA_WORKER_DO_LOG_TIMING: bool = False
-REC_DO_LOG_TIMING: bool = False
+MULTI_CAMERA_WORKER_DO_LOG_TIMING: bool = True
+REC_DO_LOG_TIMING: bool = True
 # MAIN_WINDOW_DO_LOG_TIMING: bool = False
 #### Backends
-BASLER_DO_LOG_TIMING: bool = False
+BASLER_DO_LOG_TIMING: bool = True
 
 
 class CameraSettings(BaseModel):
@@ -515,6 +515,7 @@ class RecordingSettings(BaseModel):
     container: Literal["mp4", "avi", "mov"] = "mp4"
     codec: str = "libx264"
     crf: int = Field(default=23, ge=0, le=51)
+    fast_encoding: bool = False
 
     def output_path(self) -> Path:
         """Return the absolute output path for recordings."""
@@ -528,18 +529,47 @@ def output_path(self) -> Path:
             filename = name.with_suffix(f".{self.container}")
         return directory / filename
 
-    def writegear_options(self, fps: float) -> dict[str, Any]:
-        """Return compression parameters for WriteGear."""
+    def writegear_options(self, fps: float | None) -> dict[str, Any]:
+        """Return FFmpeg/WriteGear compression parameters.
+
+        The default settings prioritize compatibility and compression quality. If
+        ``fast_encoding`` is enabled, additional low-latency encoder options are
+        added for codecs that are known to support them.
+
+        Args:
+            fps: Desired input frame rate. If missing or non-positive, falls back
+                to 30 FPS.
+
+        Returns:
+            Dictionary of WriteGear/FFmpeg options.
+        """
+        try:
+            fps_value = float(fps or 0.0)
+        except Exception:
+            fps_value = 0.0
+        if fps_value <= 0.0:
+            fps_value = 30.0
 
-        fps_value = float(fps) if fps else 30.0
         codec_value = (self.codec or "libx264").strip() or "libx264"
         crf_value = int(self.crf) if self.crf is not None else 23
-        return {
+
+        opts: dict[str, Any] = {
             "-input_framerate": f"{fps_value:.6f}",
             "-vcodec": codec_value,
             "-crf": str(crf_value),
         }
 
+        if self.fast_encoding:
+            if codec_value in {"libx264", "libx265"}:
+                opts.update(
+                    {
+                        "-preset": "ultrafast",
+                        "-tune": "zerolatency",
+                    }
+                )
+
+        return opts
+
 
 class ApplicationSettings(BaseModel):
     # optional: add a semantic version for migrations

dlclivegui/gui/main_window.py

@@ -632,13 +632,29 @@ def _build_recording_group(self) -> QGroupBox:
 
         form.addRow(grid)
 
-        # Record with overlays
+        # Recording options
         self.record_with_overlays_checkbox = QCheckBox("Record video with overlays")
         self.record_with_overlays_checkbox.setToolTip(
             "Enable to include pose overlays in recorded video (keypoints & bounding boxes)"
         )
         self.record_with_overlays_checkbox.setChecked(False)
-        form.addRow(self.record_with_overlays_checkbox)
+
+        self.fast_encoding_checkbox = QCheckBox("Use faster encoding parameters")
+        self.fast_encoding_checkbox.setToolTip(
+            "Use faster FFmpeg parameters for supported codecs.\n"
+            "For libx264/libx265 this uses preset=ultrafast and tune=zerolatency.\n"
+            "This can improve recording throughput but may increase file size."
+        )
+        self.fast_encoding_checkbox.setChecked(False)
+
+        recording_options = QWidget()
+        recording_options_layout = QHBoxLayout(recording_options)
+        recording_options_layout.setContentsMargins(0, 0, 0, 0)
+        recording_options_layout.addWidget(self.record_with_overlays_checkbox)
+        recording_options_layout.addWidget(self.fast_encoding_checkbox)
+        recording_options_layout.addStretch(1)
+
+        form.addRow(recording_options)
 
         # Wrap recording buttons in a widget to prevent shifting
         recording_button_widget = QWidget()
@@ -771,6 +787,7 @@ def _connect_signals(self) -> None:
         # Multi-camera controller signals (used for both single and multi-camera modes)
         self.multi_camera_controller.frame_ready.connect(self._on_multi_frame_processing_ready)
         self.multi_camera_controller.display_ready.connect(self._on_multi_frame_display_ready)
+        self.multi_camera_controller.recording_frame_ready.connect(self._on_recording_frame_ready)
         self.multi_camera_controller.all_started.connect(self._on_multi_camera_started)
         self.multi_camera_controller.all_stopped.connect(self._on_multi_camera_stopped)
         self.multi_camera_controller.camera_error.connect(self._on_multi_camera_error)
@@ -821,6 +838,10 @@ def _apply_config(self, config: ApplicationSettings) -> None:
             self.codec_combo.addItem(recording.codec)
             self.codec_combo.setCurrentIndex(self.codec_combo.count() - 1)
         self.crf_spin.setValue(int(recording.crf))
+
+        if hasattr(self, "fast_encoding_checkbox"):
+            self.fast_encoding_checkbox.setChecked(bool(getattr(recording, "fast_encoding", False)))
+
         ## Restore persisted session name if empty
         if hasattr(self, "session_name_edit"):
             if not self.session_name_edit.text().strip():
@@ -931,6 +952,9 @@ def _recording_settings_from_ui(self) -> RecordingSettings:
             container=self.container_combo.currentText().strip() or "mp4",
             codec=self.codec_combo.currentText().strip() or "libx264",
             crf=int(self.crf_spin.value()),
+            fast_encoding=bool(
+                getattr(self, "fast_encoding_checkbox", None) and self.fast_encoding_checkbox.isChecked()
+            ),
         )
 
     def _bbox_settings_from_ui(self) -> BoundingBoxSettings:
@@ -1372,6 +1396,24 @@ def _render_overlays_for_recording(self, cam_id, frame):
             )
         return output
 
+    def _on_recording_frame_ready(self, camera_id: str, frame: np.ndarray, timestamp: float) -> None:
+        """Handle full-rate per-camera frames for recording only.
+
+        Intentionally lean:
+        - no MultiFrameData processing
+        - no DLC routing
+        - no display state updates
+        - no FPS tracker
+        - optional overlays only if user requested recording overlays
+        """
+        if not self._rec_manager.is_active:
+            return
+
+        if self.record_with_overlays_checkbox.isChecked():
+            frame = self._render_overlays_for_recording(camera_id, frame)
+
+        self._rec_manager.write_frame(camera_id, frame, timestamp)
+
     def _on_multi_frame_processing_ready(self, frame_data: MultiFrameData) -> None:
         """Handle frames from multiple cameras.
 
@@ -1425,15 +1467,15 @@ def _on_multi_frame_processing_ready(self, frame_data: MultiFrameData) -> None:
             self._dlc.enqueue_frame(frame, timestamp)
 
         # PRIORITY 2: Recording (queued, non-blocking)
-        if self._rec_manager.is_active and src_id in frame_data.frames:
-            frame = frame_data.frames[src_id]
+        # if self._rec_manager.is_active and src_id in frame_data.frames:
+        #     frame = frame_data.frames[src_id]
 
-            if self.record_with_overlays_checkbox.isChecked():
-                # Draw overlays for recording
-                frame = self._render_overlays_for_recording(src_id, frame)
+        #     if self.record_with_overlays_checkbox.isChecked():
+        #         # Draw overlays for recording
+        #         frame = self._render_overlays_for_recording(src_id, frame)
 
-            ts = frame_data.timestamps.get(src_id, time.time())
-            self._rec_manager.write_frame(src_id, frame, ts)
+        #     ts = frame_data.timestamps.get(src_id, time.time())
+        #     self._rec_manager.write_frame(src_id, frame, ts)
 
     def _on_multi_frame_display_ready(self, frame_data: MultiFrameData) -> None:
         """Throttled UI/display path.
@@ -1514,6 +1556,7 @@ def _start_multi_camera_recording(self) -> None:
         if run_dir is None:
             self._show_error("Failed to start recording.")
             return
+        self.multi_camera_controller.set_recording_frame_do_emit(True)
 
         self._settings_store.set_session_name(session_name)
         self.start_record_button.setEnabled(False)
@@ -1524,6 +1567,9 @@ def _start_multi_camera_recording(self) -> None:
     def _stop_multi_camera_recording(self) -> None:
         if not self._rec_manager.is_active:
             return
+
+        self.multi_camera_controller.set_recording_frame_do_emit(False)
+
         self._rec_manager.stop_all()
         self.start_record_button.setEnabled(True)
         self.stop_record_button.setEnabled(False)
@@ -1715,6 +1761,8 @@ def _update_camera_controls_enabled(self) -> None:
         recording_editable = not multi_cam_recording
         self.codec_combo.setEnabled(recording_editable)
         self.crf_spin.setEnabled(recording_editable)
+        if hasattr(self, "fast_encoding_checkbox"):
+            self.fast_encoding_checkbox.setEnabled(recording_editable)
 
         # Config cameras button should be available when not in preview/recording
         self.config_cameras_button.setEnabled(allow_changes)

dlclivegui/gui/recording_manager.py

@@ -148,15 +148,19 @@ def start_all(
             frame = current_frames.get(cam_id)
             frame_size = (frame.shape[0], frame.shape[1]) if frame is not None else None
             recorder_fps = self._resolve_recording_fps(cam, cam_id, frame_rates)
+            writer_options = recording.writegear_options(recorder_fps)
 
             log.debug(
-                "Starting recorder %s -> %s frame_size=%s requested_fps=%s detected_fps=%s recorder_fps=%s",
+                "Starting recorder %s -> %s frame_size=%s requested_fps=%s detected_fps=%s "
+                "recorder_fps=%s fast_encoding=%s writer_options=%s",
                 cam_id,
                 cam_path,
                 frame_size,
                 getattr(cam, "fps", None),
                 self._backend_ns(cam).get("detected_fps"),
                 f"{recorder_fps:.3f}" if recorder_fps else "auto/fallback",
+                bool(getattr(recording, "fast_encoding", False)),
+                writer_options,
             )
 
             recorder = VideoRecorder(
@@ -166,6 +170,7 @@ def start_all(
                 codec=recording.codec,
                 crf=recording.crf,
                 convert_grayscale_to_rgb=not bool(getattr(cam, "preserve_mono", False)),
+                writer_options=writer_options,
             )
             try:
                 recorder.start()
@@ -213,19 +218,27 @@ def write_frame(self, cam_id: str, frame: np.ndarray, timestamp: float | None =
 
     def get_stats_summary(self) -> str:
         totals = {
+            "enqueued": 0,
             "written": 0,
             "dropped": 0,
             "queue": 0,
+            "buffer": 0,
+            "backlog": 0,
+            "write_fps": 0.0,
             "max_latency": 0.0,
             "avg_latencies": [],
         }
         for rec in self._recorders.values():
             stats: RecorderStats | None = rec.get_stats()
             if not stats:
                 continue
+            totals["enqueued"] += stats.frames_enqueued
             totals["written"] += stats.frames_written
             totals["dropped"] += stats.dropped_frames
             totals["queue"] += stats.queue_size
+            totals["buffer"] += stats.buffer_size
+            totals["backlog"] += stats.backlog_frames
+            totals["write_fps"] += stats.write_fps
             totals["max_latency"] = max(totals["max_latency"], stats.last_latency)
             totals["avg_latencies"].append(stats.average_latency)
 
@@ -239,8 +252,16 @@ def get_stats_summary(self) -> str:
             return "Recording..."
         else:
             avg = sum(totals["avg_latencies"]) / len(totals["avg_latencies"]) if totals["avg_latencies"] else 0.0
+
+            buffer = totals["buffer"]
+            queue_text = f"{totals['queue']}/{buffer}" if buffer > 0 else str(totals["queue"])
+            fill_pct = (100.0 * totals["queue"] / buffer) if buffer > 0 else 0.0
+
             return (
-                f"{len(self._recorders)} cams | {totals['written']} frames | "
+                f"{len(self._recorders)} cams | {totals['written']}/{totals['enqueued']} frames | "
+                f"writer {totals['write_fps']:.1f} fps | "
                 f"latency {totals['max_latency'] * 1000:.1f}ms (avg {avg * 1000:.1f}ms) | "
-                f"queue {totals['queue']} | dropped {totals['dropped']}"
+                f"queue {queue_text} ({fill_pct:.0f}%) | "
+                f"backlog {totals['backlog']} | "
+                f"dropped {totals['dropped']}"
             )

dlclivegui/services/multi_camera_controller.py

@@ -297,7 +297,8 @@ class MultiCameraController(QObject):
     """Controller for managing multiple cameras simultaneously."""
 
     # Signals
-    frame_ready = Signal(object)  # MultiFrameData (full cam FPS; recording and inference only)
+    frame_ready = Signal(object)  # MultiFrameData (full cam FPS; inference only)
+    recording_frame_ready = Signal(str, object, float)  # camera_id, frame, timestamp (full cam FPS; for recording)
     display_ready = Signal(object)  # MultiFrameData for GUI display (throttled to GUI_MAX_DISPLAY_FPS)
     camera_started = Signal(str, object)  # camera_id, settings
     camera_stopped = Signal(str)  # camera_id
@@ -318,6 +319,7 @@ def __init__(self):
         self._timestamps: dict[str, float] = {}
         self._frame_lock = Lock()
         self._running = False
+        self._recording_frame_emission_enabled: bool = False
         self._started_cameras: set = set()
         self._camera_display_order: list[str] = []
         self._display_ids: dict[str, str] = {}  # camera_id -> display_id (for labeling)
@@ -350,6 +352,14 @@ def _timing_for_camera(self, camera_id: str) -> WorkerTimingStats:
             self._timing_per_cam[camera_id] = timing
         return timing
 
+    def set_recording_frame_do_emit(self, enabled: bool) -> None:
+        """Enable/disable the lightweight per-camera recording frame signal.
+
+        This avoids sending recording-only traffic when the user is only previewing
+        or running DLC.
+        """
+        self._recording_frame_emission_enabled = bool(enabled)
+
     def _should_emit_display_ready(self) -> bool:
         """Return True when the UI/display path should be updated.
 
@@ -416,6 +426,7 @@ def start(self, camera_settings: list[CameraSettings]) -> None:
             seen[key] = camera_id
 
         self._running = True
+        self._recording_frame_emission_enabled = False
         self._frames.clear()
         self._timestamps.clear()
         self._started_cameras.clear()
@@ -481,6 +492,7 @@ def stop(self, wait: bool = True) -> None:
             return
 
         self._running = False
+        self._recording_frame_emission_enabled = False
 
         # Signal all workers to stop
         for worker in self._workers.values():
@@ -573,6 +585,10 @@ def _on_frame_captured(self, camera_id: str, frame: np.ndarray, timestamp: float
                     if crop_region:
                         frame = MultiCameraController.apply_crop(frame, crop_region)
 
+            if self._recording_frame_emission_enabled:
+                with timing.measure("Multi.emit.recording_frame_ready"):
+                    self.recording_frame_ready.emit(camera_id, frame, timestamp)
+
             with self._frame_lock:
                 with timing.measure("Multi.store_latest"):
                     self._frames[camera_id] = frame