chore: release 0.6.7 - Latest Observation polish and recording resilience

Author: Suncuss

.gitignore

@@ -38,6 +38,9 @@ AGENTS.md
 # Ignore claude code settings
 .claude/
 
+# Ignore codex settings
+.codex/
+
 # Ignore VS Code settings
 .vscode/
 

CHANGELOG.md

@@ -2,8 +2,15 @@
 
 ## [Unreleased]
 
+## [0.6.7] - 2026-05-03
+
 - Fixed audio recordings getting stuck in an infinite reprocess loop when post-analysis steps (audio extraction, spectrogram generation, BirdWeather upload, etc.) raised — the same WAV would be re-collected on the next scan and retried forever, spamming logs with duplicate "Bird detected" lines and amplifying the FD pressure that caused the failure. The processing loop now isolates per-file and per-detection failures and always removes the WAV in a `finally` block (#46)
 - Added file descriptor exhaustion diagnostics — when an EMFILE/ENFILE error is detected anywhere in the recording or processing pipeline, the next log line dumps the FD limit, current FD count, a sample of open FDs with their `/proc/self/fd` targets, and active child PIDs (catches zombie ffmpeg/sox processes holding pipes via leaked refs). Rate-limited per stage. Errno detection walks `__cause__`/`__context__`/`args` plus urllib3-style `.reason`, so EMFILE buried inside a `requests.ConnectionError(MaxRetryError(...))` chain is caught
+- Added recency-based recording protection — the latest N recordings per species are now retained alongside the existing top-N-by-confidence rule via union, so frequent recent activity isn't lost when its confidences fall below the top-N cut
+- Hid the Bird Activity Overview reverse toggle on days with no detections, since reversing an empty list is a no-op
+- Replaced the camera icon on bird detail pages with an upload arrow that animates open on hover to reveal "Upload custom image"
+- Redesigned the dashboard's Latest Observation card — bird image and text stack now sit side-by-side instead of vertically; the scientific name is shown again as a link to the bird's detail page, and the timestamp and confidence link to the Table view; long species names wrap to a second line instead of being clipped mid-word; a thin vertical divider separates the bird identity area from the spectrogram, which sits visually centered between the divider and the card edge
+- Polished the Latest Observation live spectrogram rendering — palette now matches the static spectrogram's `Greens_r` style (light background, dark green peaks), the canvas renders at 2× internal resolution for sharper downsampled output, and each new column is drawn as a horizontal gradient from the previous frame's intensities so time-axis transitions are continuous rather than stepped
 
 ## [0.6.6] - 2026-04-19
 

CLAUDE.md

@@ -67,7 +67,7 @@ backend/
 deployment/      # Runtime service scripts, Icecast streaming
 scripts/         # lint.sh, install-tests/ (BATS)
 docs/            # ARCHITECTURE, INSTALLATION, PRIVACY
-internal_docs/   # Planning notes (workflow, reviews, design docs)
+internal_docs/   # Planning notes (workflow, reviews, design docs) — gitignored, do not commit
 ```
 
 `AGENTS.md` is a symlink to this file.
@@ -97,3 +97,7 @@ Three test suites cover backend, frontend, and install scripts. Each has its own
 ## Branch Sync Rules
 
 Before syncing dev to staging or main, always run `./build.sh` first and confirm it passes.
+
+## Git Hygiene
+
+Never bypass `.gitignore` (no `git add -f`, no explicit-path force-staging) without explicit user approval. If a file genuinely needs to be in the repo, change the ignore rule instead of overriding it for one file — silently force-adding leaves a tracked file behind that the ignore pattern can't clean up later.

backend/config/settings.py

@@ -48,6 +48,7 @@ def _get_positive_int_env(name: str, default: int) -> int:
         "trigger_percent": 85,
         "target_percent": 80,
         "keep_per_species": 60,
+        "keep_recent_per_species": 16,
         "check_interval_minutes": 30
     },
     "updates": {

backend/core/db.py

@@ -843,24 +843,24 @@ def get_species_counts(self):
 
         return {row['common_name']: row['count'] for row in results}
 
-    def get_cleanup_candidates(self, keep_per_species=60, limit=None):
+    def get_cleanup_candidates(self, keep_per_species=60, keep_recent_per_species=16, limit=None):
         """Get detections eligible for cleanup, oldest first.
 
-        For each species, keeps the top N recordings by confidence.
-        Only returns recordings beyond the top N for each species.
+        For each species, protects the union of two sets:
+        - Top N by confidence (keep_per_species)
+        - Most recent N by timestamp (keep_recent_per_species)
+        A recording is a candidate only if it falls outside both sets.
 
         Args:
-            keep_per_species: Number of top recordings to keep per species (by confidence)
+            keep_per_species: Top recordings to keep per species by confidence
+            keep_recent_per_species: Most recent recordings to keep per species
             limit: Optional max number of records to return
 
         Returns:
             List of dicts with: id, common_name, confidence, timestamp,
                 audio_source, extra (raw JSON string)
             Ordered by timestamp ASC (oldest first)
         """
-        # Use window function to rank recordings within each species by confidence
-        # Only return recordings ranked beyond keep_per_species
-        # LIMIT is parameterized using -1 for unlimited (SQLite treats negative LIMIT as no limit)
         query = """
         WITH RankedDetections AS (
             SELECT
@@ -873,12 +873,16 @@ def get_cleanup_candidates(self, keep_per_species=60, limit=None):
                 ROW_NUMBER() OVER (
                     PARTITION BY common_name
                     ORDER BY confidence DESC
-                ) as confidence_rank
+                ) as confidence_rank,
+                ROW_NUMBER() OVER (
+                    PARTITION BY common_name
+                    ORDER BY timestamp DESC
+                ) as recency_rank
             FROM detections
         )
         SELECT id, common_name, confidence, timestamp, audio_source, extra
         FROM RankedDetections
-        WHERE confidence_rank > ?
+        WHERE confidence_rank > ? AND recency_rank > ?
         ORDER BY timestamp ASC
         LIMIT ?
         """
@@ -888,13 +892,14 @@ def get_cleanup_candidates(self, keep_per_species=60, limit=None):
 
         with self.get_db_connection() as conn:
             cur = conn.cursor()
-            cur.execute(query, (keep_per_species, limit_param))
+            cur.execute(query, (keep_per_species, keep_recent_per_species, limit_param))
             results = cur.fetchall()
 
         candidates = [dict(row) for row in results]
 
         logger.debug("Cleanup candidates retrieved", extra={
             'keep_per_species': keep_per_species,
+            'keep_recent_per_species': keep_recent_per_species,
             'candidates_count': len(candidates)
         })
 

backend/core/storage_manager.py

@@ -7,7 +7,8 @@
 Key features:
 - Triggers cleanup when disk usage exceeds trigger_percent (default: 80%)
 - Frees space until usage drops to target_percent (default: 70%)
-- Protects species with fewer than min_recordings_per_species (default: 60)
+- Protects the top N recordings per species by confidence (default: 60)
+- Protects the latest N recordings per species by timestamp (default: 16)
 - Deletes oldest files first
 - Preserves all database records
 """
@@ -39,6 +40,7 @@ def _get_storage_config() -> dict:
         'trigger_percent': storage.get('trigger_percent', 85),
         'target_percent': storage.get('target_percent', 80),
         'keep_per_species': storage.get('keep_per_species', 60),
+        'keep_recent_per_species': storage.get('keep_recent_per_species', 16),
         'check_interval_minutes': storage.get('check_interval_minutes', 1440),
     }
 
@@ -189,32 +191,38 @@ def delete_detection_files(detection):
     return result
 
 
-def estimate_deletable_size(db_manager, keep_per_species=None):
+def estimate_deletable_size(db_manager, keep_per_species=None, keep_recent_per_species=None):
     """Estimate total size of files that could be deleted.
 
     Args:
         db_manager: DatabaseManager instance
-        keep_per_species: Number of recordings to keep per species
+        keep_per_species: Top recordings to keep per species by confidence
+        keep_recent_per_species: Most recent recordings to keep per species
 
     Returns:
         Tuple of (estimated_bytes, candidate_count)
     """
+    config = _get_storage_config()
     if keep_per_species is None:
-        keep_per_species = _get_storage_config()['keep_per_species']
+        keep_per_species = config['keep_per_species']
+    if keep_recent_per_species is None:
+        keep_recent_per_species = config['keep_recent_per_species']
 
-    # Get candidates (without limit to count all)
-    candidates = db_manager.get_cleanup_candidates(keep_per_species=keep_per_species)
+    candidates = db_manager.get_cleanup_candidates(
+        keep_per_species=keep_per_species,
+        keep_recent_per_species=keep_recent_per_species,
+    )
 
     estimated_bytes = len(candidates) * ESTIMATED_SIZE_PER_DETECTION
     return estimated_bytes, len(candidates)
 
 
-def cleanup_storage(db_manager, target_percent=None, keep_per_species=None):
+def cleanup_storage(db_manager, target_percent=None, keep_per_species=None, keep_recent_per_species=None):
     """Run storage cleanup to free disk space.
 
     Deletes oldest audio and spectrogram files until disk usage drops
-    below target_percent. For each species, keeps the top N recordings
-    by confidence (protecting best recordings).
+    below target_percent. For each species, protects the union of the
+    top N by confidence and the latest N by timestamp.
 
     SAFETY: Will not delete files if the target is unachievable with
     available BirdNET data. This prevents mass deletion when disk is
@@ -223,7 +231,8 @@ def cleanup_storage(db_manager, target_percent=None, keep_per_species=None):
     Args:
         db_manager: DatabaseManager instance
         target_percent: Target disk usage percentage (default from settings)
-        keep_per_species: Recordings to keep per species (default from settings)
+        keep_per_species: Top recordings per species by confidence (default from settings)
+        keep_recent_per_species: Latest recordings per species (default from settings)
 
     Returns:
         dict with files_deleted, bytes_freed, target_achievable, etc.
@@ -233,6 +242,8 @@ def cleanup_storage(db_manager, target_percent=None, keep_per_species=None):
         target_percent = config['target_percent']
     if keep_per_species is None:
         keep_per_species = config['keep_per_species']
+    if keep_recent_per_species is None:
+        keep_recent_per_species = config['keep_recent_per_species']
 
     result = {
         'files_deleted': 0,
@@ -257,8 +268,10 @@ def cleanup_storage(db_manager, target_percent=None, keep_per_species=None):
     # Calculate how much we need to free
     bytes_to_free = usage['used_bytes'] - (usage['total_bytes'] * target_percent / 100)
 
-    # Fetch cleanup candidates once (oldest first, beyond top N per species)
-    candidates = db_manager.get_cleanup_candidates(keep_per_species=keep_per_species)
+    candidates = db_manager.get_cleanup_candidates(
+        keep_per_species=keep_per_species,
+        keep_recent_per_species=keep_recent_per_species,
+    )
     candidate_count = len(candidates)
 
     # SAFETY CHECK: Estimate if we can actually reach the target
@@ -281,12 +294,14 @@ def cleanup_storage(db_manager, target_percent=None, keep_per_species=None):
         'target_percent': target_percent,
         'bytes_to_free_gb': round(bytes_to_free / (1024**3), 2),
         'candidate_count': candidate_count,
-        'keep_per_species': keep_per_species
+        'keep_per_species': keep_per_species,
+        'keep_recent_per_species': keep_recent_per_species
     })
 
     if not candidates:
-        logger.info("No cleanup candidates found - all recordings within keep limit", extra={
-            'keep_per_species': keep_per_species
+        logger.info("No cleanup candidates found - all recordings within keep limits", extra={
+            'keep_per_species': keep_per_species,
+            'keep_recent_per_species': keep_recent_per_species
         })
         return result
 
@@ -369,7 +384,8 @@ def storage_monitor_loop(stop_flag, db_manager):
                     cleanup_storage(
                         db_manager,
                         target_percent=config['target_percent'],
-                        keep_per_species=config['keep_per_species']
+                        keep_per_species=config['keep_per_species'],
+                        keep_recent_per_species=config['keep_recent_per_species']
                     )
 
         except Exception as e: