Add logrotate to services (#137)

* Add logrotate to services

* Disable logrotate service

---------

Co-authored-by: Ekaterina Pantaz <katyapantaz@gmail.com>

Author: aferno

.woodpecker/e2e.yml

@@ -1071,6 +1071,7 @@ steps:
         -e dexdo_image_tag=$CI_COMMIT_SHA
         -e dexdo_seed_accounts=true
         -e dexdo_seed_notes_src="$CI_WORKSPACE/$SEED_NOTES_WS"
+        -e dexdo_logrotate_enabled=false
 
   assert_e2e_tests:
     when:

deploy/ansible/README.md

@@ -37,8 +37,40 @@ roles/dexdo/
     compose.yml.j2                image-based compose (pulls from registry) -> <deploy_dir>/compose.yml
     api.yaml.j2                   rendered to <deploy_dir>/config/api.<env>.yaml
     indexer.yaml.j2               rendered to <deploy_dir>/config/indexer.<env>.yaml
+    logrotate.j2                  rotation script for the logrotate sidecar -> <deploy_dir>/logrotate.sh
 ```
 
+## Logging
+
+The api + indexer write their logs to files under `dexdo_logs_dir/<service>/`
+(bind-mounted to `/app/logs`). That dir defaults to `<deploy_dir>/logs`; point it
+at a separate disk/mount per env to keep logs off the system disk or surviving a
+redeploy. The services themselves
+rotate **daily** (`<service>.log.<date>` + `<service>.noise.log.<date>`
+via the `dodex-logging` crate, pruned to `LOG_MAX_FILES` days), so rotation is
+handled out of the box.
+
+Daily rotation bounds age but not size. If you also need a **size** cap, the
+role ships an **optional `logrotate` sidecar** (off by default —
+`dexdo_logrotate_enabled`) that rotates the dated files by size with
+`copytruncate` (mandatory, since the services keep the files open), driven by
+busybox cron, in place; the app keeps appending to the current day's file.
+
+Knobs (in `roles/dexdo/defaults/main.yml`, overridable per env):
+
+- `dexdo_logs_dir` (default `<deploy_dir>/logs`) — host dir the logs are written
+  to; point it at a data mount to keep logs off the system disk.
+- `dexdo_logrotate_enabled` (default `false`) — the app already rotates daily;
+  turn on only when you also want a size cap on top.
+- `dexdo_logrotate_image` — image providing `logrotate` + `crond` + bash.
+- `dexdo_log_rotate_size` (default `2G`) / `dexdo_log_rotate_amount`
+  (default `10`) — rotate at this size, keep this many compressed copies.
+- `dexdo_log_rotate_spec` (default `"*/5 *"`) — cron "minute hour" for the run
+  (every 5 minutes).
+
+For the app-side `LOG_DIR` / `LOG_MAX_FILES` knobs see
+[docs/deployment.md](../../docs/deployment.md#logs).
+
 ## Environments
 
 Each environment is its own directory under `inventories/` — an inventory

deploy/ansible/roles/dexdo/defaults/main.yml

@@ -20,6 +20,11 @@ dexdo_image_tag: latest-dev
 
 # --- Host layout ------------------------------------------------------------
 dexdo_deploy_dir: /opt/dexdo
+# Where the services write their log files. Defaults under the deploy dir; break
+# it out to its own disk/mount per environment when you want logs off the system
+# disk or surviving a redeploy. Per-service subdirs `<dir>/api` and
+# `<dir>/indexer` are bind-mounted to /app/logs.
+dexdo_logs_dir: "{{ dexdo_deploy_dir }}/logs"
 # Run api + indexer with the host's network stack (network_mode: host) instead
 # of the compose bridge. Off by default. Turn on ONLY when the chain must be
 # reached over the host's 127.0.0.1 — e2e, where a co-located single-compose
@@ -89,5 +94,24 @@ dexdo_otel_endpoint: ""
 dexdo_otel_metrics_protocol: grpc
 dexdo_otel_service_name: dodex-indexer-test
 
+# --- Logging & rotation -----------------------------------------------------
+# The api + indexer already write per-service, daily-rotated files under
+# `dexdo_logs_dir/<service>` (bind-mounted to /app/logs) via the dodex-logging
+# crate: <service>.log.<date> (+ .noise.log), pruned to LOG_MAX_FILES days — so
+# rotation is already handled and the logrotate sidecar below is OFF by default.
+# Optional logrotate sidecar: a tiny container that adds a SIZE cap on top of the
+# app's daily rotation (rotate by size, copytruncate since the services keep the
+# files open). Turn on per env only if you also need a size ceiling.
+dexdo_logrotate_enabled: false
+# Image providing `logrotate` + busybox `crond` + bash.
+dexdo_logrotate_image: docker.gosh.sh/logrotate:3.13.0
+# Rotate a log file once it reaches this size; keep this many rotated copies.
+dexdo_log_rotate_size: 2G
+dexdo_log_rotate_amount: 10
+# BusyBox crontab "minute hour" fields for the rotation run (the remaining
+# day/month/weekday fields are wildcarded). "*/5 *" -> every 5 minutes, so a
+# fast-growing log can't overshoot the size cap by much between checks.
+dexdo_log_rotate_spec: "*/5 *"
+
 # --- Secrets (resolved from the env vault) ----------------------------------
 dexdo_kek_hex: "{{ vault_dexdo_kek_hex }}"

deploy/ansible/roles/dexdo/tasks/main.yml

@@ -17,8 +17,19 @@
   loop:
     - "{{ dexdo_deploy_dir }}"
     - "{{ dexdo_deploy_dir }}/config"
-    - "{{ dexdo_deploy_dir }}/logs/api"
-    - "{{ dexdo_deploy_dir }}/logs/indexer"
+
+# Logs live outside the deploy dir (see dexdo_logs_dir) so they can sit on their
+# own mount and survive a redeploy — one per-service subdir, bind-mounted to
+# /app/logs in each container.
+- name: Ensure log directories exist
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    mode: "0750"
+  loop:
+    - "{{ dexdo_logs_dir }}"
+    - "{{ dexdo_logs_dir }}/api"
+    - "{{ dexdo_logs_dir }}/indexer"
 
 - name: Render api config
   ansible.builtin.template:
@@ -58,6 +69,17 @@
     dest: "{{ dexdo_deploy_dir }}/compose.yml"
     mode: "0644"
 
+# Rotation script for the logrotate sidecar (bind-mounted into it by the compose
+# file). Bind-mounts don't trigger a container recreate, so a script-only change
+# is applied by the explicit restart below rather than by `compose up`.
+- name: Render logrotate script (for the logrotate sidecar)
+  ansible.builtin.template:
+    src: logrotate.j2
+    dest: "{{ dexdo_deploy_dir }}/logrotate.sh"
+    mode: "0755"
+  when: dexdo_logrotate_enabled | bool
+  register: dexdo_logrotate_script
+
 # Pulls the SHA-pinned images from the registry and (re)creates containers.
 # `--pull always` picks up a new tag; config-only changes are applied by the
 # "Restart dexdo stack" handler (bind-mounted YAML does not trigger a recreate).
@@ -67,3 +89,13 @@
     chdir: "{{ dexdo_deploy_dir }}"
   register: dexdo_compose_up
   changed_when: "'Started' in dexdo_compose_up.stderr or 'Recreated' in dexdo_compose_up.stderr or 'Created' in dexdo_compose_up.stderr or 'Pulled' in dexdo_compose_up.stderr"
+
+# `compose up` above recreates the sidecar only when the compose file changed;
+# a change to logrotate.sh alone (bind-mounted) needs an explicit restart so the
+# running container re-reads it.
+- name: Restart logrotate sidecar when its script changed
+  ansible.builtin.command:
+    cmd: docker compose restart logrotate
+    chdir: "{{ dexdo_deploy_dir }}"
+  when: dexdo_logrotate_enabled | bool and dexdo_logrotate_script is changed
+  changed_when: true

deploy/ansible/roles/dexdo/templates/compose.yml.j2

@@ -71,7 +71,7 @@ services:
 {% endif %}
     volumes:
       - ./config:/app/config:ro
-      - ./logs/api:/app/logs
+      - {{ dexdo_logs_dir }}/api:/app/logs
     restart: unless-stopped
     healthcheck:
       test: ["CMD", "curl", "-fsS", "http://127.0.0.1:{{ dexdo_api_port }}/readiness"]
@@ -100,5 +100,17 @@ services:
       LOG_DIR: /app/logs
     volumes:
       - ./config:/app/config:ro
-      - ./logs/indexer:/app/logs
+      - {{ dexdo_logs_dir }}/indexer:/app/logs
     restart: unless-stopped
+{% if dexdo_logrotate_enabled %}
+
+  logrotate:
+    image: {{ dexdo_logrotate_image }}
+    restart: unless-stopped
+    volumes:
+      - {{ dexdo_logs_dir }}:/logsdir
+      - ./logrotate.sh:/logrotate.sh:ro
+    command: ["/bin/bash", "/logrotate.sh"]
+    init: true
+    network_mode: "none"
+{% endif %}

deploy/ansible/roles/dexdo/templates/logrotate.j2

@@ -0,0 +1,42 @@
+#!/bin/bash
+# Rendered by Ansible (deploy/ansible/roles/dexdo/templates/logrotate.j2). Do NOT
+# edit on the host — it is overwritten on the next deploy; change the template.
+
+set -e
+
+# Guard: this rotates /etc/logrotate.d system-wide, so only ever run in a
+# container.
+if [[ -f /.dockerenv ]] || grep -Eq '(lxc|docker)' /proc/1/cgroup; then
+    :
+else
+    echo "Docker only!"
+    exit 1
+fi
+
+rm -f /etc/logrotate.d/*
+# Match the app's dated live files (<service>.log.<YYYY-MM-DD>, dodex-logging's
+# daily rotation) ONLY — the date is anchored so this does NOT also match
+# logrotate's own rotated output (<name>.1, .2.gz, ...), which would otherwise
+# be re-rotated on every run and defeat the retention count.
+cat >/etc/logrotate.d/dexdo <<EOF
+/logsdir/*/*.log.[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] {
+    missingok
+    rotate {{ dexdo_log_rotate_amount }}
+    size {{ dexdo_log_rotate_size }}
+    notifempty
+    copytruncate
+    compress
+    delaycompress
+}
+EOF
+
+cat /etc/logrotate.d/dexdo
+
+chown 0:0 /etc/logrotate.d /etc/logrotate.d/dexdo
+chmod 755 /etc/logrotate.d
+chmod 644 /etc/logrotate.d/dexdo
+
+echo "{{ dexdo_log_rotate_spec }} * * * /usr/sbin/logrotate -v /etc/logrotate.conf" > /etc/crontabs/root
+
+echo "Starting crond"
+crond -f