dodex-indexer-alerts.yaml

# Grafana-managed alert rules for the dodex indexer, provisioned (apiVersion 1).
# Grafana evaluates these itself by querying the Prometheus data source — they
# appear under Alerting → Alert rules and link to the Dodex Indexer dashboard.
#
# Install: copy to Grafana's provisioning path and restart / reload:
#   /etc/grafana/provisioning/alerting/dodex-indexer-alerts.yaml
#
# REQUIRED — set the Prometheus data source UID. Find it under
# Connections → Data sources (the edit URL ends with the uid), then:
#   sed -i 's/REPLACE_WITH_PROMETHEUS_DS_UID/<your-uid>/g' \
#       deploy/grafana/provisioning/alerting/dodex-indexer-alerts.yaml
#
# Each rule = a query node (refId A, instant) + a math condition node (refId C).
# Counter alerts use a __name__=~"...(_total)?" matcher so they fire whether or
# not the OTel→Prometheus exporter appends _total. Thresholds mirror the Grafana
# dashboard (deploy/grafana/dodex-indexer-dashboard.json). noDataState is OK so a
# fully-down indexer (series gone) does not flap these — pair with a separate
# `up == 0` liveness alert for that case.

apiVersion: 1

groups:
  - orgId: 1
    name: dodex-indexer-pipeline
    folder: Dodex Indexer
    interval: 1m
    rules:
      - uid: dodex-proj-lag-warn
        title: Indexer projection lag elevated
        condition: C
        for: 5m
        noDataState: OK
        execErrState: Error
        labels: { severity: warning, service: dodex-indexer }
        annotations:
          summary: "Indexer projection lag elevated"
          description: "Oldest unprojected raw_events row is {{ $values.A.Value }}s old on {{ $labels.instance }} (warning >30s for 5m). Read-model is falling behind capture."
        data:
          - refId: A
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: REPLACE_WITH_PROMETHEUS_DS_UID
            model:
              refId: A
              editorMode: code
              expr: indexer_projection_lag_seconds
              instant: true
              range: false
              intervalMs: 1000
              maxDataPoints: 43200
              datasource: { type: prometheus, uid: REPLACE_WITH_PROMETHEUS_DS_UID }
          - refId: C
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: __expr__
            model:
              refId: C
              type: math
              expression: "$A > 30"
              datasource: { type: __expr__, uid: __expr__ }

      - uid: dodex-proj-lag-crit
        title: Indexer projection lag critical
        condition: C
        for: 5m
        noDataState: OK
        execErrState: Error
        labels: { severity: critical, service: dodex-indexer }
        annotations:
          summary: "Indexer projection lag critical"
          description: "Oldest unprojected raw_events row is {{ $values.A.Value }}s old on {{ $labels.instance }} (critical >120s for 5m). Check the projection loop and DB pool saturation."
        data:
          - refId: A
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: REPLACE_WITH_PROMETHEUS_DS_UID
            model:
              refId: A
              editorMode: code
              expr: indexer_projection_lag_seconds
              instant: true
              range: false
              intervalMs: 1000
              maxDataPoints: 43200
              datasource: { type: prometheus, uid: REPLACE_WITH_PROMETHEUS_DS_UID }
          - refId: C
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: __expr__
            model:
              refId: C
              type: math
              expression: "$A > 120"
              datasource: { type: __expr__, uid: __expr__ }

      - uid: dodex-capture-cursor-warn
        title: Indexer capture cursor not advancing
        condition: C
        for: 5m
        noDataState: OK
        execErrState: Error
        labels: { severity: warning, service: dodex-indexer }
        annotations:
          summary: "Indexer capture cursor not advancing"
          description: "Capture cursor last advanced {{ $values.A.Value }}s ago on {{ $labels.instance }} (warning >120s for 5m). Capture may be stalled."
        data:
          - refId: A
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: REPLACE_WITH_PROMETHEUS_DS_UID
            model:
              refId: A
              editorMode: code
              expr: indexer_capture_cursor_age_seconds
              instant: true
              range: false
              intervalMs: 1000
              maxDataPoints: 43200
              datasource: { type: prometheus, uid: REPLACE_WITH_PROMETHEUS_DS_UID }
          - refId: C
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: __expr__
            model:
              refId: C
              type: math
              expression: "$A > 120"
              datasource: { type: __expr__, uid: __expr__ }

      - uid: dodex-capture-cursor-crit
        title: Indexer capture cursor stuck
        condition: C
        for: 5m
        noDataState: OK
        execErrState: Error
        labels: { severity: critical, service: dodex-indexer }
        annotations:
          summary: "Indexer capture cursor stuck"
          description: "Capture cursor last advanced {{ $values.A.Value }}s ago on {{ $labels.instance }} (critical >300s for 5m). New chain events are not being ingested."
        data:
          - refId: A
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: REPLACE_WITH_PROMETHEUS_DS_UID
            model:
              refId: A
              editorMode: code
              expr: indexer_capture_cursor_age_seconds
              instant: true
              range: false
              intervalMs: 1000
              maxDataPoints: 43200
              datasource: { type: prometheus, uid: REPLACE_WITH_PROMETHEUS_DS_UID }
          - refId: C
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: __expr__
            model:
              refId: C
              type: math
              expression: "$A > 300"
              datasource: { type: __expr__, uid: __expr__ }

      - uid: dodex-decode-errors-warn
        title: Indexer decode errors occurring
        condition: C
        for: 5m
        noDataState: OK
        execErrState: Error
        labels: { severity: warning, service: dodex-indexer }
        annotations:
          summary: "Indexer decode errors occurring"
          description: "{{ $values.A.Value }}/s event bodies failing to decode on {{ $labels.instance }} over 5m. Likely ABI drift or a malformed cell for a known event."
        data:
          - refId: A
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: REPLACE_WITH_PROMETHEUS_DS_UID
            model:
              refId: A
              editorMode: code
              expr: rate({__name__=~"indexer_decode_errors(_total)?"}[5m])
              instant: true
              range: false
              intervalMs: 1000
              maxDataPoints: 43200
              datasource: { type: prometheus, uid: REPLACE_WITH_PROMETHEUS_DS_UID }
          - refId: C
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: __expr__
            model:
              refId: C
              type: math
              expression: "$A > 0"
              datasource: { type: __expr__, uid: __expr__ }

      - uid: dodex-decode-errors-crit
        title: Indexer decode errors at high rate
        condition: C
        for: 5m
        noDataState: OK
        execErrState: Error
        labels: { severity: critical, service: dodex-indexer }
        annotations:
          summary: "Indexer decode errors at high rate"
          description: "{{ $values.A.Value }}/s event bodies failing to decode on {{ $labels.instance }} (critical >0.1/s for 5m). A known event's ABI is likely out of sync."
        data:
          - refId: A
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: REPLACE_WITH_PROMETHEUS_DS_UID
            model:
              refId: A
              editorMode: code
              expr: rate({__name__=~"indexer_decode_errors(_total)?"}[5m])
              instant: true
              range: false
              intervalMs: 1000
              maxDataPoints: 43200
              datasource: { type: prometheus, uid: REPLACE_WITH_PROMETHEUS_DS_UID }
          - refId: C
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: __expr__
            model:
              refId: C
              type: math
              expression: "$A > 0.1"
              datasource: { type: __expr__, uid: __expr__ }

      - uid: dodex-decode-ambiguous-collisions
        title: Indexer ambiguous decode collisions
        condition: C
        for: 5m
        noDataState: OK
        execErrState: Error
        labels: { severity: critical, service: dodex-indexer }
        annotations:
          summary: "Indexer ambiguous decode collisions"
          description: "{{ $values.A.Value }}/s event bodies left undecoded because their id collides across ABIs with no dst route, on {{ $labels.instance }}. This should be flat zero — a non-zero rate means a new colliding ABI was added without a decoder dst route, so those events are silently dropped. Add the route in decoder.rs."
        data:
          - refId: A
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: REPLACE_WITH_PROMETHEUS_DS_UID
            model:
              refId: A
              editorMode: code
              expr: rate({__name__=~"indexer_decode_ambiguous_collisions(_total)?"}[5m])
              instant: true
              range: false
              intervalMs: 1000
              maxDataPoints: 43200
              datasource: { type: prometheus, uid: REPLACE_WITH_PROMETHEUS_DS_UID }
          - refId: C
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: __expr__
            model:
              refId: C
              type: math
              expression: "$A > 0"
              datasource: { type: __expr__, uid: __expr__ }

      - uid: dodex-metrics-refresh-failures
        title: Indexer metrics-refresh loop failing
        condition: C
        for: 10m
        noDataState: OK
        execErrState: Error
        labels: { severity: warning, service: dodex-indexer }
        annotations:
          summary: "Indexer metrics-refresh loop failing"
          description: "The metrics-refresh loop is failing to read the DB ({{ $values.A.Value }}/s on {{ $labels.instance }}), so every gauge it feeds — projection backlog/lag, capture cursor age, inference markets/orders/lags, and wedged-books — is frozen at its last value rather than reflecting reality. A gauge alert may therefore be silently stale; treat this as loss of metrics visibility and check DB reachability. Pair with the inference wedged-books alert."
        data:
          - refId: A
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: REPLACE_WITH_PROMETHEUS_DS_UID
            model:
              refId: A
              editorMode: code
              expr: rate({__name__=~"indexer_metrics_refresh_failures(_total)?"}[5m])
              instant: true
              range: false
              intervalMs: 1000
              maxDataPoints: 43200
              datasource: { type: prometheus, uid: REPLACE_WITH_PROMETHEUS_DS_UID }
          - refId: C
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: __expr__
            model:
              refId: C
              type: math
              expression: "$A > 0"
              datasource: { type: __expr__, uid: __expr__ }

  - orgId: 1
    name: dodex-indexer-inference
    folder: Dodex Indexer
    interval: 1m
    rules:
      - uid: dodex-inf-failing-warn
        title: Inference markets stuck failing
        condition: C
        for: 15m
        noDataState: OK
        execErrState: Error
        labels: { severity: warning, service: dodex-indexer }
        annotations:
          summary: "Inference markets stuck failing"
          description: "{{ $values.A.Value }} inference market(s) invisible with a recorded reconcile failure on {{ $labels.instance }} for 15m (ABI drift, or never-deployed / wrong-dApp address). Not served by the API."
        data:
          - refId: A
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: REPLACE_WITH_PROMETHEUS_DS_UID
            model:
              refId: A
              editorMode: code
              expr: indexer_inference_markets{state="failing"}
              instant: true
              range: false
              intervalMs: 1000
              maxDataPoints: 43200
              datasource: { type: prometheus, uid: REPLACE_WITH_PROMETHEUS_DS_UID }
          - refId: C
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: __expr__
            model:
              refId: C
              type: math
              expression: "$A > 0"
              datasource: { type: __expr__, uid: __expr__ }

      - uid: dodex-inf-failing-crit
        title: Inference markets failing for over an hour
        condition: C
        for: 1h
        noDataState: OK
        execErrState: Error
        labels: { severity: critical, service: dodex-indexer }
        annotations:
          summary: "Inference markets failing for over an hour"
          description: "{{ $values.A.Value }} inference market(s) failing on {{ $labels.instance }} for 1h — reconciler backoff is not clearing them. Investigate the getter ABI / dApp deployment."
        data:
          - refId: A
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: REPLACE_WITH_PROMETHEUS_DS_UID
            model:
              refId: A
              editorMode: code
              expr: indexer_inference_markets{state="failing"}
              instant: true
              range: false
              intervalMs: 1000
              maxDataPoints: 43200
              datasource: { type: prometheus, uid: REPLACE_WITH_PROMETHEUS_DS_UID }
          - refId: C
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: __expr__
            model:
              refId: C
              type: math
              expression: "$A > 0"
              datasource: { type: __expr__, uid: __expr__ }

      - uid: dodex-inf-price-lag-warn
        title: Inference reference price stale
        condition: C
        for: 10m
        noDataState: OK
        execErrState: Error
        labels: { severity: warning, service: dodex-indexer }
        annotations:
          summary: "Inference reference price stale"
          description: "Most stale reference_price_at across visible markets is {{ $values.A.Value }}s old on {{ $labels.instance }} (warning >1h for 10m). Queue B price refresh is lagging."
        data:
          - refId: A
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: REPLACE_WITH_PROMETHEUS_DS_UID
            model:
              refId: A
              editorMode: code
              expr: indexer_inference_reference_price_lag_seconds
              instant: true
              range: false
              intervalMs: 1000
              maxDataPoints: 43200
              datasource: { type: prometheus, uid: REPLACE_WITH_PROMETHEUS_DS_UID }
          - refId: C
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: __expr__
            model:
              refId: C
              type: math
              expression: "$A > 3600"
              datasource: { type: __expr__, uid: __expr__ }

      - uid: dodex-inf-price-lag-crit
        title: Inference reference price very stale
        condition: C
        for: 10m
        noDataState: OK
        execErrState: Error
        labels: { severity: critical, service: dodex-indexer }
        annotations:
          summary: "Inference reference price very stale"
          description: "Most stale reference_price_at is {{ $values.A.Value }}s old on {{ $labels.instance }} (critical >24h for 10m). Served prices may be far out of date."
        data:
          - refId: A
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: REPLACE_WITH_PROMETHEUS_DS_UID
            model:
              refId: A
              editorMode: code
              expr: indexer_inference_reference_price_lag_seconds
              instant: true
              range: false
              intervalMs: 1000
              maxDataPoints: 43200
              datasource: { type: prometheus, uid: REPLACE_WITH_PROMETHEUS_DS_UID }
          - refId: C
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: __expr__
            model:
              refId: C
              type: math
              expression: "$A > 86400"
              datasource: { type: __expr__, uid: __expr__ }

      - uid: dodex-inf-sweep-lag-warn
        title: Inference order-book sweep lag elevated
        condition: C
        for: 10m
        noDataState: OK
        execErrState: Error
        labels: { severity: warning, service: dodex-indexer }
        annotations:
          summary: "Inference order-book sweep lag elevated"
          description: "Most stale last_swept_at across visible markets is {{ $values.A.Value }}s old on {{ $labels.instance }} (warning >5m for 10m). Order-book depth may be stale (phantom-liquidity risk)."
        data:
          - refId: A
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: REPLACE_WITH_PROMETHEUS_DS_UID
            model:
              refId: A
              editorMode: code
              expr: indexer_inference_sweep_lag_seconds
              instant: true
              range: false
              intervalMs: 1000
              maxDataPoints: 43200
              datasource: { type: prometheus, uid: REPLACE_WITH_PROMETHEUS_DS_UID }
          - refId: C
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: __expr__
            model:
              refId: C
              type: math
              expression: "$A > 300"
              datasource: { type: __expr__, uid: __expr__ }

      - uid: dodex-inf-sweep-lag-crit
        title: Inference order-book sweep lag critical
        condition: C
        for: 10m
        noDataState: OK
        execErrState: Error
        labels: { severity: critical, service: dodex-indexer }
        annotations:
          summary: "Inference order-book sweep lag critical"
          description: "Most stale last_swept_at is {{ $values.A.Value }}s old on {{ $labels.instance }} (critical >30m for 10m). Sweep is not revisiting OPEN rows."
        data:
          - refId: A
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: REPLACE_WITH_PROMETHEUS_DS_UID
            model:
              refId: A
              editorMode: code
              expr: indexer_inference_sweep_lag_seconds
              instant: true
              range: false
              intervalMs: 1000
              maxDataPoints: 43200
              datasource: { type: prometheus, uid: REPLACE_WITH_PROMETHEUS_DS_UID }
          - refId: C
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: __expr__
            model:
              refId: C
              type: math
              expression: "$A > 1800"
              datasource: { type: __expr__, uid: __expr__ }

      # Guard for the zero-visible blind spot: the reference-price and sweep lag
      # gauges read `now() - min(ts)` over VISIBLE markets and report 0 when none
      # are visible, so a stuck reconciler (e.g. the missing-CA-certs incident
      # class) that leaves every market in discovering/failing silently suppresses
      # both lag alerts above. This fires instead when markets exist but none are
      # API-visible. The `and ... > 0` clause keeps it quiet on a fresh deploy
      # before any market is discovered (noDataState OK when no series exist).
      - uid: dodex-inf-none-visible
        title: Inference markets discovered but none visible
        condition: C
        for: 15m
        noDataState: OK
        execErrState: Error
        labels: { severity: warning, service: dodex-indexer }
        annotations:
          summary: "Inference markets discovered but none visible"
          description: "Inference markets exist on {{ $labels.instance }} but none reached API-visible for 15m — the reconciler is likely stuck (the lag gauges read 0 in this state and their alerts cannot fire). Check reconciler health (getter ABI, TLS trust store, dApp deployment)."
        data:
          - refId: A
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: REPLACE_WITH_PROMETHEUS_DS_UID
            model:
              refId: A
              editorMode: code
              expr: (sum(indexer_inference_markets{state="visible"}) == bool 0) and (sum(indexer_inference_markets) > 0)
              instant: true
              range: false
              intervalMs: 1000
              maxDataPoints: 43200
              datasource: { type: prometheus, uid: REPLACE_WITH_PROMETHEUS_DS_UID }
          - refId: C
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: __expr__
            model:
              refId: C
              type: math
              expression: "$A > 0"
              datasource: { type: __expr__, uid: __expr__ }

      - uid: dodex-inf-wedged-warn
        title: Inference books wedged (fail-closed 503)
        condition: C
        for: 15m
        noDataState: OK
        execErrState: Error
        labels: { severity: warning, service: dodex-indexer }
        annotations:
          summary: "Inference books wedged (fail-closed 503)"
          description: "{{ $values.A.Value }} visible inference book(s) have an unprojected raw_events row, so the read gate returns MarketInconsistent (503) for their tokenContract live-SELL lookups (note=, side=BUY and non-LIVE reads still succeed). Sustained >15m usually means a deployed contract upgrade outran the indexer's ABI. NOTE: this gauge freezes when the metrics-refresh loop cannot read the DB — correlate with 'Indexer metrics-refresh loop failing' before trusting a 0."
        data:
          - refId: A
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: REPLACE_WITH_PROMETHEUS_DS_UID
            model:
              refId: A
              editorMode: code
              expr: max(indexer_inference_wedged_books)
              instant: true
              range: false
              intervalMs: 1000
              maxDataPoints: 43200
              datasource: { type: prometheus, uid: REPLACE_WITH_PROMETHEUS_DS_UID }
          - refId: C
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: __expr__
            model:
              refId: C
              type: math
              expression: "$A > 0"
              datasource: { type: __expr__, uid: __expr__ }

      - uid: dodex-inf-wedged-crit
        title: Inference books wedged for over an hour
        condition: C
        for: 1h
        noDataState: OK
        execErrState: Error
        labels: { severity: critical, service: dodex-indexer }
        annotations:
          summary: "Inference books wedged for over an hour"
          description: "{{ $values.A.Value }} inference book(s) have been fail-closed (503 on tokenContract live-SELL lookups) for over an hour — the unprojected events are not clearing on their own, which for an order book means a stale ABI. Sync the indexer ABI to the deployed contracts and redeploy (stop → wipe → deploy, per the runbook)."
        data:
          - refId: A
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: REPLACE_WITH_PROMETHEUS_DS_UID
            model:
              refId: A
              editorMode: code
              expr: max(indexer_inference_wedged_books)
              instant: true
              range: false
              intervalMs: 1000
              maxDataPoints: 43200
              datasource: { type: prometheus, uid: REPLACE_WITH_PROMETHEUS_DS_UID }
          - refId: C
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: __expr__
            model:
              refId: C
              type: math
              expression: "$A > 0"
              datasource: { type: __expr__, uid: __expr__ }