-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathfm_asserts.py
More file actions
197 lines (159 loc) · 6.66 KB
/
Copy pathfm_asserts.py
File metadata and controls
197 lines (159 loc) · 6.66 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
"""Shared FM assertion helpers.
Each function proves one failure-mode guarantee. Named so the call site
reads like a test: assert_fm3_poison_bounded_at_dlq(attempts, ...).
Each run_pipeline() ends with a cumulative block — FM-6's runner calls
assert_fm1 through assert_fm6 — making it obvious that every prior fix
still holds.
Assertions that take Result objects accept typed Result[NotifyPayload]
instances reconstructed from Celery's JSON result via Result.from_dict().
Integer/counter arguments are still plain ints (read from Redis before
calling).
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from typing_extensions import TypeIs
from shared.result import NotifyPayload, Result
if TYPE_CHECKING:
from shared.result import SuccessResult
# ---------------------------------------------------------------------------
# FM-1
# ---------------------------------------------------------------------------
def assert_fm1_chord_body_fired(
result: Result[NotifyPayload],
) -> "TypeIs[SuccessResult[NotifyPayload]]":
"""FM-1: chord body fired and notify ran despite header task failures.
Returns a TypeGuard so callers can narrow result.payload to NotifyPayload:
assert assert_fm1_chord_body_fired(notify_result)
notify_result.payload.pipeline_id # NotifyPayload, not None
"""
assert result.status == "SUCCESS", (
f"FM-1 regression: chord body never fired — status={result.status!r}, "
f"error={result.error!r}"
)
assert result.payload is not None, "FM-1 regression: notify payload is None"
assert (
result.payload.final is True
), "FM-1 regression: notify payload.final is not True"
return True
# ---------------------------------------------------------------------------
# FM-2
# ---------------------------------------------------------------------------
def assert_fm2_redelivery_happened(
crash_attempts: int,
control_attempts: int,
*,
expected_attempts: int = 2,
) -> None:
"""FM-2: acks_late + reject_on_worker_lost caused exactly one redelivery.
crash_attempts — parse_document entries for the CRASH_ONCE doc (doc3).
control_attempts — parse_document entries for a happy-path doc (doc2).
CRASH_ONCE guarantees exactly 2 attempts (crash + redelivery) in all FM files.
"""
assert crash_attempts == expected_attempts, (
f"FM-2 regression: crash doc should have run exactly {expected_attempts} times "
f"(crash + redelivery); got {crash_attempts}"
)
assert (
control_attempts == 1
), f"FM-2 regression: control doc should have run once; got {control_attempts}"
# ---------------------------------------------------------------------------
# FM-3
# ---------------------------------------------------------------------------
def assert_fm3_poison_bounded_at_dlq(
poison_attempts: int,
*,
delivery_limit: int,
) -> None:
"""FM-3: poison message crash loop was capped by x-delivery-limit.
poison_attempts — parse_document entries for the POISON doc (doc4).
±1 tolerance: x-delivery-count inclusive/exclusive semantics vary
slightly between RabbitMQ versions.
"""
assert delivery_limit <= poison_attempts <= delivery_limit + 1, (
f"FM-3 regression: poison doc should have crashed ~{delivery_limit}x "
f"before DLQ; got {poison_attempts}"
)
# ---------------------------------------------------------------------------
# FM-4
# ---------------------------------------------------------------------------
def assert_fm4_notify_idempotent(
first: Result[NotifyPayload],
second: Result[NotifyPayload],
pipeline_id: str,
sends: int,
contention: int,
) -> None:
"""FM-4: exactly one send_email across duplicate notify fires; busy-retry exercised."""
assert (
first.status == "SUCCESS"
), f"FM-4 regression: chord notify result status={first.status!r}"
assert first.payload is not None, "FM-4 regression: chord notify payload is None"
assert (
first.payload.sent is True
), "FM-4 regression: chord notify should have sent the email"
assert (
second.payload is not None
), "FM-4 regression: duplicate notify payload is None"
assert (
second.payload.sent is False
), "FM-4 regression: duplicate notify should have skipped send_email"
assert (
first.payload.pipeline_id == pipeline_id
), f"FM-4 regression: wrong pipeline_id: {first.payload.pipeline_id!r}"
assert (
sends == 1
), f"FM-4 regression: send_email should run exactly once; got {sends}"
assert (
contention >= 1
), f"FM-4 regression: expected ≥1 lock-contention retry; got {contention}"
# ---------------------------------------------------------------------------
# FM-5
# ---------------------------------------------------------------------------
def assert_fm5_retryable_result(
result: Result[NotifyPayload],
*,
expected_ok: int,
expected_failed: int,
) -> None:
"""FM-5: chord aggregated the expected ok/failed header-result counts."""
assert result.payload is not None, "FM-5 regression: notify payload is None"
assert (
result.payload.ok == expected_ok
), f"FM-5 regression: expected {expected_ok} ok; got {result.payload.ok}"
assert (
result.payload.failed == expected_failed
), f"FM-5 regression: expected {expected_failed} failed; got {result.payload.failed}"
def assert_fm5_doc_attempts(
doc_id: str,
actual: int,
expected: int,
*,
is_poison: bool = False,
) -> None:
"""FM-5: parse_document was entered the predicted number of times per doc."""
if is_poison:
assert (
expected <= actual <= expected + 1
), f"FM-5 regression: {doc_id} expected ~{expected} crashes; got {actual}"
else:
assert (
actual == expected
), f"FM-5 regression: {doc_id} expected {expected} calls; got {actual}"
# ---------------------------------------------------------------------------
# FM-6
# ---------------------------------------------------------------------------
def assert_fm6_hang_envelopes(
result: Result[NotifyPayload],
*,
expected_ok: int,
expected_failed: int,
) -> None:
"""FM-6: hanging docs produced FAILURE envelopes — chord didn't stall."""
assert result.payload is not None, "FM-6 regression: notify payload is None"
assert (
result.payload.ok == expected_ok
), f"FM-6 regression: expected {expected_ok} ok; got {result.payload.ok}"
assert result.payload.failed == expected_failed, (
f"FM-6 regression: expected {expected_failed} failed "
f"(hanging docs → timeout envelopes); got {result.payload.failed}"
)