Skip to content

Commit d0ae940

Browse files
committed
[slopfix] fix(pegboard): decrement kv_storage_used metric on actor delete
1 parent fc555e6 commit d0ae940

7 files changed

Lines changed: 12356 additions & 11132 deletions

File tree

engine/packages/pegboard/src/workflows/actor/metrics.rs

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,10 @@ pub(crate) async fn pegboard_actor_metrics(ctx: &mut WorkflowCtx, input: &Input)
120120
})
121121
.await?;
122122

123+
// The actor's kv is deleted on destroy, so remove its contribution from the running kv
124+
// storage total.
125+
ctx.v(2).activity(RemoveKvMetricsInput {}).await?;
126+
123127
Ok(())
124128
}
125129

@@ -323,6 +327,37 @@ async fn record_kv_metrics(ctx: &ActivityCtx, input: &RecordKvMetricsInput) -> R
323327
Ok(())
324328
}
325329

330+
#[derive(Debug, Serialize, Deserialize, Hash)]
331+
struct RemoveKvMetricsInput {}
332+
333+
#[activity(RemoveKvMetrics)]
334+
async fn remove_kv_metrics(ctx: &ActivityCtx, input: &RemoveKvMetricsInput) -> Result<()> {
335+
let mut state = ctx.state::<State>()?;
336+
337+
let namespace_id = state.namespace_id;
338+
let name = &state.name;
339+
let last_kv_storage_size = state.last_kv_storage_size;
340+
341+
if last_kv_storage_size != 0 {
342+
ctx.udb()?
343+
.txn("pegboard_actor_remove_kv_metrics", |tx| async move {
344+
namespace::keys::metric::inc(
345+
&tx.with_subspace(namespace::keys::subspace()),
346+
namespace_id,
347+
namespace::keys::metric::Metric::KvStorageUsed(name.to_string()),
348+
-last_kv_storage_size,
349+
);
350+
351+
Ok(())
352+
})
353+
.await?;
354+
}
355+
356+
state.last_kv_storage_size = 0;
357+
358+
Ok(())
359+
}
360+
326361
#[signal("pegboard_actor_metrics_pause")]
327362
pub struct Pause {
328363
pub ts: i64,

engine/packages/pegboard/src/workflows/actor2/metrics.rs

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,10 @@ pub(crate) async fn pegboard_actor2_metrics(ctx: &mut WorkflowCtx, input: &Input
119119
})
120120
.await?;
121121

122+
// The actor's kv is deleted on destroy, so remove its contribution from the running kv
123+
// storage total.
124+
ctx.v(2).activity(RemoveKvMetricsInput {}).await?;
125+
122126
Ok(())
123127
}
124128

@@ -169,6 +173,8 @@ async fn record_metrics(ctx: &ActivityCtx, input: &RecordMetricsInput) -> Result
169173
let name = &state.name;
170174
ctx.udb()?
171175
.txn("pegboard_actor2_record_metrics", |tx| async move {
176+
tx.priority(Priority::Low)?;
177+
172178
namespace::keys::metric::inc(
173179
&tx.with_subspace(namespace::keys::subspace()),
174180
namespace_id,
@@ -212,6 +218,8 @@ async fn record_kv_metrics(ctx: &ActivityCtx, input: &RecordKvMetricsInput) -> R
212218
.txn("pegboard_actor2_kv_size_scan", |tx| {
213219
let last_key = &last_key;
214220
async move {
221+
tx.priority(Priority::Low)?;
222+
215223
let start = Instant::now();
216224
let tx = tx.with_subspace(keys::subspace());
217225

@@ -296,6 +304,8 @@ async fn record_kv_metrics(ctx: &ActivityCtx, input: &RecordKvMetricsInput) -> R
296304

297305
ctx.udb()?
298306
.txn("pegboard_actor2_record_kv_metrics", |tx| async move {
307+
tx.priority(Priority::Low)?;
308+
299309
namespace::keys::metric::inc(
300310
&tx.with_subspace(namespace::keys::subspace()),
301311
namespace_id,
@@ -312,6 +322,39 @@ async fn record_kv_metrics(ctx: &ActivityCtx, input: &RecordKvMetricsInput) -> R
312322
Ok(())
313323
}
314324

325+
#[derive(Debug, Serialize, Deserialize, Hash)]
326+
struct RemoveKvMetricsInput {}
327+
328+
#[activity(RemoveKvMetrics)]
329+
async fn remove_kv_metrics(ctx: &ActivityCtx, input: &RemoveKvMetricsInput) -> Result<()> {
330+
let mut state = ctx.state::<State>()?;
331+
332+
let namespace_id = state.namespace_id;
333+
let name = &state.name;
334+
let last_kv_storage_size = state.last_kv_storage_size;
335+
336+
if last_kv_storage_size != 0 {
337+
ctx.udb()?
338+
.txn("pegboard_actor2_remove_kv_metrics", |tx| async move {
339+
tx.priority(Priority::Low)?;
340+
341+
namespace::keys::metric::inc(
342+
&tx.with_subspace(namespace::keys::subspace()),
343+
namespace_id,
344+
namespace::keys::metric::Metric::KvStorageUsed(name.to_string()),
345+
-last_kv_storage_size,
346+
);
347+
348+
Ok(())
349+
})
350+
.await?;
351+
}
352+
353+
state.last_kv_storage_size = 0;
354+
355+
Ok(())
356+
}
357+
315358
#[signal("pegboard_actor_metrics_pause")]
316359
pub struct Pause {
317360
pub ts: i64,

0 commit comments

Comments
 (0)