# HELP slot_defence_process_cpu_user_seconds_total Total user CPU time spent in seconds. # TYPE slot_defence_process_cpu_user_seconds_total counter slot_defence_process_cpu_user_seconds_total 5338.327684000001 # HELP slot_defence_process_cpu_system_seconds_total Total system CPU time spent in seconds. # TYPE slot_defence_process_cpu_system_seconds_total counter slot_defence_process_cpu_system_seconds_total 2584.400208 # HELP slot_defence_process_cpu_seconds_total Total user and system CPU time spent in seconds. # TYPE slot_defence_process_cpu_seconds_total counter slot_defence_process_cpu_seconds_total 7922.727892000001 # HELP slot_defence_process_start_time_seconds Start time of the process since unix epoch in seconds. # TYPE slot_defence_process_start_time_seconds gauge slot_defence_process_start_time_seconds 1790765391 # HELP slot_defence_process_resident_memory_bytes Resident memory size in bytes. # TYPE slot_defence_process_resident_memory_bytes gauge slot_defence_process_resident_memory_bytes 135233536 # HELP slot_defence_process_virtual_memory_bytes Virtual memory size in bytes. # TYPE slot_defence_process_virtual_memory_bytes gauge slot_defence_process_virtual_memory_bytes 44272242688 # HELP slot_defence_process_heap_bytes Process heap size in bytes. # TYPE slot_defence_process_heap_bytes gauge slot_defence_process_heap_bytes 159502336 # HELP slot_defence_process_open_fds Number of open file descriptors. # TYPE slot_defence_process_open_fds gauge slot_defence_process_open_fds 32 # HELP slot_defence_process_max_fds Maximum number of open file descriptors. # TYPE slot_defence_process_max_fds gauge slot_defence_process_max_fds 10240 # HELP slot_defence_nodejs_eventloop_lag_seconds Lag of event loop in seconds. # TYPE slot_defence_nodejs_eventloop_lag_seconds gauge slot_defence_nodejs_eventloop_lag_seconds 0.004973259 # HELP slot_defence_nodejs_eventloop_lag_min_seconds The minimum recorded event loop delay. # TYPE slot_defence_nodejs_eventloop_lag_min_seconds gauge slot_defence_nodejs_eventloop_lag_min_seconds 0.000031232 # HELP slot_defence_nodejs_eventloop_lag_max_seconds The maximum recorded event loop delay. # TYPE slot_defence_nodejs_eventloop_lag_max_seconds gauge slot_defence_nodejs_eventloop_lag_max_seconds 0.202244095 # HELP slot_defence_nodejs_eventloop_lag_mean_seconds The mean of the recorded event loop delays. # TYPE slot_defence_nodejs_eventloop_lag_mean_seconds gauge slot_defence_nodejs_eventloop_lag_mean_seconds 0.010174415013861422 # HELP slot_defence_nodejs_eventloop_lag_stddev_seconds The standard deviation of the recorded event loop delays. # TYPE slot_defence_nodejs_eventloop_lag_stddev_seconds gauge slot_defence_nodejs_eventloop_lag_stddev_seconds 0.0006006094180388646 # HELP slot_defence_nodejs_eventloop_lag_p50_seconds The 50th percentile of the recorded event loop delays. # TYPE slot_defence_nodejs_eventloop_lag_p50_seconds gauge slot_defence_nodejs_eventloop_lag_p50_seconds 0.010158079 # HELP slot_defence_nodejs_eventloop_lag_p90_seconds The 90th percentile of the recorded event loop delays. # TYPE slot_defence_nodejs_eventloop_lag_p90_seconds gauge slot_defence_nodejs_eventloop_lag_p90_seconds 0.010190847 # HELP slot_defence_nodejs_eventloop_lag_p99_seconds The 99th percentile of the recorded event loop delays. # TYPE slot_defence_nodejs_eventloop_lag_p99_seconds gauge slot_defence_nodejs_eventloop_lag_p99_seconds 0.010633215 # HELP slot_defence_nodejs_active_resources Number of active resources that are currently keeping the event loop alive, grouped by async resource type. # TYPE slot_defence_nodejs_active_resources gauge slot_defence_nodejs_active_resources{type="PipeWrap"} 2 slot_defence_nodejs_active_resources{type="TCPServerWrap"} 1 slot_defence_nodejs_active_resources{type="TCPSocketWrap"} 2 slot_defence_nodejs_active_resources{type="Immediate"} 1 # HELP slot_defence_nodejs_active_resources_total Total number of active resources. # TYPE slot_defence_nodejs_active_resources_total gauge slot_defence_nodejs_active_resources_total 6 # HELP slot_defence_nodejs_active_handles Number of active libuv handles grouped by handle type. Every handle type is C++ class name. # TYPE slot_defence_nodejs_active_handles gauge slot_defence_nodejs_active_handles{type="Socket"} 3 slot_defence_nodejs_active_handles{type="Server"} 1 slot_defence_nodejs_active_handles{type="TLSSocket"} 1 # HELP slot_defence_nodejs_active_handles_total Total number of active handles. # TYPE slot_defence_nodejs_active_handles_total gauge slot_defence_nodejs_active_handles_total 5 # HELP slot_defence_nodejs_active_requests Number of active libuv requests grouped by request type. Every request type is C++ class name. # TYPE slot_defence_nodejs_active_requests gauge # HELP slot_defence_nodejs_active_requests_total Total number of active requests. # TYPE slot_defence_nodejs_active_requests_total gauge slot_defence_nodejs_active_requests_total 0 # HELP slot_defence_nodejs_heap_size_total_bytes Process heap size from Node.js in bytes. # TYPE slot_defence_nodejs_heap_size_total_bytes gauge slot_defence_nodejs_heap_size_total_bytes 52981760 # HELP slot_defence_nodejs_heap_size_used_bytes Process heap size used from Node.js in bytes. # TYPE slot_defence_nodejs_heap_size_used_bytes gauge slot_defence_nodejs_heap_size_used_bytes 44823088 # HELP slot_defence_nodejs_external_memory_bytes Node.js external memory size in bytes. # TYPE slot_defence_nodejs_external_memory_bytes gauge slot_defence_nodejs_external_memory_bytes 4538802 # HELP slot_defence_nodejs_heap_space_size_total_bytes Process heap space size total from Node.js in bytes. # TYPE slot_defence_nodejs_heap_space_size_total_bytes gauge slot_defence_nodejs_heap_space_size_total_bytes{space="read_only"} 0 slot_defence_nodejs_heap_space_size_total_bytes{space="new"} 1048576 slot_defence_nodejs_heap_space_size_total_bytes{space="old"} 39813120 slot_defence_nodejs_heap_space_size_total_bytes{space="code"} 4980736 slot_defence_nodejs_heap_space_size_total_bytes{space="shared"} 0 slot_defence_nodejs_heap_space_size_total_bytes{space="new_large_object"} 0 slot_defence_nodejs_heap_space_size_total_bytes{space="large_object"} 6983680 slot_defence_nodejs_heap_space_size_total_bytes{space="code_large_object"} 155648 slot_defence_nodejs_heap_space_size_total_bytes{space="shared_large_object"} 0 # HELP slot_defence_nodejs_heap_space_size_used_bytes Process heap space size used from Node.js in bytes. # TYPE slot_defence_nodejs_heap_space_size_used_bytes gauge slot_defence_nodejs_heap_space_size_used_bytes{space="read_only"} 0 slot_defence_nodejs_heap_space_size_used_bytes{space="new"} 507336 slot_defence_nodejs_heap_space_size_used_bytes{space="old"} 33167912 slot_defence_nodejs_heap_space_size_used_bytes{space="code"} 4100160 slot_defence_nodejs_heap_space_size_used_bytes{space="shared"} 0 slot_defence_nodejs_heap_space_size_used_bytes{space="new_large_object"} 0 slot_defence_nodejs_heap_space_size_used_bytes{space="large_object"} 6927904 slot_defence_nodejs_heap_space_size_used_bytes{space="code_large_object"} 138432 slot_defence_nodejs_heap_space_size_used_bytes{space="shared_large_object"} 0 # HELP slot_defence_nodejs_heap_space_size_available_bytes Process heap space size available from Node.js in bytes. # TYPE slot_defence_nodejs_heap_space_size_available_bytes gauge slot_defence_nodejs_heap_space_size_available_bytes{space="read_only"} 0 slot_defence_nodejs_heap_space_size_available_bytes{space="new"} 523544 slot_defence_nodejs_heap_space_size_available_bytes{space="old"} 5900336 slot_defence_nodejs_heap_space_size_available_bytes{space="code"} 568352 slot_defence_nodejs_heap_space_size_available_bytes{space="shared"} 0 slot_defence_nodejs_heap_space_size_available_bytes{space="new_large_object"} 1048576 slot_defence_nodejs_heap_space_size_available_bytes{space="large_object"} 0 slot_defence_nodejs_heap_space_size_available_bytes{space="code_large_object"} 0 slot_defence_nodejs_heap_space_size_available_bytes{space="shared_large_object"} 0 # HELP slot_defence_nodejs_version_info Node.js version info. # TYPE slot_defence_nodejs_version_info gauge slot_defence_nodejs_version_info{version="v20.20.2",major="20",minor="20",patch="2"} 1 # HELP slot_defence_nodejs_gc_duration_seconds Garbage collection duration by kind, one of major, minor, incremental or weakcb. # TYPE slot_defence_nodejs_gc_duration_seconds histogram slot_defence_nodejs_gc_duration_seconds_bucket{le="0.001",kind="minor"} 14474 slot_defence_nodejs_gc_duration_seconds_bucket{le="0.01",kind="minor"} 25201 slot_defence_nodejs_gc_duration_seconds_bucket{le="0.1",kind="minor"} 25206 slot_defence_nodejs_gc_duration_seconds_bucket{le="1",kind="minor"} 25206 slot_defence_nodejs_gc_duration_seconds_bucket{le="2",kind="minor"} 25206 slot_defence_nodejs_gc_duration_seconds_bucket{le="5",kind="minor"} 25206 slot_defence_nodejs_gc_duration_seconds_bucket{le="+Inf",kind="minor"} 25206 slot_defence_nodejs_gc_duration_seconds_sum{kind="minor"} 24.835345680014736 slot_defence_nodejs_gc_duration_seconds_count{kind="minor"} 25206 slot_defence_nodejs_gc_duration_seconds_bucket{le="0.001",kind="incremental"} 4 slot_defence_nodejs_gc_duration_seconds_bucket{le="0.01",kind="incremental"} 213 slot_defence_nodejs_gc_duration_seconds_bucket{le="0.1",kind="incremental"} 223 slot_defence_nodejs_gc_duration_seconds_bucket{le="1",kind="incremental"} 223 slot_defence_nodejs_gc_duration_seconds_bucket{le="2",kind="incremental"} 223 slot_defence_nodejs_gc_duration_seconds_bucket{le="5",kind="incremental"} 223 slot_defence_nodejs_gc_duration_seconds_bucket{le="+Inf",kind="incremental"} 223 slot_defence_nodejs_gc_duration_seconds_sum{kind="incremental"} 1.0698340423302033 slot_defence_nodejs_gc_duration_seconds_count{kind="incremental"} 223 slot_defence_nodejs_gc_duration_seconds_bucket{le="0.001",kind="major"} 0 slot_defence_nodejs_gc_duration_seconds_bucket{le="0.01",kind="major"} 143 slot_defence_nodejs_gc_duration_seconds_bucket{le="0.1",kind="major"} 231 slot_defence_nodejs_gc_duration_seconds_bucket{le="1",kind="major"} 231 slot_defence_nodejs_gc_duration_seconds_bucket{le="2",kind="major"} 231 slot_defence_nodejs_gc_duration_seconds_bucket{le="5",kind="major"} 231 slot_defence_nodejs_gc_duration_seconds_bucket{le="+Inf",kind="major"} 231 slot_defence_nodejs_gc_duration_seconds_sum{kind="major"} 2.221571027093875 slot_defence_nodejs_gc_duration_seconds_count{kind="major"} 231 # HELP slot_defence_check_requests_total Total number of POST /check requests handled successfully. # TYPE slot_defence_check_requests_total counter slot_defence_check_requests_total 371 # HELP slot_defence_check_decisions_total Total number of individual slot decisions returned by POST /check, broken down by outcome and reason (WORK-4 12-code reason enum). # TYPE slot_defence_check_decisions_total counter slot_defence_check_decisions_total{outcome="available",reason="anchor_in_range_with_capacity"} 19901 slot_defence_check_decisions_total{outcome="unavailable",reason="slot_capacity_full"} 91 slot_defence_check_decisions_total{outcome="unavailable",reason="cluster_reserved"} 176 slot_defence_check_decisions_total{outcome="unavailable",reason="no_route_covers_customer_zone"} 116 slot_defence_check_decisions_total{outcome="unavailable",reason="outside_delivery_window"} 1486 # HELP slot_defence_over_capacity_total Sum of trimmed driver-slot counts (WORK-6 X-Slot-Defence-Trimmed-Count header value) across all /check requests. Increments only when a request produces a positive trim count. # TYPE slot_defence_over_capacity_total counter slot_defence_over_capacity_total 0 # HELP slot_defence_osrm_calls_total Total outbound OSRM HTTP calls, broken down by outcome status (ok / error / timeout). # TYPE slot_defence_osrm_calls_total counter # HELP slot_defence_osrm_route_cache_total Per-request OSRM route memo lookups by outcome: hit = served a resolved route from the request memo (no OSRM call); hit_rejected = re-threw a cached OSRM failure (no OSRM call, but the route is unavailable) — a spike here means an OSRM outage, not a healthy cache; miss = computed and cached. # TYPE slot_defence_osrm_route_cache_total counter # HELP slot_defence_volume_defence_trips_total Total number of volume-defence trips — each time the atomic-reserve Lua returned cap-exceeded (i.e. a slot transitioned into temp-closed). Fires exactly once per (callerId, zone, hourWindow). # TYPE slot_defence_volume_defence_trips_total counter slot_defence_volume_defence_trips_total 33 # HELP slot_defence_volume_defence_decisions_total Total ops decisions recorded on POST /volume-defence/decisions, broken down by outcome (reopen / hold_closed). # TYPE slot_defence_volume_defence_decisions_total counter # HELP slot_defence_slack_trip_notifications_sent_total Total successful Slack chat.postMessage calls for volume-defence trips. # TYPE slot_defence_slack_trip_notifications_sent_total counter slot_defence_slack_trip_notifications_sent_total 4 # HELP slot_defence_slack_notify_failures_total Total failed Slack outbound calls, broken down by outcome (timeout / network_error / http_error / slack_error). # TYPE slot_defence_slack_notify_failures_total counter # HELP slot_defence_slack_trip_notifications_skipped_total Total volume-defence trips that produced NO Slack notification, by reason (feature_flag_off / channel_id_unset / demo_caller / duplicate_alert_suppressed). Reconciles slot_defence_volume_defence_trips_total against slot_defence_slack_trip_notifications_sent_total: trips = sent + failures + skipped. # TYPE slot_defence_slack_trip_notifications_skipped_total counter # HELP slot_defence_persistence_failures_total Total best-effort (fire-and-forget) Postgres write failures, by target table. Non-zero means the Slice 11 read-model is missing rows the service believed it had written. # TYPE slot_defence_persistence_failures_total counter # HELP slot_defence_volume_defence_deliveries_total Total order_committed events on POST /slots/events, by volume-defence outcome. recorded = counted against the cap; duplicate / past_delivery / unknown_zone = reached the counter but not counted; dark_flag_off = volume defence off (globally or narrowed off for the business); dark_no_caller_id = unauthenticated; dark_no_business = the caller resolves to no business, so no config applies; dark_no_caps = the business's published config carries no volume-defence caps. A rising dark_* while recorded falls means orders have silently stopped filling caps. # TYPE slot_defence_volume_defence_deliveries_total counter slot_defence_volume_defence_deliveries_total{outcome="duplicate"} 13 slot_defence_volume_defence_deliveries_total{outcome="recorded"} 1 # HELP slot_defence_volume_defence_config_unavailable_total Requests affected by an unavailable business config plane (never loaded in this process), by path and outcome. fail_open = volume-defence caps were not enforced (/check, /reserve); refused = the request was refused with 502/503 (/slots/events, /slots/precheck). Stale-config serving is not counted. # TYPE slot_defence_volume_defence_config_unavailable_total counter # HELP slot_defence_volume_defence_own_hold_lookups_total Own-hold lookups on POST /check, one per request carrying holdOrderId with FEATURE_CHECK_EXCLUDES_OWN_HOLD on and volume defence governing the caller, by outcome. adjusted = the caller's own unit was added back to slotCapacityRemaining below cap (WORK-93); exempted = the caller's held slot was handed back at cap (WORK-62); no_matching_slot = a live hold was found but no capped slot offered in this response sits in its (zone, hour) cell; no_live_hold = the orderId resolved to no live hold for this caller (expired, committed, or not the id sent to /reserve); read_failed = the hold lookup errored (see the volume_defence_own_hold_read_failed warn). Only adjusted and exempted change what the caller sees. # TYPE slot_defence_volume_defence_own_hold_lookups_total counter slot_defence_volume_defence_own_hold_lookups_total{outcome="no_live_hold"} 233 slot_defence_volume_defence_own_hold_lookups_total{outcome="no_matching_slot"} 30 slot_defence_volume_defence_own_hold_lookups_total{outcome="adjusted"} 34 slot_defence_volume_defence_own_hold_lookups_total{outcome="exempted"} 1 # HELP slot_defence_clustering_decisions_total Total per-request clustering decisions on POST /check, broken down by outcome. Fires once per /check request when clustering is active (feature flag on AND distribution attached). # TYPE slot_defence_clustering_decisions_total counter slot_defence_clustering_decisions_total{outcome="below_n"} 19591 slot_defence_clustering_decisions_total{outcome="cell_over_m"} 40 slot_defence_clustering_decisions_total{outcome="flagged"} 191 # HELP slot_defence_cluster_reservation_decisions_total Per-slot cluster-reservation outcomes on POST /check. `reserved` means the slot was withheld from this customer; every other value is a fail-open path naming why the slot stayed visible. # TYPE slot_defence_cluster_reservation_decisions_total counter slot_defence_cluster_reservation_decisions_total{outcome="no_anchor"} 19751 slot_defence_cluster_reservation_decisions_total{outcome="within_radius"} 241 slot_defence_cluster_reservation_decisions_total{outcome="reserved"} 176 # HELP slot_defence_adjacency_decisions_total Per-slot adjacent-slot-defence outcomes on POST /check. `reduced` means capacity was lowered by the travel from the previous slot's cluster; `refused` means it reached zero and the slot was withheld; every other value is a fail-open path naming why capacity was untouched. # TYPE slot_defence_adjacency_decisions_total counter slot_defence_adjacency_decisions_total{outcome="no_anchor_here",ring_multiple="n/a",chains="1"} 21558 slot_defence_adjacency_decisions_total{outcome="first_anchored_slot",ring_multiple="n/a",chains="1"} 53 slot_defence_adjacency_decisions_total{outcome="within_radius",ring_multiple="n/a",chains="1"} 150 slot_defence_adjacency_decisions_total{outcome="reduced",ring_multiple="3R+",chains="1"} 4 slot_defence_adjacency_decisions_total{outcome="reduced",ring_multiple="2R",chains="1"} 5 # HELP slot_defence_fleet_feasibility_total Per-slot fleet-feasibility gate outcomes on POST /check. `refused_infeasible` withheld a slot whose window already holds driverCount distinct clusters; `allowed` fit within the roster; the rest are fail-open paths (flag reads never reach here — a dark counter would misreport adoption). # TYPE slot_defence_fleet_feasibility_total counter # HELP slot_defence_reserve_fleet_feasibility_total Fleet-feasibility verdicts on POST /reserve, one per hold the check ran for, by outcome and location_source. `refused_infeasible` refused the hold (reason fleet-infeasible); `allowed` passed it on to the zone-hour cap; `no_location` / `no_day` / `index_degraded` / `index_timeout` (index read over its 500 ms budget) / `pass_error` (the check threw) are fail-open skips that let the hold through unjudged. location_source is request / zone_centroid / none. Flag-off and past-delivery holds are not counted. # TYPE slot_defence_reserve_fleet_feasibility_total counter # HELP slot_defence_adjacency_index_writes_total Anchor-day index write attempts on POST /slots/events, by outcome (indexed / no_location / unparseable_window / redis_error). Anything other than `indexed` means adjacency defence will not see that order, and will under-penalise the day. # TYPE slot_defence_adjacency_index_writes_total counter slot_defence_adjacency_index_writes_total{outcome="indexed"} 14 # HELP slot_defence_clustering_errors_total Total clustering computation errors (WORK-15 MC-10 degrade path fires), broken down by cause. Non-zero counts here indicate the graceful-degrade safety net is firing in production. # TYPE slot_defence_clustering_errors_total counter # HELP slot_defence_slack_interactions_total Total Slack interactive-component interactions handled, by outcome (applied_reopen / applied_hold_closed / refused_* / retried_later). # TYPE slot_defence_slack_interactions_total counter # HELP slot_defence_admin_magic_link_issued_total Total admin magic-links actually issued (a provisioned admin requested a link and a token was minted). Unprovisioned-email requests do NOT increment this — they mint no token. Divergence from verifications_total{outcome=success} is the link-abandonment / delivery-failure signal. # TYPE slot_defence_admin_magic_link_issued_total counter slot_defence_admin_magic_link_issued_total 5 # HELP slot_defence_admin_magic_link_verifications_total Total admin magic-link verification attempts on POST /admin/auth/verify, by outcome. success = token consumed into a session; invalid = token missing/expired/already-used (Redis GETDEL cannot distinguish these, so they share one bucket); admin_removed (WORK-85) = the token was genuine and unspent but its admin_users row is gone, i.e. a removed person redeemed a link. The wire response is deliberately the SAME 401 as invalid — only the metric separates them. # TYPE slot_defence_admin_magic_link_verifications_total counter slot_defence_admin_magic_link_verifications_total{outcome="success"} 5 slot_defence_admin_magic_link_verifications_total{outcome="invalid"} 3 # HELP slot_defence_admin_config_total Total admin-config console actions, by action (draft_saved / published / audit_read). Lets ops see config-editing activity per business process without log search. # TYPE slot_defence_admin_config_total counter slot_defence_admin_config_total{action="audit_read"} 12 slot_defence_admin_config_total{action="draft_saved"} 5 slot_defence_admin_config_total{action="published"} 5 # HELP slot_defence_admin_config_applied_total Total per-business config resolutions where an override actually took effect (one or more keys overridden). Fires from every resolving surface, not only /check: /check, /reserve, /slots/events order_committed, /slots/precheck, journey feasibility, the admin map and the demo volume-defence panel. Makes the reach of published overrides observable — a published draft nobody's traffic hits stays flat here. # TYPE slot_defence_admin_config_applied_total counter slot_defence_admin_config_applied_total 32892 # HELP slot_defence_admin_map_reads_total Total GET /admin/map/day reads, by outcome. ok = at least one zone returned; empty = no zones (nothing to draw); persistence_off = the Postgres read model is disabled so the map can never populate; query_error = the aggregation query failed and the request 502'd. # TYPE slot_defence_admin_map_reads_total counter slot_defence_admin_map_reads_total{outcome="ok"} 5 slot_defence_admin_map_reads_total{outcome="empty"} 1 # HELP slot_defence_admin_map_points_reads_total Total GET /admin/map/points reads, by outcome. ok = at least one point returned; empty = nothing to draw; index_degraded = the Redis anchor-day index was unreachable or over its entry bound, so the map is blank for a reason that is NOT 'no orders'; persistence_off = the Postgres read model is disabled so no order can be attributed to a business; query_error = a Postgres query failed and the request errored. # TYPE slot_defence_admin_map_points_reads_total counter slot_defence_admin_map_points_reads_total{outcome="ok"} 1 # HELP slot_defence_admin_map_points_withheld_total Total order points withheld from GET /admin/map/points because the Redis index entry under this business's orderId was written by a different caller (orderId collision). Non-zero means callers are colliding on orderId; the points are correctly withheld, but the collision itself warrants investigation. # TYPE slot_defence_admin_map_points_withheld_total counter slot_defence_admin_map_points_withheld_total 0 # HELP slot_defence_check_duration_seconds End-to-end duration of a POST /check invocation, measured from controller entry to just before res.json(). Excludes JSON serialisation. # TYPE slot_defence_check_duration_seconds histogram slot_defence_check_duration_seconds_bucket{le="0.01"} 0 slot_defence_check_duration_seconds_bucket{le="0.025"} 2 slot_defence_check_duration_seconds_bucket{le="0.05"} 9 slot_defence_check_duration_seconds_bucket{le="0.1"} 35 slot_defence_check_duration_seconds_bucket{le="0.2"} 266 slot_defence_check_duration_seconds_bucket{le="0.3"} 325 slot_defence_check_duration_seconds_bucket{le="0.5"} 360 slot_defence_check_duration_seconds_bucket{le="1"} 369 slot_defence_check_duration_seconds_bucket{le="2"} 370 slot_defence_check_duration_seconds_bucket{le="3"} 370 slot_defence_check_duration_seconds_bucket{le="5"} 371 slot_defence_check_duration_seconds_bucket{le="8"} 371 slot_defence_check_duration_seconds_bucket{le="+Inf"} 371 slot_defence_check_duration_seconds_sum 73.66276362700003 slot_defence_check_duration_seconds_count 371 # HELP slot_defence_check_phase_duration_seconds Per-phase duration of a POST /check, labelled by phase (optimiser / volume_defence / adjacency / fleet / opt_redis / opt_cluster / opt_compute). The four top-level phases plus a small residual sum to slot_defence_check_duration_seconds; opt_redis (anchorStore Redis reads) + opt_cluster (identifyClusters build) + opt_compute (scoring loop + everything else) are a sub-breakdown WITHIN optimiser. # TYPE slot_defence_check_phase_duration_seconds histogram slot_defence_check_phase_duration_seconds_bucket{le="0.01",phase="optimiser"} 2 slot_defence_check_phase_duration_seconds_bucket{le="0.025",phase="optimiser"} 8 slot_defence_check_phase_duration_seconds_bucket{le="0.05",phase="optimiser"} 143 slot_defence_check_phase_duration_seconds_bucket{le="0.1",phase="optimiser"} 317 slot_defence_check_phase_duration_seconds_bucket{le="0.2",phase="optimiser"} 354 slot_defence_check_phase_duration_seconds_bucket{le="0.3",phase="optimiser"} 366 slot_defence_check_phase_duration_seconds_bucket{le="0.5",phase="optimiser"} 369 slot_defence_check_phase_duration_seconds_bucket{le="1",phase="optimiser"} 371 slot_defence_check_phase_duration_seconds_bucket{le="2",phase="optimiser"} 371 slot_defence_check_phase_duration_seconds_bucket{le="3",phase="optimiser"} 371 slot_defence_check_phase_duration_seconds_bucket{le="5",phase="optimiser"} 371 slot_defence_check_phase_duration_seconds_bucket{le="8",phase="optimiser"} 371 slot_defence_check_phase_duration_seconds_bucket{le="+Inf",phase="optimiser"} 371 slot_defence_check_phase_duration_seconds_sum{phase="optimiser"} 27.366999999999997 slot_defence_check_phase_duration_seconds_count{phase="optimiser"} 371 slot_defence_check_phase_duration_seconds_bucket{le="0.01",phase="volume_defence"} 16 slot_defence_check_phase_duration_seconds_bucket{le="0.025",phase="volume_defence"} 22 slot_defence_check_phase_duration_seconds_bucket{le="0.05",phase="volume_defence"} 50 slot_defence_check_phase_duration_seconds_bucket{le="0.1",phase="volume_defence"} 306 slot_defence_check_phase_duration_seconds_bucket{le="0.2",phase="volume_defence"} 342 slot_defence_check_phase_duration_seconds_bucket{le="0.3",phase="volume_defence"} 354 slot_defence_check_phase_duration_seconds_bucket{le="0.5",phase="volume_defence"} 367 slot_defence_check_phase_duration_seconds_bucket{le="1",phase="volume_defence"} 370 slot_defence_check_phase_duration_seconds_bucket{le="2",phase="volume_defence"} 370 slot_defence_check_phase_duration_seconds_bucket{le="3",phase="volume_defence"} 371 slot_defence_check_phase_duration_seconds_bucket{le="5",phase="volume_defence"} 371 slot_defence_check_phase_duration_seconds_bucket{le="8",phase="volume_defence"} 371 slot_defence_check_phase_duration_seconds_bucket{le="+Inf",phase="volume_defence"} 371 slot_defence_check_phase_duration_seconds_sum{phase="volume_defence"} 35.787 slot_defence_check_phase_duration_seconds_count{phase="volume_defence"} 371 slot_defence_check_phase_duration_seconds_bucket{le="0.01",phase="adjacency"} 35 slot_defence_check_phase_duration_seconds_bucket{le="0.025",phase="adjacency"} 324 slot_defence_check_phase_duration_seconds_bucket{le="0.05",phase="adjacency"} 355 slot_defence_check_phase_duration_seconds_bucket{le="0.1",phase="adjacency"} 368 slot_defence_check_phase_duration_seconds_bucket{le="0.2",phase="adjacency"} 370 slot_defence_check_phase_duration_seconds_bucket{le="0.3",phase="adjacency"} 371 slot_defence_check_phase_duration_seconds_bucket{le="0.5",phase="adjacency"} 371 slot_defence_check_phase_duration_seconds_bucket{le="1",phase="adjacency"} 371 slot_defence_check_phase_duration_seconds_bucket{le="2",phase="adjacency"} 371 slot_defence_check_phase_duration_seconds_bucket{le="3",phase="adjacency"} 371 slot_defence_check_phase_duration_seconds_bucket{le="5",phase="adjacency"} 371 slot_defence_check_phase_duration_seconds_bucket{le="8",phase="adjacency"} 371 slot_defence_check_phase_duration_seconds_bucket{le="+Inf",phase="adjacency"} 371 slot_defence_check_phase_duration_seconds_sum{phase="adjacency"} 7.646999999999997 slot_defence_check_phase_duration_seconds_count{phase="adjacency"} 371 slot_defence_check_phase_duration_seconds_bucket{le="0.01",phase="fleet"} 370 slot_defence_check_phase_duration_seconds_bucket{le="0.025",phase="fleet"} 371 slot_defence_check_phase_duration_seconds_bucket{le="0.05",phase="fleet"} 371 slot_defence_check_phase_duration_seconds_bucket{le="0.1",phase="fleet"} 371 slot_defence_check_phase_duration_seconds_bucket{le="0.2",phase="fleet"} 371 slot_defence_check_phase_duration_seconds_bucket{le="0.3",phase="fleet"} 371 slot_defence_check_phase_duration_seconds_bucket{le="0.5",phase="fleet"} 371 slot_defence_check_phase_duration_seconds_bucket{le="1",phase="fleet"} 371 slot_defence_check_phase_duration_seconds_bucket{le="2",phase="fleet"} 371 slot_defence_check_phase_duration_seconds_bucket{le="3",phase="fleet"} 371 slot_defence_check_phase_duration_seconds_bucket{le="5",phase="fleet"} 371 slot_defence_check_phase_duration_seconds_bucket{le="8",phase="fleet"} 371 slot_defence_check_phase_duration_seconds_bucket{le="+Inf",phase="fleet"} 371 slot_defence_check_phase_duration_seconds_sum{phase="fleet"} 0.022000000000000006 slot_defence_check_phase_duration_seconds_count{phase="fleet"} 371 slot_defence_check_phase_duration_seconds_bucket{le="0.01",phase="opt_redis"} 3 slot_defence_check_phase_duration_seconds_bucket{le="0.025",phase="opt_redis"} 272 slot_defence_check_phase_duration_seconds_bucket{le="0.05",phase="opt_redis"} 329 slot_defence_check_phase_duration_seconds_bucket{le="0.1",phase="opt_redis"} 358 slot_defence_check_phase_duration_seconds_bucket{le="0.2",phase="opt_redis"} 363 slot_defence_check_phase_duration_seconds_bucket{le="0.3",phase="opt_redis"} 367 slot_defence_check_phase_duration_seconds_bucket{le="0.5",phase="opt_redis"} 370 slot_defence_check_phase_duration_seconds_bucket{le="1",phase="opt_redis"} 371 slot_defence_check_phase_duration_seconds_bucket{le="2",phase="opt_redis"} 371 slot_defence_check_phase_duration_seconds_bucket{le="3",phase="opt_redis"} 371 slot_defence_check_phase_duration_seconds_bucket{le="5",phase="opt_redis"} 371 slot_defence_check_phase_duration_seconds_bucket{le="8",phase="opt_redis"} 371 slot_defence_check_phase_duration_seconds_bucket{le="+Inf",phase="opt_redis"} 371 slot_defence_check_phase_duration_seconds_sum{phase="opt_redis"} 12.525999999999991 slot_defence_check_phase_duration_seconds_count{phase="opt_redis"} 371 slot_defence_check_phase_duration_seconds_bucket{le="0.01",phase="opt_cluster"} 371 slot_defence_check_phase_duration_seconds_bucket{le="0.025",phase="opt_cluster"} 371 slot_defence_check_phase_duration_seconds_bucket{le="0.05",phase="opt_cluster"} 371 slot_defence_check_phase_duration_seconds_bucket{le="0.1",phase="opt_cluster"} 371 slot_defence_check_phase_duration_seconds_bucket{le="0.2",phase="opt_cluster"} 371 slot_defence_check_phase_duration_seconds_bucket{le="0.3",phase="opt_cluster"} 371 slot_defence_check_phase_duration_seconds_bucket{le="0.5",phase="opt_cluster"} 371 slot_defence_check_phase_duration_seconds_bucket{le="1",phase="opt_cluster"} 371 slot_defence_check_phase_duration_seconds_bucket{le="2",phase="opt_cluster"} 371 slot_defence_check_phase_duration_seconds_bucket{le="3",phase="opt_cluster"} 371 slot_defence_check_phase_duration_seconds_bucket{le="5",phase="opt_cluster"} 371 slot_defence_check_phase_duration_seconds_bucket{le="8",phase="opt_cluster"} 371 slot_defence_check_phase_duration_seconds_bucket{le="+Inf",phase="opt_cluster"} 371 slot_defence_check_phase_duration_seconds_sum{phase="opt_cluster"} 0.003 slot_defence_check_phase_duration_seconds_count{phase="opt_cluster"} 371 slot_defence_check_phase_duration_seconds_bucket{le="0.01",phase="opt_compute"} 23 slot_defence_check_phase_duration_seconds_bucket{le="0.025",phase="opt_compute"} 87 slot_defence_check_phase_duration_seconds_bucket{le="0.05",phase="opt_compute"} 287 slot_defence_check_phase_duration_seconds_bucket{le="0.1",phase="opt_compute"} 358 slot_defence_check_phase_duration_seconds_bucket{le="0.2",phase="opt_compute"} 370 slot_defence_check_phase_duration_seconds_bucket{le="0.3",phase="opt_compute"} 371 slot_defence_check_phase_duration_seconds_bucket{le="0.5",phase="opt_compute"} 371 slot_defence_check_phase_duration_seconds_bucket{le="1",phase="opt_compute"} 371 slot_defence_check_phase_duration_seconds_bucket{le="2",phase="opt_compute"} 371 slot_defence_check_phase_duration_seconds_bucket{le="3",phase="opt_compute"} 371 slot_defence_check_phase_duration_seconds_bucket{le="5",phase="opt_compute"} 371 slot_defence_check_phase_duration_seconds_bucket{le="8",phase="opt_compute"} 371 slot_defence_check_phase_duration_seconds_bucket{le="+Inf",phase="opt_compute"} 371 slot_defence_check_phase_duration_seconds_sum{phase="opt_compute"} 14.837999999999996 slot_defence_check_phase_duration_seconds_count{phase="opt_compute"} 371 # HELP slot_defence_config_load_duration_seconds Duration of each Postgres-backed cache load, by cache (business_config / business_resolution / api_keys) and outcome (success / failure). Observed when the load settles, including loads that overran CONFIG_LOAD_DEADLINE_MS. cache="api_keys" is the database API key set (WORK-87), not config: a rising failure count there means newly issued database keys are not reaching this process, and on a process where no load has EVER succeeded every database key is refused with a 401. # TYPE slot_defence_config_load_duration_seconds histogram slot_defence_config_load_duration_seconds_bucket{le="0.01",cache="business_resolution",outcome="success"} 100 slot_defence_config_load_duration_seconds_bucket{le="0.025",cache="business_resolution",outcome="success"} 101 slot_defence_config_load_duration_seconds_bucket{le="0.05",cache="business_resolution",outcome="success"} 489 slot_defence_config_load_duration_seconds_bucket{le="0.1",cache="business_resolution",outcome="success"} 555 slot_defence_config_load_duration_seconds_bucket{le="0.25",cache="business_resolution",outcome="success"} 622 slot_defence_config_load_duration_seconds_bucket{le="0.5",cache="business_resolution",outcome="success"} 631 slot_defence_config_load_duration_seconds_bucket{le="1",cache="business_resolution",outcome="success"} 805 slot_defence_config_load_duration_seconds_bucket{le="2.5",cache="business_resolution",outcome="success"} 858 slot_defence_config_load_duration_seconds_bucket{le="5",cache="business_resolution",outcome="success"} 858 slot_defence_config_load_duration_seconds_bucket{le="10",cache="business_resolution",outcome="success"} 858 slot_defence_config_load_duration_seconds_bucket{le="+Inf",cache="business_resolution",outcome="success"} 858 slot_defence_config_load_duration_seconds_sum{cache="business_resolution",outcome="success"} 229.84300000000005 slot_defence_config_load_duration_seconds_count{cache="business_resolution",outcome="success"} 858 slot_defence_config_load_duration_seconds_bucket{le="0.01",cache="business_config",outcome="success"} 547 slot_defence_config_load_duration_seconds_bucket{le="0.025",cache="business_config",outcome="success"} 557 slot_defence_config_load_duration_seconds_bucket{le="0.05",cache="business_config",outcome="success"} 610 slot_defence_config_load_duration_seconds_bucket{le="0.1",cache="business_config",outcome="success"} 630 slot_defence_config_load_duration_seconds_bucket{le="0.25",cache="business_config",outcome="success"} 631 slot_defence_config_load_duration_seconds_bucket{le="0.5",cache="business_config",outcome="success"} 637 slot_defence_config_load_duration_seconds_bucket{le="1",cache="business_config",outcome="success"} 802 slot_defence_config_load_duration_seconds_bucket{le="2.5",cache="business_config",outcome="success"} 865 slot_defence_config_load_duration_seconds_bucket{le="5",cache="business_config",outcome="success"} 865 slot_defence_config_load_duration_seconds_bucket{le="10",cache="business_config",outcome="success"} 865 slot_defence_config_load_duration_seconds_bucket{le="+Inf",cache="business_config",outcome="success"} 865 slot_defence_config_load_duration_seconds_sum{cache="business_config",outcome="success"} 197.03499999999934 slot_defence_config_load_duration_seconds_count{cache="business_config",outcome="success"} 865 slot_defence_config_load_duration_seconds_bucket{le="0.01",cache="api_keys",outcome="success"} 1 slot_defence_config_load_duration_seconds_bucket{le="0.025",cache="api_keys",outcome="success"} 1 slot_defence_config_load_duration_seconds_bucket{le="0.05",cache="api_keys",outcome="success"} 1 slot_defence_config_load_duration_seconds_bucket{le="0.1",cache="api_keys",outcome="success"} 1 slot_defence_config_load_duration_seconds_bucket{le="0.25",cache="api_keys",outcome="success"} 1 slot_defence_config_load_duration_seconds_bucket{le="0.5",cache="api_keys",outcome="success"} 1 slot_defence_config_load_duration_seconds_bucket{le="1",cache="api_keys",outcome="success"} 1 slot_defence_config_load_duration_seconds_bucket{le="2.5",cache="api_keys",outcome="success"} 1 slot_defence_config_load_duration_seconds_bucket{le="5",cache="api_keys",outcome="success"} 1 slot_defence_config_load_duration_seconds_bucket{le="10",cache="api_keys",outcome="success"} 1 slot_defence_config_load_duration_seconds_bucket{le="+Inf",cache="api_keys",outcome="success"} 1 slot_defence_config_load_duration_seconds_sum{cache="api_keys",outcome="success"} 0.003 slot_defence_config_load_duration_seconds_count{cache="api_keys",outcome="success"} 1 slot_defence_config_load_duration_seconds_bucket{le="0.01",cache="business_resolution",outcome="failure"} 0 slot_defence_config_load_duration_seconds_bucket{le="0.025",cache="business_resolution",outcome="failure"} 0 slot_defence_config_load_duration_seconds_bucket{le="0.05",cache="business_resolution",outcome="failure"} 0 slot_defence_config_load_duration_seconds_bucket{le="0.1",cache="business_resolution",outcome="failure"} 0 slot_defence_config_load_duration_seconds_bucket{le="0.25",cache="business_resolution",outcome="failure"} 0 slot_defence_config_load_duration_seconds_bucket{le="0.5",cache="business_resolution",outcome="failure"} 0 slot_defence_config_load_duration_seconds_bucket{le="1",cache="business_resolution",outcome="failure"} 0 slot_defence_config_load_duration_seconds_bucket{le="2.5",cache="business_resolution",outcome="failure"} 7 slot_defence_config_load_duration_seconds_bucket{le="5",cache="business_resolution",outcome="failure"} 7 slot_defence_config_load_duration_seconds_bucket{le="10",cache="business_resolution",outcome="failure"} 7 slot_defence_config_load_duration_seconds_bucket{le="+Inf",cache="business_resolution",outcome="failure"} 7 slot_defence_config_load_duration_seconds_sum{cache="business_resolution",outcome="failure"} 14.013999999999998 slot_defence_config_load_duration_seconds_count{cache="business_resolution",outcome="failure"} 7 slot_defence_config_load_duration_seconds_bucket{le="0.01",cache="business_config",outcome="failure"} 0 slot_defence_config_load_duration_seconds_bucket{le="0.025",cache="business_config",outcome="failure"} 0 slot_defence_config_load_duration_seconds_bucket{le="0.05",cache="business_config",outcome="failure"} 0 slot_defence_config_load_duration_seconds_bucket{le="0.1",cache="business_config",outcome="failure"} 0 slot_defence_config_load_duration_seconds_bucket{le="0.25",cache="business_config",outcome="failure"} 0 slot_defence_config_load_duration_seconds_bucket{le="0.5",cache="business_config",outcome="failure"} 0 slot_defence_config_load_duration_seconds_bucket{le="1",cache="business_config",outcome="failure"} 0 slot_defence_config_load_duration_seconds_bucket{le="2.5",cache="business_config",outcome="failure"} 6 slot_defence_config_load_duration_seconds_bucket{le="5",cache="business_config",outcome="failure"} 6 slot_defence_config_load_duration_seconds_bucket{le="10",cache="business_config",outcome="failure"} 6 slot_defence_config_load_duration_seconds_bucket{le="+Inf",cache="business_config",outcome="failure"} 6 slot_defence_config_load_duration_seconds_sum{cache="business_config",outcome="failure"} 12.02 slot_defence_config_load_duration_seconds_count{cache="business_config",outcome="failure"} 6 # HELP slot_defence_config_load_deadline_exceeded_total Postgres-backed cache loads that ran past CONFIG_LOAD_DEADLINE_MS while a request was waiting, by cache (business_config / business_resolution / api_keys). Counted once per load, not per waiting request. Those requests were served the previous value (or empty, if the cache had never loaded). On cache="api_keys" the previous value is the last-known database key set, so a sustained rate means authentication is being answered from a stale set: a key issued since the last successful load is refused with a 401 until a load completes. Pair it with the last-success timestamp below, which says how wide that window is. # TYPE slot_defence_config_load_deadline_exceeded_total counter slot_defence_config_load_deadline_exceeded_total{cache="business_resolution"} 309 slot_defence_config_load_deadline_exceeded_total{cache="business_config"} 240 # HELP slot_defence_config_last_load_success_timestamp_seconds Unix time of the last successful Postgres-backed cache load, by cache (business_config / business_resolution / api_keys). Absent until the first success. time() minus this, well beyond ADMIN_CONFIG_CACHE_TTL_MS while traffic flows, means the process is serving stale config because refreshes are failing. On cache="api_keys" that age is exactly the window of database keys this process will refuse — any key issued after this instant is not in the set it matches against — and an ABSENT api_keys series on a process booted with FEATURE_DB_API_KEYS on means no load has ever succeeded, so EVERY database key is being refused (the boot log line api_key_store_warmed, loaded:false, is the same fact). # TYPE slot_defence_config_last_load_success_timestamp_seconds gauge slot_defence_config_last_load_success_timestamp_seconds{cache="business_resolution"} 1791807550.104 slot_defence_config_last_load_success_timestamp_seconds{cache="business_config"} 1791807550.151 slot_defence_config_last_load_success_timestamp_seconds{cache="api_keys"} 1790765395.595 # HELP slot_defence_anchor_reconcile_total Anchor-index reconciliations against the durable record, by outcome. # TYPE slot_defence_anchor_reconcile_total counter # HELP slot_defence_anchor_reconcile_missing_total Anchors the durable record expected but the index did not hold. Non-zero means held slots have silently reopened. # TYPE slot_defence_anchor_reconcile_missing_total counter slot_defence_anchor_reconcile_missing_total 0 # HELP slot_defence_admin_receipts_reads_total Total GET /admin/receipts reads, by outcome. ok = at least one receipt returned; empty = none for the queried business+day; persistence_off = the Postgres read model is disabled so receipts can never populate. # TYPE slot_defence_admin_receipts_reads_total counter slot_defence_admin_receipts_reads_total{outcome="empty"} 3 slot_defence_admin_receipts_reads_total{outcome="ok"} 5 # HELP slot_defence_slots_precheck_answers_total Total GET /slots/precheck answers, by reason. window_available = at least one deliverable hour has capacity (valid); caller_not_configured / zone_ungoverned = nothing caps the zone (valid); all_windows_at_cap = every deliverable hour is at cap (invalid); no_deliverable_hours = the business delivers in no hour of that day (invalid). # TYPE slot_defence_slots_precheck_answers_total counter slot_defence_slots_precheck_answers_total{reason="window_available"} 30689 slot_defence_slots_precheck_answers_total{reason="zone_ungoverned"} 1343 # HELP slot_defence_active_anchor_resolutions_total Per-slot resolutions of activeAnchorMatch on POST /check, by what produced it. `real_anchor` = the zone anchor lookup found a committed order or dispatched driver; `hub_fallback` = no real anchor, and the business's configured cold-start hub coordinate covered the customer (activeAnchorMatch is true on config alone); `no_match` = neither, activeAnchorMatch is false. # TYPE slot_defence_active_anchor_resolutions_total counter slot_defence_active_anchor_resolutions_total{source="no_match"} 19927 slot_defence_active_anchor_resolutions_total{source="real_anchor"} 241 # HELP slot_defence_admin_team_invites_total Total admin invite attempts, by surface (team_page = the business owner's own Team page; super_console = a Slotster operator) and outcome. invited_delivered = row created and the invite email went out; invited_undelivered = row created, SMTP refused it (the admin still exists and can request a link themselves); cap_reached = the business is at its member_limit; email_conflict = that address already belongs to an admin somewhere; business_not_found = the business id did not resolve. # TYPE slot_defence_admin_team_invites_total counter slot_defence_admin_team_invites_total{surface="team_page",outcome="invited_delivered"} 1 # HELP slot_defence_admin_team_removals_total Total team-member removal attempts on DELETE /admin/team/members/:adminUserId, by outcome. removed = the row went and credentials were swept; not_found = no such member on that business (also the answer for another business's member); owner_target / super_target = a protected admin the endpoint refuses to remove. No surface label: removal exists on the Team page only. # TYPE slot_defence_admin_team_removals_total counter # HELP slot_defence_admin_credential_sweeps_total Total per-admin credential sweeps performed during team-member removal, by credential (session / magic_link) and result. revoked = the index listed digests and at least one credential key was actually deleted; index_empty = the admin held no indexed credential (expected for one minted before WORK-85, or for an admin who never signed in); index_stale = the index listed digests but no credential key existed under them. Since WORK-87 a spent link and a signed-out session evict their own digest, so this is no longer the ordinary reading. Correlate before escalating: the likeliest cause is an eviction that already failed and logged admin_credential_index_evict_failed for that admin, leaving an inert orphan digest; admin_credential_sweep_stale carries the admin_user_id. It is NOT a key-prefix misconfiguration — REDIS_KEY_PREFIX moves credential keys and index sets together and would read as index_empty. # TYPE slot_defence_admin_credential_sweeps_total counter # HELP slot_defence_flip_transitions_detected_total Total genuine (zone, hourWindow) volume-defence capacity transitions detected on POST /check while FEATURE_FLIP_TRIGGERED_PUSH is on, by direction (closed / reopened). The denominator for slot_defence_flip_webhook_total — the two should stay equal. # TYPE slot_defence_flip_transitions_detected_total counter # HELP slot_defence_flip_webhook_total Total outbound flip-triggered-push webhook attempts to cotta-backend, by outcome. `sent` = 2xx; `non_2xx` = reached the target, it refused; the rest are transport failures (timeout / dns_error / conn_refused / tls_error / conn_reset / network_error / redirect_refused); `skipped_no_config` = the flag is on but cottaWebhook config is null, i.e. misconfiguration. # TYPE slot_defence_flip_webhook_total counter