{ "slug": "postmortem", "baseline_sha256": "336ec7b129f1ade44e1b36c505f1efec847737651cbf6496cf29f3258ea665c2", "preview_sha256": "2b4660e588413516dd26f01e32123256911ac2cefd9bc3428809d533f78512b3", "source_text_nodes": 102, "text_node_multiset_equal": true, "missing_text_nodes": [], "added_text_nodes": [], "numeric_multiset_equal": true, "numeric_occurrences": 104, "keyed_record_checks": { "//body//p": true, "//body//h1": true, "//body//h2": true, "//body//h3": true, "//body//footer": true, "//body//dt": true, "//body//dd": true, "//body//th": true, "//body//td": true, "//body//figcaption": true, "//body//code": true, "impact": true, "timeline": true, "factors": true, "actions": true, "action_state_classes": true }, "links_unchanged": true, "source_svg_byte_identical": true, "svg_count": 1, "meter_style_values_unchanged": true, "added_labels": { "visible_DOM_text": [], "CSS_mobile_labels": { "ID": 4, "Action": 4, "Owner": 4, "Due": 4, "Status": 4 }, "note": "Repeated existing table column headings via data-mobile-label/::before; no new factual copy. Only relevant labels are painted at mobile. All original table header text remains in the semantic DOM." }, "all_pass": true, "original_records": { "//body//p": [ "\"Scale up\" is right for CPU saturation and harmful for pool exhaustion. It cost 72 minutes and raised the failure rate.", "09:14 – 13:01 UTC", "18% of active accounts", "39,600 recovered by retry", "A dependency bump changes the payment-provider request timeout from 2s to 30s. All six regions by 09:11, no alarms.", "A dependency bump raised the payment-provider timeout from two seconds to thirty. When the provider slowed, every stuck request held its connection fifteen times longer and the pool saturated — three lines inside a forty-one file pull request.", "A diff against 2026.9.2 surfaced the timeout constant at 11:38 and the rollback completes in the last region. Success rate returns above 99%; a retry job recovers 39,600 of 41,900 checkouts by 13:41.", "A p99 blowout raises no alert. Thirty-three minutes passed before a success-rate alert caught it indirectly.", "A pool's throughput is capacity divided by hold time. At two seconds a stuck request freed its connection fast enough that a slow provider cost latency and nothing more; at thirty it holds fifteen times longer, so the pool saturates at a fifteenth of the load it was sized for. The slowdown itself was ordinary — twice last quarter, neither an incident. What changed was our tolerance.", "A timeout raised from 2s to 30s turned a slow dependency into an outage.", "Gridlines 0/30/60%. Dashed: detected 09:47 at 6%, mitigated 13:01. Peak 52% — nine times worse than when the alert fired.", "No double charges", "Per the runbook, on-call scales pods from 12 to 40. Each opens connections against the same provider-side ceiling; the failure rate climbs from 28% to 41%.", "Provider p99 latency rises from 240ms to 4.8s. Each slow request now holds a connection for up to 30s and the 200-connection pool begins to queue in eu-west.", "The success-rate alert fires after five minutes below 95%. No latency alert ever fired: the SLO is measured at p50, which never moved.", "Three lines in a forty-one file pull request, no note in the description, approved in under four minutes." ], "//body//h1": [ "Checkout was unavailable for 3 hours 47 minutes on 12 September" ], "//body//h2": [ "Contributing factors", "Impact", "Root cause", "Timeline · 12 September, UTC", "What went well" ], "//body//h3": [ "A behaviour change hid inside a dependency bump", "Detected33 min to detect", "Impact beginsImpact start", "Mitigated3h 47m total", "Release 2026.9.3 rolls out", "Scaling up makes it worse", "The latency SLO is measured at p50", "The runbook's first move is the wrong move here" ], "//body//footer": [ "Blameless review held 18 September, sixteen attendees from Payments, Platform and Support. Timestamps come from the incident channel and the deploy log; failure rates are the per-minute success-rate metric." ], "//body//dt": [ "Authors", "Customers", "Data loss", "Duration", "Failed checkouts", "Incident", "Reviewed", "Status" ], "//body//dd": [ "12,400", "18 Sep 2026", "3h 47m", "41,900", "D. Kaur, R. Idowu", "INC-2026-041", "None", "Resolved" ], "//body//th": [ "Action", "Due", "ID", "Owner", "Status" ], "//body//td": [ "10 Oct", "19 Sep", "26 Sep", "3 Oct", "AI-1", "AI-2", "AI-3", "AI-4", "Add a circuit breaker to the provider client, opening at 25% errors over 30 seconds.", "Alert on p99 checkout latency and on success rate, not p50 alone.", "D. Kaur", "Done", "In progress", "M. Halvorsen", "Not started", "Not started", "Platform", "R. Idowu", "Rewrite the checkout runbook to diagnose saturation before scaling, pool metrics first.", "Treat timeout, retry and pool constants as behaviour changes in review — CODEOWNERS plus a required note." ], "//body//figcaption": [ "Checkout failure rate · 0–60% · 09:00–14:00 UTC" ], "//body//code": [ "2s", "30s" ], "impact": [ "Duration3h 47m09:14 – 13:01 UTC", "Customers12,40018% of active accounts", "Failed checkouts41,90039,600 recovered by retry", "Data lossNoneNo double charges" ], "timeline": [ [ "09:02", "Release 2026.9.3 rolls out", "A dependency bump changes the payment-provider request timeout from 2s to 30s. All six regions by 09:11, no alarms.", null ], [ "09:14", "Impact beginsImpact start", "Provider p99 latency rises from 240ms to 4.8s. Each slow request now holds a connection for up to 30s and the 200-connection pool begins to queue in eu-west.", "key" ], [ "09:47", "Detected33 min to detect", "The success-rate alert fires after five minutes below 95%. No latency alert ever fired: the SLO is measured at p50, which never moved.", "key" ], [ "10:26", "Scaling up makes it worse", "Per the runbook, on-call scales pods from 12 to 40. Each opens connections against the same provider-side ceiling; the failure rate climbs from 28% to 41%.", null ], [ "13:01", "Mitigated3h 47m total", "A diff against 2026.9.2 surfaced the timeout constant at 11:38 and the rollback completes in the last region. Success rate returns above 99%; a retry job recovers 39,600 of 41,900 checkouts by 13:41.", "key" ] ], "factors": [ "01The latency SLO is measured at p50 A p99 blowout raises no alert. Thirty-three minutes passed before a success-rate alert caught it indirectly.", "02A behaviour change hid inside a dependency bump Three lines in a forty-one file pull request, no note in the description, approved in under four minutes.", "03The runbook's first move is the wrong move here \"Scale up\" is right for CPU saturation and harmful for pool exhaustion. It cost 72 minutes and raised the failure rate." ], "actions": [ [ "AI-1", "Alert on p99 checkout latency and on success rate, not p50 alone.", "M. Halvorsen", "19 Sep", "Done" ], [ "AI-2", "Add a circuit breaker to the provider client, opening at 25% errors over 30 seconds.", "R. Idowu", "26 Sep", "In progress" ], [ "AI-3", "Treat timeout, retry and pool constants as behaviour changes in review — CODEOWNERS plus a required note.", "Platform", "3 Oct", "Not started" ], [ "AI-4", "Rewrite the checkout runbook to diagnose saturation before scaling, pool metrics first.", "D. Kaur", "10 Oct", "Not started" ] ], "action_state_classes": [ "st st--done", "st st--wip", "st", "st" ] } }