From 73f0b5e69db53cd7b02900f2b081ddc0ef57a456 Mon Sep 17 00:00:00 2001 From: Matt Mabrey Date: Sat, 17 Oct 2026 14:00:52 -0700 Subject: [PATCH] T14.1 (D26): a sign-in problem never destroys a queued change A 401 holds the queue: nothing is marked dead, no try is counted, and only wp-auth-ready releases it. A 403 marks only that change dead, with a reason. qWrite reports failure; a change the device cannot store stays in memory and the badge says so. Each change has an owner and is sent only under that person. tests/outbox_check.py (26 checks) fails against the old 401 rule. --- html/project-data.js | 148 +++++++++++++++++++++------ tests/outbox_check.py | 226 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 341 insertions(+), 33 deletions(-) create mode 100644 tests/outbox_check.py diff --git a/html/project-data.js b/html/project-data.js index 6447383..2a9bcd7 100644 --- a/html/project-data.js +++ b/html/project-data.js @@ -245,25 +245,63 @@ // local cache still updates immediately, so rendering never waits on the network. var OUTBOX_KEY = 'wp_sync_outbox_v1'; var _flushTimer = null, _backoff = 0, _flushing = false; + // D26. `_held` is set when the server answers 401: the session ended, so the + // queue waits for a good sign-in. It is cleared ONLY by `wp-auth-ready` (a page + // load with a good session check), never by the online event, the timer or tab + // focus, because none of those prove the session is back. + var _held = false; + // D26. Changes the device could not store (localStorage full) stay here, in page + // memory, and are still sent. They are lost if the page closes, so the badge says + // they are not saved on this device. They are never dropped without a word. + var _mem = []; + var _deviceFull = false; function qRead() { try { return JSON.parse(localStorage.getItem(OUTBOX_KEY) || '[]') || []; } catch (e) { return []; } } - function qWrite(list) { try { localStorage.setItem(OUTBOX_KEY, JSON.stringify(list)); } catch (e) {} } + // Returns true when the list is stored. A storage error used to be swallowed here, + // so a full device lost the change and nobody knew (D26, fault 2). + function qWrite(list) { + try { localStorage.setItem(OUTBOX_KEY, JSON.stringify(list)); return true; } + catch (e) { return false; } + } + // Stored ops plus any held only in memory, stored ones first (send order). + function qAll() { + var q = qRead(); + if (!_mem.length) return q; + var ids = {}; + q.forEach(function (o) { ids[o.opId] = true; }); + return q.concat(_mem.filter(function (o) { return !ids[o.opId]; })); + } + // Remove one op everywhere it can be held. + function qDrop(opId) { + _mem = _mem.filter(function (o) { return o.opId !== opId; }); + qWrite(qRead().filter(function (o) { return o.opId !== opId; })); + } + function sameUser(a, b) { return String(a || '').toLowerCase() === String(b || '').toLowerCase(); } // Append an op, coalescing by (kind,key) so only the latest write per entity is - // queued. A delete supersedes any pending upsert for the same id. + // queued. A delete supersedes any pending upsert for the same id. Returns true + // when the change is stored on this device, false when it is only in memory. function enqueue(op) { - var q = qRead(); - if (op.kind === 'wp-del') { - q = q.filter(function (o) { return !(o.key === op.key && (o.kind === 'wp' || o.kind === 'wp-del')); }); - } else { - q = q.filter(function (o) { return !(o.kind === op.kind && o.key === op.key); }); - } + var supersedes = (op.kind === 'wp-del') + ? function (o) { return o.key === op.key && (o.kind === 'wp' || o.kind === 'wp-del'); } + : function (o) { return o.kind === op.kind && o.key === op.key; }; + var q = qRead().filter(function (o) { return !supersedes(o); }); + _mem = _mem.filter(function (o) { return !supersedes(o); }); op.opId = op.kind + ':' + op.key + ':' + Date.now().toString(36) + Math.random().toString(36).slice(2, 6); op.tries = 0; + op.owner = currentUser(); // D26: sent only when this person is signed in q.push(op); - qWrite(q); + var stored = qWrite(q); + if (stored) { + // Room again: bring anything held in memory onto the device. + if (_mem.length && qWrite(q.concat(_mem))) _mem = []; + _deviceFull = _mem.length > 0; + } else { + _mem.push(op); _deviceFull = true; + } notifySync(); scheduleFlush(0); + return stored; } function opRequest(op) { @@ -275,41 +313,63 @@ }); } + // Apply `fn` to the op with this id, wherever it is held (device or memory). + function patchOp(opId, fn) { + var q = qRead(), hit = false; + for (var i = 0; i < q.length; i++) { if (q[i].opId === opId) { fn(q[i]); hit = true; break; } } + if (hit) qWrite(q); + for (var j = 0; j < _mem.length; j++) { if (_mem[j].opId === opId) { fn(_mem[j]); break; } } + } function bumpTries(opId, err) { - var q = qRead(); - for (var i = 0; i < q.length; i++) { if (q[i].opId === opId) { q[i].tries = (q[i].tries || 0) + 1; q[i].lastErr = err; break; } } - qWrite(q); + patchOp(opId, function (o) { o.tries = (o.tries || 0) + 1; o.lastErr = err; }); } // Permanently-failed op (a 4xx client error) — keep it for visibility but stop // retrying, so a rejected write can't loop forever. `err` is the server's own // explanation when it sent one; it is what the sync badge shows the user. function markDead(opId, err) { - var q = qRead(); - for (var i = 0; i < q.length; i++) { if (q[i].opId === opId) { q[i].dead = true; q[i].lastErr = err; break; } } - qWrite(q); + patchOp(opId, function (o) { o.dead = true; o.lastErr = err; }); } - // Attempt every live op; successes are removed, 4xx client errors are marked - // dead (won't succeed on retry), transient failures (429/5xx/network) stay queued. + // True when this op is waiting for another person to sign in (D26). An op with no + // owner (queued before the sign-in check finished) can be sent by anyone, which is + // what every op did before owners existed. + function foreignOp(o) { + var me = currentUser(); + return !!o.owner && !sameUser(o.owner, me); + } + function liveOps() { return qAll().filter(function (o) { return !o.dead; }); } + function sendableOps() { return liveOps().filter(function (o) { return !foreignOp(o); }); } + + // Attempt every live op that belongs to the signed-in person; successes are + // removed. A 401 HOLDS the queue (D26): the session ended, which says nothing + // about the changes, so nothing is marked dead and no try is counted. A 403 or + // other 4xx is marked dead (it will not succeed on retry); transient failures + // (429/5xx/network) stay queued. function flush() { if (_flushing) return Promise.resolve(); - var q = qRead().filter(function (o) { return !o.dead; }); + if (_held) { notifySync(); return Promise.resolve(); } + var q = sendableOps(); if (!q.length) { notifySync(); return Promise.resolve(); } _flushing = true; notifySync(); - var chain = Promise.resolve(), anyFail = false; + var chain = Promise.resolve(), anyFail = false, halted = false; q.forEach(function (op) { chain = chain.then(function () { + if (halted) return; // a 401 stops the run; later ops keep their place return opRequest(op).then(function (r) { var status = r ? r.status : 0; var done = r && (r.ok || (op.kind === 'wp-del' && status === 404)); // 404 on delete = already gone - if (done) { qWrite(qRead().filter(function (o) { return o.opId !== op.opId; })); } + if (done) { qDrop(op.opId); } + else if (status === 401) { halted = true; _held = true; } else if (status >= 400 && status < 500 && status !== 429) { // Refused once, refused forever — so the only useful thing left is the // reason. A 409 here is the archived-project gate, whose detail tells // the user the project is read-only and how to get it unarchived; a // bare "HTTP 409" would leave them staring at a change that vanished. return r.json().catch(function () { return null; }).then(function (j) { - var why = (j && typeof j.detail === 'string' && j.detail) || ('HTTP ' + status); + var detail = (j && typeof j.detail === 'string' && j.detail) || ''; + var why = status === 403 + ? 'You no longer have access to this project.' + (detail ? ' (' + detail + ')' : '') + : (detail || ('HTTP ' + status)); markDead(op.opId, why); }); } @@ -320,7 +380,8 @@ return chain.then(function () { _flushing = false; notifySync(); - if (qRead().filter(function (o) { return !o.dead; }).length) { + if (_held) return; // wait for wp-auth-ready; no timer retries a dead session + if (sendableOps().length) { _backoff = anyFail ? Math.min((_backoff || 5000) * 2, 60000) : 0; scheduleFlush(_backoff || 15000); } else { _backoff = 0; } @@ -336,14 +397,21 @@ // `failed` stays the total not-getting-through count (what listeners already // read); `dead` splits out the ops the server has permanently refused, with the // first reason it gave, because those two states need different words. + // D26 adds: `held` (the session ended; the queue waits for a sign-in), `others` + // and `otherOwner` (changes that belong to someone else, waiting for them), + // `deviceFull` and `unsaved` (changes that exist only in page memory). function syncCounts() { - var q = qRead(), pending = 0, failed = 0, dead = 0, reason = ''; + var q = qAll(), pending = 0, failed = 0, dead = 0, reason = '', others = 0, otherOwner = ''; for (var i = 0; i < q.length; i++) { if (q[i].dead) { dead++; if (!reason && q[i].lastErr) reason = String(q[i].lastErr); } + else if (foreignOp(q[i])) { others++; if (!otherOwner) otherOwner = String(q[i].owner); } else if ((q[i].tries || 0) >= 3) failed++; else pending++; } - return { pending: pending, failed: failed + dead, dead: dead, reason: reason, syncing: _flushing }; + return { + pending: pending, failed: failed + dead, dead: dead, reason: reason, syncing: _flushing, + held: _held, others: others, otherOwner: otherOwner, deviceFull: _deviceFull, unsaved: _mem.length + }; } ProjectData.syncStatus = syncCounts; function notifySync() { @@ -395,7 +463,15 @@ // Every string here now names the project explicitly. The draft indicator // (WPAutosave.mountIndicator) is the one that speaks for the form, and the two // can no longer be read as each other. - if (c.dead) { + // D26. These three are the states that used to be invisible or wrong. T14.3 + // gives them their final wording and controls; until then they are plain text. + if (c.deviceFull) { + el.textContent = '✕ ' + c.unsaved + ' change' + (c.unsaved === 1 ? ' is' : 's are') + ' not saved on this device. Keep this page open until it is sent to the project.'; + el.style.color = 'var(--wp-status-error-text)'; el.style.borderColor = 'var(--wp-status-error-border-a)'; el.style.background = 'var(--wp-status-error-bg)'; el.style.display = 'inline-flex'; + } else if (c.held) { + el.textContent = '⚠ Sign in again to send your changes to the project. They are kept on this device.'; + el.style.color = 'var(--wp-status-warning-text)'; el.style.borderColor = 'var(--cds-support-warning)'; el.style.background = 'var(--wp-status-warning-bg)'; el.style.display = 'inline-flex'; + } else if (c.dead) { el.innerHTML = '✕ ' + c.dead + ' change' + (c.dead === 1 ? '' : 's') + ' rejected by the project — not saved' + (c.reason ? '' + esc(c.reason) + '' : ''); el.style.color = 'var(--wp-status-error-text)'; el.style.borderColor = 'var(--wp-status-error-border-a)'; el.style.background = 'var(--wp-status-error-bg)'; el.style.display = 'inline-flex'; @@ -405,6 +481,9 @@ } else if (c.pending) { el.textContent = '↻ Sending ' + c.pending + ' change' + (c.pending === 1 ? '' : 's') + ' to the project…'; el.style.color = 'var(--cds-text-secondary)'; el.style.borderColor = 'var(--cds-border-subtle)'; el.style.background = 'var(--cds-layer)'; el.style.display = 'inline-flex'; + } else if (c.others) { + el.textContent = '⚠ ' + c.others + ' change' + (c.others === 1 ? ' belongs' : 's belong') + ' to ' + c.otherOwner + '. ' + (c.others === 1 ? 'It goes' : 'They go') + ' to the project when ' + c.otherOwner + ' signs in.'; + el.style.color = 'var(--wp-status-warning-text)'; el.style.borderColor = 'var(--cds-support-warning)'; el.style.background = 'var(--wp-status-warning-bg)'; el.style.display = 'inline-flex'; } else { el.textContent = '✓ Everything sent to the project'; el.style.color = 'var(--wp-status-success-text)'; el.style.borderColor = 'var(--wp-status-success-border-a)'; el.style.background = 'var(--wp-status-success-bg)'; el.style.display = 'inline-flex'; @@ -418,7 +497,11 @@ window.addEventListener('online', function () { _backoff = 0; scheduleFlush(0); }); window.addEventListener('storage', function (e) { if (e.key === OUTBOX_KEY) { notifySync(); scheduleFlush(0); } }); document.addEventListener('visibilitychange', function () { if (!document.hidden) scheduleFlush(0); }); - setInterval(function () { if (qRead().filter(function (o) { return !o.dead; }).length) scheduleFlush(0); }, 20000); + setInterval(function () { if (sendableOps().length) scheduleFlush(0); }, 20000); + // D26: the one thing that releases a held queue. auth-guard fires this after a + // good /api/auth/me (or the offline cache), and the next flush finds out for + // itself whether the session is really good: a 401 holds the queue again. + document.addEventListener('wp-auth-ready', function () { _held = false; _backoff = 0; notifySync(); scheduleFlush(0); }); } catch (e) {} if (document.readyState === 'loading') { document.addEventListener('DOMContentLoaded', function () { notifySync(); scheduleFlush(0); }); } else { setTimeout(function () { notifySync(); scheduleFlush(0); }, 0); } @@ -428,7 +511,7 @@ // project so re-completing updates the same row. ProjectData.pushSOP = function (projectId, sop, state) { if (!projectId) return Promise.resolve(null); - enqueue({ + return Promise.resolve(enqueue({ kind: 'sop', key: 'sop__' + projectId, body: { id: 'sop__' + projectId, project_id: projectId, @@ -436,22 +519,21 @@ number: (sop && sop.project && sop.project.number) || '', complete: true, created_by: currentUser(), data: { sop: sop, state: state } } - }); - return Promise.resolve(true); + })); }; // Upsert a single Work Package. The local cache stays the source of truth for // immediate rendering; the outbox guarantees the write reaches the server. ProjectData.pushWP = function (p, projectId) { if (!p || !p.id) return Promise.resolve(null); - enqueue({ kind: 'wp', key: p.id, body: pkgToServer(p, projectId) }); - return Promise.resolve(true); + // D26: resolves false when the device could not store the change (it is still + // sent from memory, but it is lost if the page closes first). + return Promise.resolve(enqueue({ kind: 'wp', key: p.id, body: pkgToServer(p, projectId) })); }; ProjectData.removeWP = function (id) { if (!id) return Promise.resolve(); - enqueue({ kind: 'wp-del', key: id }); - return Promise.resolve(true); + return Promise.resolve(enqueue({ kind: 'wp-del', key: id })); }; // Force a flush now and resolve when the queue drains (or a round-trip is done). diff --git a/tests/outbox_check.py b/tests/outbox_check.py new file mode 100644 index 0000000..b42e7f1 --- /dev/null +++ b/tests/outbox_check.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +"""Does a sign-in problem destroy queued work? - D26 / T14.1. + +The outbox in html/project-data.js queues every save and sends it later. It used to +treat a 401 (the session ended) like any other 4xx: the change was marked dead and +never sent again. It also hid a full device store and sent a change under whoever +was signed in. This check drives the real page in a real browser. It replaces +window.fetch for the write endpoints only, so no real server state is needed. + + 1. a 401 holds the whole queue: nothing dead, no try counted, state is "held" + 2. online, tab focus and the timer do not release a held queue + 3. a good sign-in (wp-auth-ready) sends every change, in the original order + 4. a 403 kills only that change and says why + 5. a full device store is reported to the caller and the badge; nothing is dropped + 6. a change is sent only under the person who made it + 7. auth-guard.js never deletes the outbox + +Exit 0 all passed, 1 a failure, 2 could not run. +""" +import json +import os +import re +import subprocess +import sys +import tempfile +import time + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import cdp # noqa: E402 +from browser_check import seed, start_server, chk, _PASS, _FAIL # noqa: E402 + +KEY = "wp_sync_outbox_v1" +HERE = os.path.dirname(os.path.abspath(__file__)) +HTML = os.path.join(os.path.dirname(HERE), "html") + +# Replaces fetch for POST /api/wps, POST /api/sops and DELETE /api/wps/ only. +# window.__plan maps a work package id to a status; the default is window.__status. +STUB = """(() => { + if (!window.__realFetch) window.__realFetch = window.fetch.bind(window); + window.__calls = []; window.__status = 200; window.__plan = {}; + window.fetch = function (url, opts) { + const m = ((opts && opts.method) || 'GET').toUpperCase(); + const u = String(url); + if ((m === 'POST' && /\\/api\\/(wps|sops)$/.test(u)) || (m === 'DELETE' && /\\/api\\/wps\\//.test(u))) { + let id = ''; + try { id = JSON.parse(opts.body).id; } catch (e) { id = decodeURIComponent(u.split('/').pop()); } + window.__calls.push(id); + const st = (id in window.__plan) ? window.__plan[id] : window.__status; + return Promise.resolve(new Response(JSON.stringify(st >= 400 ? { detail: 'stub ' + st } : { ok: true }), + { status: st, headers: { 'Content-Type': 'application/json' } })); + } + return window.__realFetch(url, opts); + }; + localStorage.removeItem('wp_sync_outbox_v1'); + return true; +})()""" + + +def boot(page, base, tok, user="root"): + page.clear_cookies() + page.set_cookie("wp_session", tok[user]) + page.goto(base + "/field.html") + for _ in range(40): + if page.eval("!!(window.WP_USER && window.ProjectData)"): + break + time.sleep(0.3) + time.sleep(0.5) + page.eval(STUB) + + +def queue(page): + return json.loads(page.eval("localStorage.getItem(%r) || '[]'" % KEY)) + + +def status(page): + return page.eval("ProjectData.syncStatus()") + + +def push(page, ids): + page.eval("(() => { %s; return true; })()" % ";".join( + "ProjectData.pushWP({id:%r, name:%r}, 'projA')" % (i, i) for i in ids)) + + +def main(): + exe = cdp.find_browser() + if not exe: + print("no headless-capable browser found; set WP_BROWSER.") + return 2 + tmpdir = tempfile.mkdtemp(prefix="wpsuite-outbox-") + db_path = os.path.join(tmpdir, "check.db") + server = None + try: + tok = seed(db_path) + port = cdp.free_port() + base = "http://127.0.0.1:%d" % port + server = start_server(port, db_path) + if server is None: + print("the test server would not start.") + return 2 + print("\nThe outbox keeps every change - D26 / T14.1\nTarget: %s" % base) + browser = cdp.Browser(exe) + page = browser.page() + try: + ids = ["wp_o1", "wp_o2", "wp_o3"] + + print("\n1. a 401 holds the whole queue") + boot(page, base, tok) + page.eval("window.__status = 401") + push(page, ids) + page.eval("ProjectData.flushSync()") + q = queue(page) + st = status(page) + chk("all 3 changes are still queued", [o["key"] for o in q] == ids, str([o["key"] for o in q])) + chk("none is marked dead", not any(o.get("dead") for o in q) and st["dead"] == 0) + chk("no try was counted", all((o.get("tries") or 0) == 0 for o in q), str([o.get("tries") for o in q])) + chk("the state is held", st["held"] is True, json.dumps(st)) + chk("the run stopped at the first 401", len(page.eval("window.__calls")) == 1, + str(page.eval("window.__calls"))) + badge = page.eval("document.getElementById('wp-sync-badge').textContent") + chk("the badge asks for a new sign-in", "Sign in again" in badge, badge) + + print("\n2. only a good sign-in releases it") + page.eval("window.__status = 200") + page.eval("window.dispatchEvent(new Event('online'))") + page.eval("document.dispatchEvent(new Event('visibilitychange'))") + page.eval("ProjectData.flushSync()") + time.sleep(1.0) + chk("online, focus and a manual flush send nothing while held", + len(page.eval("window.__calls")) == 1 and len(queue(page)) == 3, + str(page.eval("window.__calls"))) + + print("\n3. a good sign-in sends all of them, in order") + page.eval("document.dispatchEvent(new CustomEvent('wp-auth-ready', {detail: window.WP_USER}))") + for _ in range(30): + if not queue(page): + break + time.sleep(0.3) + calls = page.eval("window.__calls") + chk("the queue is empty", queue(page) == [], str(queue(page))) + chk("the changes arrived in the original order", calls[1:] == ids or calls[-3:] == ids, str(calls)) + chk("the state is no longer held", status(page)["held"] is False) + + print("\n4. a 403 kills only that change, with a reason") + boot(page, base, tok) + page.eval("window.__plan = {wp_o2: 403}") + push(page, ids) + page.eval("ProjectData.flushSync()") + q = queue(page) + dead = [o for o in q if o.get("dead")] + chk("only wp_o2 is dead", [o["key"] for o in dead] == ["wp_o2"], str([o["key"] for o in q])) + chk("the others were sent", [o["key"] for o in q if not o.get("dead")] == []) + chk("the reason names the cause", dead and "no longer have access" in dead[0].get("lastErr", ""), + dead[0].get("lastErr", "") if dead else "") + chk("a 403 does not hold the queue", status(page)["held"] is False) + + print("\n5. a full device store is never silent") + boot(page, base, tok) + page.eval("window.__status = 500") # keep it queued so the badge state stays visible + page.eval("""(() => { + window.__setItem = Storage.prototype.setItem; + Storage.prototype.setItem = function (k, v) { + if (k === 'wp_sync_outbox_v1') { const e = new Error('full'); e.name = 'QuotaExceededError'; throw e; } + return window.__setItem.call(this, k, v); + }; + return true; })()""") + stored = page.eval("ProjectData.pushWP({id:'wp_full', name:'full'}, 'projA')") + st = status(page) + chk("the caller is told it was not stored", stored is False, repr(stored)) + chk("the state says device full, 1 unsaved", st["deviceFull"] is True and st["unsaved"] == 1, json.dumps(st)) + badge = page.eval("document.getElementById('wp-sync-badge').textContent") + chk("the badge says it is not saved on this device", "not saved on this device" in badge, badge) + page.eval("window.__status = 200") + page.eval("ProjectData.flushSync()") + chk("the change is still sent from memory", "wp_full" in page.eval("window.__calls"), + str(page.eval("window.__calls"))) + page.eval("Storage.prototype.setItem = window.__setItem") + chk("the state clears once it is sent", status(page)["unsaved"] == 0) + + print("\n6. a change is sent only under the person who made it") + boot(page, base, tok) + page.eval("window.__status = 503") # keep the change queued while we look at it + push(page, ["wp_owned"]) + time.sleep(0.5) + page.eval("window.__status = 200") + q0 = queue(page) + owner = (q0[0].get("owner") if q0 else None) + chk("the change records its owner", owner == "root", repr(owner)) + page.eval("window.__calls = []; window.__me = window.WP_USER; window.WP_USER = {username: 'pat'}") + page.eval("ProjectData.flushSync()") + st = status(page) + chk("person B sends nothing", page.eval("window.__calls") == [], str(page.eval("window.__calls"))) + chk("the change is kept", [o["key"] for o in queue(page)] == ["wp_owned"]) + chk("the badge names the owner", st["others"] == 1 and st["otherOwner"] == "root", json.dumps(st)) + page.eval("window.WP_USER = window.__me") + page.eval("ProjectData.flushSync()") + chk("it is sent after the owner signs in", page.eval("window.__calls") == ["wp_owned"] + and queue(page) == [], str(page.eval("window.__calls"))) + finally: + page.close() + browser.close() + + print("\n7. auth-guard.js never deletes the outbox") + src = open(os.path.join(HTML, "auth-guard.js"), encoding="utf-8").read() + code = re.sub(r"/\*.*?\*/|//[^\n]*", "", src, flags=re.S) + chk("no code path touches the outbox key", KEY not in code) + chk("nothing clears all of localStorage", "localStorage.clear" not in code) + finally: + if server is not None: + server.kill() + try: + server.wait(timeout=10) + except subprocess.TimeoutExpired: + pass + try: + from server.db import engine + engine.dispose() + except Exception: + pass + print("\n%d passed, %d failed" % (len(_PASS), len(_FAIL))) + return 1 if _FAIL else 0 + + +if __name__ == "__main__": + sys.exit(main())