T13.3: Okta deprovisioning sync job, fail closed (D18)

run_sync() compares the Okta user list (T13.2) to active local accounts and
disables the ones Okta no longer vouches for. One AuditLog row per change,
actor system:okta_sync, action user_deprovisioned_by_sync.

Fail closed: any exception from the fetch, an empty or untrusted answer, or a
failed save changes nothing and writes nothing. The fetch runs before the
database is read, so a failed fetch has no path to a change. It only ever
disables; an already-disabled account is untouched and gets no new row.

Decided with Matt 2026-10-16: disable on missing, SUSPENDED or DEPROVISIONED.
PASSWORD_EXPIRED, LOCKED_OUT and RECOVERY do not disable (the job never
re-enables, so a literal "not ACTIVE" would lock people out for good).
Added a guard: no match between any active local account and any Okta user is
refused as a wrong identity attribute, not read as a mass offboarding.

tests/okta_sync_job_check.py: 71 checks; removing the fail-closed branch and the
guard fails 28 of them. BL-069 logs the missing per-run cap.
This commit is contained in:
2026-10-16 11:31:39 -07:00
parent 2f7de6e33b
commit 103e4d40ee
4 changed files with 402 additions and 8 deletions

View File

@@ -0,0 +1,218 @@
#!/usr/bin/env python3
"""Does the Okta deprovisioning sync fail closed? - D18 / T13.3.
The rule that must never be skipped or weakened: when Okta cannot be asked, or
its answer cannot be trusted, the sync changes NO account and writes NO audit
row. Treating "could not reach Okta" as "nobody is active" would lock the whole
company out on one bad afternoon.
Also checked: the sync only ever disables (it never re-enables, under any
input), writes one audit row per change with a non-human actor, leaves an
already-disabled account alone, logs only changes, and keeps temporary Okta
states (PASSWORD_EXPIRED, LOCKED_OUT, RECOVERY) from disabling anyone.
Self-contained: its own in-memory SQLite, no server, no network, no browser.
Exit 0 all passed, 1 a failure.
"""
import os
import sys
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, ROOT)
import httpx # noqa: E402
from sqlalchemy import create_engine, select # noqa: E402
from sqlalchemy.orm import sessionmaker # noqa: E402
from sqlalchemy.pool import StaticPool # noqa: E402
from server import models, okta_auth, okta_sync # noqa: E402
from server.okta_sync import OktaSyncError, OktaUser, run_sync # noqa: E402
fails = 0
def chk(label, ok, extra=""):
global fails
print(("PASS " if ok else "FAIL ") + label + (f" [{extra}]" if (extra and not ok) else ""))
if not ok:
fails += 1
engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
models.Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine, autoflush=False, future=True)
def seed(rows):
"""rows: list of (username, is_active[, email]). Wipes users and audit rows first."""
with Session() as db:
db.query(models.AuditLog).delete()
db.query(models.User).delete()
for r in rows:
name, active = r[0], r[1]
db.add(models.User(id="u_" + name, username=name, is_active=active,
email=(r[2] if len(r) > 2 else "")))
db.commit()
def state():
with Session() as db:
users = {u.username: u.is_active for u in db.scalars(select(models.User))}
audits = [a.to_dict() for a in db.scalars(select(models.AuditLog))]
return users, audits
def fixed(*pairs):
return lambda: [OktaUser(identity=i, status=s) for i, s in pairs]
# ── 1. the failure rule ───────────────────────────────────────────────────────
print("\n1. FAIL CLOSED: a failed or untrusted fetch changes nothing")
BASE = [("alice", True), ("bob", True), ("carol", True)]
def boom(exc):
def f():
raise exc
return f
failing = [
("OktaSyncError (auth failure)", boom(OktaSyncError("HTTP 401"))),
("RuntimeError", boom(RuntimeError("boom"))),
("ConnectionError", boom(ConnectionError("down"))),
("TimeoutError", boom(TimeoutError("slow"))),
("ValueError", boom(ValueError("bad"))),
("KeyError", boom(KeyError("x"))),
("fetch returns an empty list", lambda: []),
("fetch returns None", lambda: None),
("fetch returns garbage (strings)", lambda: ["alice", "bob"]),
("fetch returns garbage (dicts)", lambda: [{"identity": "alice"}]),
]
for label, fetch in failing:
seed(BASE)
before = state()
with Session() as db:
res = run_sync(db, fetch=fetch)
after = state()
chk(f"{label}: ok is False", res.ok is False)
chk(f"{label}: zero accounts changed", after[0] == before[0] == {"alice": True, "bob": True, "carol": True})
chk(f"{label}: zero audit rows", after[1] == [] and res.disabled == [])
print("\n ...and through the real client, not a stand-in")
okta_auth.ISSUER = "https://example.okta.com/oauth2/default"
os.environ["OKTA_API_TOKEN"] = "tok"
for code in (401, 403, 429, 500):
seed(BASE)
c = httpx.Client(transport=httpx.MockTransport(lambda r, k=code: httpx.Response(k, json={})))
with Session() as db:
res = run_sync(db, fetch=lambda c=c: okta_sync.fetch_users(http=c))
u, a = state()
chk(f"real client, HTTP {code}: nothing changed, no audit rows",
(not res.ok) and all(u.values()) and a == [])
seed(BASE)
os.environ["OKTA_API_TOKEN"] = ""
with Session() as db:
res = run_sync(db) # fetch=None: the default path, no token set
u, a = state()
chk("default fetch with no token: nothing changed, no audit rows", (not res.ok) and all(u.values()) and a == [])
print("\n ...and when saving fails")
seed(BASE)
with Session() as db:
real_commit = db.commit
def bad_commit():
raise RuntimeError("disk full")
db.commit = bad_commit
res = run_sync(db, fetch=fixed(("alice", "ACTIVE"))) # bob and carol would be disabled
u, a = state()
chk("a failed commit keeps no change and no audit row", (not res.ok) and all(u.values()) and a == [])
print("\n ...and when no local account matches any Okta user (wrong identity attribute)")
seed(BASE)
with Session() as db:
res = run_sync(db, fetch=fixed(("zed@x.com", "ACTIVE"), ("yan@x.com", "ACTIVE")))
u, a = state()
chk("zero matches is refused, not read as 'everyone left'", (not res.ok) and all(u.values()) and a == [])
chk("...and the error says why", "identity attribute" in res.error)
# ── 2. what it does disable ───────────────────────────────────────────────────
print("\n2. a good run disables the right accounts")
seed([("alice", True), ("bob", True), ("carol", True), ("dave", True),
("erin", True), ("frank", True), ("gina", True), ("hank", True)])
okta = fixed(("alice", "ACTIVE"), ("bob", "SUSPENDED"),
# carol is MISSING from the list
("dave", "DEPROVISIONED"), ("erin", "PASSWORD_EXPIRED"),
("frank", "LOCKED_OUT"), ("gina", "RECOVERY"), ("hank", "ACTIVE"))
with Session() as db:
res = run_sync(db, fetch=okta)
users, audits = state()
chk("run is ok", res.ok, res.error)
chk("missing, SUSPENDED and DEPROVISIONED are disabled",
sorted(res.disabled) == ["bob", "carol", "dave"] and
[users[n] for n in ("bob", "carol", "dave")] == [False, False, False])
chk("ACTIVE stays active", users["alice"] is True and users["hank"] is True)
chk("PASSWORD_EXPIRED, LOCKED_OUT and RECOVERY do NOT disable (temporary states)",
users["erin"] is True and users["frank"] is True and users["gina"] is True)
chk("one audit row per disabled account", len(audits) == 3, str(len(audits)))
row = next(a for a in audits if a["summary"] == "bob")
chk("audit actor is the sync job, not a person", {a["actor"] for a in audits} == {"system:okta_sync"})
chk("audit action is user_deprovisioned_by_sync", {a["action"] for a in audits} == {"user_deprovisioned_by_sync"})
chk("audit row names the user and the Okta status found",
row["entity_type"] == "user" and row["entity_id"] == "u_bob" and row["detail"].get("okta_status") == "SUSPENDED")
chk("a missing user is recorded as MISSING",
next(a for a in audits if a["summary"] == "carol")["detail"].get("okta_status") == "MISSING")
print("\n ...run it again: only changes are logged")
with Session() as db:
res2 = run_sync(db, fetch=okta)
users2, audits2 = state()
chk("second run is ok, disables nobody", res2.ok and res2.disabled == [])
chk("second run writes no new audit rows (no duplicates)", len(audits2) == 3)
chk("accounts unchanged by the second run", users2 == users)
# ── 3. never re-enables, never touches a disabled account ─────────────────────
print("\n3. never re-enables, never touches an already-disabled account")
seed([("alice", True), ("hank", False), ("gina", False)])
with Session() as db:
res = run_sync(db, fetch=fixed(("alice", "ACTIVE"), ("hank", "ACTIVE"))) # gina missing
users, audits = state()
chk("a disabled account that is ACTIVE in Okta stays disabled", users["hank"] is False)
chk("a disabled account that is missing from Okta stays as it is", users["gina"] is False)
chk("no audit row for either (nothing changed)", audits == [] and res.disabled == [])
src = open(os.path.join(ROOT, "server", "okta_sync.py")).read()
chk("no assignment of is_active to True anywhere in okta_sync.py",
"is_active = True" not in src and "is_active=True" not in src.replace("is_active.is_(True)", ""))
# ── 4. matching ───────────────────────────────────────────────────────────────
print("\n4. matching")
seed([("ALICE", True), ("bob", True, "Bob@X.com"), ("carol", True)])
with Session() as db:
res = run_sync(db, fetch=fixed(("alice", "ACTIVE"), ("bob@x.com", "ACTIVE"), ("carol", "SUSPENDED")))
users, _ = state()
chk("username match ignores case", users["ALICE"] is True)
chk("email match works, ignoring case", users["bob"] is True)
chk("a matching SUSPENDED user is disabled", users["carol"] is False)
seed([("alice", True), ("bob", True), ("carol", True)])
with Session() as db:
run_sync(db, fetch=fixed(("alice", "ACTIVE"), ("bob", "ACTIVE"), ("bob", "SUSPENDED"),
("carol", "SUSPENDED"), ("carol", "DEPROVISIONED")))
users, _ = state()
chk("an identity listed twice, one entry active: stays active", users["bob"] is True)
chk("an identity listed twice, every entry disabling: disabled", users["carol"] is False)
# ── 5. the secret never reaches the audit row or the result ───────────────────
print("\n5. no secret in what is stored or returned")
seed(BASE)
os.environ["OKTA_API_TOKEN"] = "SECRET-TOKEN-123"
c = httpx.Client(transport=httpx.MockTransport(lambda r: httpx.Response(401, json={})))
with Session() as db:
res = run_sync(db, fetch=lambda: okta_sync.fetch_users(http=c))
chk("the failure message has no token", "SECRET-TOKEN-123" not in res.error and "example.okta.com" not in res.error)
print()
print("ALL PASSED" if not fails else f"{fails} FAILED")
sys.exit(0 if not fails else 1)