T13.4: run the Okta sync on a schedule (D18)

A daemon timer thread starts from a new lifespan hook in app.py and calls the
T13.3 sync every OKTA_SYNC_INTERVAL_SECONDS (default 900, minimum 300). With no
OKTA_API_TOKEN it starts nothing and the startup log says so.

The api runs 2 gunicorn workers, so each starts its own timer. A single atomic
conditional UPDATE on one app_settings row (key okta_sync) is the gate: exactly
one worker wins each cycle, on SQLite and Postgres, with no lock held. The row
survives a restart, so a restart inside the interval starts no second run. A
fetch is now capped at 120 s in total, so a run is always shorter than the
minimum interval and cannot overlap the next. No migration needed.

docker-compose.yml passes OKTA_API_TOKEN, OKTA_ORG_URL,
OKTA_SYNC_PROFILE_ATTRIBUTE and OKTA_SYNC_INTERVAL_SECONDS to the api service.

tests/okta_sync_schedule_check.py: 38 checks including an 8-worker race on a
file database (exactly one winner, 16 rounds) and fail-closed through the
scheduled path. okta_auth_check.py still 22/22.
This commit is contained in:
2026-10-16 11:34:13 -07:00
parent 103e4d40ee
commit f30989d4b8
7 changed files with 427 additions and 6 deletions

View File

@@ -230,6 +230,23 @@ every_log = " ".join(cap.lines)
chk("no log line holds the API token", TOKEN not in every_log)
chk("no log line holds the client secret", CLIENT_SECRET not in every_log)
# ── 8. the whole fetch is time-bounded (T13.4 relies on it) ──────────────────
print("\n8. the whole fetch is time-bounded")
reset_env()
saved = okta_sync.TOTAL_DEADLINE_SECONDS
okta_sync.TOTAL_DEADLINE_SECONDS = 0.0
def slow_pages(req):
return httpx.Response(200, json=[user("alice")],
headers={"Link": f'<{ORG}/api/v1/users?after=x>; rel="next"'})
raises(slow_pages, "a fetch that runs past the total deadline")
okta_sync.TOTAL_DEADLINE_SECONDS = saved
chk("the real deadline is under the shortest allowed interval",
okta_sync.TOTAL_DEADLINE_SECONDS < okta_sync.INTERVAL_MIN_SECONDS)
print()
print("ALL PASSED" if not fails else f"{fails} FAILED")
sys.exit(0 if not fails else 1)

View File

@@ -0,0 +1,242 @@
#!/usr/bin/env python3
"""Does the Okta sync run on a schedule without doubling up? - D18 / T13.4.
The api container runs 2 gunicorn workers, and each starts its own timer. Checked
here: exactly one of them wins each cycle (including a race on a real file
database), a restart inside the interval does not start a second run, the
interval is configurable and cannot go below the minimum, the timer thread starts
and stops, nothing starts when the sync is not configured, and a failure inside
the scheduled path still changes nothing.
Self-contained: a throwaway SQLite file, threads, no server, no network, no browser.
Exit 0 all passed, 1 a failure.
"""
import os
import sys
import tempfile
import threading
import time
from datetime import datetime, timedelta, timezone
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, ROOT)
from sqlalchemy import create_engine, select # noqa: E402
from sqlalchemy.orm import sessionmaker # noqa: E402
from server import models, okta_auth, okta_sync # noqa: E402
from server.okta_sync import OktaSyncError, OktaUser # noqa: E402
fails = 0
def chk(label, ok, extra=""):
global fails
print(("PASS " if ok else "FAIL ") + label + (f" [{extra}]" if (extra and not ok) else ""))
if not ok:
fails += 1
tmp = tempfile.mkdtemp(prefix="oktasync_")
engine = create_engine(f"sqlite:///{os.path.join(tmp, 't.db')}",
connect_args={"check_same_thread": False, "timeout": 20}, future=True)
models.Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine, autoflush=False, future=True)
INTERVAL = 900
os.environ["OKTA_SYNC_INTERVAL_SECONDS"] = str(INTERVAL)
T0 = datetime(2026, 10, 16, 12, 0, 0, tzinfo=timezone.utc)
def wipe(users=()):
with Session() as db:
db.query(models.AuditLog).delete()
db.query(models.User).delete()
db.query(models.AppSetting).filter(models.AppSetting.key == okta_sync.CLAIM_KEY).delete()
for name, active in users:
db.add(models.User(id="u_" + name, username=name, is_active=active))
db.commit()
def users_state():
with Session() as db:
return ({u.username: u.is_active for u in db.scalars(select(models.User))},
db.query(models.AuditLog).count())
# ── 1. interval ───────────────────────────────────────────────────────────────
print("\n1. interval")
def iv(v):
if v is None:
os.environ.pop("OKTA_SYNC_INTERVAL_SECONDS", None)
else:
os.environ["OKTA_SYNC_INTERVAL_SECONDS"] = v
return okta_sync.interval_seconds()
chk("default is 900 seconds (15 minutes)", iv(None) == 900 and iv("") == 900)
chk("an env value is used", iv("600") == 600 and iv("3600") == 3600)
chk("not a number: the default", iv("soon") == 900 and iv("1.5") == 900)
chk("under the minimum is raised to 300 (a slow cycle can never overlap the next)",
iv("60") == 300 and iv("0") == 300 and iv("-5") == 300)
chk("a fetch is bounded below the minimum interval",
okta_sync.TOTAL_DEADLINE_SECONDS < okta_sync.INTERVAL_MIN_SECONDS)
iv(str(INTERVAL))
# ── 2. the claim ──────────────────────────────────────────────────────────────
print("\n2. the claim gate")
wipe()
with Session() as db:
chk("first ever claim wins (the insert is the claim)", okta_sync.claim(db, INTERVAL, now=T0) is True)
with Session() as db:
chk("a second claim in the same instant loses", okta_sync.claim(db, INTERVAL, now=T0) is False)
with Session() as db:
chk("one second before the interval is up: loses",
okta_sync.claim(db, INTERVAL, now=T0 + timedelta(seconds=INTERVAL - 1)) is False)
with Session() as db:
chk("when the interval is up: wins",
okta_sync.claim(db, INTERVAL, now=T0 + timedelta(seconds=INTERVAL)) is True)
with Session() as db:
chk("and then the next caller loses again",
okta_sync.claim(db, INTERVAL, now=T0 + timedelta(seconds=INTERVAL + 1)) is False)
# ── 3. the race ───────────────────────────────────────────────────────────────
print("\n3. exactly one of many workers wins (a real race on a file database)")
def race(n=8):
barrier = threading.Barrier(n)
wins, errors = [], []
def worker():
try:
with Session() as db:
barrier.wait()
wins.append(okta_sync.claim(db, INTERVAL, now=datetime.now(timezone.utc)))
except Exception as exc: # a lost race may surface as an error
errors.append(type(exc).__name__)
wins.append(False)
ts = [threading.Thread(target=worker) for _ in range(n)]
[t.start() for t in ts]
[t.join() for t in ts]
return sum(1 for w in wins if w), errors
wipe()
won, errs = race()
chk("no row yet, 8 workers at once: exactly 1 wins (the insert race)", won == 1, f"{won} {errs}")
bad = []
for i in range(15):
with Session() as db: # age the claim so the next race is due again
db.query(models.AppSetting).filter(models.AppSetting.key == okta_sync.CLAIM_KEY).update(
{"updated_at": datetime.now(timezone.utc) - timedelta(seconds=INTERVAL + 5)})
db.commit()
won, errs = race()
if won != 1:
bad.append((i, won, errs))
chk("row due, 8 workers at once, 15 rounds: exactly 1 wins every round", not bad, str(bad))
# ── 4. one tick ───────────────────────────────────────────────────────────────
print("\n4. a tick runs the sync once per cycle")
wipe([("alice", True), ("bob", True)])
calls = []
def fetch_ok():
calls.append(1)
return [OktaUser("alice", "ACTIVE"), OktaUser("bob", "SUSPENDED")]
r1 = okta_sync.run_if_due(Session, fetch=fetch_ok, now=T0)
chk("first tick claims and runs", r1 is not None and r1.ok and r1.disabled == ["bob"])
r2 = okta_sync.run_if_due(Session, fetch=fetch_ok, now=T0 + timedelta(seconds=30))
chk("a tick 30 seconds later does nothing (not due)", r2 is None and len(calls) == 1)
r3 = okta_sync.run_if_due(Session, fetch=fetch_ok, now=T0 + timedelta(seconds=INTERVAL))
chk("a tick when due runs again", r3 is not None and r3.ok and len(calls) == 2)
print("\n ...a restart inside the interval does not start a second run")
wipe([("alice", True), ("bob", True)])
calls.clear()
okta_sync.run_if_due(Session, fetch=fetch_ok, now=T0)
# "restart": a brand-new caller with nothing in memory, 2 minutes later
r = okta_sync.run_if_due(Session, fetch=fetch_ok, now=T0 + timedelta(seconds=120))
chk("the claim row outlives the process: no duplicate run", r is None and len(calls) == 1)
print("\n ...two workers tick in the same instant")
wipe([("alice", True), ("bob", True)])
calls.clear()
results = []
barrier = threading.Barrier(2)
def tick():
barrier.wait()
results.append(okta_sync.run_if_due(Session, fetch=fetch_ok))
ts = [threading.Thread(target=tick) for _ in range(2)]
[t.start() for t in ts]
[t.join() for t in ts]
chk("Okta is asked once, not twice", len(calls) == 1, str(len(calls)))
chk("exactly one worker got a result", sum(1 for r in results if r is not None) == 1)
_, audits = users_state()
chk("one audit row, not two", audits == 1, str(audits))
# ── 5. fail closed through the scheduled path ─────────────────────────────────
print("\n5. FAIL CLOSED through the scheduled wrapper (not just run_sync)")
for label, exc in (("OktaSyncError", OktaSyncError("HTTP 500")), ("RuntimeError", RuntimeError("x")),
("TimeoutError", TimeoutError("t"))):
wipe([("alice", True), ("bob", True)])
def failing(e=exc):
raise e
r = okta_sync.run_if_due(Session, fetch=failing, now=T0)
u, a = users_state()
chk(f"{label} in the scheduled path: ok is False, zero changes, zero audit rows",
r is not None and r.ok is False and all(u.values()) and a == 0)
wipe([("alice", True), ("bob", True)])
def broken_factory():
raise RuntimeError("database is down")
chk("the database failing at claim time: the tick ends quietly, returns None",
okta_sync.run_if_due(broken_factory, fetch=fetch_ok, now=T0) is None)
u, a = users_state()
chk("...and nothing changed", all(u.values()) and a == 0)
# ── 6. the thread ─────────────────────────────────────────────────────────────
print("\n6. the timer thread")
os.environ["OKTA_API_TOKEN"] = ""
okta_auth.ISSUER = "https://example.okta.com/oauth2/default"
chk("no token: nothing starts", okta_sync.start_scheduler(Session) is None)
chk("...no thread named okta-sync", not any(t.name == "okta-sync" for t in threading.enumerate()))
os.environ["OKTA_API_TOKEN"] = "tok"
wipe([("alice", True), ("bob", True)])
okta_sync.START_DELAY_SECONDS = 0.05
okta_sync.TICK_SECONDS = 0.05
ran = []
real_fetch = okta_sync.fetch_users
okta_sync.fetch_users = lambda http=None: (ran.append(1) or
[OktaUser("alice", "ACTIVE"), OktaUser("bob", "DEPROVISIONED")])
stop = okta_sync.start_scheduler(Session)
chk("configured: a stop Event comes back", stop is not None)
time.sleep(1.2) # about 20 ticks
u, a = users_state()
chk("the thread ran the sync on its own (bob was disabled without a manual call)", u["bob"] is False)
chk("many ticks, one run (the claim gate holds)", len(ran) == 1 and a == 1, f"ran={len(ran)} audits={a}")
th = [t for t in threading.enumerate() if t.name == "okta-sync"]
chk("one timer thread is alive", len(th) == 1 and th[0].is_alive())
chk("the thread is a daemon (never blocks shutdown)", th[0].daemon)
stop.set()
th[0].join(timeout=3)
chk("setting the stop Event ends the thread", not th[0].is_alive())
okta_sync.fetch_users = real_fetch
os.environ["OKTA_API_TOKEN"] = ""
print()
print("ALL PASSED" if not fails else f"{fails} FAILED")
sys.exit(0 if not fails else 1)