T13.4: run the Okta sync on a schedule (D18)
A daemon timer thread starts from a new lifespan hook in app.py and calls the T13.3 sync every OKTA_SYNC_INTERVAL_SECONDS (default 900, minimum 300). With no OKTA_API_TOKEN it starts nothing and the startup log says so. The api runs 2 gunicorn workers, so each starts its own timer. A single atomic conditional UPDATE on one app_settings row (key okta_sync) is the gate: exactly one worker wins each cycle, on SQLite and Postgres, with no lock held. The row survives a restart, so a restart inside the interval starts no second run. A fetch is now capped at 120 s in total, so a run is always shorter than the minimum interval and cannot overlap the next. No migration needed. docker-compose.yml passes OKTA_API_TOKEN, OKTA_ORG_URL, OKTA_SYNC_PROFILE_ATTRIBUTE and OKTA_SYNC_INTERVAL_SECONDS to the api service. tests/okta_sync_schedule_check.py: 38 checks including an 8-worker race on a file database (exactly one winner, 16 rounds) and fail-closed through the scheduled path. okta_auth_check.py still 22/22.
This commit is contained in:
@@ -230,6 +230,23 @@ every_log = " ".join(cap.lines)
|
||||
chk("no log line holds the API token", TOKEN not in every_log)
|
||||
chk("no log line holds the client secret", CLIENT_SECRET not in every_log)
|
||||
|
||||
# ── 8. the whole fetch is time-bounded (T13.4 relies on it) ──────────────────
|
||||
print("\n8. the whole fetch is time-bounded")
|
||||
reset_env()
|
||||
saved = okta_sync.TOTAL_DEADLINE_SECONDS
|
||||
okta_sync.TOTAL_DEADLINE_SECONDS = 0.0
|
||||
|
||||
|
||||
def slow_pages(req):
|
||||
return httpx.Response(200, json=[user("alice")],
|
||||
headers={"Link": f'<{ORG}/api/v1/users?after=x>; rel="next"'})
|
||||
|
||||
|
||||
raises(slow_pages, "a fetch that runs past the total deadline")
|
||||
okta_sync.TOTAL_DEADLINE_SECONDS = saved
|
||||
chk("the real deadline is under the shortest allowed interval",
|
||||
okta_sync.TOTAL_DEADLINE_SECONDS < okta_sync.INTERVAL_MIN_SECONDS)
|
||||
|
||||
print()
|
||||
print("ALL PASSED" if not fails else f"{fails} FAILED")
|
||||
sys.exit(0 if not fails else 1)
|
||||
|
||||
242
tests/okta_sync_schedule_check.py
Normal file
242
tests/okta_sync_schedule_check.py
Normal file
@@ -0,0 +1,242 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Does the Okta sync run on a schedule without doubling up? - D18 / T13.4.
|
||||
|
||||
The api container runs 2 gunicorn workers, and each starts its own timer. Checked
|
||||
here: exactly one of them wins each cycle (including a race on a real file
|
||||
database), a restart inside the interval does not start a second run, the
|
||||
interval is configurable and cannot go below the minimum, the timer thread starts
|
||||
and stops, nothing starts when the sync is not configured, and a failure inside
|
||||
the scheduled path still changes nothing.
|
||||
|
||||
Self-contained: a throwaway SQLite file, threads, no server, no network, no browser.
|
||||
Exit 0 all passed, 1 a failure.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.insert(0, ROOT)
|
||||
|
||||
from sqlalchemy import create_engine, select # noqa: E402
|
||||
from sqlalchemy.orm import sessionmaker # noqa: E402
|
||||
|
||||
from server import models, okta_auth, okta_sync # noqa: E402
|
||||
from server.okta_sync import OktaSyncError, OktaUser # noqa: E402
|
||||
|
||||
fails = 0
|
||||
|
||||
|
||||
def chk(label, ok, extra=""):
|
||||
global fails
|
||||
print(("PASS " if ok else "FAIL ") + label + (f" [{extra}]" if (extra and not ok) else ""))
|
||||
if not ok:
|
||||
fails += 1
|
||||
|
||||
|
||||
tmp = tempfile.mkdtemp(prefix="oktasync_")
|
||||
engine = create_engine(f"sqlite:///{os.path.join(tmp, 't.db')}",
|
||||
connect_args={"check_same_thread": False, "timeout": 20}, future=True)
|
||||
models.Base.metadata.create_all(engine)
|
||||
Session = sessionmaker(bind=engine, autoflush=False, future=True)
|
||||
INTERVAL = 900
|
||||
os.environ["OKTA_SYNC_INTERVAL_SECONDS"] = str(INTERVAL)
|
||||
T0 = datetime(2026, 10, 16, 12, 0, 0, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def wipe(users=()):
|
||||
with Session() as db:
|
||||
db.query(models.AuditLog).delete()
|
||||
db.query(models.User).delete()
|
||||
db.query(models.AppSetting).filter(models.AppSetting.key == okta_sync.CLAIM_KEY).delete()
|
||||
for name, active in users:
|
||||
db.add(models.User(id="u_" + name, username=name, is_active=active))
|
||||
db.commit()
|
||||
|
||||
|
||||
def users_state():
|
||||
with Session() as db:
|
||||
return ({u.username: u.is_active for u in db.scalars(select(models.User))},
|
||||
db.query(models.AuditLog).count())
|
||||
|
||||
|
||||
# ── 1. interval ───────────────────────────────────────────────────────────────
|
||||
print("\n1. interval")
|
||||
def iv(v):
|
||||
if v is None:
|
||||
os.environ.pop("OKTA_SYNC_INTERVAL_SECONDS", None)
|
||||
else:
|
||||
os.environ["OKTA_SYNC_INTERVAL_SECONDS"] = v
|
||||
return okta_sync.interval_seconds()
|
||||
|
||||
chk("default is 900 seconds (15 minutes)", iv(None) == 900 and iv("") == 900)
|
||||
chk("an env value is used", iv("600") == 600 and iv("3600") == 3600)
|
||||
chk("not a number: the default", iv("soon") == 900 and iv("1.5") == 900)
|
||||
chk("under the minimum is raised to 300 (a slow cycle can never overlap the next)",
|
||||
iv("60") == 300 and iv("0") == 300 and iv("-5") == 300)
|
||||
chk("a fetch is bounded below the minimum interval",
|
||||
okta_sync.TOTAL_DEADLINE_SECONDS < okta_sync.INTERVAL_MIN_SECONDS)
|
||||
iv(str(INTERVAL))
|
||||
|
||||
# ── 2. the claim ──────────────────────────────────────────────────────────────
|
||||
print("\n2. the claim gate")
|
||||
wipe()
|
||||
with Session() as db:
|
||||
chk("first ever claim wins (the insert is the claim)", okta_sync.claim(db, INTERVAL, now=T0) is True)
|
||||
with Session() as db:
|
||||
chk("a second claim in the same instant loses", okta_sync.claim(db, INTERVAL, now=T0) is False)
|
||||
with Session() as db:
|
||||
chk("one second before the interval is up: loses",
|
||||
okta_sync.claim(db, INTERVAL, now=T0 + timedelta(seconds=INTERVAL - 1)) is False)
|
||||
with Session() as db:
|
||||
chk("when the interval is up: wins",
|
||||
okta_sync.claim(db, INTERVAL, now=T0 + timedelta(seconds=INTERVAL)) is True)
|
||||
with Session() as db:
|
||||
chk("and then the next caller loses again",
|
||||
okta_sync.claim(db, INTERVAL, now=T0 + timedelta(seconds=INTERVAL + 1)) is False)
|
||||
|
||||
# ── 3. the race ───────────────────────────────────────────────────────────────
|
||||
print("\n3. exactly one of many workers wins (a real race on a file database)")
|
||||
|
||||
|
||||
def race(n=8):
|
||||
barrier = threading.Barrier(n)
|
||||
wins, errors = [], []
|
||||
|
||||
def worker():
|
||||
try:
|
||||
with Session() as db:
|
||||
barrier.wait()
|
||||
wins.append(okta_sync.claim(db, INTERVAL, now=datetime.now(timezone.utc)))
|
||||
except Exception as exc: # a lost race may surface as an error
|
||||
errors.append(type(exc).__name__)
|
||||
wins.append(False)
|
||||
|
||||
ts = [threading.Thread(target=worker) for _ in range(n)]
|
||||
[t.start() for t in ts]
|
||||
[t.join() for t in ts]
|
||||
return sum(1 for w in wins if w), errors
|
||||
|
||||
|
||||
wipe()
|
||||
won, errs = race()
|
||||
chk("no row yet, 8 workers at once: exactly 1 wins (the insert race)", won == 1, f"{won} {errs}")
|
||||
bad = []
|
||||
for i in range(15):
|
||||
with Session() as db: # age the claim so the next race is due again
|
||||
db.query(models.AppSetting).filter(models.AppSetting.key == okta_sync.CLAIM_KEY).update(
|
||||
{"updated_at": datetime.now(timezone.utc) - timedelta(seconds=INTERVAL + 5)})
|
||||
db.commit()
|
||||
won, errs = race()
|
||||
if won != 1:
|
||||
bad.append((i, won, errs))
|
||||
chk("row due, 8 workers at once, 15 rounds: exactly 1 wins every round", not bad, str(bad))
|
||||
|
||||
# ── 4. one tick ───────────────────────────────────────────────────────────────
|
||||
print("\n4. a tick runs the sync once per cycle")
|
||||
wipe([("alice", True), ("bob", True)])
|
||||
calls = []
|
||||
|
||||
|
||||
def fetch_ok():
|
||||
calls.append(1)
|
||||
return [OktaUser("alice", "ACTIVE"), OktaUser("bob", "SUSPENDED")]
|
||||
|
||||
|
||||
r1 = okta_sync.run_if_due(Session, fetch=fetch_ok, now=T0)
|
||||
chk("first tick claims and runs", r1 is not None and r1.ok and r1.disabled == ["bob"])
|
||||
r2 = okta_sync.run_if_due(Session, fetch=fetch_ok, now=T0 + timedelta(seconds=30))
|
||||
chk("a tick 30 seconds later does nothing (not due)", r2 is None and len(calls) == 1)
|
||||
r3 = okta_sync.run_if_due(Session, fetch=fetch_ok, now=T0 + timedelta(seconds=INTERVAL))
|
||||
chk("a tick when due runs again", r3 is not None and r3.ok and len(calls) == 2)
|
||||
|
||||
print("\n ...a restart inside the interval does not start a second run")
|
||||
wipe([("alice", True), ("bob", True)])
|
||||
calls.clear()
|
||||
okta_sync.run_if_due(Session, fetch=fetch_ok, now=T0)
|
||||
# "restart": a brand-new caller with nothing in memory, 2 minutes later
|
||||
r = okta_sync.run_if_due(Session, fetch=fetch_ok, now=T0 + timedelta(seconds=120))
|
||||
chk("the claim row outlives the process: no duplicate run", r is None and len(calls) == 1)
|
||||
|
||||
print("\n ...two workers tick in the same instant")
|
||||
wipe([("alice", True), ("bob", True)])
|
||||
calls.clear()
|
||||
results = []
|
||||
barrier = threading.Barrier(2)
|
||||
|
||||
|
||||
def tick():
|
||||
barrier.wait()
|
||||
results.append(okta_sync.run_if_due(Session, fetch=fetch_ok))
|
||||
|
||||
|
||||
ts = [threading.Thread(target=tick) for _ in range(2)]
|
||||
[t.start() for t in ts]
|
||||
[t.join() for t in ts]
|
||||
chk("Okta is asked once, not twice", len(calls) == 1, str(len(calls)))
|
||||
chk("exactly one worker got a result", sum(1 for r in results if r is not None) == 1)
|
||||
_, audits = users_state()
|
||||
chk("one audit row, not two", audits == 1, str(audits))
|
||||
|
||||
# ── 5. fail closed through the scheduled path ─────────────────────────────────
|
||||
print("\n5. FAIL CLOSED through the scheduled wrapper (not just run_sync)")
|
||||
for label, exc in (("OktaSyncError", OktaSyncError("HTTP 500")), ("RuntimeError", RuntimeError("x")),
|
||||
("TimeoutError", TimeoutError("t"))):
|
||||
wipe([("alice", True), ("bob", True)])
|
||||
|
||||
def failing(e=exc):
|
||||
raise e
|
||||
|
||||
r = okta_sync.run_if_due(Session, fetch=failing, now=T0)
|
||||
u, a = users_state()
|
||||
chk(f"{label} in the scheduled path: ok is False, zero changes, zero audit rows",
|
||||
r is not None and r.ok is False and all(u.values()) and a == 0)
|
||||
|
||||
wipe([("alice", True), ("bob", True)])
|
||||
|
||||
|
||||
def broken_factory():
|
||||
raise RuntimeError("database is down")
|
||||
|
||||
|
||||
chk("the database failing at claim time: the tick ends quietly, returns None",
|
||||
okta_sync.run_if_due(broken_factory, fetch=fetch_ok, now=T0) is None)
|
||||
u, a = users_state()
|
||||
chk("...and nothing changed", all(u.values()) and a == 0)
|
||||
|
||||
# ── 6. the thread ─────────────────────────────────────────────────────────────
|
||||
print("\n6. the timer thread")
|
||||
os.environ["OKTA_API_TOKEN"] = ""
|
||||
okta_auth.ISSUER = "https://example.okta.com/oauth2/default"
|
||||
chk("no token: nothing starts", okta_sync.start_scheduler(Session) is None)
|
||||
chk("...no thread named okta-sync", not any(t.name == "okta-sync" for t in threading.enumerate()))
|
||||
|
||||
os.environ["OKTA_API_TOKEN"] = "tok"
|
||||
wipe([("alice", True), ("bob", True)])
|
||||
okta_sync.START_DELAY_SECONDS = 0.05
|
||||
okta_sync.TICK_SECONDS = 0.05
|
||||
ran = []
|
||||
real_fetch = okta_sync.fetch_users
|
||||
okta_sync.fetch_users = lambda http=None: (ran.append(1) or
|
||||
[OktaUser("alice", "ACTIVE"), OktaUser("bob", "DEPROVISIONED")])
|
||||
stop = okta_sync.start_scheduler(Session)
|
||||
chk("configured: a stop Event comes back", stop is not None)
|
||||
time.sleep(1.2) # about 20 ticks
|
||||
u, a = users_state()
|
||||
chk("the thread ran the sync on its own (bob was disabled without a manual call)", u["bob"] is False)
|
||||
chk("many ticks, one run (the claim gate holds)", len(ran) == 1 and a == 1, f"ran={len(ran)} audits={a}")
|
||||
th = [t for t in threading.enumerate() if t.name == "okta-sync"]
|
||||
chk("one timer thread is alive", len(th) == 1 and th[0].is_alive())
|
||||
chk("the thread is a daemon (never blocks shutdown)", th[0].daemon)
|
||||
stop.set()
|
||||
th[0].join(timeout=3)
|
||||
chk("setting the stop Event ends the thread", not th[0].is_alive())
|
||||
okta_sync.fetch_users = real_fetch
|
||||
os.environ["OKTA_API_TOKEN"] = ""
|
||||
|
||||
print()
|
||||
print("ALL PASSED" if not fails else f"{fails} FAILED")
|
||||
sys.exit(0 if not fails else 1)
|
||||
Reference in New Issue
Block a user