A daemon timer thread starts from a new lifespan hook in app.py and calls the T13.3 sync every OKTA_SYNC_INTERVAL_SECONDS (default 900, minimum 300). With no OKTA_API_TOKEN it starts nothing and the startup log says so. The api runs 2 gunicorn workers, so each starts its own timer. A single atomic conditional UPDATE on one app_settings row (key okta_sync) is the gate: exactly one worker wins each cycle, on SQLite and Postgres, with no lock held. The row survives a restart, so a restart inside the interval starts no second run. A fetch is now capped at 120 s in total, so a run is always shorter than the minimum interval and cannot overlap the next. No migration needed. docker-compose.yml passes OKTA_API_TOKEN, OKTA_ORG_URL, OKTA_SYNC_PROFILE_ATTRIBUTE and OKTA_SYNC_INTERVAL_SECONDS to the api service. tests/okta_sync_schedule_check.py: 38 checks including an 8-worker race on a file database (exactly one winner, 16 rounds) and fail-closed through the scheduled path. okta_auth_check.py still 22/22.
243 lines
10 KiB
Python
243 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Does the Okta sync run on a schedule without doubling up? - D18 / T13.4.
|
|
|
|
The api container runs 2 gunicorn workers, and each starts its own timer. Checked
|
|
here: exactly one of them wins each cycle (including a race on a real file
|
|
database), a restart inside the interval does not start a second run, the
|
|
interval is configurable and cannot go below the minimum, the timer thread starts
|
|
and stops, nothing starts when the sync is not configured, and a failure inside
|
|
the scheduled path still changes nothing.
|
|
|
|
Self-contained: a throwaway SQLite file, threads, no server, no network, no browser.
|
|
Exit 0 all passed, 1 a failure.
|
|
"""
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
sys.path.insert(0, ROOT)
|
|
|
|
from sqlalchemy import create_engine, select # noqa: E402
|
|
from sqlalchemy.orm import sessionmaker # noqa: E402
|
|
|
|
from server import models, okta_auth, okta_sync # noqa: E402
|
|
from server.okta_sync import OktaSyncError, OktaUser # noqa: E402
|
|
|
|
fails = 0
|
|
|
|
|
|
def chk(label, ok, extra=""):
|
|
global fails
|
|
print(("PASS " if ok else "FAIL ") + label + (f" [{extra}]" if (extra and not ok) else ""))
|
|
if not ok:
|
|
fails += 1
|
|
|
|
|
|
tmp = tempfile.mkdtemp(prefix="oktasync_")
|
|
engine = create_engine(f"sqlite:///{os.path.join(tmp, 't.db')}",
|
|
connect_args={"check_same_thread": False, "timeout": 20}, future=True)
|
|
models.Base.metadata.create_all(engine)
|
|
Session = sessionmaker(bind=engine, autoflush=False, future=True)
|
|
INTERVAL = 900
|
|
os.environ["OKTA_SYNC_INTERVAL_SECONDS"] = str(INTERVAL)
|
|
T0 = datetime(2026, 10, 16, 12, 0, 0, tzinfo=timezone.utc)
|
|
|
|
|
|
def wipe(users=()):
|
|
with Session() as db:
|
|
db.query(models.AuditLog).delete()
|
|
db.query(models.User).delete()
|
|
db.query(models.AppSetting).filter(models.AppSetting.key == okta_sync.CLAIM_KEY).delete()
|
|
for name, active in users:
|
|
db.add(models.User(id="u_" + name, username=name, is_active=active))
|
|
db.commit()
|
|
|
|
|
|
def users_state():
|
|
with Session() as db:
|
|
return ({u.username: u.is_active for u in db.scalars(select(models.User))},
|
|
db.query(models.AuditLog).count())
|
|
|
|
|
|
# ── 1. interval ───────────────────────────────────────────────────────────────
|
|
print("\n1. interval")
|
|
def iv(v):
|
|
if v is None:
|
|
os.environ.pop("OKTA_SYNC_INTERVAL_SECONDS", None)
|
|
else:
|
|
os.environ["OKTA_SYNC_INTERVAL_SECONDS"] = v
|
|
return okta_sync.interval_seconds()
|
|
|
|
chk("default is 900 seconds (15 minutes)", iv(None) == 900 and iv("") == 900)
|
|
chk("an env value is used", iv("600") == 600 and iv("3600") == 3600)
|
|
chk("not a number: the default", iv("soon") == 900 and iv("1.5") == 900)
|
|
chk("under the minimum is raised to 300 (a slow cycle can never overlap the next)",
|
|
iv("60") == 300 and iv("0") == 300 and iv("-5") == 300)
|
|
chk("a fetch is bounded below the minimum interval",
|
|
okta_sync.TOTAL_DEADLINE_SECONDS < okta_sync.INTERVAL_MIN_SECONDS)
|
|
iv(str(INTERVAL))
|
|
|
|
# ── 2. the claim ──────────────────────────────────────────────────────────────
|
|
print("\n2. the claim gate")
|
|
wipe()
|
|
with Session() as db:
|
|
chk("first ever claim wins (the insert is the claim)", okta_sync.claim(db, INTERVAL, now=T0) is True)
|
|
with Session() as db:
|
|
chk("a second claim in the same instant loses", okta_sync.claim(db, INTERVAL, now=T0) is False)
|
|
with Session() as db:
|
|
chk("one second before the interval is up: loses",
|
|
okta_sync.claim(db, INTERVAL, now=T0 + timedelta(seconds=INTERVAL - 1)) is False)
|
|
with Session() as db:
|
|
chk("when the interval is up: wins",
|
|
okta_sync.claim(db, INTERVAL, now=T0 + timedelta(seconds=INTERVAL)) is True)
|
|
with Session() as db:
|
|
chk("and then the next caller loses again",
|
|
okta_sync.claim(db, INTERVAL, now=T0 + timedelta(seconds=INTERVAL + 1)) is False)
|
|
|
|
# ── 3. the race ───────────────────────────────────────────────────────────────
|
|
print("\n3. exactly one of many workers wins (a real race on a file database)")
|
|
|
|
|
|
def race(n=8):
|
|
barrier = threading.Barrier(n)
|
|
wins, errors = [], []
|
|
|
|
def worker():
|
|
try:
|
|
with Session() as db:
|
|
barrier.wait()
|
|
wins.append(okta_sync.claim(db, INTERVAL, now=datetime.now(timezone.utc)))
|
|
except Exception as exc: # a lost race may surface as an error
|
|
errors.append(type(exc).__name__)
|
|
wins.append(False)
|
|
|
|
ts = [threading.Thread(target=worker) for _ in range(n)]
|
|
[t.start() for t in ts]
|
|
[t.join() for t in ts]
|
|
return sum(1 for w in wins if w), errors
|
|
|
|
|
|
wipe()
|
|
won, errs = race()
|
|
chk("no row yet, 8 workers at once: exactly 1 wins (the insert race)", won == 1, f"{won} {errs}")
|
|
bad = []
|
|
for i in range(15):
|
|
with Session() as db: # age the claim so the next race is due again
|
|
db.query(models.AppSetting).filter(models.AppSetting.key == okta_sync.CLAIM_KEY).update(
|
|
{"updated_at": datetime.now(timezone.utc) - timedelta(seconds=INTERVAL + 5)})
|
|
db.commit()
|
|
won, errs = race()
|
|
if won != 1:
|
|
bad.append((i, won, errs))
|
|
chk("row due, 8 workers at once, 15 rounds: exactly 1 wins every round", not bad, str(bad))
|
|
|
|
# ── 4. one tick ───────────────────────────────────────────────────────────────
|
|
print("\n4. a tick runs the sync once per cycle")
|
|
wipe([("alice", True), ("bob", True)])
|
|
calls = []
|
|
|
|
|
|
def fetch_ok():
|
|
calls.append(1)
|
|
return [OktaUser("alice", "ACTIVE"), OktaUser("bob", "SUSPENDED")]
|
|
|
|
|
|
r1 = okta_sync.run_if_due(Session, fetch=fetch_ok, now=T0)
|
|
chk("first tick claims and runs", r1 is not None and r1.ok and r1.disabled == ["bob"])
|
|
r2 = okta_sync.run_if_due(Session, fetch=fetch_ok, now=T0 + timedelta(seconds=30))
|
|
chk("a tick 30 seconds later does nothing (not due)", r2 is None and len(calls) == 1)
|
|
r3 = okta_sync.run_if_due(Session, fetch=fetch_ok, now=T0 + timedelta(seconds=INTERVAL))
|
|
chk("a tick when due runs again", r3 is not None and r3.ok and len(calls) == 2)
|
|
|
|
print("\n ...a restart inside the interval does not start a second run")
|
|
wipe([("alice", True), ("bob", True)])
|
|
calls.clear()
|
|
okta_sync.run_if_due(Session, fetch=fetch_ok, now=T0)
|
|
# "restart": a brand-new caller with nothing in memory, 2 minutes later
|
|
r = okta_sync.run_if_due(Session, fetch=fetch_ok, now=T0 + timedelta(seconds=120))
|
|
chk("the claim row outlives the process: no duplicate run", r is None and len(calls) == 1)
|
|
|
|
print("\n ...two workers tick in the same instant")
|
|
wipe([("alice", True), ("bob", True)])
|
|
calls.clear()
|
|
results = []
|
|
barrier = threading.Barrier(2)
|
|
|
|
|
|
def tick():
|
|
barrier.wait()
|
|
results.append(okta_sync.run_if_due(Session, fetch=fetch_ok))
|
|
|
|
|
|
ts = [threading.Thread(target=tick) for _ in range(2)]
|
|
[t.start() for t in ts]
|
|
[t.join() for t in ts]
|
|
chk("Okta is asked once, not twice", len(calls) == 1, str(len(calls)))
|
|
chk("exactly one worker got a result", sum(1 for r in results if r is not None) == 1)
|
|
_, audits = users_state()
|
|
chk("one audit row, not two", audits == 1, str(audits))
|
|
|
|
# ── 5. fail closed through the scheduled path ─────────────────────────────────
|
|
print("\n5. FAIL CLOSED through the scheduled wrapper (not just run_sync)")
|
|
for label, exc in (("OktaSyncError", OktaSyncError("HTTP 500")), ("RuntimeError", RuntimeError("x")),
|
|
("TimeoutError", TimeoutError("t"))):
|
|
wipe([("alice", True), ("bob", True)])
|
|
|
|
def failing(e=exc):
|
|
raise e
|
|
|
|
r = okta_sync.run_if_due(Session, fetch=failing, now=T0)
|
|
u, a = users_state()
|
|
chk(f"{label} in the scheduled path: ok is False, zero changes, zero audit rows",
|
|
r is not None and r.ok is False and all(u.values()) and a == 0)
|
|
|
|
wipe([("alice", True), ("bob", True)])
|
|
|
|
|
|
def broken_factory():
|
|
raise RuntimeError("database is down")
|
|
|
|
|
|
chk("the database failing at claim time: the tick ends quietly, returns None",
|
|
okta_sync.run_if_due(broken_factory, fetch=fetch_ok, now=T0) is None)
|
|
u, a = users_state()
|
|
chk("...and nothing changed", all(u.values()) and a == 0)
|
|
|
|
# ── 6. the thread ─────────────────────────────────────────────────────────────
|
|
print("\n6. the timer thread")
|
|
os.environ["OKTA_API_TOKEN"] = ""
|
|
okta_auth.ISSUER = "https://example.okta.com/oauth2/default"
|
|
chk("no token: nothing starts", okta_sync.start_scheduler(Session) is None)
|
|
chk("...no thread named okta-sync", not any(t.name == "okta-sync" for t in threading.enumerate()))
|
|
|
|
os.environ["OKTA_API_TOKEN"] = "tok"
|
|
wipe([("alice", True), ("bob", True)])
|
|
okta_sync.START_DELAY_SECONDS = 0.05
|
|
okta_sync.TICK_SECONDS = 0.05
|
|
ran = []
|
|
real_fetch = okta_sync.fetch_users
|
|
okta_sync.fetch_users = lambda http=None: (ran.append(1) or
|
|
[OktaUser("alice", "ACTIVE"), OktaUser("bob", "DEPROVISIONED")])
|
|
stop = okta_sync.start_scheduler(Session)
|
|
chk("configured: a stop Event comes back", stop is not None)
|
|
time.sleep(1.2) # about 20 ticks
|
|
u, a = users_state()
|
|
chk("the thread ran the sync on its own (bob was disabled without a manual call)", u["bob"] is False)
|
|
chk("many ticks, one run (the claim gate holds)", len(ran) == 1 and a == 1, f"ran={len(ran)} audits={a}")
|
|
th = [t for t in threading.enumerate() if t.name == "okta-sync"]
|
|
chk("one timer thread is alive", len(th) == 1 and th[0].is_alive())
|
|
chk("the thread is a daemon (never blocks shutdown)", th[0].daemon)
|
|
stop.set()
|
|
th[0].join(timeout=3)
|
|
chk("setting the stop Event ends the thread", not th[0].is_alive())
|
|
okta_sync.fetch_users = real_fetch
|
|
os.environ["OKTA_API_TOKEN"] = ""
|
|
|
|
print()
|
|
print("ALL PASSED" if not fails else f"{fails} FAILED")
|
|
sys.exit(0 if not fails else 1)
|