Release: wave 11 + T13.2-T13.5 (D18 Okta sync), T13.1 held back until T14.1; mobile roadmap docs (BL-062..BL-069)
This commit is contained in:
@@ -55,6 +55,36 @@ OKTA_REDIRECT_URI=CHANGE_ME
|
||||
# a custom claim, …) — no code change needed, just this value.
|
||||
# OKTA_IDENTITY_CLAIM=preferred_username
|
||||
|
||||
# ── Okta deprovisioning sync (D18, wave 13) ────────────────────────────────────
|
||||
# A scheduled job asks Okta who is still active and disables local accounts
|
||||
# Okta no longer shows as active. It only ever DISABLES. It never re-enables.
|
||||
# If Okta cannot be reached or answers with anything unexpected, the job changes
|
||||
# NOTHING that cycle and logs the failure.
|
||||
#
|
||||
# A read-only Okta API token, created by the person who administers the Okta
|
||||
# tenant (Security > API > Tokens). This is NOT the OIDC client secret above.
|
||||
# It is a different credential with a different scope. Treat it like
|
||||
# AUTH_SECRET_KEY: never commit it, never log it. Leave it unset and the sync
|
||||
# stays off (the startup log says so).
|
||||
# OKTA_API_TOKEN=CHANGE_ME
|
||||
#
|
||||
# The Okta org URL that serves the Management API, e.g. https://yourorg.okta.com
|
||||
# Optional. If unset, it is the scheme and host of OKTA_ISSUER. Must be https.
|
||||
# OKTA_ORG_URL=https://your-org.okta.com
|
||||
#
|
||||
# Which Okta PROFILE attribute holds the value that matches users.username.
|
||||
# Optional. By default it follows OKTA_IDENTITY_CLAIM: preferred_username reads
|
||||
# profile.login, email reads profile.email, any other claim name reads the
|
||||
# profile attribute of the same name. Set this only if your tenant builds the
|
||||
# claim from a different attribute.
|
||||
# OKTA_SYNC_PROFILE_ATTRIBUTE=login
|
||||
#
|
||||
# How often the sync runs, in seconds. Default 900 (15 minutes), proposed and not
|
||||
# confirmed with IT. A value under 300 is raised to 300. Every api worker checks
|
||||
# once a minute and one of them wins each cycle, so raising the worker count does
|
||||
# not raise the call rate to Okta.
|
||||
# OKTA_SYNC_INTERVAL_SECONDS=900
|
||||
|
||||
# ── Email notifications (optional) ─────────────────────────────────────────────
|
||||
# WP-assignment emails are OFF by default and are turned on from the Admin
|
||||
# console (Notifications & email card), where the SMTP host/port/from-address
|
||||
|
||||
46
server/alembic/versions/8e2cb3003f8a_usage_events.py
Normal file
46
server/alembic/versions/8e2cb3003f8a_usage_events.py
Normal file
@@ -0,0 +1,46 @@
|
||||
"""usage events (CR-019, wave 11)
|
||||
|
||||
Append-only navigation/session activity, separate from audit_log on purpose —
|
||||
see the UsageEvent docstring in server/models.py. Retention is indefinite by
|
||||
decision (docs/waves/decisions-2026-09-17.md); nothing here schedules a purge.
|
||||
|
||||
Revision ID: 8e2cb3003f8a
|
||||
Revises: 1d60a608bb51
|
||||
Create Date: 2026-09-23 00:00:00.000000
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '8e2cb3003f8a'
|
||||
down_revision = '1d60a608bb51'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table('usage_events',
|
||||
sa.Column('id', sa.String(length=40), nullable=False),
|
||||
sa.Column('at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('username', sa.String(length=200), nullable=False),
|
||||
sa.Column('project_id', sa.String(length=40), nullable=True),
|
||||
sa.Column('tool', sa.String(length=40), nullable=False),
|
||||
sa.Column('event', sa.String(length=40), nullable=False),
|
||||
sa.Column('detail', sa.JSON(), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index(op.f('ix_usage_events_at'), 'usage_events', ['at'], unique=False)
|
||||
op.create_index(op.f('ix_usage_events_username'), 'usage_events', ['username'], unique=False)
|
||||
op.create_index(op.f('ix_usage_events_project_id'), 'usage_events', ['project_id'], unique=False)
|
||||
op.create_index(op.f('ix_usage_events_tool'), 'usage_events', ['tool'], unique=False)
|
||||
op.create_index(op.f('ix_usage_events_event'), 'usage_events', ['event'], unique=False)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index(op.f('ix_usage_events_event'), table_name='usage_events')
|
||||
op.drop_index(op.f('ix_usage_events_tool'), table_name='usage_events')
|
||||
op.drop_index(op.f('ix_usage_events_project_id'), table_name='usage_events')
|
||||
op.drop_index(op.f('ix_usage_events_username'), table_name='usage_events')
|
||||
op.drop_index(op.f('ix_usage_events_at'), table_name='usage_events')
|
||||
op.drop_table('usage_events')
|
||||
202
server/app.py
202
server/app.py
@@ -9,10 +9,16 @@ Run (prod): gunicorn -k uvicorn.workers.UvicornWorker -b 127.0.0.1:8000 serve
|
||||
Interactive docs: http://<host>/api/docs
|
||||
"""
|
||||
import base64
|
||||
import csv
|
||||
import contextlib
|
||||
import hashlib
|
||||
import hmac
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
@@ -22,12 +28,12 @@ from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse, RedirectResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from sqlalchemy import select, delete, func
|
||||
from sqlalchemy import select, delete, func, or_
|
||||
from sqlalchemy.orm import Session
|
||||
from starlette.middleware.sessions import SessionMiddleware
|
||||
|
||||
from .db import Base, engine, get_db
|
||||
from . import models, auth, notify, assets_db, okta_auth, okta_fake, files_http
|
||||
from .db import Base, SessionLocal, engine, get_db
|
||||
from . import models, auth, notify, assets_db, okta_auth, okta_fake, files_http, okta_sync
|
||||
|
||||
log = logging.getLogger("wpsuite.app")
|
||||
|
||||
@@ -42,7 +48,21 @@ if engine.dialect.name == "sqlite":
|
||||
# Interactive docs are handy in dev but hand an attacker the full API map in prod,
|
||||
# so enable them only on the SQLite dev fallback (production runs on Postgres).
|
||||
_docs_enabled = engine.dialect.name == "sqlite"
|
||||
@contextlib.asynccontextmanager
|
||||
async def _lifespan(_app):
|
||||
"""Start the Okta deprovisioning sync (D18 / T13.4) when the process starts and
|
||||
stop it when the process ends. With no OKTA_API_TOKEN it starts nothing and
|
||||
says so in the startup log."""
|
||||
stop = okta_sync.start_scheduler(SessionLocal)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if stop is not None:
|
||||
stop.set()
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
lifespan=_lifespan,
|
||||
title="Work Package Suite API",
|
||||
docs_url="/api/docs" if _docs_enabled else None,
|
||||
redoc_url=None,
|
||||
@@ -756,6 +776,11 @@ async def okta_callback(request: Request, db: Session = Depends(get_db)):
|
||||
user.failed_attempts = 0
|
||||
user.locked_until = None
|
||||
user.last_login_at = models.utcnow()
|
||||
# CR-019: one login event per Okta sign-in, written here rather than
|
||||
# inferred from session creation elsewhere, so there is exactly one
|
||||
# source of truth for "did this person sign in" - not one per page load
|
||||
# afterward (T11.2 covers that separately, as page_open events).
|
||||
db.add(models.UsageEvent(id=gen_id("uev"), username=user.username, tool="", event="login"))
|
||||
db.commit()
|
||||
db.refresh(user)
|
||||
|
||||
@@ -3675,6 +3700,177 @@ def create_feedback(body: CommentIn, user: models.User = Depends(auth.get_curren
|
||||
return _save_comment(body, db, user)
|
||||
|
||||
|
||||
# ── CR-019: usage/activity metrics ──────────────────────────────────────────
|
||||
class UsageIn(BaseModel):
|
||||
tool: str = ""
|
||||
project_id: Optional[str] = None
|
||||
|
||||
|
||||
@app.post("/api/usage/ping")
|
||||
def usage_ping(body: UsageIn, user: models.User = Depends(auth.get_current_user), db: Session = Depends(get_db)):
|
||||
"""One usage_events row per authenticated page load. Called exactly once,
|
||||
from auth-guard.js after wp-auth-ready fires (T11.2) - never duplicated
|
||||
per page's own script, the same lesson S4's per-page nav already taught
|
||||
this codebase. `user` comes from the session via get_current_user, never
|
||||
from anything the client claims - identity here is a server-enforced
|
||||
fact, matching every other write in this file, not a client-reported one."""
|
||||
tool = (body.tool or "").strip()[:40]
|
||||
db.add(models.UsageEvent(
|
||||
id=gen_id("uev"), username=user.username, project_id=body.project_id,
|
||||
tool=tool, event="page_open",
|
||||
))
|
||||
db.commit()
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
def _parse_date_q(v: str, end: bool = False) -> Optional[datetime]:
|
||||
"""Accepts a plain YYYY-MM-DD (what a <input type=date> sends) or a full
|
||||
ISO datetime. A date-only `to` means "through the end of that day", not
|
||||
midnight at its start - otherwise a range of "today" would match nothing
|
||||
from today at all."""
|
||||
if not v:
|
||||
return None
|
||||
try:
|
||||
d = datetime.fromisoformat(v)
|
||||
except ValueError:
|
||||
return None
|
||||
if d.tzinfo is None:
|
||||
d = d.replace(tzinfo=timezone.utc)
|
||||
if end and len(v) <= 10: # date-only
|
||||
d = d + timedelta(days=1) - timedelta(microseconds=1)
|
||||
return d
|
||||
|
||||
|
||||
def _usage_query(db: Session, caller: "models.User", date_from, date_to, project_id, username, tool):
|
||||
"""Shared by the summary and export endpoints so the two can never
|
||||
disagree about which rows a filter set matches - the export is a raw
|
||||
dump of exactly what the summary counted, not a separately-derived view."""
|
||||
stmt = select(models.UsageEvent)
|
||||
managed = managed_project_ids(db, caller)
|
||||
if managed is not None:
|
||||
# A project_super_user (never an app admin, who gets managed=None) is
|
||||
# scoped to events tied to a project they administer, plus their OWN
|
||||
# suite-wide activity (admin console opens etc. carry no project_id) -
|
||||
# never another user's activity outside what they manage.
|
||||
if managed:
|
||||
stmt = stmt.where(or_(
|
||||
models.UsageEvent.project_id.in_(managed),
|
||||
models.UsageEvent.username == caller.username,
|
||||
))
|
||||
else:
|
||||
stmt = stmt.where(models.UsageEvent.username == caller.username)
|
||||
df = _parse_date_q(date_from)
|
||||
dt = _parse_date_q(date_to, end=True)
|
||||
if df:
|
||||
stmt = stmt.where(models.UsageEvent.at >= df)
|
||||
if dt:
|
||||
stmt = stmt.where(models.UsageEvent.at <= dt)
|
||||
if project_id:
|
||||
stmt = stmt.where(models.UsageEvent.project_id == project_id)
|
||||
if username:
|
||||
stmt = stmt.where(models.UsageEvent.username == username)
|
||||
if tool:
|
||||
stmt = stmt.where(models.UsageEvent.tool == tool)
|
||||
return stmt.order_by(models.UsageEvent.at)
|
||||
|
||||
|
||||
@app.get("/api/usage/summary")
|
||||
def usage_summary(
|
||||
date_from: Optional[str] = Query(None, alias="from"),
|
||||
date_to: Optional[str] = Query(None, alias="to"),
|
||||
project_id: Optional[str] = Query(None),
|
||||
username: Optional[str] = Query(None),
|
||||
tool: Optional[str] = Query(None),
|
||||
caller: models.User = Depends(require_user_manager),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""CR-019. Same gate as the User Directory (require_user_manager): an app
|
||||
admin or a project_super_user with at least one managed project. Filters
|
||||
combine. Aggregated in Python over the filtered row set rather than a SQL
|
||||
GROUP BY - correct and simple at today's scale; if usage_events grows
|
||||
into the millions (plausible, given retention is indefinite by decision),
|
||||
the day/week/month bucketing here is the first thing to move server-side
|
||||
into SQL. Not done now because nothing currently requires it."""
|
||||
rows = db.scalars(_usage_query(db, caller, date_from, date_to, project_id, username, tool)).all()
|
||||
|
||||
by_day: dict[str, set] = {}
|
||||
by_week: dict[str, set] = {}
|
||||
by_month: dict[str, set] = {}
|
||||
per_user_last: dict[str, datetime] = {}
|
||||
per_tool: dict[str, int] = {}
|
||||
for e in rows:
|
||||
by_day.setdefault(e.at.date().isoformat(), set()).add(e.username)
|
||||
by_week.setdefault(e.at.strftime("%G-W%V"), set()).add(e.username)
|
||||
by_month.setdefault(e.at.strftime("%Y-%m"), set()).add(e.username)
|
||||
if e.username not in per_user_last or e.at > per_user_last[e.username]:
|
||||
per_user_last[e.username] = e.at
|
||||
per_tool[e.tool] = per_tool.get(e.tool, 0) + 1
|
||||
|
||||
return {
|
||||
"active_users": {
|
||||
"by_day": {k: len(v) for k, v in sorted(by_day.items())},
|
||||
"by_week": {k: len(v) for k, v in sorted(by_week.items())},
|
||||
"by_month": {k: len(v) for k, v in sorted(by_month.items())},
|
||||
},
|
||||
"per_user_last_active": {u: models._iso(t) for u, t in sorted(per_user_last.items())},
|
||||
"by_tool": dict(sorted(per_tool.items(), key=lambda kv: -kv[1])),
|
||||
"event_count": len(rows),
|
||||
}
|
||||
|
||||
|
||||
def _pseudonym(username: str) -> str:
|
||||
"""A stable per-user id for the sanitized export — the SAME input always
|
||||
produces the SAME output, within one export and across separate export
|
||||
runs, so an external system (Power BI or similar) can still group and
|
||||
trend "by user" without ever receiving a real name. HMAC rather than a
|
||||
plain hash: a plain sha256(username) is trivially reversed against a
|
||||
wordlist of the handful of usernames this app actually has; keying it
|
||||
with AUTH_SECRET_KEY (already a real secret, already required in
|
||||
production — see auth.py) means recovering a username from its
|
||||
pseudonym requires the signing key, not just guessing."""
|
||||
digest = hmac.new(auth.SECRET_KEY.encode(), username.encode(), hashlib.sha256).hexdigest()
|
||||
return "u_" + digest[:16]
|
||||
|
||||
|
||||
@app.get("/api/usage/export")
|
||||
def usage_export(
|
||||
date_from: Optional[str] = Query(None, alias="from"),
|
||||
date_to: Optional[str] = Query(None, alias="to"),
|
||||
project_id: Optional[str] = Query(None),
|
||||
username: Optional[str] = Query(None),
|
||||
tool: Optional[str] = Query(None),
|
||||
sanitize: bool = Query(False),
|
||||
caller: models.User = Depends(require_user_manager),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""CR-019. Same gate, same filters, same underlying row set as
|
||||
usage_summary() (_usage_query) — the export can never show a different
|
||||
slice of data than what the console counted for the same filters.
|
||||
|
||||
sanitize=true replaces `username` with a stable pseudonym (_pseudonym)
|
||||
and — deliberately — the `detail` column is not exported in EITHER mode.
|
||||
Every event this app writes today (login, page_open) leaves `detail`
|
||||
empty, so this costs nothing now, but it also means a future event type
|
||||
that DOES populate `detail` can't accidentally leak a real name into a
|
||||
sanitized file through a column nobody thought to scrub. If `detail`
|
||||
is ever needed in the export, it has to be sanitized explicitly, not
|
||||
assumed safe because the rest of the row was."""
|
||||
rows = db.scalars(_usage_query(db, caller, date_from, date_to, project_id, username, tool)).all()
|
||||
buf = io.StringIO()
|
||||
w = csv.writer(buf)
|
||||
w.writerow(["at", "username", "project_id", "tool", "event"])
|
||||
for e in rows:
|
||||
who = _pseudonym(e.username) if sanitize else e.username
|
||||
w.writerow([models._iso(e.at), who, e.project_id or "", e.tool, e.event])
|
||||
filename = "usage_export_%s_%s.csv" % (
|
||||
"sanitized" if sanitize else "raw", datetime.now(timezone.utc).strftime("%Y%m%d"),
|
||||
)
|
||||
return Response(
|
||||
content=buf.getvalue(), media_type="text/csv",
|
||||
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/comments")
|
||||
def list_comments(
|
||||
source: Optional[str] = Query(None),
|
||||
|
||||
@@ -191,8 +191,7 @@ def decode_token(token: str) -> Optional[dict]:
|
||||
if not token or len(token) > _MAX_TOKEN_CHARS:
|
||||
return None
|
||||
try:
|
||||
# Every token this app issues carries exp and sub (create_token,
|
||||
# maybe_refresh_token); one without them is not ours.
|
||||
# Every token this app issues carries exp and sub (create_token); one without them is not ours.
|
||||
claims = jwt.decode(token, SECRET_KEY, algorithms=[JWT_ALG],
|
||||
options={"require": ["exp", "sub"]})
|
||||
except Exception: # noqa: BLE001 — see docstring: every failure is "not signed in"
|
||||
|
||||
@@ -337,6 +337,45 @@ class AuditLog(Base):
|
||||
}
|
||||
|
||||
|
||||
class UsageEvent(Base):
|
||||
"""CR-019: append-only record of who used the suite, when, and which tool —
|
||||
navigation/session activity, not business mutations. Deliberately a SEPARATE
|
||||
table from AuditLog rather than a new `action` value there: AuditLog answers
|
||||
"who changed what" and is read by people auditing a specific record's
|
||||
history; mixing in a `page_open` row for every authenticated page load
|
||||
would make that trail noisy for its existing purpose. This table answers a
|
||||
different question — "who is active, and on what" — and CR-019's admin
|
||||
console reads from here, not from AuditLog.
|
||||
|
||||
Not a ForeignKey to `users`, matching AuditLog's own reasoning: a user who
|
||||
is later removed should still show up in historical activity rather than
|
||||
silently vanishing from it, and `D18`'s deprovisioning sync only ever sets
|
||||
`is_active=False` — it never deletes a row — so this is defensive symmetry
|
||||
rather than a live concern today.
|
||||
|
||||
Retention is indefinite (decided 2026-09-17, `decisions-2026-09-17.md`) —
|
||||
nothing purges rows written here; that is a deliberate product decision,
|
||||
not an oversight to fix later."""
|
||||
__tablename__ = "usage_events"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(40), primary_key=True)
|
||||
at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, index=True)
|
||||
username: Mapped[str] = mapped_column(String(200), default="", index=True)
|
||||
project_id: Mapped[Optional[str]] = mapped_column(String(40), nullable=True, index=True)
|
||||
# creator | wizard | field_view | dashboard | admin | directory | ...
|
||||
tool: Mapped[str] = mapped_column(String(40), default="", index=True)
|
||||
# page_open | login
|
||||
event: Mapped[str] = mapped_column(String(40), default="", index=True)
|
||||
detail: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id, "at": _iso(self.at), "username": self.username,
|
||||
"project_id": self.project_id, "tool": self.tool, "event": self.event,
|
||||
"detail": self.detail or {},
|
||||
}
|
||||
|
||||
|
||||
class AppSetting(Base):
|
||||
"""Admin-editable application settings (feature flags, SMTP config, …) stored
|
||||
as key -> JSON value. Read/written via /api/settings (admin only). Secrets like
|
||||
|
||||
479
server/okta_sync.py
Normal file
479
server/okta_sync.py
Normal file
@@ -0,0 +1,479 @@
|
||||
"""Okta Management API client - T13.2, wave 13 (D18).
|
||||
|
||||
This module asks Okta one question: who is in the org, and what is each person's
|
||||
status? It is the first half of the deprovisioning sync. It does not touch the
|
||||
database and it does not disable anyone. T13.3 does that, using the list this
|
||||
module returns.
|
||||
|
||||
Trust boundary. Sign-in uses the OIDC client (OKTA_CLIENT_ID and OKTA_CLIENT_SECRET,
|
||||
see okta_auth.py). This module uses a different credential, OKTA_API_TOKEN, a
|
||||
read-only Okta API token. The two never mix. A rotation or a leak of one does not
|
||||
touch the other.
|
||||
|
||||
Failure rule (D18, decision 4). This is the most important rule in this file:
|
||||
**an error is never an empty list.** If anything goes wrong, this module raises
|
||||
OktaSyncError. It never returns [] to mean "the call failed". T13.3 reads an empty
|
||||
list as "nobody is active", so a function that returned [] on a network error
|
||||
would disable every account on an Okta outage. The cases that raise are:
|
||||
|
||||
- the token or the org URL is not set
|
||||
- a network error or a timeout
|
||||
- any HTTP status that is not 200 (401, 403 and 429 included)
|
||||
- a body that is not a JSON list, or an item that is not shaped as expected
|
||||
- a user with no usable identity value
|
||||
- a list with no users at all (a real org is never empty; an empty list means
|
||||
the token cannot see the users)
|
||||
- a page limit reached (a cut-off list would make real users look missing)
|
||||
- a "next page" link that points to a different host
|
||||
|
||||
T13.3 (the sync job, `run_sync` below) uses that list to disable local accounts.
|
||||
Its rules are in its own docstring. In short: it only ever disables, it never
|
||||
re-enables, it writes one audit row per change, and any failure means NO change.
|
||||
|
||||
The token is read from the environment each time it is used. It is never logged,
|
||||
never put in an error message, and never returned to a browser. Errors carry the
|
||||
exception class name and the HTTP status only. httpx exceptions can include the
|
||||
request URL, so the URL is not copied into the message either.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Callable, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from . import models, okta_auth
|
||||
|
||||
log = logging.getLogger("wpsuite.okta_sync")
|
||||
|
||||
|
||||
def _make_logs_visible() -> None:
|
||||
"""Make this module's INFO lines reach `docker compose logs api`.
|
||||
|
||||
Nothing in this app configures Python logging, and gunicorn only configures its
|
||||
own loggers. So the root logger sits at WARNING with no handler, and every
|
||||
INFO line from a `wpsuite.*` logger is dropped. Found on 2026-10-16: with the
|
||||
exact production launch command, `grep "Okta sync"` found nothing on a healthy
|
||||
start, because "configured", "scheduled" and "done" are INFO lines. Warnings
|
||||
and errors (a disable, a skipped run) showed, but the lines that say the sync
|
||||
is alive did not. An operator cannot tell "off" from "on and quiet".
|
||||
|
||||
This adds one stderr handler to THIS module's logger only, and only when the
|
||||
root logger has none, so it never doubles up if logging is configured later and
|
||||
it does not change what any other module prints. (BL-028 and the general
|
||||
question of app-wide logging are separate items.)"""
|
||||
if log.handlers or logging.getLogger().handlers:
|
||||
return
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s"))
|
||||
log.addHandler(handler)
|
||||
log.setLevel(logging.INFO)
|
||||
|
||||
|
||||
_make_logs_visible()
|
||||
|
||||
# Okta's own page maximum for this endpoint. Asking for the maximum keeps a
|
||||
# 20-odd account org to a single call.
|
||||
PAGE_LIMIT = 200
|
||||
# A cut-off list is more dangerous than an error, so the loop is bounded and
|
||||
# raises when the bound is reached. 50 pages of 200 is 10,000 people.
|
||||
MAX_PAGES = 50
|
||||
TIMEOUT_SECONDS = 15.0
|
||||
# The whole fetch, all pages together, must finish inside this. T13.4 relies on it:
|
||||
# a run is always shorter than the shortest allowed interval, so one cycle can
|
||||
# never still be running when the next is due.
|
||||
TOTAL_DEADLINE_SECONDS = 120.0
|
||||
|
||||
# Which Okta *profile attribute* matches each OIDC claim that OKTA_IDENTITY_CLAIM
|
||||
# may name. The sign-in flow reads a claim from the ID token. This API returns
|
||||
# the profile instead. For the default claim, preferred_username, Okta's own
|
||||
# default maps it to profile.login. A claim that is not in this table is read
|
||||
# from profile[<claim>] (for example a custom attribute).
|
||||
# OKTA_SYNC_PROFILE_ATTRIBUTE overrides the whole table. Use it if the tenant
|
||||
# builds the claim from a different attribute.
|
||||
_CLAIM_TO_PROFILE = {
|
||||
"preferred_username": "login",
|
||||
"email": "email",
|
||||
}
|
||||
|
||||
|
||||
class OktaSyncError(Exception):
|
||||
"""The Okta user list could not be fetched with confidence. T13.3 must take no
|
||||
action on any account when it sees this."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OktaUser:
|
||||
identity: str # the value that matches users.username (lower case is NOT applied here)
|
||||
status: str # Okta's status, upper case: ACTIVE, SUSPENDED, DEPROVISIONED, ...
|
||||
|
||||
|
||||
# ── configuration ─────────────────────────────────────────────────────────────
|
||||
def _token() -> str:
|
||||
return os.getenv("OKTA_API_TOKEN", "").strip()
|
||||
|
||||
|
||||
def org_url() -> str:
|
||||
"""The Okta org base URL, for example https://yourorg.okta.com. The Management
|
||||
API lives here, not at the authorization server issuer. OKTA_ORG_URL wins if
|
||||
set. Otherwise it is the scheme and host of OKTA_ISSUER."""
|
||||
explicit = os.getenv("OKTA_ORG_URL", "").strip().rstrip("/")
|
||||
if explicit:
|
||||
return explicit
|
||||
issuer = (okta_auth.ISSUER or "").strip()
|
||||
parsed = urlparse(issuer)
|
||||
if parsed.scheme and parsed.netloc:
|
||||
return f"{parsed.scheme}://{parsed.netloc}"
|
||||
return ""
|
||||
|
||||
|
||||
def is_configured() -> bool:
|
||||
return bool(_token() and org_url().startswith("https://"))
|
||||
|
||||
|
||||
def describe() -> str:
|
||||
"""One line for the startup log. Never contains the token."""
|
||||
if not _token():
|
||||
return "Okta sync DISABLED - OKTA_API_TOKEN is not set. Deprovisioning is not detected."
|
||||
base = org_url()
|
||||
if not base:
|
||||
return "Okta sync DISABLED - no OKTA_ORG_URL and OKTA_ISSUER has no host. Deprovisioning is not detected."
|
||||
if not base.startswith("https://"):
|
||||
return f"Okta sync DISABLED - org URL {base!r} is not https. The token is never sent over http."
|
||||
return f"Okta sync configured - org {base}, identity claim {okta_auth.IDENTITY_CLAIM!r}"
|
||||
|
||||
|
||||
def _profile_attribute() -> str:
|
||||
override = os.getenv("OKTA_SYNC_PROFILE_ATTRIBUTE", "").strip()
|
||||
if override:
|
||||
return override
|
||||
claim = okta_auth.IDENTITY_CLAIM
|
||||
return _CLAIM_TO_PROFILE.get(claim, claim)
|
||||
|
||||
|
||||
# ── the client ────────────────────────────────────────────────────────────────
|
||||
def _fail(reason: str) -> "OktaSyncError":
|
||||
# One place to build the error, so no call site can add a URL or a header by
|
||||
# accident.
|
||||
return OktaSyncError(f"Okta user list could not be fetched: {reason}")
|
||||
|
||||
|
||||
def _same_origin(url: str, base: str) -> bool:
|
||||
a, b = urlparse(url), urlparse(base)
|
||||
return (a.scheme, a.netloc) == (b.scheme, b.netloc)
|
||||
|
||||
|
||||
def _parse_user(item, attribute: str) -> OktaUser:
|
||||
if not isinstance(item, dict):
|
||||
raise _fail("a list item is not an object")
|
||||
status = item.get("status")
|
||||
profile = item.get("profile")
|
||||
if not isinstance(status, str) or not status.strip():
|
||||
raise _fail("a user has no status")
|
||||
if not isinstance(profile, dict):
|
||||
raise _fail("a user has no profile")
|
||||
value = profile.get(attribute)
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise _fail(f"a user has no value for profile attribute {attribute!r}")
|
||||
return OktaUser(identity=value.strip(), status=status.strip().upper())
|
||||
|
||||
|
||||
def fetch_users(http: Optional[httpx.Client] = None) -> "list[OktaUser]":
|
||||
"""Return every user Okta lists, as OktaUser values. Raises OktaSyncError on
|
||||
any problem; never returns an empty list. `http` is for tests (a client built
|
||||
on httpx.MockTransport). In production it is None and a short-lived client is
|
||||
made here."""
|
||||
token = _token()
|
||||
base = org_url()
|
||||
if not token:
|
||||
raise _fail("OKTA_API_TOKEN is not set")
|
||||
if not base.startswith("https://"):
|
||||
raise _fail("the org URL is missing or is not https")
|
||||
attribute = _profile_attribute()
|
||||
|
||||
own_client = http is None
|
||||
client = http if http is not None else httpx.Client(timeout=TIMEOUT_SECONDS)
|
||||
headers = {"Accept": "application/json", "Authorization": f"SSWS {token}"}
|
||||
url: Optional[str] = f"{base}/api/v1/users"
|
||||
params: Optional[dict] = {"limit": PAGE_LIMIT}
|
||||
users: "list[OktaUser]" = []
|
||||
started = time.monotonic()
|
||||
try:
|
||||
for _ in range(MAX_PAGES):
|
||||
if not url:
|
||||
break
|
||||
if time.monotonic() - started > TOTAL_DEADLINE_SECONDS:
|
||||
raise _fail(f"took longer than {int(TOTAL_DEADLINE_SECONDS)} seconds in total")
|
||||
try:
|
||||
resp = client.get(url, params=params, headers=headers,
|
||||
follow_redirects=False)
|
||||
except httpx.HTTPError as exc:
|
||||
# Class name only. The exception text can carry the URL.
|
||||
raise _fail(f"network error ({type(exc).__name__})") from None
|
||||
if resp.status_code != 200:
|
||||
raise _fail(f"Okta answered HTTP {resp.status_code}")
|
||||
try:
|
||||
body = resp.json()
|
||||
except ValueError:
|
||||
raise _fail("the response is not JSON") from None
|
||||
if not isinstance(body, list):
|
||||
raise _fail("the response is not a list of users")
|
||||
users.extend(_parse_user(item, attribute) for item in body)
|
||||
|
||||
nxt = resp.links.get("next", {}).get("url")
|
||||
if not nxt:
|
||||
url = None
|
||||
break
|
||||
if not _same_origin(nxt, base):
|
||||
# Never send the token to a host that the Link header names.
|
||||
raise _fail("a next-page link points to a different host")
|
||||
url, params = nxt, None # the next link already carries its query
|
||||
else:
|
||||
# The loop used every page and the last one still named a next page.
|
||||
raise _fail(f"more than {MAX_PAGES} pages; the list may be cut off")
|
||||
finally:
|
||||
if own_client:
|
||||
client.close()
|
||||
|
||||
if not users:
|
||||
raise _fail("Okta returned no users, which is not credible for a real org")
|
||||
log.info("Okta user list fetched: %d user(s)", len(users))
|
||||
return users
|
||||
|
||||
|
||||
# ── T13.3: the sync job ───────────────────────────────────────────────────────
|
||||
# Which Okta statuses mean "this person is gone or was blocked on purpose".
|
||||
# Decided by Matt, 2026-10-16: a literal "anything that is not ACTIVE" would also
|
||||
# catch PASSWORD_EXPIRED, LOCKED_OUT and RECOVERY, which people leave on their
|
||||
# own, and because this job never re-enables, one AD password expiry would lock a
|
||||
# person out of this app until an admin stepped in. A user who is MISSING from the
|
||||
# list disables too (Okta's default list leaves out DEPROVISIONED users).
|
||||
DISABLING_STATUSES = frozenset({"SUSPENDED", "DEPROVISIONED"})
|
||||
|
||||
SYNC_ACTOR = "system:okta_sync"
|
||||
SYNC_ACTION = "user_deprovisioned_by_sync"
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncResult:
|
||||
ok: bool # False = the run took NO action at all
|
||||
error: str = "" # why, when ok is False
|
||||
okta_users: int = 0 # size of the Okta list
|
||||
checked: int = 0 # active local accounts looked at
|
||||
disabled: "list[str]" = field(default_factory=list) # usernames disabled this run
|
||||
|
||||
|
||||
def _gen_id(prefix: str) -> str:
|
||||
# Same shape as server/app.py gen_id. Not imported: app.py will import this
|
||||
# module (T13.4), and the reverse import would be circular.
|
||||
return f"{prefix}_{uuid.uuid4().hex[:12]}"
|
||||
|
||||
|
||||
def run_sync(db: Session, fetch: Optional[Callable[[], "list[OktaUser]"]] = None) -> SyncResult:
|
||||
"""Compare the Okta user list to local accounts and disable the ones Okta no
|
||||
longer vouches for.
|
||||
|
||||
The rules, none of which may be relaxed (D18):
|
||||
|
||||
1. FAIL CLOSED. If fetching raises ANY exception, or the answer cannot be
|
||||
trusted, this function changes nothing and writes nothing. A failure is
|
||||
never read as "nobody is active". The fetch happens before the database
|
||||
is read, so there is no code path from a failed fetch to a change.
|
||||
2. ONLY DISABLE. It sets is_active to False. No input makes it set True.
|
||||
An account that is already disabled is not touched and gets no new
|
||||
audit row, so a run that finds nothing new writes nothing.
|
||||
3. One AuditLog row per disabled account, actor "system:okta_sync", so it
|
||||
is never confused with an admin's action. Rows and changes are saved in
|
||||
one transaction. If saving fails, none of it is kept.
|
||||
4. Only changes are logged. A person who is still active in Okta and still
|
||||
active here produces no row.
|
||||
|
||||
One guard beyond the rules above, inside decision 4's "anything the sync can't
|
||||
confidently parse": if there are active local accounts and NONE of them
|
||||
matches any Okta user, the identity attribute is almost certainly wrong (see
|
||||
OKTA_SYNC_PROFILE_ATTRIBUTE). That is a setup mistake, not an offboarding of
|
||||
the whole company, so the run is refused and nothing changes.
|
||||
|
||||
A local account matches an Okta user by username or by email, ignoring case,
|
||||
the same two fields auth.find_user() uses. If the Okta list holds the same
|
||||
identity twice with different statuses, the account is disabled only when
|
||||
every one of them is a disabling status.
|
||||
|
||||
`fetch` is for tests. In production it is None and fetch_users() is used.
|
||||
"""
|
||||
fetch = fetch or fetch_users
|
||||
|
||||
# 1. The fetch. Nothing below runs unless it returned a list.
|
||||
try:
|
||||
okta_users = fetch()
|
||||
except Exception as exc: # an error of ANY kind, not only OktaSyncError
|
||||
msg = str(exc) if isinstance(exc, OktaSyncError) else f"unexpected {type(exc).__name__}"
|
||||
log.error("Okta sync SKIPPED, no account was changed: %s", msg)
|
||||
return SyncResult(ok=False, error=msg)
|
||||
if not okta_users:
|
||||
# fetch_users() already raises on an empty list. This covers a stand-in
|
||||
# fetch (and any future change) that returns one: same rule.
|
||||
msg = "the fetch returned no users"
|
||||
log.error("Okta sync SKIPPED, no account was changed: %s", msg)
|
||||
return SyncResult(ok=False, error=msg)
|
||||
|
||||
try:
|
||||
by_identity: "dict[str, list[str]]" = {}
|
||||
for u in okta_users:
|
||||
by_identity.setdefault(u.identity.strip().lower(), []).append(u.status.strip().upper())
|
||||
|
||||
locals_ = list(db.scalars(select(models.User).where(models.User.is_active.is_(True))))
|
||||
plan: "list[tuple[models.User, str]]" = []
|
||||
matched = 0
|
||||
for local in locals_:
|
||||
statuses: "list[str]" = []
|
||||
for key in {(local.username or "").strip().lower(), (local.email or "").strip().lower()}:
|
||||
if key:
|
||||
statuses.extend(by_identity.get(key, []))
|
||||
if statuses:
|
||||
matched += 1
|
||||
if all(st in DISABLING_STATUSES for st in statuses):
|
||||
plan.append((local, sorted(set(statuses))[0]))
|
||||
else:
|
||||
plan.append((local, "MISSING"))
|
||||
|
||||
if locals_ and matched == 0:
|
||||
msg = (f"none of {len(locals_)} active local account(s) matches any of "
|
||||
f"{len(okta_users)} Okta user(s); the identity attribute is probably wrong")
|
||||
log.error("Okta sync SKIPPED, no account was changed: %s", msg)
|
||||
return SyncResult(ok=False, error=msg, okta_users=len(okta_users), checked=len(locals_))
|
||||
|
||||
planned = [(local.username, status) for local, status in plan]
|
||||
checked = len(locals_)
|
||||
for local, status in plan:
|
||||
local.is_active = False
|
||||
db.add(models.AuditLog(
|
||||
id=_gen_id("ev"), actor=SYNC_ACTOR, action=SYNC_ACTION,
|
||||
entity_type="user", entity_id=local.id, project_id=None,
|
||||
summary=local.username[:400],
|
||||
detail={"okta_status": status, "via": "okta_sync"},
|
||||
))
|
||||
db.commit()
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
log.error("Okta sync FAILED, nothing was kept: unexpected %s", type(exc).__name__)
|
||||
return SyncResult(ok=False, error=f"unexpected {type(exc).__name__}")
|
||||
|
||||
for name, status in planned:
|
||||
log.warning("Okta sync disabled %r (Okta status: %s)", name, status)
|
||||
log.info("Okta sync done: %d Okta user(s), %d active local account(s) checked, %d disabled",
|
||||
len(okta_users), checked, len(planned))
|
||||
return SyncResult(ok=True, okta_users=len(okta_users), checked=checked,
|
||||
disabled=[name for name, _ in planned])
|
||||
|
||||
|
||||
# ── T13.4: running it on a schedule ───────────────────────────────────────────
|
||||
# The api container runs gunicorn with 2 workers, and a later deploy may run more
|
||||
# than one container. Each worker starts its own timer thread below. Without a
|
||||
# gate, all of them would call Okta every cycle and write overlapping changes.
|
||||
# The gate is one row in app_settings (key "okta_sync"). Its updated_at is "when
|
||||
# the last run was claimed". A worker claims a run with ONE atomic conditional
|
||||
# UPDATE: it succeeds only if the last claim is older than the interval. Exactly
|
||||
# one worker wins each cycle, on SQLite and on Postgres, with no lock held.
|
||||
#
|
||||
# The row survives a restart, so a restart inside the interval does not start a
|
||||
# second run. A slow run cannot overlap the next: a fetch is bounded to
|
||||
# TOTAL_DEADLINE_SECONDS (120) and the interval can never be set below
|
||||
# INTERVAL_MIN_SECONDS (300). Inside one process the timer thread runs one cycle at
|
||||
# a time, so cycles cannot stack there either.
|
||||
CLAIM_KEY = "okta_sync"
|
||||
INTERVAL_DEFAULT_SECONDS = 900 # 15 minutes (D18, proposed, not confirmed)
|
||||
INTERVAL_MIN_SECONDS = 300
|
||||
TICK_SECONDS = 60 # how often each worker checks if a run is due
|
||||
START_DELAY_SECONDS = 10 # let the app finish starting first
|
||||
|
||||
|
||||
def interval_seconds() -> int:
|
||||
"""OKTA_SYNC_INTERVAL_SECONDS, default 900. A value that is not a number uses
|
||||
the default. A value under 300 is raised to 300 (see above for why)."""
|
||||
raw = os.getenv("OKTA_SYNC_INTERVAL_SECONDS", "").strip()
|
||||
if not raw:
|
||||
return INTERVAL_DEFAULT_SECONDS
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
log.warning("OKTA_SYNC_INTERVAL_SECONDS=%r is not a whole number; using %d",
|
||||
raw, INTERVAL_DEFAULT_SECONDS)
|
||||
return INTERVAL_DEFAULT_SECONDS
|
||||
if value < INTERVAL_MIN_SECONDS:
|
||||
log.warning("OKTA_SYNC_INTERVAL_SECONDS=%d is under the minimum; using %d",
|
||||
value, INTERVAL_MIN_SECONDS)
|
||||
return INTERVAL_MIN_SECONDS
|
||||
return value
|
||||
|
||||
|
||||
def claim(db: Session, interval: int, now: Optional[datetime] = None) -> bool:
|
||||
"""Try to claim this cycle. True for exactly one caller per interval."""
|
||||
now = now or datetime.now(timezone.utc)
|
||||
cutoff = now - timedelta(seconds=interval)
|
||||
try:
|
||||
row = db.get(models.AppSetting, CLAIM_KEY)
|
||||
if row is None:
|
||||
db.add(models.AppSetting(key=CLAIM_KEY, value={}, updated_at=now))
|
||||
db.commit() # the insert is the claim
|
||||
return True
|
||||
except IntegrityError:
|
||||
db.rollback() # another worker inserted first: it won
|
||||
return False
|
||||
db.rollback() # end the read; the UPDATE starts fresh
|
||||
result = db.execute(
|
||||
update(models.AppSetting)
|
||||
.where(models.AppSetting.key == CLAIM_KEY, models.AppSetting.updated_at <= cutoff)
|
||||
.values(updated_at=now)
|
||||
)
|
||||
db.commit()
|
||||
return result.rowcount == 1
|
||||
|
||||
|
||||
def run_if_due(session_factory: Callable[[], Session],
|
||||
fetch: Optional[Callable[[], "list[OktaUser]"]] = None,
|
||||
now: Optional[datetime] = None) -> Optional[SyncResult]:
|
||||
"""One timer tick: claim the cycle, and if it is ours, run the sync. Returns
|
||||
the SyncResult, or None when the cycle was not ours (or could not be claimed).
|
||||
Nothing here can turn a failure into a change: run_sync() owns that rule, and
|
||||
anything that goes wrong around it is logged and ends the tick."""
|
||||
try:
|
||||
with session_factory() as db:
|
||||
if not claim(db, interval_seconds(), now=now):
|
||||
return None
|
||||
with session_factory() as db:
|
||||
return run_sync(db, fetch=fetch)
|
||||
except Exception as exc:
|
||||
log.error("Okta sync tick failed, no account was changed: unexpected %s", type(exc).__name__)
|
||||
return None
|
||||
|
||||
|
||||
def start_scheduler(session_factory: Callable[[], Session]) -> "Optional[threading.Event]":
|
||||
"""Start the timer thread. Returns a stop Event (set it to end the thread), or
|
||||
None when the sync is off because OKTA_API_TOKEN or the org URL is missing.
|
||||
Logs one line either way, so an operator can see it in `docker compose logs api`."""
|
||||
log.info(describe())
|
||||
if not is_configured():
|
||||
return None
|
||||
stop = threading.Event()
|
||||
|
||||
def loop():
|
||||
if stop.wait(START_DELAY_SECONDS):
|
||||
return
|
||||
while not stop.is_set():
|
||||
run_if_due(session_factory)
|
||||
if stop.wait(TICK_SECONDS):
|
||||
return
|
||||
|
||||
threading.Thread(target=loop, name="okta-sync", daemon=True).start()
|
||||
log.info("Okta sync scheduled: every %d seconds (checked each %d s by every worker; "
|
||||
"one wins each cycle)", interval_seconds(), TICK_SECONDS)
|
||||
return stop
|
||||
Reference in New Issue
Block a user