Add Micron asset picker to work package creator
Adds an optional read-only Micron asset catalog lookup for the WP creator, with searchable asset IDs, CSV import, and graceful fallback to manual asset entry when the catalog is absent or unreachable. This includes the backend /api/assets endpoint, SQL Server connector configuration, Docker network changes for outbound access, and UI updates/documentation to make the catalog read-only and clearly distinguish Micron-vetted assets from manual entries.
This commit is contained in:
@@ -30,3 +30,25 @@ AUTH_SECRET_KEY=CHANGE_ME_run_the_command_above
|
||||
# notifications are marked "skipped", nothing is sent) until both the toggle is
|
||||
# on and SMTP is configured.
|
||||
# SMTP_PASSWORD=your-smtp-app-password
|
||||
|
||||
# ── Micron asset catalog (optional) ───────────────────────────────────────────
|
||||
# Backs the searchable asset picker in the work package creator. READ-ONLY: the
|
||||
# app only ever runs the single SELECT in server/assets_db.py, so give it a
|
||||
# db_datareader login and nothing more.
|
||||
#
|
||||
# Leave this unset and the suite works normally — the picker reports that no
|
||||
# catalog is configured and people type asset tags in by hand.
|
||||
#
|
||||
# URL-encode special characters in the password (@ = %40, # = %23, / = %2F …).
|
||||
# MICRON_DB_URL=mssql+pymssql://readonly_user:PASSWORD@sqlhost.example.com:1433/MicronDB
|
||||
#
|
||||
# To use pyodbc instead of pymssql you must also add pyodbc to requirements.txt
|
||||
# and install the Microsoft ODBC driver in the image:
|
||||
# MICRON_DB_URL=mssql+pyodbc://readonly_user:PASSWORD@sqlhost.example.com/MicronDB?driver=ODBC+Driver+18+for+SQL+Server
|
||||
#
|
||||
# Two things to check when the picker says the catalog is unreachable:
|
||||
# 1. The table/column names in ASSET_QUERY (server/assets_db.py) match the real
|
||||
# Micron schema — that one constant is the whole schema contract.
|
||||
# 2. The api container is on the `outbound` network in docker-compose.yml. The
|
||||
# `internal` network has no default gateway, which blocks the VPN as well as
|
||||
# the internet.
|
||||
|
||||
@@ -25,7 +25,7 @@ from sqlalchemy import select, delete, func
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .db import Base, engine, get_db
|
||||
from . import models, auth, notify
|
||||
from . import models, auth, notify, assets_db
|
||||
|
||||
# Schema management:
|
||||
# • Local dev (SQLite) auto-creates tables for a zero-config run.
|
||||
@@ -1810,6 +1810,29 @@ def list_comments(
|
||||
return [c.to_dict() for c in rows]
|
||||
|
||||
|
||||
# ── Micron asset catalog (read-only lookup) ────────────────────────────────────
|
||||
# Backs the asset picker in the work package creator. This is a *lookup*, not a
|
||||
# resource this app owns: there is no POST, and nothing here ever writes to the
|
||||
# Micron database. It is deliberately not project-scoped by the app's own access
|
||||
# rules — the catalog is reference data, and any signed-in user who can build a
|
||||
# work package needs to be able to name the assets it covers. Authentication is
|
||||
# still required (the auth_gate middleware covers every /api/ path).
|
||||
@app.get("/api/assets")
|
||||
def list_assets(_user: models.User = Depends(auth.get_current_user)):
|
||||
"""The whole catalog, fetched once when the creator loads. Searching happens
|
||||
in the browser — there is no per-keystroke endpoint by design."""
|
||||
if not assets_db.configured():
|
||||
# Not an error — the suite is designed to run without Micron wired up.
|
||||
# The picker reads this and switches to manual entry.
|
||||
return {"configured": False, "assets": [], "detail": assets_db.status()["detail"]}
|
||||
try:
|
||||
return {"configured": True, "assets": assets_db.load()}
|
||||
except assets_db.AssetSourceError as exc:
|
||||
# 503, not 500: the suite is healthy, its upstream lookup is not. The
|
||||
# picker degrades to manual entry rather than blocking the package.
|
||||
raise HTTPException(status_code=503, detail=str(exc))
|
||||
|
||||
|
||||
# ── Local dev convenience: serve the static site from this app ──────────────────
|
||||
# In production NGINX serves html/ and only proxies /api/ here, so this app never
|
||||
# receives "/" requests, and the api Docker image doesn't even include html/ — so
|
||||
|
||||
199
server/assets_db.py
Normal file
199
server/assets_db.py
Normal file
@@ -0,0 +1,199 @@
|
||||
"""Read-only reader for the Micron asset catalog.
|
||||
|
||||
The work package creator used to ask people to paste a controls.dev link for
|
||||
every asset. Assets actually live in the Micron database — a SQL Server instance
|
||||
that is NOT part of this repo and whose schema is not managed here. This module
|
||||
gives the API a *read-only* window onto it so the creator can offer a searchable
|
||||
picker instead of free-text links.
|
||||
|
||||
How it works: the whole catalog is fetched in one query and handed to the browser
|
||||
when the creator loads. Searching then happens in the browser with no round trip
|
||||
at all. The catalog is a list of asset IDs — about 9k of them today and not
|
||||
expected past 100k — so it is small enough to send whole, and it is slow-moving
|
||||
reference data, so there is nothing to gain from querying it per keystroke and a
|
||||
lot of latency to lose. A short server-side cache keeps a room full of people
|
||||
opening the page from turning into a query each.
|
||||
|
||||
Other deliberate constraints:
|
||||
|
||||
* **Read-only, always.** The only statement in this file is the SELECT below.
|
||||
Point it at a login with `db_datareader` and nothing else.
|
||||
* **No prime_db dependency.** A plain SQLAlchemy connection built from a
|
||||
connection string, kept separate from the app's own engine in `db.py`, so a
|
||||
Micron outage can never affect the suite's own database.
|
||||
|
||||
Unconfigured is a first-class state: with no `MICRON_DB_URL` set, `configured()`
|
||||
returns False, the API says so, and the UI falls back to manual entry. The suite
|
||||
boots and runs fine without the Micron database being reachable.
|
||||
"""
|
||||
import os
|
||||
import time
|
||||
import logging
|
||||
import threading
|
||||
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ── The query ─────────────────────────────────────────────────────────────────
|
||||
# The only place the Micron schema appears; everything else here is plumbing.
|
||||
# Returns one row per asset, aliased `tag`. No row cap: the catalog is small
|
||||
# enough to hand over whole, and a partial list would silently hide assets.
|
||||
#
|
||||
# Add a WHERE clause here if some rows should never be offered at all
|
||||
# (decommissioned assets, other sites, …). Filtering at the source keeps the
|
||||
# payload small, which matters more than anything else here.
|
||||
ASSET_QUERY = """
|
||||
SELECT a.AssetID AS tag
|
||||
FROM Asset.Asset AS a
|
||||
ORDER BY a.AssetID
|
||||
"""
|
||||
|
||||
# How long a fetched catalog is reused before the next page load re-queries.
|
||||
CACHE_SECONDS = int(os.getenv("MICRON_ASSETS_CACHE_SECONDS", "300")) # 5 min
|
||||
CONNECT_TIMEOUT = 8
|
||||
|
||||
|
||||
class AssetSourceError(RuntimeError):
|
||||
"""The catalog is configured but could not be read."""
|
||||
|
||||
|
||||
# ── Engine (lazy, process-wide) ───────────────────────────────────────────────
|
||||
# A full SQLAlchemy URL, e.g.
|
||||
# mssql+pymssql://user:pass@host:1433/MicronDB
|
||||
# mssql+pyodbc://user:pass@host/MicronDB?driver=ODBC+Driver+18+for+SQL+Server
|
||||
# URL-encode any special characters in the password.
|
||||
_engine = None
|
||||
_engine_lock = threading.Lock()
|
||||
|
||||
|
||||
def _db_url() -> str:
|
||||
return os.getenv("MICRON_DB_URL", "").strip()
|
||||
|
||||
|
||||
def configured() -> bool:
|
||||
return bool(_db_url())
|
||||
|
||||
|
||||
def _validate_url(url: str) -> None:
|
||||
"""Catch the one URL mistake that produces a baffling error message.
|
||||
|
||||
A password containing an unencoded '@' makes the URL ambiguous: the parser
|
||||
splits on the first '@', so part of the password ends up parsed as the host.
|
||||
The driver then reports a connection failure against a nonsense hostname that
|
||||
happens to contain a fragment of the password — confusing to read and unsafe
|
||||
to display. Detect it here and say plainly what is wrong.
|
||||
|
||||
Nothing from the URL is included in the message; it never leaves this process.
|
||||
"""
|
||||
authority = url.split("://", 1)[-1].split("/", 1)[0]
|
||||
if authority.count("@") > 1:
|
||||
raise AssetSourceError(
|
||||
"MICRON_DB_URL is ambiguous: the username or password contains an "
|
||||
"unencoded '@'. Percent-encode the special characters — @ = %40, "
|
||||
": = %3A, / = %2F, # = %23, ? = %3F, % = %25."
|
||||
)
|
||||
|
||||
|
||||
def _connect_args(url: str) -> dict:
|
||||
"""Per-driver connect timeouts, so an unreachable Micron host fails fast
|
||||
instead of tying up a worker until the OS gives up."""
|
||||
if url.startswith("mssql+pymssql"):
|
||||
return {"login_timeout": CONNECT_TIMEOUT, "timeout": CONNECT_TIMEOUT}
|
||||
if url.startswith("mssql+pyodbc"):
|
||||
return {"timeout": CONNECT_TIMEOUT}
|
||||
return {}
|
||||
|
||||
|
||||
def _get_engine():
|
||||
global _engine
|
||||
if _engine is not None:
|
||||
return _engine
|
||||
url = _db_url()
|
||||
if not url:
|
||||
raise AssetSourceError("The Micron DB is not configured.")
|
||||
_validate_url(url)
|
||||
with _engine_lock:
|
||||
if _engine is None:
|
||||
try:
|
||||
_engine = create_engine(
|
||||
url,
|
||||
connect_args=_connect_args(url),
|
||||
pool_pre_ping=True, # a recycled dead connection retries instead of erroring
|
||||
pool_recycle=1800,
|
||||
pool_size=1, # one catalog query now and then, not a workload
|
||||
max_overflow=1,
|
||||
future=True,
|
||||
)
|
||||
except Exception as exc: # bad URL, missing driver package, …
|
||||
# See the note on load() — the exception text can echo the
|
||||
# connection string, so it is logged and not propagated.
|
||||
log.error("Micron asset catalog: could not open the connection: %s", exc)
|
||||
raise AssetSourceError(
|
||||
"Could not open a connection to the Micron DB. "
|
||||
"Check MICRON_DB_URL and the API log for the driver error."
|
||||
) from exc
|
||||
return _engine
|
||||
|
||||
|
||||
# ── Cache ─────────────────────────────────────────────────────────────────────
|
||||
# Every page load asks for the whole catalog, so without this a shift change
|
||||
# would be one full-table query per person. Held per worker process.
|
||||
_cache: list[dict] | None = None
|
||||
_cached_at = 0.0
|
||||
_cache_lock = threading.Lock()
|
||||
|
||||
|
||||
def load(force: bool = False) -> list[dict]:
|
||||
"""Return the whole catalog as [{'tag': …}, …]. Never writes."""
|
||||
global _cache, _cached_at
|
||||
with _cache_lock:
|
||||
if _cache is not None and not force and (time.monotonic() - _cached_at) < CACHE_SECONDS:
|
||||
return _cache
|
||||
|
||||
engine = _get_engine()
|
||||
try:
|
||||
with engine.connect() as conn:
|
||||
result = conn.execute(text(ASSET_QUERY)).mappings().all()
|
||||
except SQLAlchemyError as exc:
|
||||
# The driver's message is NOT propagated. AssetSourceError text reaches the
|
||||
# browser, and connection errors quote the host, the login, and — when the
|
||||
# URL is malformed — fragments of the password. Operators get the detail
|
||||
# from the API log, where it belongs; users get a message they can act on.
|
||||
log.error("Micron asset catalog query failed: %s", exc)
|
||||
raise AssetSourceError(
|
||||
"The Micron DB could not be read. Check that the host is "
|
||||
"reachable, that the login has SELECT on the asset table, and that "
|
||||
"ASSET_QUERY matches the real schema — the API log has the driver error."
|
||||
) from exc
|
||||
|
||||
# Drop rows with no identifier — an asset with no tag is not selectable and
|
||||
# would render as a blank line in the picker.
|
||||
rows = [{"tag": str(r["tag"])} for r in result if r.get("tag") not in (None, "")]
|
||||
with _cache_lock:
|
||||
_cache, _cached_at = rows, time.monotonic()
|
||||
return rows
|
||||
|
||||
|
||||
def status() -> dict:
|
||||
"""Describe the source for the UI, so it can explain itself rather than just
|
||||
showing an empty dropdown."""
|
||||
if not configured():
|
||||
return {
|
||||
"configured": False, "ok": False, "count": 0,
|
||||
"detail": "The Micron DB is not configured — enter assets manually.",
|
||||
}
|
||||
try:
|
||||
rows = load()
|
||||
except AssetSourceError as exc:
|
||||
return {"configured": True, "ok": False, "count": 0, "detail": str(exc)}
|
||||
return {"configured": True, "ok": True, "count": len(rows),
|
||||
"detail": f"{len(rows):,} asset IDs from the Micron DB."}
|
||||
@@ -9,6 +9,12 @@ gunicorn==26.0.0
|
||||
sqlalchemy==2.0.51
|
||||
alembic==1.18.5 # database migrations
|
||||
psycopg[binary]==3.3.4
|
||||
pymssql==2.3.13 # read-only lookups against the Micron asset DB (SQL Server).
|
||||
# Chosen over pyodbc because it ships self-contained wheels —
|
||||
# pyodbc would also need msodbcsql18 + unixODBC installed in
|
||||
# the image. To use pyodbc instead, add it here, install the
|
||||
# Microsoft ODBC driver in the Dockerfile, and switch
|
||||
# MICRON_DB_URL to mssql+pyodbc://…?driver=ODBC+Driver+18+for+SQL+Server
|
||||
pydantic==2.13.4
|
||||
python-dotenv==1.2.2
|
||||
bcrypt==5.0.0 # password hashing
|
||||
|
||||
Reference in New Issue
Block a user