forked from b.peck/BAT
exchange work
This commit is contained in:
@@ -20,6 +20,11 @@ Checks (stdlib only, exit non-zero on any FAIL):
|
||||
only PrimeControls; dead view params reported as failures
|
||||
|
||||
Usage: python3 tools/lint_project.py [--strict] [--project PrimeBAT]
|
||||
[--namespace PrimeControls]
|
||||
|
||||
--namespace names the single folder every view / style class / script package
|
||||
must live under. It defaults to PrimeControls (PrimeBAT); the shippable,
|
||||
de-branded copy passes --project AlarmDashboard --namespace AlarmDashboard.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
@@ -116,7 +121,9 @@ def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--strict", action="store_true")
|
||||
ap.add_argument("--project", default="PrimeBAT")
|
||||
ap.add_argument("--namespace", default="PrimeControls")
|
||||
args = ap.parse_args()
|
||||
ns = args.namespace
|
||||
|
||||
proj = os.path.join(ROOT, "ignition", "gateway", "projects", args.project)
|
||||
if not os.path.isdir(proj):
|
||||
@@ -171,8 +178,8 @@ def main():
|
||||
session_schema = sp.get("custom") or {}
|
||||
if args.strict:
|
||||
for key in session_schema:
|
||||
if key != "PrimeControls":
|
||||
fail(rel(sp_path), "session custom key %r outside PrimeControls" % key)
|
||||
if key != ns:
|
||||
fail(rel(sp_path), "session custom key %r outside %s" % (key, ns))
|
||||
|
||||
# ---------- page-config contest rules ----------
|
||||
pc_path = os.path.join(proj, PERSPECTIVE, "page-config", "config.json")
|
||||
@@ -181,18 +188,18 @@ def main():
|
||||
pages = pc.get("pages") or {}
|
||||
if list(pages.keys()) != ["/"]:
|
||||
fail(rel(pc_path), "pages must be exactly ['/'], got %s" % list(pages.keys()))
|
||||
elif (pages["/"] or {}).get("viewPath") != "PrimeControls/Dashboard":
|
||||
fail(rel(pc_path), "page '/' must map to PrimeControls/Dashboard")
|
||||
elif (pages["/"] or {}).get("viewPath") != "%s/Dashboard" % ns:
|
||||
fail(rel(pc_path), "page '/' must map to %s/Dashboard" % ns)
|
||||
if pc.get("sharedDocks"):
|
||||
fail(rel(pc_path), "sharedDocks must be empty (docked views prohibited)")
|
||||
|
||||
# ---------- namespace rule: everything under PrimeControls/ ----------
|
||||
for p in sorted(view_paths):
|
||||
if not p.startswith("PrimeControls/"):
|
||||
fail("views/%s" % p, "view outside PrimeControls/ folder")
|
||||
if not p.startswith(ns + "/"):
|
||||
fail("views/%s" % p, "view outside %s/ folder" % ns)
|
||||
for p in sorted(style_paths):
|
||||
if not p.startswith("PrimeControls/"):
|
||||
fail("style-classes/%s" % p, "style class outside PrimeControls/ folder")
|
||||
if not p.startswith(ns + "/"):
|
||||
fail("style-classes/%s" % p, "style class outside %s/ folder" % ns)
|
||||
|
||||
# ---------- per-view checks ----------
|
||||
for vp in sorted(view_paths):
|
||||
@@ -222,8 +229,8 @@ def main():
|
||||
if isinstance(path_prop, str) and path_prop:
|
||||
if path_prop not in view_paths:
|
||||
fail(rv, "%s embeds missing view %r" % (cpath, path_prop))
|
||||
elif not path_prop.startswith("PrimeControls/"):
|
||||
fail(rv, "%s embeds non-PrimeControls view %r" % (cpath, path_prop))
|
||||
elif not path_prop.startswith(ns + "/"):
|
||||
fail(rv, "%s embeds view %r outside %s/" % (cpath, path_prop, ns))
|
||||
elif not bound:
|
||||
warn(rv, "%s embed with no static path and no binding" % cpath)
|
||||
classes = ((node.get("props") or {}).get("style") or {}).get("classes")
|
||||
@@ -241,7 +248,8 @@ def main():
|
||||
if btype not in KNOWN_BINDING_TYPES:
|
||||
fail(rv, "%s %s: unknown binding type %r" % (owner, key, btype))
|
||||
if btype == "tag":
|
||||
fail(rv, "%s %s: tag binding (prohibited in PrimeBAT — journal API only)" % (owner, key))
|
||||
fail(rv, "%s %s: tag binding (prohibited in %s — journal API only)"
|
||||
% (owner, key, args.project))
|
||||
text_blobs = [json.dumps(binding.get("config") or {})]
|
||||
for tr in binding.get("transforms") or []:
|
||||
code = tr.get("code", "")
|
||||
@@ -288,8 +296,8 @@ def main():
|
||||
fail("ignition/%s" % entry, "unexpected project resource type for submission")
|
||||
if os.path.isdir(sp_root):
|
||||
for entry in sorted(os.listdir(sp_root)):
|
||||
if entry != "PrimeControls":
|
||||
fail("script-python/%s" % entry, "script package outside PrimeControls")
|
||||
if entry != ns:
|
||||
fail("script-python/%s" % entry, "script package outside %s" % ns)
|
||||
|
||||
for w in WARNS:
|
||||
print(w)
|
||||
|
||||
@@ -4,12 +4,13 @@
|
||||
The alarm simulator (SimHarness/alarmsim) generates activity in real time, so
|
||||
"give me 8 hours of alarms" would take 8 hours. This writes the same kind of
|
||||
activity straight into the journal tables with backdated timestamps, shaped to
|
||||
land on a target Alarm Health grade.
|
||||
land on a target Alarm Health grade or an exact Alarm Health score.
|
||||
|
||||
python3 tools/seed_journal.py --hours 8 --grade B --dry-run # plan only
|
||||
python3 tools/seed_journal.py --hours 8 --grade B # insert
|
||||
python3 tools/seed_journal.py --hours 8 --score 85 --dry-run # plan only
|
||||
python3 tools/seed_journal.py --hours 8 --score 85 # insert
|
||||
python3 tools/seed_journal.py --hours 8 --grade B # canned profile
|
||||
|
||||
How the grade is hit (weights from PrimeControls.calc.DEFAULTS):
|
||||
How the grade/score is hit (weights from PrimeControls.calc.DEFAULTS):
|
||||
|
||||
rate 0.30 activations/hr vs the 6/hr ISA target
|
||||
flood 0.25 % of window inside a flood episode (>10 activations/10 min)
|
||||
@@ -19,11 +20,28 @@ How the grade is hit (weights from PrimeControls.calc.DEFAULTS):
|
||||
|
||||
Only `rate`, `flood` and `priority` are seedable. `standing` reads live gateway
|
||||
alarm state via system.alarm.queryStatus, so whatever is actually active now is
|
||||
measured as-is and the profile budgets for it (--standing-now).
|
||||
measured as-is and the profile budgets for it (--standing-now). `chatter` is
|
||||
structurally 0 (SOURCE_CAP < calc's chatter_min_count).
|
||||
|
||||
--score solves for the three seedable dials instead of using a canned profile:
|
||||
total activations (rate), whether exactly one 10-min cell floods, and the exact
|
||||
low/medium/high split (priority deviation). Rows already in the window - a live
|
||||
active alarm, an earlier seed - are read back from the journal and counted as
|
||||
part of the target, so the solve describes the window the gateway will actually
|
||||
score. Two shapes are available:
|
||||
|
||||
--shape stable (default) rate held at/below the ISA target, so the rate
|
||||
sub-score sits pinned at 100 on the flat part of its curve
|
||||
and the score does NOT drift as the dashboard's rolling
|
||||
window slides forward and old activations fall out. The
|
||||
whole deficit lands on flood + priority mix.
|
||||
--shape balanced deficit spread across rate, flood and priority - a more
|
||||
typical-looking plant, but the score climbs a few tenths
|
||||
per 10 minutes as the window slides past the seeded data.
|
||||
|
||||
The plan is verified offline against the real PrimeControls.calc before anything
|
||||
is written - the same module the gateway runs - and the run aborts if the
|
||||
predicted grade misses the target.
|
||||
predicted grade/score misses the target.
|
||||
|
||||
Journal conventions this mirrors (harvested from existing rows):
|
||||
eventtype 0=active 1=clear 2=ack eventflags 0 for tag alarms
|
||||
@@ -101,6 +119,12 @@ MAX_DURATION_S = 420
|
||||
ACK_RATE = 0.85
|
||||
CLEAR_RATE = 0.94
|
||||
|
||||
# --score solver
|
||||
BURST_COUNT = 11 # > calc flood_per_10min (10) => one flooding cell
|
||||
SOLVE_N_MAX = 260 # activation ceiling searched (32/hr over 8 h)
|
||||
PRIOR_WORSE = 1.25 # prior-window activations, as a multiple of current
|
||||
HIGH_SHARE_PRIOR = 0.18 # tie-break: how much of the mix should be High/Critical
|
||||
|
||||
|
||||
# ---------------- the gateway's own calc module ----------------
|
||||
|
||||
@@ -160,6 +184,105 @@ def cell_start(ms):
|
||||
return (ms // BIN_MS) * BIN_MS
|
||||
|
||||
|
||||
def bucket_capacity(sources):
|
||||
"""Activations each bucket can absorb without any source reaching
|
||||
SOURCE_CAP - i.e. without manufacturing a chattering alarm."""
|
||||
cap = {"low": 0, "medium": 0, "high": 0}
|
||||
for rel, lvl in sources:
|
||||
cap[BUCKET_OF[lvl]] += SOURCE_CAP
|
||||
return cap
|
||||
|
||||
|
||||
def solve_spec(calc, target, hours, standing_now, existing, shape, tol, capacity):
|
||||
"""Search the seedable dial space for a window that scores `target`.
|
||||
|
||||
Dials: total activations N (rate sub-score), whether exactly one 10-min cell
|
||||
floods (flood sub-score), and the integer low/medium/high split (priority
|
||||
deviation). chatter is structurally 0 and standing is live gateway state, so
|
||||
both are inputs. Every candidate's score comes from the gateway's own
|
||||
calc.health_score, and `existing` (activations already in the window, per
|
||||
bucket) is part of the total - only the remainder gets seeded.
|
||||
|
||||
Returns a PROFILES-style spec for the seeded remainder, plus the totals and
|
||||
the predicted sub-scores for reporting.
|
||||
"""
|
||||
span_ms = int(hours * HOUR_MS)
|
||||
ex_total = sum(existing.values())
|
||||
weight_p = calc.DEFAULTS["health_weights"]["priority"]
|
||||
ptarget = calc.DEFAULTS["priority_target"]
|
||||
cands = []
|
||||
|
||||
for flood_bins in (1, 0):
|
||||
pct_flood = 100.0 * flood_bins * BIN_MS / span_ms
|
||||
burst = BURST_COUNT if flood_bins else 0
|
||||
for n in range(max(ex_total, burst, 1), SOLVE_N_MAX + 1):
|
||||
rate = n / hours
|
||||
# score(dev) is linear in the priority deviation while priority_s > 0,
|
||||
# so two probes give the deviation the target needs - no duplicated
|
||||
# copy of calc's weighting here.
|
||||
base = calc.health_score(rate, pct_flood, 0, standing_now, 0.0, True)["score"]
|
||||
slope = base - calc.health_score(rate, pct_flood, 0, standing_now,
|
||||
1.0, True)["score"]
|
||||
if slope <= 0:
|
||||
continue
|
||||
dev_needed = (base - target) / slope
|
||||
if dev_needed < 0 or dev_needed > 100.0 / max(weight_p, 1e-9):
|
||||
continue
|
||||
best = None
|
||||
hi_max = min(int(0.30 * n), existing["high"] + capacity["high"])
|
||||
med_max = min(int(0.40 * n), existing["medium"] + capacity["medium"])
|
||||
for h in range(existing["high"], hi_max + 1):
|
||||
for m in range(existing["medium"], med_max + 1):
|
||||
lo = n - m - h
|
||||
if lo < existing["low"] or lo < m or lo < h:
|
||||
continue # low stays the biggest bucket
|
||||
if lo - existing["low"] > capacity["low"]:
|
||||
continue # more low activations than sources can carry
|
||||
dev = (abs(100.0 * lo / n - ptarget["low"]) +
|
||||
abs(100.0 * m / n - ptarget["medium"]) +
|
||||
abs(100.0 * h / n - ptarget["high"]))
|
||||
key = (abs(dev - dev_needed), abs(h / float(n) - HIGH_SHARE_PRIOR))
|
||||
if best is None or key < best[0]:
|
||||
best = (key, lo, m, h, dev)
|
||||
if best is None:
|
||||
continue
|
||||
_key, lo, m, h, dev = best
|
||||
hs = calc.health_score(rate, pct_flood, 0, standing_now, dev, True)
|
||||
miss = abs(hs["score"] - target)
|
||||
if miss > tol:
|
||||
continue
|
||||
subs = dict((s["key"], s["score"]) for s in hs["subs"])
|
||||
cands.append({"flood_bins": flood_bins, "burst": burst, "n": n,
|
||||
"total": {"low": lo, "medium": m, "high": h},
|
||||
"dev": dev, "score": hs["score"], "grade": hs["grade"],
|
||||
"subs": subs, "miss": miss})
|
||||
if not cands:
|
||||
raise SystemExit(
|
||||
"no seedable window scores %.2f +/-%.2f with %d standing alarm(s) and "
|
||||
"%d activation(s) already in the window (score reachable range with "
|
||||
"these inputs is roughly 60-100; wipe the window or adjust --hours)"
|
||||
% (target, tol, standing_now, ex_total))
|
||||
|
||||
def rank(c):
|
||||
# accuracy first, in 0.02-score buckets (the UI shows one decimal), then
|
||||
# the shape preference, then more data over less.
|
||||
bucket = int(c["miss"] / 0.02)
|
||||
if shape == "stable":
|
||||
shape_cost = (0 if c["subs"]["rate"] >= 100.0 else 1,
|
||||
abs(c["total"]["high"] / float(c["n"]) - HIGH_SHARE_PRIOR))
|
||||
else:
|
||||
# spread the deficit: penalise deep single-sub holes quadratically
|
||||
spread = sum((100.0 - c["subs"][k]) ** 2 * w for k, w in
|
||||
calc.DEFAULTS["health_weights"].items()
|
||||
if c["subs"].get(k) is not None)
|
||||
shape_cost = (round(spread / 100.0), 0.0)
|
||||
return (bucket, shape_cost, -c["n"])
|
||||
|
||||
best = sorted(cands, key=rank)[0]
|
||||
best["counts"] = dict((b, best["total"][b] - existing[b]) for b in best["total"])
|
||||
return best
|
||||
|
||||
|
||||
def allocate(sources, counts, burst_count, burst_area, rng):
|
||||
"""Pick (rel, level) for every activation, honouring the exact bucket split
|
||||
and SOURCE_CAP. Returns (burst_picks, baseline_picks)."""
|
||||
@@ -194,12 +317,14 @@ def allocate(sources, counts, burst_count, burst_area, rng):
|
||||
return burst, baseline
|
||||
|
||||
|
||||
def place_times(start_ms, end_ms, burst, baseline, burst_at_ms, rng):
|
||||
def place_times(start_ms, end_ms, burst, baseline, burst_at_ms, rng, pre_cells=None):
|
||||
"""Assign an activation timestamp to every pick.
|
||||
|
||||
Burst picks land inside the single 10-minute cell containing burst_at_ms
|
||||
(with an edge margin so they cannot spill into the neighbouring cell).
|
||||
Baseline picks spread over the remaining cells, capped at BASELINE_BIN_CAP.
|
||||
`pre_cells` seeds the per-cell counter with activations already in the
|
||||
journal, so the cap accounts for them too.
|
||||
"""
|
||||
burst_cell = cell_start(burst_at_ms)
|
||||
events = []
|
||||
@@ -225,7 +350,7 @@ def place_times(start_ms, end_ms, burst, baseline, burst_at_ms, rng):
|
||||
raise SystemExit("baseline %d exceeds capacity %d (cap %d/cell over %d cells)"
|
||||
% (len(baseline), capacity, BASELINE_BIN_CAP, len(cells)))
|
||||
|
||||
per_cell = collections.Counter()
|
||||
per_cell = collections.Counter(pre_cells or {})
|
||||
for pick in baseline:
|
||||
# Free choice among under-cap cells, so activity clumps the way real
|
||||
# activity does. BASELINE_BIN_CAP (4) is below both the flood-start
|
||||
@@ -268,14 +393,14 @@ def build_instances(events, end_ms, rng):
|
||||
return instances
|
||||
|
||||
|
||||
def plan_window(sources, spec, start_ms, end_ms, rng):
|
||||
def plan_window(sources, spec, start_ms, end_ms, rng, pre_cells=None):
|
||||
counts = spec["counts"]
|
||||
burst_hours, burst_count = spec["burst"]
|
||||
burst, baseline = allocate(sources, counts, burst_count,
|
||||
spec.get("burst_area"), rng)
|
||||
burst_at = end_ms - int(burst_hours * HOUR_MS)
|
||||
events, burst_cell, per_cell = place_times(start_ms, end_ms, burst, baseline,
|
||||
burst_at, rng)
|
||||
burst_at, rng, pre_cells)
|
||||
instances = build_instances(events, end_ms, rng)
|
||||
return instances, burst_cell, per_cell
|
||||
|
||||
@@ -371,6 +496,61 @@ def max_id():
|
||||
return int(out.split("\n")[0])
|
||||
|
||||
|
||||
def parse_local(text):
|
||||
"""'YYYY-MM-DD HH:MM:SS' in GATEWAY_TZ (how the journal stores it) -> ms."""
|
||||
dt = datetime.strptime(text.strip(), "%Y-%m-%d %H:%M:%S")
|
||||
if ZoneInfo is not None:
|
||||
dt = dt.replace(tzinfo=ZoneInfo(GATEWAY_TZ))
|
||||
return int(dt.timestamp() * 1000)
|
||||
|
||||
|
||||
STATE_OF_EVENTTYPE = {0: "active", 1: "clear", 2: "ack"}
|
||||
|
||||
|
||||
def read_existing(calc, start_ms, end_ms):
|
||||
"""Journal rows already in [start_ms, end_ms], shaped like the dicts
|
||||
alarms._norm_event hands calc - so the offline prediction sees the window the
|
||||
gateway will actually score, not just the rows this run adds.
|
||||
|
||||
Ack attribution is left off: it feeds MTTA/ack-user lists, never health.
|
||||
"""
|
||||
out = mariadb("SELECT eventid,source,priority,eventtype,eventtime FROM %s "
|
||||
"WHERE eventtime >= '%s' AND eventtime <= '%s' ORDER BY eventtime;"
|
||||
% (EVENTS_TABLE, local_dt(start_ms), local_dt(end_ms)))
|
||||
rows = []
|
||||
for line in (out or "").split("\n"):
|
||||
if not line.strip():
|
||||
continue
|
||||
eventid, source, priority, etype, eventtime = line.split("\t")
|
||||
state = STATE_OF_EVENTTYPE.get(int(etype))
|
||||
if state is None:
|
||||
continue
|
||||
lvl, name = calc.normalize_priority(int(priority))
|
||||
rows.append({"event_id": eventid, "source": source, "display_path": "",
|
||||
"priority": lvl, "priority_name": name, "state": state,
|
||||
"ts": parse_local(eventtime), "ack_user": None,
|
||||
"is_system": source.startswith("evt:") or ":/alm:" not in source})
|
||||
return rows
|
||||
|
||||
|
||||
def existing_activations(rows, start_ms, end_ms):
|
||||
"""(bucket counts, per-10-min-cell counts) for the real activations already
|
||||
inside the scored window - the head start the solver has to plan around."""
|
||||
buckets = {"low": 0, "medium": 0, "high": 0}
|
||||
cells = collections.Counter()
|
||||
for r in rows:
|
||||
if r["is_system"] or r["state"] != "active":
|
||||
continue
|
||||
if not (start_ms <= r["ts"] <= end_ms):
|
||||
continue
|
||||
bucket = BUCKET_OF.get(r["priority"])
|
||||
if bucket is None:
|
||||
continue # unbucketed priority: calc ignores it
|
||||
buckets[bucket] += 1
|
||||
cells[cell_start(r["ts"])] += 1
|
||||
return buckets, cells
|
||||
|
||||
|
||||
# ---------------- reporting ----------------
|
||||
|
||||
|
||||
@@ -383,13 +563,44 @@ def show_health(label, bundle):
|
||||
% (sub["key"], sub["weight"], score, sub["detail"]))
|
||||
|
||||
|
||||
def prior_spec(total_counts, burst_count, burst_hours, existing_prior, capacity):
|
||||
"""A deliberately worse preceding window, so the period-over-period delta
|
||||
chips read as improving. Never scored - only the chips consume it, so its
|
||||
counts just get clamped to what the source pool can carry."""
|
||||
counts = {}
|
||||
for bucket in total_counts:
|
||||
want = int(round(total_counts[bucket] * PRIOR_WORSE))
|
||||
counts[bucket] = max(0, min(want - existing_prior.get(bucket, 0),
|
||||
capacity[bucket]))
|
||||
burst = burst_count + 2 if burst_count else 0
|
||||
if sum(counts.values()) < burst:
|
||||
burst = 0
|
||||
return {"counts": counts, "burst": (burst_hours + 0.4, burst),
|
||||
"burst_area": "BoilerHouse"}
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--hours", type=float, default=8.0,
|
||||
help="window length; must match the dashboard preset (default 8)")
|
||||
ap.add_argument("--grade", default="B", choices=sorted(PROFILES),
|
||||
help="target Alarm Health grade (default B)")
|
||||
ap.add_argument("--score", type=float, default=None,
|
||||
help="target Alarm Health score (e.g. 85). Solves for the "
|
||||
"seedable dials instead of using a canned profile")
|
||||
ap.add_argument("--grade", default=None, choices=sorted(PROFILES),
|
||||
help="target Alarm Health grade via a canned profile "
|
||||
"(assumed to be B when --score is omitted)")
|
||||
ap.add_argument("--shape", default="stable", choices=("stable", "balanced"),
|
||||
help="--score only: stable holds the rate sub-score pinned at 100 so "
|
||||
"the score does not drift as the rolling window slides; balanced "
|
||||
"spreads the deficit across rate/flood/priority (default stable)")
|
||||
ap.add_argument("--tolerance", type=float, default=0.05,
|
||||
help="--score only: allowed miss on the predicted score; the UI shows "
|
||||
"one decimal (default 0.05)")
|
||||
ap.add_argument("--burst-hours", type=float, default=2.2,
|
||||
help="--score only: put the flood burst this many hours before the "
|
||||
"window end - it survives window drift until it ages out "
|
||||
"(default 2.2)")
|
||||
ap.add_argument("--standing-now", type=int, default=1,
|
||||
help="live alarms active >24h, which cost 20 pts each on the "
|
||||
"standing sub-score. NOT seedable - standing() reads "
|
||||
@@ -397,39 +608,101 @@ def main():
|
||||
"the SimHarness ProbeTick probe (default 1)")
|
||||
ap.add_argument("--no-prior", action="store_true",
|
||||
help="skip the preceding comparison window (delta chips go 'new')")
|
||||
ap.add_argument("--attempts", type=int, default=6,
|
||||
help="re-plan with a fresh rng draw this many times before giving up "
|
||||
"(placement can interact with rows already in the window)")
|
||||
ap.add_argument("--seed", type=int, default=20260731)
|
||||
ap.add_argument("--dry-run", action="store_true",
|
||||
help="plan and predict only; write nothing")
|
||||
args = ap.parse_args()
|
||||
if args.score is None and args.grade is None:
|
||||
args.grade = "B"
|
||||
|
||||
calc = load_calc()
|
||||
sources = load_sources()
|
||||
rng = random.Random(args.seed)
|
||||
|
||||
now_ms = int(datetime.now().timestamp() * 1000)
|
||||
span = int(args.hours * HOUR_MS)
|
||||
end_ms, start_ms = now_ms, now_ms - span
|
||||
prior_start = start_ms - span
|
||||
|
||||
profile = PROFILES[args.grade]
|
||||
standing_n = args.standing_now
|
||||
|
||||
print("window %s -> %s (%.1f h)"
|
||||
% (local_dt(start_ms), local_dt(end_ms), args.hours))
|
||||
print("sources %d real sim alarms (bad-actor tags excluded)" % len(sources))
|
||||
|
||||
cur, burst_cell, per_cell = plan_window(sources, profile["current"],
|
||||
start_ms, end_ms, rng)
|
||||
instances = list(cur)
|
||||
if not args.no_prior:
|
||||
pri, _, _ = plan_window(sources, profile["prior"], prior_start, start_ms, rng)
|
||||
instances = pri + instances
|
||||
|
||||
standing_n = args.standing_now
|
||||
# Rows already in the window (a live active alarm, an earlier seed) are part
|
||||
# of what the gateway will score, so they are part of the target.
|
||||
existing_rows = read_existing(calc, prior_start, end_ms)
|
||||
ex_buckets, ex_cells = existing_activations(existing_rows, start_ms, end_ms)
|
||||
ex_prior, _ = existing_activations(existing_rows, prior_start, start_ms - 1)
|
||||
print("existing %d journal row(s) in the window -> %d activation(s) %d/%d/%d "
|
||||
"low/medium/high (folded into the target)"
|
||||
% (len([r for r in existing_rows if start_ms <= r["ts"] <= end_ms]),
|
||||
sum(ex_buckets.values()), ex_buckets["low"], ex_buckets["medium"],
|
||||
ex_buckets["high"]))
|
||||
print("standing %d live alarm(s) >24h (measured, not seeded)" % standing_n)
|
||||
|
||||
rows = to_journal_rows(instances, calc)
|
||||
active_now = synthetic_standing(calc, standing_n, now_ms)
|
||||
bundle = predict(calc, rows, active_now, start_ms, end_ms, now_ms)
|
||||
if args.score is not None:
|
||||
capacity = bucket_capacity(sources)
|
||||
solved = solve_spec(calc, args.score, args.hours, standing_n, ex_buckets,
|
||||
args.shape, args.tolerance, capacity)
|
||||
spec_current = {"counts": solved["counts"],
|
||||
"burst": (args.burst_hours, solved["burst"]),
|
||||
"burst_area": "Packaging"}
|
||||
spec_prior = prior_spec(solved["total"], solved["burst"], args.burst_hours,
|
||||
ex_prior, capacity)
|
||||
t = solved["total"]
|
||||
print()
|
||||
print("solved (%s shape) for score %.2f:" % (args.shape, args.score))
|
||||
print(" %d activations in window (%.2f/hr), %d/%d/%d low/medium/high, "
|
||||
"dev %.2f, %d flooding 10-min cell(s)"
|
||||
% (solved["n"], solved["n"] / args.hours, t["low"], t["medium"],
|
||||
t["high"], solved["dev"], solved["flood_bins"]))
|
||||
print(" sub-scores rate %.1f flood %.1f chatter %.1f standing %.1f "
|
||||
"priority %.1f -> %.4f (%s)"
|
||||
% (solved["subs"]["rate"], solved["subs"]["flood"],
|
||||
solved["subs"]["chatter"], solved["subs"]["standing"],
|
||||
solved["subs"]["priority"], solved["score"], solved["grade"]))
|
||||
print(" to seed: %d/%d/%d low/medium/high (%d activations)"
|
||||
% (solved["counts"]["low"], solved["counts"]["medium"],
|
||||
solved["counts"]["high"], sum(solved["counts"].values())))
|
||||
else:
|
||||
profile = PROFILES[args.grade]
|
||||
spec_current = profile["current"]
|
||||
spec_prior = profile["prior"]
|
||||
|
||||
def attempt(seed):
|
||||
rng = random.Random(seed)
|
||||
cur, _burst_cell, per_cell = plan_window(sources, spec_current, start_ms,
|
||||
end_ms, rng, ex_cells)
|
||||
instances = list(cur)
|
||||
if not args.no_prior:
|
||||
pri, _, _ = plan_window(sources, spec_prior, prior_start, start_ms, rng)
|
||||
instances = pri + instances
|
||||
rows = existing_rows + to_journal_rows(instances, calc)
|
||||
active_now = synthetic_standing(calc, standing_n, now_ms)
|
||||
bundle = predict(calc, rows, active_now, start_ms, end_ms, now_ms)
|
||||
return cur, instances, rows, per_cell, bundle
|
||||
|
||||
def missed(bundle):
|
||||
if args.score is not None:
|
||||
s = bundle["health"]["score"]
|
||||
if s is None:
|
||||
return "no score (empty window)"
|
||||
if abs(s - args.score) > args.tolerance:
|
||||
return "score %.4f != %.2f +/-%.2f" % (s, args.score, args.tolerance)
|
||||
return None
|
||||
g = bundle["health"]["grade"]
|
||||
return None if g == args.grade else "grade %s != %s" % (g, args.grade)
|
||||
|
||||
why = None
|
||||
for i in range(max(1, args.attempts)):
|
||||
cur, instances, rows, per_cell, bundle = attempt(args.seed + i)
|
||||
why = missed(bundle)
|
||||
if why is None:
|
||||
break
|
||||
print(" attempt %d missed (%s); re-planning" % (i + 1, why))
|
||||
|
||||
m = bundle["meta"]
|
||||
print()
|
||||
@@ -444,18 +717,22 @@ def main():
|
||||
pct = bundle["priority"]["pct"]
|
||||
print(" priority %.1f/%.1f/%.1f vs 80/15/5 (dev %.1f)"
|
||||
% (pct["low"], pct["medium"], pct["high"], bundle["priority"]["sum_abs_dev"]))
|
||||
print(" busiest 10-min cell (non-burst): %d burst cell: %d"
|
||||
% (max(per_cell.values()) if per_cell else 0,
|
||||
profile["current"]["burst"][1]))
|
||||
print(" busiest 10-min cell (burst cell excluded): %d burst cell: %d"
|
||||
% (max([c for k, c in per_cell.items()
|
||||
if c < spec_current["burst"][1] or not spec_current["burst"][1]] or [0]),
|
||||
spec_current["burst"][1]))
|
||||
show_health("health", bundle)
|
||||
|
||||
grade = bundle["health"]["grade"]
|
||||
if grade != args.grade:
|
||||
raise SystemExit("\nABORT: predicted grade %s != target %s; nothing written."
|
||||
% (grade, args.grade))
|
||||
print("\nprediction matches target grade %s" % args.grade)
|
||||
if why is not None:
|
||||
raise SystemExit("\nABORT after %d attempt(s): %s; nothing written."
|
||||
% (max(1, args.attempts), why))
|
||||
if args.score is not None:
|
||||
print("\nprediction matches target score %.2f (%.4f, grade %s)"
|
||||
% (args.score, bundle["health"]["score"], bundle["health"]["grade"]))
|
||||
else:
|
||||
print("\nprediction matches target grade %s" % args.grade)
|
||||
|
||||
event_rows = sum(1 for _ in rows)
|
||||
event_rows = len(rows) - len(existing_rows)
|
||||
if args.dry_run:
|
||||
print("dry run: would insert %d event rows for %d activations "
|
||||
"(%d in the scored window)"
|
||||
|
||||
Reference in New Issue
Block a user