#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ verify_scores.py — the only source of numbers for the `brainstorm` skill. WHY THIS FILE EXISTS A model asked to average six weighted scores across forty ideas will produce a confident, well-formatted, wrong table. That is not a hypothetical: hand-computed rankings in earlier runs contained arithmetic errors that nobody caught, because the output looked exactly like a correct one. So: no score, average, count or rank is ever written by the model. It comes from here, recomputed from the raw axes every time. Stored sigma values are never trusted — not even the ones this script wrote a minute ago. THE HARD GATE The skill declares a bias audit to be mandatory. Declaring it was not enough: it ran on 5 of 9 runs. Now, until the state file records `bias_audit.done = true`, every ranking this script prints carries a PRELIMINARY watermark. The model cannot clear that by asserting the audit happened; only `--bias-done` clears it. USAGE python verify_scores.py rank + KPIs (+ watermark) python verify_scores.py --all rank + superlatives + weight sweep python verify_scores.py --init create a valid empty state file python verify_scores.py --bias-done record the audit as performed python verify_scores.py --superlatives ties at axis maxima python verify_scores.py --profile NAME re-rank under another weighting python verify_scores.py --write persist recomputed sigma LICENSE MIT. """ import json import os import sys import datetime if hasattr(sys.stdout, "reconfigure"): sys.stdout.reconfigure(encoding="utf-8", errors="replace") AXES = ["E", "R", "F", "S", "P", "K"] # Weight profiles. `default` is deliberately unopinionated; the other two exist so a # ranking can be tested against a different worldview. An item that only wins under one # profile is an artefact of the weights, not a finding — the sweep below surfaces those. PROFILES = { "default": {"E": 0.25, "R": 0.20, "F": 0.15, "S": 0.15, "P": 0.15, "K": 0.10}, "growth-first": {"E": 0.40, "R": 0.15, "F": 0.15, "S": 0.05, "P": 0.20, "K": 0.05}, "safe-first": {"E": 0.15, "R": 0.30, "F": 0.15, "S": 0.15, "P": 0.05, "K": 0.20}, } LABELS = { "BIZ": {"E": "effect", "R": "risk", "F": "feasibility", "S": "speed", "P": "strategic fit", "K": "cost"}, "TECH": {"E": "impact", "R": "risk", "F": "feasibility", "S": "speed", "P": "maintainability", "K": "cost"}, "PERSONAL": {"E": "payoff", "R": "risk", "F": "feasibility", "S": "speed", "P": "values fit", "K": "cost"}, "DECISION": {"E": "upside", "R": "downside", "F": "reversibility", "S": "speed", "P": "fit", "K": "cost"}, } WILD_LEVELS = ("safe", "stretch", "wild") WILD_QUOTA = 0.15 # below this the field is a to-do list, not a brainstorm EXEC_NOBODY = "nobody" # a valid answer — marks a proposal without an owner # ───────────────────────────────────────────────────────────────────────────── # core maths # ───────────────────────────────────────────────────────────────────────────── def normalize(weights): """Weights must sum to 1. A state file edited by hand rarely does.""" total = sum(weights.get(a, 0) for a in AXES) if total <= 0: raise SystemExit("ERROR: weights sum to zero — cannot rank.") return {a: weights.get(a, 0) / total for a in AXES} def sigma(idea, weights): """Weighted score. Recomputed from `axes` every time — a stored sigma is a claim, not a fact, and this function exists precisely so that claims are not load-bearing.""" ax = idea.get("axes") or {} missing = [a for a in AXES if a not in ax] if missing: raise SystemExit("ERROR: idea id=%s is missing axes %s" % (idea.get("id"), missing)) return round(sum(float(ax[a]) * weights[a] for a in AXES), 3) def rank(ideas, weights): """Deterministic order: sigma desc, then first axis desc, then id asc. The tiebreak matters more than it looks. Without it, two runs over identical data can print different orders, and the difference reads as new information.""" scored = [(sigma(i, weights), float((i.get("axes") or {}).get("E", 0)), -int(i["id"]), i) for i in ideas] scored.sort(key=lambda t: (t[0], t[1], t[2]), reverse=True) return [(s, i) for s, _, _, i in scored] # ───────────────────────────────────────────────────────────────────────────── # the gate # ───────────────────────────────────────────────────────────────────────────── def watermark(state): """Returns the PRELIMINARY banner, or empty string once the audit is recorded. This is the whole enforcement mechanism. It is three lines because it has to be impossible to argue with: either the state file says the audit ran, or every ranking printed says it did not.""" if (state.get("bias_audit") or {}).get("done"): return "" return ("\n⚠ PRELIMINARY — bias audit has not run. The ranking below is not final.\n" " Run the audit, write findings into state, then: --bias-done\n") def do_bias_done(path, state): findings = (state.get("bias_audit") or {}).get("findings") if not findings: print("REFUSED: bias_audit.findings is empty.") print(" The flag records that an audit happened. Setting it without findings") print(" is the exact failure this gate exists to prevent. Write findings first.") sys.exit(2) state["bias_audit"]["done"] = True state["bias_audit"]["date"] = datetime.date.today().isoformat() json.dump(state, open(path, "w", encoding="utf-8"), ensure_ascii=False, indent=1) print("OK — bias audit recorded %s. Watermark cleared." % state["bias_audit"]["date"]) # ───────────────────────────────────────────────────────────────────────────── # reports # ───────────────────────────────────────────────────────────────────────────── def report_rank(state, weights, limit=20): ideas = state["ideas"] labels = state.get("criteria_labels") or LABELS.get(state.get("criteria_set"), LABELS["BIZ"]) ranked = rank(ideas, weights) print(watermark(state), end="") print("\n%s · %s · profile %s · n=%d" % (state.get("slug", "?"), state.get("criteria_set", "?"), state.get("weights_profile", "default"), len(ideas))) print("axes: " + " · ".join("%s=%s" % (a, labels.get(a, a)) for a in AXES)) print("-" * 78) for pos, (s, i) in enumerate(ranked[:limit], 1): flag = "!" if i.get("exec") == EXEC_NOBODY else " " print("%3d. %-5.3f %s [%s] %s" % (pos, s, flag, i.get("wild", "?")[:6], i.get("name", "")[:52])) top = [s for s, _ in ranked] if len(top) > 3: spread = top[0] - top[3] if spread < 0.25: print("\nBAND: top 4 differ by %.3f — read as a tie, not a ranking." % spread) print("\nrange: %.3f — %.3f [derived: one reviewer, written rubric, weights fixed" " before scoring. Not a measurement.]" % (top[-1], top[0])) return ranked def report_kpis(state): ideas = state["ideas"] n = len(ideas) wild = sum(1 for i in ideas if i.get("wild") == "wild") stretch = sum(1 for i in ideas if i.get("wild") == "stretch") nobody = sum(1 for i in ideas if i.get("exec") == EXEC_NOBODY) unknown = sum(1 for i in ideas if not i.get("exec")) nomech = sum(1 for i in ideas if not (i.get("mech") or "").strip()) print("\nKPI") print(" wild %d/%d = %.0f%% %s" % ( wild, n, 100.0 * wild / n, "OK" if wild / n >= WILD_QUOTA else "BELOW %.0f%% QUOTA — field is too safe" % (WILD_QUOTA * 100))) print(" stretch %d · safe %d" % (stretch, n - wild - stretch)) print(" no executor: %d (valid answer, but they are proposals, not plans)" % nobody) if unknown: print(" MISSING exec field: %d — cannot tell proposal from plan" % unknown) if nomech: print(" MISSING mechanism: %d — these got scored on how they sound" % nomech) def report_superlatives(state, weights): """Ties at axis maxima. Exists because hand-written 'the highest X' claims were false often enough to be worth a subcommand.""" ideas = state["ideas"] print("\nSUPERLATIVES (verify before writing 'highest' / 'the only one')") for a in AXES: best = max(float((i.get("axes") or {}).get(a, 0)) for i in ideas) holders = [i for i in ideas if float((i.get("axes") or {}).get(a, 0)) == best] note = "UNIQUE" if len(holders) == 1 else "TIE ×%d — do not write 'the only'" % len(holders) print(" %s max=%g %s" % (a, best, note)) for h in holders[:4]: print(" id=%s %s" % (h["id"], (h.get("name") or "")[:52])) def report_sweep(state): """How much of the ranking is the ideas, and how much is the weights.""" ideas = state["ideas"] tops = {} for name, prof in PROFILES.items(): tops[name] = [i["id"] for _, i in rank(ideas, normalize(prof))[:10]] stable = set(tops["default"]) for name in tops: stable &= set(tops[name]) print("\nWEIGHT SWEEP — top 10 under each profile") for name, ids in tops.items(): print(" %-13s %s" % (name, ids)) print(" stable across all profiles: %s" % sorted(stable)) fragile = set(tops["default"]) - stable if fragile: print(" ONLY under 'default': %s ← these are weight artefacts, not findings" % sorted(fragile)) # ───────────────────────────────────────────────────────────────────────────── # init # ───────────────────────────────────────────────────────────────────────────── def do_init(path, argv): cset = argv[argv.index("--set") + 1] if "--set" in argv else "BIZ" if cset not in LABELS: raise SystemExit("ERROR: unknown criteria set %r (use %s)" % (cset, "/".join(LABELS))) if os.path.exists(path): raise SystemExit("ERROR: %s already exists — refusing to overwrite a run in progress." % path) skeleton = { "problem": "FILL IN — one sentence", "slug": os.path.basename(path).replace("_state.json", ""), "created": datetime.date.today().isoformat(), "mode": "light|standard|deep — FILL IN", "criteria_set": cset, "criteria_labels": LABELS[cset], "weights": PROFILES["default"], "weights_profile": "default", "categories": [], "commit_slot": "FILL IN — when will the choice be made? (asked at F0 on purpose)", "anchors": [], "ideas": [{"id": 1, "cat": "", "name": "", "mech": "1-2 sentences on how it works", "wild": "safe", "exec": EXEC_NOBODY, "axes": {a: 5 for a in AXES}, "sigma": None, "desc": None}], "promoted": [], "initiatives": [], "bias_audit": {"done": False, "date": None, "findings": None}, "phase_done": ["F0"], } d = os.path.dirname(os.path.abspath(path)) if d: os.makedirs(d, exist_ok=True) json.dump(skeleton, open(path, "w", encoding="utf-8"), ensure_ascii=False, indent=1) print("OK — state skeleton (%s) written to %s" % (cset, path)) print("Fill in: problem, mode, categories, commit_slot, ideas (wild + exec are required).") # ───────────────────────────────────────────────────────────────────────────── def main(): if len(sys.argv) < 2: print(__doc__) sys.exit(1) path = sys.argv[1] argv = sys.argv flags = set(a for a in argv[2:] if a.startswith("--")) if "--init" in flags: do_init(path, argv) return if not os.path.exists(path): raise SystemExit("ERROR: %s not found. Create one with --init." % path) state = json.load(open(path, encoding="utf-8")) if "--bias-done" in flags: do_bias_done(path, state) return profile = argv[argv.index("--profile") + 1] if "--profile" in argv else None weights = normalize(PROFILES[profile] if profile else state["weights"]) ranked = report_rank(state, weights) report_kpis(state) if "--all" in flags or "--superlatives" in flags: report_superlatives(state, weights) if "--all" in flags: report_sweep(state) if "--write" in flags: by_id = {i["id"]: s for s, i in ranked} for i in state["ideas"]: i["sigma"] = by_id[i["id"]] json.dump(state, open(path, "w", encoding="utf-8"), ensure_ascii=False, indent=1) print("\nOK — recomputed sigma written back to %s" % path) print(watermark(state), end="") if __name__ == "__main__": main()