v5.0: 11-judge error-detection thesis, 48-check verification script
This commit is contained in:
root
Executable
+220
@@ -0,0 +1,220 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Ad-hoc verification for VerdictTank v5.0 proposal page.
|
||||
|
||||
Not a test suite. Checks the four things that can actually be wrong here:
|
||||
1. Content rules (no em/en dashes, no double hyphens in prose, no excluded model)
|
||||
2. HTML structural integrity (tag balance, required sections, no dangling anchors)
|
||||
3. Factual grounding (every headline number traces to the validation data)
|
||||
4. Deploy parity (local == served == remote, archive intact)
|
||||
"""
|
||||
import hashlib
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
from html.parser import HTMLParser
|
||||
|
||||
LOCAL = "/root/projects/verdicttank/index-v5.0.html"
|
||||
BASE = "https://proposals.itpropartner.com/verdicttank"
|
||||
SSH = ["ssh", "-o", "StrictHostKeyChecking=no", "-i", "/root/.ssh/itpp-infra",
|
||||
"root@152.53.241.111"]
|
||||
DOCROOT = "/home/ippadmin/htdocs/proposals.itpropartner.com/verdicttank"
|
||||
|
||||
results = []
|
||||
|
||||
|
||||
def check(name, ok, detail=""):
|
||||
results.append((name, ok, detail))
|
||||
print(f"{'PASS' if ok else 'FAIL'} {name}" + (f" | {detail}" if detail else ""))
|
||||
|
||||
|
||||
class TextExtractor(HTMLParser):
|
||||
"""Rendered-text extractor: skips style/script so CSS class names
|
||||
(.callout--fail, .stat--hero) don't produce false double-hyphen hits."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.text, self.skip = [], False
|
||||
self.stack, self.mismatched = [], []
|
||||
self.ids, self.hrefs = set(), []
|
||||
self.void = {"meta", "br", "img", "hr", "input", "link", "source"}
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
a = dict(attrs)
|
||||
if a.get("id"):
|
||||
self.ids.add(a["id"])
|
||||
if a.get("href", "").startswith("#"):
|
||||
self.hrefs.append(a["href"][1:])
|
||||
if tag in ("style", "script"):
|
||||
self.skip = True
|
||||
if tag not in self.void:
|
||||
self.stack.append(tag)
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if tag in ("style", "script"):
|
||||
self.skip = False
|
||||
if self.stack and self.stack[-1] == tag:
|
||||
self.stack.pop()
|
||||
else:
|
||||
self.mismatched.append((tag, list(self.stack[-3:])))
|
||||
|
||||
def handle_data(self, data):
|
||||
if not self.skip:
|
||||
self.text.append(data)
|
||||
|
||||
|
||||
html = open(LOCAL, encoding="utf-8").read()
|
||||
p = TextExtractor()
|
||||
p.feed(html)
|
||||
prose = " ".join(p.text)
|
||||
|
||||
print("=" * 72)
|
||||
print("1. CONTENT RULES (checked against rendered prose, not raw markup)")
|
||||
print("=" * 72)
|
||||
for label, needle in [("em dash", "\u2014"), ("en dash", "\u2013"),
|
||||
("double hyphen", "--"), ("excluded model family", "5.6")]:
|
||||
n = prose.count(needle)
|
||||
check(f"zero {label}", n == 0, f"{n} occurrence(s)" if n else "clean")
|
||||
|
||||
# rfptank must appear ONLY as a defensive holding, never as a product surface
|
||||
rfp_ctx = [prose[max(0, m.start() - 90):m.start() + 60]
|
||||
for m in re.finditer(r"rfptank\.com", prose)]
|
||||
defensive = all(re.search(r"defensive|legacy|holding", c, re.I) for c in rfp_ctx)
|
||||
check("rfptank.com only as defensive holding", defensive and len(rfp_ctx) > 0,
|
||||
f"{len(rfp_ctx)} mention(s), all qualified")
|
||||
check("verdicttank.com is primary domain", prose.count("verdicttank.com") >= 2)
|
||||
|
||||
print()
|
||||
print("=" * 72)
|
||||
print("2. HTML STRUCTURAL INTEGRITY")
|
||||
print("=" * 72)
|
||||
check("no unclosed tags", not p.stack, str(p.stack[:5]) if p.stack else "balanced")
|
||||
check("no mismatched close tags", not p.mismatched, str(p.mismatched[:3]) if p.mismatched else "clean")
|
||||
|
||||
REQUIRED = ["thesis", "pipeline", "validation", "errors", "panel",
|
||||
"worked", "pricing", "competitive", "deployment", "legal"]
|
||||
missing = [s for s in REQUIRED if s not in p.ids]
|
||||
check(f"all {len(REQUIRED)} required sections present", not missing,
|
||||
f"missing: {missing}" if missing else "thesis..legal")
|
||||
|
||||
dangling = sorted(set(h for h in p.hrefs if h and h not in p.ids))
|
||||
check("no dangling in-page anchors", not dangling,
|
||||
f"broken: {dangling}" if dangling else f"{len(p.hrefs)} anchors resolve")
|
||||
|
||||
nsec = len(re.findall(r"<section", html))
|
||||
check("section open/close parity", nsec == html.count("</section>"),
|
||||
f"{nsec} sections")
|
||||
|
||||
print()
|
||||
print("=" * 72)
|
||||
print("3. FACTUAL GROUNDING (headline numbers must trace to validation data)")
|
||||
print("=" * 72)
|
||||
# Ground truth from the 2026-08-12 validation run
|
||||
FACTS = {
|
||||
"14": "material errors caught",
|
||||
"-0.40": "aggregate delta vs solo",
|
||||
"4.94": "panel aggregate mean",
|
||||
"5.34": "solo aggregate mean",
|
||||
"4.40": "RFP Tank panel",
|
||||
"4.93": "RFP Tank solo",
|
||||
"6.14": "VentureBuilt panel",
|
||||
"6.10": "VentureBuilt solo",
|
||||
"4.29": "CartMySupply panel",
|
||||
"5.00": "CartMySupply solo",
|
||||
"8.2": "Financial Integrity high seat",
|
||||
"2.8": "Execution Feasibility outlier",
|
||||
}
|
||||
for val, desc in FACTS.items():
|
||||
check(f"cites {val} ({desc})", val in prose)
|
||||
|
||||
# Arithmetic self-consistency: the deltas must actually subtract correctly
|
||||
for panel, solo, delta, name in [(4.40, 4.93, -0.53, "RFP Tank"),
|
||||
(6.14, 6.10, 0.04, "VentureBuilt"),
|
||||
(4.29, 5.00, -0.71, "CartMySupply"),
|
||||
(4.94, 5.34, -0.40, "Aggregate")]:
|
||||
check(f"{name} delta arithmetic", abs((panel - solo) - delta) < 0.005,
|
||||
f"{panel} - {solo} = {panel - solo:+.2f}")
|
||||
|
||||
# Error taxonomy: parse the summary cards from the page itself rather than
|
||||
# trusting a hardcoded dict, then reconcile against the per-proposal detail rows.
|
||||
cards = re.findall(
|
||||
r'<div class="errcard"[^>]*><div class="n">(\d+)</div><div class="t">([^<]+)</div>',
|
||||
html)
|
||||
classes = [(int(n), t) for n, t in cards if "Total" not in t]
|
||||
total_card = [int(n) for n, t in cards if "Total" in t]
|
||||
check("taxonomy summary card present", len(total_card) == 1)
|
||||
check("taxonomy classes sum to headline",
|
||||
sum(n for n, _ in classes) == total_card[0] == 14,
|
||||
" + ".join(f"{n}" for n, _ in classes) + f" = {sum(n for n, _ in classes)}")
|
||||
|
||||
# Every class card must be backed by that many tagged rows in the detail tables
|
||||
tagged_rows = len(re.findall(r'<span class="tag tag-red">(?:Revenue|Competitive|Execution|Legal|Team)</span>', html))
|
||||
check("detail rows match headline count", tagged_rows == 14, f"{tagged_rows} tagged findings")
|
||||
|
||||
per_class = {}
|
||||
for cls in ["Revenue", "Competitive", "Execution", "Legal", "Team"]:
|
||||
per_class[cls] = len(re.findall(rf'<span class="tag tag-red">{cls}</span>', html))
|
||||
for n, label in classes:
|
||||
key = next(k for k in per_class if k.lower() in label.lower())
|
||||
check(f"'{label.strip()}' card == detail rows", per_class[key] == n,
|
||||
f"card {n} vs {per_class[key]} rows")
|
||||
|
||||
check("mean is labelled mean, not median",
|
||||
"Median 4.7" not in prose and "Mean 4.7" in prose)
|
||||
|
||||
# The page must NOT claim score elevation anywhere
|
||||
elevation_claims = re.findall(
|
||||
r"(higher score|raises? (?:your )?scores?|score elevation works|boosts? (?:your )?scores?)",
|
||||
prose, re.I)
|
||||
bad = [c for c in elevation_claims if "higher score" not in c.lower()]
|
||||
check("makes no score-elevation claim", not bad, f"{bad}" if bad else
|
||||
"'higher score' appears only in negation")
|
||||
check("publishes the FAIL result", "THESIS FAIL" in prose or "thesis fail" in prose.lower())
|
||||
check("discloses 8-of-11 seat coverage", "8 of 11" in prose)
|
||||
|
||||
print()
|
||||
print("=" * 72)
|
||||
print("4. DEPLOY PARITY")
|
||||
print("=" * 72)
|
||||
local_md5 = hashlib.md5(html.encode()).hexdigest()
|
||||
|
||||
served = urllib.request.urlopen(f"{BASE}/index.html", timeout=30).read()
|
||||
check("served index.html == local source",
|
||||
hashlib.md5(served).hexdigest() == local_md5, local_md5[:12])
|
||||
|
||||
remote = subprocess.run(SSH + [f"md5sum {DOCROOT}/index.html"],
|
||||
capture_output=True, text=True, timeout=60)
|
||||
check("on-disk index.html == local source",
|
||||
remote.stdout.split()[0] == local_md5 if remote.stdout else False,
|
||||
"verified against disk, not cache")
|
||||
|
||||
for path in ["index.html", "index-v4.0.html", "architecture.html", "judge-pool-spec.md"]:
|
||||
try:
|
||||
code = urllib.request.urlopen(f"{BASE}/{path}", timeout=20).status
|
||||
except Exception as e:
|
||||
code = str(e)
|
||||
check(f"HTTP 200 {path}", code == 200, str(code))
|
||||
|
||||
v4 = urllib.request.urlopen(f"{BASE}/index-v4.0.html", timeout=30).read().decode(errors="replace")
|
||||
check("archive is genuinely v4.0", "VerdictTank v4.0" in v4,
|
||||
re.search(r"<title>([^<]*)</title>", v4).group(1).strip())
|
||||
check("v5 served page is v5.0",
|
||||
"Error Detection Density" in served.decode(errors="replace"))
|
||||
|
||||
ownership = subprocess.run(
|
||||
SSH + [f"stat -c '%U:%G' {DOCROOT}/index.html {DOCROOT}/index-v4.0.html"],
|
||||
capture_output=True, text=True, timeout=60)
|
||||
owners = ownership.stdout.split()
|
||||
check("ownership is ippadmin:ippadmin",
|
||||
bool(owners) and all(o == "ippadmin:ippadmin" for o in owners), " ".join(owners))
|
||||
|
||||
print()
|
||||
print("=" * 72)
|
||||
failed = [n for n, ok, _ in results if not ok]
|
||||
print(f"RESULT: {len(results) - len(failed)}/{len(results)} checks passed")
|
||||
if failed:
|
||||
print("FAILED:")
|
||||
for f in failed:
|
||||
print(f" - {f}")
|
||||
print("=" * 72)
|
||||
sys.exit(1 if failed else 0)
|
||||
Reference in New Issue
Block a user