"""Analyzer B — version registry. Each version is a named configuration of speed/accuracy knobs. The benchmark harness (bench.py) runs each version over the 4 fixed benchmark pages and records per-step wall-time + detection accuracy, so we can compare which version is the fastest and which is the most accurate. Knobs ----- page_workers : pages processed concurrently (env HVAC_PAGE_WORKERS). Pages are independent; >1 overlaps API latency across pages. tile_workers : tiles per page sent concurrently (env HVAC_TILE_WORKERS). dpi : render DPI. Lower = faster + smaller images, but boxes are coarser and tiny tags may be missed. Scorer rescales boxes to GT DPI (300). tile_size : detection tile edge in px. Bigger = fewer API calls (faster) but more units per call (model may miss some); smaller = more calls. runs : repeat detection N times and union (recall up, slower). consensus : 1/2/3 — agreement passes (precision up, slower). fp_filter : run the false-positive filter pass (precision up, +1 step). all_tags : search every schedule tag (recall up) vs only qty>0 tags. Edit/extend this list to define up to 10 (or more) versions. v10_best is meant to be filled in after the first sweep, combining the winning knobs. """ # Common detector defaults shared by every version unless overridden. _BASE = dict( page_workers=1, tile_workers=1, dpi=300, tile_size=3000, runs=1, consensus=1, fp_filter=True, all_tags=True, fast_ocr=False, # short-circuit OCR-snap PSM/rotation passes (HVAC_FAST_OCR) recall_prompt=False, # inclusive "miss nothing" detection prompt (--recall-prompt) detector="gemini", # "gemini" = direct API; "qwen" = OpenRouter (set model=) model=None, # OpenRouter model slug when detector="qwen" (--qwen-model) reasoning="", # OpenRouter thinking effort: high|medium|low (HVAC_OR_REASONING) adaptive_fp=0, # post-detection density-adaptive fp-filter (HVAC_ADAPTIVE_FP_THRESHOLD) prescan=0, # pre-scan density probe threshold (HVAC_ADAPTIVE_PRESCAN); 0=off ensemble=None, # list of 2 OpenRouter slugs for --detector ensemble tiebreak="", # 'gemini' to adjudicate ensemble yellows with direct Gemini ) def _v(**over): d = dict(_BASE) d.update(over) return d VERSIONS = { # id knobs "v01_baseline": _v(page_workers=1, tile_workers=1), # fully serial "v02_tiles4": _v(page_workers=1, tile_workers=4), # parallel tiles (model-B default) "v03_tiles8": _v(page_workers=1, tile_workers=8), # more tile parallelism "v04_pages2": _v(page_workers=2, tile_workers=4), # page-level parallelism "v05_pages2_t8": _v(page_workers=2, tile_workers=8), # both axes parallel "v06_dpi250": _v(page_workers=1, tile_workers=4, dpi=250), # lower DPI for speed "v07_tile4500": _v(page_workers=1, tile_workers=4, tile_size=4500),# fewer, larger tiles "v08_tile2000_t8":_v(page_workers=1, tile_workers=8, tile_size=2000),# many small tiles, high parallelism "v09_no_fpfilter":_v(page_workers=1, tile_workers=4, fp_filter=False),# drop FP-filter step "v11_turbo16": _v(page_workers=2, tile_workers=16), # max parallelism (tiles+snap+verify+fp) "v12_fastocr": _v(page_workers=1, tile_workers=8, fast_ocr=True), # v03 + short-circuit OCR-snap # ── Recall-focused (dense pages): smaller tiles = fewer units per API call, # runs>1 unions jittery misses. Proven recipe was tile=1800, runs=3. "v13_recall": _v(tile_workers=12, fast_ocr=True, tile_size=1800, runs=2), "v14_recall_max": _v(tile_workers=12, fast_ocr=True, tile_size=1800, runs=3), "v15_recall_cons":_v(tile_workers=12, fast_ocr=True, tile_size=1800, runs=2, consensus=2), # ── Ultra-recall: every recall lever — inclusive prompt + small tiles + 3 # runs + consensus (shifted tile edges). Aims for ~100% recall, precision # handled by the fp-filter. Most expensive; still targets <5 min/page. "v17_ultra": _v(tile_workers=16, fast_ocr=True, tile_size=1500, runs=3, consensus=2, recall_prompt=True), "v18_ultra_nofp": _v(tile_workers=16, fast_ocr=True, tile_size=1500, runs=3, consensus=2, recall_prompt=True, fp_filter=False), # Max-recall workflow config: winning v14 knobs but NO fp-filter, so nothing # is auto-dropped — uncertain boxes survive flagged (orange/yellow) for human # review instead of being deleted. Favours recall; human checks only flags. "v19_maxrecall": _v(tile_workers=12, fast_ocr=True, tile_size=1800, runs=3, fp_filter=False), # Direct API, max recall + consensus: v19 plus shifted-tile-edge passes to # catch units split across tile boundaries (a real miss source on dense plans). "v26_consensus": _v(tile_workers=12, fast_ocr=True, tile_size=1800, runs=2, consensus=2, fp_filter=False), # SHIP CONFIG as of 2026-07-02: v26 knobs + the inclusive (recall) prompt. # A/B on the 4-building gold set: strict prompt 39% center-recall vs 79% # inclusive (Lewis 13→60, Carroll 52→90, 1326 66→80, 23042 4→70). FP/review # burden rises ~26 boxes/page — attacked by dedupe + FP guardrails + Track C. "v30_inclusive": _v(tile_workers=12, fast_ocr=True, tile_size=1800, runs=2, consensus=2, fp_filter=False, recall_prompt=True), # Push recall further: consensus=3 adds a 3rd pass at a different TILE SIZE # (scale diversity), to catch systematic misses that edge-shifts alone don't. "v27_consensus3": _v(tile_workers=12, fast_ocr=True, tile_size=1800, runs=2, consensus=3, fp_filter=False), # Validation of the size-relative merge fix: same knobs as v26, but the engine # now uses size-relative merge distance — should recover back-to-back twins. "v29_mergefix": _v(tile_workers=12, fast_ocr=True, tile_size=1800, runs=2, consensus=2, fp_filter=False), # Density-adaptive: v26 recipe everywhere, but auto-enable the precision filter # on SPARSE sheets (<15 detected units) so they don't get FP-laden green boxes, # while DENSE sheets keep full recall. The all-page-types production config. "v28_adaptive": _v(tile_workers=12, fast_ocr=True, tile_size=1800, runs=2, consensus=2, fp_filter=False, adaptive_fp=15), # Direct API, max recall: v19 + inclusive "miss-nothing" prompt. Generates # more candidates; no fp-filter so borderline ones survive flagged for review. "v25_max_direct": _v(tile_workers=12, fast_ocr=True, tile_size=1800, runs=3, fp_filter=False, recall_prompt=True), # ── OpenRouter-routed Gemini (bypasses the direct-API spend cap) ───────── # Same google/gemini-2.5-flash model, via OpenRouter. Lower worker count to # respect OpenRouter rate limits. v20 ≈ v14 (precision filter on); v21 ≈ v19 # (no filter, max recall, uncertain flagged for review). "v20_or_recall": _v(tile_workers=8, fast_ocr=True, tile_size=1800, runs=3, detector="qwen", model="google/gemini-2.5-flash"), "v21_or_maxrecall": _v(tile_workers=8, fast_ocr=True, tile_size=1800, runs=3, fp_filter=False, detector="qwen", model="google/gemini-2.5-flash"), # Stronger OpenRouter model to close the recall gap vs the direct API. "v22_or_g35": _v(tile_workers=8, fast_ocr=True, tile_size=1800, runs=3, fp_filter=False, detector="qwen", model="google/gemini-3.5-flash"), # Same model as v21 but with "thinking" ON via OpenRouter — should recover the # recall lost vs the direct API (which runs Gemini 2.5 with thinking by default). "v23_or_think": _v(tile_workers=8, fast_ocr=True, tile_size=1800, runs=3, fp_filter=False, detector="qwen", model="google/gemini-2.5-flash", reasoning="high"), # Best OpenRouter model + inclusive "miss-nothing" prompt + no filter = max # recall, uncertain flagged for review (the user's ideal workflow). "v24_or_max": _v(tile_workers=8, fast_ocr=True, tile_size=1800, runs=3, fp_filter=False, recall_prompt=True, detector="qwen", model="google/gemini-3.5-flash"), # ── Cost-vs-recall: v26 recipe but runs=1 (one detection pass instead of two). # consensus=2 is kept (the real edge-split recall lever); runs only fights # run-to-run VLM jitter. ~78 tile calls/floor vs 156 for v26 → ~half the API # cost. Run head-to-head with v26_consensus to measure the recall delta. "v30_runs1": _v(tile_workers=12, fast_ocr=True, tile_size=1800, runs=1, consensus=2, fp_filter=False), # Cheaper still: bigger tiles (2400) + runs=1 — ~half the tiles of v30 again. # Use if v30 holds recall and you want a further cost cut on less-dense sheets. "v31_tile2400": _v(tile_workers=12, fast_ocr=True, tile_size=2400, runs=1, consensus=2, fp_filter=False), # ── Adaptive density PRE-SCAN: a cheap precision-first probe (big tiles + # strict prompt) estimates real unit count BEFORE the recall recipe runs, # then auto-picks the recipe per page — dense→no fp-filter (recall), sparse # →fp-filter ON (precision). Targets the sparse-page F1 weakness (355 ~0.6) # without hurting dense (Dean ~0.95). Probe hits fold into the union. "v32_adaptive_prescan": _v(tile_workers=12, fast_ocr=True, tile_size=1800, runs=1, consensus=2, fp_filter=False, prescan=15), # ── OpenRouter MODEL SEARCH (same pipeline, different detector model) ────── # The detector="qwen" path routes tiles through OpenRouter. These swap only # the model to compare cost/recall vs direct Gemini. NOTE: OpenRouter doesn't # run Gemini "thinking", so Google models score lower here than direct (known). # Slugs may need adjusting to your OpenRouter catalog. "v40_or_gpt4o_mini": _v(tile_workers=8, fast_ocr=True, tile_size=1800, runs=1, consensus=2, fp_filter=False, detector="qwen", model="openai/gpt-4o-mini"), "v41_or_flash_lite": _v(tile_workers=8, fast_ocr=True, tile_size=1800, runs=1, consensus=2, fp_filter=False, detector="qwen", model="google/gemini-2.5-flash-lite"), "v42_or_qwen25vl72": _v(tile_workers=8, fast_ocr=True, tile_size=1800, runs=1, consensus=2, fp_filter=False, detector="qwen", model="qwen/qwen2.5-vl-72b-instruct"), "v43_or_llama_vision": _v(tile_workers=8, fast_ocr=True, tile_size=1800, runs=1, consensus=2, fp_filter=False, detector="qwen", model="meta-llama/llama-3.2-11b-vision-instruct"), # GLM-5 vision (glm-5.2 itself is text-only — glm-5v-turbo is the 5-series # vision model). Same dpi300/tile1800/runs1/consensus2 as the best config. "v80_or_glm5v": _v(tile_workers=8, fast_ocr=True, tile_size=1800, runs=1, consensus=2, fp_filter=False, detector="qwen", model="z-ai/glm-5v-turbo"), # ── Curated cheap cost/accuracy search (all confirmed image-capable on OR) ── "v44_or_nemotron_omni": _v(tile_workers=6, fast_ocr=True, tile_size=1800, runs=1, consensus=2, fp_filter=False, detector="qwen", model="nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free"), "v45_or_nova_lite": _v(tile_workers=8, fast_ocr=True, tile_size=1800, runs=1, consensus=2, fp_filter=False, detector="qwen", model="amazon/nova-lite-v1"), "v46_or_gemma3_27b": _v(tile_workers=8, fast_ocr=True, tile_size=1800, runs=1, consensus=2, fp_filter=False, detector="qwen", model="google/gemma-3-27b-it"), "v47_or_qwen3vl_32b": _v(tile_workers=8, fast_ocr=True, tile_size=1800, runs=1, consensus=2, fp_filter=False, detector="qwen", model="qwen/qwen3-vl-32b-instruct"), "v48_or_gemini31_lite": _v(tile_workers=8, fast_ocr=True, tile_size=1800, runs=1, consensus=2, fp_filter=False, detector="qwen", model="google/gemini-3.1-flash-lite"), # ── LOCAL OCR detector (EasyOCR) — read tag labels locally, $0 API cost. # Finds units with a readable printed label; misses faint/symbol-only ones. # The cost floor to compare every paid detector against. "v50_ocr_easy": _v(tile_workers=4, fast_ocr=True, detector="ocr"), # OCR through the recall machinery: smaller tiles + consensus (shifted edges) # + per-tile upscale — the same levers that lift the VLM, applied to free OCR. "v51_ocr_tiled": _v(tile_workers=4, fast_ocr=True, detector="ocr", tile_size=1600, consensus=2), # ── SPARSE recipe (bigger tiles 3000 + fp-filter ON) on the cheap models, to # test whether the per-density recipe lifts sparse-page precision. (The # gemini-direct version is v12_fastocr — deferred while the cap is exhausted.) "v70_sparse_qwen3vl": _v(tile_workers=8, fast_ocr=True, tile_size=3000, runs=1, consensus=1, fp_filter=True, detector="qwen", model="qwen/qwen3-vl-32b-instruct"), "v71_sparse_flashlite": _v(tile_workers=8, fast_ocr=True, tile_size=3000, runs=1, consensus=1, fp_filter=True, detector="qwen", model="google/gemini-2.5-flash-lite"), # qwen3-vl with OpenRouter REASONING on — does thinking improve it (esp. on # sparse, where rejecting tag-like non-units needs reasoning)? Compare to v47. "v73_qwen3vl_reason": _v(tile_workers=8, fast_ocr=True, tile_size=1800, runs=1, consensus=2, fp_filter=False, detector="qwen", model="qwen/qwen3-vl-32b-instruct", reasoning="high"), # ── ENSEMBLE: two cheap models; a unit found by BOTH is green (confirmed), # one found by only one is yellow (review). Agreement kills the solo FPs # that wreck sparse precision. v60 adds a direct-Gemini tie-break on yellows; # v61 leaves yellows for human review (isolates the agreement effect). "v60_ensemble": _v(tile_workers=8, fast_ocr=True, tile_size=1800, runs=1, consensus=2, fp_filter=False, detector="ensemble", ensemble=["qwen/qwen3-vl-32b-instruct", "google/gemini-3.1-flash-lite"], tiebreak="gemini"), "v61_ensemble_notb": _v(tile_workers=8, fast_ocr=True, tile_size=1800, runs=1, consensus=2, fp_filter=False, detector="ensemble", ensemble=["qwen/qwen3-vl-32b-instruct", "google/gemini-3.1-flash-lite"]), # v10_best — proven winner (= v26_consensus): direct API, parallel everything, # fast-OCR, small tiles, 2 runs + consensus=2 (shifted tile edges catch # units split across tile boundaries — the big recall lever), NO fp-filter so # uncertain units are flagged for review not dropped. Dean recall 0.94-0.96, # F1 0.97, ~4 min/page, ~1 box/page to review. Standard prompt. "v10_best": _v(tile_workers=12, fast_ocr=True, tile_size=1800, runs=2, consensus=2, fp_filter=False), } def engine_flags(cfg: dict) -> list[str]: """Translate a version config into analyze_blueprint_b.py CLI flags.""" flags = [ "--dpi", str(cfg["dpi"]), "--tile-size", str(cfg["tile_size"]), "--runs", str(cfg["runs"]), "--consensus", str(cfg["consensus"]), "--refresh-detections", # force real detection work (honest timing) "--no-yolo", # skip YOLO label export (not needed for the bench) "-y", ] if cfg["fp_filter"]: flags.append("--fp-filter") if cfg["all_tags"]: flags.append("--all-tags") if cfg.get("recall_prompt"): flags.append("--recall-prompt") if cfg.get("detector") and cfg["detector"] != "gemini": flags += ["--detector", cfg["detector"]] if cfg.get("model"): flags += ["--qwen-model", cfg["model"]] if cfg.get("detector") == "ensemble" and cfg.get("ensemble"): flags += ["--ensemble-models", ",".join(cfg["ensemble"])] if cfg.get("tiebreak"): flags += ["--ensemble-tiebreak", cfg["tiebreak"]] return flags def engine_env(cfg: dict) -> dict: return { "HVAC_PAGE_WORKERS": str(cfg["page_workers"]), "HVAC_TILE_WORKERS": str(cfg["tile_workers"]), "HVAC_FAST_OCR": "1" if cfg.get("fast_ocr") else "0", "HVAC_OR_REASONING": str(cfg.get("reasoning") or ""), "HVAC_ADAPTIVE_FP_THRESHOLD": str(cfg.get("adaptive_fp") or 0), "HVAC_ADAPTIVE_PRESCAN": str(cfg.get("prescan") or 0), }