"""Measurement correctness, not product fixes or tests demanding better scores.""" import json from pathlib import Path from scripts.measure_generation_reliability_13a import ( PRODUCT_BLOCKS, corpus, plan_observation, rate, score, ) from src.domain.projects.discovery_schemas import DiscoverySpec, ProductDefinition ROOT = Path(__file__).resolve().parents[1] def test_corpus_has_fixed_unique_denominator_and_explicit_ambiguous_cases(): # GR-05 appended a 10-row "support" stratum (13A-121..130) and GR-06 # appended a 10-row "approval" stratum (13A-131..140) after the frozen # 13A-01/13A-02 120-row set. Appended, never inserted -- every # pre-existing 13A-NNN id, including admission_required 119/120, is # unchanged. See docs/audits/GR05_EXIT_REVIEW.md and # docs/audits/GR06_EXIT_REVIEW.md. rows = corpus(ROOT / "scripts/phase_13a_corpus.txt") assert len(rows) == len({row["prompt"] for row in rows}) == 140 assert len({row["stratum"] for row in rows}) == 14 assert sum(row["oracle_kind"] == "admitted" for row in rows) == 128 assert sum(row["oracle_kind"] == "clarify" for row in rows) == 10 assert sum(row["oracle_kind"] == "admission_required" for row in rows) == 2 assert all(set(row["expected_product_blocks"]) <= PRODUCT_BLOCKS for row in rows) def test_micro_precision_recall_and_false_positive_denominators(): result = score( [ { "true_positive": ["crm.customer"], "false_positive": ["forms.contact"], "false_negative": ["crm.lead"], "exact_product_match": False, "plan_satisfiable": True, } ] ) assert result["planner_precision"]["rate"] == 0.5 assert result["planner_recall"]["rate"] == 0.5 assert result["false_negative_capability_rate"]["rate"] == 0.5 assert result["false_positive_capability_rate"]["denominator"] == len(PRODUCT_BLOCKS) - 2 assert result["exact_product_match_rate"]["rate"] == 0 assert result["plan_satisfiability_rate"]["rate"] == 1 def test_no_trials_is_unmeasured_not_a_green_score(): assert rate(0, 0, "no generated project")["rate"] is None assert rate(0, 0, "no generated project")["unmeasured_reason"] == "no generated project" assert all(metric["rate"] is None for metric in score([]).values()) def test_empty_expected_static_product_penalizes_false_positive(): result = score( [ { "true_positive": [], "false_positive": ["forms.contact"], "false_negative": [], "exact_product_match": False, "plan_satisfiable": True, } ] ) assert result["planner_precision"]["rate"] == 0 assert result["planner_recall"]["rate"] is None assert result["false_positive_capability_rate"]["denominator"] == len(PRODUCT_BLOCKS) def test_gold_changes_scoring_but_never_planner_input_or_generated_plan(): spec = DiscoverySpec(product=ProductDefinition(purpose="Staff product catalog")) right = plan_observation(spec, ["catalog.product"]) wrong = plan_observation(spec, ["booking.appointment"]) assert right["predicted_product_blocks"] == wrong["predicted_product_blocks"] assert right["plan"] == wrong["plan"] assert right["exact_product_match"] is True assert wrong["exact_product_match"] is False assert right["required_ui_slots"] > 0 def test_satisfiable_plan_does_not_hide_intent_miss(): spec = DiscoverySpec(product=ProductDefinition(purpose="A place for our office")) result = plan_observation(spec, ["crm.customer"]) assert result["plan_satisfiable"] is True assert result["exact_product_match"] is False assert result["false_negative"] == ["crm.customer"] def test_natural_crm_false_negative_is_recorded_not_repaired(): import tempfile from pathlib import Path from src.domain.app_blocks.generation_reliability_13a import ( evaluate_structural, spec_from_prompt, ) spec = spec_from_prompt("Build me a CRM for the sales team.") observation = plan_observation(spec, ["crm.lead"]) assert "crm.lead" in observation["false_negative"] with tempfile.TemporaryDirectory(prefix="p2hs-13a-crm-") as raw: structural = evaluate_structural(spec, Path(raw) / "crm-fn") assert structural["plan_satisfiable"] is True assert "crm.lead" not in structural["planned_blocks"] def test_quote_html_materialize_mounts_without_claiming_a_production_build(): import tempfile from pathlib import Path from src.domain.app_blocks.generation_reliability_13a import evaluate_structural spec = DiscoverySpec( product=ProductDefinition(purpose="Roofing company website where visitors request a quote.") ) with tempfile.TemporaryDirectory(prefix="p2hs-13a-quote-") as raw: dest = Path(raw) / "quote" result = evaluate_structural(spec, dest) routes = dest / "p2hs" / "generated-routes.json" assert routes.is_file() assert result["codegen_success"] is True assert result["must_mount_conformance"] is True assert result["structural_validation"] is True assert result["repair_success"] is False def test_structural_score_keeps_unmeasured_repair_success_when_nothing_failed(): from src.domain.app_blocks.generation_reliability_13a import score_structural scored = score_structural( [ { "codegen_success": True, "must_mount_conformance": True, "unexpected_semantic_surface": False, "structural_validation": True, "repair_required": False, "terminal_generation_failure": False, } ] ) assert scored["codegen_success_rate"]["rate"] == 1 assert scored["repair_success_rate"]["rate"] is None assert scored["repair_success_rate"]["unmeasured_reason"] == "No repairs attempted in 13A." def test_empty_or_wrong_plans_can_still_structurally_succeed(): from src.domain.app_blocks.generation_reliability_13a import gold_join, score_intent_joined empty = gold_join( {"planned_blocks": [], "structural_validation": True}, {"expected_product_blocks": ["crm.lead"], "oracle_kind": "admitted"}, ) matched = gold_join( {"planned_blocks": ["crm.customer", "admin.shell"], "structural_validation": True}, {"expected_product_blocks": ["crm.customer"], "oracle_kind": "admitted"}, ) assert empty["empty_plan"] is True assert empty["intended_product_structurally_valid"] is False assert matched["exact_product_match"] is True scored = score_intent_joined( [ {**empty, "structural_validation": True}, {**matched, "structural_validation": True}, ] ) assert scored["empty_plan_rate"]["rate"] == 0.5 assert scored["admitted_exact_product_and_structural_rate"]["rate"] == 0.5 def test_structural_scorecard_keeps_unmeasured_13e_columns(): payload = json.loads( (ROOT / "docs/audits/fixtures/13a_reliability_scorecard.json").read_text(encoding="utf-8") ) assert payload["corpus_size"] == 120 assert payload["phase_13b_authorized"] is False assert payload["generation_reliability_certified"] is False assert payload["codegen_success_rate"]["denominator"] == 120 assert payload["actual_build_rate"]["rate"] is None assert payload["runtime_smoke_rate"]["rate"] is None assert payload["repair_success_rate"]["rate"] is None def test_run_02_freeze_keeps_catalog_parity_and_does_not_authorize_13b(): freeze = json.loads( (ROOT / "docs/audits/fixtures/13a_run_02_freeze.json").read_text(encoding="utf-8") ) brief = (ROOT / "docs/audits/PHASE_13B_IMPLEMENTATION_BRIEF.md").read_text(encoding="utf-8") run_01 = ROOT / "docs/audits/fixtures/13a_run_01/scorecard.json" run_02 = ROOT / "docs/audits/fixtures/13a_run_02/scorecard.json" assert run_01.is_file() assert run_02.is_file() assert freeze["phase_13a_status"] == "complete_partial_baseline" assert freeze["phase_13a_frozen_baseline"] == "13A-02" assert freeze["phase_13a_run_01_preserved"] is True assert freeze["phase_13a_run_02_catalog_revision"] == "145" assert freeze["catalog_revision_caused_planner_change"] is False assert freeze["structural_control_is_live_path"] is False assert freeze["raw_planner_exact_product"] == "76/108" assert freeze["discovery_completion"] == "36/120" assert freeze["discovery_timeout_failures"] == 84 assert freeze["phase_13b_brief"] == "prepared" assert freeze["phase_13b_authorized"] is False assert freeze["phase_13b_gate"] == "none" assert freeze["implementation_started"] is False assert "timeout_increase_is_not_success=yes" in brief assert "AUTHORIZE PHASE 13B." in brief assert "Do not begin Gate B2 until B1" in brief assert "phase_13c_authorized=no" in brief assert "phase_13e_authorized=no" in brief