#!/usr/bin/env python3
"""
Generate comprehensive Round 2 comparison report with additional analyses:
1. Channel breakdown
2. Cluster size analysis
3. Combined round 1 + round 2 summary
4. "Honest low scores" test
5. Title categorization
6. Blind scoring comparison
"""

import json
import os
import re
from datetime import datetime
from collections import defaultdict

SANDBOX_DIR = os.path.dirname(os.path.abspath(__file__))

def load_results(round_num: int) -> dict:
    """Load results from a round."""
    if round_num == 1:
        path = os.path.join(SANDBOX_DIR, 'comparison_results.json')
    else:
        path = os.path.join(SANDBOX_DIR, f'round{round_num}_results.json')
    with open(path) as f:
        return json.load(f)

def load_blind_scores(round_num: int) -> dict:
    """Load blind scores from a round."""
    path = os.path.join(SANDBOX_DIR, f'round{round_num}_blind_scores.json')
    with open(path) as f:
        return json.load(f)

def categorize_title(title: str) -> str:
    """Categorize a title into one of the categories."""
    title_lower = title.lower()

    # Specific story markers (references real person/event/number with personal angle)
    story_markers = [
        r'\bi\b.*(?:paid|spent|found|got|was|have|had)',
        r'he\s+(?:said|asked|left|spent|was)',
        r'she\s+(?:said|asked|left|spent|was)',
        r'my\s+\$?\d+',
        r'\d+\s*(?:years?|months?|days?)\s*(?:ago|later)',
        r"'one night'",
        r'survivor',
    ]
    for marker in story_markers:
        if re.search(marker, title_lower):
            return "Specific story"

    # Specific detail markers (one surprising fact, number, or counterintuitive claim)
    detail_markers = [
        r'\d+%',
        r'\$\d+',
        r'the\s+\d+',
        r'never\s+(?:had|was|did)',
        r'actually\s+(?:a|the|doing)',
        r"doesn't exist",
        r"didn't exist",
        r'opposite of',
        r'loophole',
        r'secret',
    ]
    for marker in detail_markers:
        if re.search(marker, title_lower):
            return "Specific detail"

    # Framework/listicle markers
    framework_markers = [
        r'^\d+\s+(?:ways?|tricks?|steps?|things?|tips?)',
        r'the\s+\w+\s+framework',
        r'how\s+to\s+',
        r'a\s+guide\s+to',
        r'decision\s+tree',
        r'cheat\s+sheet',
        r'playbook',
    ]
    for marker in framework_markers:
        if re.search(marker, title_lower):
            return "Framework/listicle"

    # Default to theme-based
    return "Theme-based"

def analyze_verdicts(results_list: list) -> dict:
    """Analyze verdict distribution across conditions."""
    verdict_counts = {
        "A": defaultdict(int),
        "B": defaultdict(int),
        "C": defaultdict(int),
    }

    for r in results_list:
        for cond in ["A", "B", "C"]:
            verdict = r["results"][cond].get("verdict", "UNKNOWN")
            verdict_counts[cond][verdict] += 1

    return {k: dict(v) for k, v in verdict_counts.items()}

def analyze_by_channel(results_list: list) -> dict:
    """Analyze performance breakdown by channel."""
    channel_data = defaultdict(lambda: {"A": [], "B": [], "C": []})

    for r in results_list:
        channel = r.get("channel", "unknown")
        for cond in ["A", "B", "C"]:
            channel_data[channel][cond].append({
                "title": r["results"][cond].get("title", ""),
                "verdict": r["results"][cond].get("verdict", ""),
                "score": r["results"][cond].get("score", 0)
            })

    return dict(channel_data)

def analyze_by_cluster_size(results_list: list) -> dict:
    """Analyze performance by cluster size."""
    size_bins = {
        "small (<20)": {"A": [], "B": [], "C": []},
        "medium (20-50)": {"A": [], "B": [], "C": []},
        "large (>50)": {"A": [], "B": [], "C": []}
    }

    for r in results_list:
        topic_count = r.get("topic_count", 0)
        if topic_count < 20:
            bin_key = "small (<20)"
        elif topic_count <= 50:
            bin_key = "medium (20-50)"
        else:
            bin_key = "large (>50)"

        for cond in ["A", "B", "C"]:
            size_bins[bin_key][cond].append({
                "verdict": r["results"][cond].get("verdict", ""),
                "score": r["results"][cond].get("score", 0)
            })

    return size_bins

def analyze_honest_low_scores(results_list: list) -> list:
    """Find clusters where B/C scored low but A gave HIGH_PRIORITY."""
    mismatches = []

    for r in results_list:
        a_verdict = r["results"]["A"].get("verdict", "")
        b_score = r["results"]["B"].get("score", 0)
        c_score = r["results"]["C"].get("score", 0)

        # Check for scoring mismatch
        if a_verdict in ["HIGH_PRIORITY"] and (b_score < 4 or c_score < 4):
            mismatches.append({
                "test_id": r.get("test_id", ""),
                "channel": r.get("channel", ""),
                "A_verdict": a_verdict,
                "A_title": r["results"]["A"].get("title", ""),
                "B_score": b_score,
                "B_title": r["results"]["B"].get("title", ""),
                "C_score": c_score,
                "C_title": r["results"]["C"].get("title", "")
            })

    return mismatches

def categorize_all_titles(results_list: list) -> dict:
    """Categorize all titles by condition."""
    categories = {
        "A": defaultdict(list),
        "B": defaultdict(list),
        "C": defaultdict(list)
    }

    for r in results_list:
        for cond in ["A", "B", "C"]:
            title = r["results"][cond].get("title", "")
            category = categorize_title(title)
            categories[cond][category].append(title)

    return {k: dict(v) for k, v in categories.items()}

def generate_report():
    """Generate the comprehensive round 2 report."""
    # Load all data
    round1_results = load_results(1).get("results", [])
    round2_results = load_results(2).get("results", [])
    all_results = round1_results + round2_results

    round1_blind = load_blind_scores(1)
    round2_blind = load_blind_scores(2)

    report = []
    report.append("=" * 80)
    report.append("ROUND 2 COMPARISON REPORT: VALIDATION RUN")
    report.append("=" * 80)
    report.append(f"Generated: {datetime.now().isoformat()}")
    report.append(f"Round 1 clusters: {len(round1_results)} (brain_is_lying, money_traps)")
    report.append(f"Round 2 clusters: {len(round2_results)} (body_is_weird, power_works, what_actually_happened)")
    report.append(f"Total clusters: {len(all_results)}")
    report.append("")

    # =========================================================================
    # SECTION 1: BLIND SCORING COMPARISON (THE KEY METRIC)
    # =========================================================================
    report.append("-" * 80)
    report.append("SECTION 1: BLIND SCROLL-STOP SCORES")
    report.append("-" * 80)
    report.append("")
    report.append("This is the apples-to-apples comparison: same judge, same criteria,")
    report.append("no knowledge of which prompt produced which title.")
    report.append("")

    # Aggregate blind scores
    blind_stats = {"A": [], "B": [], "C": []}
    for round_data in [round1_blind, round2_blind]:
        for cond in ["A", "B", "C"]:
            cond_data = round_data.get("by_condition", {}).get(cond, [])
            for item in cond_data:
                if item.get("blind_score", 0) > 0:
                    blind_stats[cond].append(item["blind_score"])

    report.append("COMBINED BLIND SCORES (60 titles total, 20 per condition):")
    report.append("")
    for cond, label in [("A", "Baseline"), ("B", "Curiosity Mining"), ("C", "Unclustered")]:
        scores = blind_stats[cond]
        if scores:
            mean = sum(scores) / len(scores)
            high = sum(1 for s in scores if s >= 7)
            low = sum(1 for s in scores if s <= 3)
            report.append(f"[{cond}] {label}:")
            report.append(f"    Mean scroll-stop score: {mean:.2f}")
            report.append(f"    Range: {min(scores)}-{max(scores)}")
            report.append(f"    Titles scoring 7+ (\"Wait, what?\"): {high}")
            report.append(f"    Titles scoring 3 or less (scroll-past): {low}")
            report.append("")

    # Round-by-round breakdown
    report.append("BY ROUND:")
    for round_num, round_data in [(1, round1_blind), (2, round2_blind)]:
        stats = round_data.get("statistics", {})
        report.append(f"\n  Round {round_num}:")
        for cond in ["A", "B", "C"]:
            s = stats.get(cond, {})
            report.append(f"    [{cond}] Mean: {s.get('mean', 0):.2f}, 7+: {s.get('count_7plus', 0)}, ≤3: {s.get('count_3minus', 0)}")

    report.append("")

    # =========================================================================
    # SECTION 2: CHANNEL BREAKDOWN
    # =========================================================================
    report.append("-" * 80)
    report.append("SECTION 2: CHANNEL BREAKDOWN")
    report.append("-" * 80)
    report.append("")
    report.append("Do some channels benefit more from B/C than others?")
    report.append("")

    channel_data = analyze_by_channel(all_results)

    for channel, cond_data in sorted(channel_data.items()):
        report.append(f"📊 {channel}:")

        for cond in ["A", "B", "C"]:
            items = cond_data[cond]
            if items:
                make_now = sum(1 for i in items if "MAKE_NOW" in str(i.get("verdict", "")))
                high_pri = sum(1 for i in items if "HIGH_PRIORITY" in str(i.get("verdict", "")))
                avg_score = sum(i.get("score", 0) for i in items) / len(items) if items else 0
                label = {"A": "Baseline", "B": "Curiosity", "C": "Unclustered"}[cond]
                report.append(f"    [{cond}] {label}: {make_now} MAKE_NOW, {high_pri} HIGH_PRIORITY, avg score {avg_score:.1f}")
        report.append("")

    # =========================================================================
    # SECTION 3: CLUSTER SIZE ANALYSIS
    # =========================================================================
    report.append("-" * 80)
    report.append("SECTION 3: CLUSTER SIZE ANALYSIS")
    report.append("-" * 80)
    report.append("")
    report.append("Do smaller clusters produce better hooks? Or do larger clusters")
    report.append("benefit more from B/C because there's more raw material to mine?")
    report.append("")

    size_data = analyze_by_cluster_size(all_results)

    for size_bin, cond_data in size_data.items():
        report.append(f"📊 {size_bin}:")

        for cond in ["A", "B", "C"]:
            items = cond_data[cond]
            if items:
                make_now = sum(1 for i in items if "MAKE_NOW" in str(i.get("verdict", "")))
                avg_score = sum(i.get("score", 0) for i in items) / len(items) if items else 0
                label = {"A": "Baseline", "B": "Curiosity", "C": "Unclustered"}[cond]
                report.append(f"    [{cond}] {label}: {make_now} MAKE_NOW, avg score {avg_score:.1f} (n={len(items)})")
        report.append("")

    # =========================================================================
    # SECTION 4: COMBINED VERDICT SUMMARY
    # =========================================================================
    report.append("-" * 80)
    report.append("SECTION 4: COMBINED VERDICT SUMMARY (ALL 20 CLUSTERS)")
    report.append("-" * 80)
    report.append("")

    verdicts = analyze_verdicts(all_results)

    for cond, label in [("A", "Baseline"), ("B", "Curiosity Mining"), ("C", "Unclustered")]:
        report.append(f"[{cond}] {label}:")
        for verdict, count in sorted(verdicts[cond].items(), key=lambda x: -x[1]):
            report.append(f"    {verdict}: {count}")
        report.append("")

    # Calculate MAKE_NOW rate
    report.append("MAKE_NOW RATE:")
    for cond, label in [("A", "Baseline"), ("B", "Curiosity Mining"), ("C", "Unclustered")]:
        make_now = verdicts[cond].get("MAKE_NOW", 0)
        total = sum(verdicts[cond].values())
        rate = (make_now / total * 100) if total > 0 else 0
        report.append(f"    [{cond}] {label}: {make_now}/{total} = {rate:.0f}%")

    report.append("")

    # =========================================================================
    # SECTION 5: "HONEST LOW SCORES" TEST
    # =========================================================================
    report.append("-" * 80)
    report.append("SECTION 5: SCORING MISCALIBRATION TEST")
    report.append("-" * 80)
    report.append("")
    report.append("Clusters where Prompt A gave HIGH_PRIORITY but B/C scored < 4.0")
    report.append("(honest assessment: \"this material isn't that interesting\")")
    report.append("")

    mismatches = analyze_honest_low_scores(all_results)

    if mismatches:
        for m in mismatches:
            report.append(f"📊 {m['test_id']} ({m['channel']}):")
            report.append(f"    [A] {m['A_verdict']}: \"{m['A_title'][:60]}...\"")
            report.append(f"    [B] Score {m['B_score']:.1f}: \"{m['B_title'][:60]}...\"")
            report.append(f"    [C] Score {m['C_score']:.1f}: \"{m['C_title'][:60]}...\"")
            report.append("")
    else:
        report.append("No significant miscalibration found.")
        report.append("")

    report.append(f"Total mismatches: {len(mismatches)}/20 clusters")
    report.append("")

    # =========================================================================
    # SECTION 6: TITLE CATEGORIZATION
    # =========================================================================
    report.append("-" * 80)
    report.append("SECTION 6: TITLE CATEGORIZATION")
    report.append("-" * 80)
    report.append("")
    report.append("Categories:")
    report.append("  - Specific story: References a real person/event/number with personal angle")
    report.append("  - Specific detail: One surprising fact or counterintuitive claim")
    report.append("  - Theme-based: Describes a category or pattern")
    report.append("  - Framework/listicle: N tricks, N steps, the X playbook")
    report.append("")

    categories = categorize_all_titles(all_results)

    report.append("DISTRIBUTION (20 titles per condition):")
    report.append("")
    for cond, label in [("A", "Baseline"), ("B", "Curiosity Mining"), ("C", "Unclustered")]:
        report.append(f"[{cond}] {label}:")
        for cat in ["Specific story", "Specific detail", "Theme-based", "Framework/listicle"]:
            count = len(categories[cond].get(cat, []))
            report.append(f"    {cat}: {count}")
        report.append("")

    # =========================================================================
    # SECTION 7: ALL TITLES
    # =========================================================================
    report.append("-" * 80)
    report.append("SECTION 7: ALL GENERATED TITLES")
    report.append("-" * 80)
    report.append("")

    report.append("ROUND 1 (brain_is_lying, money_traps):")
    report.append("")
    for r in round1_results:
        report.append(f"📊 {r['test_id']}")
        for cond in ["A", "B", "C"]:
            title = r["results"][cond].get("title", "N/A")
            verdict = r["results"][cond].get("verdict", "N/A")
            score = r["results"][cond].get("score", 0)
            category = categorize_title(title)
            report.append(f"   [{cond}] ({verdict}, {score:.1f}, {category}) {title[:65]}...")
        report.append("")

    report.append("")
    report.append("ROUND 2 (body_is_weird, power_works, what_actually_happened):")
    report.append("")
    for r in round2_results:
        report.append(f"📊 {r['test_id']}")
        for cond in ["A", "B", "C"]:
            title = r["results"][cond].get("title", "N/A")
            verdict = r["results"][cond].get("verdict", "N/A")
            score = r["results"][cond].get("score", 0)
            category = categorize_title(title)
            report.append(f"   [{cond}] ({verdict}, {score:.1f}, {category}) {title[:65]}...")
        report.append("")

    return "\n".join(report)


if __name__ == "__main__":
    report = generate_report()

    # Print to console
    print(report)

    # Save to file
    report_path = os.path.join(SANDBOX_DIR, 'round2_report.txt')
    with open(report_path, 'w') as f:
        f.write(report)

    print(f"\n\nReport saved to: {report_path}")