#!/usr/bin/env python3
"""
Multi-Source Trending Topics Pipeline
Collects weekly trends from Google Trends, YouTube, Google News, and Reddit.
Supports channel-specific collection via --channel flag.
"""

import argparse
import json
import sqlite3
import sys
from datetime import datetime
from pathlib import Path
from typing import Optional, List, Set

from dotenv import load_dotenv

# Load environment variables from .env file
load_dotenv()

from collectors import get_all_collectors
from collectors.google_autocomplete import GoogleAutocompleteEnricher
from collectors.reddit_search import RedditSearchEnricher
from collectors.curiosity_queries import CuriosityQueryCollector
from aggregator.merger import merge_by_region, merge_all_regions
from config import REGIONS
from channel_config import (
    get_enabled_channels,
    get_channel,
    init_channel_tables,
    sync_channels_to_db,
    tag_item_to_channel,
    Channel
)

# Subreddits that represent "question" signal (others are "curiosity")
QUESTION_SUBREDDITS = {
    "OutOfTheLoop", "explainlikeimfive", "NoStupidQuestions", "TooAfraidToAsk",
    "answers", "askscience", "askhistorians", "changemyview"
}
CURIOSITY_SUBREDDITS = {"AskReddit", "whatisthisthing", "YouShouldKnow", "howdoesthiswork"}

OUTPUT_DIR = Path(__file__).parent / "output"
DB_PATH = OUTPUT_DIR / "trends_history.db"


def get_channel_sources(channel: Channel) -> dict:
    """Extract source identifiers from a channel config."""
    # Extract subreddit names from RedditSource objects
    reddit_names = set(r.name for r in channel.sources.reddit)
    # Extract RSS names from RSSSource objects
    rss_names = set(r.name for r in channel.sources.rss)

    return {
        "reddit_subreddits": reddit_names,
        "rss_feeds": rss_names,
        "youtube_categories": set(channel.sources.youtube_categories),
        "keywords_include": set(kw.lower() for kw in channel.keywords.include),
        "keywords_exclude": set(kw.lower() for kw in channel.keywords.exclude)
    }


def match_item_to_channels(item: dict, channels: dict) -> List[tuple]:
    """
    Match an item to channels based on source and keywords.
    Returns list of (channel_id, confidence, matched_by) tuples.
    """
    matches = []
    topic_lower = item.get("topic_name", "").lower()
    source = item.get("source", "")
    subreddit = item.get("metadata", {}).get("subreddit", "")

    for channel_id, channel in channels.items():
        ch_sources = get_channel_sources(channel)
        confidence = 0.0
        matched_by = []

        # Check source match
        if source == "reddit" and subreddit:
            # Check if subreddit matches (with or without r/ prefix)
            subreddit_clean = subreddit.lower().replace("r/", "")
            for ch_sub in ch_sources["reddit_subreddits"]:
                ch_sub_clean = ch_sub.lower().replace("r/", "")
                if subreddit_clean == ch_sub_clean:
                    confidence += 0.8
                    matched_by.append(f"subreddit:{subreddit}")
                    break

        # Check keyword match (both include and exclude)
        keywords_include = ch_sources["keywords_include"]
        keywords_exclude = ch_sources["keywords_exclude"]

        if keywords_include:
            for kw in keywords_include:
                if kw in topic_lower:
                    confidence += 0.3
                    matched_by.append(f"keyword:{kw}")
                    break  # Only count one keyword match

        if keywords_exclude:
            for kw in keywords_exclude:
                if kw in topic_lower:
                    confidence = 0  # Exclude this channel
                    matched_by = []
                    break

        # If confidence > threshold, add match
        if confidence >= 0.3:
            matches.append((channel_id, min(confidence, 1.0), ",".join(matched_by)))

    return matches


def tag_items_to_channels(items: list, run_id: int, channel_filter: Optional[str] = None):
    """Tag collected items to their matching channels."""
    channels = get_enabled_channels()

    if channel_filter:
        # Only match to specific channel
        channels = {k: v for k, v in channels.items() if k == channel_filter}

    if not channels:
        print("  No channels configured with sources. Skipping channel tagging.")
        return

    # Check if any channel has sources configured
    has_sources = any(ch.has_sources() for ch in channels.values())
    if not has_sources:
        print("  Channels exist but no sources configured yet. Skipping channel tagging.")
        return

    print(f"\n--- Tagging Items to Channels ---")

    # Get item IDs from database for this run
    conn = sqlite3.connect(DB_PATH)
    cursor = conn.cursor()
    cursor.execute(
        "SELECT id, topic_name, source, metadata FROM trend_items WHERE run_id = ?",
        (run_id,)
    )
    db_items = cursor.fetchall()
    conn.close()

    # Create lookup from topic_name to item_id
    topic_to_id = {}
    for item_id, topic_name, source, metadata_json in db_items:
        metadata = json.loads(metadata_json) if metadata_json else {}
        topic_to_id[topic_name] = {
            "id": item_id,
            "source": source,
            "metadata": metadata
        }

    tagged_count = {ch_id: 0 for ch_id in channels}

    for item in items:
        topic_name = item.get("topic_name", "")
        if topic_name not in topic_to_id:
            continue

        item_info = topic_to_id[topic_name]
        item_with_meta = {
            "topic_name": topic_name,
            "source": item_info["source"],
            "metadata": item_info["metadata"]
        }

        matches = match_item_to_channels(item_with_meta, channels)

        for channel_id, confidence, matched_by in matches:
            tag_item_to_channel(
                item_id=item_info["id"],
                channel_id=channel_id,
                confidence=confidence,
                matched_by=matched_by
            )
            tagged_count[channel_id] += 1

    for ch_id, count in tagged_count.items():
        if count > 0:
            print(f"  {ch_id}: {count} items tagged")


def init_db():
    """Initialize SQLite database for history tracking."""
    conn = sqlite3.connect(DB_PATH)
    cursor = conn.cursor()

    cursor.execute(
        """
        CREATE TABLE IF NOT EXISTS collection_runs (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            run_date TEXT NOT NULL,
            total_items INTEGER,
            sources_succeeded TEXT,
            sources_failed TEXT,
            errors TEXT
        )
    """
    )

    cursor.execute(
        """
        CREATE TABLE IF NOT EXISTS trend_items (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            run_id INTEGER,
            topic_name TEXT NOT NULL,
            source TEXT NOT NULL,
            original_rank INTEGER,
            final_rank INTEGER,
            region TEXT NOT NULL,
            final_score REAL,
            source_count INTEGER,
            boosted INTEGER,
            url TEXT,
            metadata TEXT,
            collected_at TEXT,
            autocomplete TEXT,
            reddit_score INTEGER,
            FOREIGN KEY (run_id) REFERENCES collection_runs(id)
        )
    """
    )

    # Add new columns if they don't exist (for existing databases)
    new_columns = [
        ("autocomplete", "TEXT"),
        ("reddit_score", "INTEGER"),
        ("selftext", "TEXT"),
        ("num_comments", "INTEGER"),
        ("subreddit", "TEXT"),
        ("category", "TEXT"),
        ("view_count", "INTEGER"),
        ("reddit_subreddits", "TEXT"),
    ]

    for col_name, col_type in new_columns:
        try:
            cursor.execute(f"ALTER TABLE trend_items ADD COLUMN {col_name} {col_type}")
        except sqlite3.OperationalError:
            pass  # Column already exists

    cursor.execute(
        "CREATE INDEX IF NOT EXISTS idx_trends_region ON trend_items(region)"
    )
    cursor.execute(
        "CREATE INDEX IF NOT EXISTS idx_trends_source ON trend_items(source)"
    )
    cursor.execute(
        "CREATE INDEX IF NOT EXISTS idx_trends_date ON collection_runs(run_date)"
    )
    cursor.execute(
        "CREATE INDEX IF NOT EXISTS idx_trends_topic ON trend_items(topic_name)"
    )

    # Reddit deduplication table for incremental collection
    cursor.execute(
        """
        CREATE TABLE IF NOT EXISTS reddit_seen (
            post_id TEXT PRIMARY KEY,
            subreddit TEXT,
            first_seen_at TEXT
        )
    """
    )

    # Signal type column for items
    try:
        cursor.execute("ALTER TABLE trend_items ADD COLUMN signal_type TEXT")
    except sqlite3.OperationalError:
        pass  # Column already exists

    conn.commit()
    conn.close()


def save_to_db(run_date: str, items: list, succeeded: list, failed: list, errors: list):
    """Save results to SQLite history database."""
    conn = sqlite3.connect(DB_PATH)
    cursor = conn.cursor()

    # Save run info
    cursor.execute(
        """
        INSERT INTO collection_runs (run_date, total_items,
                                     sources_succeeded, sources_failed, errors)
        VALUES (?, ?, ?, ?, ?)
    """,
        (
            run_date,
            len(items),
            ",".join(succeeded),
            ",".join(failed),
            "\n".join(errors),
        ),
    )
    run_id = cursor.lastrowid

    # Save each trend item
    for item in items:
        metadata = item.get("metadata", {})
        reddit_engagement = item.get("reddit_engagement", {})
        reddit_score = reddit_engagement.get("total_score", 0) + reddit_engagement.get("total_comments", 0)
        reddit_subreddits = reddit_engagement.get("subreddits", [])

        cursor.execute(
            """
            INSERT INTO trend_items (run_id, topic_name, source, original_rank,
                                    final_rank, region, final_score, source_count,
                                    boosted, url, metadata, collected_at,
                                    autocomplete, reddit_score, selftext, num_comments,
                                    subreddit, category, view_count, reddit_subreddits,
                                    signal_type)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        """,
            (
                run_id,
                item["topic_name"],
                item["source"],
                item["rank"],
                item.get("final_rank"),
                item["region"],
                item.get("final_score"),
                item.get("source_count", 1),
                1 if item.get("boosted") else 0,
                item.get("url"),
                json.dumps(metadata),
                item.get("timestamp"),
                json.dumps(item.get("autocomplete_suggestions", [])),
                reddit_score,
                metadata.get("selftext", ""),
                metadata.get("num_comments"),
                metadata.get("subreddit", ""),
                metadata.get("category", ""),
                metadata.get("view_count"),
                json.dumps(reddit_subreddits),
                item.get("signal_type", "unknown"),
            ),
        )

    conn.commit()
    conn.close()
    return run_id


def generate_markdown(results: dict, all_merged: list, run_date: str) -> str:
    """Generate Markdown report."""
    lines = [
        "# Weekly Trending Topics Report",
        f"**Generated:** {run_date}",
        f"**Period:** Last 7 days",
        "",
        "## Top Trending (All Regions)",
        "",
        "| Rank | Topic | Sources | Regions | Score | Boosted |",
        "|------|-------|---------|---------|-------|---------|",
    ]

    for item in all_merged[:30]:
        sources = ", ".join(item.get("sources", [item["source"]]))
        regions = ", ".join(item.get("regions", [item["region"]]))
        boosted = "Yes" if item.get("boosted") else ""
        score = f"{item.get('final_score', 0):.2f}"
        topic = item["topic_name"][:50] + "..." if len(item["topic_name"]) > 50 else item["topic_name"]
        lines.append(
            f"| {item['final_rank']} | {topic} | {sources} | {regions} | {score} | {boosted} |"
        )

    lines.append("")

    # Per-region breakdown
    for region, items in results.items():
        lines.append(f"## {region} Trends")
        lines.append("")
        lines.append("| Rank | Topic | Sources | Score |")
        lines.append("|------|-------|---------|-------|")

        for item in items[:25]:
            sources = ", ".join(item.get("sources", [item["source"]]))
            score = f"{item.get('final_score', 0):.2f}"
            topic = item["topic_name"][:50] + "..." if len(item["topic_name"]) > 50 else item["topic_name"]
            lines.append(f"| {item['final_rank']} | {topic} | {sources} | {score} |")

        lines.append("")

    return "\n".join(lines)


def run_pipeline(skip_enrichment: bool = False, channel: Optional[str] = None,
                 all_channels: bool = False, primary_only: bool = False):
    """Main pipeline execution.

    Args:
        skip_enrichment: Skip autocomplete and Reddit enrichment
        channel: If specified, only collect from this channel's sources
        all_channels: If True, collect from all enabled channels
        primary_only: If True, only collect from primary subreddits (skip secondary)
    """
    print("=" * 60)
    print("Multi-Source Trending Topics Pipeline")
    if channel:
        ch = get_channel(channel)
        if ch:
            print(f"Channel: {ch.name}")
        else:
            print(f"[ERROR] Unknown channel: {channel}")
            return None
    print("=" * 60)

    run_date = datetime.utcnow().isoformat()
    OUTPUT_DIR.mkdir(exist_ok=True)
    init_db()

    # Initialize channel tables
    init_channel_tables()
    sync_channels_to_db()

    # Collect from all sources
    collectors = get_all_collectors()

    # If channel or all_channels specified, replace Reddit collector with channel-specific one
    if channel or all_channels:
        from collectors.reddit_scraper import RedditScraperCollector

        all_subreddits = set()

        if all_channels:
            # Collect from all enabled channels
            enabled = get_enabled_channels()
            for ch_id, ch in enabled.items():
                if ch.sources.reddit:
                    for r in ch.sources.reddit:
                        if not primary_only or r.priority == "primary":
                            all_subreddits.add(r.name)
            print(f"Collecting from {len(enabled)} channels")
        else:
            # Single channel
            ch = get_channel(channel)
            if ch and ch.sources.reddit:
                for r in ch.sources.reddit:
                    if not primary_only or r.priority == "primary":
                        all_subreddits.add(r.name)

        channel_subreddits = list(all_subreddits)
        priority_label = "primary" if primary_only else "all"
        print(f"Using {len(channel_subreddits)} {priority_label} subreddits")

        # Replace the default Reddit collector with channel-specific one
        for i, collector in enumerate(collectors):
            if collector.name == "reddit":
                collectors[i] = RedditScraperCollector(subreddits=channel_subreddits)
                break

    all_items = []
    all_errors = []
    succeeded = []
    failed = []

    for collector in collectors:
        print(f"\n--- Collecting from {collector.name} ---")
        collector_succeeded = False

        # Reddit is global, not region-specific
        if collector.name == "reddit":
            print(f"  (global)...", end=" ")
            items = collector.safe_collect("global")
            if items:
                all_items.extend([item.to_dict() for item in items])
                print(f"Collected {len(items)} items")
                collector_succeeded = True
            else:
                print("No items (or error)")
        else:
            # Region-specific collectors
            for region in REGIONS:
                print(f"  Region: {region}...", end=" ")
                items = collector.safe_collect(region)

                if items:
                    all_items.extend([item.to_dict() for item in items])
                    print(f"Collected {len(items)} items")
                    collector_succeeded = True
                else:
                    print("No items (or error)")

        if collector_succeeded:
            if collector.name not in succeeded:
                succeeded.append(collector.name)
        else:
            if collector.name not in failed:
                failed.append(collector.name)

        all_errors.extend(collector.errors)

    if not all_items:
        print("\n[ERROR] No items collected from any source!")
        return None

    # Step 2: Generate curiosity queries from non-Reddit sources
    print("\n--- Generating Curiosity Queries ---")

    # Extract unique topics from Google Trends, Google News, and YouTube (prioritize in that order)
    non_reddit_topics = []
    seen_topics = set()

    # Priority order: google_trends first (highest signal), then google_news, then youtube
    for source_priority in ["google_trends", "google_news", "youtube"]:
        for item in all_items:
            if item["source"] == source_priority:
                topic = item["topic_name"]
                if topic.lower() not in seen_topics:
                    seen_topics.add(topic.lower())
                    non_reddit_topics.append({
                        "topic_name": topic,
                        "source": item["source"]
                    })

    print(f"  Found {len(non_reddit_topics)} unique non-Reddit topics")

    # Generate curiosity queries via Groq LLM
    curiosity_collector = CuriosityQueryCollector()
    curiosity_items = curiosity_collector.generate_queries_for_topics(non_reddit_topics, max_items=150)

    if curiosity_items:
        all_items.extend([item.to_dict() for item in curiosity_items])
        succeeded.append("curiosity_query")
        print(f"  Generated {len(curiosity_items)} curiosity queries")
        print(f"    Sent to Groq: {curiosity_collector.stats['items_sent_to_groq']}")
        print(f"    Questions generated: {curiosity_collector.stats['questions_generated']}")
        print(f"    Skipped by LLM: {curiosity_collector.stats['skipped_by_llm']}")
        print(f"    YouTube validated: {curiosity_collector.stats['youtube_validated']}")
        if curiosity_collector.stats['groq_errors'] > 0:
            print(f"    Groq errors: {curiosity_collector.stats['groq_errors']}")
    else:
        print("  No curiosity queries generated")

    all_errors.extend(curiosity_collector.errors)

    # Step 3: Assign signal_type to all items
    print("\n--- Assigning Signal Types ---")
    for item in all_items:
        source = item.get("source", "")
        subreddit = item.get("metadata", {}).get("subreddit", "")

        if source == "reddit":
            if subreddit in QUESTION_SUBREDDITS:
                item["signal_type"] = "question"
            elif subreddit in CURIOSITY_SUBREDDITS:
                item["signal_type"] = "curiosity"
            else:
                item["signal_type"] = "question"  # Default for Reddit
        elif source == "curiosity_query":
            item["signal_type"] = "question"
        elif source == "google_trends":
            item["signal_type"] = "search_volume"
        elif source == "youtube":
            item["signal_type"] = "video_demand"
        elif source == "google_news":
            item["signal_type"] = "news_context"
        else:
            item["signal_type"] = "unknown"

    signal_counts = {}
    for item in all_items:
        st = item.get("signal_type", "unknown")
        signal_counts[st] = signal_counts.get(st, 0) + 1
    print(f"  Signal type distribution: {signal_counts}")

    # Merge and score
    print("\n--- Aggregating Results ---")
    results_by_region = merge_by_region(all_items)
    all_merged = merge_all_regions(all_items)

    for region, items in results_by_region.items():
        print(f"  {region}: {len(items)} unique topics after deduplication")

    print(f"  All regions combined: {len(all_merged)} unique topics")

    # Enrichment (skip with --no-enrich for faster runs)
    autocomplete_data = {}
    reddit_data = {}

    if not skip_enrichment:
        # Enrich with Google Autocomplete
        print("\n--- Enriching with Google Autocomplete ---")
        autocomplete_enricher = GoogleAutocompleteEnricher()
        topic_names = [item["topic_name"] for item in all_merged]
        autocomplete_data = autocomplete_enricher.enrich(topic_names)
        print(f"  Autocomplete enrichment complete. Errors: {len(autocomplete_enricher.errors)}")
        all_errors.extend(autocomplete_enricher.errors)

        # Enrich with Reddit search
        print("\n--- Enriching with Reddit Search ---")
        reddit_enricher = RedditSearchEnricher()
        reddit_data = reddit_enricher.enrich(topic_names)
        print(f"  Reddit enrichment complete. Errors: {len(reddit_enricher.errors)}")
        all_errors.extend(reddit_enricher.errors)
    else:
        print("\n--- Skipping enrichment (--no-enrich flag set) ---")

    # Attach enrichment data to each topic
    for item in all_merged:
        topic = item["topic_name"]
        item["autocomplete_suggestions"] = autocomplete_data.get(topic, [])
        item["reddit_engagement"] = reddit_data.get(topic, {})

    # Also attach to by_region items
    for region, items in results_by_region.items():
        for item in items:
            topic = item["topic_name"]
            item["autocomplete_suggestions"] = autocomplete_data.get(topic, [])
            item["reddit_engagement"] = reddit_data.get(topic, {})

    # Save to database
    run_id = save_to_db(run_date, all_merged, succeeded, failed, all_errors)
    print(f"\nSaved to database (run_id: {run_id})")

    # Tag items to channels
    tag_items_to_channels(all_merged, run_id, channel_filter=channel)

    # Save JSON output
    json_path = OUTPUT_DIR / "trends_weekly.json"
    output_data = {
        "run_date": run_date,
        "run_id": run_id,
        "all_trends": all_merged,
        "by_region": results_by_region,
        "sources_succeeded": succeeded,
        "sources_failed": failed,
        "errors": all_errors,
    }
    with open(json_path, "w") as f:
        json.dump(output_data, f, indent=2)
    print(f"JSON saved: {json_path}")

    # Save Markdown output
    md_path = OUTPUT_DIR / "trends_weekly.md"
    with open(md_path, "w") as f:
        f.write(generate_markdown(results_by_region, all_merged, run_date))
    print(f"Markdown saved: {md_path}")

    # Summary
    print("\n" + "=" * 60)
    print("Pipeline Complete")
    print("=" * 60)
    print(f"Sources succeeded: {', '.join(succeeded) or 'None'}")
    print(f"Sources failed: {', '.join(failed) or 'None'}")
    print(f"Total errors: {len(all_errors)}")
    print(f"Total unique trends: {len(all_merged)}")

    return output_data


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Trending Topics Pipeline")
    parser.add_argument("--enrich", action="store_true", help="Run enrichment (autocomplete + Reddit search)")
    parser.add_argument("--channel", type=str, help="Collect only for a specific channel")
    parser.add_argument("--all-channels", action="store_true", help="Collect for all enabled channels")
    parser.add_argument("--primary-only", action="store_true", help="Only collect from primary subreddits (skip secondary)")
    parser.add_argument("--list-channels", action="store_true", help="List available channels and exit")
    args = parser.parse_args()

    if args.list_channels:
        print("Available channels:")
        for ch_id, ch in get_enabled_channels().items():
            sources_count = len(ch.sources.reddit) + len(ch.sources.rss)
            print(f"  {ch_id}: {ch.name} ({sources_count} sources)")
        sys.exit(0)

    # Skip enrichment by default in data accumulation mode
    run_pipeline(
        skip_enrichment=not args.enrich,
        channel=args.channel,
        all_channels=args.all_channels,
        primary_only=args.primary_only
    )