
    	iZ              
          d Z ddlZddlZddlZddlZddlmZ ddlmZ ddlm	Z	m
Z
mZ ddlmZ  e        ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZmZmZm Z m!Z! h dZ"h dZ# ee$      jJ                  dz  Z&e&dz  Z'de!de(fdZ)de(de(de
e*   fdZ+d9de,de-de	e.   fdZ/d Z0de.de,de,de,d e,f
d!Z1d"e(d#e,de.de.fd$Z2d:d%e3de	e.   fd&Z4e5d'k(  r ejl                  d()      Z7e7jq                  d*d+d,-       e7jq                  d.e.d/0       e7jq                  d1d+d2-       e7js                         Z:e:jv                  r e<d3        e       j{                         D ]Z  \  Z>Z? e@e?j                  j                         e@e?j                  j                        z   ZD e<d4e> d5e?j                   d6eD d7       \  ej                  d        e4e:j                   e:j                  8       yy);z
Multi-Source Trending Topics Pipeline
Collects weekly trends from Google Trends, YouTube, Google News, and Reddit.
Supports channel-specific collection via --channel flag.
    N)datetime)Path)OptionalListSet)load_dotenv)get_all_collectors)GoogleAutocompleteEnricher)RedditSearchEnricher)CuriosityQueryCollector)merge_by_regionmerge_all_regions)REGIONS)get_enabled_channelsget_channelinit_channel_tablessync_channels_to_dbtag_item_to_channelChannel>   answers
askscienceOutOfTheLoopchangemyviewaskhistoriansTooAfraidToAskNoStupidQuestionsexplainlikeimfive>   	AskRedditYouShouldKnowhowdoesthisworkwhatisthisthingoutputztrends_history.dbchannelreturnc                 t   t        d | j                  j                  D              }t        d | j                  j                  D              }||t        | j                  j                        t        d | j
                  j                  D              t        d | j
                  j                  D              dS )z1Extract source identifiers from a channel config.c              3   4   K   | ]  }|j                     y wNname.0rs     main.py	<genexpr>z&get_channel_sources.<locals>.<genexpr>2   s     >!qvv>   c              3   4   K   | ]  }|j                     y wr'   r(   r*   s     r-   r.   z&get_channel_sources.<locals>.<genexpr>4   s     8qAFF8r/   c              3   <   K   | ]  }|j                           y wr'   lowerr+   kws     r-   r.   z&get_channel_sources.<locals>.<genexpr>:        Nr
N   c              3   <   K   | ]  }|j                           y wr'   r2   r4   s     r-   r.   z&get_channel_sources.<locals>.<genexpr>;   r6   r7   )reddit_subreddits	rss_feedsyoutube_categorieskeywords_includekeywords_exclude)setsourcesredditrssr;   keywordsincludeexclude)r#   reddit_names	rss_namess      r-   get_channel_sourcesrG   /   s     >w'='=>>L8GOO$7$788I *!'//"D"DENW5E5E5M5MNNNW5E5E5M5MNN     itemchannelsc                    g }| j                  dd      j                         }| j                  dd      }| j                  di       j                  dd      }|j                         D ]  \  }}t        |      }d}	g }
|dk(  rl|rj|j                         j	                  dd      }|d	   D ]B  }|j                         j	                  dd      }||k(  s)|	d
z  }	|
j                  d|         n |d   }|d   }|r&|D ]!  }||v s|	dz  }	|
j                  d|         n |r|D ]  }||v sd}	g }
 n |	dk\  s|j                  |t        |	d      dj                  |
      f        |S )z
    Match an item to channels based on source and keywords.
    Returns list of (channel_id, confidence, matched_by) tuples.
    
topic_name sourcemetadata	subredditg        r@   zr/r9   g?z
subreddit:r<   r=   g333333?zkeyword:r   g      ?,)getr3   itemsrG   replaceappendminjoin)rI   rJ   matchestopic_lowerrN   rP   
channel_idr#   
ch_sources
confidence
matched_bysubreddit_cleanch_subch_sub_cleanr<   r=   r5   s                    r-   match_item_to_channelsra   ?   s   
 G((<,224KXXh#FR(,,["=I'~~/ $U
G(1


 X)'oo/77bAO$%89 %||~55dB?"l2#%J%%
9+&>? &&89%&89& $#%J%%o6	 & $!"J!#J	 NNJJ(<chhz>RSTI$UL NrH   rS   run_idchannel_filterc                 t   t               }|r)|j                         D ci c]  \  }}||k(  s|| }}}|st        d       yt        d |j	                         D              }|st        d       yt        d       t        j                  t              }|j                         }|j                  d|f       |j                         }	|j                          i }
|	D ])  \  }}}}|rt        j                  |      ni }|||d|
|<   + |D ci c]  }|d }}| D ]`  }|j                  d	d
      }||
vr|
|   }||d   |d   d}t        ||      }|D ]%  \  }}}t!        |d   |||       ||xx   dz  cc<   ' b |j                         D ]  \  }}|dkD  st        d| d| d        yc c}}w c c}w )z/Tag collected items to their matching channels.z@  No channels configured with sources. Skipping channel tagging.Nc              3   <   K   | ]  }|j                           y wr'   )has_sources)r+   chs     r-   r.   z(tag_items_to_channels.<locals>.<genexpr>   s     C2bnn&Cr7   zI  Channels exist but no sources configured yet. Skipping channel tagging.z"
--- Tagging Items to Channels ---zISELECT id, topic_name, source, metadata FROM trend_items WHERE run_id = ?)idrN   rO   r   rL   rM   rN   rO   )rL   rN   rO   rh   )item_idrZ   r\   r]        : z items tagged)r   rS   printanyvaluessqlite3connectDB_PATHcursorexecutefetchallclosejsonloadsrR   ra   r   )rS   rb   rc   rJ   kvrf   connrs   db_itemstopic_to_idri   rL   rN   metadata_jsonrO   ch_idtagged_countrI   	item_infoitem_with_metarX   rZ   r\   r]   counts                             r-   tag_items_to_channelsr   r   s    #%H%-^^%5MTQn9LAqDMMPQ C1BCCKYZ	/1 ??7#D[[]F
NNS		  HJJL K6> 
2V]0=4::m,2 #
J
 +33E1H3L3 *XXlB/
[(
+	$)!*-
 )B29 	*.J
J!$%%%	 $)$	**. %**, 6u19BugRwm456w ND 4s   F/F/
F5c                  X   t        j                  t              } | j                         }|j	                  d       |j	                  d       g d}|D ]  \  }}	 |j	                  d| d|         |j	                  d       |j	                  d       |j	                  d       |j	                  d	       |j	                  d
       	 |j	                  d       | j                          | j                          y# t         j
                  $ r Y w xY w# t         j
                  $ r Y Ow xY w)z0Initialize SQLite database for history tracking.a  
        CREATE TABLE IF NOT EXISTS collection_runs (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            run_date TEXT NOT NULL,
            total_items INTEGER,
            sources_succeeded TEXT,
            sources_failed TEXT,
            errors TEXT
        )
    ai  
        CREATE TABLE IF NOT EXISTS trend_items (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            run_id INTEGER,
            topic_name TEXT NOT NULL,
            source TEXT NOT NULL,
            original_rank INTEGER,
            final_rank INTEGER,
            region TEXT NOT NULL,
            final_score REAL,
            source_count INTEGER,
            boosted INTEGER,
            url TEXT,
            metadata TEXT,
            collected_at TEXT,
            autocomplete TEXT,
            reddit_score INTEGER,
            FOREIGN KEY (run_id) REFERENCES collection_runs(id)
        )
    ))autocompleteTEXT)reddit_scoreINTEGER)selftextr   )num_commentsr   )rP   r   )categoryr   )
view_countr   )r9   r   z#ALTER TABLE trend_items ADD COLUMN  zCCREATE INDEX IF NOT EXISTS idx_trends_region ON trend_items(region)zCCREATE INDEX IF NOT EXISTS idx_trends_source ON trend_items(source)zGCREATE INDEX IF NOT EXISTS idx_trends_date ON collection_runs(run_date)zFCREATE INDEX IF NOT EXISTS idx_trends_topic ON trend_items(topic_name)z
        CREATE TABLE IF NOT EXISTS reddit_seen (
            post_id TEXT PRIMARY KEY,
            subreddit TEXT,
            first_seen_at TEXT
        )
    z3ALTER TABLE trend_items ADD COLUMN signal_type TEXTN)rp   rq   rr   rs   rt   OperationalErrorcommitrv   )r{   rs   new_columnscol_namecol_types        r-   init_dbr      s,   ??7#D[[]F
NN		 NN	0	K * (	NN@
!H:VW NNM NNM NNQ NNP
 NN	LM 	KKMJJLE '' 		< ## s$   C:D :DDD)(D)run_date	succeededfailederrorsc                    t        j                  t              }|j                         }|j	                  d| t        |      dj                  |      dj                  |      dj                  |      f       |j                  }|D ]  }|j                  di       }	|j                  di       }
|
j                  dd      |
j                  dd      z   }|
j                  d	g       }|j	                  d
||d   |d   |d   |j                  d      |d   |j                  d      |j                  dd      |j                  d      rdnd|j                  d      t        j                  |	      |j                  d      t        j                  |j                  dg             ||	j                  dd      |	j                  d      |	j                  dd      |	j                  dd      |	j                  d      t        j                  |      |j                  dd      f        |j                          |j                          |S )z(Save results to SQLite history database.z
        INSERT INTO collection_runs (run_date, total_items,
                                     sources_succeeded, sources_failed, errors)
        VALUES (?, ?, ?, ?, ?)
    rQ   
rO   reddit_engagementtotal_scorer   total_comments
subredditsa*  
            INSERT INTO trend_items (run_id, topic_name, source, original_rank,
                                    final_rank, region, final_score, source_count,
                                    boosted, url, metadata, collected_at,
                                    autocomplete, reddit_score, selftext, num_comments,
                                    subreddit, category, view_count, reddit_subreddits,
                                    signal_type)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        rL   rN   rank
final_rankregionfinal_scoresource_countrj   boostedurl	timestampautocomplete_suggestionsr   rM   r   rP   r   r   signal_typeunknown)rp   rq   rr   rs   rt   lenrW   	lastrowidrR   rw   dumpsr   rv   )r   rS   r   r   r   r{   rs   rb   rI   rO   r   r   r9   s                r-   
save_to_dbr     s    ??7#D[[]F NN	 JHHYHHVIIf	
 F  '
88J+ HH%8"=(,,]A>ARAVAVWgijAkk-11,C \"XV&X'+XXi(a

8$%

488$>CDZ,^,["-Z,\*

,-	2+!	
'
R 	KKMJJLMrH   results
all_mergedc                    dd| ddddddg}|dd	 D ]  }d
j                  |j                  d|d   g            }d
j                  |j                  d|d   g            }|j                  d      rdnd}|j                  dd      d}t        |d         dkD  r|d   dd dz   n|d   }	|j                  d|d    d|	 d| d| d| d| d        |j                  d       | j	                         D ]  \  }
}|j                  d|
 d       |j                  d       |j                  d       |j                  d       |dd D ]}  }d
j                  |j                  d|d   g            }|j                  dd      d}t        |d         dkD  r|d   dd dz   n|d   }	|j                  d|d    d|	 d| d| d	        |j                  d        d j                  |      S )!zGenerate Markdown report.z# Weekly Trending Topics Reportz**Generated:** z**Period:** Last 7 daysrM   z## Top Trending (All Regions)z6| Rank | Topic | Sources | Regions | Score | Boosted |z6|------|-------|---------|---------|-------|---------|N   , r?   rN   regionsr   r   Yesr   r   z.2frL   2   ...z| r   z | z |z## z Trendsz"| Rank | Topic | Sources | Score |z"|------|-------|---------|-------|   r   )rW   rR   r   rU   rS   )r   r   r   linesrI   r?   r   r   scoretopicr   rS   s               r-   generate_markdownr   Z  sF    	*
($
!
'
@@	E 3B 
))DHHYh0@AB))DHHYh0@AB88I.%B88M1-c236tL7I3JR3O\"3B'%/UYZfUgl#$Cwc'#gYc%PST[S\\^_	

 
LL ! s6('*+R9:9:#2J 	VDiiT(^4D EFGxxq1#6E7:4;M7NQS7SD&s+e3Y]^jYkELL2d<01UG3wis5'QSTU		V 	R 99UrH   skip_enrichmentc           	         t        d       t        d       |r5t        |      }|rt        d|j                          nt        d|        yt        d       t        j                         j                         }t        j                  d       t                t                t                t               }|rt        |      }|r|j                  j                  r{|j                  j                  D cg c]  }|j                   }}t        dt        |       d	       d
dlm} t#        |      D ]"  \  }}	|	j                  dk(  s ||      ||<    n g }
g }g }g }|D ]  }	t        d|	j                   d       d}|	j                  dk(  rtt        dd       |	j%                  d      }|rH|
j'                  |D cg c]  }|j)                          c}       t        dt        |       d       d}nt        d       nt*        D ]y  }t        d| dd       |	j%                  |      }|rH|
j'                  |D cg c]  }|j)                          c}       t        dt        |       d       d}ot        d       { |r*|	j                  |vrE|j-                  |	j                         n)|	j                  |vr|j-                  |	j                         |j'                  |	j.                          |
st        d       yt        d       g }t1               }dD ]`  }|
D ]Y  }|d   |k(  s|d   }|j3                         |vs$|j5                  |j3                                |j-                  ||d   d       [ b t        d t        |       d!       t7               }|j9                  |d"#      }|r|
j'                  |D cg c]  }|j)                          c}       |j-                  d$       t        d%t        |       d&       t        d'|j:                  d(           t        d)|j:                  d*           t        d+|j:                  d,           t        d-|j:                  d.           |j:                  d/   d
kD  r't        d0|j:                  d/           nt        d1       |j'                  |j.                         t        d2       |
D ]  }|j=                  dd3      }|j=                  d4i       j=                  d5d3      }|dk(  r"|t>        v rd6|d7<   J|t@        v rd8|d7<   Xd6|d7<   ^|d$k(  rd6|d7<   i|d9k(  rd:|d7<   t|d;k(  rd<|d7<   |d=k(  rd>|d7<   d?|d7<    i }|
D ],  }|j=                  d7d?      }|j=                  |d
      d@z   ||<   . t        dA|        t        dB       tC        |
      }tE        |
      }|jG                         D ]   \  }}t        dC| dDt        |       dE       " t        dFt        |       dG       i }i }| st        dH       tI               } |D cg c]  }|d   	 }!}| jK                  |!      }t        dIt        | j.                                |j'                  | j.                         t        dJ       tM               }"|"jK                  |!      }t        dKt        |"j.                                |j'                  |"j.                         nt        dL       |D ]1  }|d   }|j=                  |g       |dM<   |j=                  |i       |dN<   3 |jG                         D ];  \  }}|D ]1  }|d   }|j=                  |g       |dM<   |j=                  |i       |dN<   3 = tO        |||||      }#t        dO|# dP       tQ        ||#|Q       t        dRz  }$||#|||||dS}%tS        |$dT      5 }&tU        jV                  |%|&dUV       ddd       t        dW|$        t        dXz  }'tS        |'dT      5 }&|&jY                  t[        |||             ddd       t        dY|'        t        dZ       t        d[       t        d       t        d\d]j]                  |      xs d^        t        d_d]j]                  |      xs d^        t        d`t        |              t        dat        |              |%S c c}w c c}w c c}w c c}w c c}w # 1 sw Y   xY w# 1 sw Y   xY w)bzMain pipeline execution.

    Args:
        skip_enrichment: Skip autocomplete and Reddit enrichment
        channel: If specified, only collect from this channel's sources
    z<============================================================z%Multi-Source Trending Topics Pipelinez	Channel: z[ERROR] Unknown channel: NT)exist_okzUsing z channel-specific subredditsr   )RedditScraperCollectorr@   )r   z
--- Collecting from z ---Fz  (global)...r   )endglobalz
Collected z itemszNo items (or error)z
  Region: r   z,
[ERROR] No items collected from any source!z%
--- Generating Curiosity Queries ---)google_trendsgoogle_newsyoutuberN   rL   )rL   rN   z  Found z unique non-Reddit topics   )	max_itemscuriosity_queryz  Generated z curiosity queriesz    Sent to Groq: items_sent_to_groqz    Questions generated: questions_generatedz    Skipped by LLM: skipped_by_llmz    YouTube validated: youtube_validatedgroq_errorsz    Groq errors: z   No curiosity queries generatedz
--- Assigning Signal Types ---rM   rO   rP   questionr   	curiosityr   search_volumer   video_demandr   news_contextr   rj   z  Signal type distribution: z
--- Aggregating Results ---rk   rl   z" unique topics after deduplicationz  All regions combined: z unique topicsz+
--- Enriching with Google Autocomplete ---z,  Autocomplete enrichment complete. Errors: z%
--- Enriching with Reddit Search ---z&  Reddit enrichment complete. Errors: z3
--- Skipping enrichment (--no-enrich flag set) ---r   r   z
Saved to database (run_id: ))rc   ztrends_weekly.json)r   rb   
all_trends	by_regionsources_succeededsources_failedr   w   )indentzJSON saved: ztrends_weekly.mdzMarkdown saved: z=
============================================================zPipeline CompletezSources succeeded: r   NonezSources failed: zTotal errors: zTotal unique trends: )/rm   r   r)   r   utcnow	isoformat
OUTPUT_DIRmkdirr   r   r   r	   r?   r@   r   collectors.reddit_scraperr   	enumeratesafe_collectextendto_dictr   rU   r   r>   r3   addr   generate_queries_for_topicsstatsrR   QUESTION_SUBREDDITSCURIOSITY_SUBREDDITSr   r   rS   r
   enrichr   r   r   openrw   dumpwriter   rW   )(r   r#   rg   r   
collectorsr,   channel_subredditsr   i	collector	all_items
all_errorsr   r   collector_succeededrS   rI   r   non_reddit_topicsseen_topicssource_priorityr   curiosity_collectorcuriosity_itemsrN   rP   signal_countsstresults_by_regionr   autocomplete_datareddit_dataautocomplete_enrichertopic_namesreddit_enricherrb   	json_pathoutput_datafmd_paths(                                           r-   run_pipeliner    sX	    
(O	
12!IbggY'(-gY78	(O **,Hd#I  $%J !"**##24**2C2C!DQ!&&!D!DF31233OPQ I )* 5 9>>X-$:FX$YJqM
 IJIF ",	&y~~&6d;<# >>X%M,**84E  U!CT$,,.!CD
3u:,f56&*#+, " 	1
6(#.C8!..v6$$%Gdlln%GHJs5zl&9:*.'/0	1 ~~Y.  0~~V+inn-)**+E",H => 

23 %K G 	 	DH~0\*;;=3OOEKKM2%,,&+"&x.. 		 
HS*+,,E
FG 23)EEFWcfEgO_ET$,,.EF*+S122DEF"#6#<#<=Q#R"STU)*=*C*CDY*Z)[\]$%8%>%>?O%P$QRS'(;(A(ABU(V'WXY$$]3a7%&9&?&?&N%OPQ01)001 

,- ,(B'HHZ,00bA	X//&0]#22&1]#&0]#((",D&"1Dy "0D}$"0D"+D),, M 9XXmY/)--b!4q8b9 
(
89 

)*'	2"9-J*002 M6("SZL(JKLM 
$S_$5^
DE K<= : <6@AdtL)AA188E<SAVA]A]=^<_`a/667 	67.0%,,[96s?;Q;Q7R6STU/001DE  ?\"+<+@+@+K'($/OOE2$> !? +002 C 	CD&E/@/D/DUB/OD+,(3r(BD$%	CC *iLF	)&
34 *fWE 11I && K 
i	 ,		+q+,	L
$% --G	gs	 Lq	!"3ZJKL	WI
&' 
/	
	(O			) 4 >?
@A	TYYv.8&9
:;	N3z?+
,-	!#j/!2
34_ "E0 "D &HZ F| BZ, ,L Ls6   6c c"
c'
c,;c1c6d6d d__main__zTrending Topics Pipeline)descriptionz--enrich
store_truez-Run enrichment (autocomplete + Reddit search))actionhelpz	--channelz#Collect only for a specific channel)typer
  z--list-channelsz List available channels and exitzAvailable channels:rk   rl   z (z	 sources))r   r#   r'   )FN)I__doc__argparserw   rp   sysr   pathlibr   typingr   r   r   dotenvr   r   r	   collectors.google_autocompleter
   collectors.reddit_searchr   collectors.curiosity_queriesr   aggregator.mergerr   r   configr   channel_configr   r   r   r   r   r   r   r   __file__parentr   rr   dictrG   tuplera   listintstrr   r   r   r   boolr  __name__ArgumentParserparseradd_argument
parse_argsargslist_channelsrm   rS   r   rg   r   r?   r@   rA   sources_countr)   exitr   r#    rH   r-   <module>r*     sU      
   & &   ) E 9 @ @    \ (^""X-

*
* T  0 0 0$u+ 0fC6 C6s C6HSM C6LZzB BT Bd BD BRV BJ(t ( ( ( (Vq$ q# qh z$X$$1KLF

<>mn
#4YZ
),EghD#$-/557 	EIE2

 1 12S5HHMBugRy=/CD	E 	 T[[$,,G rH   