
    pi-                         d Z ddlZddlZddlZddlZddlmZmZmZm	Z	m
Z
mZ ddlmZmZ ddlmZ ddlmZ ddlmZ  e         G d	 d
      Zy)a
  Curiosity Query Generator - transforms trending topics into questions using LLM.

Uses Groq Llama 3.3 70B to analyze headlines and extract the explainer opportunity
hidden behind each one. Replaces mechanical concept extraction with intelligent
question generation.
    N)ListDictAnySetTupleOptional)datetimetimezone)load_dotenv)Groq   )	TrendItemc                       e Zd ZdZd Zdeeeef      deeeef      fdZ	ddeeeef      de
deeeeef         fdZdeeeef      deeeeef      fd	Zd
edee   fdZ	 ddeeeef      de
dee   fdZy)CuriosityQueryCollectorz>Generates curiosity queries from trending topics via Groq LLM.c                     d| _         g | _        t        t        j                  d            | _        ddi| _        dddddd| _        y )Ncuriosity_queryGROQ_API_KEY)api_keyz
User-Agentz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36r   )items_sent_to_groqquestions_generatedskipped_by_llmyoutube_validatedgroq_errors)nameerrorsr   osgetenvgroq_clientheadersstats)selfs    F/home/sietch6/trending-topics-pipeline/collectors/curiosity_queries.py__init__z CuriosityQueryCollector.__init__   sO    %	!#		.(ABX
 #$#$!"

    itemsreturnc                     t               }g }|D ]c  }|j                  dd      }t        j                  dd|j	                               dd }||vsB|j                  |       |j                  |       e |S )zCRemove near-duplicate headlines (same story from multiple outlets).
topic_name z[^\w\s]N2   )setgetresubloweraddappend)r!   r%   seen_normalizeduniqueitemheadline
normalizeds          r"   _dedupe_headlinesz)CuriosityQueryCollector._dedupe_headlines)   sv    % 	$Dxxb1H
B0@A#2FJ0##J/d#	$ r$   
batch_sizec                 n    g }t        dt        |      |      D ]  }|j                  ||||z            |S )z'Split items into batches for API calls.r   )rangelenr1   )r!   r%   r8   batchesis        r"   _batch_itemsz$CuriosityQueryCollector._batch_items9   s?    q#e*j1 	4ANN51z>23	4r$   batchc                 ^   d}t        |d      D ]:  \  }}|j                  dd      dd }|j                  dd      }|| d| d	| d
z  }< d| d}	 | j                  j                  j                  j                  dd|dgdd      }|j                  d   j                  j                  j                         }	g }
|	j                  d
      }|D ]  }|j                         }|st        j                  d|      }|s0t        |j                  d            dz
  }|j                  d      j                         }d|cxk  rt        |      k  sn ||   j                  dd      }||   j                  dd      }|j!                         dk(  r| j"                  dxx   dz  cc<   |
j%                  |||f       | j"                  dxx   dz  cc<    |
S # t&        $ rJ}| j(                  j%                  dt+        |              | j"                  dxx   dz  cc<   g cY d}~S d}~ww xY w)z
        Call Groq LLM to generate questions from a batch of headlines/trends.

        Returns list of (original_headline, generated_question, source) tuples.
        r)   r   r(   N   sourceunknownz. [z] 
u  You are analyzing trending news headlines and search terms to identify what questions a curious but uninformed person would ask after seeing each one. Your goal is to find the EXPLAINER OPPORTUNITY hidden behind each headline.

For each item below, respond with either:
- A specific question that a regular person would want answered (phrased naturally, like how someone would type it into Google or ask on Reddit)
- "SKIP" if there's no natural explainer question (e.g., sports scores, celebrity gossip with no deeper angle, weather, stock prices, game results, athlete performances)

Rules:
- The question should be about understanding WHY or HOW something works, not just WHAT happened
- Prefer questions that could sustain a 5-15 minute explanation
- If a headline mentions a policy, law, technology, or scientific concept, the question should be about that concept, not the specific news event
- If a headline is about a person, the question should be about what they represent or why they matter, not biographical facts
- Sports game results, scores, player performances → SKIP
- Celebrity news without deeper societal angle → SKIP
- Weather updates, stock prices → SKIP

Items:
z

Respond in this exact format (one line per item, same numbering):
1. Why does [concept] work this way?
2. SKIP
3. How does [thing] actually affect [people]?
...zllama-3.3-70b-versatileuser)rolecontenti  g333333?)modelmessages
max_tokenstemperaturer   z^(\d+)[.)\s]+(.+)$   SKIPr   r   zGroq batch error: r   )	enumerater,   r   chatcompletionscreatechoicesmessagerG   stripsplitr-   matchintgroupr;   upperr    r1   	Exceptionr   str)r!   r?   
items_textr=   r4   r5   rB   promptresponseresponse_textresultslineslinerV   idxquestionoriginales                     r"   _call_groq_for_questionsz0CuriosityQueryCollector._call_groq_for_questions@   s6    
 * 	:GAtxxb1$37HXXh	2FQCs6("XJb99J	:
   !0(	'',,88??/#)f=>	 @ H %,,Q/77??EEGM G!''-E Czz| !6=ekk!n-1C${{1~335HC,#e*,#(:>>,#C!&s)!D#>>+v5 JJ'78A=8#NNHh+GH JJ'<=B='C* N 	KK!3CF8<=JJ}%*%I	s-   B,G  AG BG 	H,"?H'!H,'H,queryc           	      D   	 d}dd|d}t        j                  ||| j                  d      }|j                          |j                  }ddl}	 |j                  |      }t        |      d	k\  rt        |d
   t              r|d
   dd S t        j                  d|      }|r|j                  |j                               }t        |      d	k\  ryt        |d
   t              rfg }	|d
   D ]W  }
t        |
t              r#t        |
      dkD  r|	j                  |
d          6t        |
t              sG|	j                  |
       Y |	dd S g S # |j                  $ r Y w xY w# t         $ r9}| j"                  j                  d|dd  dt        |              g cY d}~S d}~ww xY w)z
        Get suggestions from YouTube search for a query.
        Used to validate if people are searching for the LLM-generated questions.
        z1https://suggestqueries.google.com/complete/searchyoutubeyt)clientdsq
   )paramsr   timeoutr   NrL   r      z\[.*\]zYouTube suggest error for '   z...': )requestsr,   r   raise_for_statustextjsonloadsr;   
isinstancelistJSONDecodeErrorr-   searchrX   r1   r[   rZ   r   )r!   rh   urlrp   r^   rv   rw   datarV   suggestionsr4   rf   s               r"   youtube_search_suggestz.CuriosityQueryCollector.youtube_search_suggest   s   
*	EC#F
  ||	H %%'==Dzz$'t9>ja$&?72A;&
 IIi.Ezz%++-0t9>ja$&?"$K $Q 5%dD1c$i!m'..tAw7'c2'..t4	5
 'r?*I! '' $  	KK!<U3BZLsSTvhWXI	sO   AE 9E B%E .E E EE EE 	F&.FFFtopics	max_itemsc                    |D cg c]  }|j                  d      dk(  s| }}|D cg c]  }|j                  d      dk(  s| }}|D cg c]  }|j                  d      dk(  s| }}t        dt        |       dt        |       dt        |       d       | j                  |      }t        d	t        |       d
       g }|j	                  |dd        |j	                  |dd        |j	                  |dd        |d| }t        |      | j
                  d<   t        dt        |       d       | j                  |d      }g }	t        |      D ]  \  }
}t        d|
dz    dt        |       dt        |       dd       | j                  |      }|	j	                  |       t        t        |       d       |
t        |      dz
  k  s}t        j                  d        t        dt        |	       d| j
                  d    d       g }|	D ]g  \  }}}|j                  t        |d t        |      dz   d!t        j                  t        j                         j#                         d"||d#d$d%&             i t        d'       d(}|dd) D ]  }|j$                  }| j'                  |      }|r|dz  }|dd* D ]  }|j)                         |j)                         k7  s%|j                  t        |d t        |      dz   d!t        j                  t        j                         j#                         d"|j*                  j                  d+d"      |j*                  j                  d,d"      d-d.|d/&              t        j                  d0        || j
                  d1<   t        d2| d3       t        |      | j
                  d4<   t        d5t        |              |S c c}w c c}w c c}w )6a9  
        Generate curiosity queries from trending topics using Groq LLM.

        Args:
            topics: List of topic dicts with 'topic_name' and 'source' keys
            max_items: Maximum items to process through Groq

        Returns:
            List of TrendItem objects for generated questions
        rB   google_newsgoogle_trendsrj   z    Input: z news, z	 trends, z videosz    After dedup: z unique news headlinesNx      ro   r   z    Sending z items to Groq LLM...   )r8   z      Batch r   /z (z
 items)... )endz
 questionsz    Groq complete: z questions, r   z skippedr   globalr)   Tgroq_llm)original_headlineoriginal_sourcellm_generatedquery_source)r(   rB   rankregion	timestampr}   metadataz7    Validating top 30 questions with YouTube suggest...r   rs   rL   r   r   Fyoutube_suggest)r   r   r   r   validated_fromg      ?r   z    YouTube validation: z/30 questions had suggestionsqueries_generatedz    Total curiosity queries: )r,   printr;   r7   extendr    r>   rN   rg   timesleepr1   r   r	   nowr
   utc	isoformatr(   r   r/   r   )r!   r   r   tr   r   rj   combinedr<   all_questionsr=   r?   	questionsr%   re   rd   rB   validated_countr4   r   
suggestions                        r"   generate_queries_for_topicsz3CuriosityQueryCollector.generate_queries_for_topics   s     #)MQAEE(O},LqMM$*QqaeeHo.PQQ$Eh9(D1EEC,-WS5G4H	RUV]R^Q__fgh ,,[9!#k"2!33IJK DS)*cr*+%JY'+.x=

'(S]O+@AB ##H#<!'* 	HAuL1Qs7|nBs5zl*MSVW55e<I  +S^$J/0 3w<!##

1	 	#C$6#7|DJJO_D`Caaijk *7 	&HhLL#(Z!^",,x||4>>@)1'-%)$.	 	" 	GI#2J 	DH55h?K1$"-bq/ J!'')X^^-==Y'1#4!$Ua#+&.ll8<<&@&J&J&L "59]]5F5FGZ\^5_37==3D3DEVXZ3[160A2:&& $ JJsO3	6 +:

&'((99VWX*-e*

&'-c%j\:;u NQEs!   O$O$O)O)O.'O.N)r   )rA   )__name__
__module____qualname____doc__r#   r   r   r[   r   r7   rW   r>   r   rg   r   r   r    r$   r"   r   r      s   H
tDcN'; T#s(^@T  $tCH~"6 C QUVZ[_`ceh`h[iVjQk Md4S>.B MtERUWZ\_R_L`Ga M^/C /DI /h jT#s(^$j j 
i	jr$   r   )r   r   r-   r   rt   typingr   r   r   r   r   r   r	   r
   dotenvr   groqr   baser   r   r   r$   r"   <module>r      s;    
 	   8 8 '    S Sr$   