i> d Z ddlZddlZddlZddlmZ ddlmZ ddlmZm Z m
Z
ee j dz Z
e
dz Ze
dz Zg d Zd
d d Zd
efdZd Zd
efdZdeded
efdZded
efdZd
e fdZd
e
e fdZd!dee fdZ e dk( rK ej< d Z e jA d ed e jC Z" e e"jF yy)"z
Generate dashboard data JSON from analysis results and database stats.
Supports channel filtering and includes channel metadata for the UI.
Run: python3 generate_dashboard_data.py [--channel CHANNEL_ID]
N)Path)datetime)OptionalDictListoutputztrends_history.dbzdashboard_data.json)how_things_workbrain_is_lying power_works money_traps
body_is_weirdwhat_actually_happened )defaultc | }|D ]- }||c S t |t r|j | }+|c S ||S t |t r dD ]
}||v s|| c S t | S |S )z$Safely traverse nested dict/objects.)textvaluecontent descriptionprimarymain)
isinstancedictgetstr)objr keysvalkeyks A/home/sietch6/trending-topics-pipeline/generate_dashboard_data.pysafe_getr" s
C ;N c4 ''#,CN
{#t O ACx1v
3x J returnc | yt | t r| S t | t rg }| D ] }t |t r|j d| (t |t s9|j d xs1 |j d xs |j d xs t | }|j d| dj
| S t | t rg }| j D ] \ }}|j dd j }t |t r?d
j
d |dd D }t | d kD r=|d
t | d z
dz
}n't |t r t | }n t | }|j d| d| dj
| S t | S )z@Format a potentially nested content field as readable HTML text.Nr u • r r r z
_ z; c 3 8 K | ] }t | d d y w)Nd )r ).0cs r! z'format_content_field..K s (K!Q (Ks z (+z more)zz : z
) r r listappendr r joinitemsreplacetitlelenformat_content_field) r r1 itemr partsr r label content_texts r! r5 r5 4 s
{#s
#t ,D$$ tD6]+D$'xx'f488I+>f$((=BYf]`ae]f tD6]+
, {{5!!#t IIK
FLCKKS)//1E'4( #yy(Kwr{(K K w> ??7 #D
[[]F NN56!??,Q/E- NNRS H V'' 4B'){{3C'DE# $ JJL L V;CA;x{3B'7IE# $s ,C C'oppr= c | j dg }t |t r|r t | }nt }| j d| j dg }|D ]9 }t |t s|j d s&|j |d ; | j dg D ]9 }t |t s|j d s&|j |d ; | j dd }|dk rt
|d z n
t
| }| j d
i }t |t rd |i}nt |t si }t |d xs t |d } |j d
|j dg }
t |
t r5|
D cg c]) } t | t r t | d n
t | + }
} ng }
| j di } | j di }
i }ddddddd}|j D ]o \ }}|
j |i }t |t r)|j dd |j dd d ||< Qt |t t f sh|dd ||< q | j d i }|r|j d s|j d g }t | D cg c] \ }}|j d d!|j d"|d#z |j d$d |j d%d |j d&d |j d'd |j d(d |j d)d d* }}}|j d+d |j d,d |j d-g |j d.g d/}n| j d0g }t |t rnt | D cg c]W \ }}t |d1 xs t |d xs d2|d#z t |d3 xs dt |d' xs t |d4 xs dd5Y }}}ng }d6}i d7t | d8| j d8d9 d|d | j d:| j d;d< d=t | d= xs dd|r t | nd>gd?| j d?d# d@| j d@d dAt t | dB dCt | dC xs t | dD xs ddBt | dB xs ddE| xs d|
r|
d6d ng dFdGt | dH xs t | dI xs dt | dJ xs dt | dK xs dt | dL xs ddMdNt | j dNd dOt | j dOd dPt | dQ xs ddRt | dS xs d|||dTS c c} w c c}}w c c}}w )Uz3Transform analysis opportunity to dashboard format.sourcestopics_in_clustercluster_topicssource merged_fromweighted_scorer confusion_analysisprimary_questionr secondary_questions secondaryquestiontarget_audiencescoresDemandGapExplainability EvergreenBreadth Competition)
demand_signal content_gapexplainabilityevergreen_potentialaudience_breadthcompetition_gapscore rationaler )rv reasoningvideo_structurelegacysectionsr3 zSection section_numberr; timestampduration_secondspurposer visuals hook_to_next)sectionr} r~ r r r r total_durationretention_strategykey_retention_momentsshare_triggers)r r r r structurer zPart durationr )r r} r NidverdictUNKNOWNsuggested_titletopicUntitledthemeunknown topic_countchannelone_lineverdict_reasoning
missing_angleangle confusion)rd re audience
the_momentmoment
click_triggerwhat_they_knowwhat_they_think_is_wrong)r r r
what_is_wrongexisting_contentwhy_existing_failshook opening_hookcsfcritical_success_factor)r
video_meta
dimensions)r r r. setr addintr r" r1 float enumerater? r5 )rY r= r[ topicstm raw_score score_100r primary_q secondary_qsqr ri r dimension_namesr r8 dimry r{ isr r
raw_structures r! transform_opportunityr sM ggi$G'4 Wg,%,cgg6F.KL )A!T"quuX AhK( ) + )A!T"quuX AhK( )
(!,I'0A~IN#3y>I ,b1I)S!'3
4
( $67Y8Iy;YI ==!6
kSU8VWL,%\hiWX:a3FJ/CPQFRi i ww("-H WWXr
"FJ! * * %(
O &++- @
Ujjb! c4 !, WW[" 5!Ju c5\
**-B ?Ju @ gg /4O 2 28 <"&&z26 "(+
1 55HQUU;KQqS5Q4R*ST UU; 3$%EE*B '9 5 Y!]9S YWYI I
c%j 3779i0 *CGGGZ,HI
#w'-2 G4=)
sww}a0 3779b) &x5H'IJ #7W8C;QWUW Xc+> ? E2 )R7C<#3
"8\:`hxQY>Z`^` %h @ FB &x1A B Hb %h0J K Qr
! , 09KR1P Q- . 2377;OQS3T U/ 0 n-31 2 x 67=23 4 9 G jB
,s ".W
BW "A W%
analysis_timec | rt j syt j t }|j } t j | j dd }|j d j d }|j d|f |j d }|j |S # | }Y >xY w)z.Count items collected since the last analysis.r Z+00:00N)tzinfoz%Y-%m-%dT%H:%M:%Sz7SELECT COUNT(*) FROM trend_items WHERE collected_at > ?) rJ rK rL rM rN r rQ r2 rR rO rP rS )r rU rN analysis_dt analysis_strcounts r! count_items_since_analysisr s 0 ??7 #D
[[]F% ,,]-B-B3-QR "**$*7@@ATU NNA
OO a EJJL L%$ s
AC Cc
ddl m} m} i } | j D ]W \ }} || }||j |j
|j |j |j |d |d d||< Y |S # t $ r i cY S w xY w)z.Load channel configurations for the dashboard.r )get_enabled_channelsget_channel_stats
item_countrD )r name
short_namer colorr[ r rD )
channel_configr r r1 r r r r get_all_sources ImportError)r r channelsch_idchrT s r! load_channels_configr s J-/557 IE2%e,E mm !~~ --/ #L 1!&!7 HUO s A:A= = B
B c g } t D ]1 }t d| dz }|j s!| j | 3 t dz }|j r| s| j | | S )a
Find all channel-specific opportunity files.
Returns files like:
- analysis_opportunities_how_things_work.json
- analysis_opportunities_brain_is_lying.json
But NOT timestamped backups like:
- analysis_opportunities_how_things_work_20260212_123456.json
analysis_opportunities_z.jsonzanalysis_opportunities.json) CHANNEL_IDS
OUTPUT_DIRrK r/ )files
channel_idpath generic_paths r! get_channel_opportunity_filesr 2 sm
E!
5j\ GG ;;= LL ==L U
\" Lr# channel_filterc $ t d | rt d| t }t d|d dd t }|rt dt | t }t dt | |D ] }t d |j
g }d
}d d d d d }i }d } |D ] }
t
|
d
5 }t j | } d
d
d
j d } | j d xs | j d }
|
r ||
|kD r|
}| j dg }| r| | k7 r| rt | || < |D ]k }| r|j d s| |d< t || }|j d s| r| |d< |j | | dz
} |d }||v s_||xx dz
cc< m
|j d d t | D ] \ }}t | |d< t dt | |j D ] \ }}t d | d| |r: t! j" |j% dd }|j' d |d < |r t) | nd |d!< t | d"g}d D ]4 }|| d kD s |j || d#|j% d$d# 6 d%j+ | |d&< t d'|d& d
}| r | |v r|| }||||d(}t
t, d) 5 }t j. ||d*+ d
d
d
t d,t, t d-t0 d. t d/t0 y
# 1 sw Y xY w# |r|d
d nd |d < Y xY w# 1 sw Y bxY w)0z?Generate dashboard data, merging all channel opportunity files.z Generating dashboard data...z Channel filter: z Database: rB ,z itemsz Channels configured: z Opportunity files found: z - Nr )
HIGH_PRIORITY WORTH_MAKING CONDITIONALSKIPrr r} analysis_timestamp
opportunitiesr r; r c & | j dd S )Nrv r )r )xs r! zmain.. s QUU7A%6 r# T)r reverser z Total opportunities: z: r r rG rD rH rI rE z opportunitiesr' r& z, rF z Summary: )rT r r current_channelw )indentz
Dashboard data written to: z
Open: file://z/dashboard.htmlz. Or: python3 -m http.server 8000 --directory )printrX r r4 r r openjsonloadr r r/ sortr r r1 r rQ r2 rR r r0 DASHBOARD_DATA_PATHdumpr )r rT r opp_filesfr r verdict_countschannel_opp_counts global_indexopp_filedatafile_channel_idfile_analysis_timeraw_oppsrY transformedvr r r rW r7 r dashboard_datas r! r r L s
()
">"234
E L}-a0
78 $%H
'H
78 ./I 'I'7
89
!
qvvh ! MM'(!AWXYNL %'
(C
A99Q
2 3E
E G ! q LLN1 -.a #s0C/DE FG yy'E) Ki()
*+ O.H4">2 & * N
!3 ' /1 .!A./
)*=)>
?@ M* _
56 ::,
GHm l X;H]3B%7iE/ "4/ /s$ =M%09M2 N%M/ 2NN__main__zGenerate dashboard data)r z --channelz Filter to a specific channel)typehelp)r )N)$__doc__argparser rL pathlibr r typingr r r __file__parentr rJ r r r" r r5 r? r rX r r r r r r __name__ArgumentParserparser add_argument
parse_argsargsr r# r! r s9 ' '
(^
"
"X
-
*
* #88 "$ * F; D D}t }C }D }@c c .d .tDz 4sI# sIl z
$X
$
$1J
KF
+
D % r#