WiU> d dl Zd dlZd dlmZ d dlZd dlm Z m
Z
dde e dej de e
dz dej dej de
ee
f dz d ed
ed e
d e d
e
ee f de de
de
dej" f dZy) N)ListUniondocshierarchical_topicstopics
embeddingsreduced_embeddingssamplehide_annotationshide_document_hover nr_levels level_scale
custom_labelstitlewidthheightreturnc | j }||dkD rd}g }t | D ] }t j t j | |k( d }t | dk r t | nt
t | |z }|j t j j ||d t j | }t j dt j | | i }|D cg c] }|| c}|d< |D cg c] }|| c}|d< |3|.|,| j |j j d
}n:|}n7||| }n/|-|+| j |j j d
}|/ dd lm} |d d
dd j# }|j$ }n|||| }n|||}dddf |d< |dddf |d< |j* j }|
dk( s|
dk( rt j, t j. t1 j2 dd t1 j2 t | dz
d | j5 t j7 }|j9 |D cg c] } || } } nX|
dk( s|
dk( rCt j: t= t | | D cg c]
}||d c}ddd } n t? d tA | D ]x \ }} |jB jE D ci c] }|| } }|jF |j* | k ddf } | jH j5 t | _$ | jK d } | jM D ]( }!|!d jN D ] }|!d jH | |< * | D "cg c] }"d }#}"tQ |# rUtA | jS D ]- \ } \ }$}%|%| jU v r|$|%k7 r | |% | |$< )d|#| < / tQ |# rU|jB jW | |d |dz < |d |dz j5 t |d |dz < { g }&i }'t= |jH j5 t jY D ] }||jH j5 t j[ k r| j] | sFt_ | t` r:| d d jc te tg | jh | | dd z }(nj| jj | r | jj || jl z }(n?| d d jc | j] | D )"cg c]
\ })}"|)dd c}"})dd z }(|(dd! |(dd! d"|'|< |&jo |( | d |jF |jH ta | k( d#f jp d z }(d jc |(js d dd D *cg c] }*|*dd c}* }+|(dd! |+dd! d"|'|< |&jo |( g },t= t | D ] }-g }.| jl r|.jo tu jv |jF |d |-dz dk( df |jF |d |-dz dk( df d$d%d&|s |jF |d |-dz dk( df nddty d'd(d)* + |rf|jF |jB j{ | ddf } t} | d |-dz jE D cg c]
}t
| c} }/n9t} |d |-dz jE D cg c]
}t
| c} }/|/D ] }|dk7 s
|r<|jF |d |-dz |k( |jB j{ | z ddf } n |jF |d |-dz |k( ddf } |sd| jF t | ddf< d,| d&< | j~ j | jF t | dz
df< | j j | jF t | dz
df< |'t
| d- | jF t | dz
d&f< |.jo tu jv | j~ | j |s | j nd|s | j ndd&|'t
| d. d$ty d(d)/ 0 |,jo |. |,D .cg c]
}.t |. }0}.d|0d fg}1tA |0dd D ]% \ }}2|1| d }3|2|3z }4|1jo |3|4f ' tu j }5|,D ] }.|.D ] }6|5j |6 t= t |5j D ] }||0d k\ s d|5j | _F ! g }7tA |1 D ]t \ }}ty d1ta | d2dgt |5j z ig3 }8t= |d |d z
D ] }d |8d4 d d2 ||d z < |7jo |8 v ty d5d6id7d i|78 g}9|j~ j[ t |j~ j[ d9z z
|j~ jY t |j~ jY d9z z f}:|j j[ t |j j[ d9z z
|j jY t |j jY d9z z f};|5j d:t |: d
z |;d t |: d
z |;d ty d'd
; < |5j d:|:d t |; d
z |:d t |; d
z ty d=d
; < |5j |:d t |; d
z d>dd ? |5j |;d t |: d
z d@dd A |5j |9dB| d)dCdDty dEdFG dH|
|I |5j dJ |5j dJ |5S c c}w c c}w # t& t( f$ r t) d w xY wc c} w c c}w c c}w c c}"w c c}"})w c c}*w c c}w c c}w c c}.w )Ka Visualize documents and their topics in 2D at different levels of hierarchy.
Arguments:
topic_model: A fitted BERTopic instance.
docs: The documents you used when calling either `fit` or `fit_transform`
hierarchical_topics: A dataframe that contains a hierarchy of topics
represented by their parents and their children
topics: A selection of topics to visualize.
Not to be confused with the topics that you get from `.fit_transform`.
For example, if you want to visualize only topics 1 through 5:
`topics = [1, 2, 3, 4, 5]`.
embeddings: The embeddings of all documents in `docs`.
reduced_embeddings: The 2D reduced embeddings of all documents in `docs`.
sample: The percentage of documents in each topic that you would like to keep.
Value can be between 0 and 1. Setting this value to, for example,
0.1 (10% of documents in each topic) makes it easier to visualize
millions of documents as a subset is chosen.
hide_annotations: Hide the names of the traces on top of each cluster.
hide_document_hover: Hide the content of the documents when hovering over
specific points. Helps to speed up generation of visualizations.
nr_levels: The number of levels to be visualized in the hierarchy. First, the distances
in `hierarchical_topics.Distance` are split in `nr_levels` lists of distances.
Then, for each list of distances, the merged topics are selected that have a
distance less or equal to the maximum distance of the selected list of distances.
NOTE: To get all possible merged steps, make sure that `nr_levels` is equal to
the length of `hierarchical_topics`.
level_scale: Whether to apply a linear or logarithmic (log) scale levels of the distance
vector. Linear scaling will perform an equal number of merges at each level
while logarithmic scaling will perform more mergers in earlier levels to
provide more resolution at higher levels (this can be used for when the number
of topics is large).
custom_labels: If bool, whether to use custom topic labels that were defined using
`topic_model.set_topic_labels`.
If `str`, it uses labels from other aspects, e.g., "Aspect1".
NOTE: Custom labels are only generated for the original
un-merged topics.
title: Title of the plot.
width: The width of the figure.
height: The height of the figure.
Examples:
To visualize the topics simply run:
```python
topic_model.visualize_hierarchical_documents(docs, hierarchical_topics)
```
Do note that this re-calculates the embeddings and reduces them to 2D.
The advised and preferred pipeline for using this function is as follows:
```python
from sklearn.datasets import fetch_20newsgroups
from sentence_transformers import SentenceTransformer
from bertopic import BERTopic
from umap import UMAP
# Prepare embeddings
docs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']
sentence_model = SentenceTransformer("all-MiniLM-L6-v2")
embeddings = sentence_model.encode(docs, show_progress_bar=False)
# Train BERTopic and extract hierarchical topics
topic_model = BERTopic().fit(docs, embeddings)
hierarchical_topics = topic_model.hierarchical_topics(docs)
# Reduce dimensionality of embeddings, this step is optional
# reduced_embeddings = UMAP(n_neighbors=10, n_components=2, min_dist=0.0, metric='cosine').fit_transform(embeddings)
# Run the visualization with the original embeddings
topic_model.visualize_hierarchical_documents(docs, hierarchical_topics, embeddings=embeddings)
# Or, if you have reduced the original embeddings already:
topic_model.visualize_hierarchical_documents(docs, hierarchical_topics, reduced_embeddings=reduced_embeddings)
```
Or if you want to save the resulting figure:
```python
fig = topic_model.visualize_hierarchical_documents(docs, hierarchical_topics, reduced_embeddings=reduced_embeddings)
fig.write_html("path/to/file.html")
```
Note:
This visualization was inspired by the scatter plot representation of Doc2Map:
https://github.com/louisgeisler/Doc2Map
N r d F)sizereplacetopicdocdocument)method)UMAP
g cosine) n_neighbors n_componentsmin_distmetricz{UMAP is required if the embeddings are not yet reduced in dimensionality. Please install it using `pip install umap-learn`.xylog logarithmic)startstopnumlinlinearz0level_scale needs to be one of 'log' or 'linear' Parent_IDTlevel__ ( )
trace_name plot_text Parent_Namez markers+textothertextz#CFD8DC g ?)colorr opacity)r% r& modename hoverinfo hovertext
showlegendmarker r6 r5 )r r< )r% r& r9 r@ r? r> r= rB updatevisible)r labelargsrG prefixzLevel: t) currentvaluepadstepsg333333?line)r; r )typex0y0x1y1rM z#9E9E9ED1)r% r& r9 showarrowyshiftD2)r& r% r9 rT xshift simple_whitecentertop Black)r r; )r9 r% xanchoryanchorfont)sliderstemplater r r )rE )Ntopics_setnpwherearraylenintextendrandomchoicepd DataFrame_extract_embeddingsr to_listumapr fit
embedding_ ImportErrorModuleNotFoundErrorDistanceroundlogspacemathr' astypetolistreverse array_splitrange
ValueError enumerater uniquelocr/ sort_valuesiterrowsTopicsanyitemskeysmapmaxmin get_topic
isinstancestrjoinnextziptopic_aspects_custom_labels_ _outliersappendvaluessplitgo Scattergldictisinsortedr% meanr& r9 Figure add_tracedatarE abs add_shapesumadd_annotation
update_layout update_xaxes update_yaxes)< topic_modelr r r r r r
r r r
r r r r r
topic_per_docindicesr sr dfindexembeddings_to_reducer
umap_model
embeddings_2d distances log_indicesi
max_distances max_distancemapping selectionrowr1 mappingskeyvalue trace_names topic_namesr5 wordr> r6
all_tracesleveltraces
unique_topicsnr_traces_per_set
trace_indices nr_tracesr) endfigtracerL stepr` x_rangey_ranges< u/home/sietch6/trending-topics-pipeline/venv/lib/python3.12/site-packages/bertopic/plotting/_hierarchical_documents.py visualize_hierarchical_documentsr st
T ''M ~!G]# F HHRXXm,5 6q 9 Q# s1v3s1v+?ryy''e'DEF hhw G w 7 @A BB*12e2BuI5<=E='=BwK ~ "4"<#.#B#B266>>CS\f#B#g #- !#-g#6
$6$>#.#B#B266>>CS\f#B#g ! ! "1sS[\``auvJ & 1 1M
2 >*73
. :*
AqD!BsGAqD!BsG $,,446Ie{m ; HH ((1b/ #i.1"4b9!
VC[
VX /:;!1 ;
; !824..sK^G_A`bk2l
'.Igbk "
B$
KLL ( 7 H|-/XX__->?E5%<??'++,?,H,HL,XZ[,[\ ' 1 1 8 8 = ))+6 %%' 2C Q
2!$Q!1!1
2 2
#**QD**(m#,W]]_#=
(GCL"'HQK
( (m $&88<<#8VEAI; #%uqyk&:#;#B#B3#GVEAI; -H2 KK*44;;C@DDFG + &0077<@@B B$$U+ mS1$)7!sxx S+"<"<]"KE"R STUWVWX0 "J !//;
!,!;!;EKDYDY"&Du%L Mnr $ iaE
f 5 9:I "IuWXykFZ<['?5'H!'K LI'7;IMM#i.!"34(*If%=F[[=M=M=OIMM#i.1"4c"9:=F[[=M=M=OIMM#i.1"4c"9:@KCPUJ@WXc@dIMM#i.1"4f"<=
LL#++#++3CY^^7J)--PT"( (U 4\ B + #C 8 6 &!i4"n 4>>V>>*1-./M %&7&; < +y e$Q'% eS\*+
))+C ! !E MM% !! s388}% , %a( (&+CHHUO #,
E#M2 we* ugCHH
567
71:
23 BE=ADLOI &uwqz'9 : B
T (I!6S"IUSTG
S"$$((*,--
S"$$((*,--G
S"$$((*,--
S"$$((*,--G MM
wu t) NNNNFTr r- Fz(Hierarchical Documents and Topicsi i )numpyrd pandasrl plotly.graph_objects
graph_objectsr rx typingr r r rm rh ndarrayfloatboolr r r r s ! $ !%)'+ " $ &+; n
s)n n
I n
n
n
%* $n n n n n s#n n n
n YY!nr