
    Wiw                       d dl Z d dlZ ej                  de        ej                  de       	 de j
                  d<   d dl	Z	d dl
Z
d dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZmZ d dlm Z  d dl!m"Z# d dl$m%Z% d dl&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/  e%d      d dl0m1Z1  e1       Z2nd dl3m2Z2 e.rd dl4m5Z6 d dl7m8Z9 	 d dl:m;Z; dZ<d dlAmBZB d dlCmDZE d dl?mFZF d dlGmHZH d dlImJZJ d dlKmLZLmMZM d dlNmOZO d dlPmQZQ d dlRmSZS d dlTmUZU d dlVmWZW d d lXmYZYmZZZ d d!l[m\Z\ d d"l]m^Z^m_Z_ d d#l0m`Z`maZambZbmcZcmdZdmeZemfZf d dlgmhZi  e`       Zjejj                  d$        G d% d&      Zl G d' d(      Zm	 	 	 	 d1d)e*ene+f   d*e*ene+f   d+e*enej                  f   d,e*ene+f   dz  d-e*ene+f   dz  d.e*epe+f   dz  d/eqfd0Zry# eeef$ r Y w xY w# e=e>f$ r dZ<d dl?m;Z@ Y w xY w)2    Nignore)categoryFYAMLLoadWarning)deepcopy)tqdm)Path)version)TemporaryDirectory)defaultdictCounter)
csr_matrix)	hierarchy)	find_spec)	ListTupleUnionMappingAnyCallableIterableTYPE_CHECKINGLiteralplotly)MockPlotlyModule)plotting)HDBSCANT)	normalize)__version__)AgglomerativeClustering)PCAcosine_similarity)CountVectorizerTfidfTransformer)BaseCluster)BaseEmbedder)mmr)select_backend)ClassTfidfTransformer)BaseRepresentationKeyBERTInspired)BaseDimensionalityReduction)hdbscan_delegatoris_supported_hdbscan)MyLoggercheck_documents_typecheck_embeddings_shapecheck_is_fittedvalidate_distance_matrixselect_topic_representationget_unique_distancesWARNINGc                   >   e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddededeeef   dedeeef   dz  d	ed
ede	e	e      dz  de	e   dz  de
dedededefdZed        Zed        Z	 	 	 dde	e   dej&                  de	e   dz  dee	e   ej&                  f   fdZ	 	 	 dde	e   dej&                  de	e   dz  dee	e   ej&                  f   dee	e   eej&                  df   f   f
dZ	 	 ddeee	e   f   dej&                  de	e   dz  dee	e   ej&                  f   fdZ	 	 dde	e   dej&                  dee	e   ej&                  f   fdZ	 	 	 	 	 dde	e   dee	e   e	e   f   d e	e   dz  d!edz  d"edz  d#ed$edej2                  fd%Z	 dde	e   d&ee	e   e	e   f   d$edej2                  fd'Z	 	 	 dde	e   d(ed)eegej&                  f   dz  d*eegef   dz  dej2                  f
d+Z	 	 	 	 	 	 	 	 ddeee	e   f   d,ed-ed.e
d/ed0ed1ed2ed3edeej&                  ee	ej&                     df   f   fd4Z	 dd5edz  d6edz  d7edee	e   e	e
   f   fd8Z 	 	 	 	 	 	 	 dde	e   de	e   dz  d e	e   dz  dedeeef   dz  dede!defd9Z"dd:ede#eeee
f   f   fd;Z$dd<ed:edee#eeee
f   f   ef   fd=Z%dd<edz  dej2                  fd>Z&dd<edz  deej2                  ef   fd?Z'	 	 dde	e   d@ej2                  dAe#ee(f   dz  dej2                  fdBZ)dd<edz  de	e   fdCZ*e+	 	 ddDej2                  dEe
dz  dFedefdG       Z,dHee	e   e#eef   f   ddfdIZ-	 	 	 	 	 ddJedKedLedz  d3edMedz  de	e   fdNZ.	 dde	e   dOe	ee/e   ef      de	e   dz  ddfdPZ0dQe	e   ddfdRZ1	 	 	 dde	e   deeef   de	e   dz  d(eddf
dSZ2ddTddUdi fde	e   d e	e   de	e   dz  dVedWej&                  dXe
dej&                  dYe#ee(f   de	e   fdZZ3	 	 	 	 	 	 	 dd e	e   dz  d[edz  d(ed\ed]ed^ed_edd`fdaZ4	 	 	 	 	 	 	 	 	 	 dde	e   d e	e   dz  dej&                  ddej&                  dee
dz  dfedged\ed]ed^ed_edd`fdhZ5ddddddiddbdcdddi i fde	e   dz  d e	e   dz  dej&                  ddej&                  d\eeef   d]edjeedf   d^ed_edkedledKedme6dne6ddofdpZ7	 	 	 	 	 	 	 	 	 	 	 	 dde	e   dqej2                  d e	e   dz  dej&                  ddej&                  deee
ef   dz  dfedgedredsed\ed]ed^ed_edd`fdtZ8	 	 	 	 	 	 dd e	e   dz  dued\ed]ed^ed_edd`fdvZ9	 	 	 	 	 	 	 ddwej2                  d[edz  d e	e   dz  dxed\ed]ed^ed_edd`fdyZ:	 	 	 	 	 	 	 ddzej2                  d[ed e	e   dz  dxed\ed]ed^ed_edd`fd{Z;	 	 	 	 	 ddWej&                  d|e
d\ed]ed^ed_edd`fd}Z<	 dd~edej&                  defdZ=	 	 	 	 	 	 	 	 	 	 	 	 dded e	e   dz  d[edz  d(ed\ed]ed^ed_edqej2                  d)eegej&                  f   dz  d*eegef   dz  dedd`fdZ>	 	 	 	 	 	 	 	 dd e	e   dz  d[edz  dedz  d(ed\ed]ed^ed_edd`fdZ?	 	 	 	 	 	 	 	 dd e	e   dz  d[eded\ed]ed^ed_ededd`fdZ@	 	 	 ddeAd   deeef   defdZBeCddefd       ZDeCdd.e
fd       ZE	 	 	 	 	 	 	 	 	 ddedededz  dedz  dededededeeef   defdZFddede#ee(f   fdZG	 	 	 ddee	e   ef   de	e   dz  dededz  dej&                  f
dZHdej2                  dej&                  dej2                  fdZIde	e   de	e   fdZJ	 	 ddeej&                  ef   dee	e   ej&                  f   dedej&                  fdZK	 	 ddej&                  dej2                  dedej&                  deej2                  ej&                  f   f
dZLdej2                  dej&                  deej2                  ej                  ej2                  ej                  f   fdZNd ZOdej2                  dej&                  dej2                  dej&                  deej2                  ej&                  f   f
dZPdej&                  dee	e   ej                  f   fdZQ	 	 	 	 ddej2                  dej&                  dedefdZRdej2                  fdZS	 	 	 ddedej2                  d e#ee	eee
f      f   dedede
dz  dee	e   e	e	e      f   fdZT	 	 	 ddej2                  dej&                  fdZU	 	 ddej2                  dededeee	e   f   fdZVdej2                  fdZW	 	 	 	 dde	e   dej2                  dedededej&                  de#ee	eee
f      f   fdZXddej2                  d(edej2                  fdZYddej2                  d(edej2                  fdZZddej2                  d(edej2                  fdZ[dej2                  dej2                  fdZ\	 ddWeej&                  df   dedeej&                  df   fdZ]dej&                  de	e   fdZ^e+dededej&                  fdĄ       Z_e+dedej&                  dej&                  fdƄ       Z`eCdǄ        ZadȄ Zby)BERTopica  BERTopic is a topic modeling technique that leverages BERT embeddings and
    c-TF-IDF to create dense clusters allowing for easily interpretable topics
    whilst keeping important words in the topic descriptions.

    The default embedding model is `all-MiniLM-L6-v2` when selecting `language="english"`
    and `paraphrase-multilingual-MiniLM-L12-v2` when selecting `language="multilingual"`.

    Attributes:
        topics_ (List[int]) : The topics that are generated for each document after training or updating
                              the topic model. The most recent topics are tracked.
        probabilities_ (List[float]): The probability of the assigned topic per document. These are
                                      only calculated if a HDBSCAN model is used for the clustering step.
                                      When `calculate_probabilities=True`, then it is the probabilities
                                      of all topics per document.
        topic_sizes_ (Mapping[int, int]) : The size of each topic.
        topic_mapper_ (TopicMapper) : A class for tracking topics and their mappings anytime they are
                                      merged, reduced, added, or removed.
        topic_representations_ (Mapping[int, Tuple[int, float]]) : The top n terms per topic and their respective
                                                                   c-TF-IDF values.
        c_tf_idf_ (csr_matrix) : The topic-term matrix as calculated through c-TF-IDF. To access its respective
                                 words, run `.vectorizer_model.get_feature_names()`  or
                                 `.vectorizer_model.get_feature_names_out()`
        topic_labels_ (Mapping[int, str]) : The default labels for each topic.
        custom_labels_ (List[str]) : Custom labels for each topic.
        topic_embeddings_ (np.ndarray) : The embeddings for each topic. They are calculated by taking the
                                         centroid embedding of each cluster.
        representative_docs_ (Mapping[int, str]) : The representative documents for each topic.

    Examples:
    ```python
    from bertopic import BERTopic
    from sklearn.datasets import fetch_20newsgroups

    docs = fetch_20newsgroups(subset='all')['data']
    topic_model = BERTopic()
    topics, probabilities = topic_model.fit_transform(docs)
    ```

    If you want to use your own embedding model, use it as follows:

    ```python
    from bertopic import BERTopic
    from sklearn.datasets import fetch_20newsgroups
    from sentence_transformers import SentenceTransformer

    docs = fetch_20newsgroups(subset='all')['data']
    sentence_model = SentenceTransformer("all-MiniLM-L6-v2")
    topic_model = BERTopic(embedding_model=sentence_model)
    ```

    Due to the stochastic nature of UMAP, the results from BERTopic might differ
    and the quality can degrade. Using your own embeddings allows you to
    try out BERTopic several times until you find the topics that suit
    you best.
    NFlanguagetop_n_wordsn_gram_rangemin_topic_size	nr_topics
low_memorycalculate_probabilitiesseed_topic_listzeroshot_topic_listzeroshot_min_similarityvectorizer_modelctfidf_modelrepresentation_modelverbosec                    |dkD  rt         j                  d       || _        || _        || _        || _        || _        || _        || _        |	| _	        |
| _
        |s|nd| _        || _        || _        |xs t        | j                        | _        |xs
 t!               | _        || _        ||| _        n#	 ddlm}  |ddd	d
| j
                        | _        ||| _        nCt4        rt7        | j                  ddd      | _        nt9        | j                  ddd      | _        d| _        d| _        d| _        d| _         d| _!        d| _"        i | _#        d| _$        d| _%        d| _&        i | _'        i | _(        d| _)        |rt         jU                  d       yt         jU                  d       y# t,        t.        f$ r t1        d      | _        Y w xY w)a?  BERTopic initialization.

        Arguments:
            language: The main language used in your documents. The default sentence-transformers
                      model for "english" is `all-MiniLM-L6-v2`. For a full overview of
                      supported languages see bertopic.backend.languages. Select
                      "multilingual" to load in the `paraphrase-multilingual-MiniLM-L12-v2`
                      sentence-transformers model that supports 50+ languages.
                      NOTE: This is not used if `embedding_model` is used.
            top_n_words: The number of words per topic to extract. Setting this
                         too high can negatively impact topic embeddings as topics
                         are typically best represented by at most 10 words.
            n_gram_range: The n-gram range for the CountVectorizer.
                          Advised to keep high values between 1 and 3.
                          More would likely lead to memory issues.
                          NOTE: This param will not be used if you pass in your own
                          CountVectorizer.
            min_topic_size: The minimum size of the topic. Increasing this value will lead
                            to a lower number of clusters/topics and vice versa.
                            It is the same parameter as `min_cluster_size` in HDBSCAN.
                            NOTE: This param will not be used if you are using `hdbscan_model`.
            nr_topics: Specifying the number of topics will reduce the initial
                       number of topics to the value specified. This reduction can take
                       a while as each reduction in topics (-1) activates a c-TF-IDF
                       calculation. If this is set to None, no reduction is applied. Use
                       "auto" to automatically reduce topics using HDBSCAN.
                       NOTE: Controlling the number of topics is best done by adjusting
                       `min_topic_size` first before adjusting this parameter.
            low_memory: Sets UMAP low memory to True to make sure less memory is used.
                        NOTE: This is only used in UMAP. For example, if you use PCA instead of UMAP
                        this parameter will not be used.
            calculate_probabilities: Calculate the probabilities of all topics
                                     per document instead of the probability of the assigned
                                     topic per document. This could slow down the extraction
                                     of topics if you have many documents (> 100_000).
                                     NOTE: If false you cannot use the corresponding
                                     visualization method `visualize_probabilities`.
                                     NOTE: This is an approximation of topic probabilities
                                     as used in HDBSCAN and not an exact representation.
            seed_topic_list: A list of seed words per topic to converge around
            zeroshot_topic_list: A list of topic names to use for zero-shot classification
            zeroshot_min_similarity: The minimum similarity between a zero-shot topic and
                                     a document for assignment. The higher this value, the more
                                     confident the model needs to be to assign a zero-shot topic to a document.
            verbose: Changes the verbosity of the model, Set to True if you want
                     to track the stages of the model.
            embedding_model: Use a custom embedding model.
                             The following backends are currently supported
                               * SentenceTransformers
                               * Flair
                               * Spacy
                               * Gensim
                               * USE (TF-Hub)
                             You can also pass in a string that points to one of the following
                             sentence-transformers models:
                               * https://www.sbert.net/docs/pretrained_models.html
            umap_model: Pass in a UMAP model to be used instead of the default.
                        NOTE: You can also pass in any dimensionality reduction algorithm as long
                        as it has `.fit` and `.transform` functions.
            hdbscan_model: Pass in a hdbscan.HDBSCAN model to be used instead of the default
                           NOTE: You can also pass in any clustering algorithm as long as it has
                           `.fit` and `.predict` functions along with the `.labels_` variable.
            vectorizer_model: Pass in a custom `CountVectorizer` instead of the default model.
            ctfidf_model: Pass in a custom ClassTfidfTransformer instead of the default model.
            representation_model: Pass in a model that fine-tunes the topic representations
                                  calculated through c-TF-IDF. Models from `bertopic.representation`
                                  are supported.
        d   ]Note that extracting more than 100 words from a sparse can slow down computation quite a bit.Nngram_ranger   )UMAP      g        cosine)n_neighborsn_componentsmin_distmetricr>   )rQ   	euclideaneomTmin_cluster_sizerS   cluster_selection_methodprediction_datarW   rS   rX   n_jobsDEBUGr6   )+loggerwarningr:   r<   r=   r>   r?   rF   r@   rA   rB   r9   embedding_modelr;   r#   rC   r)   rD   rE   
umap_modelumaprL   ImportErrorModuleNotFoundErrorr    hdbscan_modelHAS_HDBSCANr   
SK_HDBSCANtopics_probabilities_topic_sizes_topic_mapper_topic_representations_topic_embeddings__topic_id_to_zeroshot_topic_idxcustom_labels_	c_tf_idf_representative_images_representative_docs_topic_aspects__merged_topics	set_level)selfr9   r:   r;   r<   r=   r>   r?   r@   rA   rB   r`   ra   re   rC   rD   rE   rF   rL   s                      ^/home/sietch6/trending-topics-pipeline/venv/lib/python3.12/site-packages/bertopic/_bertopic.py__init__zBERTopic.__init__   s   r NNo ',"$'>$.#6 '>$ )8T. ) 0 bOPTPaPa4b(C,A,C %9! !(DO6%"& "!" ### $!.D!(!%!4!4"). $	"D ",!%!4!4[chqs"D
 " !&*#!%/1,"&*#$&!  #W%Y'Q  !45 6"%1"56s   7"F/ / GGc                 &    d| j                   v rdS dS )aE  Some algorithms have outlier labels (-1) that can be tricky to work
        with if you are slicing data based on that labels. Therefore, we
        track if there are outlier labels and act accordingly when slicing.

        Returns:
            An integer indicating whether outliers are present in the topic model
        rZ      r   )rj   rv   s    rw   	_outlierszBERTopic._outliers=  s     $+++q22    c                    | j                   j                         D ci c]1  \  }}|| ddj                  |dd D cg c]  }|d   	 c}      z   3 }}}}| j                         rK| j                  j                         D ci c]  \  }}|| j
                  |    }}}|j                  |       |S c c}w c c}}}w c c}}w )ab  Map topic IDs to their labels.
        A label is the topic ID, along with the first four words of the topic representation, joined using '_'.
        Zeroshot topic labels come from self.zeroshot_topic_list rather than the calculated representation.

        Returns:
            topic_labels: a dict mapping a topic ID (int) to its label (str)
        _N   r   )rl   itemsjoin_is_zeroshotrn   rA   update)rv   keyvalueswordtopic_labelstopic_idzeroshot_topic_idxtopic_id_to_zeroshot_labels           rw   topic_labels_zBERTopic.topic_labels_H  s      $::@@B
 
V C5SXX6"1:&F4tAw&FGGG
 
  594X4X4^4^4`*0H0 $223EFF*& *  :; 'G
*s   B;B6
B;C6B;	documents
embeddingsimagesyc                 0    | j                  ||||       | S )a"  Fit the models on a collection of documents and generate topics.

        Arguments:
            documents: A list of documents to fit on
            embeddings: Pre-trained document embeddings. These can be used
                        instead of the sentence-transformer model
            images: A list of paths to the images to fit on or the images themselves
            y: The target class for (semi)-supervised modeling. Use -1 if no class for a
               specific instance is specified.

        Examples:
        ```python
        from bertopic import BERTopic
        from sklearn.datasets import fetch_20newsgroups

        docs = fetch_20newsgroups(subset='all')['data']
        topic_model = BERTopic().fit(docs)
        ```

        If you want to use your own embeddings, use it as follows:

        ```python
        from bertopic import BERTopic
        from sklearn.datasets import fetch_20newsgroups
        from sentence_transformers import SentenceTransformer

        # Create embeddings
        docs = fetch_20newsgroups(subset='all')['data']
        sentence_model = SentenceTransformer("all-MiniLM-L6-v2")
        embeddings = sentence_model.encode(docs, show_progress_bar=True)

        # Create topic model
        topic_model = BERTopic().fit(docs, embeddings)
        ```
        )r   r   r   r   )fit_transform)rv   r   r   r   r   s        rw   fitzBERTopic.fit^  s"    T 	Y:SYZr}   returnc                    |t        |       t        ||       |t        t        |            nt        t        |            }t	        j
                  ||d|d      }|t        j                  d       t        | j                  | j                  | j                        | _	        | j                  |j                  j                  j                         |d| j                        }t        j                  d       n=| j                  1t        | j                  | j                  | j                        | _	        | j                    | j                  | j#                  |      \  }}| j%                  ||      }| j'                         r@| j)                  ||      \  }}}}t        |      dkD  r| j*                  j-                  |      }t        |      dkD  rM| j/                  |||	      \  }}	| j'                         r*t              dkD  r| j1                  |||      \  }}n}}| j2                  s| j5                  |      }|j                  j                  d   s| j7                  ||      }
| j9                  |
|| j2                   
       | j2                  r| j;                  |
      }
| j=                  ||       | j?                  |
       nX| j9                  ||| j                  | j2                          | j2                  r| j;                  |      }| j?                  |       | j'                         rwt              dkD  ritA               | _!        tE        |tG        jH                  | jJ                              }| jL                  r|| _'        n5tG        jP                  |d      | _'        n| jS                  	d      | _'        |jT                  jW                         }|| jN                  fS )a  Fit the models on a collection of documents, generate topics,
        and return the probabilities and topic per document.

        Arguments:
            documents: A list of documents to fit on
            embeddings: Pre-trained document embeddings. These can be used
                        instead of the sentence-transformer model
            images: A list of paths to the images to fit on or the images themselves
            y: The target class for (semi)-supervised modeling. Use -1 if no class for a
               specific instance is specified.

        Returns:
            predictions: Topic predictions for each documents
            probabilities: The probability of the assigned topic per document.
                           If `calculate_probabilities` in BERTopic is set to True, then
                           it calculates the probabilities of all topics across all documents
                           instead of only the assigned topic. This, however, slows down
                           computation and may increase memory usage.

        Examples:
        ```python
        from bertopic import BERTopic
        from sklearn.datasets import fetch_20newsgroups

        docs = fetch_20newsgroups(subset='all')['data']
        topic_model = BERTopic()
        topics, probs = topic_model.fit_transform(docs)
        ```

        If you want to use your own embeddings, use it as follows:

        ```python
        from bertopic import BERTopic
        from sklearn.datasets import fetch_20newsgroups
        from sentence_transformers import SentenceTransformer

        # Create embeddings
        docs = fetch_20newsgroups(subset='all')['data']
        sentence_model = SentenceTransformer("all-MiniLM-L6-v2")
        embeddings = sentence_model.encode(docs, show_progress_bar=True)

        # Create topic model
        topic_model = BERTopic()
        topics, probs = topic_model.fit_transform(docs, embeddings)
        ```
        N)DocumentIDTopicImagez1Embedding - Transforming documents to embeddings.r9   rF   documentr   methodrF   u   Embedding - Completed ✓r   r   )r   fine_tune_representation)r   r   )r   rF   r   rz   axisToriginal_topics),r0   r1   rangelenpd	DataFramer^   infor(   r`   r9   rF   _extract_embeddingsr   r   tolistr@   _guided_topic_modeling_reduce_dimensionalityr   _zeroshot_topic_modelingra   	transform_cluster_embeddings_combine_zeroshot_topicsr=   _sort_mappings_by_frequency_images_to_text_extract_topics_reduce_topics_create_topic_vectors_save_representative_docsr%   re   r"   nparrayrm   r?   ri   max_map_probabilitiesr   to_list)rv   r   r   r   r   doc_idsumap_embeddingsassigned_documentsassigned_embeddingsprobabilitiescustom_documents
sim_matrixpredictionss                rw   r   zBERTopic.fit_transform  s   j   +":y9+4+@%I'eCPVKFXLLiwQU`f!gh	 KKKL#1$2F2FQUQ^Q^hlhtht#uD 11""))002!	 2 J KK67##/'5((4==$,,($
 +0D0D0P 77
CMAz 55j!D MQMjMj:NJIz#57J
 9~!"&//";";J"Gy>A'+'?'?QZ^_'?'`$I}  "s+='>'B(,(E(Ez+=?R)%	:
 +I,J ~~88CI $$Q'/#33IzJ   !1jgkgugucu v~~#'#6#67G#H &&z&R **+;<   j$,,eiesesas !  ~~ //	:	 **95 3'9#:Q#>!,D*:rxx@V@V7WXJ++&0#&(ffZa&@#"&"9"9-Y]"9"^Doo--/D////r}   c                 l   t        |        t        ||       t        |t              s||g}|| j	                  ||d| j
                        }|t        d      t        | j                        t        u rt        j                  d       t        |t        j                  | j                              }t        j                   |d      | j"                  z
  }| j$                  r|}||fS t        j&                  |d      }||fS t        j                  d       | j(                  j+                  |      }t        j                  d	       t        j                  d
       t-        | j                        rht/        | j                  d|      \  }}| j$                  r_t        j                  d       t/        | j                  d|      }t        j                  d       n| j                  j1                  |      }d}t        j                  d       | j3                  |d      }| j5                  |      }||fS )a  After having fit a model, use transform to predict new instances.

        Arguments:
            documents: A single document or a list of documents to predict on
            embeddings: Pre-trained document embeddings. These can be used
                        instead of the sentence-transformer model.
            images: A list of paths to the images to predict on or the images themselves

        Returns:
            predictions: Topic predictions for each documents
            probabilities: The topic probability distribution which is returned by default.
                           If `calculate_probabilities` in BERTopic is set to False, then the
                           probabilities are not calculated to speed up computation and
                           decrease memory usage.

        Examples:
        ```python
        from bertopic import BERTopic
        from sklearn.datasets import fetch_20newsgroups

        docs = fetch_20newsgroups(subset='all')['data']
        topic_model = BERTopic().fit(docs)
        topics, probs = topic_model.transform(docs)
        ```

        If you want to use your own embeddings:

        ```python
        from bertopic import BERTopic
        from sklearn.datasets import fetch_20newsgroups
        from sentence_transformers import SentenceTransformer

        # Create embeddings
        docs = fetch_20newsgroups(subset='all')['data']
        sentence_model = SentenceTransformer("all-MiniLM-L6-v2")
        embeddings = sentence_model.encode(docs, show_progress_bar=True)

        # Create topic model
        topic_model = BERTopic().fit(docs, embeddings)
        topics, probs = topic_model.transform(docs, embeddings)
        ```
        Nr   r   zNo embedding model was found to embed the documents.Make sure when loading in the model using BERTopic.load()to also specify the embedding model.zXPredicting topic assignments through cosine similarity of topic and document embeddings.rz   r   z=Dimensionality - Reducing dimensionality of input embeddings.   Dimensionality - Completed ✓z:Clustering - Approximating new points with `hdbscan_model`approximate_predictz?Probabilities - Start calculation of probabilities with HDBSCANmembership_vectoru   Probabilities - Completed ✓   Cluster - Completed ✓Tr   )r2   r1   
isinstancestrr   rF   
ValueErrortypere   r%   r^   r   r"   r   r   rm   argmaxr|   r?   r   ra   r   r.   r-   predictr   _map_predictions)rv   r   r   r   r   r   r   r   s           rw   r   zBERTopic.transform!  s   ` 	z95i%):"I11)FS]gkgsgs1tJ 7  $$$%4KKrs*:rxx@V@V7WXJ))JQ7$..HK++ *< M))9 !#z :8 M))1 KKWX"oo77
COKK;< KKTU#D$6$67->&&(=.*]
 //KK ab$5d6H6HJ]_n$oMKK BC"0088I $KK45 !33MSW3XM//<KM))r}   c           	      T
   t        ||       t        | j                  d      st        d      t	        |t
              r|g}t        j                  |t        t        |            dd      }|~| j                  1t        | j                  | j                  | j                        | _        | j                  |j                   j"                  j%                         d| j                        }nI| j                  =| j                  1t        | j                  | j                  | j                        | _        | j&                   | j                  | j)                  |      \  }}| j+                  ||d	      }| j-                  ||d	      \  }| _        |j0                  j3                         }| j4                  st7        |      | _        | j4                  j9                         }t;        |      j=                  t;        |j?                                     }tA        |      D 	ci c]%  \  }}	|	tC        |j#                               |z   d
z   ' }
}}	| j4                  jE                  |
       | j4                  j9                         }|D 	cg c]  }	||	   	 }}	||d<   | j                  rkt;        | j                  j?                               j=                  t;        |            }|D ]*  }dt        |      |g|jF                  t        |      ddf<   , ni }|jI                  d      jK                  dgd      }|jM                         j0                  jO                  tP              }|jS                  ddjT                  i      }| jW                  |d	      \  | _,        }| j[                  ||| jX                  d      | _
        | j]                          t        |      dkD  r|j^                  dt        |        }| j`                  | jc                  |       | S |jK                  dgd      je                         }|jg                         D ]  \  }}tQ        |j0                        }	| j`                  ji                  |	      /|	|vr+| j`                  |	xx   tQ        |j                         z  cc<   e| j`                  ji                  |	      tQ        |j                         | j`                  |	<    |j0                  jO                  tP              j%                         | _5        | S c c}	}w c c}	w )a%
  Fit BERTopic on a subset of the data and perform online learning
        with batch-like data.

        Online topic modeling in BERTopic is performed by using dimensionality
        reduction and cluster algorithms that support a `partial_fit` method
        in order to incrementally train the topic model.

        Likewise, the `bertopic.vectorizers.OnlineCountVectorizer` is used
        to dynamically update its vocabulary when presented with new data.
        It has several parameters for modeling decay and updating the
        representations.

        In other words, although the main algorithm stays the same, the training
        procedure now works as follows:

        For each subset of the data:

        1. Generate embeddings with a pre-trained language model
        2. Incrementally update the dimensionality reduction algorithm with `partial_fit`
        3. Incrementally update the cluster algorithm with `partial_fit`
        4. Incrementally update the OnlineCountVectorizer and apply some form of decay

        Note that it is advised to use `partial_fit` with batches and
        not single documents for the best performance.

        Arguments:
            documents: A list of documents to fit on
            embeddings: Pre-trained document embeddings. These can be used
                        instead of the sentence-transformer model
            y: The target class for (semi)-supervised modeling. Use -1 if no class for a
               specific instance is specified.

        Examples:
        ```python
        from sklearn.datasets import fetch_20newsgroups
        from sklearn.cluster import MiniBatchKMeans
        from sklearn.decomposition import IncrementalPCA
        from bertopic.vectorizers import OnlineCountVectorizer
        from bertopic import BERTopic

        # Prepare documents
        docs = fetch_20newsgroups(subset=subset,  remove=('headers', 'footers', 'quotes'))["data"]

        # Prepare sub-models that support online learning
        umap_model = IncrementalPCA(n_components=5)
        cluster_model = MiniBatchKMeans(n_clusters=50, random_state=0)
        vectorizer_model = OnlineCountVectorizer(stop_words="english", decay=.01)

        topic_model = BERTopic(umap_model=umap_model,
                               hdbscan_model=cluster_model,
                               vectorizer_model=vectorizer_model)

        # Incrementally fit the topic model by training on 1000 documents at a time
        for index in range(0, len(docs), 1000):
            topic_model.partial_fit(docs[index: index+1000])
        ```
        partial_fitzXIn order to use `.partial_fit`, the cluster model should have a `.partial_fit` function.Nr   r   r   r   r   r   rF   T)r   rz   r    Fas_indexr   calculate_aspectsr   )6r1   hasattrre   r   r   r   r   r   r   r   rl   r(   r`   r9   rF   r   r   r   r   r@   r   r   r   ri   r   r   rk   TopicMapperget_mappingsset
differencekeys	enumerater   add_new_topicslocsort_valuesgroupbyfirstastypeintaggr   	_c_tf_idfrp   _extract_words_per_topicr   ilocrj   _update_topic_sizecountiterrowsgetrh   )rv   r   r   r   r   topicsmappings
new_topicsindextopicnew_topic_idsupdated_mappingsupdated_topicsmissing_topicsmissing_topicdocuments_per_topicupdated_wordssizesr   rows                       rw   r   zBERTopic.partial_fit  s   @ 	z95t))=9wxx i%"ILLiuS^?T_c!de	 **2'5((4==$,,($ 11""))002! 2 J ##/D4O4O4W'5((4==$,,($
 +0D0D0P 77
CMAz55j!QU5V *.)A)A/S\jn)A)o&	4&((* !!!,V!4D%%224[++C,@A
W`akWlm|ueHOO$5 6 > BBmm))-8--::<?EFe*51FF+	' && !<!<!A!A!CDOOPSTbPcdN!/ X47Y3W	c)na/0X  N (33G<DDgYY^D_,224::AA#F155z3886LM )-7JX\(]%&*&C&C9dnn 'D '
# 	""$ ~"!'=#n*=)=>I$##I.  %%wi%%@FFHE..* A3CII$$((/;^@[%%e,CLL0AA,&&**519/23<</@D%%e,A %??11#6==?DLU n Gs    *TT%docs
timestampsr   nr_binsdatetime_formatevolution_tuningglobal_tuningc                    t        |        t        |       |r|n| j                  }t        j                  |||d      }	t        | j                  ddd      }
t        t        |	j                  j                                     }t        |      D ci c]  \  }}||
 }}}t        |d   t              r$|sdnd}t        j                  |	d   ||	      |	d<   |r:t        j                  |	j                   |
      |	d<   |	j#                  d d      |	d<   |	j%                  d      }	|	j                   j                         }t'        |      dkD  r"t(        j+                  dt'        |       d       g }t-        t        |      | j.                         D ]r  \  }}|	j0                  |	j                   |k(  ddf   }|j3                  dgd      j5                  dj6                  dd      }| j9                  |d      \  }}|s|rt        |ddd      }|r|dk7  rt        t        |j                  j:                              }t        t        t=              j?                  t=        |                        }|D cg c]  }|jA                  |       }}|D cg c]  }|jA                  |       }}||   |   z   dz  jC                         |jC                         |<   |r1|j                  j:                  D cg c]  }||   	 }}|
|   |z   dz  }| jE                  |||d      }t        jF                  |j                   j:                  |j                        jI                         }|jK                         D cg c]1  \  }}|dj7                  |D cg c]  }|d   	 c}dd       ||   |f3 }}}}|jM                  |       |s;t        t        |j                  j:                              }|jO                         }u t        j                  |g d      S c c}}w c c}w c c}w c c}w c c}w c c}}}w )a  Create topics over time.

        To create the topics over time, BERTopic needs to be already fitted once.
        From the fitted models, the c-TF-IDF representations are calculate at
        each timestamp t. Then, the c-TF-IDF representations at timestamp t are
        averaged with the global c-TF-IDF representations in order to fine-tune the
        local representations.

        Note:
            Make sure to use a limited number of unique timestamps (<100) as the
            c-TF-IDF representation will be calculated at each single unique timestamp.
            Having a large number of unique timestamps can take some time to be calculated.
            Moreover, there aren't many use-cases where you would like to see the difference
            in topic representations over more than 100 different timestamps.

        Arguments:
            docs: The documents you used when calling either `fit` or `fit_transform`
            timestamps: The timestamp of each document. This can be either a list of strings or ints.
                        If it is a list of strings, then the datetime format will be automatically
                        inferred. If it is a list of ints, then the documents will be ordered in
                        ascending order.
            topics: A list of topics where each topic is related to a document in `docs` and
                    a timestamp in `timestamps`. You can use this to apply topics_over_time on
                    a subset of the data. Make sure that `docs`, `timestamps`, and `topics`
                    all correspond to one another and have the same size.
            nr_bins: The number of bins you want to create for the timestamps. The left interval will
                     be chosen as the timestamp. An additional column will be created with the
                     entire interval.
            datetime_format: The datetime format of the timestamps if they are strings, eg "%d/%m/%Y".
                             Set this to None if you want to have it automatically detect the format.
                             See strftime documentation for more information on choices:
                             https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior.
            evolution_tuning: Fine-tune each topic representation at timestamp *t* by averaging its
                              c-TF-IDF matrix with the c-TF-IDF matrix at timestamp *t-1*. This creates
                              evolutionary topic representations.
            global_tuning: Fine-tune each topic representation at timestamp *t* by averaging its c-TF-IDF matrix
                       with the global c-TF-IDF matrix. Turn this off if you want to prevent words in
                       topic representations that could not be found in the documents at timestamp *t*.

        Returns:
            topics_over_time: A dataframe that contains the topic, words, and frequency of topic
                              at timestamp *t*.

        Examples:
        The timestamps variable represents the timestamp of each document. If you have over
        100 unique timestamps, it is advised to bin the timestamps as shown below:

        ```python
        from bertopic import BERTopic
        topic_model = BERTopic()
        topics, probs = topic_model.fit_transform(docs)
        topics_over_time = topic_model.topics_over_time(docs, timestamps, nr_bins=20)
        ```
        )r   r   
Timestampsrz   l1Fr   normcopyr   Tr  )infer_datetime_formatformat)binsBinsc                 .    | j                   j                  S N)r  left)r   s    rw   <lambda>z+BERTopic.topics_over_time.<locals>.<lambda>p  s    #((-- r}   rH   z1There are more than 100 unique timestamps (i.e., z) which significantly slows down the application. Consider setting `nr_bins` to a value lower than 100 to speed up calculation. disableNr   r   r   r   )r   r  r          @r   r   , rN   )r   Words	Frequency	Timestampcolumns)(r2   r0   rh   r   r   r   rp   sortedlistr   uniquer   r   r   to_datetimecutr  applyr   r   r^   r_   r   rF   r   r   r   r   r   r   r   intersectionr   tolilr   Seriesto_dictr   extendr  ) rv   r   r   r   r   r   r  r  selected_topicsr   global_c_tf_idf
all_topicsr   r   all_topics_indicesr	  topics_over_time	timestamp	selectionr   c_tf_idfwordscurrent_topicsprevious_topicsoverlapping_topicscurrent_overlap_idxprevious_overlap_idxprevious_c_tf_idfwords_per_topictopic_frequencyr   topics_at_timestamps                                    rw   r+  zBERTopic.topics_over_time  s8   @ 	T"$*&LLd_\f!gh	#DNNERD!7!7!9:;
?H?TU|ueeUlUUjmS)0?DU!&(nn,'&;&'Il#  "y';';' JIf&/oo6OQR&SIl# )),7	))002
z?S NNCC
OCT UF F  $Yz%:DT U :	4E9!i&:&:i&G&JKI"+"3"3WI"3"N"R"R XXW=# #nn-@enLOHe 0$XADuM  EQJ!'-@-F-F-M-M(N!O%+_-::3~;NOP&" Qc&cu~';';E'B&c#&c "4( $))%0($ ( !!458IJ^8__ %'  !45 J]JcJcJjJj"k#5e#<"k"k+O<xG3N #;;E9hjo;pO ii#..55=P=V=Vgi  &5%:%:%<# # "E6 IIV<EuQx<Ra@A#E*	# # ##$78"(.A.G.G.N.N)O"P$,MMO!u:	4x ||,6bccm VZ 'd( #l =#s0   P#<P)P.;P3 P=
P8$P=
8P=
classesc                    t        |       t        j                  || j                  |d      }t	        | j
                  ddd      }g }t        t        t        |            | j                         D ]`  \  }}|j                  |j                  |k(  ddf   }	|	j                  dgd	      j                  d
j                  dd      }
| j                  |
d      \  }}|r;t	        |ddd      }||
j                   j"                  | j$                  z      |z   dz  }| j'                  ||	|d      }t        j(                  |
j                  j"                  |
j                         j+                         }|j-                         D cg c]1  \  }}|dj                  |D cg c]  }|d   	 c}dd       ||   |f3 }}}}|j/                  |       c t        j                  |g d      }|S c c}w c c}}}w )al  Create topics per class.

        To create the topics per class, BERTopic needs to be already fitted once.
        From the fitted models, the c-TF-IDF representations are calculated at
        each class c. Then, the c-TF-IDF representations at class c are
        averaged with the global c-TF-IDF representations in order to fine-tune the
        local representations. This can be turned off if the pure representation is
        needed.

        Note:
            Make sure to use a limited number of unique classes (<100) as the
            c-TF-IDF representation will be calculated at each single unique class.
            Having a large number of unique classes can take some time to be calculated.

        Arguments:
            docs: The documents you used when calling either `fit` or `fit_transform`
            classes: The class of each document. This can be either a list of strings or ints.
            global_tuning: Fine-tune each topic representation for class c by averaging its c-TF-IDF matrix
                           with the global c-TF-IDF matrix. Turn this off if you want to prevent words in
                           topic representations that could not be found in the documents for class c.

        Returns:
            topics_per_class: A dataframe that contains the topic, words, and frequency of topics
                              for each class.

        Examples:
        ```python
        from bertopic import BERTopic
        topic_model = BERTopic()
        topics, probs = topic_model.fit_transform(docs)
        topics_per_class = topic_model.topics_per_class(docs, classes)
        ```
        )r   r   Classrz   r  Fr  r  Nr   r   r   r   )r   r;  r  r  r   r  r  r   rN   )r   r  r  r;  r  )r0   r   r   rh   r   rp   r   r   r   rF   r   r;  r   r   r   r   r   r   r|   r   r$  r%  r   r&  )rv   r   r9  r  r   r(  topics_per_classr   class_r-  r   r.  r/  r6  r7  r   r   topics_at_classs                     rw   r<  zBERTopic.topics_per_class  s   N 	T"LLdT\\T[!\]	#DNNER iG54<<?OP 	5IAv!ioo&?&BCI"+"3"3WI"3"N"R"R XX8# #nn-@enLOHe $XADuM+,?,E,E,L,Lt~~,]^aiimpp #;;E9hjo;pO ii(;(A(A(H(HPcPiPijrrtO &5%:%:%<  "E6 IIV<EuQx<Ra@A#E*	O  ##O49	5< <<(8Bjk =s   
G8
"G3.G8
3G8

use_ctfidflinkage_functiondistance_functionc           
      
   t        |       |d }|d }t        | j                  | j                  |      d   | j                  d } ||      }t        ||j                  d         } ||      }t        |dddf         t        t        j                  |dddf               k7  rt        |dddf         |dddf<   t        j                  |t        t        |            | j                  d      }|j                  dgd	      j!                  d
dj"                  i      }	|	j$                  |	j&                  dk7  ddf   }	| j)                  |	j*                  j,                        }
t/        j0                  t2              t/        j0                  d      k\  r| j4                  j7                         }n| j4                  j9                         }| j4                  j;                  |
      }t        j                  g d      }t=        t        t        |                  D ]  }t?        j@                  |||   d   d      | j                  z
  }t        |      }d}||   d   }|=|t        |      z
  dk  rtC        |      }n|tC        |t        |      z
           d   }|=tE        |      D cg c]  \  }}|||   k(  s| }}}tG        ||   jI                  d            }| jJ                  j;                  |      }|j$                  |j&                  jM                  |      ddf   }d|_        | jO                  |||d      }|t        |      z   }dj#                  |d   D cg c]  }|d   	 c}dd       }||   d   }||z
  dk  r|n||z
  }||z
  dk  r5dj#                  | jQ                  |      D cg c]  }|d   	 c}dd       }n"|jR                  tC        |         jT                  }||   d   }||z
  dk  r|n||z
  }||z
  dk  r5dj#                  | jQ                  |      D cg c]  }|d   	 c}dd       } n"|jR                  tC        |         jT                  } |||tC        ||   d         |tC        ||   d         | g|j$                  t        |      ddf<    |dddf   |d<   |jW                  dd      }|g d   jY                  tZ              |g d<   |S c c}}w c c}w c c}w c c}w )a	  Create a hierarchy of topics.

        To create this hierarchy, BERTopic needs to be already fitted once.
        Then, a hierarchy is calculated on the distance matrix of the c-TF-IDF or topic embeddings
        representation using `scipy.cluster.hierarchy.linkage`.

        Based on that hierarchy, we calculate the topic representation at each
        merged step. This is a local representation, as we only assume that the
        chosen step is merged and not all others which typically improves the
        topic representation.

        Arguments:
            docs: The documents you used when calling either `fit` or `fit_transform`
            use_ctfidf: Whether to calculate distances between topics based on c-TF-IDF embeddings. If False, the
                        embeddings from the embedding model are used.
            linkage_function: The linkage function to use. Default is:
                              `lambda x: sch.linkage(x, 'ward', optimal_ordering=True)`
            distance_function: The distance function to use on the c-TF-IDF matrix. Default is:
                               `lambda x: 1 - cosine_similarity(x)`.
                               You can pass any function that returns either a square matrix of
                               shape (n_samples, n_samples) with zeros on the diagonal and
                               non-negative values or condensed distance matrix of shape
                               (n_samples * (n_samples - 1) / 2,) containing the upper
                               triangular of the distance matrix.

        Returns:
            hierarchical_topics: A dataframe that contains a hierarchy of topics
                                 represented by their parents and their children

        Examples:
        ```python
        from bertopic import BERTopic
        topic_model = BERTopic()
        topics, probs = topic_model.fit_transform(docs)
        hierarchical_topics = topic_model.hierarchical_topics(docs)
        ```

        A custom linkage function can be used as follows:

        ```python
        from scipy.cluster import hierarchy as sch
        from bertopic import BERTopic
        topic_model = BERTopic()
        topics, probs = topic_model.fit_transform(docs)

        # Hierarchical topics
        linkage_function = lambda x: sch.linkage(x, 'ward', optimal_ordering=True)
        hierarchical_topics = topic_model.hierarchical_topics(docs, linkage_function=linkage_function)
        ```
        Nc                     dt        |       z
  S )Nrz   r!   xs    rw   r  z.BERTopic.hierarchical_topics.<locals>.<lambda>F  s    !.?.B*B r}   c                 2    t        j                  | dd      S )NwardT)optimal_ordering)schlinkagerD  s    rw   r  z.BERTopic.hierarchical_topics.<locals>.<lambda>I  s    QQU)V r}   r      r   r   Fr   r   r   rZ   1.0.0)	Parent_IDParent_NameTopicsChild_Left_IDChild_Left_NameChild_Right_IDChild_Right_Namer  distance)t	criterionr   r   r   rN   rz   DistancerM  	ascending)rM  rP  rR  ).r0   r4   rp   rm   r|   r3   shaper   r   r  r5   r   r   r   rh   r   r   r   r   r   _preprocess_textr   r   r	   parsesklearn_versionrC   get_feature_names_outget_feature_namesr   r   rI  fclusterr   r   r   sumrD   isinr   	get_topicr   rN  r   r   r   )!rv   r   r?  r@  rA  r   XZr   r   clean_documentsr/  bowhier_topicsr   clustersnr_clustersr   valirE  clustered_topicsgroupedr.  r-  r6  	parent_idparent_nameZ_idchild_left_idchild_left_namechild_right_idchild_right_names!                                    rw   hierarchical_topicszBERTopic.hierarchical_topics  s   r 	T"$ B#V 1AWAWYcdefgNN

 j)$Q
(8(8(;< Q qAw<3ryy1a4122*1QT73AadG LLd%D	:JUYUaUa!bc	'//	E/JNNPZ\_\d\dOef1556I6O6OSU6UWX6XY//0C0L0L0S0ST ==)W]]7-CC))??AE));;=E##--o> ll

 %A-( 5	E||A5!
Kdnn\Hh-K EE(1+C-X&*HECc(m 345a8C	 -
 /8.AZdaQ(SX/EYZZ !%5!6!:!:!:!BCG((227;H!ioo&:&:;K&La&OPIIO";;E9hjo;pO H-I((/!2D#EQAaD#Ebq#IJK U8A;D$(;$6$:D{@RMk!A%"%(($..:N+OQAaD+OPRQR+S"T"-"2"23}3E"F"R"R U8A;D%)K%7!%;TASNk!A%#&884>>$;O,PaQqT,PQSRS,T#U #.#3#3C4G#H#T#T   AeHQK AeHQK  4KOOC,a/0[5	n #$AqD'J!--kU-KHS<I

&+ 	DE a  [ $F ,P -Qs   /S. S.(S4
;S9
1S>
windowstridemin_similarity
batch_sizepaddinguse_embedding_modelcalculate_tokens	separatorc
                    t        |t              r|g}|t        |      }d}
n!t        j                  t        |      |z        }
g }g }t        t        |
      | j                         D ]  }|||z  |dz   |z   }| j                  j                         }|D cg c]
  } ||       }}g }dg}g }|D ]  }t        |      |k  r#|g}t        t        t        |                  g}n&t        t        t        |                  dd|   }g }g }|D ]\  }||||z    }|st        |      |k(  s|j                  |       |j                  t        t        ||t        |      z                      ^ |rg }g }t        j                  ||z        dz
  }t        t        j                  ||z        dz
        D ]L  }|j                  |d|||z
  |z  z
          |j                  t        t        d|||z
  |z  z
                     N ||z   }||z   }|D cg c]  }|	j                  |       }}|j                  |       |j                  |       |j                  |d   t        |      z           |r8| j                  |dd      } t        | | j                   | j"                  d       }!nY| j                  j%                  |      }"| j&                  j%                  |"      }#t        |#| j(                  | j"                  d       }!d|!|!|k  <   |rg }$g }%t+        |      D ]  \  }&}||&   }'||&dz      }(|'|(k(  r|(dz   }(t        t        |            D cg c]  }| })}|)D &ci c]  }&|&g  }*}&t-        |!|'|( ||'|(       D ]%  \  }+},|,D ]  }||*v s|*|   j                  |+        ' g }-|*j/                         D ]3  \  }.}/|-j                  t0        j2                  j5                  |/             5 t1        j6                  |-      }-t        |-j8                        dk(  r7t1        j:                  dt        | j<                        | j"                  z
  f      }-|%j                  t1        j6                  |-             |$j                  t0        j2                  j5                  |-              t?        |$d	d
      }$n~g }$t        t        |      dz
        D ]?  }&||&   }'||&dz      }(|'|(k(  r|(dz   }(|!|'|( jA                  d      }0|$j                  |0       A t?        t1        j6                  |$      d	d
      }$d}%|j                  |$       |%d}|j                  |%        t1        jB                  |      }||fS c c}w c c}w c c}w c c}&w )a  A post-hoc approximation of topic distributions across documents.

        In order to perform this approximation, each document is split into tokens
        according to the provided tokenizer in the `CountVectorizer`. Then, a
        sliding window is applied on each document creating subsets of the document.
        For example, with a window size of 3 and stride of 1, the sentence:

        `Solving the right problem is difficult.`

        can be split up into `solving the right`, `the right problem`, `right problem is`,
        and `problem is difficult`. These are called tokensets. For each of these
        tokensets, we calculate their c-TF-IDF representation and find out
        how similar they are to the previously generated topics. Then, the
        similarities to the topics for each tokenset are summed up in order to
        create a topic distribution for the entire document.

        We can also dive into this a bit deeper by then splitting these tokensets
        up into individual tokens and calculate how much a word, in a specific sentence,
        contributes to the topics found in that document. This can be enabled by
        setting `calculate_tokens=True` which can be used for visualization purposes
        in `topic_model.visualize_approximate_distribution`.

        The main output, `topic_distributions`, can also be used directly in
        `.visualize_distribution(topic_distributions[index])` by simply selecting
        a single distribution.

        Arguments:
            documents: A single document or a list of documents for which we
                       approximate their topic distributions
            window: Size of the moving window which indicates the number of
                    tokens being considered.
            stride: How far the window should move at each step.
            min_similarity: The minimum similarity of a document's tokenset
                            with respect to the topics.
            batch_size: The number of documents to process at a time. If None,
                        then all documents are processed at once.
                        NOTE: With a large number of documents, it is not
                        advised to process all documents at once.
            padding: Whether to pad the beginning and ending of a document with
                     empty tokens.
            use_embedding_model: Whether to use the topic model's embedding
                                 model to calculate the similarity between
                                 tokensets and topics instead of using c-TF-IDF.
            calculate_tokens: Calculate the similarity of tokens with all topics.
                              NOTE: This is computation-wise more expensive and
                              can require more memory. Using this over batches of
                              documents might be preferred.
            separator: The separator used to merge tokens into tokensets.

        Returns:
            topic_distributions: A `n` x `m` matrix containing the topic distributions
                                 for all input documents with `n` being the documents
                                 and `m` the topics.
            topic_token_distributions: A list of `t` x `m` arrays with `t` being the
                                       number of tokens for the respective document
                                       and `m` the topics.

        Examples:
        After fitting the model, the topic distributions can be calculated regardless
        of the clustering model and regardless of whether the documents were previously
        seen or not:

        ```python
        topic_distr, _ = topic_model.approximate_distribution(docs)
        ```

        As a result, the topic distributions are calculated in `topic_distr` for the
        entire document based on a token set with a specific window size and stride.

        If you want to calculate the topic distributions on a token-level:

        ```python
        topic_distr, topic_token_distr = topic_model.approximate_distribution(docs, calculate_tokens=True)
        ```

        The `topic_token_distr` then contains, for each token, the best fitting topics.
        As with `topic_distr`, it can contain multiple topics for a single token.
        Nrz   r  r   rZ   r   Tr   r  )r  r   r   )"r   r   r   mathceilr   r   rF   rC   build_tokenizerr  appendr   r&  r   r"   rm   r|   r   rD   rp   r   zipr   r   addreducer   rZ  zerosr   r   ra  vstack)1rv   r   rw  rx  ry  rz  r{  r|  r}  r~  batchestopic_distributionstopic_token_distributionsrl  doc_setanalyzerr   tokensall_sentencesall_indicesall_token_sets_idstokenset
token_setstoken_sets_idsstride_indicesstride_indexselected_tokenspadded
padded_idsrU  token	sentencesr   
similaritybow_docc_tf_idf_doctopic_distributiontopic_token_distributionr   startendtoken_id	token_valsim	token_setmatrixr   valuegroups1                                                    rw   approximate_distributionz!BERTopic.approximate_distribution  s   t i%"IYJGiiI ;<G $&!eGn$,,.>? w	KAJ!a%:1EFG ,,<<>H9@AXhx(AFA M#K!#" 'Ex=6)"*J&*5X+?&@%AN &*%H*>%?&%IN!#J%'N(6 *2<,QWBW*X"c/&:f&D&--o>*11 $$)(4(4s?7K(K%&!" !#%'
 IIfvo6:!&tyy&'AA'E!F [A"MM(3PVA?O5P*QR&--d5Fq1uPVFV<W3X.YZ[ &,j%8
)3n)D AKKuY^^E2K	K$$Y/")).9"";r?S^#CDO'ET #!55mJ`d5e
.z4;Q;QRVR`R`Rb;cd
 //99-H#00::7C.|T^^DNNL\=]^
 78JzN23  %'"+-($-f$5 ELE5'.E%eai0C|!Ag ,1U+<=a=H=8@ Au AI A*-js.CEWX]^aEb*c =Y%. =E$	1 )% 0 7 7 <==
  F$-OO$5 <5bffmmE&:;<  XXf-F6<<(A-!#1c$2D2D.E.V*W!X,33BHHV4DE&--bffmmF.CD3E6 &//AST%U" &(""3{#3a#78 5E'.E%eai0C|!Ag&uS15515=E&--e45 &/rxx8J/KRV]^%_"+/(  &&'9:'/,0))001IJow	Kr !ii(;<"$===m BV L>  > As   U:U?	V"
V	search_termimagetop_nc                 J   | j                   t        d      t        | j                  j	                               }|j                          |$| j                  |gdd      j                         }n'|%| j                  dg|gdd      j                         }t        j                  dd	      | j                        j                         }t        j                  |      | d }|D cg c]  }||   	 c}ddd	   }	|D 
cg c]  }
||
   	 c}
ddd	   }||	fS c c}w c c}
w )
a  Find topics most similar to a search_term.

        Creates an embedding for a search query and compares that with
        the topic embeddings. The most similar topics are returned
        along with their similarity values.

        The query is specified using search_term for text queries or image for image queries.

        The search_term can be of any size but since it is compared
        with the topic representation it is advised to keep it
        below 5 words.

        Arguments:
            search_term: the term you want to use to search for topics.
            image: path to the image you want to use to search for topics.
            top_n: the number of topics to return

        Returns:
            similar_topics: the most similar topics from high to low
            similarity: the similarity scores from high to low

        Examples:
        You can use the underlying embedding model to find topics that
        best represent the search term:

        ```python
        topics, similarity = topic_model.find_topics("sports", top_n=5)
        ```

        Note that the search query is typically more accurate if the
        search_term consists of a phrase or multiple words.
        NzBThis method can only be used if you did not use custom embeddings.r   Fr   r   r   rz   rZ   )r`   	Exceptionr  rl   r   sortr   flattenr"   reshaperm   r   argsort)rv   r  r  r  
topic_listsearch_embeddingsimsidsrl  r  r   similar_topicss               rw   find_topicszBERTopic.find_topics  s0   F '`aa$55::<=
 "#77f^c7dlln#77wz5  8  gi  !!1!9!9!R!@$BXBXYaac jjvw''*+!d1g+DbD1
9<=*U+=ddCz)) ,=s   )DD c	           	         t        |       t        |        |s| j                  }|dkD  rt        j	                  d       || _        |xs t        |      | _        |xs
 t               | _	        || _
        || j                  }nt        j	                  d       t        j                  ||t        t        |            |d      }	|	j!                  dgd	      j#                  d
dj$                  i      }
| j'                  |	       | j)                  |
      \  | _        }| j-                  ||	      | _        t1        |      t1        | j                        k7  rxt3        t5        | j                  |      D cg c]  \  }}|dk7  r	||k(  rdnd c}}      }|r'd|vr#d| j                  v r| j6                  dd | _        y| j9                          yyc c}}w )a`
  Updates the topic representation by recalculating c-TF-IDF with the new
        parameters as defined in this function.

        When you have trained a model and viewed the topics and the words that represent them,
        you might not be satisfied with the representation. Perhaps you forgot to remove
        stop_words or you want to try out a different n_gram_range. This function allows you
        to update the topic representation after they have been formed.

        Arguments:
            docs: The documents you used when calling either `fit` or `fit_transform`
            images: The images you used when calling either `fit` or `fit_transform`
            topics: A list of topics where each topic is related to a document in `docs`.
                    Use this variable to change or map the topics.
                    NOTE: Using a custom list of topic assignments may lead to errors if
                          topic reduction techniques are used afterwards. Make sure that
                          manually assigning topics is the last step in the pipeline
            top_n_words: The number of words per topic to extract. Setting this
                         too high can negatively impact topic embeddings as topics
                         are typically best represented by at most 10 words.
            n_gram_range: The n-gram range for the CountVectorizer.
            vectorizer_model: Pass in your own CountVectorizer from scikit-learn
            ctfidf_model: Pass in your own c-TF-IDF model to update the representations
            representation_model: Pass in a model that fine-tunes the topic representations
                                  calculated through c-TF-IDF. Models from `bertopic.representation`
                                  are supported.

        Examples:
        In order to update the topic representation, you will need to first fit the topic
        model and extract topics from them. Based on these, you can update the representation:

        ```python
        topic_model.update_topics(docs, n_gram_range=(2, 3))
        ```

        You can also use a custom vectorizer to update the representation:

        ```python
        from sklearn.feature_extraction.text import CountVectorizer
        vectorizer_model = CountVectorizer(ngram_range=(1, 2), stop_words="english")
        topic_model.update_topics(docs, vectorizer_model=vectorizer_model)
        ```

        You can also use this function to change or map the topics to something else.
        You can update them as follows:

        ```python
        topic_model.update_topics(docs, my_updated_topics)
        ```
        rH   rI   rJ   Na,  Using a custom list of topic assignments may lead to errors if topic reduction techniques are used afterwards. Make sure that manually assigning topics is the last step in the pipeline.Note that topic embeddings will also be created through weightedc-TF-IDF embeddings instead of centroid embeddings.)r   r   r   r   r   Fr   r   r   rZ   Trz   )r0   r2   r;   r^   r_   r:   r#   rC   r)   rD   rE   rh   r   r   r   r   r   r   r   r   r   rp   r   rl   r   allr  rm   r   )rv   r   r   r   r:   r;   rC   rD   rE   r   r   r/  	old_topic	new_topicsame_positions                  rw   update_topicszBERTopic.update_topics  s   x 	T",,LNNo ' 0 ]OP\4](C,A,C$8!>\\FNNF LLdV5QTUYQZK[fl!mn	'//	E/JNNPZ\_\d\dOef 		* !%/B C&*&C&CE9&U# v;#dll++ 14DLL&0I,	9 B &2D=M 6!1bDLL6H)-)?)?)C&**, ,s   4G
fullc                     t        |        |r+d| j                  i}|j                  | j                         |S | j                  S )a  Return topics with top n words and their c-TF-IDF score.

        Arguments:
            full: If True, returns all different forms of topic representations
                  for each topic, including aspects

        Returns:
            self.topic_representations_: The top n words per topic and the corresponding c-TF-IDF score

        Examples:
        ```python
        all_topics = topic_model.get_topics()
        ```
        Main)r2   rl   r   rs   )rv   r  topic_representationss      rw   
get_topicszBERTopic.get_topics>  sF     	%+T-H-H$I!!(()<)<=((...r}   r   c                    t        |        || j                  v re|rTd| j                  |   i}| j                  j                         D ci c]  \  }}|||    }}}j	                  |       |S | j                  |   S yc c}}w )a  Return top n words for a specific topic and their c-TF-IDF scores.

        Arguments:
            topic: A specific topic for which you want its representation
            full: If True, returns all different forms of topic representations
                  for a topic, including aspects

        Returns:
            The top n words for a specific word and its respective c-TF-IDF scores

        Examples:
        ```python
        topic = topic_model.get_topic(12)
        ```
        r  F)r2   rl   rs   r   r   )rv   r   r  representationsaspectaspectss         rw   rc  zBERTopic.get_topicV  s      	D///#)4+F+Fu+M"NY]YlYlYrYrYtu>Ufo6?5#99uu&&w/&&22599 vs   
B c                    t        |        t        j                  | j                  j	                         ddg      j                  d      }|j                  j                  | j                        |d<   | j                  mt        | j                        t        |      k(  rLt        | j                        D ci c]  \  }}|| j                  z
  | }}}|d   j                  |      |d<   | j                  j	                         D ci c]   \  }}|t        t        t!        |             " c}}}|d   j                  |      |d<   | j"                  ri| j"                  j	                         D ]K  \  }}t%        t        |j'                               d   t              rt%        t        |j'                               d   d	   t(              s-t%        t        |j'                               d   d	   t              r<|j	                         D ci c]   \  }}|t        t        t!        |             " }}}nmt%        t        |j'                               d   d	   t*              r@|j	                         D ci c]%  \  }}|d
j-                  |      j/                         ' }}}|d   j                  |      ||<   N | j0                  !|d   j                  | j0                        |d<   | j2                  !|d   j                  | j2                        |d<    |j4                  |j                  |k(  ddf   }|j7                  d      S c c}}w c c}}w c c}}w c c}}w )am  Get information about each topic including its ID, frequency, and name.

        Arguments:
            topic: A specific topic for which you want the frequency

        Returns:
            info: The information relating to either a single topic or all topics

        Examples:
        ```python
        info_df = topic_model.get_topic_info()
        ```
        r   Countr  NameN
CustomNameRepresentationrZ   r   r   Representative_DocsRepresentative_ImagesT)drop)r2   r   r   rj   r   r   r   mapr   ro   r   r   r|   rl   r  nextr  rs   r   r   tupler   r   striprr   rq   r   reset_index)rv   r   r   labellabelsr   r  r  s           rw   get_topic_infozBERTopic.get_topic_infor  s    	||D--335?QR^^_fgzz~~d&8&89V *4&&'3t94LUVZViViLjkLE5%$..0%7kk%)']%6%6v%>\" HLGbGbGhGhGjkmeV%d3<011k!%g!2!26!: "&"5"5";";"= 9d6==?3B7>!$v}}"7";A">F*V]]_-b1!4dK V\UaUaUc!d\UE%d3;.?)@"@!d!d#D$9"$=a$@#FU[UaUaUc!d\UE%%)>)>)@"@!d!d#G}008V9 $$0*.w-*;*;D<U<U*VD&'&&2,0M,=,=d>Y>Y,ZD() 88DJJ%/23DT**; l l "e!ds   9M
%M%M9*M c                     t        |        t        |t              r| j                  |   S t	        j
                  | j                  j                         ddg      j                  dd      S )a&  Return the size of topics (descending order).

        Arguments:
            topic: A specific topic for which you want the frequency

        Returns:
            Either the frequency of a single topic or dataframe with
            the frequencies of all topics

        Examples:
        To extract the frequency of all topics:

        ```python
        frequency = topic_model.get_topic_freq()
        ```

        To get the frequency of a single topic:

        ```python
        frequency = topic_model.get_topic_freq(12)
        ```
        r   r  r  FrX  )r2   r   r   rj   r   r   r   r   rv   r   s     rw   get_topic_freqzBERTopic.get_topic_freq  sd    . 	eS!$$U++<< 1 1 7 7 9GWCUVbb5 c  r}   dfmetadatac                 >   t        |       |%|j                         }||d<   | j                  |d<   n"t        j                  || j                  d      }| j                         j                  dd      }t        j                  ||dd      }t        | j                        D ci c]2  }|d	j                  t        t        | j                  |                   4 }}|j                  j                  |      |d
<   | j                  t!        | j                  j"                        dk(  r| j                  |d<   nRt        | j                  | j                        D cg c]#  \  }}|dk7  rt%        |      ndt'        |      z
  % c}}|d<   | j(                  j+                         D 	
cg c]  }	|	D ]  }
|
  c}
}	}	d|d<   d|j,                  |j.                  j1                  |	      df<   ||j3                         D ]
  \  }}|||<    |S c c}w c c}}w c c}
}	w )ap	  Get information about the documents on which the topic was trained
        including the documents themselves, their respective topics, the name
        of each topic, the top n words of each topic, whether it is a
        representative document, and probability of the clustering if the cluster
        model supports it.

        There are also options to include other meta data, such as the topic
        distributions or the x and y coordinates of the reduced embeddings.

        Arguments:
            docs: The documents on which the topic model was trained.
            df: A dataframe containing the metadata and the documents on which
                the topic model was originally trained on.
            metadata: A dictionary with meta data for each document in the form
                      of column name (key) and the respective values (value).

        Returns:
            document_info: A dataframe with several statistics regarding
                           the documents on which the topic model was trained.

        Usage:

        To get the document info, you will only need to pass the documents on which
        the topic model was trained:

        ```python
        document_info = topic_model.get_document_info(docs)
        ```

        There are additionally options to include meta data, such as the topic
        distributions. Moreover, we can pass the original dataframe that contains
        the documents and extend it with the information retrieved from BERTopic:

        ```python
        from sklearn.datasets import fetch_20newsgroups

        # The original data in a dataframe format to include the target variable
        data = fetch_20newsgroups(subset='all',  remove=('headers', 'footers', 'quotes'))
        df = pd.DataFrame({"Document": data['data'], "Class": data['target']})

        # Add information about the percentage of the document that relates to the topic
        topic_distr, _ = topic_model.approximate_distribution(docs, batch_size=1000)
        distributions = [distr[topic] if topic != -1 else 0 for topic, distr in zip(topics, topic_distr)]

        # Create our documents dataframe using the original dataframe and meta data about
        # the topic distributions
        document_info = topic_model.get_document_info(docs, df=df,
                                                      metadata={"Topic_distribution": distributions})
        r   r   )r   r   r  rz   r   r  )onhowz - Top_n_wordsProbabilityrZ   FRepresentative_documentT)r0   r  rh   r   r   r  r  merger   r   r  r  rc  r   r  ri   r   rZ  r   ra  rr   r   r   r   rb  r   )rv   r   r  r  document_info
topic_infor   r:   probs	repr_docsrepr_doccolumnr   s                rw   get_document_infozBERTopic.get_document_info  s   n 	T">GGIM(,M*%%)\\M'"LLdT\\)RSM ((*//a/@

wFS Z]]a]i]iYjkPUuejjc4>>%3H.I)JKKkk'4':':'>'>{'Km$ *4&&,,-2/3/B/Bm, ),DLL$:M:M(N0$u #(2+CJ1s5z>A0m, 04/H/H/O/O/Ql)bklV^XlXl	38/0_c-0055i@B[[\ "*.."2 /(.f%/- l0 ms   &7H(H*Hc                     t        |        t        |t              r+| j                  j	                  |      r| j                  |   S y| j                  S )a  Extract the best representing documents per topic.

        Note:
            This does not extract all documents per topic as all documents
            are not saved within BERTopic. To get all documents, please
            run the following:

            ```python
            # When you used `.fit_transform`:
            df = pd.DataFrame({"Document": docs, "Topic": topic})

            # When you used `.fit`:
            df = pd.DataFrame({"Document": docs, "Topic": topic_model.topics_})
            ```

        Arguments:
            topic: A specific topic for which you want
                   the representative documents

        Returns:
            Representative documents of the chosen topic

        Examples:
        To extract the representative docs of all topics:

        ```python
        representative_docs = topic_model.get_representative_docs()
        ```

        To get the representative docs of a single topic:

        ```python
        representative_docs = topic_model.get_representative_docs(12)
        ```
        N)r2   r   r   rr   r   r  s     rw   get_representative_docsz BERTopic.get_representative_docs"  sK    H 	eS!((,,U30077,,,r}   rh  max_distancetight_layoutc           	      0   	
 |rdnd j                   j                         dz    j                  j                  t              j                         dz
  	t        t         j                   j                              

j                  t        t         j                   j                                     
j                         D ci c]  \  }}||dd  c}}
 j                         D ci c]I  }t        |d   j                        t        |d   j                        t        |d   j                        gK }} 	
fd}t         j                  j                  t              j                               } |||      S c c}}w c c}w )u  Extract the topic tree such that it can be printed.

        Arguments:
            hier_topics: A dataframe containing the structure of the topic tree.
                         This is the output of `topic_model.hierarchical_topics()`
            max_distance: The maximum distance between two topics. This value is
                          based on the Distance column in `hier_topics`.
            tight_layout: Whether to use a tight layout (narrow width) for
                          easier readability if you have hundreds of topics.

        Returns:
            A tree that has the following structure when printed:
                .
                .
                └─health_medical_disease_patients_hiv
                    ├─patients_medical_disease_candida_health
                    │    ├─■──candida_yeast_infection_gonorrhea_infections ── Topic: 48
                    │    └─patients_disease_cancer_medical_doctor
                    │         ├─■──hiv_medical_cancer_patients_doctor ── Topic: 34
                    │         └─■──pain_drug_patients_disease_diet ── Topic: 26
                    └─■──health_newsgroup_tobacco_vote_votes ── Topic: 9

            The blocks (■) indicate that the topic is one you can directly access
            from `topic_model.get_topic`. In other words, they are the original un-grouped topics.

        Examples:
        ```python
        # Train model
        from bertopic import BERTopic
        topic_model = BERTopic()
        topics, probs = topic_model.fit_transform(docs)
        hierarchical_topics = topic_model.hierarchical_topics(docs)

        # Print topic tree
        tree = topic_model.get_topic_tree(hierarchical_topics)
        print(tree)
        ```
        rz   r   NrH   c                 :    dfd	d} || | |      }|S )z8Based on: https://stackoverflow.com/a/51920869/10532563.c                    	j                   	j                  |k(  	j                  |k(  z  df   }t        |      dkD  r|j                  d   nd}||k7  rD|	| |   z  } n9t        |      k  r |
k  r| d|   z   d| z   dz   z  } n| dz  } n| |   dz   z  } ||vr| S ||   d d D ]#  }| |d	z   d
z   z  }  | |||||dz   dz  z         } % ||   d   }| |dz   d
z   z  }  | |||||ddz   z  z         } | S )NrW  r   
   u	   ■──u    ── Topic: 
zO 
rZ   u   ├u   ─u   │r   u   └rz   )r   rP  rR  r   r   r   )to_printr  parenttreegrandpaindentrT  child_treerh  r  max_original_topictopic_to_namewidths           rw   r  z8BERTopic.get_topic_tree.<locals>.get_tree.<locals>._tree  s{   &?? ..&8[=W=W[a=ab  25X1B8??1-U? M&$99v;*<<',6 (K-:O,OTcdjckRl,los,s s (F 2$f(=(DDH%#O!&\#2. iE 66H$XueT66TY>\_bg\gKghHi VR(FUNU22 5%vvPSW\_`W`PaGabr}   z.
)N  )	r  r  r  r  rh  r  r  r  r  s	      @rw   get_treez)BERTopic.get_topic_tree.<locals>.get_tree  s*         D "HXueT:HOr}   )rW  r   rM  r   r   mindictr  rP  rQ  r   rR  rS  r   r   r   )rh  r  r  r   namer   r  r  r  r  r  r  s   ``       @@@rw   get_topic_treezBERTopic.get_topic_treeO  sq   X "q&//3359L(2299#>BBDqH S!:!:K<W<WXYT#k&@&@+B^B^"_`a>K>Q>Q>ST{udTc
*T #++-

 	 A  !CF(()CF))*$ 
 
'	 '	R K))00599;<t$$k U
s   F5AFr   c                 R   t        t        | j                              }t        |t              r| j
                  *t        || j
                        D ci c]  \  }}||
 }}}n9| j                         }t	        t        |j                  |j                              }|D cg c])  }|j                  |      r|j                  |      n||   + }}|| _        yt        |t              r,t        |      t        |      k(  r
|}|| _        yt        d      | _        yc c}}w c c}w )af  Set custom topic labels in your fitted BERTopic model.

        Arguments:
            topic_labels: If a list of topic labels, it should contain the same number
                          of labels as there are topics. This must be ordered
                          from the topic with the lowest ID to the highest ID,
                          including topic -1 if it exists.
                          If a dictionary of `topic ID`: `topic_label`, it can have
                          any number of topics as it will only map the topics found
                          in the dictionary.

        Examples:
        First, we define our topic labels with `.generate_topic_labels` in which
        we can customize our topic labels:

        ```python
        topic_labels = topic_model.generate_topic_labels(nr_words=2,
                                                    topic_prefix=True,
                                                    word_length=10,
                                                    separator=", ")
        ```

        Then, we pass these `topic_labels` to our topic model which
        can be accessed at any time with `.custom_labels_`:

        ```python
        topic_model.set_topic_labels(topic_labels)
        topic_model.custom_labels_
        ```

        You might want to change only a few topic labels instead of all of them.
        To do so, you can pass a dictionary where the keys are the topic IDs and
        its keys the topic labels:

        ```python
        topic_model.set_topic_labels({0: "Space", 1: "Sports", 2: "Medicine"})
        topic_model.custom_labels_
        ```
        NzUMake sure that `topic_labels` contains the same number of labels as there are topics.)r  r   rh   r   r  ro   r  r  r   r  r   r  r   r   )rv   r   unique_topicsr   r  original_labelsr   custom_labelss           rw   set_topic_labelszBERTopic.set_topic_labels  s   P s4<<01lD)"".DGW[WjWjDk"lLE55%<"l"l**,"&s4::tyy'A"B + ,8+;+;E+B  'X]H^^M  , d+< C$66 , ,	 !k  ,# #ms   D".D$nr_wordstopic_prefixword_lengthr  c                    t        t        | j                              }g }|D ]  }|rt        | j                  |   |    \  }	}
nt        | j                  |       \  }	}
|r|	D cg c]  }|d| 	 c}d| }	nt        |	      d| }	|r| | |j                  |	      z   }n|j                  |	      }|j                  |        |S c c}w )a8  Get labels for each topic in a user-defined format.

        Arguments:
            nr_words: Top `n` words per topic to use
            topic_prefix: Whether to use the topic ID as a prefix.
                          If set to True, the topic ID will be separated
                          using the `separator`
            word_length: The maximum length of each word in the topic label.
                         Some words might be relatively long and setting this
                         value helps to make sure that all labels have relatively
                         similar lengths.
            separator: The string with which the words and topic prefix will be
                       separated. Underscores are the default but a nice alternative
                       is `", "`.
            aspect: The aspect from which to generate topic labels

        Returns:
            topic_labels: A list of topic labels sorted from the lowest topic ID to the highest.
                          If the topic model was trained using HDBSCAN, the lowest topic ID is -1,
                          otherwise it is 0.

        Examples:
        To create our custom topic labels, usage is rather straightforward:

        ```python
        topic_labels = topic_model.generate_topic_labels(nr_words=2, separator=", ")
        ```
        N)	r  r   rh   r  rs   rc  r  r   r  )rv   r  r  r  r~  r  r  r   r   r/  r   r   topic_labels                rw   generate_topic_labelszBERTopic.generate_topic_labels  s    H s4<<01" 	-E 3 3F ;E BCqu 56q8=>l{+>yIUIX.!&	{3innU6KK'nnU3,!	-$  ?s   $Ctopics_to_mergec                 ,   t        |        t        |       t        j                  || j                  |t        t        |            d      }t        | j                        D ci c]  }|| }}t        |d   t              rt        |      D ]
  }|d   ||<    n>t        |d   t              r t        |      D ]  }|D ]
  }|d   ||<     nt        d      t        t              }t        |j                               D ]  \  }	}
||
   j!                  |	        |j                         D ci c]&  \  }}|||D cg c]  }| j"                  |    c}d( }}}}|j$                  j'                  |      |_        | j(                  j+                  ||        | j-                  |      }| j/                  ||       | j1                  |       | j3                  |       | j5                  | j6                        | _        yc c}w c c}w c c}}}w )a  Arguments:
            docs: The documents you used when calling either `fit` or `fit_transform`
            topics_to_merge: Either a list of topics or a list of list of topics
                             to merge. For example:
                                [1, 2, 3] will merge topics 1, 2 and 3
                                [[1, 2], [3, 4]] will merge topics 1 and 2, and
                                separately merge topics 3 and 4.
            images: A list of paths to the images used when calling either
                    `fit` or `fit_transform`.

        Examples:
        If you want to merge topics 1, 2, and 3:

        ```python
        topics_to_merge = [1, 2, 3]
        topic_model.merge_topics(docs, topics_to_merge)
        ```

        or if you want to merge topics 1 and 2, and separately
        merge topics 3 and 4:

        ```python
        topics_to_merge = [[1, 2],
                            [3, 4]]
        topic_model.merge_topics(docs, topics_to_merge)
        ```
        r   r   r   r   r   zWMake sure that `topics_to_merge` is eithera list of topics or a list of list of topics.topics_fromtopic_sizestopic_model)r   N)r2   r0   r   r   rh   r   r   r   r   r   r  r   r   r   r  r   r  rj   r   r  rk   add_mappingsr   r   r   r   r   ri   )rv   r   r  r   r   r   mappingtopic_groupr   r   rk  topic_tor  s                rw   merge_topicszBERTopic.merge_topics5  s   B 	T"LL CI&	
	 .1->?E5%<??oa(#.0 4!0!34*H5%o6 4( 4E%0^GEN44 vww t$w}}/ 	&HCSM  %	& *2)9
 

 &+	 *FQRU 1 1% 8R 
 
 $//--g6	''T'B44Y?	Y:	*&&y1"55d6I6IJ= @$  S
s   $
H2H H
H
Htopics_to_deletec                    t        |        t        j                  d| j                  i      }dt	        | j                        v }|sRt        fd| j                  D              r3t        j                  d| j                  j                  d   f      }t        j                  |      }t        j                  || j                  g      | _        t        j                  d| j                  j                  d   f      }t        j                  || j                  g      | _        dg| j                  d<   g | j                  d<   | j                   Tt        j                  d| j                   j                  d   f      }t        j                  || j                   g      | _        t#        | d      r| j$                  d| j$                  d<   t#        | d	      r| j&                  }| j&                  j(                  j                  d   }t        j                  d
gdgdgffd|f      }t        j                  || j&                  j(                  g      | j&                  _        t#        | d      r/| j*                  #| j*                  D ]  }	i | j*                  |	   d<    t	        | j                        D 
ci c]  }
|
|
v rdn|
 }}
d|d<   t-        t.              }t1        |j3                               D ]  \  }}||   j5                  |        |j3                         D 
ci c]&  \  }}|||D 
cg c]  }
| j6                  |
    c}
d( }}}}
|j8                  j;                  |      |_        | j<                  j?                  |tA        |              | jC                  |      }| jE                  |       | jG                  | jH                        | _$        | j<                  jK                  d      }t#        | d      rt| j*                  h| j*                  j3                         D 	ci c]8  \  }	}|	|j3                         D ci c]  \  }}|vr|dk7  r||   nd| c}}: }}}}	}|| _        t#        | d      rN| j$                  B| j$                  j3                         D ci c]  \  }}|vr|dk7  r||   nd| }}}|| _        | j                  j3                         D ci c]  \  }}|vr|dk7  r||   nd| }}}|| _        | j                  j3                         D ci c]  \  }}|vr|dk7  r||   nd| }}}|| _        | j                   gt        jL                  tO        tQ        | j                               D 
cg c]  }
|
v c}
      }|j                         r| j                   |   nd| _        dD ]Y  }tS        | |      }t        jL                  tO        |j                  d         D 
cg c]  }
|
v c}
      }tU        | |||          [ t#        | d	      r| j&                  xt        jL                  tO        | j&                  j(                  j                  d         D 
cg c]  }
|
v c}
      }| j&                  j(                  |   | j&                  _        yyyc c}
w c c}
w c c}
}}w c c}}w c c}}}}	w c c}}w c c}}w c c}}w c c}
w c c}
w c c}
w )a  Delete topics from the topic model.

        The deleted topics will be mapped to -1 (outlier topic). Core topic attributes
        like topic embeddings and c-TF-IDF will be automatically updated.

        Arguments:
            topics_to_delete: List of topics to delete
        r   rZ   c              3   &   K   | ]  }|v  
 y wr  r   ).0r   r  s     rw   	<genexpr>z)BERTopic.delete_topics.<locals>.<genexpr>  s     #X%E-=$=#Xs   rz   r  gh㈵>Nro   r  rD   g      ?r   rZ  rs   r  r  Fr   )rm   rp   )+r2   r   r   rh   r   anyr   r  rp   rZ  spr   r  rm   rl   rr   rq   r   ro   rD   	_idf_diagrs   r   r  r  r   r  rj   r   r  rk   r  r   r   r   r   ri   r   r   r   r   getattrsetattr)rv   r  	topics_dfhad_outliersoutlier_rowoutlier_embeddingoutlier_image
n_featuresoutlier_diagr  r   r  r   r   rk  r  r  final_mappingr   r  contentnew_aspectsr  
new_labelsnew_representationsr   new_representative_docsmaskattrr  s    `                            rw   delete_topicszBERTopic.delete_topics  s    	LL'4<<!89	 S.. #X4<<#X X((At~~';';A'>#?@K--4KYYT^^'DEDN !#!T-C-C-I-I!-L)M N%'YY0A4CYCY/Z%[D" 0;mD''+ -/D%%b) **6 "!T-H-H-N-Nq-Q)R S.0iiHcHc8d.e+ t-.43F3F3R*,##B' t^,1B1B1N!..88>>qA
!}}seqcA3Z-@JX.0iitGXGXGbGb8c.d!!+ t-.43F3F3R"11 9F68D''/39 SVVZVbVbRcd5)9 9"uDdd t$w}}/ 	&HCSM  %	& *2)9
 

 &+	 *FQRU 1 1% 8R 
 
 $//--g6	''Xd^'L44Y?		*"55d6I6IJ**777N 4)*t/B/B/N '+&9&9&?&?&A  #FF .4lln*	7 (88 2;b]9-b7R K  #.D 4)*t/B/B/N )-(;(;(A(A(C$Iu$44 .7"_y)"uLJ 
 #-D
 '+&A&A&G&G&I
"	7 00 *3b]9%b7J
 

 ':#
 $(#<#<#B#B#D#
	4 00 *3b]9%b4G#
 #

 %<! &&288cRVRmRmNnHopuU*::pqDOSxxz$*E*Ed*K_cD' 7 	.DT4(F88fll[\oH^_uU*::_`DD$t-	. 4(T->->-J88dN_N_NiNiNoNopqNrHstuU*::tuD*.*;*;*E*Ed*KD' .K(W e  S
*
#
 q `
 usZ   1Z4 Z>.Z9Z>*[
[![
([*[,[[%>[*
7[/9Z>[
c           
      @   t        |        t        |       || _        t        j                  || j
                  |t        t        |            d      }| j                  ||      }d| _	        | j                  |       | j                  | j                        | _        | S )a6  Reduce the number of topics to a fixed number of topics
        or automatically.

        If nr_topics is an integer, then the number of topics is reduced
        to nr_topics using `AgglomerativeClustering` on the cosine distance matrix
        of the topic c-TF-IDF or semantic embeddings.

        If nr_topics is `"auto"`, then HDBSCAN is used to automatically
        reduce the number of topics by running it on the topic embeddings.

        The topics, their sizes, and representations are updated.

        Arguments:
            docs: The docs you used when calling either `fit` or `fit_transform`
            nr_topics: The number of topics you want reduced to
            images: A list of paths to the images used when calling either
                    `fit` or `fit_transform`
            use_ctfidf: Whether to calculate distances between topics based on c-TF-IDF embeddings. If False, the
                        embeddings from the embedding model are used.

        Updates:
            topics_ : Assigns topics to their merged representations.
            probabilities_ : Assigns probabilities to their merged representations.

        Examples:
        You can further reduce the topics by passing the documents with their
        topics and probabilities (if they were calculated):

        ```python
        topic_model.reduce_topics(docs, nr_topics=30)
        ```

        You can then access the updated topics and probabilities with:

        ```python
        topics = topic_model.topics_
        probabilities = topic_model.probabilities_
        ```
        r  N)r2   r0   r=   r   r   rh   r   r   r   rt   r   r   ri   )rv   r   r=   r   r?  r   s         rw   reduce_topicszBERTopic.reduce_topics		  s    \ 	T""LL CI&	
	 ''	:>	"&&y1"55d6I6IJr}   distributionsr   strategyr   	thresholddistributions_paramsc	                 n   | j                   st        d      |d}|j                         dk(  r|t        d      |j                         dk(  rRt        ||      D 	
cg c]9  \  }	}
t	        j
                  |
      |k\  r|	dk(  rt	        j                  |
      n|	; }}	}
|S |j                         dk(  rt        |      D 	cg c]  \  }}	|	dk(  s| }}}	|D cg c]  }||   	 }} | j                  |fd|i|\  }}t        |D 
cg c]'  }
t        |
      d	kD  rt	        j                  |
      nd) c}
      }|D 	cg c]  }	|	dk7  r|	n
t        |       }}	|S |j                         d
k(  rt        |      D 	cg c]  \  }}	|	dk(  s| }}}	|D cg c]  }||   	 }}| j                  j                  |      }| j                  j                  |      }t        || j                   | j                   d       }d	|||k  <   t        |D cg c]'  }t        |      d	kD  rt	        j                  |      nd) c}      }|D 	cg c]  }	|	dk7  r|	n
t        |       }}	|S |j                         dk(  ri| j"                  |t        d      t        |      D 	cg c]  \  }}	|	dk(  s| }}}	||D cg c]  }||   	 }}n|D cg c]  }||   	 }}|'t	        j$                  |D cg c]  }||   	 c}      }nW|:|D cg c]  }||   	 }}| j"                  j'                  || j(                        }n| j"                  j+                  |      }t        || j,                  | j                   d       }d	|||k  <   t        |D cg c]'  }t        |      d	kD  rt	        j                  |      nd) c}      }|D 	cg c]  }	|	dk7  r|	n
t        |       }}	S c c}
}	w c c}	}w c c}w c c}
w c c}	w c c}	}w c c}w c c}w c c}	w c c}	}w c c}w c c}w c c}w c c}w c c}w c c}	w )aD  Reduce outliers by merging them with their nearest topic according
        to one of several strategies.

        When using HDBSCAN, DBSCAN, or OPTICS, a number of outlier documents might be created
        that do not fall within any of the created topics. These are labeled as -1.
        This function allows the user to match outlier documents with their nearest topic
        using one of the following strategies using the `strategy` parameter:
            * "probabilities"
                This uses the soft-clustering as performed by HDBSCAN to find the
                best matching topic for each outlier document. To use this, make
                sure to calculate the `probabilities` beforehand by instantiating
                BERTopic with `calculate_probabilities=True`.
            * "distributions"
                Use the topic distributions, as calculated with `.approximate_distribution`
                to find the most frequent topic in each outlier document. You can use the
                `distributions_params` variable to tweak the parameters of
                `.approximate_distribution`.
            * "c-tf-idf"
                Calculate the c-TF-IDF representation for each outlier document and
                find the best matching c-TF-IDF topic representation using
                cosine similarity.
            * "embeddings"
                Using the embeddings of each outlier documents, find the best
                matching topic embedding using cosine similarity.

        Arguments:
            documents: A list of documents for which we reduce or remove the outliers.
            topics: The topics that correspond to the documents
            images: A list of paths to the images used when calling either
                    `fit` or `fit_transform`
            strategy: The strategy used for reducing outliers.
                    Options:
                        * "probabilities"
                            This uses the soft-clustering as performed by HDBSCAN
                            to find the best matching topic for each outlier document.

                        * "distributions"
                            Use the topic distributions, as calculated with `.approximate_distribution`
                            to find the most frequent topic in each outlier document.

                        * "c-tf-idf"
                            Calculate the c-TF-IDF representation for outlier documents and
                            find the best matching c-TF-IDF topic representation.

                        * "embeddings"
                            Calculate the embeddings for outlier documents and
                            find the best matching topic embedding.
            probabilities: Probabilities generated by HDBSCAN for each document when using the strategy `"probabilities"`.
            threshold: The threshold for assigning topics to outlier documents. This value
                       represents the minimum probability when `strategy="probabilities"`.
                       For all other strategies, it represents the minimum similarity.
            embeddings: The pre-computed embeddings to be used when `strategy="embeddings"`.
                        If this is None, then it will compute the embeddings for the outlier documents.
            distributions_params: The parameters used in `.approximate_distribution` when using
                                  the strategy `"distributions"`.

        Returns:
            new_topics: The updated topics

        Usage:

        The default settings uses the `"distributions"` strategy:

        ```python
        new_topics = topic_model.reduce_outliers(docs, topics)
        ```

        When you use the `"probabilities"` strategy, make sure to also pass the probabilities
        as generated through HDBSCAN:

        ```python
        from bertopic import BERTopic
        topic_model = BERTopic(calculate_probabilities=True)
        topics, probs = topic_model.fit_transform(docs)

        new_topics = topic_model.reduce_outliers(docs, topics, probabilities=probs, strategy="probabilities")
        ```
        zNo outliers to reduce.Nr   r   zOMake sure to pass in `probabilities` in order to use the probabilities strategyrZ   r<  ry  r   zc-tf-idfzdTo use this strategy, you will need to pass a model to `embedding_model`when instantiating BERTopic.rF   )r|   r   lowerr  r   r   r   r   r  iterra  r  rC   r   rD   r"   rp   r`   r   embed_imagesrF   embed_documentsrm   )rv   r   r   r   r=  r   r>  r   r?  r   probr   r   outlier_idsoutlier_docstopic_distrr   outlier_topicsr  r  r  r  outlier_embeddingsoutlier_imagess                           rw   reduce_outlierszBERTopic.reduce_outliersL	  s[   r ~~566#H >>.=3Hnoo >>. $'v}#=E4 $&66$<9#<"		$RWWJ t i ^^05>v5FV\UE%SU+5VKV:EFIe,FLF:T::-6:NNK "Xc"dPTc$i!m299T?#K"deNV\]U5B;%D4HH]J]Z U ^^+5>v5FV\UE%SU+5VKV:EFIe,FLF ++55lCG,,66w?L*<HX9YZJ 23JzI-.!U_"`cSX\299S>r#I"`aNV\]U5B;%D4HH]J]< 7 ^^-##+
0B 3  6?v5FV\UE%SU+5VKV!;FG%uGG>IJU	% 0JJ %%'XXk.ZUz%/@.Z%["#=H!IE&-!I!I%)%9%9%F%F~_c_k_k%F%l"%)%9%9%I%I,%W"*+=t?U?UVZVdVdVf?ghJ 23JzI-.!U_"`cSX\299S>r#I"`aNV\]U5B;%D4HH]J]u WF #e] WF #a] WGJ /[!I #a]sr   >O#O)O)O/
,O4O9?O>O>P,P	PP-P:PP0P#
P(,P-P2top_n_topicsr	  titler  heightz	go.Figurec           
      R    t        |        t        j                  | |||||||      S )a  Visualize topics, their sizes, and their corresponding words.

        This visualization is highly inspired by LDAvis, a great visualization
        technique typically reserved for LDA.

        Arguments:
            topics: A selection of topics to visualize
                    Not to be confused with the topics that you get from `.fit_transform`.
                    For example, if you want to visualize only topics 1 through 5:
                    `topics = [1, 2, 3, 4, 5]`.
            top_n_topics: Only select the top n most frequent topics
            use_ctfidf: Whether to use c-TF-IDF representations instead of the embeddings from the embedding model.
            custom_labels: Whether to use custom topic labels that were defined using
                           `topic_model.set_topic_labels`.
            title: Title of the plot.
            width: The width of the figure.
            height: The height of the figure.

        Examples:
        To visualize the topics simply run:

        ```python
        topic_model.visualize_topics()
        ```

        Or if you want to save the resulting figure:

        ```python
        fig = topic_model.visualize_topics()
        fig.write_html("path/to/file.html")
        ```
        )r   rN  r?  r	  rO  r  rP  )r2   r   visualize_topics)rv   r   rN  r?  r	  rO  r  rP  s           rw   rR  zBERTopic.visualize_topics	  s8    T 	((%!'	
 		
r}       reduced_embeddingssamplehide_annotationshide_document_hoverc                 p    t        |        t        |       t        j                  | |||||||||	|
|      S )a  Visualize documents and their topics in 2D.

        Arguments:
            topic_model: A fitted BERTopic instance.
            docs: The documents you used when calling either `fit` or `fit_transform`
            topics: A selection of topics to visualize.
                    Not to be confused with the topics that you get from `.fit_transform`.
                    For example, if you want to visualize only topics 1 through 5:
                    `topics = [1, 2, 3, 4, 5]`.
            embeddings: The embeddings of all documents in `docs`.
            reduced_embeddings: The 2D reduced embeddings of all documents in `docs`.
            sample: The percentage of documents in each topic that you would like to keep.
                    Value can be between 0 and 1. Setting this value to, for example,
                    0.1 (10% of documents in each topic) makes it easier to visualize
                    millions of documents as a subset is chosen.
            hide_annotations: Hide the names of the traces on top of each cluster.
            hide_document_hover: Hide the content of the documents when hovering over
                                specific points. Helps to speed up generation of visualization.
            custom_labels: Whether to use custom topic labels that were defined using
                           `topic_model.set_topic_labels`.
            title: Title of the plot.
            width: The width of the figure.
            height: The height of the figure.

        Examples:
        To visualize the topics simply run:

        ```python
        topic_model.visualize_documents(docs)
        ```

        Do note that this re-calculates the embeddings and reduces them to 2D.
        The advised and preferred pipeline for using this function is as follows:

        ```python
        from sklearn.datasets import fetch_20newsgroups
        from sentence_transformers import SentenceTransformer
        from bertopic import BERTopic
        from umap import UMAP

        # Prepare embeddings
        docs = fetch_20newsgroups(subset='all',  remove=('headers', 'footers', 'quotes'))['data']
        sentence_model = SentenceTransformer("all-MiniLM-L6-v2")
        embeddings = sentence_model.encode(docs, show_progress_bar=False)

        # Train BERTopic
        topic_model = BERTopic().fit(docs, embeddings)

        # Reduce dimensionality of embeddings, this step is optional
        # reduced_embeddings = UMAP(n_neighbors=10, n_components=2, min_dist=0.0, metric='cosine').fit_transform(embeddings)

        # Run the visualization with the original embeddings
        topic_model.visualize_documents(docs, embeddings=embeddings)

        # Or, if you have reduced the original embeddings already:
        topic_model.visualize_documents(docs, reduced_embeddings=reduced_embeddings)
        ```

        Or if you want to save the resulting figure:

        ```python
        fig = topic_model.visualize_documents(docs, reduced_embeddings=reduced_embeddings)
        fig.write_html("path/to/file.html")
        ```

        <iframe src="../getting_started/visualization/documents.html"
        style="width:1000px; height: 800px; border: 0px;""></iframe>
        )r   r   r   rU  rV  rW  rX  r	  rO  r  rP  )r2   r0   r   visualize_documents)rv   r   r   r   rU  rV  rW  rX  r	  rO  r  rP  s               rw   rZ  zBERTopic.visualize_documents#
  sL    d 	T"++!1- 3'
 	
r}   zDocuments and Topics	sub_titleinteractiveenable_searchdatamap_kwdsint_datamap_kwdsz
fig.Figurec                 ^    t        |        t        j                  | |||||||||	|
||||      S )a#  Visualize documents and their topics in 2D as a static plot for publication using
        DataMapPlot. This works best if there are between 5 and 60 topics. It is therefore best
        to use a sufficiently large `min_topic_size` or set `nr_topics` when building the model.

        Arguments:
            topic_model:  A fitted BERTopic instance.
            docs: The documents you used when calling either `fit` or `fit_transform`.
            topics: A selection of topics to visualize.
                    Not to be confused with the topics that you get from `.fit_transform`.
                    For example, if you want to visualize only topics 1 through 5:
                    `topics = [1, 2, 3, 4, 5]`. Documents not in these topics will be shown
                    as noise points.
            embeddings:  The embeddings of all documents in `docs`.
            reduced_embeddings:  The 2D reduced embeddings of all documents in `docs`.
            custom_labels:  If bool, whether to use custom topic labels that were defined using
                        `topic_model.set_topic_labels`.
                        If `str`, it uses labels from other aspects, e.g., "Aspect1".
            title: Title of the plot.
            sub_title: Sub-title of the plot.
            width: The width of the figure.
            height: The height of the figure.
            interactive: Whether to create an interactive plot using DataMapPlot's `create_interactive_plot`.
            enable_search: Whether to enable search in the interactive plot. Only works if `interactive=True`.
            topic_prefix: Prefix to add to the topic number when displaying the topic name.
            datamap_kwds:  Keyword args be passed on to DataMapPlot's `create_plot` function
                        if you are not using the interactive version.
                        See the DataMapPlot documentation for more details.
            int_datamap_kwds:  Keyword args be passed on to DataMapPlot's `create_interactive_plot` function
                            if you are using the interactive version.
                            See the DataMapPlot documentation for more details.

        Returns:
            figure: A Matplotlib Figure object.

        Examples:
        To visualize the topics simply run:

        ```python
        topic_model.visualize_document_datamap(docs)
        ```

        Do note that this re-calculates the embeddings and reduces them to 2D.
        The advised and preferred pipeline for using this function is as follows:

        ```python
        from sklearn.datasets import fetch_20newsgroups
        from sentence_transformers import SentenceTransformer
        from bertopic import BERTopic
        from umap import UMAP

        # Prepare embeddings
        docs = fetch_20newsgroups(subset='all',  remove=('headers', 'footers', 'quotes'))['data']
        sentence_model = SentenceTransformer("all-MiniLM-L6-v2")
        embeddings = sentence_model.encode(docs, show_progress_bar=False)

        # Train BERTopic
        topic_model = BERTopic(min_topic_size=36).fit(docs, embeddings)

        # Reduce dimensionality of embeddings, this step is optional
        # reduced_embeddings = UMAP(n_neighbors=10, n_components=2, min_dist=0.0, metric='cosine').fit_transform(embeddings)

        # Run the visualization with the original embeddings
        topic_model.visualize_document_datamap(docs, embeddings=embeddings)

        # Or, if you have reduced the original embeddings already:
        topic_model.visualize_document_datamap(docs, reduced_embeddings=reduced_embeddings)
        ```

        Or if you want to save the resulting figure:

        ```python
        fig = topic_model.visualize_document_datamap(docs, reduced_embeddings=reduced_embeddings)
        fig.savefig("path/to/file.png", bbox_inches="tight")
        ```
        )r2   r   visualize_document_datamap)rv   r   r   r   rU  r	  rO  r[  r  rP  r\  r]  r  r^  r_  s                  rw   ra  z#BERTopic.visualize_document_datamap
  sM    x 	22
 	
r}   rv  	nr_levelslevel_scalec                 v    t        |        t        |       t        j                  | |||||||||	|
||||      S )a  Visualize documents and their topics in 2D at different levels of hierarchy.

        Arguments:
            docs: The documents you used when calling either `fit` or `fit_transform`
            hierarchical_topics: A dataframe that contains a hierarchy of topics
                                represented by their parents and their children
            topics: A selection of topics to visualize.
                    Not to be confused with the topics that you get from `.fit_transform`.
                    For example, if you want to visualize only topics 1 through 5:
                    `topics = [1, 2, 3, 4, 5]`.
            embeddings: The embeddings of all documents in `docs`.
            reduced_embeddings: The 2D reduced embeddings of all documents in `docs`.
            sample: The percentage of documents in each topic that you would like to keep.
                    Value can be between 0 and 1. Setting this value to, for example,
                    0.1 (10% of documents in each topic) makes it easier to visualize
                    millions of documents as a subset is chosen.
            hide_annotations: Hide the names of the traces on top of each cluster.
            hide_document_hover: Hide the content of the documents when hovering over
                                 specific points. Helps to speed up generation of visualizations.
            nr_levels: The number of levels to be visualized in the hierarchy. First, the distances
                       in `hierarchical_topics.Distance` are split in `nr_levels` lists of distances with
                       equal length. Then, for each list of distances, the merged topics, that have
                       a distance less or equal to the maximum distance of the selected list of distances, are selected.
                       NOTE: To get all possible merged steps, make sure that `nr_levels` is equal to
                       the length of `hierarchical_topics`.
            level_scale: Whether to apply a linear or logarithmic ('log') scale levels of the distance
                         vector. Linear scaling will perform an equal number of merges at each level
                         while logarithmic scaling will perform more mergers in earlier levels to
                         provide more resolution at higher levels (this can be used for when the number
                         of topics is large).
            custom_labels: Whether to use custom topic labels that were defined using
                           `topic_model.set_topic_labels`.
                           NOTE: Custom labels are only generated for the original
                           un-merged topics.
            title: Title of the plot.
            width: The width of the figure.
            height: The height of the figure.

        Examples:
        To visualize the topics simply run:

        ```python
        topic_model.visualize_hierarchical_documents(docs, hierarchical_topics)
        ```

        Do note that this re-calculates the embeddings and reduces them to 2D.
        The advised and preferred pipeline for using this function is as follows:

        ```python
        from sklearn.datasets import fetch_20newsgroups
        from sentence_transformers import SentenceTransformer
        from bertopic import BERTopic
        from umap import UMAP

        # Prepare embeddings
        docs = fetch_20newsgroups(subset='all',  remove=('headers', 'footers', 'quotes'))['data']
        sentence_model = SentenceTransformer("all-MiniLM-L6-v2")
        embeddings = sentence_model.encode(docs, show_progress_bar=False)

        # Train BERTopic and extract hierarchical topics
        topic_model = BERTopic().fit(docs, embeddings)
        hierarchical_topics = topic_model.hierarchical_topics(docs)

        # Reduce dimensionality of embeddings, this step is optional
        # reduced_embeddings = UMAP(n_neighbors=10, n_components=2, min_dist=0.0, metric='cosine').fit_transform(embeddings)

        # Run the visualization with the original embeddings
        topic_model.visualize_hierarchical_documents(docs, hierarchical_topics, embeddings=embeddings)

        # Or, if you have reduced the original embeddings already:
        topic_model.visualize_hierarchical_documents(docs, hierarchical_topics, reduced_embeddings=reduced_embeddings)
        ```

        Or if you want to save the resulting figure:

        ```python
        fig = topic_model.visualize_hierarchical_documents(docs, hierarchical_topics, reduced_embeddings=reduced_embeddings)
        fig.write_html("path/to/file.html")
        ```

        <iframe src="../getting_started/visualization/hierarchical_documents.html"
        style="width:1000px; height: 770px; border: 0px;""></iframe>
        )r   rv  r   r   rU  rV  rW  rX  rb  rc  r	  rO  r  rP  )r2   r0   r    visualize_hierarchical_documents)rv   r   rv  r   r   rU  rV  rW  rX  rb  rc  r	  rO  r  rP  s                  rw   re  z)BERTopic.visualize_hierarchical_documents
  sU    H 	T"88 3!1- 3#'
 	
r}   	log_scalec           	      P    t        |        t        j                  | ||||||      S )aO  Visualize the ranks of all terms across all topics.

        Each topic is represented by a set of words. These words, however,
        do not all equally represent the topic. This visualization shows
        how many words are needed to represent a topic and at which point
        the beneficial effect of adding words starts to decline.

        Arguments:
            topics: A selection of topics to visualize. These will be colored
                    red where all others will be colored black.
            log_scale: Whether to represent the ranking on a log scale
            custom_labels: Whether to use custom topic labels that were defined using
                           `topic_model.set_topic_labels`.
            title: Title of the plot.
            width: The width of the figure.
            height: The height of the figure.

        Returns:
            fig: A plotly figure

        Examples:
        To visualize the ranks of all words across
        all topics simply run:

        ```python
        topic_model.visualize_term_rank()
        ```

        Or if you want to save the resulting figure:

        ```python
        fig = topic_model.visualize_term_rank()
        fig.write_html("path/to/file.html")
        ```

        Reference:

        This visualization was heavily inspired by the
        "Term Probability Decline" visualization found in an
        analysis by the amazing [tmtoolkit](https://tmtoolkit.readthedocs.io/).
        Reference to that specific analysis can be found
        [here](https://wzbsocialsciencecenter.github.io/tm_corona/tm_analysis.html).
        )r   rf  r	  rO  r  rP  )r2   r   visualize_term_rank)rv   r   rf  r	  rO  r  rP  s          rw   rh  zBERTopic.visualize_term_rankm  s5    h 	++'
 	
r}   r+  normalize_frequencyc	                 T    t        |        t        j                  | ||||||||	      S )a  Visualize topics over time.

        Arguments:
            topics_over_time: The topics you would like to be visualized with the
                              corresponding topic representation
            top_n_topics: To visualize the most frequent topics instead of all
            topics: Select which topics you would like to be visualized
            normalize_frequency: Whether to normalize each topic's frequency individually
            custom_labels: Whether to use custom topic labels that were defined using
                           `topic_model.set_topic_labels`.
            title: Title of the plot.
            width: The width of the figure.
            height: The height of the figure.

        Returns:
            A plotly.graph_objects.Figure including all traces

        Examples:
        To visualize the topics over time, simply run:

        ```python
        topics_over_time = topic_model.topics_over_time(docs, timestamps)
        topic_model.visualize_topics_over_time(topics_over_time)
        ```

        Or if you want to save the resulting figure:

        ```python
        fig = topic_model.visualize_topics_over_time(topics_over_time)
        fig.write_html("path/to/file.html")
        ```
        )r+  rN  r   ri  r	  rO  r  rP  )r2   r   visualize_topics_over_time)	rv   r+  rN  r   ri  r	  rO  r  rP  s	            rw   rk  z#BERTopic.visualize_topics_over_time  ;    V 	22-% 3'

 
	
r}   r<  c	                 T    t        |        t        j                  | ||||||||	      S )a  Visualize topics per class.

        Arguments:
            topics_per_class: The topics you would like to be visualized with the
                              corresponding topic representation
            top_n_topics: To visualize the most frequent topics instead of all
            topics: Select which topics you would like to be visualized
            normalize_frequency: Whether to normalize each topic's frequency individually
            custom_labels: Whether to use custom topic labels that were defined using
                           `topic_model.set_topic_labels`.
            title: Title of the plot.
            width: The width of the figure.
            height: The height of the figure.

        Returns:
            A plotly.graph_objects.Figure including all traces

        Examples:
        To visualize the topics per class, simply run:

        ```python
        topics_per_class = topic_model.topics_per_class(docs, classes)
        topic_model.visualize_topics_per_class(topics_per_class)
        ```

        Or if you want to save the resulting figure:

        ```python
        fig = topic_model.visualize_topics_per_class(topics_per_class)
        fig.write_html("path/to/file.html")
        ```
        )r<  rN  r   ri  r	  rO  r  rP  )r2   r   visualize_topics_per_class)	rv   r<  rN  r   ri  r	  rO  r  rP  s	            rw   rn  z#BERTopic.visualize_topics_per_class  rl  r}   min_probabilityc           	      P    t        |        t        j                  | ||||||      S )a  Visualize the distribution of topic probabilities.

        Arguments:
            probabilities: An array of probability scores
            min_probability: The minimum probability score to visualize.
                             All others are ignored.
            custom_labels: Whether to use custom topic labels that were defined using
                           `topic_model.set_topic_labels`.
            title: Title of the plot.
            width: The width of the figure.
            height: The height of the figure.

        Examples:
        Make sure to fit the model before and only input the
        probabilities of a single document:

        ```python
        topic_model.visualize_distribution(topic_model.probabilities_[0])
        ```

        Or if you want to save the resulting figure:

        ```python
        fig = topic_model.visualize_distribution(topic_model.probabilities_[0])
        fig.write_html("path/to/file.html")
        ```
        )r   ro  r	  rO  r  rP  )r2   r   visualize_distribution)rv   r   ro  r	  rO  r  rP  s          rw   rq  zBERTopic.visualize_distribution  s5    H 	..'+'
 	
r}   r   r  r   c                 J    t        |        t        j                  | |||      S )a  Visualize the topic distribution calculated by `.approximate_topic_distribution`
        on a token level. Thereby indicating the extent to which a certain word or phrase belongs
        to a specific topic. The assumption here is that a single word can belong to multiple
        similar topics and as such can give information about the broader set of topics within
        a single document.

        Arguments:
            topic_model: A fitted BERTopic instance.
            document: The document for which you want to visualize
                      the approximated topic distribution.
            topic_token_distribution: The topic-token distribution of the document as
                                      extracted by `.approximate_topic_distribution`
            normalize: Whether to normalize, between 0 and 1 (summing up to 1), the
                       topic distribution values.

        Returns:
            df: A stylized dataframe indicating the best fitting topics
                for each token.

        Examples:
        ```python
        # Calculate the topic distributions on a token level
        # Note that we need to have `calculate_token_level=True`
        topic_distr, topic_token_distr = topic_model.approximate_distribution(
                docs, calculate_token_level=True
        )

        # Visualize the approximated topic distributions
        df = topic_model.visualize_approximate_distribution(docs[0], topic_token_distr[0])
        df
        ```

        To revert this stylized dataframe back to a regular dataframe,
        you can run the following:

        ```python
        df.data.columns = [column.strip() for column in df.data.columns]
        df = df.data
        ```
        )r   r  r   )r2   r   "visualize_approximate_distribution)rv   r   r  r   s       rw   rs  z+BERTopic.visualize_approximate_distributionK  s,    \ 	::%=	
 	
r}   orientationcolor_thresholdc                 \    t        |        t        j                  | |||||||||	|
||      S )a4  Visualize a hierarchical structure of the topics.

        A ward linkage function is used to perform the
        hierarchical clustering based on the cosine distance
        matrix between c-TF-IDF or semantic embeddings of the topics.

        Arguments:
            topic_model: A fitted BERTopic instance.
            orientation: The orientation of the figure.
                         Either 'left' or 'bottom'
            topics: A selection of topics to visualize
            top_n_topics: Only select the top n most frequent topics
            use_ctfidf: Whether to calculate distances between topics based on c-TF-IDF embeddings. If False, the
                        embeddings from the embedding model are used.
            custom_labels: Whether to use custom topic labels that were defined using
                           `topic_model.set_topic_labels`.
                           NOTE: Custom labels are only generated for the original
                           un-merged topics.
            title: Title of the plot.
            width: The width of the figure. Only works if orientation is set to 'left'
            height: The height of the figure. Only works if orientation is set to 'bottom'
            hierarchical_topics: A dataframe that contains a hierarchy of topics
                                 represented by their parents and their children.
                                 NOTE: The hierarchical topic names are only visualized
                                 if both `topics` and `top_n_topics` are not set.
            linkage_function: The linkage function to use. Default is:
                              `lambda x: sch.linkage(x, 'ward', optimal_ordering=True)`
                              NOTE: Make sure to use the same `linkage_function` as used
                              in `topic_model.hierarchical_topics`.
            distance_function: The distance function to use on the c-TF-IDF matrix. Default is:
                               `lambda x: 1 - cosine_similarity(x)`
                               NOTE: Make sure to use the same `distance_function` as used
                               in `topic_model.hierarchical_topics`.
            color_threshold: Value at which the separation of clusters will be made which
                             will result in different colors for different clusters.
                             A higher value will typically lead to less colored clusters.

        Returns:
            fig: A plotly figure

        Examples:
        To visualize the hierarchical structure of
        topics simply run:

        ```python
        topic_model.visualize_hierarchy()
        ```

        If you also want the labels of hierarchical topics visualized,
        run the following:

        ```python
        # Extract hierarchical topics and their representations
        hierarchical_topics = topic_model.hierarchical_topics(docs)

        # Visualize these representations
        topic_model.visualize_hierarchy(hierarchical_topics=hierarchical_topics)
        ```

        If you want to save the resulting figure:

        ```python
        fig = topic_model.visualize_hierarchy()
        fig.write_html("path/to/file.html")
        ```
        <iframe src="../getting_started/visualization/hierarchy.html"
        style="width:1000px; height: 680px; border: 0px;""></iframe>
        )rt  r   rN  r?  r	  rO  r  rP  rv  r@  rA  ru  )r2   r   visualize_hierarchy)rv   rt  r   rN  r?  r	  rO  r  rP  rv  r@  rA  ru  s                rw   rw  zBERTopic.visualize_hierarchy  sG    f 	++#%!' 3-/+
 	
r}   
n_clustersc	                 T    t        |        t        j                  | ||||||||	      S )a>  Visualize a heatmap of the topic's similarity matrix.

        Based on the cosine similarity matrix between c-TF-IDFs or semantic embeddings of the topics,
        a heatmap is created showing the similarity between topics.

        Arguments:
            topics: A selection of topics to visualize.
            top_n_topics: Only select the top n most frequent topics.
            n_clusters: Create n clusters and order the similarity
                        matrix by those clusters.
            use_ctfidf: Whether to calculate distances between topics based on c-TF-IDF embeddings. If False, the
                        embeddings from the embedding model are used.
            custom_labels: Whether to use custom topic labels that were defined using
                           `topic_model.set_topic_labels`.
            title: Title of the plot.
            width: The width of the figure.
            height: The height of the figure.

        Returns:
            fig: A plotly figure

        Examples:
        To visualize the similarity matrix of
        topics simply run:

        ```python
        topic_model.visualize_heatmap()
        ```

        Or if you want to save the resulting figure:

        ```python
        fig = topic_model.visualize_heatmap()
        fig.write_html("path/to/file.html")
        ```
        )r   rN  rx  r?  r	  rO  r  rP  )r2   r   visualize_heatmap)	rv   r   rN  rx  r?  r	  rO  r  rP  s	            rw   rz  zBERTopic.visualize_heatmap  s;    ^ 	))%!!'

 
	
r}   n_words	autoscalec	                 T    t        |        t        j                  | ||||||||	      S )a  Visualize a barchart of selected topics.

        Arguments:
            topics: A selection of topics to visualize.
            top_n_topics: Only select the top n most frequent topics.
            n_words: Number of words to show in a topic
            custom_labels: Whether to use custom topic labels that were defined using
                           `topic_model.set_topic_labels`.
            title: Title of the plot.
            width: The width of each figure.
            height: The height of each figure.
            autoscale: Whether to automatically calculate the height of the figures to fit the whole bar text

        Returns:
            fig: A plotly figure

        Examples:
        To visualize the barchart of selected topics
        simply run:

        ```python
        topic_model.visualize_barchart()
        ```

        Or if you want to save the resulting figure:

        ```python
        fig = topic_model.visualize_barchart()
        fig.write_html("path/to/file.html")
        ```
        )r   rN  r{  r	  rO  r  rP  r|  )r2   r   visualize_barchart)	rv   r   rN  r{  r	  rO  r  rP  r|  s	            rw   r~  zBERTopic.visualize_barchart!  s;    T 	**%'

 
	
r}   serialization)safetensorspicklepytorchsave_embedding_modelsave_ctfidfc                    |dk(  rt         j                  d       t        |d      5 }d| j                  _        |s1| j
                  }d| _        t        j                  | |       || _        nt        j                  | |       ddd       y|dk(  s|dk(  r3t        |      }|j                  dd       |r=t        | j
                  d	      r't        |t              s| j
                  j                  }n|st         j                  d
       t        j                  | ||       t        j                   | |dz         t        j"                  | |dz         t        j$                  | |dz  |       |rU| j&                  t         j                  d       yt        j(                  | ||       t        j*                  | |dz         yyy# 1 sw Y   yxY w)a}  Saves the model to the specified path or folder.

        When saving the model, make sure to also keep track of the versions
        of dependencies and Python used. Loading and saving the model should
        be done using the same dependencies and Python. Moreover, models
        saved in one version of BERTopic should not be loaded in other versions.

        Arguments:
            path: If `serialization` is 'safetensors' or `pytorch`, this is a directory.
                  If `serialization` is `pickle`, then this is a file.
            serialization: If `pickle`, the entire model will be pickled. If `safetensors`
                           or `pytorch` the model will be saved without the embedding,
                           dimensionality reduction, and clustering algorithms.
                           This is a very efficient format and typically advised.
            save_embedding_model: If serialization is `pickle`, then you can choose to skip
                                  saving the embedding model. If serialization is `safetensors`
                                  or `pytorch`, this variable can be used as a string pointing
                                  towards a huggingface model.
            save_ctfidf: Whether to save c-TF-IDF information if serialization is `safetensors`
                         or `pytorch`

        Examples:
        To save the model in an efficient and safe format (safetensors) with c-TF-IDF information:

        ```python
        topic_model.save("model_dir", serialization="safetensors", save_ctfidf=True)
        ```

        If you wish to also add a pointer to the embedding model, which will be downloaded from
        HuggingFace upon loading:

        ```python
        embedding_model = "sentence-transformers/all-MiniLM-L6-v2"
        topic_model.save("model_dir", serialization="safetensors", save_embedding_model=embedding_model)
        ```

        or if you want save the full model with pickle:

        ```python
        topic_model.save("my_model")
        ```

        NOTE: Pickle can run arbitrary code and is generally considered to be less safe than
        safetensors.
        r  zWhen you use `pickle` to save/load a BERTopic model,please make sure that the environments in which you saveand load the model are **exactly** the same. The version of BERTopic,its dependencies, and python need to remain the same.wbNr  r  T)exist_okparents	_hf_modelaA  You are saving a BERTopic model without explicitly defining an embedding model.If you are using a sentence-transformers model or a HuggingFace model supportedby sentence-transformers, please save the model by using a pointer towards that model.For example, `save_embedding_model='sentence-transformers/all-mpnet-base-v2'`)modelsave_directoryr  ztopics.json)r  pathr   zconfig.json)r  r  r`   zThe c-TF-IDF matrix could not be saved as it was not found. This typically occurs when merging BERTopic models with `BERTopic.merge_models`.zctfidf_config.json)r^   r_   openrC   stop_words_r`   joblibdumpr   mkdirr   r   r   r  
save_utilssave_hfsave_topicssave_imagessave_configrp   r  save_ctfidf_config)rv   r  r  r  r  filer`   r  s           rw   savezBERTopic.saveX  s   h H$NNH dD! ,T 59%%1+&*&:&:O+/D(KKd++:D(KKd+, , m+}	/I!$ZN  $ = %D00+>"#7='+';';'E'E$)d T.Xef""N]4RS""NX4MN""#m3 4 >>)NNk
 **"'5&3
 11>ThChi ? 0J, ,s   AG

Gr  c           
      >   t        |      }|j                         rat        |d      5 }|r2t        j                  |      }t        ||j                        |_        nt        j                  |      }|cddd       S |j                         rt        j                  |      \  }}}}	}
}n5dt        |      v rt        j                  |      \  }}}}	}
}nt        d      t        ||||	|
||du       }|t        ||j                        |_        |S # 1 sw Y   xY w)ap  Loads the model from the specified path or directory.

        Arguments:
            path: Either load a BERTopic model from a file (`.pickle`) or a folder containing
                  `.safetensors` or `.bin` files.
            embedding_model: Additionally load in an embedding model if it was not saved
                             in the BERTopic model file or directory.

        Examples:
        ```python
        BERTopic.load("model_dir")
        ```

        or if you did not save the embedding model:

        ```python
        BERTopic.load("model_dir", embedding_model="all-MiniLM-L6-v2")
        ```
        rbrA  N/z7Make sure to either pass a valid directory or HF model.warn_no_backend)r   is_filer  r  loadr(   rF   r`   is_dirr  load_local_filesr   load_files_from_hfr   _create_model_from_files)clsr  r`   file_or_dirr  r  r   paramstensorsctfidf_tensorsctfidf_configr   s               rw   r  zBERTopic.load  s#   * 4j  k4( #D""(++d"3K2@ZeZmZm2nK/"(++d"3K"# # MWMhMhitMuJFFG^]FCIMWMjMjkoMpJFFG^]FVWW.,4
 &*8R]ReRe*fK';# #s   ADDc           
      \   d }t               5 }g g g }}}t        |      D ]  \  }	}
|
j                  | |              t        j                  t        |            \  }}}}}}|j                  |       |j                  |       |j                  t        j                  |d                |	dk(  s|}|}t        j                  |d         }d|d<    	 ddd       t        dd dd       D ]  \  }}t        |      }t        j                  |d      }t        t        |      D 	cg c]  \  }	}||k  s|	|d	   z
   c}}	      }t        t        d
               }i }|D ])  }|dk7  s
|dz  }|||<   |d   t        |         |d   t        |      <   |d   t        |         |d   t        |      <   |d   rt        |d   j                               }t        |d   j                               }|j!                  |      }|r|D ]
  }i |d   |<    |j#                  d      s	|d   |d<   n;|d   j%                         D ]%  \  }}|t        |         |d   |   t        |      <   ' |||d	   z      }t        j&                  ||g      }, t)        t+        t-        d|dz   d                  j.                  |d<   t        j0                  |d      } t        j                  |d      }t        t        | |            D !"#ci c]  \  }!\  }"}#|#|k\  r|!|d	   z
  |"|d	   z
   }$}"}!}#|$j3                  |       d|$d<   |d
   D %cg c]  }%|$|%   	 }}%|d
   j5                  |       t7        t9        |d
               |d<    di}t;        |dddd      }&|d   j<                  |&_        t?        |D 
cg c]  }
|
j@                   c}
      }'|-tC        |&j<                        tD        u rtG        ||'      |&_        |&S # 1 sw Y   6xY wc c}}	w c c}#}"}!w c c}%w c c}
w )a  Merge multiple pre-trained BERTopic models into a single model.

        The models are merged as if they were all saved using pytorch or
        safetensors, so a minimal version without c-TF-IDF.

        To do this, we choose the first model in the list of
        models as a baseline. Then, we check each model whether
        they contain topics that are not in the baseline.
        This check is based on the cosine similarity between
        topics embeddings. If topic embeddings between two models
        are similar, then the topic of the second model is re-assigned
        to the first. If they are dissimilar, the topic of the second
        model is assigned to the first.

        In essence, we simply check whether sufficiently "new"
        topics emerge and add them.

        Arguments:
            models: A list of fitted BERTopic models
            min_similarity: The minimum similarity for when topics are merged.
            embedding_model: Additionally load in an embedding model if necessary.

        Returns:
            A new BERTopic model that was created as if you were
            loading a model from the HuggingFace Hub without c-TF-IDF

        Examples:
        ```python
        from bertopic import BERTopic
        from sklearn.datasets import fetch_20newsgroups

        docs = fetch_20newsgroups(subset='all',  remove=('headers', 'footers', 'quotes'))['data']

        # Create three separate models
        topic_model_1 = BERTopic(min_topic_size=5).fit(docs[:4000])
        topic_model_2 = BERTopic(min_topic_size=5).fit(docs[4000:8000])
        topic_model_3 = BERTopic(min_topic_size=5).fit(docs[8000:])

        # Combine all models into one
        merged_model = BERTopic.merge_models([topic_model_1, topic_model_2, topic_model_3])
        ```
        c                  ~    	 ddl } y# t        t        f$ r& 	 ddl}Y y# t        t        f$ r t        d      w xY ww xY w)z/Choose the backend to use for saving the model.r   Nr  r  zNeither pytorch nor safetensors is installed. Please install at least one of these packages:
  pip install torch
  pip install safetensors)torchrd   rc   r  )r  r  s     rw   choose_backendz-BERTopic.merge_models.<locals>.choose_backend2  sR     '5 
&(+[9 %4 s    <8<)r  topic_embeddingsr   Nr	  rz   r   r|   r   rZ   r  r   topic_aspectstopic_mapperr  Fr  rA  )$r
   r   r  r  r  r   r  r   r   r  r"   r   r  r   r   r   r   r   r   r  r   r  r   	mappings_r   r   r&  r  r   r  r`   r%  rF   r   r&   r(   )(r  modelsry  r`   r  tmpdirr)  
all_paramsall_tensorsr   r  r   r  r  r   merged_topicsmerged_paramsmerged_tensorsr'  r   r  r  r   	max_topicnew_topics_dictr  	aspects_1	aspects_2aspects_diffr  r   new_tensorssims_idxabrk  to_merger   merged_modelrF   s(                                           rw   merge_modelszBERTopic.merge_models  s   Z	(  ! 	:V24b"K
J )& 1 :u

61A
B3=3N3NtTZ|3\0Aq!!&)!!&)""288G4F,G#HI A:$*M$*M%'XXg6H.I%JN59M/2:	:" ),KOZ^(L ;	R$G_*7NCJ66*1-D  HQRVp*%[^ao[o55pJ Ch 789I !O' N	?NI1:OI.M\]tMuINM"9:3y>J ETTbDcdghqdrDsM.1#i.A '7$'o(F(K(K(M$N	$'(H(M(M(O$P	'0';';I'F'*6 LIKo >v FL  -00A=L_=]M/: 3B/2R2X2X2Z pY_`cdm`nYoo >v Fs9~ Vp #*)ok6R*R"SK%'YY/L%MN;N@ -8U2yST}VW=X8Y,Z,d,dM.) yy!4H66*1-D $-S4-@#A Ax3.( OK00!mK6P2PPH 
 OOO,HRL3B83LM%huoMFM(#**62+/h8O0P+QM-(w;	R| -n=/!
 (.ay'@'@$ &9u}}9:&40L0L+MQ]+]+9/SZ+[L(	: 	:0 qV N$ :s0   B#P
2#P
-P;
P!P
P$?P)
Prepo_idcommit_messager  revisionprivate	create_pr
model_cardc                 B    t        j                  | |||||||||	|
      S )a  Push your BERTopic model to a HuggingFace Hub.

        Whenever you want to upload files to the Hub, you need to log in to your HuggingFace account:

        * Log in to your HuggingFace account with the following command:
            ```bash
            huggingface-cli login

            # or using an environment variable
            huggingface-cli login --token $HUGGINGFACE_TOKEN
            ```
        * Alternatively, you can programmatically login using login() in a notebook or a script:
            ```python
            from huggingface_hub import login
            login()
            ```
        * Or you can give a token with the `token` variable

        Arguments:
            repo_id: The name of your HuggingFace repository
            commit_message: A commit message
            token: Token to add if not already logged in
            revision: Repository revision
            private: Whether to create a private repository
            create_pr: Whether to upload the model as a Pull Request
            model_card: Whether to automatically create a modelcard
            serialization: The type of serialization.
                           Either `safetensors` or `pytorch`
            save_embedding_model: A pointer towards a HuggingFace model to be loaded in with
                                  SentenceTransformers. E.g.,
                                  `sentence-transformers/all-MiniLM-L6-v2`
            save_ctfidf: Whether to save c-TF-IDF information


        Examples:
        ```python
        topic_model.push_to_hf_hub(
            repo_id="ArXiv",
            save_ctfidf=True,
            save_embedding_model="sentence-transformers/all-MiniLM-L6-v2"
        )
        ```
        )r  r  r  r  r  r  r  r  r  r  r  )r  push_to_hf_hub)rv   r  r  r  r  r  r  r  r  r  r  s              rw   r  zBERTopic.push_to_hf_hub  s9    p (()!'!5#
 	
r}   deepc                     t               }| j                         D ]Y  t        |       }|rDt        |d      r8|j	                         j                         }|j                  fd|D               ||<   [ |S )a  Get parameters for this estimator.

        Adapted from:
            https://github.com/scikit-learn/scikit-learn/blob/b3ea3ed6a/sklearn/base.py#L178

        Arguments:
            deep: bool, default=True
                  If True, will return the parameters for this estimator and
                  contained subobjects that are estimators.

        Returns:
            out: Parameter names mapped to their values.
        
get_paramsc              3   8   K   | ]  \  }}d z   |z   |f  yw)__Nr   )r!  krk  r   s      rw   r"  z&BERTopic.get_params.<locals>.<genexpr>   s"     JVQC$JNC0Js   )r  _get_param_namesr(  r   r  r   r   )rv   r  outr  
deep_itemsr   s        @rw   r  zBERTopic.get_params  su     f((* 	CD#&E|4"--/557


JzJJCH	 
r}   r   c                 b   t        |t              r|g}|6t        | j                  d      r | j                  j	                  |||      }|S |dk(  r| j                  j                  ||      }|S |dk(  r| j                  j                  ||      }|S |d   |t        d      t        d	      )
am  Extract sentence/document embeddings through pre-trained embeddings
        For an overview of pre-trained models: https://www.sbert.net/docs/pretrained_models.html.

        Arguments:
            documents: Dataframe with documents and their corresponding IDs
            images: A list of paths to the images to fit on or the images themselves
            method: Whether to extract document or word-embeddings, options are "document" and "word"
            verbose: Whether to show a progressbar demonstrating the time to extract embeddings

        Returns:
            embeddings: The extracted embeddings.
        rD  )r   r   rF   r   )r/  rF   r   rA  r   zrMake sure to use an embedding model that can either embed documentsor images depending on which you want to embed.zhWrong method for extracting document/word embeddings. Either choose 'word' or 'document' as the method. )r   r   r   r`   embedembed_wordsrE  r   )rv   r   r   r   rF   r   s         rw   r   zBERTopic._extract_embeddings  s    & i%"I'$*>*>"O--33iPV`g3hJ  v--99	SZ9[J  z!--==iQX=YJ  q\!fnB 
 E r}   c                 @   t         j                  d       t        | j                  t              r@| j                  j                         D ]"  }t        |dd      s|j                  ||      }$ nt        | j                  t              r2| j                  D ]"  }t        |dd      s|j                  ||      }$ nMt        | j                  t              r3t        | j                  dd      r| j                  j                  ||      }t         j                  d       |S )zConvert images to text.z<Images - Converting images to text. This might take a while.image_to_text_modelFu   Images - Completed ✓)
r^   r   r   rE   r  r   r(  image_to_textr  r*   )rv   r   r   tuners       rw   r   zBERTopic._images_to_text,  s    RSd//62299; K5"7? % 3 3Iz JIK 114822 K5"7? % 3 3Iz JIK 113EFt002GO 55CCIzZ	/0r}   r   c                 x    | j                   j                  d      }|D cg c]  }||v r||   nd }}|S c c}w )z=Map predictions to the correct topics if topics were reduced.Tr   rZ   )rk   r   )rv   r   r   
predictionmapped_predictionss        rw   r   zBERTopic._map_predictions=  sN    %%2242HgrsYcjH6Lhz2RTTss!! ts   7r   c                 N   t         j                  d       |rt        | j                  d      r=| j                  j	                  |      | _        | j                  j                  |      }n| j                  it        | j                  d      r| j                  j                  |      }n| j                  j                  |       | j                  j                  |      }n| j                  j                  |      }n	 |t        j                  |      nd}t        | j                  d      r| j                  j                  ||      }n8| j                  j                  ||       | j                  j                  |      }t         j                  d       t        j                  |      S # t        $ ro t        | j                  d      r| j                  j                  ||      }n8| j                  j                  ||       | j                  j                  |      }Y w xY w)a  Reduce dimensionality of embeddings using UMAP and train a UMAP model.

        Arguments:
            embeddings: The extracted embeddings using the sentence transformer module.
            y: The target class for (semi)-supervised dimensionality reduction
            partial_fit: Whether to run `partial_fit` for online learning

        Returns:
            umap_embeddings: The reduced embeddings
        z?Dimensionality - Fitting the dimensionality reduction algorithmr   Nr   r   r   )r^   r   r   ra   r   r   rl   r   r   r   r   	TypeError
nan_to_num)rv   r   r   r   r   s        rw   r   zBERTopic._reduce_dimensionalityC  s     	UVt6"&//"="=j"I"&//";";J"G,,44??O<&*oo&C&CJ&OOOO''
3&*oo&?&?
&KO"&//";";J"GL#$=BHHQKd4??O<&*oo&C&CJRS&C&TOOO''
a'8&*oo&?&?
&KO 	78}}_--  L4??O<&*oo&C&CJRS&C&TOOO''
a'8&*oo&?&?
&KOLs   =BF, ,A5H$#H$r   c                    t         j                  d       |rC| j                  j                  |      | _        | j                  j                  }||d<   || _        nK	 | j                  j                  ||       	 | j                  j                  }||d<   | j                  |       d}t        | j                  d      rM| j                  j                  }| j                  r+t        | j                        rt        | j                  d      }|st        | j
                        | _        t         j                  d       ||fS # t        $ r | j                  j                  |       Y w xY w# t        $ r |}Y w xY w)a  Cluster UMAP reduced embeddings with HDBSCAN.

        Arguments:
            umap_embeddings: The reduced sentence embeddings with UMAP
            documents: Dataframe with documents and their corresponding IDs
            partial_fit: Whether to run `partial_fit` for online learning
            y: Array of topics to use

        Returns:
            documents: Updated dataframe with documents and their corresponding IDs
                       and newly added Topics
            probabilities: The distribution of probabilities
        z1Cluster - Start clustering the reduced embeddingsr   r   Nri   all_points_membership_vectorsr   )r^   r   re   r   labels_rh   r   r  AttributeErrorr   r   ri   r?   r.   r-   r   rk   )rv   r   r   r   r   r  r   s          rw   r   zBERTopic._cluster_embeddingsv  sV   ( 	GH!%!3!3!?!?!PD''//F!'Ig!DL8""&&!&<++33 "(Ig##I. 4%%'78 ..==M++0DTEWEW0X 1$2D2DFe f!,T\\!:D01-''+  8""&&78
 " s$   E  :E*  $E'&E'*E87E8c           	         t         j                  d       | j                  | j                        }t	        ||      }t        j                  |d      }t        j                  |d      }t        |      D cg c]  \  }}|| j                  k\  s| }	}}t        |      D cg c]  \  }}|| j                  k  s| }
}}|j                  |	   }||	   D cg c]  }| c}|d<   |d   j                         |d<   t        t        |            |d<   ||	   }t        |d   j                               }| j                  dk7  rD| j                  r8| j                  |kD  s)t!        d| j                   d| d	| j                   d
      |j                  |
   }|d   j                         |d<   t        t        |            |d<   ||
   }t        |      dk(  r<|d   j"                  j%                         | _        t)        | j&                        | _        t         j                  d       ||||fS c c}}w c c}}w c c}w )a  Find documents that could be assigned to either one of the topics in self.zeroshot_topic_list.

        We transform the topics in `self.zeroshot_topic_list` to embeddings and
        compare them through cosine similarity with the document embeddings.
        If they pass the `self.zeroshot_min_similarity` threshold, they are assigned.

        Arguments:
            documents: Dataframe with documents and their corresponding IDs
            embeddings: The document embeddings

        Returns:
            documents: The leftover documents that were not assigned to any topic
            embeddings: The leftover embeddings that were not assigned to any topic
        z`Zeroshot Step 1 - Finding documents that could be assigned to either one of the zero-shot topicsrz   r   r   Old_IDautozThe set nr_topics (z6) must exceed the number of matched zero-shot topics (zF). Consider raising nr_topics or raising the zeroshot_min_similarity ().r   u   Zeroshot Step 1 - Completed ✓)r^   r   r   rA   r"   r   r   r   r   rB   r   r  r   r   r  r=   r   r   r   rh   r   rk   )rv   r   r   zeroshot_embeddingscosine_similarities
assignmentassignment_valsr   r  assigned_idsnon_assigned_idsr   r   r   num_zeroshot_topicss                  rw   r   z!BERTopic._zeroshot_topic_modeling  s?   " 	vw"66t7O7OP/
<OPYY2A6
&&!4a82;O2Lv,%PUY]YuYuPuvv&/&@
"eUEDLhLhDhE
 

 '^^L9:D\:R&Su&S7#'9$'?'D'D'F8$#(-?)@#A4 (6 ""4W"="D"D"FG>>V#~~dnn7J&J )$..)9 :+, -0040L0L/MRQ  NN#34	'o224	(I/	$ 01
y>Q-g6==DDFDL!,T\\!:D89*&8:MMMC w
 'Ts   7H/H/$H5<H5	H;c                 6    | j                   | j                  yy)zCheck whether zero-shot topic modeling is possible.

        * Embedding model is necessary to convert zero-shot topics to embeddings
        * Zero-shot topics should be defined
        TF)rA   r`   r{   s    rw   r   zBERTopic._is_zeroshot  s!     ##/D4H4H4Tr}   r   r   c                   	 t         j                  d       t        t        |j                              D ci c]  \  }}||
 }}}t        t        |j                              D ci c]  \  }}||
 c}}| _        |j                  j                  |      |_        t        |      	|j                  j                  	fd      |_        t        j                  ||g      }t        j                  ||g      }|j                  |_        |j                  j                         }|j                   |   }||   }| j#                  |       t%        | j&                        | _        t         j                  d       ||fS c c}}w c c}}w )a)  Combine the zero-shot topics with the clustered topics.

        The zero-shot topics will be inserted between the outlier topic (that may or may not exist) and the rest of the
        topics from clustering. The rest of the topics from clustering will be given new IDs to correspond to topics
        after zero-shot topics.

        Documents and embeddings used in zero-shot topic modeling and clustering and re-merged.

        Arguments:
            documents: DataFrame with clustered documents and their corresponding IDs
            embeddings: The document embeddings for clustered documents
            assigned_documents: DataFrame with documents and their corresponding IDs
                                that were assigned to a zero-shot topic
            assigned_embeddings: The document embeddings for documents that were assigned to a zero-shot topic

        Returns:
            documents: DataFrame with all the original documents with their topic assignments
            embeddings: np.ndarray of embeddings aligned with the documents
        z_Zeroshot Step 2 - Combining topics from zero-shot topic modeling with topics from clustering...c                     | dk7  r| z   S | S )NrZ   r   )r   r  s    rw   r  z3BERTopic._combine_zeroshot_topics.<locals>.<lambda>  s    x2~X(;; S[ r}   u   Zeroshot Step 2 - Completed ✓)r^   r   r   r   r   rn   r  r   r!  r   concatr   r  r  r   r  r   r   r   rh   rk   )
rv   r   r   r   r   new_topic_idzeroshot_topic_idzeroshot_topic_idx_to_topic_idsorted_indicesr  s
            @rw   r   z!BERTopic._combine_zeroshot_topics  sp   4 	uv 4=SASAYAY=Z3[*
// |+*
& *
 4=SASAYAY=Z3[0
// ++0
, $6#;#;#?#?@^#_ !"@A $////[
	
 IIy*<=>	YY
,?@A
 ''	"))113NN>2	/
 		*(689*$$;*
0
s   E>)Fc                    t         j                  d       | j                  D cg c]  }dj                  |       }}| j	                  || j
                        }t        j                  ||j                  d      g      }t        ||      }t        |j                  d         D cg c]  }t        j                  ||          }}|D cg c]  }|t        |      k7  r|nd }}t        t        |            D ]9  }t        |      D 	cg c]  \  }}	|	|k(  s| }
}}	||
   dz  ||   dz  z   ||
<   ; t         j                  d	       ||fS c c}w c c}w c c}w c c}	}w )
a  Apply Guided Topic Modeling.

        We transform the seeded topics to embeddings using the
        same embedder as used for generating document embeddings.

        Then, we apply cosine similarity between the embeddings
        and set labels for documents that are more similar to
        one of the topics than the average document.

        If a document is more similar to the average document
        than any of the topics, it gets the -1 label and is
        thereby not included in UMAP.

        Arguments:
            embeddings: The document embeddings

        Returns:
            y: The labels for each seeded topic
            embeddings: Updated embeddings
        z9Guided - Find embeddings highly related to seeded topics.r   rA  r   r   rZ   g      ?g      ?u   Guided - Completed ✓)r^   r   r@   r   r   rF   r   r  meanr"   r   rZ  r   r   r   )rv   r   
seed_topicr@   seed_topic_embeddingsr   r   r   rk  r   indicess              rw   r   zBERTopic._guided_topic_modeling)  sh   * 	OPBFBVBVWJ388J/WW $ 8 8RVR^R^ 8 _ "		+@*//WX/BY*Z [ 'z3HI
7<Z=M=Ma=P7QReRYYz%()RRCDECC3//SR7EE  O 45 	hJ1:1U*ATuUGU",W"5"<?TU_?`cg?g"gJw	h 	/0*} X SE
 Vs   E(EEE E r   c                 v   |r(|rdnd}|rdnd}t         j                  d| d| d       |j                  dgd	
      j                  ddj                  i      }| j                  |      \  | _        }	| j                  |	||||      | _        | j                  |||       |rt         j                  d       yy)a  Extract topics from the clusters using a class-based TF-IDF.

        Arguments:
            documents: Dataframe with documents and their corresponding IDs
            embeddings: The document embeddings
            mappings: The mappings from topic to word
            verbose: Whether to log the process of extracting topics
            fine_tune_representation: If True, the topic representation will be fine-tuned using representation models.
                                      If False, the topic representation will remain as the base c-TF-IDF representation.

        Returns:
            c_tf_idf: The resulting matrix giving a value (importance score) for each word per topic
        zFine-tuning
Extractingzrepresentation modelszc-TF-IDF for topic reductionzRepresentation - z topics using .r   Fr   r   r   )r   r   r   )r   r   r   u   Representation - Completed ✓N)
r^   r   r   r   r   r   rp   r   rl   r   )
rv   r   r   r   rF   r   actionr   r   r/  s
             rw   r   zBERTopic._extract_topicsQ  s    * &>]LF0H,NlFKK+F8>&KL'//	E/JNNPZ\_\d\dOef $/B C&*&C&C%=6! 'D '
# 	""Y:X`"aKK;< r}   c                 p    | j                  | j                  || j                  dd      \  }}}}|| _        y)zSave the 3 most representative docs per topic.

        Arguments:
            documents: Dataframe with documents and their corresponding IDs

        Updates:
            self.representative_docs_: Populate each topic with 3 representative docs
             )
nr_samplesnr_repr_docsN)_extract_representative_docsrp   rl   rr   )rv   r   r  r   s       rw   r   z"BERTopic._save_representative_docsy  sE     ">>NN'' ? 
	1a %.!r}   r.  r  r  	diversityc           	         |j                  ddd      j                  d      j                  |dd      j                         }g }g }	i }
g }t	        t        |j                                     }t        |      D ]  \  }}|j                  |j                  |k(  d	d	f   }|d
   j                  }|j                  j                         }t        |      |kD  r|n
t        |      }| j                  j                  |      }| j                   j                  |      }t#        |||         }|rt%        ||   ||||      }n@t'        j(                  |j+                  dd      d   |       | d	 }|D cg c]  }||   	 }}t        |      D cg c]  \  }}||v s||    }}}|j-                  |       |j/                  |       |	j-                  t1        |      D cg c]  }dk7  r|	d   d   |z   dz   n| c}        t3        |j                         |	      D ci c]  \  }}|||d   |d   dz     }
}}|
||	|fS c c}w c c}}w c c}w c c}}w )a  Approximate most representative documents per topic by sampling
        a subset of the documents in each topic and calculating which are
        most representative to their topic based on the cosine similarity between
        c-TF-IDF representations.

        Arguments:
            c_tf_idf: The topic c-TF-IDF representation
            documents: All input documents
            topics: The candidate topics as calculated with c-TF-IDF
            nr_samples: The number of candidate documents to extract per topic
            nr_repr_docs: The number of representative documents to extract per topic
            diversity: The diversity between the most representative documents.
                       If None, no MMR is used. Otherwise, accepts values between 0 and 1.

        Returns:
            repr_docs_mappings: A dictionary from topic to representative documents
            representative_docs: A flat list of representative documents
            repr_doc_indices: Ordered indices of representative documents
                              that belong to each topic
            repr_doc_ids: The indices of representative documents
                          that belong to each topic
        r   rz   r   )r   errorsr   T*   )nreplacerandom_stateNr   )r  r
  rZ   r   )r  r   rV  drop_duplicatesr  r  r   r   r   r   r   r   r   r   rC   r   rD   r"   r'   r   argpartitionr  r  r&  r   r  )rv   r.  r   r   r  r  r
  r   r  repr_docs_indicesrepr_docs_mappingsrepr_docs_idsr  r   r   r-  selected_docsselected_docs_idsnr_docsrg  ctfidfr   r   r  docr   rl  s                              rw   r	  z%BERTopic._extract_representative_docs  s   B NN78N<WWVj$RV@_	 	 	V[[]+,%f- 	wLE5+//0C0I0IU0RTU0UVI%j188M ) 6 6 8 '*-&8<&GlSQ^M_G''11-@C&&005F*68E?CJ UO!!' //**<*<Q*CA*FQSZRZR[\:ABe,BBBKMBZjJE3^aei^i(/jGj  )T"$$fklsft%uab5TU:&7&;B&?!&Ca&G[\&\%uv=	w> RUU[U`U`UbduQvwXUAeYqtaeai%@@ww!9.?NN Cj &vws   1IIII
!Ic                 ~   ||g }|j                  d      j                  j                         }|d   j                  }|d   j                  j	                  t
              }|D ].  }||k(  }	|j                  |||	      j                  d             0 t        j                  |      | _
        y| j                  ?|<i }
|j                         D ]q  \  }}|d   }|d   }|st        j                  | j                        t        j                  |      | j                  z      }t        j                  |d|      }||
|<   s t        j                  | j                  j                        ddd	df   D ci c]  }|d   |d
    }}i }|
j                         D ]  \  }}||||   <    t!        t#        |j%                                     }t        j                  |D cg c]  }||   	 c}      | _
        y| j&                  t)        | j&                        t*        urzt#        | j,                  j%                               }|j/                          t1        | j,                  |d            }| j2                  |k  r| j2                  }|D cg c]  }| j5                  |       }}|D cg c]  }|D ]  }|d   	  }}}| j7                  |dd      }g }t9        |      D ]  \  }}| j5                  |      D cg c]  }|d
   	 }}t;        |      dk(  r0t=        t1        | j5                  |                  D cg c]  }d
 }}t        j                  |||z  |||z  z    |d      }|j                  |        t        j                  |      | _
        yyyc c}w c c}w c c}w c c}}w c c}w c c}w )a  Creates embeddings per topics based on their topic representation.

        As a default, topic vectors (topic embeddings) are created by taking
        the average of all document embeddings within a topic. If topics are
        merged, then a weighted average of topic embeddings is taken based on
        the initial topic sizes.

        For the `.partial_fit` and `.update_topics` method, the average
        of all document embeddings is not taken since those are not known.
        Instead, the weighted average of the embeddings of the top n words
        is taken for each topic. The weighting is done based on the c-TF-IDF
        score. This will put more emphasis to words that represent a topic best.
        Nr   r   r   r   r  r  )r   weightsrz   r   Fr   )r  r   )r   r   r  r   r   r   r  r  r   r   rm   r   r|   averagerk   r  r  r  r   r`   r   r&   rl   r  r   r:   rc  r   r   ra  r   )rv   r   r   r   r  r   	topic_idsr   r   r7  topic_embeddings_dictr  r  r  embdstopic_embeddingtopic_mappingtopics_to_mapr  r  r  topic_wordsr   word_embeddingsrl  rk  word_importancer   s                               rw   r   zBERTopic._create_topic_vectors  s   ( !i&;!**7399@@BF!'*11Io,,33C8G P E) ''
74=(A(F(FA(F(NOP &(XX.>%?D" ##/H4H$&!)1)9 F%+'6	)-8HHT%;%;<RXXi=PSWSaSa=abE&(jjQ&TO6E)(3F IKQUQcQcQmQmHnoprtruouHv7Da -"22M   " 5 ; ; = ?u9> u!56?"4(8(=(=(?#@AM%'XXTa.b5/?/F.b%cD" !!-$t7K7K2LT`2`d99>>@AJOO D//
1>?A!#$$ ?IIU4>>%0IKI0;NuN47N7NKN"66{6[`6aO
  "%j1 	9559^^E5J"Kc3q6"K"K'1,27DNN5<Q8R2S&TQq&TO&T"$**#AEAQK8+#
 !''8	9 &(XX.>%?D"; 3a- /c JN #L&Ts$   9N "N%N*0N/ N5	N:r   r   c                    | j                  |j                  j                        }|r+| j                  j	                  |      j                  |      }n9|r| j                  j                  |      }n| j                  j                  |      }t        j                  t              t        j                  d      k\  r| j                  j                         }n| j                  j                         }d}| j                  j                  r| j                  r| j                  D 	cg c]  }|D ]  }	|	  }
}}	t!        j"                  |D cg c]2  }|| j                  j                  v r| j                  j$                  nd4 c}      }t!        j"                  t'        ||      D cg c]  \  }}||
v rdn| c}}      }n| j                  j                  rRt!        j"                  |D cg c]2  }|| j                  j                  v r| j                  j$                  nd4 c}      }nW| j                  rK| j                  D 	cg c]  }|D ]  }	|	  }
}}	t!        j"                  |D cg c]
  }||
v rdnd c}      }|r"| j                  j)                  ||      | _        | j                  j                  |      }||fS c c}	}w c c}w c c}}w c c}w c c}	}w c c}w )a  Calculate a class-based TF-IDF where m is the number of total documents.

        Arguments:
            documents_per_topic: The joined documents per topic such that each topic has a single
                                 string made out of multiple documents
            m: The total number of documents (unjoined)
            fit: Whether to fit a new vectorizer or use the fitted self.vectorizer_model
            partial_fit: Whether to run `partial_fit` for online learning

        Returns:
            tf_idf: The resulting matrix giving a value (importance score) for each word per topic
            words: The names of the words to which values were given
        rL  Nrz   g333333?)
multiplier)r[  r   r   rC   r   
update_bowr   r   r	   r\  r]  r^  r_  rD   
seed_wordsr@   r   r   seed_multiplierr  r   )rv   r   r   r   r   rd  r/  r)  seedsseedr@   r   r  r.  s                 rw   r   zBERTopic._c_tf_idf.  sx   & ))*=*F*F*M*MN	%%11)<GG	RA%%33I>A%%//	:A ==)W]]7-CC))??AE));;=E
''D,@,@151E1EXRWX$tXtXOXnstfjdd>O>O>Z>Z6Z""22`aatJ `cdnpu`v"wQ\QVX\$/*A3u#L"wxJ))nstfjdd>O>O>Z>Z6Z""22`aatJ !!151E1EXRWX$tXtXOXUZ"[T$/*A3q#H"[\J $ 1 1 5 5aJ 5 OD$$..q1% Yt"w u Y"[s$   #K	7K%K
)7KK)Kc                     t        j                  |j                  j                  j	                               | _        |j                  j                  t              j	                         | _        y)zCalculate the topic sizes.

        Arguments:
            documents: Updated dataframe with documents and their corresponding IDs and newly added Topics
        N)	collectionsr   r   r   r   rj   r   r   rh   )rv   r   s     rw   r   zBERTopic._update_topic_sizeg  sI     (//	0F0F0M0M0OP --c299;r}   r/  r   c                 	   || j                   }t        t        |j                  j	                                     }|D cg c]  }t        |       }}t        | j                  d      }	| j                  ||	      }
| j                  ||
      }t        j                  |d      }t        j                  |
|d      }
t        j                  ||d      }t        |      D ci c]E  \  }}|t        |
|   ddd   ||   ddd         D cg c]  \  }}||dkD  r||   |fnd c}}G }}}}}|j                         }| j                   r|s2|j#                         D ci c]  \  }}||d| j                    }}}n|rAt%        | j                   t              r'| j                   D ]  }|j'                  | |||      } n|r;t%        | j                   t(              r!| j                   j'                  | ||||      }nR|r:t%        | j                   t*              r | j                   j'                  | |||      }n|rt%        | j                   t,              r| j                   j/                  d      r| j                   d   }t%        |t*              r|j'                  | |||      }nt%        |t              r|D ]  }|j'                  | |||      } nt1        d	t3        |      j4                   d
      |j#                         D ci c]  \  }}||d| j                    }}}n,t1        d	t3        | j                         j4                   d      |r	t%        | j                   t,              r| j                   j#                         D ]  \  }}|dk7  s|j                         }|s0|j#                         D ci c]  \  }}||d| j                    }}}t%        |t              r|D ]  }|j'                  | |||      } nJt%        |t*              r|j'                  | |||      }n%t1        d	t3        |      j4                   d|d      || j6                  |<    |S c c}w c c}}w c c}}}}w c c}}w c c}}w c c}}w )a\  Based on tf_idf scores per topic, extract the top n words per topic.

        If the top words per topic need to be extracted, then only the `words` parameter
        needs to be passed. If the top words per topic in a specific timestamp, then it
        is important to pass the timestamp-based c-TF-IDF matrix and its corresponding
        labels.

        Arguments:
            words: List of all words (sorted according to tf_idf matrix position)
            documents: DataFrame with documents and their topic IDs
            c_tf_idf: A c-TF-IDF matrix from which to calculate the top words
            fine_tune_representation: If True, the topic representation will be fine-tuned using representation models.
                                      If False, the topic representation will remain as the base c-TF-IDF representation.
            calculate_aspects: Whether to calculate additional topic aspects
            embeddings: Pre-trained document embeddings. These can be used
                        instead of an embedding model

        Returns:
            topics: The top words per topic
        N   rz   r   rZ   r   r#  r  zunsupported type z! for representation_model['Main']z for representation_modelz for representation_model[])rp   r  r  r   r  r   r   r:   _top_n_idx_sparse_top_n_values_sparser   r  take_along_axisr   r  r  rE   r   r   extract_topicsr+   r*   r  r   r  r   __name__rs   )rv   r/  r   r.  r   r   r   r  r  r:   r  scoresr  r   
word_indexscorebase_topicsr   r   r  
main_modelr  aspect_modelr  s                           rw   r   z!BERTopic._extract_words_per_topicp  s   : ~~HY__33567*01#e*11 $**B/((;?**8W=FA.$$Wn1E##FNC !*& 1
 

 u	 ),WU^DbD-A6%=QUSUQUCV)W%J /9.DQRz"E*Xee 
 
 !!#((0HMS\\^\ME6eV$6d&6&677\F\%*T5N5NPT*U22 Q--dIxPQ%*T5N5NP_*`..==dIxY_aklF%*T5N5NPb*c..==dIxY_`F%*T5N5NPT*U((,,V4!66v>
j*<='66tYRXYF
D1!+ Y!&!5!5dIxQW!XY $&7Z8H8Q8Q7RRs$tuu RXQ]Q]Q_`v%(:$*:*:!;;``/T5N5N0O0X0X/YYrstt D,E,Et!L(,(A(A(G(G(I :$V#)..0G'ZaZgZgZi"jPV5&1C43C3C*D#D"j"j!,5%1 _E&+&:&:4HV]&^G_#L2DE"."="=dIxY`"a'/\0B0K0K/LLfgmfppqr  3:D''/:" E 2
 ]( a #ks/   Q"#*Q-
Q''Q-
#Q5
Q;<R'Q-
c           	      f   t         j                  d       t        | j                               }t	        | j
                  t              r~| j
                  |k  r| j                  ||      }nt         j                  d| j
                   dt        | j                                d       | j                  || j                         |S t	        | j
                  t              r| j                  ||      }nt        d      t         j                  d| dt        | j                                       |S )	a  Reduce topics to self.nr_topics.

        Arguments:
            documents: Dataframe with documents and their corresponding IDs and Topics
            use_ctfidf: Whether to calculate distances between topics based on c-TF-IDF embeddings. If False, semantic
                        embeddings are used.

        Returns:
            documents: Updated dataframe with documents and the reduced number of Topics
        z+Topic reduction - Reducing number of topicsz$Topic reduction - Number of topics (z/) is equal or higher than the clustered topics(r  rA  z(nr_topics needs to be an int or 'auto'! z0Topic reduction - Reduced number of topics from z to )r^   r   r   r  r   r=   r   _reduce_to_n_topicsr   rF   r   _auto_reduce_topicsr   r  )rv   r   r?  initial_nr_topicss       rw   r   zBERTopic._reduce_topics  s(    	AB 12dnnc*~~ 11 44Y
K	:4>>:JJyz}  C  N  N  P  {Q  zR  RT  U $$Y$E  ,00JGIGHH>?P>QQUVYZ^ZmZmZoVpUqr	
 r}   c                    |j                   j                         j                         }t        | j                  | j
                  |d      d   | j                  d }dt        |      z
  }t        j                  |d       t        j                  t              t        j                  d      k\  r&t        | j                  | j                  z
  dd	      }n%t        | j                  | j                  z
  dd
      }|j                  |       |D cg c]  }|dk7  r|j                   |   nd }}t#        ||      D 	
ci c]  \  }	}
|	|

 }}	}
t%        t&              }t)        |j+                               D ]  \  }}||   j-                  |        |j+                         D ci c]&  \  }}|||D cg c]  }| j.                  |    c}d( }}}}||_         | j1                  |       | j2                  j5                  ||        | j7                  |      }| j9                  ||| j:                         | j1                  |       |S c c}w c c}
}	w c c}w c c}}}w )a  Reduce topics to self.nr_topics.

        Arguments:
            documents: Dataframe with documents and their corresponding IDs and Topics
            use_ctfidf: Whether to calculate distances between topics based on c-TF-IDF embeddings. If False, semantic
                        embedding are used.

        Returns:
            documents: Updated dataframe with documents and the reduced number of Topics
        Toutput_ndarrayr   Nrz   z1.4.0precomputedr  )rS   rJ  )affinityrJ  rZ   r  r  r   rF   )r   r   r  r4   rp   rm   r|   r"   r   fill_diagonalr	   r\  r]  r   r=   r   r  r  r   r  r  r   r  rj   r   rk   r  r   r   rF   )rv   r   r?  r   r  distance_matrixclusterr   r   
from_topicto_topicmapped_topicsbasic_mappingsr   rk  r  r  r   s                     rw   r@  zBERTopic._reduce_to_n_topics  sM    '')..0 7NND22Jt

^^ /0@AA
!, ==)W]]7-CC-dnnt~~.MVcmvwG-/&!G
 	O$QWXgooe,CX
X KNfV`Jab2F*hX-bb$T*}2245 	,HC3&&s+	, *8)=)=)?
 

 &+	 *FQRU 1 1% 8R 
 
 %		*''4'H 44Y?	Y4<<P	*3 Y c  S
s$   I=I%I3I	IIc                    |j                   j                         j                         }t        t	        |j                   j                                     | j                  d }t        | j                  | j                  |d      d   }t        |d      }t        r,t        ddd	d
      j                  || j                  d       }n+t        ddd	d      j                  || j                  d       }i }t        ||      D ]'  \  }	}
|	dk7  s|	|vr|
||	<   t!        ||	   |
      ||	<   ) i }t        ||      D ]  \  }	}
|	dk(  r|
||
<   ||	   ||
<    |j                   j#                  |      j%                  |j                         j'                  t(              |_         t        ||j                   j                               D ci c]  \  }}||
 }}}t+        t              }t        |j-                               D ]  \  }}||   j/                  |        |j-                         D ci c]&  \  }}|||D cg c]  }| j0                  |    c}d( }}}}| j2                  j5                  ||        | j7                  |      }| j9                  ||| j:                         | j=                  |       |S c c}}w c c}w c c}}}w )a  Reduce the number of topics automatically using HDBSCAN.

        Arguments:
            documents: Dataframe with documents and their corresponding IDs and Topics
            use_ctfidf: Whether to calculate distances between topics based on c-TF-IDF embeddings. If False, the
                        embeddings from the embedding model are used.

        Returns:
            documents: Updated dataframe with documents and the reduced number of Topics
        NTrD  r   l2)r  rK  rT   rU   rV   rZ   r[   r  r  rH  )r   r   r  r  r  r  r|   r4   rp   rm   r   rf   r   fit_predictrg   r  r  r  fillnar   r   r   r   r  rj   rk   r  r   r   rF   r   )rv   r   r?  r   r  r   	norm_datar   cluster_to_lowestrK  r   rN  rL  rM  r   r   rk  r  r  r   s                       rw   rA  zBERTopic._auto_reduce_topics/  s    '')..0tIOO$:$:$<=>t~~?OP 1NND22Jt


 jt4	!!""). $	
 k)DNN$456  %!";QV_ak)DNN$456 
 !$[-!@ 	[GX"}"3319%g.145Fw5OQY1Z%g.	[ !$[-!@ 	EGX"}*2h'*;G*Dh'		E $//--m<CCIOOT[[\_`	JMfV_VeVeVlVlVnJop2F*hX-pp t$}2245 	&HCSM  %	& *2)9
 

 &+	 *FQRU 1 1% 8R 
 
 	''4'H44Y?	Y4<<P	*' q  S
s   J<.K<KKKc                    t        | j                        }| j                  r| j                  s|dkD  r|S | j	                  |       t        j                  | j                  j                         ddg      j                  dd      }||j                  dk7     }i ddit        t        |j                  t        t        |                        }| j                  j                  ||        |j                   j#                  |      j%                  |j                         j'                  t(              |_        | j	                  |       |S )	aQ  Reorder mappings by their frequency.

        For example, if topic 88 was mapped to topic
        5 and topic 5 turns out to be the largest topic,
        then topic 5 will be topic 0. The second largest
        will be topic 1, etc.

        If there are no mappings since no reduction of topics
        took place, then the topics will simply be ordered
        by their frequency and will get the topic ids based
        on that order.

        This means that -1 will remain the outlier class, and
        that the rest of the topics will be in descending order
        of ids and frequency.

        Arguments:
            documents: Dataframe with documents and their corresponding IDs and Topics

        Returns:
            documents: Updated dataframe with documents and the mapped
                       and re-ordered topic ids
        r   	Old_TopicSizer  FrX  rZ   r  )r   rn   r   r=   r   r   r   rj   r   r   rW  r  r  r   rk   r  r   r  rS  r   r   )rv   r   nr_zeroshotr  sorted_topicss        rw   r   z$BERTopic._sort_mappings_by_frequencyv  s   2 $>>?T^^a 		*\\$++113k6=RS__`frw_x"#OB8OtCeCGn,M'NO''4'H $//--m<CCIOOT[[\_`		*r}   r   c           	         | j                   j                  |      }|t        |j                        dk(  rt	        j
                  |j                  d   t        t        |j                                     | j                  z
  f      }|j                         D ])  \  }}|dk7  s|dk7  s|dd|fxx   |dd|f   z  cc<   + |S |S )ai  Map the probabilities to the reduced topics.
        This is achieved by adding together the probabilities
        of all topics that are mapped to the same topic. Then,
        the topics that were mapped from are set to 0 as they
        were reduced.

        Arguments:
            probabilities: An array containing probabilities
            original_topics: Whether we want to map from the
                             original topics to the most recent topics
                             or from the second-most recent topics.

        Returns:
            mapped_probabilities: Updated probabilities
        NrK  r   rZ   )
rk   r   r   rZ  r   r  r   r   r|   r   )rv   r   r   r   mapped_probabilitiesrL  rM  s          rw   r   zBERTopic._map_probabilities  s    $ %%22?C $=&&'1,')xx%++A.C 123dnnD($ -5NN,< Z(J2~**:,Q[9]1j==YY9Z ,+r}   c                 <   |D cg c]  }|j                  dd       }}|D cg c]  }|j                  dd       }}| j                  dk(  r$|D cg c]  }t        j                  dd|       }}|D cg c]  }|dk7  r|nd }}|S c c}w c c}w c c}w c c}w )zBasic preprocessing of text.

        Steps:
            * Replace \n and \t with whitespace
            * Only keep alpha-numerical characters
        r  r   	englishz[^A-Za-z0-9 ]+r  emptydoc)r  r9   resub)rv   r   r  cleaned_documentss       rw   r[  zBERTopic._preprocess_text  s     @IIS[[s3II?PQS[[s3QQ==I%O` a(92s!C a aIZ[#C2IS:=[[   JQ a[s   B
BB6Br  r  c                    g }t        | j                  dd | j                  dd       D ]  \  }}t        |||z
        }| j                  |t	        j
                  | j                  || |       | d z      }t        |      D cg c]  }t        |      |dz   k\  r||   nd }}|j                  |        t	        j                  |      S c c}w )a  Return indices of top n values in each row of a sparse matrix.

        Retrieved from:
            https://stackoverflow.com/questions/49207275/finding-the-top-n-values-in-a-row-of-a-scipy-sparse-matrix

        Arguments:
            matrix: The sparse matrix from which to get the top n indices per row
            n: The number of highest values to extract from each row

        Returns:
            indices: The top n indices per row
        NrZ   rz   )r  indptrr  r  r   r  datar   r   r  r   )r  r  r  leri
n_row_pickr   r   s           rw   r4  zBERTopic._top_n_idx_sparse  s     &--,fmmAB.?@ 	#FBQRJ^^BR9Kj[)Y[eZeZf)g$ghFW\]^W_`es6{eai'?fUmTI`F`NN6"		#
 xx   as   Cr  c           	          g }t        |      D ]B  \  }}t        j                  |D cg c]  }|| ||f   nd c}      }|j                  |       D t        j                  |      S c c}w )a  Return the top n values for each row in a sparse matrix.

        Arguments:
            matrix: The sparse matrix from which to get the top n indices per row
            indices: The top n indices per row

        Returns:
            top_values: The top n scores per row
        r   )r   r   r   r  )r  r  
top_valuesr   r   r  r9  s          rw   r5  zBERTopic._top_n_values_sparse  st     
$W- 	&KCXX[abRWU5Fvc5j1AMbcFf%	& xx
## cs   A(
c                    t        j                  | j                        }t        |j                  j                         D cg c]8  }|j                  dk7  s|j                  |j                  k7  s-|j                  : c}      }|S c c}w )zGet parameter names for the estimator.

        Adapted from:
            https://github.com/scikit-learn/scikit-learn/blob/b3ea3ed6a/sklearn/base.py#L178
        rv   )	inspect	signaturerx   r  
parametersr   r  kindVAR_KEYWORD)r  init_signaturepro  s       rw   r  zBERTopic._get_param_names  sq     !**3<<8+66==?p166VCSXYX^X^bcboboXoQVVp

  qs   BB0Bc                     d}| j                         j                         D ]>  \  }}t        |      }d|v r|d   dk7  r|j                  d      d   dz   }|| d| dz  }@ d|dd	  d
S )zGet a string representation of the current object.

        Returns:
            str: Human readable representation of the most important model parameters.
                 The parameters that represent models are ignored due to their length.
        r  (r   z(...)=r  z	BERTopic(Nr  ))r  r   r   split)rv   ro  	parameterr  s       rw   __str__zBERTopic.__str__  s     
 $ 1 7 7 9 	3IuJEe|aCC(+g5YKqr22J		3 :cr?+1--r}   )r_  r  )rz   rz   r  NFFNNffffff?NNNNNNF)NNN)NN)NNNTTT)TNN)r   rz   g?  FFFr   )NNrN   )NNr  NNNN)Fr  )NF)r  TNr   N)   NF)NNFFz<b>Intertopic Distance Map</b>  r  )
NNNNFFFz<b>Documents and Topics</b>rS  rT  )NNNNFTr  linearFz(<b>Hierarchical Documents and Topics</b>rS  rT  )NFFz#<b>Term score decline per Topic</b>   r  )NNFFz<b>Topics over Time</b>  i  )r  NFFz<b>Topics per Class</b>r  i  )gQ?Fz%<b>Topic Probability Distribution</b>r  X  )r  NNTFz<b>Hierarchical Clustering</b>r}  r  NNNrz   )NNNFFz<b>Similarity Matrix</b>r  r  )N   rN   FzTopic Word Scores   r  F)r  TF)r{  N)	zAdd BERTopic modelNNFFTr  TF)Nr   N)FN)NNFT)r  rN   N)TF)NTTN)cr8  
__module____qualname____doc__r   r   r   r   boolr   floatr#   r$   r*   rx   propertyr|   r   r   ndarrayr   r   r   r   r   r   r+  r<  r   r   rv  r  r  r)   r  r   r  rc  r  r  r   r  r  staticmethodr  r
  r  r   r  r9  r;  rM  rR  rZ  r  ra  re  rh  rk  rn  rq  rs  rw  rz  r~  r   r  classmethodr  r  r  r  r   r   r   r   r   r   r   r   r   r   r   r   r	  r   r   r   r   r   r@  rA  r   r   r[  r4  r5  r  rz  r   r}   rw   r8   r8   X   s   6t "(. ,0 (-2604),,0)-37%j(j( j( CHo	j(
 j( c?T)j( j( "&j( d3i4/j( "#Y-j( "'j( *j(  '!j(" 1#j($ %j(X 3 3  0 "&#'*.+9+ JJ+ S	D 	+
 cBJJ&'+` "&#'*.T09T0 JJT0 S	D 	T0
 cBJJ&'T0 
tCy%

D 011	2T0r "&#'	f*d3i(f* JJf* S	D 	f*
 
tCy"**$	%f*V "&*.	R9R JJR cBJJ&'	Rp $("&*!%"]d3i]d $s)T#Y./]d S	D 	]d
 t]d t]d ]d ]d 
]dF #	M 3iM  tCy$s)+,M  	M 
 
M d  FJGKg3ig g #J<#;<tC	g
 $ZL*$<=Dg 
gX  #$)!&a>d3i(a> a> 	a>
 a> a> a> "a> a> a> 
rzz5bjj!14!788	9a>H VW7*:7*584Z7*OR7*	tCy$u+%	&7*x $(#'/3,0.237l-3il- S	D l- S	D 	l-
 l- CHo,l- *l- ,l- 1l-\/t /U3:=N8N0O /0s $ 5eTWY^T^N_I_A`bfAf;g 83+C$J 3+",, 3+jC$J %c@Q:R D  -1	Z3iZ LLZ #s(#d*	Z
 
Zx+-S4Z +-49 +-Z  &*"i%\\i%dli% i% 
	i% i%V=,U49gc3h>O3O-P =,UY =,B !"&!99 9 4Z	9
 9 d
9 
c9~ $(	JK3iJK eHSM3$678JK S	D 	JK
 
JKXFLs)FL 
FLV &(#' A3iA c?A S	D 	A
 A 
AN $('$(!%24_9_ S	_ S	D 	_
 _ zz_ _ JJ_ &c3h/_ 
c_F $(#' #54
S	D 4
 Dj4
 	4

 4
 4
 4
 4
 
4
r $(!%)-#!&$)#2a
3ia
 S	D a
 JJ	a

 JJa
 a
 a
 "a
 a
 a
 a
 a
 
a
J "&#'!%)-*/+&*!#"!#m
3i$m
 S	D m
 JJ	m

 JJm
 T3Y'm
 m
 d#m
 m
 m
 m
 m
 m
 m
 m
  
!m
f $(!%)-+/!&$(##?v
3iv
  \\v
 S	D 	v

 JJv
 JJv
 eSj!D(v
 v
 "v
 v
 v
 v
 v
 v
 v
  
!v
t $(#:=
S	D =
 =
 	=

 =
 =
 =
 
=
D $(#'$)#.6
,,6
 Dj6
 S	D 	6

 "6
 6
 6
 6
 6
 
6
v #'$)#.6
,,6
 6
 S	D 	6

 "6
 6
 6
 6
 6
 
6
v "'#<-
zz-
 -
 	-

 -
 -
 -
 
-
f  	4
4
 #%**4
 	4
p "#'#'#5,0FJGK b
b
 S	D b
 Dj	b

 b
 b
 b
 b
 b
  \\b
 #J<#;<tCb
 $ZL*$<=Db
 b
 
b
L $(#'!% #/:
S	D :
 Dj:
 $J	:

 :
 :
 :
 :
 :
 
:
| $(#(5
S	D 5
 5
 	5

 5
 5
 5
 5
 5
 
5
t FN15!sj ABsj $D#I.	sj
 sjj 5 5 5n _% _ _H 3 #*15!D
D
 D
 Tz	D

 *D
 D
 D
 D
 D
 $CI.D
 D
Lt S0A 4 $( #&cC(& S	D & 	&
 & 
&P 2:: RTR^R^ ""DI "$s) " +/!	1."**j011. cBJJ&'1. 	1.
 
1.n "2(2( <<2( 	2(
 ::2( 
r||RZZ'	(2(h8N8N35::8N	r||RXXr||RXX=	>8Nt9%<<9% JJ9% LL	9%
  ZZ9% 
r||RZZ'	(9%v& &d3iQSQYQYFY@Z &V "&)-&=<<&= JJ&=
 &= #'&=P.2<< .. "&NONO <<NO T%U
"3445	NO
 NO NO 4<NO 
tCy$tCy/)	*NOd #'!%	Q@<<Q@ JJQ@l !	7\\7 7 	7
 
z49$	%7r<BLL <  $)-"&!%cCyc <<c 	c
 #'c  c JJc 
d5e,--	.cJ $ SUS_S_ B7R\\ 7t 7XZXdXd 7rER\\ Et EXZXdXd EN'R\\ 'bll 'T OT#"2::t#34#GK#	rzz4	 #J!"** !c ! !* ! ! ! !* $Z $"** $ $ $  
 
.r}   r8   c                   n    e Zd ZdZdee   fdZddedeeef   fdZ	deeef   de
fd	Zdeeef   fd
Zy)r   a  Keep track of Topic Mappings.

    The number of topics can be reduced
    by merging them together. This mapping
    needs to be tracked in BERTopic as new
    predictions need to be mapped to the new
    topics.

    These mappings are tracked in the `self.mappings_`
    attribute where each set of topic is stacked horizontally.
    For example, the most recent topics can be found in the
    last column. To get a mapping, simply take two columns
    of topics.

    In other words, it is represented as graph:
    Topic 1 --> Topic 11 --> Topic 4 --> etc.

    Attributes:
        self.mappings_ (np.ndarray) : A  matrix indicating the mappings from one topic
                                      to another. The columns represent a collection of topics
                                      at any time. The last column represents the current state
                                      of topics and the first column represents the initial state
                                      of topics.
    r   c                    t        j                  t        t        |                  }|j	                         j                  dd      }t        j                  |j	                         |j	                         g      j                         | _        y)znInitialization of Topic Mapper.

        Arguments:
            topics: A list of topics per document
        rZ   rz   N)	r   r   r  r   r  r  hstackr   r  )rv   r   r<  s      rw   rx   zTopicMapper.__init__1  s`     hhvc&k23!!#++B2FKKM6;;=#ABIIKr}   r   r   c                 :   |rMt        j                  | j                        ddddgf   }t        t	        |dddf   |dddf               }|S t        j                  | j                        ddddgf   }t        t	        |dddf   |dddf               }|S )a^  Get mappings from either the original topics or
        the second-most recent topics to the current topics.

        Arguments:
            original_topics: Whether we want to map from the
                             original topics to the most recent topics
                             or from the second-most recent topics.

        Returns:
            mappings: The mappings from old topics to new topics

        Examples:
        To get mappings, simply call:
        ```python
        mapper = TopicMapper(topics)
        mappings = mapper.get_mappings(original_topics=False)
        ```
        Nr   rZ   rz   )r   r   r  r  r  )rv   r   r   s      rw   r   zTopicMapper.get_mappings;  s    & xx/Ar7
;HCAA?@H  xx/B8<HCAA?@Hr}   r   r  c                    | j                   D ]1  }|d   }||v r|j                  ||          !|j                  d       3 |j                         rt        |j                        dkD  rqi }t        j                  |j                  j                         ddddf   D ci c]  }|d   |d    }}t        t              }|j                         D ]  \  }	}
||
   j                  |	        |j                         D ]  \  }}|D cg c]  }||j                  v s| }}t        |      dk(  r3|D cg c]  }|j                  |j                  |        }}|j                  |      }t        ||j                  |   g      j                         }t        j                   |      }||   }||j"                  k\  s|j                  ||      ||<    ||_        yyyc c}w c c}w c c}w )zAdd new column(s) of topic mappings.

        Arguments:
            mappings: The mappings to add
            topic_model: The topic model this TopicMapper belongs to
        rZ   r   Nr  rz   )r  r  r   r   rn   r   r   rk   r   r  r   rA   r   r"   rm   r  r   rB   )rv   r   r  r   r   "new_topic_id_to_zeroshot_topic_idxr#  r$  r  r   r  r  r  r   zeroshot_topic_idszeroshot_labelsr  r  best_zeroshot_topic_idxbest_cosine_similaritys                       rw   r  zTopicMapper.add_mappingsV  s0    nn 	"F2JE huo.b!	" ##%#k.Y.Y*Z]^*^13. &(XXk.G.G.Q.Q%RSTVXVYSY%Z! a -"22M  "$'G+113 +
U%%c*+ *1 %+ .9&!)HHsHs<sH&" & )*a/ %7#   33K4_4_`h4ij# # '2&E&Eo&V#&7'+*G*G*Q)R'') $ +-))4G*H')<=T)U&)[-P-PPCNCnCn*+BCD6x@-2 ;]K7K +_%&#s   !GG#G#5#G(c                     t        | j                  d         }|j                         D ]1  \  }}|gdg|dz
  z  z   |gz   }| j                  j                  |       3 y)zhAdd new row(s) of topic mappings.

        Arguments:
            mappings: The mappings to add
        r   NrK  )r   r  r   r  )rv   r   lengthr   r  	to_appends         rw   r   zTopicMapper.add_new_topics  sa     T^^A&'"..* 	-JC$6A:!675'AINN!!),	-r}   Nr|  )r8  r  r  r  r   r   rx   r  r   r   r8   r  r   r   r}   rw   r   r     sj    2LtCy LD GCH<M 66]WS#X%6 6]X 6]p	-wsCx'8 	-r}   r   r   r  r  r  r  r   r  c                    t        |d         |d<   ||d   d   d   }t        |      |d   d   d<   t        |d         |d<   	 ddlm} t         ||d               }	|j                  d      |d= t               }
t               }t        d|	|
|d	|}|d
   |_        | d   j                         D ci c]  \  }}t        |      | c}}|_        | d   |_        | d   j                         D ci c]  \  }}t        |      | c}}|_        | d   |_        | j                  d      rki }| d   j                         D ];  \  }}|dk7  s|j                         D ci c]  \  }}t        |      | c}}||<   = ||_        |||j$                  d<   t'        dg      |_        | d   |j(                  _        |t-        |d   |d   |d   f|d         |_        t1        di |d   d   |_        |d   d   |j2                  _        |d   d   |j6                  _        |d   d   |j6                  _        |d   }t=        j>                  |dtA        |      tA        |      fdtB        jD                        |j6                  _#        |S #  t	               }	|rt
        j                  d       Y VxY wc c}}w c c}}w c c}}w )aJ  Create a BERTopic model from a variety of inputs.

    Arguments:
        topics: A dictionary containing topic metadata, including:
                - Topic representations, labels, sizes, custom labels, etc.
        params: BERTopic-specific hyperparams, including HF embedding_model ID
                if given.
        tensors: The topic embeddings
        ctfidf_tensors: The c-TF-IDF representations
        ctfidf_config: The config for CountVectorizer and c-TF-IDF
        images: The images per topic
        warn_no_backend: Whether to warn the user if no backend is given
    r;   rC   r  rK   r   )SentenceTransformerr`   zYou are loading a BERTopic model without explicitly defining an embedding model. If you want to also load in an embedding model, make sure to use `BERTopic.load(my_model, embedding_model=my_embedding_model)`.)r`   ra   re   r  r  r   r  r	  r  Visual_Aspectr  rf  r  re  rZ  r$  vocabrD   reduce_frequent_wordsbm25_weightingdiagcsr)offsetsrZ  r
  dtyper   )$r  sentence_transformersr  r(   r&   r^   r_   r   r,   r%   r8   rm   r   r   rl   rh   rj   ro   rs   r   rk   r  r   rp   r#   rC   vocabulary_rD   r  r  r&  diagsr   r   float64r'  )r   r  r  r  r  r   r  rK   r  r`   empty_dimensionality_modelempty_cluster_modelr  r   rk  r  r  r   r   r  idfs                        rw   r  r    s   , #6.#9:F> #$67A-PEJ;EW()(3MB"6.#9:F>=()<VDU=V)WX zz#$0$% "=!>%-  '-) 	K %,,>$?K!HNOfHgHmHmHo)pHC#c(C-)pK& *K>D]>S>Y>Y>[\(#sC#\K!'!8Kzz/"$_5;;= 	_NFF(OU||~(^|ueUU):(^f%	_ &3":@K&&7 !,QC 0K*0*@K'! *v&y)x(
 !)!
 (7'eGY9Z[c9d'e$3@AS3TU\3]$$0 :G~9VWn9o  62?2OP`2a  /V$-/XX3s8SX"6uBJJ.
  * C&.NNR* *q\ )_s   J 6J85J>#K#J5)NNNT)syamlwarningsfilterwarningsFutureWarningUserWarning_warnings_enabledKeyErrorr  r  ra  r  r  rm  r0  numpyr   pandasr   scipy.sparsesparser&  r  r   r   pathlibr   	packagingr	   tempfiler
   r   r   r   scipy.clusterr   rI  importlib.utilr   typingr   r   r   r   r   r   r   r   r   bertopic._utilsr   r   bertopicplotly.graph_objs
graph_objsgomatplotlib.figurefigurefighdbscanr   rf   rc   rd   sklearn.clusterrg   sklearn.preprocessingr   sklearnr   r]  r   sklearn.decompositionr    sklearn.metrics.pairwiser"   sklearn.feature_extraction.textr#   r$   bertopic.clusterr%   bertopic.backendr&   bertopic.representation._mmrr'   bertopic.backend._utilsr(   bertopic.vectorizersr)   bertopic.representationr*   r+   bertopic.dimensionalityr,   bertopic.cluster._utilsr-   r.   r/   r0   r1   r2   r3   r4   r5   bertopic._save_utils_save_utilsr  r^   	configurer8   r   r   r   r   r  r  r   r}   rw   <module>r     s       = 9   ; 7	05D,- 
            ' , # * $ _ _ _ X0!H "&'6K
 , 2 3 % 6 M ) ) , 2 6 G ? K   *	    |J. |J.~U@- @-N 04.2'+ dCHdCHd S"((]#d CH%,	d
 38$t+d CH$d da\ 	.), 		V 	() 6K556s#   G G GGG-,G-