#!/usr/bin/env python3 """ Select 5 diverse test clusters from each channel for the sandbox comparison test. Criteria: - Different sizes (variety of cluster sizes) - Good selftext coverage (clusters with substantial raw post content) - Thematic variety """ import json import os from collections import Counter SANDBOX_DIR = os.path.dirname(os.path.abspath(__file__)) def analyze_cluster(cluster): """Analyze a cluster and return metrics for selection.""" posts = cluster.get('reddit_posts', []) topics = cluster.get('topics', []) # Count posts with substantial selftext (>100 chars) posts_with_content = sum(1 for p in posts if len(p.get('selftext', '')) > 100) # Calculate average selftext length selftexts = [len(p.get('selftext', '')) for p in posts] avg_selftext = sum(selftexts) / len(selftexts) if selftexts else 0 # Get unique subreddits subreddits = set(p.get('subreddit', '') for p in posts) # Sample topics for theme detection topic_names = [t.get('topic_name', '')[:50] for t in topics[:5]] return { 'cluster_id': cluster.get('cluster_id'), 'topic_count': cluster.get('topic_count', len(topics)), 'post_count': len(posts), 'posts_with_content': posts_with_content, 'content_ratio': posts_with_content / len(posts) if posts else 0, 'avg_selftext_len': avg_selftext, 'subreddit_count': len(subreddits), 'subreddits': list(subreddits)[:5], 'sample_topics': topic_names } def select_diverse_clusters(clusters, n=5): """Select n diverse clusters with good content coverage.""" # Analyze all clusters analyzed = [] for cluster in clusters: metrics = analyze_cluster(cluster) # Only consider clusters with at least some content if metrics['posts_with_content'] >= 3: metrics['_cluster'] = cluster analyzed.append(metrics) # Sort by content ratio (higher = more posts with selftext) analyzed.sort(key=lambda x: (x['content_ratio'], x['avg_selftext_len']), reverse=True) # Select diverse clusters: # - Pick from different size ranges (small, medium, large) # - Ensure variety in subreddits selected = [] seen_subreddits = set() # Bin by size small = [c for c in analyzed if c['topic_count'] < 20] medium = [c for c in analyzed if 20 <= c['topic_count'] < 50] large = [c for c in analyzed if c['topic_count'] >= 50] print(f" Clusters by size: small={len(small)}, medium={len(medium)}, large={len(large)}") # Select from each bin for diversity bins = [large, medium, small, analyzed] # Priority order bin_quotas = [2, 2, 1, n] # How many from each bin for bin_clusters, quota in zip(bins, bin_quotas): for cluster in bin_clusters: if len(selected) >= n: break cluster_id = cluster['cluster_id'] # Check if already selected if any(c['cluster_id'] == cluster_id for c in selected): continue # Check subreddit overlap (want diversity) new_subs = set(cluster['subreddits']) - seen_subreddits if len(selected) > 0 and len(new_subs) == 0: continue # Skip if no new subreddits selected.append(cluster) seen_subreddits.update(cluster['subreddits']) # If still need more, just take top content-ratio clusters if len(selected) < n: for cluster in analyzed: if len(selected) >= n: break if not any(c['cluster_id'] == cluster['cluster_id'] for c in selected): selected.append(cluster) return selected def extract_test_data(selected_clusters, channel_id): """Extract full test data for selected clusters.""" test_data = [] for i, metrics in enumerate(selected_clusters): cluster = metrics['_cluster'] # Get all raw posts with their full content raw_posts = [] for post in cluster.get('reddit_posts', []): raw_posts.append({ 'title': post.get('title', ''), 'selftext': post.get('selftext', ''), 'subreddit': post.get('subreddit', ''), 'num_comments': post.get('num_comments', 0), 'score': post.get('score', 0), 'url': post.get('url', '') }) test_cluster = { 'test_id': f"{channel_id}_cluster_{i+1}", 'cluster_id': metrics['cluster_id'], 'channel': channel_id, 'topic_count': metrics['topic_count'], 'posts_with_content': metrics['posts_with_content'], 'sample_topics': metrics['sample_topics'], 'subreddits': metrics['subreddits'], 'raw_posts': raw_posts } test_data.append(test_cluster) return test_data def main(): channels = ['brain_is_lying', 'money_traps'] all_test_clusters = [] for channel_id in channels: print(f"\n{'='*60}") print(f"Processing {channel_id}") print(f"{'='*60}") # Load cluster data input_path = os.path.join(SANDBOX_DIR, f'{channel_id}_clusters.json') with open(input_path) as f: data = json.load(f) clusters = data.get('clusters', []) print(f"Total clusters: {len(clusters)}") # Select diverse clusters selected = select_diverse_clusters(clusters, n=5) print(f"\nSelected {len(selected)} clusters:") for c in selected: print(f" - Cluster {c['cluster_id']}: {c['topic_count']} topics, " f"{c['posts_with_content']} posts with content, " f"subs: {c['subreddits']}") print(f" Sample topics: {c['sample_topics'][:2]}") # Extract full test data test_data = extract_test_data(selected, channel_id) all_test_clusters.extend(test_data) # Save combined test data output_path = os.path.join(SANDBOX_DIR, 'test_clusters.json') with open(output_path, 'w') as f: json.dump({ 'total_clusters': len(all_test_clusters), 'channels': channels, 'clusters': all_test_clusters }, f, indent=2) print(f"\n{'='*60}") print(f"Saved {len(all_test_clusters)} test clusters to {output_path}") print(f"{'='*60}") # Summary statistics total_posts = sum(len(c['raw_posts']) for c in all_test_clusters) total_with_content = sum(c['posts_with_content'] for c in all_test_clusters) print(f"\nSummary:") print(f" Total raw posts: {total_posts}") print(f" Posts with content: {total_with_content}") if __name__ == "__main__": main()