#!/usr/bin/env python3 """ Validation test for the updated Layer 3 curiosity-mining prompt. Runs 5 clusters (3 from money_traps, 2 from body_is_weird) through the updated pipeline. """ import json import subprocess import re from pathlib import Path from datetime import datetime SANDBOX_DIR = Path(__file__).parent OUTPUT_FILE = SANDBOX_DIR / "validation_results.json" # New curiosity-mining Layer 3 prompt (v2) def build_prompt(cluster, channel_context): """Build the new curiosity-mining prompt for a cluster.""" # Build topic titles all_titles = [t.get("topic_name", t.get("title", "")) for t in cluster.get("topics", [])] titles_text = "\n".join([f"- {t}" for t in all_titles]) # Build Reddit context reddit_text = "No Reddit posts in this cluster." raw_posts = cluster.get("raw_posts", []) if raw_posts: reddit_lines = [] for r in raw_posts: comments = r.get("num_comments", 0) score = r.get("score", 0) subreddit = r.get("subreddit", "unknown") title = r.get("title", "") selftext = r.get("selftext", "")[:200] reddit_lines.append(f"- r/{subreddit}: \"{title}\" (score: {score}, comments: {comments})") if selftext: reddit_lines.append(f" Context: {selftext}...") reddit_text = "\n".join(reddit_lines) subreddits = cluster.get("subreddits", []) sources = ', '.join(subreddits) if subreddits else "reddit" topic_count = cluster.get("topic_count", len(all_titles)) prompt = f"""You are a short-form video director scanning raw material for the ONE specific detail that would stop someone mid-scroll on TikTok. You are not a content strategist evaluating themes. You are a treasure hunter looking for the single most surprising, counterintuitive, or unexplainable specific detail buried in this cluster of Reddit posts. {channel_context} HERE ARE THE RAW POSTS IN THIS CLUSTER: {titles_text} REDDIT SIGNALS: {reddit_text} SOURCES: {sources} TOPIC COUNT: {topic_count} YOUR TASK HAS THREE PHASES. Do them in order. ═══════════════════════════════════════════════════════════ PHASE 1: THE NUGGET HUNT ═══════════════════════════════════════════════════════════ Read every post title in this cluster. You are looking for ONE specific thing that would make a random person scrolling TikTok think "wait, what?" — a detail so surprising, counterintuitive, or unexplainable that they cannot scroll past it. The best hooks share these qualities: - SPECIFIC: They contain a concrete detail — a number, a name, a date, a measurement, a specific person's experience. Not a broad theme. - COUNTERINTUITIVE: They violate everyday expectations. The viewer's internal model of the world is wrong, and this detail proves it. - UNRESOLVED: The viewer cannot explain the detail without watching. It creates genuine confusion that demands resolution. - RELATABLE ENTRY POINT: Even if the topic is obscure, the hook connects to something the viewer already cares about (their body, their money, their daily habits, their assumptions about how things work). Look for UP TO 3 candidate "nuggets" — specific details that could anchor a video. For each, note: 1. The exact post title it came from 2. The specific detail (one sentence) 3. Why this creates a "wait, what?" moment If you cannot find any genuinely surprising details, say so. Do not manufacture intrigue from mundane patterns. ═══════════════════════════════════════════════════════════ PHASE 2: BUILD THE CONCEPT ═══════════════════════════════════════════════════════════ Select the BEST nugget from Phase 1. Now build the video concept around that specific detail: TITLE: Write a title that leads with the specific detail, not a general theme. BAD: "Why Your Sleep Schedule Matters More Than You Think" GOOD: "The 3am Blood Test That Changed Everything We Know About Sleep" BAD: "The Truth About Credit Card Rewards" GOOD: "Why a $50 Grocery Trip Actually Cost You $73" THE ITCH: What specific information gap does this create? What does the viewer NEED to know that they can't figure out themselves? OPENING HOOK (first 10 seconds): Write the exact words. This must: - State the specific counterintuitive detail immediately - Create an unresolved curiosity gap - NOT be a question ("Have you ever wondered...") - NOT be a promise ("In this video I'll show you...") CORE REVEAL: What is the satisfying explanation? Can you actually deliver the answer? DEPTH CHECK: Can this specific detail sustain a full video? Is there enough to explain, or will you run out of content after 20 seconds? EMOTIONAL PAYOFF: What does the viewer feel at the end? (empowered / validated / smarter / relieved / amused) ═══════════════════════════════════════════════════════════ PHASE 3: VIABILITY ASSESSMENT ═══════════════════════════════════════════════════════════ Score on each dimension (0-5 scale), where 3 = average: CURIOSITY INTENSITY (30% weight): How strong is the "wait, what?" reaction? Does the specific detail genuinely violate expectations, or is it just mildly interesting? DEMAND SIGNAL (20% weight): Are people actively searching for this? Do the Reddit posts show genuine confusion or just casual interest? CONTENT GAP (15% weight): Is anyone else explaining THIS SPECIFIC detail, or just the general topic? DEPTH/EXPLAINABILITY (15% weight): Can you actually explain this in 60-90 seconds in a satisfying way? EVERGREEN POTENTIAL (10% weight): Will this still be interesting in 6 months? AUDIENCE BREADTH (10% weight): How many people would find this specific detail surprising? WEIGHTED SCORE: Calculate using the weights above. VERDICT: - MAKE_NOW: The specific detail is genuinely surprising, you can explain it satisfyingly, and nothing else covers it. This is rare — maybe 1 in 5 clusters. - WORTH_MAKING: Good detail, worth developing, but not urgent. - NEEDS_RESEARCH: Interesting hook, but you're not sure you can deliver the explanation. What specific research would you need? - SKIP: No genuinely surprising details in this cluster. The posts are mundane, already well-covered, or the "hook" requires manufacturing intrigue that isn't there. Note on verdict distribution: Not every cluster contains a video. If you're giving MAKE_NOW to more than 20% of clusters, you're not being selective enough. If you're never giving SKIP, you're not being honest enough. The pipeline's value comes from surfacing the genuine gems, not from inflating everything. Respond in this JSON format: {{ "phase1_nuggets": [ {{ "source_post_title": "exact Reddit post title that contains the hook", "the_detail": "the specific surprising detail in one sentence", "wait_what": "why this creates an unresolvable curiosity gap" }} ], "phase1_best_nugget_index": 0, "phase1_honest_assessment": "Is this cluster genuinely interesting or just analytically viable?", "theme": "cluster theme (for reference only, not the video topic)", "suggested_title": "title built around the specific detail", "the_itch": "the specific information gap in 1-2 sentences", "opening_hook": "first 10 seconds of the video", "core_reveal": "the satisfying explanation", "depth_check": "can this sustain a full video? why or why not?", "emotional_payoff": "empowered|validated|smarter|relieved|amused", "target_audience": "who is the specific person who would watch this, described as a moment", "existing_content": "what already exists and why it fails", "missing_angle": "what this video does that nothing else does", "scores": {{ "curiosity_intensity": {{"score": 0, "reasoning": "..."}}, "demand_signal": {{"score": 0, "reasoning": "..."}}, "content_gap": {{"score": 0, "reasoning": "..."}}, "depth_explainability": {{"score": 0, "reasoning": "..."}}, "evergreen_potential": {{"score": 0, "reasoning": "..."}}, "audience_breadth": {{"score": 0, "reasoning": "..."}} }}, "weighted_score": 0.0, "verdict": "MAKE_NOW|WORTH_MAKING|NEEDS_RESEARCH|SKIP", "verdict_reasoning": "one sentence on why", "research_needed": "if NEEDS_RESEARCH, what specifically needs to be verified", "source_post_title": "the specific Reddit post that inspired this concept", "structure": "3-4 section outline for the video", "critical_success_factor": "the one thing that must be right for this video to work" }}""" return prompt def run_layer3(cluster, channel_id): """Run the new Layer 3 prompt on a cluster using Claude CLI.""" # Channel context (simplified for validation) channel_contexts = { "money_traps": """CHANNEL CONTEXT: "Money Traps" This channel exposes hidden financial mechanisms that cost viewers money without them realizing it. Target: People who consider themselves financially savvy but don't know they're being exploited.""", "body_is_weird": """CHANNEL CONTEXT: "Body Is Weird" This channel explains surprising biological facts about the human body. Target: People who are curious about how their own bodies work in ways they've never thought about.""" } channel_context = channel_contexts.get(channel_id, f"CHANNEL: {channel_id}") prompt = build_prompt(cluster, channel_context) try: result = subprocess.run( ["claude", "--print", "--model", "opus", "-p", "-"], input=prompt, capture_output=True, text=True, timeout=300 ) if result.returncode != 0: return {"error": f"Claude CLI failed: {result.stderr[:200]}"} response_text = result.stdout.strip() # Parse JSON response json_match = re.search(r'\{.*\}', response_text, re.DOTALL) if json_match: analysis = json.loads(json_match.group()) return analysis else: return {"error": "Could not parse JSON", "raw_response": response_text[:1000]} except subprocess.TimeoutExpired: return {"error": "Claude CLI timed out"} except json.JSONDecodeError as e: return {"error": f"JSON parse error: {str(e)}", "raw_response": response_text[:1000]} except Exception as e: return {"error": str(e)} def main(): print("=" * 60) print("VALIDATION TEST: Curiosity-Mining Layer 3 Prompt") print("=" * 60) # Load test clusters with open(SANDBOX_DIR / "test_clusters.json") as f: round1_data = json.load(f) with open(SANDBOX_DIR / "round2_clusters.json") as f: round2_data = json.load(f) # Select 3 money_traps clusters from round 1 money_traps_clusters = [c for c in round1_data["clusters"] if c.get("channel") == "money_traps"][:3] # Select 2 body_is_weird clusters from round 2 body_is_weird_clusters = [c for c in round2_data["clusters"] if c.get("channel") == "body_is_weird"][:2] validation_clusters = money_traps_clusters + body_is_weird_clusters print(f"\nSelected {len(validation_clusters)} clusters for validation:") for c in validation_clusters: print(f" - {c.get('test_id', c.get('cluster_id'))}: {c.get('channel')}") results = [] for i, cluster in enumerate(validation_clusters): test_id = cluster.get("test_id", f"cluster_{cluster.get('cluster_id')}") channel = cluster.get("channel") print(f"\n[{i+1}/{len(validation_clusters)}] Running: {test_id} ({channel})") analysis = run_layer3(cluster, channel) # Check result if "error" in analysis: print(f" ERROR: {analysis['error']}") else: verdict = analysis.get("verdict", "UNKNOWN") score = analysis.get("weighted_score", 0) title = analysis.get("suggested_title", "No title")[:60] print(f" -> {verdict} (score: {score:.1f})") print(f" Title: {title}...") results.append({ "test_id": test_id, "channel": channel, "original_title": cluster.get("pipeline_title", "N/A"), "original_verdict": cluster.get("pipeline_verdict", "N/A"), "original_score": cluster.get("pipeline_score", 0), "new_analysis": analysis }) # Save results output = { "validation_run": datetime.now().isoformat(), "prompt_version": "v2_curiosity_mining", "clusters_tested": len(results), "results": results, "summary": { "verdicts": {}, "avg_score": 0 } } # Calculate summary verdicts = {} scores = [] for r in results: if "error" not in r["new_analysis"]: v = r["new_analysis"].get("verdict", "UNKNOWN") verdicts[v] = verdicts.get(v, 0) + 1 scores.append(r["new_analysis"].get("weighted_score", 0)) output["summary"]["verdicts"] = verdicts output["summary"]["avg_score"] = sum(scores) / len(scores) if scores else 0 with open(OUTPUT_FILE, "w") as f: json.dump(output, f, indent=2) print(f"\n{'=' * 60}") print("VALIDATION COMPLETE") print(f"{'=' * 60}") print(f"\nResults saved to: {OUTPUT_FILE}") print(f"\nSummary:") print(f" Clusters tested: {len(results)}") print(f" Verdicts: {verdicts}") print(f" Average score: {output['summary']['avg_score']:.2f}") # Compare old vs new print(f"\n{'=' * 60}") print("COMPARISON: Old vs New") print(f"{'=' * 60}") for r in results: old_title = r["original_title"][:40] old_verdict = r["original_verdict"] new_title = r["new_analysis"].get("suggested_title", "ERROR")[:40] new_verdict = r["new_analysis"].get("verdict", "ERROR") print(f"\n{r['test_id']}:") print(f" OLD: [{old_verdict}] {old_title}...") print(f" NEW: [{new_verdict}] {new_title}...") if __name__ == "__main__": main()