#!/usr/bin/env python3 """ Blind Scoring: Uniform evaluation of all titles without context. Runs AFTER all conditions have been evaluated. Strips titles of all context (scores, verdicts, condition labels) and presents them as a randomized flat list for fair comparison. """ import json import os import subprocess import re import random from datetime import datetime SANDBOX_DIR = os.path.dirname(os.path.abspath(__file__)) BLIND_SCORING_PROMPT = """You are a TikTok user scrolling your feed. You have no special interest in any of these topics. For each title below, rate how likely you are to stop scrolling and watch on a scale of 1-10: 1-2: I'd scroll past without reading the full title 3-4: I'd read it but keep scrolling. "I guess that's interesting." 5-6: I'd pause. Mildly curious. Might watch if nothing better is coming. 7-8: I'd stop scrolling. "Wait, what?" I need to know this. 9-10: I'd watch AND send it to someone. "You need to see this." Rate each title. Be harsh. Most titles are 3-5. A 7+ should be rare. A 9+ should be exceptional. IMPORTANT: You are not evaluating whether the topic is important, educational, or well-researched. You are evaluating ONE thing: does this title make you stop scrolling? A title about something trivial that creates genuine curiosity scores higher than a title about something important that sounds like homework. Here are the titles to rate: {titles_list} Respond in JSON format: {{ "ratings": [ {{"id": 1, "score": X, "reaction": "One sentence explaining your gut reaction"}}, {{"id": 2, "score": X, "reaction": "..."}}, ... ], "distribution": {{ "1-2": N, "3-4": N, "5-6": N, "7-8": N, "9-10": N }}, "top_3": [ {{"id": X, "title": "...", "why": "Why this one stood out"}} ], "bottom_3": [ {{"id": X, "title": "...", "why": "Why this one failed"}} ] }} Be honest. Be harsh. Most titles are mediocre.""" def load_round_results(round_num: int) -> dict: """Load results from a round.""" if round_num == 1: path = os.path.join(SANDBOX_DIR, 'comparison_results.json') else: path = os.path.join(SANDBOX_DIR, f'round{round_num}_results.json') with open(path) as f: return json.load(f) def extract_titles_from_round(results: dict) -> list: """Extract all titles from round results with metadata.""" titles = [] for r in results.get('results', []): test_id = r.get('test_id', '') for condition in ['A', 'B', 'C']: title = r['results'][condition].get('title', '') if title and not title.startswith('[ERROR'): titles.append({ 'test_id': test_id, 'condition': condition, 'title': title, 'original_score': r['results'][condition].get('score', 0), 'original_verdict': r['results'][condition].get('verdict', '') }) return titles def randomize_titles(titles: list) -> tuple: """Randomize titles and create mapping for de-randomization.""" # Create random order indices = list(range(len(titles))) random.shuffle(indices) # Build randomized list and mapping randomized = [] mapping = {} # id -> original data for new_id, original_idx in enumerate(indices, 1): randomized.append({ 'id': new_id, 'title': titles[original_idx]['title'] }) mapping[new_id] = titles[original_idx] return randomized, mapping def call_claude(prompt: str, model: str = "opus", timeout: int = 300) -> dict: """Call Claude CLI and return parsed JSON response.""" try: result = subprocess.run( ["claude", "--print", "--model", model, "-p", "-"], input=prompt, capture_output=True, text=True, timeout=timeout ) if result.returncode != 0: return {"error": f"Claude CLI failed: {result.stderr[:200]}"} response_text = result.stdout.strip() try: json_match = re.search(r'\{.*\}', response_text, re.DOTALL) if json_match: return json.loads(json_match.group()) else: return json.loads(response_text) except json.JSONDecodeError as e: return { "error": "JSON parse error", "raw_response": response_text[:3000], "parse_error": str(e) } except subprocess.TimeoutExpired: return {"error": "Claude CLI timed out"} except Exception as e: return {"error": str(e)} def run_blind_scoring(round_num: int): """Run blind scoring for a round.""" print(f"\n{'='*70}") print(f"BLIND SCORING: ROUND {round_num}") print(f"{'='*70}") # Load results results = load_round_results(round_num) titles = extract_titles_from_round(results) print(f"Total titles to score: {len(titles)}") # Randomize randomized, mapping = randomize_titles(titles) # Build prompt titles_list = "\n".join([f"{t['id']}. \"{t['title']}\"" for t in randomized]) prompt = BLIND_SCORING_PROMPT.format(titles_list=titles_list) print(f"Calling Claude for blind scoring...") # Call Claude response = call_claude(prompt, timeout=600) if "error" in response: print(f"ERROR: {response['error']}") return None # De-randomize and group by condition ratings = response.get('ratings', []) by_condition = {'A': [], 'B': [], 'C': []} for rating in ratings: rating_id = rating.get('id') if rating_id in mapping: original = mapping[rating_id] entry = { 'test_id': original['test_id'], 'title': original['title'], 'condition': original['condition'], 'blind_score': rating.get('score', 0), 'reaction': rating.get('reaction', ''), 'original_score': original['original_score'], 'original_verdict': original['original_verdict'] } by_condition[original['condition']].append(entry) # Calculate statistics stats = {} for cond in ['A', 'B', 'C']: scores = [e['blind_score'] for e in by_condition[cond] if e['blind_score'] > 0] if scores: stats[cond] = { 'mean': sum(scores) / len(scores), 'max': max(scores), 'min': min(scores), 'scores': scores, 'count_7plus': sum(1 for s in scores if s >= 7), 'count_3minus': sum(1 for s in scores if s <= 3) } else: stats[cond] = {'mean': 0, 'max': 0, 'min': 0, 'scores': [], 'count_7plus': 0, 'count_3minus': 0} # Build output output = { 'round': round_num, 'timestamp': datetime.now().isoformat(), 'total_titles': len(titles), 'raw_response': response, 'by_condition': by_condition, 'statistics': stats, 'randomization_mapping': {str(k): v for k, v in mapping.items()} } # Save output_path = os.path.join(SANDBOX_DIR, f'round{round_num}_blind_scores.json') with open(output_path, 'w') as f: json.dump(output, f, indent=2) print(f"\nResults saved to: {output_path}") # Print summary print(f"\n{'='*50}") print("BLIND SCORING SUMMARY") print(f"{'='*50}") for cond, label in [('A', 'Baseline'), ('B', 'Curiosity Mining'), ('C', 'Unclustered')]: s = stats[cond] print(f"[{cond}] {label}:") print(f" Mean score: {s['mean']:.2f}") print(f" Range: {s['min']}-{s['max']}") print(f" Titles scoring 7+: {s['count_7plus']}") print(f" Titles scoring 3 or less: {s['count_3minus']}") print() return output def main(): import sys if len(sys.argv) > 1: round_num = int(sys.argv[1]) run_blind_scoring(round_num) else: # Run both rounds print("Running blind scoring for both rounds...") run_blind_scoring(1) run_blind_scoring(2) if __name__ == "__main__": main()