#!/usr/bin/env python3 """ Cross-channel blind scoring test for v4 premises. Strips all context, randomizes order, sends to Opus as TikTok scroller. """ import json import random import subprocess import sys from datetime import datetime CHANNELS = [ 'why_you_do_that', 'how_it_actually_works', 'sixty_second_rabbit_hole', 'designed_to_trick_you', 'the_money_thing', 'what_happens_next', 'one_minute_history' ] OUTPUT_DIR = '/home/sietch6/trending-topics-pipeline/output' SANDBOX_DIR = '/home/sietch6/trending-topics-pipeline/sandbox' def run_cross_channel_blind(): # Collect all MAKE_NOW and WORTH_MAKING premises all_premises = [] for channel in CHANNELS: filepath = f"{OUTPUT_DIR}/analysis_opportunities_{channel}.json" try: with open(filepath) as f: data = json.load(f) for opp in data.get('opportunities', []): verdict = opp.get('verdict', 'SKIP') if verdict in ('MAKE_NOW', 'WORTH_MAKING'): premise = opp.get('premise', opp.get('suggested_title', '')) if premise and premise.strip(): all_premises.append({ 'premise': premise, 'actual_channel': channel, 'actual_verdict': verdict, 'actual_weighted_score': opp.get('weighted_score', 0) }) except Exception as e: print(f"Error loading {channel}: {e}") print(f"Collected {len(all_premises)} premises from {len(CHANNELS)} channels") print(f" MAKE_NOW: {sum(1 for p in all_premises if p['actual_verdict'] == 'MAKE_NOW')}") print(f" WORTH_MAKING: {sum(1 for p in all_premises if p['actual_verdict'] == 'WORTH_MAKING')}") # Shuffle for blind test random.seed(42) # Reproducible shuffled = all_premises.copy() random.shuffle(shuffled) # Build prompt with just premises premises_list = "\n".join([f"{i+1}. {p['premise']}" for i, p in enumerate(shuffled)]) prompt = f"""You are scrolling TikTok at 11pm. You're tired. You don't want to learn anything. Between each premise below, imagine you just watched a cat video and a cooking video. Rate each premise 1-10 on ONE thing only: would your thumb physically stop moving? 10 = I literally cannot scroll past this 8-9 = I'm stopping and watching 6-7 = I'd pause, maybe watch 4-5 = Only if I'm already interested in this topic 2-3 = Generic, would scroll past 1 = Would not register at all For each one, give the score and ONE sentence explaining why you stopped or didn't. No other commentary. Here are the premises: {premises_list} Respond in JSON format: {{ "scores": [ {{"index": 1, "score": N, "reason": "one sentence"}}, ... ] }} """ print(f"\nSending {len(shuffled)} premises to Opus for blind scoring...") result = subprocess.run( ["claude", "--print", "--model", "opus", "-p", "-"], input=prompt, capture_output=True, text=True, timeout=900 ) if result.returncode != 0: print(f"Error: {result.stderr}") return # Parse response import re response = result.stdout.strip() json_match = re.search(r'\{.*\}', response, re.DOTALL) if not json_match: print("Could not parse JSON response") print(response[:2000]) return blind_scores = json.loads(json_match.group()) # Map scores back to premises results = [] for score_entry in blind_scores.get('scores', []): idx = score_entry['index'] - 1 if 0 <= idx < len(shuffled): original = shuffled[idx] results.append({ 'premise': original['premise'], 'blind_score': score_entry['score'], 'blind_reason': score_entry.get('reason', ''), 'actual_channel': original['actual_channel'], 'actual_verdict': original['actual_verdict'], 'actual_weighted_score': original['actual_weighted_score'] }) # Sort by blind score descending results.sort(key=lambda x: x['blind_score'], reverse=True) # Calculate statistics make_now = [r for r in results if r['actual_verdict'] == 'MAKE_NOW'] worth_making = [r for r in results if r['actual_verdict'] == 'WORTH_MAKING'] avg_make_now = sum(r['blind_score'] for r in make_now) / len(make_now) if make_now else 0 avg_worth_making = sum(r['blind_score'] for r in worth_making) / len(worth_making) if worth_making else 0 # Per-channel averages channel_scores = {} for r in results: ch = r['actual_channel'] if ch not in channel_scores: channel_scores[ch] = [] channel_scores[ch].append(r['blind_score']) channel_avgs = {ch: sum(scores)/len(scores) for ch, scores in channel_scores.items()} # Weak premises (scored 4 or below) weak = [r for r in results if r['blind_score'] <= 4] output = { 'test_date': datetime.now().isoformat(), 'total_premises': len(results), 'statistics': { 'avg_make_now': round(avg_make_now, 2), 'avg_worth_making': round(avg_worth_making, 2), 'make_now_count': len(make_now), 'worth_making_count': len(worth_making), 'channel_averages': {ch: round(avg, 2) for ch, avg in sorted(channel_avgs.items(), key=lambda x: -x[1])}, 'weak_premises_count': len(weak) }, 'results': results } # Save output_path = f"{SANDBOX_DIR}/v4_cross_channel_blind_scores.json" with open(output_path, 'w') as f: json.dump(output, f, indent=2) print(f"\n{'='*60}") print("CROSS-CHANNEL BLIND SCORING RESULTS") print(f"{'='*60}") print(f"\nTotal premises scored: {len(results)}") print(f"\nAVERAGE BLIND SCORE BY VERDICT:") print(f" MAKE_NOW ({len(make_now)}): {avg_make_now:.2f}") print(f" WORTH_MAKING ({len(worth_making)}): {avg_worth_making:.2f}") print(f" Delta: {avg_make_now - avg_worth_making:+.2f}") print(f"\n{'='*60}") print("TOP 10 PREMISES (by blind score)") print(f"{'='*60}") for i, r in enumerate(results[:10]): print(f" [{r['blind_score']}] ({r['actual_verdict'][:4]}) {r['premise'][:60]}...") print(f" Channel: {r['actual_channel']}, Internal score: {r['actual_weighted_score']}") print(f"\n{'='*60}") print("BOTTOM 10 PREMISES (MAKE_NOW/WORTH_MAKING only)") print(f"{'='*60}") for r in results[-10:]: print(f" [{r['blind_score']}] ({r['actual_verdict'][:4]}) {r['premise'][:60]}...") print(f" Channel: {r['actual_channel']}, Internal score: {r['actual_weighted_score']}") print(f"\n{'='*60}") print("AVERAGE BLIND SCORE BY CHANNEL") print(f"{'='*60}") for ch, avg in sorted(channel_avgs.items(), key=lambda x: -x[1]): count = len(channel_scores[ch]) print(f" {ch}: {avg:.2f} (n={count})") print(f"\n{'='*60}") print(f"WEAK PREMISES (blind score <= 4): {len(weak)}") print(f"{'='*60}") if weak: for r in weak: print(f" [{r['blind_score']}] ({r['actual_verdict'][:4]}) {r['premise'][:60]}...") print(f" Channel: {r['actual_channel']}") print(f" Reason: {r['blind_reason']}") else: print(" None - all premises scored 5+") print(f"\nResults saved to: {output_path}") if __name__ == '__main__': run_cross_channel_blind()