138 lines
No EOL
4.9 KiB
Python
138 lines
No EOL
4.9 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Test script to demonstrate one-at-a-time cost tracking functionality
|
|
"""
|
|
|
|
import subprocess
|
|
import sys
|
|
import json
|
|
from pathlib import Path
|
|
|
|
def run_one_at_a_time_test():
|
|
"""Run a test with one-at-a-time mode and cost tracking"""
|
|
print("Testing one-at-a-time mode with cost tracking...")
|
|
print("=" * 60)
|
|
|
|
# Test command with one-at-a-time mode and cost tracking
|
|
cmd = [
|
|
sys.executable, "cli.py",
|
|
"--test",
|
|
"--openai",
|
|
"--one-at-a-time",
|
|
"--concurrent-workers", "3", # Lower concurrency for testing
|
|
"--enable-cost-tracking",
|
|
"--cost-report"
|
|
]
|
|
|
|
print(f"Running command: {' '.join(cmd)}")
|
|
print("This will make 41 separate API calls (one per master image)")
|
|
print("-" * 60)
|
|
|
|
try:
|
|
result = subprocess.run(cmd, capture_output=True, text=True, cwd=Path(__file__).parent)
|
|
|
|
print("STDOUT:")
|
|
print(result.stdout)
|
|
|
|
if result.stderr:
|
|
print("\nSTDERR:")
|
|
print(result.stderr)
|
|
|
|
print(f"\nReturn code: {result.returncode}")
|
|
|
|
# Check if cost report was generated and analyze it
|
|
cost_reports = list(Path("results").glob("cost_report_*.json"))
|
|
if cost_reports:
|
|
latest_report = cost_reports[-1]
|
|
print(f"\n✅ Cost report generated: {latest_report}")
|
|
|
|
# Analyze the cost report
|
|
try:
|
|
with open(latest_report, 'r') as f:
|
|
cost_data = json.load(f)
|
|
|
|
session_summary = cost_data.get('session_summary', {})
|
|
if session_summary.get('tracking_enabled'):
|
|
totals = session_summary.get('session_totals', {})
|
|
operation_breakdown = session_summary.get('operation_breakdown', {})
|
|
|
|
print(f"\n📊 Cost Analysis:")
|
|
print(f" Total cost: ${totals.get('total_cost', 0):.4f}")
|
|
print(f" Total API calls: {totals.get('total_api_calls', 0)}")
|
|
print(f" Total tokens: {totals.get('total_input_tokens', 0) + totals.get('total_output_tokens', 0):,}")
|
|
|
|
if operation_breakdown:
|
|
print(f"\n🔍 Operation Breakdown:")
|
|
for op_type, count in operation_breakdown.items():
|
|
print(f" {op_type}: {count} calls")
|
|
|
|
# Check for one-at-a-time detection calls
|
|
one_at_a_time_calls = operation_breakdown.get('one_at_a_time_detection', 0)
|
|
if one_at_a_time_calls > 0:
|
|
print(f"\n✅ One-at-a-time cost tracking working: {one_at_a_time_calls} individual API calls tracked")
|
|
else:
|
|
print(f"\n❌ One-at-a-time cost tracking not working: No individual API calls found")
|
|
|
|
except Exception as e:
|
|
print(f"❌ Error analyzing cost report: {e}")
|
|
else:
|
|
print("\n❌ No cost report found")
|
|
|
|
except Exception as e:
|
|
print(f"❌ Error running test: {e}")
|
|
|
|
def run_hybrid_comparison():
|
|
"""Run hybrid mode for comparison"""
|
|
print("\n\nTesting hybrid mode for cost comparison...")
|
|
print("=" * 60)
|
|
|
|
# Test hybrid mode with cost tracking
|
|
cmd = [
|
|
sys.executable, "cli.py",
|
|
"--test",
|
|
"--hybrid",
|
|
"--enable-cost-tracking"
|
|
]
|
|
|
|
print(f"Running command: {' '.join(cmd)}")
|
|
print("This will make 1 API call (panel counting + censorship)")
|
|
print("-" * 60)
|
|
|
|
try:
|
|
result = subprocess.run(cmd, capture_output=True, text=True, cwd=Path(__file__).parent)
|
|
|
|
print("STDOUT:")
|
|
print(result.stdout)
|
|
|
|
if result.stderr:
|
|
print("\nSTDERR:")
|
|
print(result.stderr)
|
|
|
|
print(f"\nReturn code: {result.returncode}")
|
|
|
|
except Exception as e:
|
|
print(f"❌ Error running test: {e}")
|
|
|
|
if __name__ == "__main__":
|
|
print("One-at-a-Time Cost Tracking Test")
|
|
print("=" * 60)
|
|
|
|
print("This test will demonstrate the cost difference between:")
|
|
print("1. One-at-a-time mode: 41 API calls (one per master)")
|
|
print("2. Hybrid mode: 1 API call (panel counting only)")
|
|
print()
|
|
|
|
# Test 1: One-at-a-time mode with cost tracking
|
|
run_one_at_a_time_test()
|
|
|
|
# Test 2: Hybrid mode for comparison
|
|
run_hybrid_comparison()
|
|
|
|
print("\n" + "=" * 60)
|
|
print("Cost comparison test completed!")
|
|
print("=" * 60)
|
|
print("\n💡 Key takeaways:")
|
|
print("- One-at-a-time mode: High accuracy, high cost (41 API calls)")
|
|
print("- Hybrid mode: Good accuracy, low cost (1 API call)")
|
|
print("- Cost tracking shows the exact difference in API usage")
|
|
print("=" * 60) |