Merge pull request #14 from Sandy247/multi-agent-comparison

feat: Multi-Agent Comparison with Scenario-Based Testing

Author: dimittal

.gitignore

@@ -1,81 +1,12 @@
-# Dependencies
-node_modules/
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-
-# Next.js
-frontend/.next/
-frontend/.next/**
-frontend/out/
-
-# Production builds
-frontend/build/
-frontend/dist/
-
-# Environment variables
-.env
-.env.local
-.env.development.local
-.env.test.local
-.env.production.local
-
-# Logs
-logs
-*.log
-
-# Runtime data
-pids
-*.pid
-*.seed
-*.pid.lock
-
-# Coverage directory used by tools like istanbul
-coverage/
-
-# nyc test coverage
-.nyc_output
-
-# Dependency directories
-jspm_packages/
-
-# Optional npm cache directory
-.npm
-
-# Optional REPL history
-.node_repl_history
-
-# Output of 'npm pack'
-*.tgz
-
-# Yarn Integrity file
-.yarn-integrity
-
-# dotenv environment variables file
-.env
-
-# IDE files
-.vscode/
-.idea/
-*.swp
-*.swo
-*~
-
-# OS generated files
-.DS_Store
-.DS_Store?
-._*
-.Spotlight-V100
-.Trashes
-ehthumbs.db
-Thumbs.db
-
 # Python
 __pycache__/
 *.py[cod]
 *$py.class
 *.so
 .Python
+env/
+venv/
+ENV/
 build/
 develop-eggs/
 dist/
@@ -91,32 +22,97 @@ wheels/
 *.egg-info/
 .installed.cfg
 *.egg
-MANIFEST
+.pytest_cache/
+.coverage
+htmlcov/
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Node
+node_modules/
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.npm
+.eslintcache
+.node_repl_history
+*.tgz
 
-# Virtual environments
+# React build
+webapp/frontend/build/
+
+# Environment files
 .env
-# Virtual environments (managed by uv)
-.venv/
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+# But keep these for reference
+!.env.production
+!.env.example
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+.DS_Store
+
+# Logs
+*.log
+logs/
+*.log.*
 
 # Database
 *.db
 *.sqlite3
 
-# Audio files (if you want to ignore local recordings)
-recordings/
-*.wav
-*.mp3
-*.m4a
-*.aac
-*.ogg
-*.flac
+# Transcripts and data
+transcripts/
+reports/*.csv
+reports/*.xlsx
+reports/*.json
 
 # Temporary files
-*.tmp
-*.temp
-.cache/
+tmp/
+temp/
+.cache/
+
+# Docker
+.dockerignore.local
+
+# Kubernetes secrets
+k8s/secrets/
+k8s/*.secret.yaml
+
+# Additional test/coverage files
+*.coverage.xml
+coverage.xml
+htmlcov/
+
+# Playwright
+.playwright-mcp/
+
+# Temporary logs
+backend-logs.txt
+*-logs.txt
+
+# Test artifacts
+test_single_redaction.py
+metrics_extraction_evaluation.py
+
+# Local scripts
+generate_shubham_metrics_csv.py
+recover_bolna_executions.py
+schedule-agent-tasks.sh
+
+# Claude Config
+CLAUDE.md
+.claude/
+
+# Next.js build artifacts
+frontend/.next/
+frontend/out/
+*.tsbuildinfo

README.md

@@ -51,6 +51,7 @@ Perfect for **call centers**, **voice bot developers**, **customer service teams
 - **🏷️ Smart Classification & Labeling**: AI-driven call categorization, sentiment analysis, and business action labeling
 - **📝 Enhanced Transcript Processing**: Automatic timestamp alignment, turn-by-turn conversation analysis, and transcript normalization
 - **🎵 Advanced Audio Analysis**: AI-powered voice analysis with pause detection, speech segmentation, and conversation health scoring
+- **🔄 Multi-Agent Comparison**: Scenario-based testing to compare multiple voice agents with AI-powered metrics
 - **☁️ S3 Integration**: Secure audio file storage with automatic format detection
 - **🌐 Modern Web UI**: Beautiful React/Next.js frontend with real-time timeline visualization
 - **🔌 Flexible Data Ingestion**: Support for both direct API calls and Bolna platform integration
@@ -413,13 +414,18 @@ python test_all_calls_pipeline.py --help
 voicesummary/
 ├── app/                          # Backend application
 │   ├── api/                      # API endpoints
-│   │   └── calls.py             # Call management API
+│   │   ├── calls.py             # Call management API
+│   │   └── agent_comparison.py   # Agent comparison API
 │   ├── integrations/             # External platform integrations
-│   │   └── fetch_bolna_calls_simple.py  # Bolna integration
+│   │   ├── fetch_bolna_calls_simple.py  # Bolna integration
+│   │   ├── bolna_agent_config_fetcher.py  # Agent config fetching
+│   │   └── manual_agent_manager.py        # Manual agent management
 │   ├── utils/                    # Utility modules
 │   │   ├── audio_processor.py    # Audio analysis & processing
 │   │   ├── improved_voice_analyzer.py  # AI voice analysis
 │   │   ├── call_data_pipeline.py # AI data extraction pipeline
+│   │   ├── comparison_orchestrator.py     # Comparison execution
+│   │   ├── conversation_simulator.py      # AI dialogue simulation
 │   │   └── s3.py                # S3 operations
 │   ├── models.py                 # Database models
 │   ├── schemas.py                # API schemas
@@ -469,6 +475,16 @@ voicesummary/
 | `GET` | `/api/calls/{call_id}/extracted-data` | Get AI-extracted data for a call |
 | `GET` | `/api/calls/{call_id}/extracted-data/status` | Get AI processing status of extracted data |
 
+### Agent Comparison
+
+| Method | Endpoint | Description |
+|--------|----------|-------------|
+| `POST` | `/api/comparisons/` | Create new agent comparison |
+| `GET` | `/api/comparisons/` | List all comparisons |
+| `GET` | `/api/comparisons/{comparison_id}` | Get comparison details |
+| `POST` | `/api/comparisons/{comparison_id}/execute` | Execute comparison |
+| `POST` | `/api/comparisons/{comparison_id}/rerun` | Re-run comparison |
+
 ## 🎯 Use Cases
 
 ### Voice Agent Analytics & AI Insights

alembic/versions/001_add_processed_data_column.py

@@ -1,16 +1,18 @@
 """Add processed_data column to audio_calls table
 
 Revision ID: 001
-Revises: 
+Revises:
 Create Date: 2025-01-27 10:00:00.000000
 
 """
-from alembic import op
+
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
+from alembic import op
+
 # revision identifiers, used by Alembic.
-revision = '001'
+revision = "001"
 down_revision = None
 branch_labels = None
 depends_on = None
@@ -21,10 +23,12 @@ def upgrade():
     # Check if the column already exists
     connection = op.get_bind()
     inspector = sa.inspect(connection)
-    columns = [col['name'] for col in inspector.get_columns('audio_calls')]
-    
-    if 'processed_data' not in columns:
-        op.add_column('audio_calls', sa.Column('processed_data', postgresql.JSON, nullable=True))
+    columns = [col["name"] for col in inspector.get_columns("audio_calls")]
+
+    if "processed_data" not in columns:
+        op.add_column(
+            "audio_calls", sa.Column("processed_data", postgresql.JSON, nullable=True)
+        )
         print("Added processed_data column to audio_calls table")
     else:
         print("processed_data column already exists in audio_calls table")
@@ -35,10 +39,10 @@ def downgrade():
     # Check if the column exists before trying to drop it
     connection = op.get_bind()
     inspector = sa.inspect(connection)
-    columns = [col['name'] for col in inspector.get_columns('audio_calls')]
-    
-    if 'processed_data' in columns:
-        op.drop_column('audio_calls', 'processed_data')
+    columns = [col["name"] for col in inspector.get_columns("audio_calls")]
+
+    if "processed_data" in columns:
+        op.drop_column("audio_calls", "processed_data")
         print("Removed processed_data column from audio_calls table")
     else:
         print("processed_data column does not exist in audio_calls table")