major-domo-database/validate_migration.py

#!/usr/bin/env python3

import os
import logging
from collections import defaultdict

logger = logging.getLogger(f'{__name__}.validate_migration')

def compare_table_counts():
    """Compare record counts between SQLite and PostgreSQL"""
    logger.info("Comparing table record counts...")
    
    # Get all models
    os.environ['DATABASE_TYPE'] = 'sqlite'
    from app.db_engine import (
        Current, Manager, Division, SbaPlayer, Team, Player,
        Result, Schedule, Transaction, BattingStat, PitchingStat,
        Standings, BattingCareer, PitchingCareer, FieldingCareer,
        BattingSeason, PitchingSeason, FieldingSeason,
        DraftPick, DraftData, DraftList, Award, DiceRoll,
        Keeper, Injury, StratGame, StratPlay, Decision,
        CustomCommandCreator, CustomCommand
    )
    
    all_models = [
        Current, Manager, Division, SbaPlayer, Team, Player,
        Result, Schedule, Transaction, BattingStat, PitchingStat,
        Standings, BattingCareer, PitchingCareer, FieldingCareer,
        BattingSeason, PitchingSeason, FieldingSeason,
        DraftPick, DraftData, DraftList, Award, DiceRoll,
        Keeper, Injury, StratGame, StratPlay, Decision,
        CustomCommandCreator, CustomCommand
    ]
    
    results = {}
    
    for model in all_models:
        table_name = model._meta.table_name
        
        try:
            # SQLite count
            os.environ['DATABASE_TYPE'] = 'sqlite'
            from peewee import SqliteDatabase
            sqlite_db = SqliteDatabase('storage/sba_master.db')
            model._meta.database = sqlite_db
            sqlite_db.connect()
            sqlite_count = model.select().count()
            sqlite_db.close()
            
            # PostgreSQL count  
            os.environ['DATABASE_TYPE'] = 'postgresql'
            from peewee import PostgresqlDatabase
            
            # Debug: Print what credentials we're using
            db_name = os.environ.get('SBA_DATABASE', 'sba_master')
            db_user = os.environ.get('SBA_DB_USER', 'sba_admin')  
            db_password = os.environ.get('SBA_DB_USER_PASSWORD', 'your_production_password')
            db_host = os.environ.get('POSTGRES_HOST', 'localhost')
            db_port = int(os.environ.get('POSTGRES_PORT', '5432'))
            
            if table_name == 'current':  # Only print debug info once
                logger.info(f"Debug - Connecting to: {db_host}:{db_port}, DB: {db_name}, User: {db_user}, Pass: {'*' * len(db_password)}")
            
            postgres_db = PostgresqlDatabase(
                db_name, user=db_user, password=db_password,
                host=db_host, port=db_port
            )
            model._meta.database = postgres_db
            postgres_db.connect()
            postgres_count = model.select().count()
            postgres_db.close()
            
            results[table_name] = {
                'sqlite': sqlite_count,
                'postgres': postgres_count,
                'match': sqlite_count == postgres_count
            }
            
            status = "✓" if sqlite_count == postgres_count else "✗"
            logger.info(f"  {status} {table_name:20} SQLite: {sqlite_count:6} | PostgreSQL: {postgres_count:6}")
            
        except Exception as e:
            logger.error(f"  ✗ {table_name:20} Error: {e}")
            results[table_name] = {
                'sqlite': 'ERROR',
                'postgres': 'ERROR', 
                'match': False
            }
    
    return results

def validate_sample_data():
    """Validate specific records exist in both databases"""
    logger.info("Validating sample data integrity...")
    
    validations = []
    
    try:
        # Check Current table
        os.environ['DATABASE_TYPE'] = 'sqlite'
        from app.db_engine import Current
        from peewee import SqliteDatabase
        
        sqlite_db = SqliteDatabase('storage/sba_master.db')
        Current._meta.database = sqlite_db
        sqlite_db.connect()
        
        if Current.select().exists():
            sqlite_current = Current.select().first()
            sqlite_season = sqlite_current.season if sqlite_current else None
            sqlite_db.close()
            
            # Check in PostgreSQL
            os.environ['DATABASE_TYPE'] = 'postgresql'
            from peewee import PostgresqlDatabase
            
            postgres_db = PostgresqlDatabase(
                'sba_master', user='sba_admin',
                password='sba_dev_password_2024', 
                host='localhost', port=5432
            )
            Current._meta.database = postgres_db
            postgres_db.connect()
            
            if Current.select().exists():
                postgres_current = Current.select().first()
                postgres_season = postgres_current.season if postgres_current else None
                
                if sqlite_season == postgres_season:
                    logger.info(f"  ✓ Current season matches: {sqlite_season}")
                    validations.append(True)
                else:
                    logger.error(f"  ✗ Current season mismatch: SQLite={sqlite_season}, PostgreSQL={postgres_season}")
                    validations.append(False)
            else:
                logger.error("  ✗ No Current record in PostgreSQL")
                validations.append(False)
                
            postgres_db.close()
        else:
            logger.info("  - No Current records to validate")
            validations.append(True)
            sqlite_db.close()
            
    except Exception as e:
        logger.error(f"  ✗ Sample data validation error: {e}")
        validations.append(False)
    
    return all(validations)

def generate_migration_report(count_results, sample_validation):
    """Generate comprehensive migration report"""
    logger.info("\n" + "="*60)
    logger.info("MIGRATION VALIDATION REPORT")
    logger.info("="*60)
    
    # Count summary
    total_tables = len(count_results)
    matching_tables = sum(1 for r in count_results.values() if r['match'])
    
    logger.info(f"Tables analyzed: {total_tables}")
    logger.info(f"Count matches: {matching_tables}/{total_tables}")
    
    if matching_tables == total_tables:
        logger.info("✅ ALL TABLE COUNTS MATCH!")
    else:
        logger.error(f"❌ {total_tables - matching_tables} tables have count mismatches")
        
        # Show mismatches
        logger.info("\nMismatched tables:")
        for table, result in count_results.items():
            if not result['match']:
                logger.error(f"  {table}: SQLite={result['sqlite']}, PostgreSQL={result['postgres']}")
    
    # Sample data validation
    if sample_validation:
        logger.info("✅ Sample data validation passed")
    else:
        logger.error("❌ Sample data validation failed")
    
    # Overall status
    migration_success = (matching_tables == total_tables) and sample_validation
    
    logger.info("\n" + "="*60)
    if migration_success:
        logger.info("🎉 MIGRATION VALIDATION: SUCCESS")
        logger.info("✅ Ready for production migration")
    else:
        logger.error("❌ MIGRATION VALIDATION: FAILED")
        logger.error("⚠️  Issues must be resolved before production")
    logger.info("="*60)
    
    return migration_success

def main():
    logging.basicConfig(
        level=logging.INFO,
        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    )
    
    logger.info("Starting migration validation...")
    
    # Run validations
    count_results = compare_table_counts()
    sample_validation = validate_sample_data()
    
    # Generate report
    success = generate_migration_report(count_results, sample_validation)
    
    return 0 if success else 1

if __name__ == "__main__":
    exit(main())