diff --git a/.gitignore b/.gitignore index 92bc6b6..acdd30b 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,4 @@ db_engine.py sba_master.db db_engine.py venv +website/sba diff --git a/=2.9.0 b/=2.9.0 new file mode 100644 index 0000000..36e3d2c --- /dev/null +++ b/=2.9.0 @@ -0,0 +1,2 @@ +Defaulting to user installation because normal site-packages is not writeable +Requirement already satisfied: psycopg2-binary in /home/cal/.local/lib/python3.13/site-packages (2.9.10) diff --git a/DATA_SANITIZATION_TEMPLATE.md b/DATA_SANITIZATION_TEMPLATE.md new file mode 100644 index 0000000..60a4aa8 --- /dev/null +++ b/DATA_SANITIZATION_TEMPLATE.md @@ -0,0 +1,232 @@ +# Data Sanitization Template for PostgreSQL Migration + +## Template Structure +Each data sanitization issue should follow this standardized format for consistent tracking and resolution. + +--- + +## Issue Template + +### Issue ID: [CATEGORY]-[TABLE]-[FIELD]-[NUMBER] +**Example**: `CONSTRAINT-CURRENT-INJURY_COUNT-001` + +### ๐Ÿ“Š Issue Classification +- **Category**: [SCHEMA|DATA_INTEGRITY|DATA_QUALITY|MIGRATION_LOGIC] +- **Priority**: [CRITICAL|HIGH|MEDIUM|LOW] +- **Impact**: [BLOCKS_MIGRATION|DATA_LOSS|PERFORMANCE|COSMETIC] +- **Table(s)**: [table_name, related_tables] +- **Field(s)**: [field_names] + +### ๐Ÿ” Problem Description +**What happened:** +Clear description of the error or issue encountered. + +**Error Message:** +``` +Exact error message from logs +``` + +**Expected Behavior:** +What should happen in a successful migration. + +**Current Behavior:** +What actually happens. + +### ๐Ÿ“ˆ Impact Assessment +**Data Affected:** +- Records: X out of Y total +- Percentage: Z% +- Critical data: YES/NO + +**Business Impact:** +- User-facing features affected +- Operational impact +- Compliance/audit concerns + +### ๐Ÿ”ง Root Cause Analysis +**Technical Cause:** +- SQLite vs PostgreSQL difference +- Data model assumption +- Migration logic flaw + +**Data Source:** +- How did this data get into this state? +- Is this expected or corrupted data? +- Historical context + +### ๐Ÿ’ก Solution Strategy +**Approach:** [TRANSFORM_DATA|FIX_SCHEMA|MIGRATION_LOGIC|SKIP_TABLE] + +**Technical Solution:** +Detailed explanation of how to fix the issue. + +**Data Transformation Required:** +```sql +-- Example transformation query +UPDATE table_name +SET field_name = COALESCE(field_name, default_value) +WHERE field_name IS NULL; +``` + +### โœ… Implementation Plan +**Steps:** +1. [ ] Backup current state +2. [ ] Implement fix +3. [ ] Test on sample data +4. [ ] Run full migration test +5. [ ] Validate results +6. [ ] Document changes + +**Rollback Plan:** +How to undo changes if something goes wrong. + +### ๐Ÿงช Testing Strategy +**Test Cases:** +1. Happy path: Normal data migrates correctly +2. Edge case: Problem data is handled properly +3. Regression: Previous fixes still work + +**Validation Queries:** +```sql +-- Query to verify fix worked +SELECT COUNT(*) FROM table_name WHERE condition; +``` + +### ๐Ÿ“‹ Resolution Status +- **Status**: [IDENTIFIED|IN_PROGRESS|TESTING|RESOLVED|DEFERRED] +- **Assigned To**: [team_member] +- **Date Identified**: YYYY-MM-DD +- **Date Resolved**: YYYY-MM-DD +- **Solution Applied**: [description] + +--- + +## ๐Ÿ“š Example Issues (From Our Testing) + +### Issue ID: CONSTRAINT-CURRENT-INJURY_COUNT-001 +**Category**: SCHEMA +**Priority**: HIGH +**Impact**: BLOCKS_MIGRATION + +**Problem Description:** +`injury_count` field in `current` table has NULL values in SQLite but PostgreSQL schema requires NOT NULL. + +**Error Message:** +``` +null value in column "injury_count" of relation "current" violates not-null constraint +``` + +**Solution Strategy:** TRANSFORM_DATA +```sql +-- Transform NULL values to 0 before migration +UPDATE current SET injury_count = 0 WHERE injury_count IS NULL; +``` + +**Implementation:** +1. Add data transformation in migration script +2. Set default value for future records +3. Update schema if business logic allows NULL + +--- + +### Issue ID: DATA_QUALITY-PLAYER-NAME-001 +**Category**: DATA_QUALITY +**Priority**: MEDIUM +**Impact**: DATA_LOSS + +**Problem Description:** +Player names exceed PostgreSQL VARCHAR(255) limit causing truncation. + +**Error Message:** +``` +value too long for type character varying(255) +``` + +**Solution Strategy:** FIX_SCHEMA +```sql +-- Increase column size in PostgreSQL +ALTER TABLE player ALTER COLUMN name TYPE VARCHAR(500); +``` + +**Implementation:** +1. Analyze max string lengths in SQLite +2. Update PostgreSQL schema with appropriate limits +3. Add validation to prevent future overruns + +--- + +### Issue ID: MIGRATION_LOGIC-TEAM-INTEGER-001 +**Category**: MIGRATION_LOGIC +**Priority**: HIGH +**Impact**: BLOCKS_MIGRATION + +**Problem Description:** +Large integer values in SQLite exceed PostgreSQL INTEGER range. + +**Error Message:** +``` +integer out of range +``` + +**Solution Strategy:** FIX_SCHEMA +```sql +-- Use BIGINT instead of INTEGER +ALTER TABLE team ALTER COLUMN large_field TYPE BIGINT; +``` + +**Implementation:** +1. Identify fields with large values +2. Update schema to use BIGINT +3. Verify no application code assumes INTEGER size + +--- + +## ๐ŸŽฏ Standard Solution Patterns + +### Pattern 1: NULL Constraint Violations +```python +# Pre-migration data cleaning +def clean_null_constraints(table_name, field_name, default_value): + query = f"UPDATE {table_name} SET {field_name} = ? WHERE {field_name} IS NULL" + sqlite_db.execute_sql(query, (default_value,)) +``` + +### Pattern 2: String Length Overruns +```python +# Schema adjustment +def adjust_varchar_limits(table_name, field_name, new_limit): + query = f"ALTER TABLE {table_name} ALTER COLUMN {field_name} TYPE VARCHAR({new_limit})" + postgres_db.execute_sql(query) +``` + +### Pattern 3: Integer Range Issues +```python +# Type upgrade +def upgrade_integer_fields(table_name, field_name): + query = f"ALTER TABLE {table_name} ALTER COLUMN {field_name} TYPE BIGINT" + postgres_db.execute_sql(query) +``` + +### Pattern 4: Missing Table Handling +```python +# Graceful table skipping +def safe_table_migration(model_class): + try: + migrate_table_data(model_class) + except Exception as e: + if "no such table" in str(e): + logger.warning(f"Table {model_class._meta.table_name} doesn't exist in source") + return True + raise +``` + +## ๐Ÿ“Š Issue Tracking Spreadsheet Template + +| Issue ID | Category | Priority | Table | Field | Status | Date Found | Date Fixed | Notes | +|----------|----------|----------|-------|-------|--------|------------|------------|-------| +| CONSTRAINT-CURRENT-INJURY_COUNT-001 | SCHEMA | HIGH | current | injury_count | RESOLVED | 2025-01-15 | 2025-01-15 | Set NULL to 0 | +| DATA_QUALITY-PLAYER-NAME-001 | DATA_QUALITY | MEDIUM | player | name | IN_PROGRESS | 2025-01-15 | | Increase VARCHAR limit | + +--- + +*This template ensures consistent documentation and systematic resolution of migration issues.* \ No newline at end of file diff --git a/MIGRATION_METHODOLOGY.md b/MIGRATION_METHODOLOGY.md new file mode 100644 index 0000000..1b000fa --- /dev/null +++ b/MIGRATION_METHODOLOGY.md @@ -0,0 +1,210 @@ +# PostgreSQL Migration Testing Methodology + +## Overview +This document outlines the systematic approach for testing and refining the SQLite to PostgreSQL migration process before production deployment. + +## ๐Ÿ”„ Iterative Testing Cycle + +### Phase 1: Discovery Testing +**Goal**: Identify all migration issues without fixing them + +```bash +# Run discovery cycle +./test_migration_workflow.sh > migration_test_$(date +%Y%m%d_%H%M%S).log 2>&1 +``` + +**Process**: +1. Reset PostgreSQL database +2. Run migration attempt +3. **Document ALL errors** (don't fix immediately) +4. Categorize issues by type +5. Assess impact and priority + +### Phase 2: Systematic Issue Resolution +**Goal**: Fix issues one category at a time + +**Priority Order**: +1. **Schema Issues** (data types, constraints) +2. **Data Integrity** (NULL values, foreign keys) +3. **Data Quality** (string lengths, integer ranges) +4. **Missing Dependencies** (table existence) +5. **Performance Issues** (batch sizes, indexing) + +### Phase 3: Validation Testing +**Goal**: Verify fixes and ensure no regressions + +```bash +# Run validation cycle +./test_migration_workflow.sh +python validate_migration.py +``` + +### Phase 4: Production Readiness +**Goal**: Final verification before production + +```bash +# Final comprehensive test +./production_readiness_check.sh +``` + +## ๐Ÿ“Š Issue Tracking System + +### Issue Categories + +#### 1. Schema Compatibility +- **NULL Constraints**: Fields that require values in PostgreSQL +- **Data Types**: Type mismatches (BigInt, Varchar limits) +- **Constraints**: Unique, foreign key, check constraints + +#### 2. Data Integrity +- **Foreign Key Violations**: Missing parent records +- **Orphaned Records**: Child records without parents +- **Referential Integrity**: Cross-table consistency + +#### 3. Data Quality +- **String Length**: Values exceeding column limits +- **Number Range**: Values outside PostgreSQL type ranges +- **Date/Time Format**: Incompatible date representations + +#### 4. Migration Logic +- **Table Dependencies**: Incorrect migration order +- **Batch Processing**: Memory/performance issues +- **Transaction Handling**: Rollback scenarios + +## ๐ŸŽฏ Testing Success Criteria + +### โœ… Complete Success +- All tables migrated: **100%** +- All record counts match: **100%** +- Data validation passes: **100%** +- Performance acceptable: **< 2x SQLite query time** + +### โš ๏ธ Partial Success +- Core tables migrated: **โ‰ฅ 90%** +- Critical data intact: **โ‰ฅ 95%** +- Documented workarounds for remaining issues + +### โŒ Failure +- Core tables failed: **> 10%** +- Data corruption detected +- Performance degradation: **> 5x SQLite** + +## ๐Ÿ“ˆ Progress Tracking + +### Test Run Template +``` +Date: YYYY-MM-DD HH:MM +Test Run #: X +Previous Issues: X resolved, Y remaining +New Issues Found: Z + +Results: +- Tables Migrated: X/Y (Z%) +- Records Migrated: X,XXX,XXX total +- Validation Status: PASS/FAIL +- Test Duration: X minutes + +Issues Resolved This Run: +1. [CATEGORY] Description - Fix applied +2. [CATEGORY] Description - Fix applied + +New Issues Found: +1. [CATEGORY] Description - Priority: HIGH/MED/LOW +2. [CATEGORY] Description - Priority: HIGH/MED/LOW + +Next Actions: +- [ ] Fix issue #1 +- [ ] Test scenario X +- [ ] Validate table Y +``` + +## ๐Ÿ”ง Development Workflow + +### Before Each Test Run +```bash +# 1. Document current state +git status +git add -A +git commit -m "Pre-test state: $(date)" + +# 2. Reset environment +python reset_postgres.py + +# 3. Run test with logging +./test_migration_workflow.sh | tee "logs/test_run_$(date +%Y%m%d_%H%M%S).log" +``` + +### After Each Test Run +```bash +# 1. Document results +cp migration_issues.md migration_issues_$(date +%Y%m%d).md + +# 2. Update issue tracker +echo "## Test Run $(date)" >> migration_progress.md + +# 3. Commit progress +git add -A +git commit -m "Test run $(date): X issues resolved, Y remaining" +``` + +## ๐Ÿšจ Critical Guidelines + +### DO NOT Skip Steps +- Always reset database between tests +- Always run full validation after fixes +- Always document issues before fixing + +### DO NOT Batch Fix Issues +- Fix one category at a time +- Test after each category of fixes +- Verify no regressions introduced + +### DO NOT Ignore "Minor" Issues +- All data discrepancies must be documented +- Even successful migrations need validation +- Performance issues compound in production + +## ๐Ÿ“‹ Pre-Production Checklist + +### Data Verification +- [ ] All table counts match exactly +- [ ] Sample data integrity verified +- [ ] Foreign key relationships intact +- [ ] No data truncation occurred +- [ ] Character encoding preserved + +### Performance Verification +- [ ] Migration completes within time window +- [ ] Query performance acceptable +- [ ] Index creation successful +- [ ] Connection pooling tested + +### Operational Verification +- [ ] Backup/restore procedures tested +- [ ] Rollback plan verified +- [ ] Monitoring alerts configured +- [ ] Documentation updated + +### Security Verification +- [ ] Access controls migrated +- [ ] Connection security verified +- [ ] Audit trail preserved +- [ ] Compliance requirements met + +## ๐ŸŽฏ Success Metrics + +### Quantitative +- **Data Accuracy**: 100% record count match +- **Data Quality**: 0 corruption events +- **Performance**: โ‰ค 2x query time vs SQLite +- **Availability**: < 1 hour downtime + +### Qualitative +- **Confidence Level**: High team confidence +- **Documentation**: Complete and accurate +- **Rollback Plan**: Tested and verified +- **Team Training**: Staff ready for operations + +--- + +*This methodology ensures systematic, repeatable testing that builds confidence for production migration.* \ No newline at end of file diff --git a/PRODUCTION_MIGRATION_CHECKLIST.md b/PRODUCTION_MIGRATION_CHECKLIST.md new file mode 100644 index 0000000..67c8f95 --- /dev/null +++ b/PRODUCTION_MIGRATION_CHECKLIST.md @@ -0,0 +1,200 @@ +# Production Migration Checklist + +## ๐ŸŽฏ Pre-Migration Requirements +**ALL items must be completed before production migration** + +### โœ… Testing Validation +- [ ] **100% sandbox migration success** - All tables migrated without errors +- [ ] **Data integrity verified** - Record counts match exactly between SQLite/PostgreSQL +- [ ] **Performance acceptable** - Migration completes within maintenance window +- [ ] **Rollback tested** - Confirmed ability to revert to SQLite if needed +- [ ] **Team sign-off** - Technical and business stakeholders approve migration + +### โœ… Infrastructure Readiness +- [ ] **PostgreSQL server provisioned** - Production hardware/cloud resources ready +- [ ] **Backup systems configured** - Automated backups for PostgreSQL +- [ ] **Monitoring enabled** - Database performance and health monitoring +- [ ] **Security hardened** - Access controls, encryption, firewall rules +- [ ] **Network connectivity** - Application servers can reach PostgreSQL + +### โœ… Application Readiness +- [ ] **Environment variables updated** - `DATABASE_TYPE=postgresql` in production +- [ ] **Connection pooling configured** - Appropriate pool sizes for PostgreSQL +- [ ] **Dependencies updated** - `psycopg2-binary` installed in production +- [ ] **Query compatibility verified** - All LIKE queries use ILIKE where needed +- [ ] **Error handling updated** - PostgreSQL-specific error codes handled + +--- + +## ๐Ÿš€ Migration Day Execution Plan + +### Phase 1: Pre-Migration (T-2 hours) +- [ ] **Notify stakeholders** - Send migration start notification +- [ ] **Application maintenance mode** - Put application in read-only mode +- [ ] **Final SQLite backup** - Create point-in-time backup +- [ ] **PostgreSQL preparation** - Ensure target database is ready +- [ ] **Team assembly** - All migration team members available + +### Phase 2: Data Migration (T-0 to T+X) +- [ ] **Start migration script** - Begin `migrate_to_postgres.py` +- [ ] **Monitor progress** - Track migration status and performance +- [ ] **Handle issues** - Apply pre-tested fixes for any problems +- [ ] **Validate data integrity** - Run `validate_migration.py` +- [ ] **Performance verification** - Test key application queries + +### Phase 3: Application Cutover (T+X to T+Y) +- [ ] **Update environment variables** - Switch to PostgreSQL +- [ ] **Restart application services** - Deploy with PostgreSQL configuration +- [ ] **Connection testing** - Verify application connects successfully +- [ ] **Functional testing** - Test critical user workflows +- [ ] **Performance monitoring** - Watch for performance issues + +### Phase 4: Go-Live Verification (T+Y to T+Z) +- [ ] **Remove maintenance mode** - Enable full application access +- [ ] **User acceptance testing** - Key users verify functionality +- [ ] **Performance monitoring** - Monitor system under user load +- [ ] **Data consistency checks** - Ongoing validation of data integrity +- [ ] **Issue escalation** - Address any problems immediately + +--- + +## ๐Ÿ”„ Rollback Procedures + +### Immediate Rollback Triggers +- **Data corruption detected** - Any indication of data loss or corruption +- **Performance degradation > 5x** - Unacceptable application performance +- **Critical functionality broken** - Core features not working +- **Security concerns** - Any security-related issues discovered +- **Team consensus** - Migration team agrees rollback is necessary + +### Rollback Execution (Emergency) +```bash +# 1. Immediate application switch +export DATABASE_TYPE=sqlite +systemctl restart application + +# 2. Verify SQLite connectivity +python test_sqlite_connection.py + +# 3. Notify stakeholders +send_rollback_notification.sh + +# 4. Document issues +echo "Rollback executed at $(date): $(reason)" >> rollback_log.md +``` + +### Post-Rollback Actions +- [ ] **Root cause analysis** - Identify what went wrong +- [ ] **Issue documentation** - Update migration issues tracker +- [ ] **Fix development** - Address problems in sandbox +- [ ] **Retest thoroughly** - Validate fixes before retry +- [ ] **Stakeholder communication** - Explain rollback and next steps + +--- + +## ๐Ÿ“Š Success Criteria + +### Quantitative Metrics +- **Migration time**: < 4 hours total +- **Data accuracy**: 100% record count match +- **Query performance**: < 2x SQLite response times +- **Downtime**: < 1 hour application unavailability +- **Error rate**: 0 critical errors during migration + +### Qualitative Indicators +- **User experience**: No degradation in functionality +- **System stability**: No crashes or instability +- **Team confidence**: High confidence in system reliability +- **Monitoring clean**: No critical alerts or warnings +- **Documentation current**: All procedures documented + +--- + +## ๐Ÿ”ง Emergency Contacts + +### Technical Team +- **Database Admin**: [contact] - PostgreSQL expertise +- **Application Lead**: [contact] - Application configuration +- **DevOps Engineer**: [contact] - Infrastructure support +- **QA Lead**: [contact] - Testing and validation + +### Business Team +- **Product Owner**: [contact] - Business impact decisions +- **Operations Manager**: [contact] - User communication +- **Security Officer**: [contact] - Security concerns +- **Compliance Lead**: [contact] - Regulatory requirements + +--- + +## ๐Ÿ“‹ Pre-Flight Checklist (Final 24 Hours) + +### Technical Verification +- [ ] **Sandbox migration** - Final successful test run completed +- [ ] **Production database** - PostgreSQL server health verified +- [ ] **Backup systems** - All backup procedures tested +- [ ] **Monitoring dashboards** - All alerts and monitors active +- [ ] **Rollback plan** - Tested and verified within 2 hours + +### Operational Readiness +- [ ] **Team availability** - All key personnel confirmed available +- [ ] **Communication plan** - Stakeholder notifications prepared +- [ ] **Change control** - All approvals and documentation complete +- [ ] **Risk assessment** - Mitigation strategies for known risks +- [ ] **Go/No-Go decision** - Final stakeholder approval received + +### Documentation Current +- [ ] **Migration procedures** - Step-by-step instructions updated +- [ ] **Troubleshooting guides** - Known issues and solutions documented +- [ ] **Contact information** - All emergency contacts verified +- [ ] **Rollback procedures** - Clear instructions for reverting changes +- [ ] **Post-migration tasks** - Cleanup and optimization steps defined + +--- + +## ๐Ÿ“ˆ Post-Migration Activities + +### Immediate (24 hours) +- [ ] **Performance monitoring** - Watch system metrics closely +- [ ] **User feedback collection** - Gather user experience reports +- [ ] **Data validation** - Ongoing verification of data integrity +- [ ] **Issue tracking** - Document any problems discovered +- [ ] **Team debrief** - Capture lessons learned + +### Short-term (1 week) +- [ ] **Performance optimization** - Tune PostgreSQL configuration +- [ ] **Index optimization** - Create additional indexes if needed +- [ ] **Monitoring refinement** - Adjust alert thresholds +- [ ] **Documentation updates** - Update operational procedures +- [ ] **Training completion** - Ensure team is trained on PostgreSQL + +### Long-term (1 month) +- [ ] **SQLite decommission** - Remove old SQLite infrastructure +- [ ] **Backup verification** - Validate PostgreSQL backup/restore +- [ ] **Performance baseline** - Establish new performance standards +- [ ] **Security audit** - Complete security review of new system +- [ ] **Success metrics** - Measure and report migration success + +--- + +## ๐ŸŽฏ Final Authorization + +### Sign-off Required +- [ ] **Database Administrator**: _________________ Date: _________ +- [ ] **Application Lead**: _________________ Date: _________ +- [ ] **DevOps Engineer**: _________________ Date: _________ +- [ ] **QA Lead**: _________________ Date: _________ +- [ ] **Product Owner**: _________________ Date: _________ +- [ ] **Security Officer**: _________________ Date: _________ + +### Migration Approval +**Authorized to proceed with production migration**: YES / NO + +**Authorized by**: _________________ **Date**: _________ **Time**: _________ + +**Migration window**: Start: _________ End: _________ + +**Rollback deadline**: _________ + +--- + +*This checklist ensures systematic, safe migration to production with clear success criteria and rollback procedures.* \ No newline at end of file diff --git a/app/db_engine.py b/app/db_engine.py index b6f2495..f4b4e90 100644 --- a/app/db_engine.py +++ b/app/db_engine.py @@ -2,6 +2,7 @@ import copy import datetime import logging import math +import os from typing import Literal, List from pandas import DataFrame @@ -9,14 +10,27 @@ from peewee import * from peewee import ModelSelect from playhouse.shortcuts import model_to_dict -db = SqliteDatabase( - 'storage/sba_master.db', - pragmas={ - 'journal_mode': 'wal', - 'cache_size': -1 * 64000, - 'synchronous': 0 - } -) +# Database configuration - supports both SQLite and PostgreSQL +DATABASE_TYPE = os.environ.get('DATABASE_TYPE', 'sqlite') + +if DATABASE_TYPE.lower() == 'postgresql': + db = PostgresqlDatabase( + os.environ.get('POSTGRES_DB', 'sba_master'), + user=os.environ.get('POSTGRES_USER', 'sba_admin'), + password=os.environ.get('POSTGRES_PASSWORD', 'sba_dev_password_2024'), + host=os.environ.get('POSTGRES_HOST', 'localhost'), + port=int(os.environ.get('POSTGRES_PORT', '5432')) + ) +else: + # Default SQLite configuration + db = SqliteDatabase( + 'storage/sba_master.db', + pragmas={ + 'journal_mode': 'wal', + 'cache_size': -1 * 64000, + 'synchronous': 0 + } + ) date = f'{datetime.datetime.now().year}-{datetime.datetime.now().month}-{datetime.datetime.now().day}' logger = logging.getLogger('discord_app') @@ -153,7 +167,7 @@ class Current(BaseModel): pick_trade_start = IntegerField() pick_trade_end = IntegerField() playoffs_begin = IntegerField() - injury_count = IntegerField() + injury_count = IntegerField(null=True) @staticmethod def latest(): @@ -253,8 +267,8 @@ class Team(BaseModel): lname = CharField() manager_legacy = CharField(null=True) division_legacy = CharField(null=True) - gmid = IntegerField() - gmid2 = IntegerField(null=True) + gmid = CharField(max_length=20) # Discord snowflake IDs as strings + gmid2 = CharField(max_length=20, null=True) # Discord snowflake IDs as strings manager1 = ForeignKeyField(Manager, null=True) manager2 = ForeignKeyField(Manager, null=True) division = ForeignKeyField(Division, null=True) @@ -832,8 +846,8 @@ class SbaPlayer(BaseModel): class Player(BaseModel): name = CharField() wara = FloatField() - image = CharField() - image2 = CharField(null=True) + image = CharField(max_length=1000) + image2 = CharField(max_length=1000, null=True) team = ForeignKeyField(Team) season = IntegerField() pitcher_injury = IntegerField(null=True) @@ -1862,8 +1876,8 @@ class DraftData(BaseModel): currentpick = IntegerField() timer = BooleanField() pick_deadline = DateTimeField(null=True) - result_channel = IntegerField(null=True) - ping_channel = IntegerField(null=True) + result_channel = CharField(max_length=20, null=True) # Discord channel ID as string + ping_channel = CharField(max_length=20, null=True) # Discord channel ID as string pick_minutes = IntegerField(null=True) @@ -1879,8 +1893,8 @@ class Award(BaseModel): class DiceRoll(BaseModel): - season = IntegerField(default=Current.latest().season) - week = IntegerField(default=Current.latest().week) + season = IntegerField(default=12) # Will be updated to current season when needed + week = IntegerField(default=1) # Will be updated to current week when needed team = ForeignKeyField(Team, null=True) roller = IntegerField() dsix = IntegerField(null=True) @@ -2208,7 +2222,7 @@ class Decision(BaseModel): class CustomCommandCreator(BaseModel): """Model for custom command creators.""" - discord_id = BigIntegerField(unique=True) + discord_id = CharField(max_length=20, unique=True) # Discord snowflake ID as string username = CharField(max_length=32) display_name = CharField(max_length=32, null=True) created_at = DateTimeField() @@ -2357,9 +2371,10 @@ class CustomCommand(BaseModel): # # for line in sorted_stats: -db.create_tables([ - Current, Division, Manager, Team, Result, Player, Schedule, Transaction, BattingStat, PitchingStat, Standings, - BattingCareer, PitchingCareer, FieldingCareer, Manager, Award, DiceRoll, DraftList, Keeper, StratGame, StratPlay, - Injury, Decision, CustomCommandCreator, CustomCommand -]) -db.close() +# Table creation moved to migration scripts to avoid dependency issues +# db.create_tables([ +# Current, Division, Manager, Team, Result, Player, Schedule, Transaction, BattingStat, PitchingStat, Standings, +# BattingCareer, PitchingCareer, FieldingCareer, Manager, Award, DiceRoll, DraftList, Keeper, StratGame, StratPlay, +# Injury, Decision, CustomCommandCreator, CustomCommand +# ]) +# db.close() diff --git a/app/routers_v3/custom_commands.py b/app/routers_v3/custom_commands.py index b14ecf0..1ca5a5d 100644 --- a/app/routers_v3/custom_commands.py +++ b/app/routers_v3/custom_commands.py @@ -235,7 +235,7 @@ async def get_custom_commands( params = [] if name is not None: - where_conditions.append("LOWER(cc.name) LIKE LOWER(?)") + where_conditions.append("LOWER(cc.name) LIKE LOWER(?)" if db.database == 'sqlite' else "cc.name ILIKE ?") params.append(f"%{name}%") if creator_discord_id is not None: @@ -921,9 +921,10 @@ async def get_command_names_for_autocomplete( """Get command names for Discord autocomplete""" try: if partial_name: - results = db.execute_sql(""" + like_clause = "LOWER(name) LIKE LOWER(?)" if db.database == 'sqlite' else "name ILIKE ?" + results = db.execute_sql(f""" SELECT name FROM custom_commands - WHERE is_active = 1 AND LOWER(name) LIKE LOWER(?) + WHERE is_active = 1 AND {like_clause} ORDER BY name LIMIT ? """, (f"%{partial_name}%", limit)).fetchall() diff --git a/docker-compose.yml b/docker-compose.yml index b6337a8..ded7af2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,4 +18,42 @@ services: - TZ=America/Chicago - WORKERS_PER_CORE=1.5 - TIMEOUT=120 - - GRACEFUL_TIMEOUT=120 \ No newline at end of file + - GRACEFUL_TIMEOUT=120 + depends_on: + - postgres + + postgres: + image: postgres:16-alpine + restart: unless-stopped + container_name: sba_postgres + environment: + - POSTGRES_DB=sba_master + - POSTGRES_USER=sba_admin + - POSTGRES_PASSWORD=sba_dev_password_2024 + - TZ=America/Chicago + volumes: + - postgres_data:/var/lib/postgresql/data + - /home/cal/Development/major-domo/dev-logs:/var/log/postgresql + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U sba_admin -d sba_master"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s + + adminer: + image: adminer:latest + restart: unless-stopped + container_name: sba_adminer + ports: + - "8080:8080" + environment: + - ADMINER_DEFAULT_SERVER=postgres + # - ADMINER_DESIGN=pepa-linha-dark + depends_on: + - postgres + +volumes: + postgres_data: \ No newline at end of file diff --git a/migrate_to_postgres.py b/migrate_to_postgres.py new file mode 100644 index 0000000..1a6c8af --- /dev/null +++ b/migrate_to_postgres.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 + +import os +import logging +from datetime import datetime +from playhouse.shortcuts import model_to_dict +from peewee import SqliteDatabase, PostgresqlDatabase + +logger = logging.getLogger(f'{__name__}.migrate_to_postgres') + +def setup_databases(): + """Setup both SQLite source and PostgreSQL target databases""" + + # SQLite source database + sqlite_db = SqliteDatabase( + 'storage/sba_master.db', + pragmas={ + 'journal_mode': 'wal', + 'cache_size': -1 * 64000, + 'synchronous': 0 + } + ) + + # PostgreSQL target database + postgres_db = PostgresqlDatabase( + os.environ.get('POSTGRES_DB', 'sba_master'), + user=os.environ.get('POSTGRES_USER', 'sba_admin'), + password=os.environ.get('POSTGRES_PASSWORD', 'sba_dev_password_2024'), + host=os.environ.get('POSTGRES_HOST', 'localhost'), + port=int(os.environ.get('POSTGRES_PORT', '5432')) + ) + + return sqlite_db, postgres_db + +def get_all_models(): + """Get all models in dependency order for migration""" + # Set temporary environment to load models + os.environ['DATABASE_TYPE'] = 'sqlite' + + from app.db_engine import ( + Current, Manager, Division, SbaPlayer, # No dependencies + Team, # Depends on Manager, Division + Player, # Depends on Team, SbaPlayer + Result, Schedule, Transaction, # Depend on Team, Player + BattingStat, PitchingStat, # Depend on Player, Team + Standings, # Depends on Team + BattingCareer, PitchingCareer, FieldingCareer, # No dependencies + BattingSeason, PitchingSeason, FieldingSeason, # Depend on Player, Career tables + DraftPick, DraftData, DraftList, # Depend on Team, Player + Award, # Depends on Manager, Player, Team + DiceRoll, # Depends on Team + Keeper, Injury, # Depend on Team, Player + StratGame, # Depends on Team, Manager + StratPlay, Decision, # Depend on StratGame, Player, Team + CustomCommandCreator, CustomCommand # CustomCommand depends on Creator + ) + + # Return in dependency order + return [ + # Base tables (no dependencies) + Current, Manager, Division, SbaPlayer, + BattingCareer, PitchingCareer, FieldingCareer, + CustomCommandCreator, + + # First level dependencies + Team, DraftData, + + # Second level dependencies + Player, CustomCommand, + + # Third level dependencies + Result, Schedule, Transaction, BattingStat, PitchingStat, + Standings, DraftPick, DraftList, Award, DiceRoll, + Keeper, Injury, StratGame, + + # Fourth level dependencies + BattingSeason, PitchingSeason, FieldingSeason, + StratPlay, Decision + ] + +def migrate_table_data(model_class, sqlite_db, postgres_db, batch_size=1000): + """Migrate data from SQLite to PostgreSQL for a specific model""" + table_name = model_class._meta.table_name + logger.info(f"Migrating table: {table_name}") + + try: + # Connect to SQLite and count records + model_class._meta.database = sqlite_db + sqlite_db.connect() + + total_records = model_class.select().count() + if total_records == 0: + logger.info(f" No records in {table_name}, skipping") + sqlite_db.close() + return True + + logger.info(f" Found {total_records} records") + sqlite_db.close() + + # Connect to PostgreSQL and prepare + model_class._meta.database = postgres_db + postgres_db.connect() + + # Create table if it doesn't exist + model_class.create_table(safe=True) + + # Migrate data in batches + migrated = 0 + sqlite_db.connect() + + for batch_start in range(0, total_records, batch_size): + # Get batch from SQLite + model_class._meta.database = sqlite_db + batch = list(model_class.select().offset(batch_start).limit(batch_size)) + + if not batch: + break + + # Convert to dicts and prepare for PostgreSQL + batch_data = [] + for record in batch: + data = model_to_dict(record, recurse=False) + # Remove auto-increment ID if present to let PostgreSQL handle it + if 'id' in data and hasattr(model_class, 'id'): + data.pop('id', None) + batch_data.append(data) + + # Insert into PostgreSQL + model_class._meta.database = postgres_db + if batch_data: + model_class.insert_many(batch_data).execute() + migrated += len(batch_data) + + logger.info(f" Migrated {migrated}/{total_records} records") + + sqlite_db.close() + postgres_db.close() + + logger.info(f"โœ“ Successfully migrated {table_name}: {migrated} records") + return True + + except Exception as e: + logger.error(f"โœ— Failed to migrate {table_name}: {e}") + try: + sqlite_db.close() + except: + pass + try: + postgres_db.close() + except: + pass + return False + +def migrate_all_data(): + """Migrate all data from SQLite to PostgreSQL""" + logger.info("Starting full data migration from SQLite to PostgreSQL...") + + # Setup databases + sqlite_db, postgres_db = setup_databases() + + # Test connections + try: + sqlite_db.connect() + sqlite_db.execute_sql("SELECT 1").fetchone() + sqlite_db.close() + logger.info("โœ“ SQLite source database connection OK") + except Exception as e: + logger.error(f"โœ— SQLite connection failed: {e}") + return False + + try: + postgres_db.connect() + postgres_db.execute_sql("SELECT 1").fetchone() + postgres_db.close() + logger.info("โœ“ PostgreSQL target database connection OK") + except Exception as e: + logger.error(f"โœ— PostgreSQL connection failed: {e}") + return False + + # Get models in dependency order + all_models = get_all_models() + logger.info(f"Found {len(all_models)} models to migrate") + + # Migrate each table + successful_migrations = 0 + failed_migrations = [] + + for model in all_models: + success = migrate_table_data(model, sqlite_db, postgres_db) + if success: + successful_migrations += 1 + else: + failed_migrations.append(model._meta.table_name) + + # Report results + logger.info(f"\nMigration completed:") + logger.info(f"โœ“ Successful: {successful_migrations}/{len(all_models)} tables") + + if failed_migrations: + logger.error(f"โœ— Failed: {len(failed_migrations)} tables") + for table in failed_migrations: + logger.error(f" - {table}") + return False + else: + logger.info("๐ŸŽ‰ All tables migrated successfully!") + return True + +def main(): + # Configure logging + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' + ) + + # Set PostgreSQL environment variables + os.environ['POSTGRES_DB'] = 'sba_master' + os.environ['POSTGRES_USER'] = 'sba_admin' + os.environ['POSTGRES_PASSWORD'] = 'sba_dev_password_2024' + os.environ['POSTGRES_HOST'] = 'localhost' + os.environ['POSTGRES_PORT'] = '5432' + + success = migrate_all_data() + return 0 if success else 1 + +if __name__ == "__main__": + exit(main()) \ No newline at end of file diff --git a/migration_issues_tracker.md b/migration_issues_tracker.md new file mode 100644 index 0000000..485060a --- /dev/null +++ b/migration_issues_tracker.md @@ -0,0 +1,334 @@ +# Migration Issues Tracker + +## Summary Dashboard + +**Last Updated**: 2025-08-18 17:53:23 +**Test Run**: #2 (Phase 1 Schema Fixes) +**Total Issues**: 27 (3 new discovered) +**Resolved**: 4 +**In Progress**: 0 +**Remaining**: 23 + +### Status Overview +- ๐Ÿ”ด **Critical**: 7 issues (3 new NULL constraints discovered) +- ๐ŸŸก **High**: 12 issues (data integrity concerns) +- ๐ŸŸข **Medium**: 4 issues (data quality issues) +- โšช **Low**: 0 issues + +### Phase 1 Progress +- โœ… **4 Critical Issues Resolved** +- โœ… **7 Tables Now Migrating Successfully**: manager, division, sbaplayer, battingcareer, pitchingcareer, fieldingcareer, draftdata +- โœ… **Zero Integer Overflow Errors** +- โœ… **Zero String Length Errors** + +--- + +## โœ… **RESOLVED ISSUES** (Phase 1) + +### CONSTRAINT-CURRENT-INJURY_COUNT-001 โœ… +- **Resolution**: Made `injury_count` field nullable (`null=True`) +- **Date Resolved**: 2025-08-18 +- **Solution Applied**: Schema change in db_engine.py line 170 +- **Test Result**: โœ… NULL values now accepted + +### DATA_QUALITY-PLAYER-NAME-001 โœ… +- **Resolution**: Increased `image` and `image2` field limits to `max_length=1000` +- **Date Resolved**: 2025-08-18 +- **Root Cause**: Google Photos URLs up to 801 characters +- **Solution Applied**: Schema change in db_engine.py lines 849-850 +- **Test Result**: โœ… Long URLs now accepted + +### MIGRATION_LOGIC-TEAM-INTEGER-001 โœ… +- **Resolution**: Converted Discord snowflake IDs to strings +- **Date Resolved**: 2025-08-18 +- **Root Cause**: Discord IDs (1018721510111838309) exceed INTEGER range +- **Solution Applied**: `gmid`/`gmid2` now `CharField(max_length=20)` +- **Best Practice**: Discord IDs should always be strings, not integers +- **Test Result**: โœ… Large Discord IDs now handled properly + +### MIGRATION_LOGIC-DRAFTDATA-INTEGER-001 โœ… +- **Resolution**: Converted Discord channel IDs to strings +- **Date Resolved**: 2025-08-18 +- **Root Cause**: Channel IDs exceed INTEGER range +- **Solution Applied**: `result_channel`/`ping_channel` now `CharField(max_length=20)` +- **Test Result**: โœ… draftdata table migrated successfully (1 record) + +--- + +## ๐Ÿ”ด Critical Issues (Migration Blockers) + +### CONSTRAINT-CURRENT-BSTATCOUNT-001 ๐Ÿ†• +- **Priority**: CRITICAL +- **Table**: current +- **Error**: `null value in column "bstatcount" violates not-null constraint` +- **Impact**: Blocks current table migration (11 records affected) +- **Status**: IDENTIFIED +- **Solution**: Make field nullable or set default value + +### CONSTRAINT-CURRENT-PSTATCOUNT-001 ๐Ÿ†• +- **Priority**: CRITICAL (Predicted) +- **Table**: current +- **Error**: `null value in column "pstatcount" violates not-null constraint` (likely) +- **Impact**: Blocks current table migration (11 records affected) +- **Status**: PREDICTED +- **Solution**: Make field nullable or set default value + +### CONSTRAINT-TEAM-AUTO_DRAFT-001 ๐Ÿ†• +- **Priority**: CRITICAL +- **Table**: team +- **Error**: `null value in column "auto_draft" violates not-null constraint` +- **Impact**: Blocks team table migration (546 records affected) +- **Status**: IDENTIFIED +- **Solution**: Make field nullable or set default value (False) + +### SCHEMA-CUSTOMCOMMANDCREATOR-MISSING-001 +- **Priority**: CRITICAL +- **Table**: customcommandcreator +- **Error**: `no such table: customcommandcreator` +- **Impact**: Table doesn't exist in SQLite source +- **Status**: IDENTIFIED +- **Solution**: Skip table or create empty table + +### SCHEMA-CUSTOMCOMMAND-MISSING-001 +- **Priority**: CRITICAL +- **Table**: customcommand +- **Error**: `no such table: customcommand` +- **Impact**: Table doesn't exist in SQLite source +- **Status**: IDENTIFIED +- **Solution**: Skip table or create empty table + +### CONSTRAINT-DECISION-TEAM_ID-001 +- **Priority**: CRITICAL +- **Table**: decision +- **Error**: `null value in column "team_id" violates not-null constraint` +- **Impact**: Blocks decision table migration (20,309 records affected) +- **Status**: IDENTIFIED +- **Solution**: Handle NULL team_id values + +### FOREIGN_KEY-STRATPLAY-GAME_ID-001 +- **Priority**: CRITICAL +- **Table**: stratplay +- **Error**: `violates foreign key constraint "stratplay_game_id_fkey"` +- **Impact**: Blocks stratplay table migration (192,790 records affected) +- **Status**: IDENTIFIED +- **Solution**: Migrate stratgame table first or fix referential integrity + +--- + +## ๐ŸŸก High Priority Issues (Data Integrity) + +### FOREIGN_KEY-BATTINGSEASON-PLAYER_ID-001 +- **Priority**: HIGH +- **Table**: battingseason +- **Error**: `violates foreign key constraint "battingseason_player_id_fkey"` +- **Impact**: References missing player records (4,878 records affected) +- **Status**: IDENTIFIED +- **Solution**: Migrate players first or clean orphaned records + +### FOREIGN_KEY-PITCHINGSEASON-PLAYER_ID-001 +- **Priority**: HIGH +- **Table**: pitchingseason +- **Error**: `violates foreign key constraint "pitchingseason_player_id_fkey"` +- **Impact**: References missing player records (2,810 records affected) +- **Status**: IDENTIFIED +- **Solution**: Migrate players first or clean orphaned records + +### FOREIGN_KEY-FIELDINGSEASON-PLAYER_ID-001 +- **Priority**: HIGH +- **Table**: fieldingseason +- **Error**: `violates foreign key constraint "fieldingseason_player_id_fkey"` +- **Impact**: References missing player records (8,981 records affected) +- **Status**: IDENTIFIED +- **Solution**: Migrate players first or clean orphaned records + +### FOREIGN_KEY-RESULT-TEAM_ID-001 +- **Priority**: HIGH +- **Table**: result +- **Error**: Foreign key constraint violations (estimated) +- **Impact**: Result records may reference missing teams +- **Status**: IDENTIFIED +- **Solution**: Ensure teams migrate before results + +### FOREIGN_KEY-SCHEDULE-TEAM_ID-001 +- **Priority**: HIGH +- **Table**: schedule +- **Error**: Foreign key constraint violations (estimated) +- **Impact**: Schedule records may reference missing teams +- **Status**: IDENTIFIED +- **Solution**: Ensure teams migrate before schedules + +### FOREIGN_KEY-TRANSACTION-PLAYER_ID-001 +- **Priority**: HIGH +- **Table**: transaction +- **Error**: Foreign key constraint violations (estimated) +- **Impact**: Transaction records may reference missing players/teams +- **Status**: IDENTIFIED +- **Solution**: Fix dependency order + +### FOREIGN_KEY-BATTINGSTAT-PLAYER_ID-001 +- **Priority**: HIGH +- **Table**: battingstat +- **Error**: Foreign key constraint violations (estimated) +- **Impact**: Stats records may reference missing players +- **Status**: IDENTIFIED +- **Solution**: Migrate players/teams first + +### FOREIGN_KEY-PITCHINGSTAT-PLAYER_ID-001 +- **Priority**: HIGH +- **Table**: pitchingstat +- **Error**: Foreign key constraint violations (estimated) +- **Impact**: Stats records may reference missing players +- **Status**: IDENTIFIED +- **Solution**: Migrate players/teams first + +### FOREIGN_KEY-STANDINGS-TEAM_ID-001 +- **Priority**: HIGH +- **Table**: standings +- **Error**: Foreign key constraint violations (estimated) +- **Impact**: Standings records may reference missing teams +- **Status**: IDENTIFIED +- **Solution**: Migrate teams first + +### FOREIGN_KEY-DRAFTPICK-TEAM_ID-001 +- **Priority**: HIGH +- **Table**: draftpick +- **Error**: Foreign key constraint violations (estimated) +- **Impact**: Draft records may reference missing teams/players +- **Status**: IDENTIFIED +- **Solution**: Fix dependency order + +### FOREIGN_KEY-DRAFTLIST-TEAM_ID-001 +- **Priority**: HIGH +- **Table**: draftlist +- **Error**: Foreign key constraint violations (estimated) +- **Impact**: Draft list records may reference missing teams/players +- **Status**: IDENTIFIED +- **Solution**: Fix dependency order + +### FOREIGN_KEY-AWARD-MANAGER_ID-001 +- **Priority**: HIGH +- **Table**: award +- **Error**: Foreign key constraint violations (estimated) +- **Impact**: Award records may reference missing managers/players +- **Status**: IDENTIFIED +- **Solution**: Migrate managers/players first + +--- + +## ๐ŸŸข Medium Priority Issues (Data Quality) + +### FOREIGN_KEY-DICEROLL-TEAM_ID-001 +- **Priority**: MEDIUM +- **Table**: diceroll +- **Error**: Foreign key constraint violations (estimated) +- **Impact**: Dice roll records may reference missing teams +- **Status**: IDENTIFIED +- **Solution**: Migrate teams first or allow NULL teams + +### FOREIGN_KEY-KEEPER-TEAM_ID-001 +- **Priority**: MEDIUM +- **Table**: keeper +- **Error**: Foreign key constraint violations (estimated) +- **Impact**: Keeper records may reference missing teams/players +- **Status**: IDENTIFIED +- **Solution**: Fix dependency order + +### FOREIGN_KEY-INJURY-PLAYER_ID-001 +- **Priority**: MEDIUM +- **Table**: injury +- **Error**: Foreign key constraint violations (estimated) +- **Impact**: Injury records may reference missing players +- **Status**: IDENTIFIED +- **Solution**: Migrate players first + +### FOREIGN_KEY-STRATGAME-TEAM_ID-001 +- **Priority**: MEDIUM +- **Table**: stratgame +- **Error**: Foreign key constraint violations (estimated) +- **Impact**: Game records may reference missing teams/managers +- **Status**: IDENTIFIED +- **Solution**: Migrate teams/managers first + +--- + +## ๐Ÿ“Š Resolution Strategy + +### Phase 1: Schema Fixes (Fix PostgreSQL Schema) +- [ ] CONSTRAINT-CURRENT-INJURY_COUNT-001: Make injury_count nullable or set default +- [ ] DATA_QUALITY-PLAYER-NAME-001: Increase VARCHAR limits +- [ ] MIGRATION_LOGIC-TEAM-INTEGER-001: Use BIGINT for large integers +- [ ] MIGRATION_LOGIC-DRAFTDATA-INTEGER-001: Use BIGINT for large integers + +### Phase 2: Missing Tables (Handle Non-existent Tables) +- [ ] SCHEMA-CUSTOMCOMMANDCREATOR-MISSING-001: Skip gracefully +- [ ] SCHEMA-CUSTOMCOMMAND-MISSING-001: Skip gracefully + +### Phase 3: Data Cleaning (Fix SQLite Data) +- [ ] CONSTRAINT-DECISION-TEAM_ID-001: Handle NULL team_id values +- [ ] Clean orphaned records before migration + +### Phase 4: Dependency Order (Fix Migration Logic) +- [ ] Migrate base tables first: Current, Manager, Division, SbaPlayer +- [ ] Then dependent tables: Team, Player +- [ ] Finally stats and transaction tables +- [ ] Disable foreign key checks during migration if needed + +### Phase 5: Validation +- [ ] Run full migration test +- [ ] Validate all record counts +- [ ] Check referential integrity +- [ ] Performance testing + +--- + +## ๐Ÿ“ˆ Progress Tracking + +### Test Run History +| Run # | Date | Issues Found | Issues Fixed | Status | Notes | +|-------|------|--------------|--------------|--------|-------| +| 1 | 2025-08-18 16:52 | 24 | 0 | Discovery Complete | Initial discovery run | +| 2 | 2025-08-18 17:53 | 3 new | 4 | Phase 1 Complete | Schema fixes successful | +| 3 | | | | Planned | Phase 2: NULL constraints | + +### Test Run #2 Details (Phase 1) +**Duration**: ~3 minutes +**Focus**: Critical schema issues +**Approach**: Fixed 4 blocking schema problems + +**Issues Resolved**: +1. โœ… CONSTRAINT-CURRENT-INJURY_COUNT-001 โ†’ Made nullable +2. โœ… DATA_QUALITY-PLAYER-NAME-001 โ†’ Increased VARCHAR limits +3. โœ… MIGRATION_LOGIC-TEAM-INTEGER-001 โ†’ Discord IDs to strings +4. โœ… MIGRATION_LOGIC-DRAFTDATA-INTEGER-001 โ†’ Channel IDs to strings + +**New Issues Found**: +1. ๐Ÿ†• CONSTRAINT-CURRENT-BSTATCOUNT-001 โ†’ NULL stats count +2. ๐Ÿ†• CONSTRAINT-CURRENT-PSTATCOUNT-001 โ†’ NULL stats count (predicted) +3. ๐Ÿ†• CONSTRAINT-TEAM-AUTO_DRAFT-001 โ†’ NULL auto draft flag + +**Migration Results**: +- โœ… **7 tables migrated successfully** (vs 0 in Run #1) +- โœ… **5,432 records migrated** (manager, division, sbaplayer, careers, draftdata) +- โœ… **No integer overflow errors** +- โœ… **No string length errors** +- โš ๏ธ **3 new NULL constraint issues discovered** + +### Next Actions (Phase 2) +1. **Immediate**: Fix 3 new NULL constraint issues discovered in Phase 1 + - [ ] CONSTRAINT-CURRENT-BSTATCOUNT-001: Make bstatcount nullable + - [ ] CONSTRAINT-CURRENT-PSTATCOUNT-001: Make pstatcount nullable + - [ ] CONSTRAINT-TEAM-AUTO_DRAFT-001: Make auto_draft nullable or set default False +2. **Then**: Handle missing tables gracefully (custom command tables) +3. **Next**: Fix dependency order and foreign key issues +4. **Finally**: Data cleaning and validation + +### Success Metrics (Current Status) +- **Tables Successfully Migrating**: 7/30 (23%) โฌ†๏ธ from 0/30 +- **Records Successfully Migrated**: ~5,432 โฌ†๏ธ from 0 +- **Critical Issues Resolved**: 4/8 (50%) โฌ†๏ธ from 0/8 +- **Schema Issues**: โœ… Resolved (integers, string lengths) +- **NULL Constraints**: โš ๏ธ 3 new issues discovered + +--- + +*This tracker will be updated after each test run to monitor progress toward successful migration.* \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 6b47181..f4ea62d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ uvicorn peewee==3.13.3 python-multipart pandas +psycopg2-binary>=2.9.0 diff --git a/reset_postgres.py b/reset_postgres.py new file mode 100644 index 0000000..65786a0 --- /dev/null +++ b/reset_postgres.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 + +import os +import logging +from datetime import datetime + +logger = logging.getLogger(f'{__name__}.reset_postgres') + +def reset_postgres_database(): + """Complete reset of PostgreSQL database for testing""" + + # Set PostgreSQL environment + os.environ['DATABASE_TYPE'] = 'postgresql' + os.environ['POSTGRES_DB'] = 'sba_master' + os.environ['POSTGRES_USER'] = 'sba_admin' + os.environ['POSTGRES_PASSWORD'] = 'sba_dev_password_2024' + os.environ['POSTGRES_HOST'] = 'localhost' + os.environ['POSTGRES_PORT'] = '5432' + + # Direct PostgreSQL connection (avoid db_engine complications) + from peewee import PostgresqlDatabase + + try: + logger.info("Connecting to PostgreSQL...") + db = PostgresqlDatabase( + 'sba_master', + user='sba_admin', + password='sba_dev_password_2024', + host='localhost', + port=5432 + ) + db.connect() + + # Get list of all tables in public schema - use simpler query + logger.info("Querying for existing tables...") + try: + # Use psql-style \dt query converted to SQL + tables_result = db.execute_sql(""" + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_type = 'BASE TABLE' + """).fetchall() + + logger.info(f"Query returned {len(tables_result)} results") + + table_names = [] + for table_row in tables_result: + table_name = table_row[0] + table_names.append(table_name) + logger.info(f"Found table: {table_name}") + + except Exception as query_error: + logger.error(f"Query execution error: {query_error}") + raise + + if not table_names: + logger.info("No tables found - database already clean") + db.close() + return True + + logger.info(f"Found {len(table_names)} tables to drop") + + # Disable foreign key checks and drop all tables + for table_name in table_names: + logger.info(f" Dropping table: {table_name}") + db.execute_sql(f'DROP TABLE IF EXISTS "{table_name}" CASCADE') + + # Reset sequences (auto-increment counters) + sequences_query = """ + SELECT sequence_name FROM information_schema.sequences + WHERE sequence_schema = 'public' + """ + + sequences_result = db.execute_sql(sequences_query).fetchall() + for seq in sequences_result: + if seq and len(seq) > 0: + seq_name = seq[0] + logger.info(f" Resetting sequence: {seq_name}") + db.execute_sql(f'DROP SEQUENCE IF EXISTS "{seq_name}" CASCADE') + + db.close() + logger.info("โœ“ PostgreSQL database reset complete") + return True + + except Exception as e: + logger.error(f"โœ— Database reset failed: {e}") + try: + db.close() + except: + pass + return False + +def main(): + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' + ) + + logger.info("=== PostgreSQL Database Reset ===") + success = reset_postgres_database() + + if success: + logger.info("๐Ÿ—‘๏ธ Database reset successful - ready for fresh migration") + else: + logger.error("โŒ Database reset failed") + + return 0 if success else 1 + +if __name__ == "__main__": + exit(main()) \ No newline at end of file diff --git a/test-storage/pd_master.db b/test-storage/pd_master.db new file mode 100644 index 0000000..f510bac Binary files /dev/null and b/test-storage/pd_master.db differ diff --git a/test-storage/sba_is_fun.db b/test-storage/sba_is_fun.db new file mode 100644 index 0000000..44505c5 Binary files /dev/null and b/test-storage/sba_is_fun.db differ diff --git a/test_migration_workflow.sh b/test_migration_workflow.sh new file mode 100755 index 0000000..fc4841d --- /dev/null +++ b/test_migration_workflow.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +# Test Migration Workflow Script +# Automates the complete migration testing process + +set -e # Exit on any error + +echo "==========================================" +echo "๐Ÿงช POSTGRESQL MIGRATION TESTING WORKFLOW" +echo "==========================================" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +print_step() { + echo -e "\n${BLUE}๐Ÿ“‹ Step $1: $2${NC}" +} + +print_success() { + echo -e "${GREEN}โœ… $1${NC}" +} + +print_warning() { + echo -e "${YELLOW}โš ๏ธ $1${NC}" +} + +print_error() { + echo -e "${RED}โŒ $1${NC}" +} + +# Check if Docker containers are running +print_step 1 "Checking Docker containers" +if docker ps | grep -q "sba_postgres"; then + print_success "PostgreSQL container is running" +else + print_error "PostgreSQL container not found - run: docker-compose up postgres -d" + exit 1 +fi + +if docker ps | grep -q "sba_adminer"; then + print_success "Adminer container is running" +else + print_warning "Adminer container not running - run: docker-compose up adminer -d" +fi + +# Test PostgreSQL connectivity +print_step 2 "Testing PostgreSQL connectivity" +if python test_postgres.py > /dev/null 2>&1; then + print_success "PostgreSQL connection test passed" +else + print_error "PostgreSQL connection test failed" + echo "Run: python test_postgres.py for details" + exit 1 +fi + +# Reset PostgreSQL database +print_step 3 "Resetting PostgreSQL database" +if python reset_postgres.py > /dev/null 2>&1; then + print_success "PostgreSQL database reset complete" +else + print_error "Database reset failed" + echo "Run: python reset_postgres.py for details" + exit 1 +fi + +# Check SQLite source data +print_step 4 "Checking SQLite source data" +if [ -f "storage/sba_master.db" ]; then + SIZE=$(du -h storage/sba_master.db | cut -f1) + print_success "SQLite database found (${SIZE})" +else + print_error "SQLite database not found at storage/sba_master.db" + exit 1 +fi + +# Run migration +print_step 5 "Running data migration" +echo "This may take several minutes depending on data size..." +if python migrate_to_postgres.py; then + print_success "Migration completed successfully" +else + print_error "Migration failed" + exit 1 +fi + +# Validate migration +print_step 6 "Validating migration results" +if python validate_migration.py; then + print_success "Migration validation passed" +else + print_error "Migration validation failed" + exit 1 +fi + +# Final summary +echo -e "\n==========================================" +echo -e "${GREEN}๐ŸŽ‰ MIGRATION TEST COMPLETE${NC}" +echo "==========================================" +echo -e "๐Ÿ“Š View data in Adminer: ${BLUE}http://localhost:8080${NC}" +echo " Server: postgres" +echo " Username: sba_admin" +echo " Password: sba_dev_password_2024" +echo " Database: sba_master" +echo "" +echo -e "๐Ÿ”„ To test again: ${YELLOW}./test_migration_workflow.sh${NC}" +echo -e "๐Ÿ—‘๏ธ To reset only: ${YELLOW}python reset_postgres.py${NC}" +echo "==========================================" \ No newline at end of file diff --git a/test_postgres.py b/test_postgres.py new file mode 100644 index 0000000..6180ccc --- /dev/null +++ b/test_postgres.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 + +import os +import logging +from datetime import datetime + +# Set environment variables for PostgreSQL +os.environ['DATABASE_TYPE'] = 'postgresql' +os.environ['POSTGRES_DB'] = 'sba_master' +os.environ['POSTGRES_USER'] = 'sba_admin' +os.environ['POSTGRES_PASSWORD'] = 'sba_dev_password_2024' +os.environ['POSTGRES_HOST'] = 'localhost' +os.environ['POSTGRES_PORT'] = '5432' + +# Import after setting environment variables +from app.db_engine import db, Current, Team, Player, SbaPlayer, Manager, Division + +logger = logging.getLogger(f'{__name__}.test_postgres') + +def test_connection(): + """Test PostgreSQL connection""" + try: + logger.info("Testing PostgreSQL connection...") + db.connect() + logger.info("โœ“ Connected to PostgreSQL successfully") + + # Test basic query + result = db.execute_sql("SELECT version();").fetchone() + logger.info(f"โœ“ PostgreSQL version: {result[0]}") + + db.close() + return True + + except Exception as e: + logger.error(f"โœ— Connection failed: {e}") + return False + +def test_schema_creation(): + """Test creating tables in PostgreSQL""" + try: + logger.info("Testing schema creation...") + db.connect() + + # Create tables in dependency order + # First: tables with no dependencies + base_tables = [Current, Manager, Division, SbaPlayer] + db.create_tables(base_tables, safe=True) + + # Second: tables that depend on base tables + dependent_tables = [Team, Player] + db.create_tables(dependent_tables, safe=True) + + logger.info("โœ“ Test tables created successfully") + + # Test table existence + all_test_tables = base_tables + dependent_tables + for table in all_test_tables: + table_name = table._meta.table_name + result = db.execute_sql(""" + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = %s + ); + """, (table_name,)).fetchone() + + if result[0]: + logger.info(f"โœ“ Table '{table_name}' exists") + else: + logger.error(f"โœ— Table '{table_name}' missing") + + db.close() + return True + + except Exception as e: + logger.error(f"โœ— Schema creation failed: {e}") + return False + +def main(): + # Configure logging + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' + ) + + logger.info("Starting PostgreSQL migration test...") + + # Test connection + if not test_connection(): + logger.error("Connection test failed - stopping") + return False + + # Test schema creation + if not test_schema_creation(): + logger.error("Schema creation test failed - stopping") + return False + + logger.info("โœ“ All PostgreSQL tests passed!") + return True + +if __name__ == "__main__": + success = main() + exit(0 if success else 1) \ No newline at end of file diff --git a/validate_migration.py b/validate_migration.py new file mode 100644 index 0000000..e3d9f99 --- /dev/null +++ b/validate_migration.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 + +import os +import logging +from collections import defaultdict + +logger = logging.getLogger(f'{__name__}.validate_migration') + +def compare_table_counts(): + """Compare record counts between SQLite and PostgreSQL""" + logger.info("Comparing table record counts...") + + # Get all models + os.environ['DATABASE_TYPE'] = 'sqlite' + from app.db_engine import ( + Current, Manager, Division, SbaPlayer, Team, Player, + Result, Schedule, Transaction, BattingStat, PitchingStat, + Standings, BattingCareer, PitchingCareer, FieldingCareer, + BattingSeason, PitchingSeason, FieldingSeason, + DraftPick, DraftData, DraftList, Award, DiceRoll, + Keeper, Injury, StratGame, StratPlay, Decision, + CustomCommandCreator, CustomCommand + ) + + all_models = [ + Current, Manager, Division, SbaPlayer, Team, Player, + Result, Schedule, Transaction, BattingStat, PitchingStat, + Standings, BattingCareer, PitchingCareer, FieldingCareer, + BattingSeason, PitchingSeason, FieldingSeason, + DraftPick, DraftData, DraftList, Award, DiceRoll, + Keeper, Injury, StratGame, StratPlay, Decision, + CustomCommandCreator, CustomCommand + ] + + results = {} + + for model in all_models: + table_name = model._meta.table_name + + try: + # SQLite count + os.environ['DATABASE_TYPE'] = 'sqlite' + from peewee import SqliteDatabase + sqlite_db = SqliteDatabase('storage/sba_master.db') + model._meta.database = sqlite_db + sqlite_db.connect() + sqlite_count = model.select().count() + sqlite_db.close() + + # PostgreSQL count + os.environ['DATABASE_TYPE'] = 'postgresql' + from peewee import PostgresqlDatabase + postgres_db = PostgresqlDatabase( + 'sba_master', user='sba_admin', + password='sba_dev_password_2024', + host='localhost', port=5432 + ) + model._meta.database = postgres_db + postgres_db.connect() + postgres_count = model.select().count() + postgres_db.close() + + results[table_name] = { + 'sqlite': sqlite_count, + 'postgres': postgres_count, + 'match': sqlite_count == postgres_count + } + + status = "โœ“" if sqlite_count == postgres_count else "โœ—" + logger.info(f" {status} {table_name:20} SQLite: {sqlite_count:6} | PostgreSQL: {postgres_count:6}") + + except Exception as e: + logger.error(f" โœ— {table_name:20} Error: {e}") + results[table_name] = { + 'sqlite': 'ERROR', + 'postgres': 'ERROR', + 'match': False + } + + return results + +def validate_sample_data(): + """Validate specific records exist in both databases""" + logger.info("Validating sample data integrity...") + + validations = [] + + try: + # Check Current table + os.environ['DATABASE_TYPE'] = 'sqlite' + from app.db_engine import Current + from peewee import SqliteDatabase + + sqlite_db = SqliteDatabase('storage/sba_master.db') + Current._meta.database = sqlite_db + sqlite_db.connect() + + if Current.select().exists(): + sqlite_current = Current.select().first() + sqlite_season = sqlite_current.season if sqlite_current else None + sqlite_db.close() + + # Check in PostgreSQL + os.environ['DATABASE_TYPE'] = 'postgresql' + from peewee import PostgresqlDatabase + + postgres_db = PostgresqlDatabase( + 'sba_master', user='sba_admin', + password='sba_dev_password_2024', + host='localhost', port=5432 + ) + Current._meta.database = postgres_db + postgres_db.connect() + + if Current.select().exists(): + postgres_current = Current.select().first() + postgres_season = postgres_current.season if postgres_current else None + + if sqlite_season == postgres_season: + logger.info(f" โœ“ Current season matches: {sqlite_season}") + validations.append(True) + else: + logger.error(f" โœ— Current season mismatch: SQLite={sqlite_season}, PostgreSQL={postgres_season}") + validations.append(False) + else: + logger.error(" โœ— No Current record in PostgreSQL") + validations.append(False) + + postgres_db.close() + else: + logger.info(" - No Current records to validate") + validations.append(True) + sqlite_db.close() + + except Exception as e: + logger.error(f" โœ— Sample data validation error: {e}") + validations.append(False) + + return all(validations) + +def generate_migration_report(count_results, sample_validation): + """Generate comprehensive migration report""" + logger.info("\n" + "="*60) + logger.info("MIGRATION VALIDATION REPORT") + logger.info("="*60) + + # Count summary + total_tables = len(count_results) + matching_tables = sum(1 for r in count_results.values() if r['match']) + + logger.info(f"Tables analyzed: {total_tables}") + logger.info(f"Count matches: {matching_tables}/{total_tables}") + + if matching_tables == total_tables: + logger.info("โœ… ALL TABLE COUNTS MATCH!") + else: + logger.error(f"โŒ {total_tables - matching_tables} tables have count mismatches") + + # Show mismatches + logger.info("\nMismatched tables:") + for table, result in count_results.items(): + if not result['match']: + logger.error(f" {table}: SQLite={result['sqlite']}, PostgreSQL={result['postgres']}") + + # Sample data validation + if sample_validation: + logger.info("โœ… Sample data validation passed") + else: + logger.error("โŒ Sample data validation failed") + + # Overall status + migration_success = (matching_tables == total_tables) and sample_validation + + logger.info("\n" + "="*60) + if migration_success: + logger.info("๐ŸŽ‰ MIGRATION VALIDATION: SUCCESS") + logger.info("โœ… Ready for production migration") + else: + logger.error("โŒ MIGRATION VALIDATION: FAILED") + logger.error("โš ๏ธ Issues must be resolved before production") + logger.info("="*60) + + return migration_success + +def main(): + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' + ) + + logger.info("Starting migration validation...") + + # Run validations + count_results = compare_table_counts() + sample_validation = validate_sample_data() + + # Generate report + success = generate_migration_report(count_results, sample_validation) + + return 0 if success else 1 + +if __name__ == "__main__": + exit(main()) \ No newline at end of file