0cba52cea5
- Add db_helpers.py with cross-database upsert functions for SQLite/PostgreSQL - Replace 12 on_conflict_replace() calls with PostgreSQL-compatible upserts - Add unique indexes: StratPlay(game, play_num), Decision(game, pitcher) - Add max_length to Team model fields (abbrev, sname, lname) - Fix boolean comparison in teams.py (== 0/1 to == False/True) - Create migrate_to_postgres.py with ID-preserving migration logic - Create audit_sqlite.py for pre-migration data integrity checks - Add PROJECT_PLAN.json for migration tracking - Add .secrets/ to .gitignore for credentials Audit results: 658,963 records across 29 tables, 2,390 orphaned stats (expected) Based on Major Domo migration lessons learned (33 issues resolved there)
483 lines
16 KiB
JSON
483 lines
16 KiB
JSON
{
|
|
"meta": {
|
|
"version": "1.1.0",
|
|
"created": "2026-01-25",
|
|
"lastUpdated": "2026-01-25",
|
|
"planType": "migration",
|
|
"description": "SQLite to PostgreSQL migration for Paper Dynasty database API",
|
|
"branch": "postgres-migration",
|
|
"totalEstimatedHours": 22,
|
|
"totalTasks": 16,
|
|
"completedTasks": 13
|
|
},
|
|
"context": {
|
|
"sourceDatabase": {
|
|
"type": "SQLite",
|
|
"file": "storage/pd_master.db",
|
|
"size": "110 MB",
|
|
"tables": 29,
|
|
"totalRecords": 515000,
|
|
"largestTable": {
|
|
"name": "stratplay",
|
|
"records": 332737
|
|
}
|
|
},
|
|
"targetDatabase": {
|
|
"type": "PostgreSQL 17",
|
|
"server": "sba_postgres (same server as Major Domo)",
|
|
"database": "pd_master",
|
|
"user": "pd_admin",
|
|
"credentialsFile": ".secrets/pd_admin_credentials.txt"
|
|
},
|
|
"lessonsFromMajorDomo": [
|
|
"CRITICAL: Primary key IDs must be explicitly preserved during migration",
|
|
"PostgreSQL GROUP BY requires ALL non-aggregated columns",
|
|
"Boolean fields cannot be summed directly - cast to integer first",
|
|
"Discord snowflake IDs must be strings, not integers (N/A for Paper Dynasty)",
|
|
"VARCHAR fields need explicit max_length",
|
|
"NULL constraints are stricter in PostgreSQL",
|
|
"Foreign key orphaned records need smart fallback handling",
|
|
"Reset sequences after ID-preserving inserts"
|
|
],
|
|
"devServer": {
|
|
"access": "ssh sba-db",
|
|
"composeLocation": "cd container-data/dev-sba-database/"
|
|
}
|
|
},
|
|
"categories": {
|
|
"critical": "Must complete before migration - blocks production",
|
|
"high": "Required for successful migration",
|
|
"medium": "Improves migration quality/reliability",
|
|
"low": "Polish and nice-to-have",
|
|
"completed": "Already done on postgres-migration branch"
|
|
},
|
|
"tasks": [
|
|
{
|
|
"id": "MIG-001",
|
|
"name": "Environment-based database configuration",
|
|
"description": "Add PostgreSQL support with environment variable switching between SQLite/PostgreSQL",
|
|
"category": "completed",
|
|
"priority": 1,
|
|
"completed": true,
|
|
"tested": true,
|
|
"dependencies": [],
|
|
"files": [
|
|
{
|
|
"path": "app/db_engine.py",
|
|
"lines": [11, 35],
|
|
"issue": "Now supports DATABASE_TYPE env var for SQLite/PostgreSQL switching"
|
|
}
|
|
],
|
|
"suggestedFix": "Already implemented with PooledPostgresqlDatabase",
|
|
"estimatedHours": 2,
|
|
"notes": "Includes connection pooling (20 max, 5-min stale timeout, autorollback)"
|
|
},
|
|
{
|
|
"id": "MIG-002",
|
|
"name": "Add table_name to all models",
|
|
"description": "Explicit table naming for PostgreSQL compatibility",
|
|
"category": "completed",
|
|
"priority": 2,
|
|
"completed": true,
|
|
"tested": true,
|
|
"dependencies": [],
|
|
"files": [
|
|
{
|
|
"path": "app/db_engine.py",
|
|
"lines": [],
|
|
"issue": "All 29 models now have Meta.table_name defined"
|
|
}
|
|
],
|
|
"suggestedFix": "Already implemented",
|
|
"estimatedHours": 1,
|
|
"notes": "Prevents Peewee naming inconsistencies"
|
|
},
|
|
{
|
|
"id": "MIG-003",
|
|
"name": "Fix GROUP BY queries for PostgreSQL",
|
|
"description": "PostgreSQL requires all non-aggregated SELECT fields in GROUP BY clause",
|
|
"category": "completed",
|
|
"priority": 3,
|
|
"completed": true,
|
|
"tested": false,
|
|
"dependencies": [],
|
|
"files": [
|
|
{
|
|
"path": "app/routers_v2/stratplays.py",
|
|
"lines": [342, 456, 645, 733],
|
|
"issue": "Conditionally build SELECT fields based on group_by parameter"
|
|
}
|
|
],
|
|
"suggestedFix": "Already implemented - needs testing with all group_by variations",
|
|
"estimatedHours": 4,
|
|
"notes": "Pattern: only include non-aggregated fields that will be in GROUP BY"
|
|
},
|
|
{
|
|
"id": "MIG-004",
|
|
"name": "Add psycopg2-binary dependency",
|
|
"description": "PostgreSQL adapter for Python",
|
|
"category": "completed",
|
|
"priority": 4,
|
|
"completed": true,
|
|
"tested": true,
|
|
"dependencies": [],
|
|
"files": [
|
|
{
|
|
"path": "requirements.txt",
|
|
"lines": [],
|
|
"issue": "psycopg2-binary added"
|
|
}
|
|
],
|
|
"suggestedFix": "Already implemented",
|
|
"estimatedHours": 0.1,
|
|
"notes": ""
|
|
},
|
|
{
|
|
"id": "MIG-005",
|
|
"name": "Docker Compose for local testing",
|
|
"description": "Local PostgreSQL environment for development testing",
|
|
"category": "completed",
|
|
"priority": 5,
|
|
"completed": true,
|
|
"tested": true,
|
|
"dependencies": [],
|
|
"files": [
|
|
{
|
|
"path": "docker-compose.yml",
|
|
"lines": [],
|
|
"issue": "PostgreSQL 17 + Adminer configured"
|
|
},
|
|
{
|
|
"path": "QUICK_START.md",
|
|
"lines": [],
|
|
"issue": "Testing guide created"
|
|
}
|
|
],
|
|
"suggestedFix": "Already implemented",
|
|
"estimatedHours": 1,
|
|
"notes": "Adminer on port 8081"
|
|
},
|
|
{
|
|
"id": "MIG-006",
|
|
"name": "Migration script auto-detection",
|
|
"description": "db_migrations.py auto-selects PostgresqlMigrator or SqliteMigrator",
|
|
"category": "completed",
|
|
"priority": 6,
|
|
"completed": true,
|
|
"tested": false,
|
|
"dependencies": ["MIG-001"],
|
|
"files": [
|
|
{
|
|
"path": "db_migrations.py",
|
|
"lines": [],
|
|
"issue": "Migrator selection based on DATABASE_TYPE"
|
|
}
|
|
],
|
|
"suggestedFix": "Already implemented",
|
|
"estimatedHours": 0.5,
|
|
"notes": ""
|
|
},
|
|
{
|
|
"id": "MIG-007",
|
|
"name": "Create data migration script with ID preservation",
|
|
"description": "CRITICAL: Migrate all data from SQLite to PostgreSQL while preserving primary key IDs exactly",
|
|
"category": "critical",
|
|
"priority": 1,
|
|
"completed": false,
|
|
"tested": false,
|
|
"dependencies": ["MIG-001", "MIG-002"],
|
|
"files": [
|
|
{
|
|
"path": "scripts/migrate_to_postgres.py",
|
|
"lines": [],
|
|
"issue": "New file - must explicitly insert IDs and reset sequences"
|
|
}
|
|
],
|
|
"suggestedFix": "1. Read all records from SQLite\n2. Insert into PostgreSQL with explicit ID values\n3. Reset PostgreSQL sequences: SELECT setval('table_id_seq', MAX(id))\n4. Validate record counts match\n5. Smart FK error handling (batch insert with individual fallback)",
|
|
"estimatedHours": 3,
|
|
"notes": "Major Domo's #1 lesson: Without explicit ID preservation, PostgreSQL auto-assigns sequential IDs starting from 1, causing all FK references to point to wrong records"
|
|
},
|
|
{
|
|
"id": "MIG-008",
|
|
"name": "Fix on_conflict_replace() calls (Player model)",
|
|
"description": "Convert SQLite on_conflict_replace() to PostgreSQL on_conflict() for Player model",
|
|
"category": "critical",
|
|
"priority": 2,
|
|
"completed": false,
|
|
"tested": false,
|
|
"dependencies": [],
|
|
"files": [
|
|
{
|
|
"path": "main.py",
|
|
"lines": [1696],
|
|
"issue": "Player.insert_many(batch).on_conflict_replace()"
|
|
},
|
|
{
|
|
"path": "app/routers_v2/players.py",
|
|
"lines": [808],
|
|
"issue": "Player.insert_many(batch).on_conflict_replace()"
|
|
}
|
|
],
|
|
"suggestedFix": "Player.insert_many(batch).on_conflict(\n conflict_target=[Player.player_id],\n action='update',\n update={Player.p_name: EXCLUDED.p_name, ...all fields}\n).execute()",
|
|
"estimatedHours": 0.5,
|
|
"notes": "Player has explicit player_id primary key - straightforward"
|
|
},
|
|
{
|
|
"id": "MIG-009",
|
|
"name": "Fix on_conflict_replace() calls (Card models)",
|
|
"description": "Convert SQLite on_conflict_replace() for BattingCard, PitchingCard, CardPosition, ratings",
|
|
"category": "critical",
|
|
"priority": 3,
|
|
"completed": false,
|
|
"tested": false,
|
|
"dependencies": [],
|
|
"files": [
|
|
{
|
|
"path": "app/routers_v2/battingcards.py",
|
|
"lines": [134],
|
|
"issue": "BattingCard - unique on (player, variant)"
|
|
},
|
|
{
|
|
"path": "app/routers_v2/pitchingcards.py",
|
|
"lines": [130],
|
|
"issue": "PitchingCard - unique on (player, variant)"
|
|
},
|
|
{
|
|
"path": "app/routers_v2/cardpositions.py",
|
|
"lines": [131],
|
|
"issue": "CardPosition - unique on (player, variant, position)"
|
|
},
|
|
{
|
|
"path": "app/routers_v2/battingcardratings.py",
|
|
"lines": [549],
|
|
"issue": "BattingCardRatings - unique on (battingcard, vs_hand)"
|
|
},
|
|
{
|
|
"path": "app/routers_v2/pitchingcardratings.py",
|
|
"lines": [432],
|
|
"issue": "PitchingCardRatings - unique on (pitchingcard, vs_hand)"
|
|
}
|
|
],
|
|
"suggestedFix": "All have existing unique indexes - use those as conflict_target",
|
|
"estimatedHours": 2,
|
|
"notes": "These have many fields to update - consider helper function"
|
|
},
|
|
{
|
|
"id": "MIG-010",
|
|
"name": "Fix on_conflict_replace() calls (Game models)",
|
|
"description": "Convert SQLite on_conflict_replace() for StratPlay, Decision, GauntletReward",
|
|
"category": "critical",
|
|
"priority": 4,
|
|
"completed": false,
|
|
"tested": false,
|
|
"dependencies": ["MIG-011"],
|
|
"files": [
|
|
{
|
|
"path": "app/routers_v2/stratplays.py",
|
|
"lines": [1082],
|
|
"issue": "StratPlay - needs unique index on (game, play_num)"
|
|
},
|
|
{
|
|
"path": "app/routers_v2/decisions.py",
|
|
"lines": [217],
|
|
"issue": "Decision - needs unique index on (game, pitcher)"
|
|
},
|
|
{
|
|
"path": "main.py",
|
|
"lines": [4978],
|
|
"issue": "GauntletReward - investigate if id provided or needs refactor"
|
|
},
|
|
{
|
|
"path": "app/routers_v2/gauntletrewards.py",
|
|
"lines": [127],
|
|
"issue": "GauntletReward - same as main.py"
|
|
}
|
|
],
|
|
"suggestedFix": "Add unique indexes first (MIG-011), then implement on_conflict()",
|
|
"estimatedHours": 1.5,
|
|
"notes": "StratPlay and Decision need new unique indexes to be created first"
|
|
},
|
|
{
|
|
"id": "MIG-011",
|
|
"name": "Add missing unique indexes for upserts",
|
|
"description": "Create unique indexes needed for PostgreSQL on_conflict() operations",
|
|
"category": "high",
|
|
"priority": 5,
|
|
"completed": false,
|
|
"tested": false,
|
|
"dependencies": [],
|
|
"files": [
|
|
{
|
|
"path": "app/db_engine.py",
|
|
"lines": [779, 848],
|
|
"issue": "Add unique indexes for StratPlay and Decision"
|
|
}
|
|
],
|
|
"suggestedFix": "StratPlay: ModelIndex(StratPlay, (StratPlay.game, StratPlay.play_num), unique=True)\nDecision: ModelIndex(Decision, (Decision.game, Decision.pitcher), unique=True)",
|
|
"estimatedHours": 1,
|
|
"notes": "These are natural business keys - a play number should be unique within a game, and a pitcher should have one decision per game"
|
|
},
|
|
{
|
|
"id": "MIG-012",
|
|
"name": "Fix on_conflict_replace() for MlbPlayer",
|
|
"description": "Convert or remove on_conflict_replace() for MlbPlayer",
|
|
"category": "medium",
|
|
"priority": 6,
|
|
"completed": false,
|
|
"tested": false,
|
|
"dependencies": [],
|
|
"files": [
|
|
{
|
|
"path": "app/routers_v2/mlbplayers.py",
|
|
"lines": [185],
|
|
"issue": "MlbPlayer.insert_many(batch).on_conflict_replace()"
|
|
}
|
|
],
|
|
"suggestedFix": "Code already checks for duplicates before insert (lines 170-179) and raises HTTPException. The on_conflict_replace() may be unnecessary. Option 1: Remove it and use plain insert_many(). Option 2: Use on_conflict with id as target.",
|
|
"estimatedHours": 0.25,
|
|
"notes": "Low risk - pre-check rejects duplicates"
|
|
},
|
|
{
|
|
"id": "MIG-013",
|
|
"name": "Fix boolean comparison in teams.py",
|
|
"description": "PostgreSQL requires True/False instead of 1/0 for boolean comparisons",
|
|
"category": "low",
|
|
"priority": 7,
|
|
"completed": false,
|
|
"tested": false,
|
|
"dependencies": [],
|
|
"files": [
|
|
{
|
|
"path": "app/routers_v2/teams.py",
|
|
"lines": [110, 112],
|
|
"issue": "Team.has_guide == 0 / Team.has_guide == 1"
|
|
}
|
|
],
|
|
"suggestedFix": "Change to Team.has_guide == False / Team.has_guide == True",
|
|
"estimatedHours": 0.25,
|
|
"notes": "Peewee may handle this automatically, but explicit is better"
|
|
},
|
|
{
|
|
"id": "MIG-014",
|
|
"name": "SQLite data integrity audit",
|
|
"description": "Check for NULL values, orphaned FKs, VARCHAR lengths before migration",
|
|
"category": "high",
|
|
"priority": 8,
|
|
"completed": false,
|
|
"tested": false,
|
|
"dependencies": [],
|
|
"files": [
|
|
{
|
|
"path": "scripts/audit_sqlite.py",
|
|
"lines": [],
|
|
"issue": "New file - pre-migration data validation"
|
|
}
|
|
],
|
|
"suggestedFix": "Create script to check:\n1. NULL values in NOT NULL fields\n2. Orphaned foreign key records\n3. VARCHAR field max lengths\n4. Table record counts for baseline",
|
|
"estimatedHours": 1.5,
|
|
"notes": "Major Domo found 206 orphaned decisions and VARCHAR violations"
|
|
},
|
|
{
|
|
"id": "MIG-015",
|
|
"name": "Test on dev PostgreSQL server",
|
|
"description": "Full migration test on sba-db dev server with production data copy",
|
|
"category": "high",
|
|
"priority": 9,
|
|
"completed": false,
|
|
"tested": false,
|
|
"dependencies": ["MIG-007", "MIG-008", "MIG-009", "MIG-010", "MIG-014"],
|
|
"files": [],
|
|
"suggestedFix": "1. ssh sba-db\n2. Create pd_master database with pd_admin user\n3. Copy production SQLite to dev\n4. Run migration script\n5. Verify record counts\n6. Test API endpoints",
|
|
"estimatedHours": 3,
|
|
"notes": "Dev server access: ssh sba-db, then cd container-data/dev-sba-database/"
|
|
},
|
|
{
|
|
"id": "MIG-016",
|
|
"name": "Production migration execution",
|
|
"description": "Execute migration on production server within maintenance window",
|
|
"category": "critical",
|
|
"priority": 10,
|
|
"completed": false,
|
|
"tested": false,
|
|
"dependencies": ["MIG-015"],
|
|
"files": [],
|
|
"suggestedFix": "1. Notify users of maintenance window\n2. Stop Paper Dynasty API\n3. Create SQLite backup\n4. Create pd_master database and pd_admin user\n5. Run migration script\n6. Verify data integrity\n7. Update docker-compose.yml with PostgreSQL env vars\n8. Start API\n9. Smoke test critical endpoints\n10. Announce migration complete",
|
|
"estimatedHours": 3,
|
|
"notes": "Downtime window: 1-4 hours. Have rollback plan ready."
|
|
}
|
|
],
|
|
"quickWins": [
|
|
{
|
|
"taskId": "MIG-013",
|
|
"estimatedMinutes": 15,
|
|
"impact": "Prevents boolean comparison issues in team queries"
|
|
},
|
|
{
|
|
"taskId": "MIG-012",
|
|
"estimatedMinutes": 15,
|
|
"impact": "Simplify MlbPlayer insert logic"
|
|
}
|
|
],
|
|
"productionBlockers": [
|
|
{
|
|
"taskId": "MIG-007",
|
|
"reason": "Without ID-preserving migration, all foreign key references will break"
|
|
},
|
|
{
|
|
"taskId": "MIG-008",
|
|
"reason": "Player upserts will fail without PostgreSQL-compatible syntax"
|
|
},
|
|
{
|
|
"taskId": "MIG-009",
|
|
"reason": "Card data upserts will fail without PostgreSQL-compatible syntax"
|
|
},
|
|
{
|
|
"taskId": "MIG-010",
|
|
"reason": "Game data upserts will fail without PostgreSQL-compatible syntax"
|
|
}
|
|
],
|
|
"weeklyRoadmap": {
|
|
"week1": {
|
|
"theme": "Code Changes - Make PostgreSQL Compatible",
|
|
"tasks": ["MIG-007", "MIG-008", "MIG-009", "MIG-010", "MIG-011", "MIG-012", "MIG-013"],
|
|
"estimatedHours": 8.5
|
|
},
|
|
"week2": {
|
|
"theme": "Testing & Validation",
|
|
"tasks": ["MIG-014", "MIG-015"],
|
|
"estimatedHours": 4.5
|
|
},
|
|
"week3": {
|
|
"theme": "Production Migration",
|
|
"tasks": ["MIG-016"],
|
|
"estimatedHours": 3
|
|
}
|
|
},
|
|
"rollbackPlan": {
|
|
"triggers": [
|
|
"Data corruption detected",
|
|
"More than 5% of endpoints failing",
|
|
"Performance more than 5x worse than SQLite",
|
|
"Critical functionality broken"
|
|
],
|
|
"duringTesting": {
|
|
"steps": [
|
|
"Set DATABASE_TYPE=sqlite",
|
|
"API immediately uses SQLite",
|
|
"No data loss - PostgreSQL was a copy"
|
|
]
|
|
},
|
|
"afterProduction": {
|
|
"steps": [
|
|
"Stop API: docker-compose down",
|
|
"Update docker-compose.yml: DATABASE_TYPE=sqlite",
|
|
"Restore SQLite backup if needed",
|
|
"Start API: docker-compose up -d",
|
|
"Verify SQLite connectivity",
|
|
"Document issues for retry"
|
|
],
|
|
"timeLimit": "24 hours from migration"
|
|
}
|
|
}
|
|
}
|