paper-dynasty-database/PROJECT_PLAN.json
Cal Corum e1c39cfb17 Fix PostgreSQL timestamp conversion for GET filters
Convert milliseconds timestamps to datetime for created_after filter
in notifications and created_after/before filters in paperdex.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-30 22:38:06 -06:00

620 lines
21 KiB
JSON

{
"meta": {
"version": "1.2.0",
"created": "2026-01-25",
"lastUpdated": "2026-01-30",
"planType": "migration",
"description": "SQLite to PostgreSQL migration for Paper Dynasty database API",
"branch": "postgres-migration",
"totalEstimatedHours": 26,
"totalTasks": 28,
"completedTasks": 20
},
"context": {
"sourceDatabase": {
"type": "SQLite",
"file": "storage/pd_master.db",
"size": "110 MB",
"tables": 29,
"totalRecords": 515000,
"largestTable": {
"name": "stratplay",
"records": 332737
}
},
"targetDatabase": {
"type": "PostgreSQL 17",
"server": "sba_postgres (same server as Major Domo)",
"database": "pd_master",
"user": "pd_admin",
"credentialsFile": ".secrets/pd_admin_credentials.txt"
},
"lessonsFromMajorDomo": [
"CRITICAL: Primary key IDs must be explicitly preserved during migration",
"PostgreSQL GROUP BY requires ALL non-aggregated columns",
"Boolean fields cannot be summed directly - cast to integer first",
"Discord snowflake IDs must be strings, not integers (N/A for Paper Dynasty)",
"VARCHAR fields need explicit max_length",
"NULL constraints are stricter in PostgreSQL",
"Foreign key orphaned records need smart fallback handling",
"Reset sequences after ID-preserving inserts"
],
"devServer": {
"access": "ssh sba-db",
"composeLocation": "cd container-data/dev-sba-database/"
}
},
"categories": {
"critical": "Must complete before migration - blocks production",
"high": "Required for successful migration",
"medium": "Improves migration quality/reliability",
"low": "Polish and nice-to-have",
"completed": "Already done on postgres-migration branch"
},
"tasks": [
{
"id": "MIG-001",
"name": "Environment-based database configuration",
"description": "Add PostgreSQL support with environment variable switching between SQLite/PostgreSQL",
"category": "completed",
"priority": 1,
"completed": true,
"tested": true,
"dependencies": [],
"files": [
{
"path": "app/db_engine.py",
"lines": [11, 35],
"issue": "Now supports DATABASE_TYPE env var for SQLite/PostgreSQL switching"
}
],
"suggestedFix": "Already implemented with PooledPostgresqlDatabase",
"estimatedHours": 2,
"notes": "Includes connection pooling (20 max, 5-min stale timeout, autorollback)"
},
{
"id": "MIG-002",
"name": "Add table_name to all models",
"description": "Explicit table naming for PostgreSQL compatibility",
"category": "completed",
"priority": 2,
"completed": true,
"tested": true,
"dependencies": [],
"files": [
{
"path": "app/db_engine.py",
"lines": [],
"issue": "All 29 models now have Meta.table_name defined"
}
],
"suggestedFix": "Already implemented",
"estimatedHours": 1,
"notes": "Prevents Peewee naming inconsistencies"
},
{
"id": "MIG-003",
"name": "Fix GROUP BY queries for PostgreSQL",
"description": "PostgreSQL requires all non-aggregated SELECT fields in GROUP BY clause",
"category": "completed",
"priority": 3,
"completed": true,
"tested": false,
"dependencies": [],
"files": [
{
"path": "app/routers_v2/stratplays.py",
"lines": [342, 456, 645, 733],
"issue": "Conditionally build SELECT fields based on group_by parameter"
}
],
"suggestedFix": "Already implemented - needs testing with all group_by variations",
"estimatedHours": 4,
"notes": "Pattern: only include non-aggregated fields that will be in GROUP BY"
},
{
"id": "MIG-004",
"name": "Add psycopg2-binary dependency",
"description": "PostgreSQL adapter for Python",
"category": "completed",
"priority": 4,
"completed": true,
"tested": true,
"dependencies": [],
"files": [
{
"path": "requirements.txt",
"lines": [],
"issue": "psycopg2-binary added"
}
],
"suggestedFix": "Already implemented",
"estimatedHours": 0.1,
"notes": ""
},
{
"id": "MIG-005",
"name": "Docker Compose for local testing",
"description": "Local PostgreSQL environment for development testing",
"category": "completed",
"priority": 5,
"completed": true,
"tested": true,
"dependencies": [],
"files": [
{
"path": "docker-compose.yml",
"lines": [],
"issue": "PostgreSQL 17 + Adminer configured"
},
{
"path": "QUICK_START.md",
"lines": [],
"issue": "Testing guide created"
}
],
"suggestedFix": "Already implemented",
"estimatedHours": 1,
"notes": "Adminer on port 8081"
},
{
"id": "MIG-006",
"name": "Migration script auto-detection",
"description": "db_migrations.py auto-selects PostgresqlMigrator or SqliteMigrator",
"category": "completed",
"priority": 6,
"completed": true,
"tested": false,
"dependencies": ["MIG-001"],
"files": [
{
"path": "db_migrations.py",
"lines": [],
"issue": "Migrator selection based on DATABASE_TYPE"
}
],
"suggestedFix": "Already implemented",
"estimatedHours": 0.5,
"notes": ""
},
{
"id": "MIG-007",
"name": "Create data migration script with ID preservation",
"description": "CRITICAL: Migrate all data from SQLite to PostgreSQL while preserving primary key IDs exactly",
"category": "critical",
"priority": 1,
"completed": false,
"tested": false,
"dependencies": ["MIG-001", "MIG-002"],
"files": [
{
"path": "scripts/migrate_to_postgres.py",
"lines": [],
"issue": "New file - must explicitly insert IDs and reset sequences"
}
],
"suggestedFix": "1. Read all records from SQLite\n2. Insert into PostgreSQL with explicit ID values\n3. Reset PostgreSQL sequences: SELECT setval('table_id_seq', MAX(id))\n4. Validate record counts match\n5. Smart FK error handling (batch insert with individual fallback)",
"estimatedHours": 3,
"notes": "Major Domo's #1 lesson: Without explicit ID preservation, PostgreSQL auto-assigns sequential IDs starting from 1, causing all FK references to point to wrong records"
},
{
"id": "MIG-008",
"name": "Fix on_conflict_replace() calls (Player model)",
"description": "Convert SQLite on_conflict_replace() to PostgreSQL on_conflict() for Player model",
"category": "critical",
"priority": 2,
"completed": false,
"tested": false,
"dependencies": [],
"files": [
{
"path": "main.py",
"lines": [1696],
"issue": "Player.insert_many(batch).on_conflict_replace()"
},
{
"path": "app/routers_v2/players.py",
"lines": [808],
"issue": "Player.insert_many(batch).on_conflict_replace()"
}
],
"suggestedFix": "Player.insert_many(batch).on_conflict(\n conflict_target=[Player.player_id],\n action='update',\n update={Player.p_name: EXCLUDED.p_name, ...all fields}\n).execute()",
"estimatedHours": 0.5,
"notes": "Player has explicit player_id primary key - straightforward"
},
{
"id": "MIG-009",
"name": "Fix on_conflict_replace() calls (Card models)",
"description": "Convert SQLite on_conflict_replace() for BattingCard, PitchingCard, CardPosition, ratings",
"category": "critical",
"priority": 3,
"completed": false,
"tested": false,
"dependencies": [],
"files": [
{
"path": "app/routers_v2/battingcards.py",
"lines": [134],
"issue": "BattingCard - unique on (player, variant)"
},
{
"path": "app/routers_v2/pitchingcards.py",
"lines": [130],
"issue": "PitchingCard - unique on (player, variant)"
},
{
"path": "app/routers_v2/cardpositions.py",
"lines": [131],
"issue": "CardPosition - unique on (player, variant, position)"
},
{
"path": "app/routers_v2/battingcardratings.py",
"lines": [549],
"issue": "BattingCardRatings - unique on (battingcard, vs_hand)"
},
{
"path": "app/routers_v2/pitchingcardratings.py",
"lines": [432],
"issue": "PitchingCardRatings - unique on (pitchingcard, vs_hand)"
}
],
"suggestedFix": "All have existing unique indexes - use those as conflict_target",
"estimatedHours": 2,
"notes": "These have many fields to update - consider helper function"
},
{
"id": "MIG-010",
"name": "Fix on_conflict_replace() calls (Game models)",
"description": "Convert SQLite on_conflict_replace() for StratPlay, Decision, GauntletReward",
"category": "critical",
"priority": 4,
"completed": false,
"tested": false,
"dependencies": ["MIG-011"],
"files": [
{
"path": "app/routers_v2/stratplays.py",
"lines": [1082],
"issue": "StratPlay - needs unique index on (game, play_num)"
},
{
"path": "app/routers_v2/decisions.py",
"lines": [217],
"issue": "Decision - needs unique index on (game, pitcher)"
},
{
"path": "main.py",
"lines": [4978],
"issue": "GauntletReward - investigate if id provided or needs refactor"
},
{
"path": "app/routers_v2/gauntletrewards.py",
"lines": [127],
"issue": "GauntletReward - same as main.py"
}
],
"suggestedFix": "Add unique indexes first (MIG-011), then implement on_conflict()",
"estimatedHours": 1.5,
"notes": "StratPlay and Decision need new unique indexes to be created first"
},
{
"id": "MIG-011",
"name": "Add missing unique indexes for upserts",
"description": "Create unique indexes needed for PostgreSQL on_conflict() operations",
"category": "high",
"priority": 5,
"completed": false,
"tested": false,
"dependencies": [],
"files": [
{
"path": "app/db_engine.py",
"lines": [779, 848],
"issue": "Add unique indexes for StratPlay and Decision"
}
],
"suggestedFix": "StratPlay: ModelIndex(StratPlay, (StratPlay.game, StratPlay.play_num), unique=True)\nDecision: ModelIndex(Decision, (Decision.game, Decision.pitcher), unique=True)",
"estimatedHours": 1,
"notes": "These are natural business keys - a play number should be unique within a game, and a pitcher should have one decision per game"
},
{
"id": "MIG-012",
"name": "Fix on_conflict_replace() for MlbPlayer",
"description": "Convert or remove on_conflict_replace() for MlbPlayer",
"category": "medium",
"priority": 6,
"completed": false,
"tested": false,
"dependencies": [],
"files": [
{
"path": "app/routers_v2/mlbplayers.py",
"lines": [185],
"issue": "MlbPlayer.insert_many(batch).on_conflict_replace()"
}
],
"suggestedFix": "Code already checks for duplicates before insert (lines 170-179) and raises HTTPException. The on_conflict_replace() may be unnecessary. Option 1: Remove it and use plain insert_many(). Option 2: Use on_conflict with id as target.",
"estimatedHours": 0.25,
"notes": "Low risk - pre-check rejects duplicates"
},
{
"id": "MIG-013",
"name": "Fix boolean comparison in teams.py",
"description": "PostgreSQL requires True/False instead of 1/0 for boolean comparisons",
"category": "low",
"priority": 7,
"completed": false,
"tested": false,
"dependencies": [],
"files": [
{
"path": "app/routers_v2/teams.py",
"lines": [110, 112],
"issue": "Team.has_guide == 0 / Team.has_guide == 1"
}
],
"suggestedFix": "Change to Team.has_guide == False / Team.has_guide == True",
"estimatedHours": 0.25,
"notes": "Peewee may handle this automatically, but explicit is better"
},
{
"id": "MIG-014",
"name": "SQLite data integrity audit",
"description": "Check for NULL values, orphaned FKs, VARCHAR lengths before migration",
"category": "high",
"priority": 8,
"completed": false,
"tested": false,
"dependencies": [],
"files": [
{
"path": "scripts/audit_sqlite.py",
"lines": [],
"issue": "New file - pre-migration data validation"
}
],
"suggestedFix": "Create script to check:\n1. NULL values in NOT NULL fields\n2. Orphaned foreign key records\n3. VARCHAR field max lengths\n4. Table record counts for baseline",
"estimatedHours": 1.5,
"notes": "Major Domo found 206 orphaned decisions and VARCHAR violations"
},
{
"id": "MIG-015",
"name": "Test on dev PostgreSQL server",
"description": "Full migration test on sba-db dev server with production data copy",
"category": "high",
"priority": 9,
"completed": false,
"tested": false,
"dependencies": ["MIG-007", "MIG-008", "MIG-009", "MIG-010", "MIG-014"],
"files": [],
"suggestedFix": "1. ssh sba-db\n2. Create pd_master database with pd_admin user\n3. Copy production SQLite to dev\n4. Run migration script\n5. Verify record counts\n6. Test API endpoints",
"estimatedHours": 3,
"notes": "Dev server access: ssh sba-db, then cd container-data/dev-sba-database/"
},
{
"id": "MIG-016",
"name": "Production migration execution",
"description": "Execute migration on production server within maintenance window",
"category": "critical",
"priority": 10,
"completed": false,
"tested": false,
"dependencies": ["MIG-015"],
"files": [],
"suggestedFix": "1. Notify users of maintenance window\n2. Stop Paper Dynasty API\n3. Create SQLite backup\n4. Create pd_master database and pd_admin user\n5. Run migration script\n6. Verify data integrity\n7. Update docker-compose.yml with PostgreSQL env vars\n8. Start API\n9. Smoke test critical endpoints\n10. Announce migration complete",
"estimatedHours": 3,
"notes": "Downtime window: 1-4 hours. Have rollback plan ready."
}
],
"quickWins": [
{
"taskId": "MIG-013",
"estimatedMinutes": 15,
"impact": "Prevents boolean comparison issues in team queries"
},
{
"taskId": "MIG-012",
"estimatedMinutes": 15,
"impact": "Simplify MlbPlayer insert logic"
}
],
"productionBlockers": [
{
"taskId": "MIG-007",
"reason": "Without ID-preserving migration, all foreign key references will break"
},
{
"taskId": "MIG-008",
"reason": "Player upserts will fail without PostgreSQL-compatible syntax"
},
{
"taskId": "MIG-009",
"reason": "Card data upserts will fail without PostgreSQL-compatible syntax"
},
{
"taskId": "MIG-010",
"reason": "Game data upserts will fail without PostgreSQL-compatible syntax"
}
],
"weeklyRoadmap": {
"week1": {
"theme": "Code Changes - Make PostgreSQL Compatible",
"tasks": ["MIG-007", "MIG-008", "MIG-009", "MIG-010", "MIG-011", "MIG-012", "MIG-013"],
"estimatedHours": 8.5
},
"week2": {
"theme": "Testing & Validation",
"tasks": ["MIG-014", "MIG-015"],
"estimatedHours": 4.5
},
"week3": {
"theme": "Production Migration",
"tasks": ["MIG-016"],
"estimatedHours": 3
}
},
"timestampFixTasks": {
"description": "Post-migration timestamp type mismatch fixes - Discord bot sends milliseconds integers but PostgreSQL uses DateTimeField",
"pattern": {
"postHandler": "datetime.fromtimestamp(field / 1000)",
"getFilter": "param_dt = datetime.fromtimestamp(param / 1000)",
"nullableField": "Use None instead of 0",
"nullCheck": "Model.field.is_null() instead of == 0"
},
"tasks": [
{
"id": "TS-001",
"name": "Fix rewards.py GET created_after filter",
"category": "critical",
"completed": true,
"file": "app/routers_v2/rewards.py",
"lines": [46, 47],
"notes": "Fixed 2026-01-30. Was causing /comeonmanineedthis to fail."
},
{
"id": "TS-002",
"name": "Fix notifications.py POST created field",
"category": "critical",
"completed": true,
"file": "app/routers_v2/notifications.py",
"lines": [23, 115],
"botUsage": ["helpers/main.py:524", "helpers/main.py:779"],
"notes": "Breaks rare card pull notifications"
},
{
"id": "TS-003",
"name": "Fix notifications.py GET created_after filter",
"category": "critical",
"completed": true,
"file": "app/routers_v2/notifications.py",
"lines": [34, 44],
"notes": "Used for notification polling"
},
{
"id": "TS-004",
"name": "Fix packs.py POST open_time field",
"category": "critical",
"completed": true,
"file": "app/routers_v2/packs.py",
"lines": [29, 158, 184],
"botUsage": ["cogs/economy.py:1287", "cogs/economy_new/team_setup.py:242", "cogs/economy_new/admin_tools.py:76,115,147"],
"notes": "Breaks starter pack creation and admin pack distribution"
},
{
"id": "TS-005",
"name": "Fix packs.py PATCH open_time field",
"category": "critical",
"completed": true,
"file": "app/routers_v2/packs.py",
"lines": [199, 230, 234],
"notes": "Used when updating pack open times"
},
{
"id": "TS-006",
"name": "Fix paperdex.py POST created field",
"category": "critical",
"completed": true,
"file": "app/routers_v2/paperdex.py",
"lines": [26, 121],
"botUsage": ["api_calls.py:post_to_dex()"],
"notes": "Breaks collection tracking"
},
{
"id": "TS-007",
"name": "Fix paperdex.py GET created_after/before filters",
"category": "high",
"completed": true,
"file": "app/routers_v2/paperdex.py",
"lines": [31, 32, 48, 50],
"notes": "Used for filtering paperdex by date range"
},
{
"id": "TS-008",
"name": "Fix gauntletruns.py GET timestamp filters (4 fields)",
"category": "high",
"completed": false,
"file": "app/routers_v2/gauntletruns.py",
"lines": [36, 37, 58, 60, 62, 64],
"notes": "created_after, created_before, ended_after, ended_before"
},
{
"id": "TS-009",
"name": "Fix gauntletruns.py GET is_active filter (NULL vs 0)",
"category": "high",
"completed": false,
"file": "app/routers_v2/gauntletruns.py",
"lines": [67, 69],
"notes": "Should use is_null() not == 0"
},
{
"id": "TS-010",
"name": "Fix gauntletruns.py PATCH timestamp fields",
"category": "high",
"completed": false,
"file": "app/routers_v2/gauntletruns.py",
"lines": [122, 124, 129, 131],
"notes": "Assigns int to DateTimeField, uses 0 instead of None"
},
{
"id": "TS-011",
"name": "Fix gauntletruns.py POST timestamp fields",
"category": "high",
"completed": false,
"file": "app/routers_v2/gauntletruns.py",
"lines": [28, 29, 152],
"notes": "Pydantic model uses int with default 0"
},
{
"id": "TS-012",
"name": "Fix batstats.py GET created filter",
"category": "medium",
"completed": false,
"file": "app/routers_v2/batstats.py",
"lines": [74, 98],
"notes": "Not actively used - commented out in gameplay_legacy.py"
},
{
"id": "TS-013",
"name": "Fix pitstats.py GET created filter",
"category": "medium",
"completed": false,
"file": "app/routers_v2/pitstats.py",
"lines": [60, 85],
"notes": "Not actively used - commented out in gameplay_legacy.py"
}
],
"implementationOrder": [
{"phase": 1, "name": "Critical POST Endpoints", "tasks": ["TS-002", "TS-004", "TS-005", "TS-006"]},
{"phase": 2, "name": "Critical GET Filters", "tasks": ["TS-003", "TS-007"]},
{"phase": 3, "name": "Gauntlet System", "tasks": ["TS-008", "TS-009", "TS-010", "TS-011"]},
{"phase": 4, "name": "Stats Endpoints", "tasks": ["TS-012", "TS-013"]}
]
},
"rollbackPlan": {
"triggers": [
"Data corruption detected",
"More than 5% of endpoints failing",
"Performance more than 5x worse than SQLite",
"Critical functionality broken"
],
"duringTesting": {
"steps": [
"Set DATABASE_TYPE=sqlite",
"API immediately uses SQLite",
"No data loss - PostgreSQL was a copy"
]
},
"afterProduction": {
"steps": [
"Stop API: docker-compose down",
"Update docker-compose.yml: DATABASE_TYPE=sqlite",
"Restore SQLite backup if needed",
"Start API: docker-compose up -d",
"Verify SQLite connectivity",
"Document issues for retry"
],
"timeLimit": "24 hours from migration"
}
}
}