{
  "meta": {
    "version": "1.2.0",
    "created": "2026-01-25",
    "lastUpdated": "2026-01-30",
    "planType": "migration",
    "description": "SQLite to PostgreSQL migration for Paper Dynasty database API",
    "branch": "postgres-migration",
    "totalEstimatedHours": 26,
    "totalTasks": 28,
    "completedTasks": 18
  },
  "context": {
    "sourceDatabase": {
      "type": "SQLite",
      "file": "storage/pd_master.db",
      "size": "110 MB",
      "tables": 29,
      "totalRecords": 515000,
      "largestTable": {
        "name": "stratplay",
        "records": 332737
      }
    },
    "targetDatabase": {
      "type": "PostgreSQL 17",
      "server": "sba_postgres (same server as Major Domo)",
      "database": "pd_master",
      "user": "pd_admin",
      "credentialsFile": ".secrets/pd_admin_credentials.txt"
    },
    "lessonsFromMajorDomo": [
      "CRITICAL: Primary key IDs must be explicitly preserved during migration",
      "PostgreSQL GROUP BY requires ALL non-aggregated columns",
      "Boolean fields cannot be summed directly - cast to integer first",
      "Discord snowflake IDs must be strings, not integers (N/A for Paper Dynasty)",
      "VARCHAR fields need explicit max_length",
      "NULL constraints are stricter in PostgreSQL",
      "Foreign key orphaned records need smart fallback handling",
      "Reset sequences after ID-preserving inserts"
    ],
    "devServer": {
      "access": "ssh sba-db",
      "composeLocation": "cd container-data/dev-sba-database/"
    }
  },
  "categories": {
    "critical": "Must complete before migration - blocks production",
    "high": "Required for successful migration",
    "medium": "Improves migration quality/reliability",
    "low": "Polish and nice-to-have",
    "completed": "Already done on postgres-migration branch"
  },
  "tasks": [
    {
      "id": "MIG-001",
      "name": "Environment-based database configuration",
      "description": "Add PostgreSQL support with environment variable switching between SQLite/PostgreSQL",
      "category": "completed",
      "priority": 1,
      "completed": true,
      "tested": true,
      "dependencies": [],
      "files": [
        {
          "path": "app/db_engine.py",
          "lines": [11, 35],
          "issue": "Now supports DATABASE_TYPE env var for SQLite/PostgreSQL switching"
        }
      ],
      "suggestedFix": "Already implemented with PooledPostgresqlDatabase",
      "estimatedHours": 2,
      "notes": "Includes connection pooling (20 max, 5-min stale timeout, autorollback)"
    },
    {
      "id": "MIG-002",
      "name": "Add table_name to all models",
      "description": "Explicit table naming for PostgreSQL compatibility",
      "category": "completed",
      "priority": 2,
      "completed": true,
      "tested": true,
      "dependencies": [],
      "files": [
        {
          "path": "app/db_engine.py",
          "lines": [],
          "issue": "All 29 models now have Meta.table_name defined"
        }
      ],
      "suggestedFix": "Already implemented",
      "estimatedHours": 1,
      "notes": "Prevents Peewee naming inconsistencies"
    },
    {
      "id": "MIG-003",
      "name": "Fix GROUP BY queries for PostgreSQL",
      "description": "PostgreSQL requires all non-aggregated SELECT fields in GROUP BY clause",
      "category": "completed",
      "priority": 3,
      "completed": true,
      "tested": false,
      "dependencies": [],
      "files": [
        {
          "path": "app/routers_v2/stratplays.py",
          "lines": [342, 456, 645, 733],
          "issue": "Conditionally build SELECT fields based on group_by parameter"
        }
      ],
      "suggestedFix": "Already implemented - needs testing with all group_by variations",
      "estimatedHours": 4,
      "notes": "Pattern: only include non-aggregated fields that will be in GROUP BY"
    },
    {
      "id": "MIG-004",
      "name": "Add psycopg2-binary dependency",
      "description": "PostgreSQL adapter for Python",
      "category": "completed",
      "priority": 4,
      "completed": true,
      "tested": true,
      "dependencies": [],
      "files": [
        {
          "path": "requirements.txt",
          "lines": [],
          "issue": "psycopg2-binary added"
        }
      ],
      "suggestedFix": "Already implemented",
      "estimatedHours": 0.1,
      "notes": ""
    },
    {
      "id": "MIG-005",
      "name": "Docker Compose for local testing",
      "description": "Local PostgreSQL environment for development testing",
      "category": "completed",
      "priority": 5,
      "completed": true,
      "tested": true,
      "dependencies": [],
      "files": [
        {
          "path": "docker-compose.yml",
          "lines": [],
          "issue": "PostgreSQL 17 + Adminer configured"
        },
        {
          "path": "QUICK_START.md",
          "lines": [],
          "issue": "Testing guide created"
        }
      ],
      "suggestedFix": "Already implemented",
      "estimatedHours": 1,
      "notes": "Adminer on port 8081"
    },
    {
      "id": "MIG-006",
      "name": "Migration script auto-detection",
      "description": "db_migrations.py auto-selects PostgresqlMigrator or SqliteMigrator",
      "category": "completed",
      "priority": 6,
      "completed": true,
      "tested": false,
      "dependencies": ["MIG-001"],
      "files": [
        {
          "path": "db_migrations.py",
          "lines": [],
          "issue": "Migrator selection based on DATABASE_TYPE"
        }
      ],
      "suggestedFix": "Already implemented",
      "estimatedHours": 0.5,
      "notes": ""
    },
    {
      "id": "MIG-007",
      "name": "Create data migration script with ID preservation",
      "description": "CRITICAL: Migrate all data from SQLite to PostgreSQL while preserving primary key IDs exactly",
      "category": "critical",
      "priority": 1,
      "completed": false,
      "tested": false,
      "dependencies": ["MIG-001", "MIG-002"],
      "files": [
        {
          "path": "scripts/migrate_to_postgres.py",
          "lines": [],
          "issue": "New file - must explicitly insert IDs and reset sequences"
        }
      ],
      "suggestedFix": "1. Read all records from SQLite\n2. Insert into PostgreSQL with explicit ID values\n3. Reset PostgreSQL sequences: SELECT setval('table_id_seq', MAX(id))\n4. Validate record counts match\n5. Smart FK error handling (batch insert with individual fallback)",
      "estimatedHours": 3,
      "notes": "Major Domo's #1 lesson: Without explicit ID preservation, PostgreSQL auto-assigns sequential IDs starting from 1, causing all FK references to point to wrong records"
    },
    {
      "id": "MIG-008",
      "name": "Fix on_conflict_replace() calls (Player model)",
      "description": "Convert SQLite on_conflict_replace() to PostgreSQL on_conflict() for Player model",
      "category": "critical",
      "priority": 2,
      "completed": false,
      "tested": false,
      "dependencies": [],
      "files": [
        {
          "path": "main.py",
          "lines": [1696],
          "issue": "Player.insert_many(batch).on_conflict_replace()"
        },
        {
          "path": "app/routers_v2/players.py",
          "lines": [808],
          "issue": "Player.insert_many(batch).on_conflict_replace()"
        }
      ],
      "suggestedFix": "Player.insert_many(batch).on_conflict(\n    conflict_target=[Player.player_id],\n    action='update',\n    update={Player.p_name: EXCLUDED.p_name, ...all fields}\n).execute()",
      "estimatedHours": 0.5,
      "notes": "Player has explicit player_id primary key - straightforward"
    },
    {
      "id": "MIG-009",
      "name": "Fix on_conflict_replace() calls (Card models)",
      "description": "Convert SQLite on_conflict_replace() for BattingCard, PitchingCard, CardPosition, ratings",
      "category": "critical",
      "priority": 3,
      "completed": false,
      "tested": false,
      "dependencies": [],
      "files": [
        {
          "path": "app/routers_v2/battingcards.py",
          "lines": [134],
          "issue": "BattingCard - unique on (player, variant)"
        },
        {
          "path": "app/routers_v2/pitchingcards.py",
          "lines": [130],
          "issue": "PitchingCard - unique on (player, variant)"
        },
        {
          "path": "app/routers_v2/cardpositions.py",
          "lines": [131],
          "issue": "CardPosition - unique on (player, variant, position)"
        },
        {
          "path": "app/routers_v2/battingcardratings.py",
          "lines": [549],
          "issue": "BattingCardRatings - unique on (battingcard, vs_hand)"
        },
        {
          "path": "app/routers_v2/pitchingcardratings.py",
          "lines": [432],
          "issue": "PitchingCardRatings - unique on (pitchingcard, vs_hand)"
        }
      ],
      "suggestedFix": "All have existing unique indexes - use those as conflict_target",
      "estimatedHours": 2,
      "notes": "These have many fields to update - consider helper function"
    },
    {
      "id": "MIG-010",
      "name": "Fix on_conflict_replace() calls (Game models)",
      "description": "Convert SQLite on_conflict_replace() for StratPlay, Decision, GauntletReward",
      "category": "critical",
      "priority": 4,
      "completed": false,
      "tested": false,
      "dependencies": ["MIG-011"],
      "files": [
        {
          "path": "app/routers_v2/stratplays.py",
          "lines": [1082],
          "issue": "StratPlay - needs unique index on (game, play_num)"
        },
        {
          "path": "app/routers_v2/decisions.py",
          "lines": [217],
          "issue": "Decision - needs unique index on (game, pitcher)"
        },
        {
          "path": "main.py",
          "lines": [4978],
          "issue": "GauntletReward - investigate if id provided or needs refactor"
        },
        {
          "path": "app/routers_v2/gauntletrewards.py",
          "lines": [127],
          "issue": "GauntletReward - same as main.py"
        }
      ],
      "suggestedFix": "Add unique indexes first (MIG-011), then implement on_conflict()",
      "estimatedHours": 1.5,
      "notes": "StratPlay and Decision need new unique indexes to be created first"
    },
    {
      "id": "MIG-011",
      "name": "Add missing unique indexes for upserts",
      "description": "Create unique indexes needed for PostgreSQL on_conflict() operations",
      "category": "high",
      "priority": 5,
      "completed": false,
      "tested": false,
      "dependencies": [],
      "files": [
        {
          "path": "app/db_engine.py",
          "lines": [779, 848],
          "issue": "Add unique indexes for StratPlay and Decision"
        }
      ],
      "suggestedFix": "StratPlay: ModelIndex(StratPlay, (StratPlay.game, StratPlay.play_num), unique=True)\nDecision: ModelIndex(Decision, (Decision.game, Decision.pitcher), unique=True)",
      "estimatedHours": 1,
      "notes": "These are natural business keys - a play number should be unique within a game, and a pitcher should have one decision per game"
    },
    {
      "id": "MIG-012",
      "name": "Fix on_conflict_replace() for MlbPlayer",
      "description": "Convert or remove on_conflict_replace() for MlbPlayer",
      "category": "medium",
      "priority": 6,
      "completed": false,
      "tested": false,
      "dependencies": [],
      "files": [
        {
          "path": "app/routers_v2/mlbplayers.py",
          "lines": [185],
          "issue": "MlbPlayer.insert_many(batch).on_conflict_replace()"
        }
      ],
      "suggestedFix": "Code already checks for duplicates before insert (lines 170-179) and raises HTTPException. The on_conflict_replace() may be unnecessary. Option 1: Remove it and use plain insert_many(). Option 2: Use on_conflict with id as target.",
      "estimatedHours": 0.25,
      "notes": "Low risk - pre-check rejects duplicates"
    },
    {
      "id": "MIG-013",
      "name": "Fix boolean comparison in teams.py",
      "description": "PostgreSQL requires True/False instead of 1/0 for boolean comparisons",
      "category": "low",
      "priority": 7,
      "completed": false,
      "tested": false,
      "dependencies": [],
      "files": [
        {
          "path": "app/routers_v2/teams.py",
          "lines": [110, 112],
          "issue": "Team.has_guide == 0 / Team.has_guide == 1"
        }
      ],
      "suggestedFix": "Change to Team.has_guide == False / Team.has_guide == True",
      "estimatedHours": 0.25,
      "notes": "Peewee may handle this automatically, but explicit is better"
    },
    {
      "id": "MIG-014",
      "name": "SQLite data integrity audit",
      "description": "Check for NULL values, orphaned FKs, VARCHAR lengths before migration",
      "category": "high",
      "priority": 8,
      "completed": false,
      "tested": false,
      "dependencies": [],
      "files": [
        {
          "path": "scripts/audit_sqlite.py",
          "lines": [],
          "issue": "New file - pre-migration data validation"
        }
      ],
      "suggestedFix": "Create script to check:\n1. NULL values in NOT NULL fields\n2. Orphaned foreign key records\n3. VARCHAR field max lengths\n4. Table record counts for baseline",
      "estimatedHours": 1.5,
      "notes": "Major Domo found 206 orphaned decisions and VARCHAR violations"
    },
    {
      "id": "MIG-015",
      "name": "Test on dev PostgreSQL server",
      "description": "Full migration test on sba-db dev server with production data copy",
      "category": "high",
      "priority": 9,
      "completed": false,
      "tested": false,
      "dependencies": ["MIG-007", "MIG-008", "MIG-009", "MIG-010", "MIG-014"],
      "files": [],
      "suggestedFix": "1. ssh sba-db\n2. Create pd_master database with pd_admin user\n3. Copy production SQLite to dev\n4. Run migration script\n5. Verify record counts\n6. Test API endpoints",
      "estimatedHours": 3,
      "notes": "Dev server access: ssh sba-db, then cd container-data/dev-sba-database/"
    },
    {
      "id": "MIG-016",
      "name": "Production migration execution",
      "description": "Execute migration on production server within maintenance window",
      "category": "critical",
      "priority": 10,
      "completed": false,
      "tested": false,
      "dependencies": ["MIG-015"],
      "files": [],
      "suggestedFix": "1. Notify users of maintenance window\n2. Stop Paper Dynasty API\n3. Create SQLite backup\n4. Create pd_master database and pd_admin user\n5. Run migration script\n6. Verify data integrity\n7. Update docker-compose.yml with PostgreSQL env vars\n8. Start API\n9. Smoke test critical endpoints\n10. Announce migration complete",
      "estimatedHours": 3,
      "notes": "Downtime window: 1-4 hours. Have rollback plan ready."
    }
  ],
  "quickWins": [
    {
      "taskId": "MIG-013",
      "estimatedMinutes": 15,
      "impact": "Prevents boolean comparison issues in team queries"
    },
    {
      "taskId": "MIG-012",
      "estimatedMinutes": 15,
      "impact": "Simplify MlbPlayer insert logic"
    }
  ],
  "productionBlockers": [
    {
      "taskId": "MIG-007",
      "reason": "Without ID-preserving migration, all foreign key references will break"
    },
    {
      "taskId": "MIG-008",
      "reason": "Player upserts will fail without PostgreSQL-compatible syntax"
    },
    {
      "taskId": "MIG-009",
      "reason": "Card data upserts will fail without PostgreSQL-compatible syntax"
    },
    {
      "taskId": "MIG-010",
      "reason": "Game data upserts will fail without PostgreSQL-compatible syntax"
    }
  ],
  "weeklyRoadmap": {
    "week1": {
      "theme": "Code Changes - Make PostgreSQL Compatible",
      "tasks": ["MIG-007", "MIG-008", "MIG-009", "MIG-010", "MIG-011", "MIG-012", "MIG-013"],
      "estimatedHours": 8.5
    },
    "week2": {
      "theme": "Testing & Validation",
      "tasks": ["MIG-014", "MIG-015"],
      "estimatedHours": 4.5
    },
    "week3": {
      "theme": "Production Migration",
      "tasks": ["MIG-016"],
      "estimatedHours": 3
    }
  },
  "timestampFixTasks": {
    "description": "Post-migration timestamp type mismatch fixes - Discord bot sends milliseconds integers but PostgreSQL uses DateTimeField",
    "pattern": {
      "postHandler": "datetime.fromtimestamp(field / 1000)",
      "getFilter": "param_dt = datetime.fromtimestamp(param / 1000)",
      "nullableField": "Use None instead of 0",
      "nullCheck": "Model.field.is_null() instead of == 0"
    },
    "tasks": [
      {
        "id": "TS-001",
        "name": "Fix rewards.py GET created_after filter",
        "category": "critical",
        "completed": true,
        "file": "app/routers_v2/rewards.py",
        "lines": [46, 47],
        "notes": "Fixed 2026-01-30. Was causing /comeonmanineedthis to fail."
      },
      {
        "id": "TS-002",
        "name": "Fix notifications.py POST created field",
        "category": "critical",
        "completed": true,
        "file": "app/routers_v2/notifications.py",
        "lines": [23, 115],
        "botUsage": ["helpers/main.py:524", "helpers/main.py:779"],
        "notes": "Breaks rare card pull notifications"
      },
      {
        "id": "TS-003",
        "name": "Fix notifications.py GET created_after filter",
        "category": "critical",
        "completed": false,
        "file": "app/routers_v2/notifications.py",
        "lines": [34, 44],
        "notes": "Used for notification polling"
      },
      {
        "id": "TS-004",
        "name": "Fix packs.py POST open_time field",
        "category": "critical",
        "completed": true,
        "file": "app/routers_v2/packs.py",
        "lines": [29, 158, 184],
        "botUsage": ["cogs/economy.py:1287", "cogs/economy_new/team_setup.py:242", "cogs/economy_new/admin_tools.py:76,115,147"],
        "notes": "Breaks starter pack creation and admin pack distribution"
      },
      {
        "id": "TS-005",
        "name": "Fix packs.py PATCH open_time field",
        "category": "critical",
        "completed": true,
        "file": "app/routers_v2/packs.py",
        "lines": [199, 230, 234],
        "notes": "Used when updating pack open times"
      },
      {
        "id": "TS-006",
        "name": "Fix paperdex.py POST created field",
        "category": "critical",
        "completed": true,
        "file": "app/routers_v2/paperdex.py",
        "lines": [26, 121],
        "botUsage": ["api_calls.py:post_to_dex()"],
        "notes": "Breaks collection tracking"
      },
      {
        "id": "TS-007",
        "name": "Fix paperdex.py GET created_after/before filters",
        "category": "high",
        "completed": false,
        "file": "app/routers_v2/paperdex.py",
        "lines": [31, 32, 48, 50],
        "notes": "Used for filtering paperdex by date range"
      },
      {
        "id": "TS-008",
        "name": "Fix gauntletruns.py GET timestamp filters (4 fields)",
        "category": "high",
        "completed": false,
        "file": "app/routers_v2/gauntletruns.py",
        "lines": [36, 37, 58, 60, 62, 64],
        "notes": "created_after, created_before, ended_after, ended_before"
      },
      {
        "id": "TS-009",
        "name": "Fix gauntletruns.py GET is_active filter (NULL vs 0)",
        "category": "high",
        "completed": false,
        "file": "app/routers_v2/gauntletruns.py",
        "lines": [67, 69],
        "notes": "Should use is_null() not == 0"
      },
      {
        "id": "TS-010",
        "name": "Fix gauntletruns.py PATCH timestamp fields",
        "category": "high",
        "completed": false,
        "file": "app/routers_v2/gauntletruns.py",
        "lines": [122, 124, 129, 131],
        "notes": "Assigns int to DateTimeField, uses 0 instead of None"
      },
      {
        "id": "TS-011",
        "name": "Fix gauntletruns.py POST timestamp fields",
        "category": "high",
        "completed": false,
        "file": "app/routers_v2/gauntletruns.py",
        "lines": [28, 29, 152],
        "notes": "Pydantic model uses int with default 0"
      },
      {
        "id": "TS-012",
        "name": "Fix batstats.py GET created filter",
        "category": "medium",
        "completed": false,
        "file": "app/routers_v2/batstats.py",
        "lines": [74, 98],
        "notes": "Not actively used - commented out in gameplay_legacy.py"
      },
      {
        "id": "TS-013",
        "name": "Fix pitstats.py GET created filter",
        "category": "medium",
        "completed": false,
        "file": "app/routers_v2/pitstats.py",
        "lines": [60, 85],
        "notes": "Not actively used - commented out in gameplay_legacy.py"
      }
    ],
    "implementationOrder": [
      {"phase": 1, "name": "Critical POST Endpoints", "tasks": ["TS-002", "TS-004", "TS-005", "TS-006"]},
      {"phase": 2, "name": "Critical GET Filters", "tasks": ["TS-003", "TS-007"]},
      {"phase": 3, "name": "Gauntlet System", "tasks": ["TS-008", "TS-009", "TS-010", "TS-011"]},
      {"phase": 4, "name": "Stats Endpoints", "tasks": ["TS-012", "TS-013"]}
    ]
  },
  "rollbackPlan": {
    "triggers": [
      "Data corruption detected",
      "More than 5% of endpoints failing",
      "Performance more than 5x worse than SQLite",
      "Critical functionality broken"
    ],
    "duringTesting": {
      "steps": [
        "Set DATABASE_TYPE=sqlite",
        "API immediately uses SQLite",
        "No data loss - PostgreSQL was a copy"
      ]
    },
    "afterProduction": {
      "steps": [
        "Stop API: docker-compose down",
        "Update docker-compose.yml: DATABASE_TYPE=sqlite",
        "Restore SQLite backup if needed",
        "Start API: docker-compose up -d",
        "Verify SQLite connectivity",
        "Document issues for retry"
      ],
      "timeLimit": "24 hours from migration"
    }
  }
}