diff --git a/CLAUDE.md b/CLAUDE.md index f3c3426..d7b9c71 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,7 +14,7 @@ docker build -t paper-dynasty-db . # Build image ## Architecture - **Routers**: Domain-based in `app/routers_v2/` (cards, players, teams, packs, stats, gauntlets, scouting) -- **ORM**: Peewee with PostgreSQL +- **ORM**: Peewee with PostgreSQL (`pd_master` database via PooledPostgresqlDatabase) - **Card images**: Playwright/Chromium renders HTML templates → screenshots (see `routers_v2/players.py`) - **Logging**: Rotating files in `logs/database/{date}.log` @@ -42,7 +42,7 @@ docker build -t paper-dynasty-db . # Build image - **API docs**: `/api/docs` and `/api/redoc` ### Key Env Vars -`API_TOKEN`, `LOG_LEVEL`, `DATABASE_TYPE`, `POSTGRES_HOST`, `POSTGRES_DB`, `POSTGRES_USER`, `POSTGRES_PASSWORD` +`API_TOKEN`, `LOG_LEVEL`, `POSTGRES_HOST`, `POSTGRES_DB`, `POSTGRES_USER`, `POSTGRES_PASSWORD` ### Common Issues - 502 Bad Gateway → API container crashed; check `docker logs pd_api` diff --git a/app/db_engine.py b/app/db_engine.py index f132f47..f05d864 100644 --- a/app/db_engine.py +++ b/app/db_engine.py @@ -9,40 +9,20 @@ from peewee import * from peewee import ModelSelect from playhouse.shortcuts import model_to_dict -# Database configuration - supports both SQLite and PostgreSQL -DATABASE_TYPE = os.environ.get("DATABASE_TYPE", "sqlite") -# Skip table creation for PostgreSQL (tables already exist in production) -SKIP_TABLE_CREATION = DATABASE_TYPE.lower() == "postgresql" +from playhouse.pool import PooledPostgresqlDatabase -if DATABASE_TYPE.lower() == "postgresql": - from playhouse.pool import PooledPostgresqlDatabase - - db = PooledPostgresqlDatabase( - os.environ.get("POSTGRES_DB", "pd_master"), - user=os.environ.get("POSTGRES_USER", "pd_admin"), - password=os.environ.get("POSTGRES_PASSWORD"), - host=os.environ.get("POSTGRES_HOST", "localhost"), - port=int(os.environ.get("POSTGRES_PORT", "5432")), - max_connections=20, - stale_timeout=300, # 5 minutes - timeout=0, - autoconnect=True, - autorollback=True, # Automatically rollback failed transactions - ) -else: - # SQLite configuration for local development only. - # Production always uses PostgreSQL (see DATABASE_TYPE env var). - # - # synchronous=0 (OFF): SQLite skips fsync() after every write, maximising - # throughput at the cost of durability — a hard crash could corrupt the DB. - # This is an acceptable trade-off in dev where data loss is tolerable and - # write speed matters. WAL journal mode reduces (but does not eliminate) - # the corruption window by keeping the main database file consistent while - # writes land in the WAL file first. - db = SqliteDatabase( - "storage/pd_master.db", - pragmas={"journal_mode": "wal", "cache_size": -1 * 64000, "synchronous": 0}, - ) +db = PooledPostgresqlDatabase( + os.environ.get("POSTGRES_DB", "pd_master"), + user=os.environ.get("POSTGRES_USER", "pd_admin"), + password=os.environ.get("POSTGRES_PASSWORD"), + host=os.environ.get("POSTGRES_HOST", "localhost"), + port=int(os.environ.get("POSTGRES_PORT", "5432")), + max_connections=20, + stale_timeout=300, # 5 minutes + timeout=0, + autoconnect=True, + autorollback=True, # Automatically rollback failed transactions +) # Refractor stat accumulation starts at this season — stats from earlier seasons # are excluded from evaluation queries. Override via REFRACTOR_START_SEASON env var. @@ -203,10 +183,6 @@ class Current(BaseModel): return latest_current -if not SKIP_TABLE_CREATION: - db.create_tables([Current], safe=True) - - class Rarity(BaseModel): value = IntegerField() name = CharField(unique=True) @@ -220,10 +196,6 @@ class Rarity(BaseModel): return self.name -if not SKIP_TABLE_CREATION: - db.create_tables([Rarity], safe=True) - - class Event(BaseModel): name = CharField() short_desc = CharField(null=True) @@ -237,10 +209,6 @@ class Event(BaseModel): table_name = "event" -if not SKIP_TABLE_CREATION: - db.create_tables([Event], safe=True) - - class Cardset(BaseModel): name = CharField() description = CharField() @@ -258,10 +226,6 @@ class Cardset(BaseModel): return self.name -if not SKIP_TABLE_CREATION: - db.create_tables([Cardset], safe=True) - - class MlbPlayer(BaseModel): first_name = CharField() last_name = CharField() @@ -276,10 +240,6 @@ class MlbPlayer(BaseModel): table_name = "mlbplayer" -if not SKIP_TABLE_CREATION: - db.create_tables([MlbPlayer], safe=True) - - class Player(BaseModel): player_id = IntegerField(primary_key=True) p_name = CharField() @@ -378,10 +338,6 @@ class Player(BaseModel): table_name = "player" -if not SKIP_TABLE_CREATION: - db.create_tables([Player], safe=True) - - class Team(BaseModel): abbrev = CharField(max_length=20) # Gauntlet teams use prefixes like "Gauntlet-NCB" sname = CharField(max_length=100) @@ -437,10 +393,6 @@ class Team(BaseModel): table_name = "team" -if not SKIP_TABLE_CREATION: - db.create_tables([Team], safe=True) - - class PackType(BaseModel): name = CharField() card_count = IntegerField() @@ -453,10 +405,6 @@ class PackType(BaseModel): table_name = "packtype" -if not SKIP_TABLE_CREATION: - db.create_tables([PackType], safe=True) - - class Pack(BaseModel): team = ForeignKeyField(Team) pack_type = ForeignKeyField(PackType) @@ -469,10 +417,6 @@ class Pack(BaseModel): table_name = "pack" -if not SKIP_TABLE_CREATION: - db.create_tables([Pack], safe=True) - - class Card(BaseModel): player = ForeignKeyField(Player, null=True) team = ForeignKeyField(Team, null=True) @@ -495,10 +439,6 @@ class Card(BaseModel): table_name = "card" -if not SKIP_TABLE_CREATION: - db.create_tables([Card], safe=True) - - class Roster(BaseModel): team = ForeignKeyField(Team) name = CharField() @@ -728,26 +668,6 @@ class GauntletRun(BaseModel): table_name = "gauntletrun" -if not SKIP_TABLE_CREATION: - db.create_tables( - [ - Roster, - RosterSlot, - BattingStat, - PitchingStat, - Result, - Award, - Paperdex, - Reward, - GameRewards, - Notification, - GauntletReward, - GauntletRun, - ], - safe=True, - ) - - class BattingCard(BaseModel): player = ForeignKeyField(Player) variant = IntegerField() @@ -914,19 +834,6 @@ pos_index = ModelIndex( CardPosition.add_index(pos_index) -if not SKIP_TABLE_CREATION: - db.create_tables( - [ - BattingCard, - BattingCardRatings, - PitchingCard, - PitchingCardRatings, - CardPosition, - ], - safe=True, - ) - - class StratGame(BaseModel): season = IntegerField() game_type = CharField() @@ -1168,20 +1075,6 @@ class ProcessedGame(BaseModel): table_name = "processed_game" -if not SKIP_TABLE_CREATION: - db.create_tables( - [ - StratGame, - StratPlay, - Decision, - BattingSeasonStats, - PitchingSeasonStats, - ProcessedGame, - ], - safe=True, - ) - - class ScoutOpportunity(BaseModel): pack = ForeignKeyField(Pack, null=True) opener_team = ForeignKeyField(Team) @@ -1213,10 +1106,6 @@ scout_claim_index = ModelIndex( ScoutClaim.add_index(scout_claim_index) -if not SKIP_TABLE_CREATION: - db.create_tables([ScoutOpportunity, ScoutClaim], safe=True) - - class RefractorTrack(BaseModel): name = CharField(unique=True) card_type = CharField() # 'batter', 'sp', 'rp' @@ -1277,189 +1166,4 @@ class RefractorBoostAudit(BaseModel): table_name = "refractor_boost_audit" -if not SKIP_TABLE_CREATION: - db.create_tables( - [ - RefractorTrack, - RefractorCardState, - RefractorBoostAudit, - ], - safe=True, - ) - - db.close() - -# scout_db = SqliteDatabase( -# 'storage/card_creation.db', -# pragmas={ -# 'journal_mode': 'wal', -# 'cache_size': -1 * 64000, -# 'synchronous': 0 -# } -# ) -# -# -# class BaseModelScout(Model): -# class Meta: -# database = scout_db -# -# -# class ScoutCardset(BaseModelScout): -# set_title = CharField() -# set_subtitle = CharField(null=True) -# -# -# class ScoutPlayer(BaseModelScout): -# sba_id = IntegerField(primary_key=True) -# name = CharField() -# fg_id = IntegerField() -# br_id = CharField() -# offense_col = IntegerField() -# hand = CharField(default='R') -# -# -# scout_db.create_tables([ScoutCardset, ScoutPlayer], safe=True) -# -# -# class BatterRatings(BaseModelScout): -# id = CharField(unique=True, primary_key=True) -# player = ForeignKeyField(ScoutPlayer) -# cardset = ForeignKeyField(ScoutCardset) -# vs_hand = FloatField() -# is_prep = BooleanField() -# homerun = FloatField() -# bp_homerun = FloatField() -# triple = FloatField() -# double_three = FloatField() -# double_two = FloatField() -# double_pull = FloatField() -# single_two = FloatField() -# single_one = FloatField() -# single_center = FloatField() -# bp_single = FloatField() -# hbp = FloatField() -# walk = FloatField() -# strikeout = FloatField() -# lineout = FloatField() -# popout = FloatField() -# flyout_a = FloatField() -# flyout_bq = FloatField() -# flyout_lf_b = FloatField() -# flyout_rf_b = FloatField() -# groundout_a = FloatField() -# groundout_b = FloatField() -# groundout_c = FloatField() -# avg = FloatField(null=True) -# obp = FloatField(null=True) -# slg = FloatField(null=True) -# -# -# class PitcherRatings(BaseModelScout): -# id = CharField(unique=True, primary_key=True) -# player = ForeignKeyField(ScoutPlayer) -# cardset = ForeignKeyField(ScoutCardset) -# vs_hand = CharField() -# is_prep = BooleanField() -# homerun = FloatField() -# bp_homerun = FloatField() -# triple = FloatField() -# double_three = FloatField() -# double_two = FloatField() -# double_cf = FloatField() -# single_two = FloatField() -# single_one = FloatField() -# single_center = FloatField() -# bp_single = FloatField() -# hbp = FloatField() -# walk = FloatField() -# strikeout = FloatField() -# fo_slap = FloatField() -# fo_center = FloatField() -# groundout_a = FloatField() -# groundout_b = FloatField() -# xcheck_p = FloatField() -# xcheck_c = FloatField() -# xcheck_1b = FloatField() -# xcheck_2b = FloatField() -# xcheck_3b = FloatField() -# xcheck_ss = FloatField() -# xcheck_lf = FloatField() -# xcheck_cf = FloatField() -# xcheck_rf = FloatField() -# avg = FloatField(null=True) -# obp = FloatField(null=True) -# slg = FloatField(null=True) -# -# -# # scout_db.create_tables([BatterRatings, PitcherRatings], safe=True) -# -# -# class CardColumns(BaseModelScout): -# id = CharField(unique=True, primary_key=True) -# player = ForeignKeyField(ScoutPlayer) -# hand = CharField() -# b_ratings = ForeignKeyField(BatterRatings, null=True) -# p_ratings = ForeignKeyField(PitcherRatings, null=True) -# one_dice = CharField() -# one_results = CharField() -# one_splits = CharField() -# two_dice = CharField() -# two_results = CharField() -# two_splits = CharField() -# three_dice = CharField() -# three_results = CharField() -# three_splits = CharField() -# -# -# class Position(BaseModelScout): -# player = ForeignKeyField(ScoutPlayer) -# cardset = ForeignKeyField(ScoutCardset) -# position = CharField() -# innings = IntegerField() -# range = IntegerField() -# error = IntegerField() -# arm = CharField(null=True) -# pb = IntegerField(null=True) -# overthrow = IntegerField(null=True) -# -# -# class BatterData(BaseModelScout): -# player = ForeignKeyField(ScoutPlayer) -# cardset = ForeignKeyField(ScoutCardset) -# stealing = CharField() -# st_low = IntegerField() -# st_high = IntegerField() -# st_auto = BooleanField() -# st_jump = FloatField() -# bunting = CharField(null=True) -# hit_and_run = CharField(null=True) -# running = CharField() -# -# -# class PitcherData(BaseModelScout): -# player = ForeignKeyField(ScoutPlayer) -# cardset = ForeignKeyField(ScoutCardset) -# balk = IntegerField(null=True) -# wild_pitch = IntegerField(null=True) -# hold = CharField() -# starter_rating = IntegerField() -# relief_rating = IntegerField() -# closer_rating = IntegerField(null=True) -# batting = CharField(null=True) -# -# -# scout_db.create_tables([CardColumns, Position, BatterData, PitcherData], safe=True) -# -# -# class CardOutput(BaseModelScout): -# name = CharField() -# hand = CharField() -# positions = CharField() -# stealing = CharField() -# bunting = CharField() -# hitandrun = CharField() -# running = CharField() -# -# -# scout_db.close() diff --git a/app/db_helpers.py b/app/db_helpers.py index 9bf6577..12f5390 100644 --- a/app/db_helpers.py +++ b/app/db_helpers.py @@ -1,15 +1,8 @@ """ -Database helper functions for PostgreSQL compatibility. - -This module provides cross-database compatible upsert operations that work -with both SQLite and PostgreSQL. - -The key difference: -- SQLite: .on_conflict_replace() works directly -- PostgreSQL: Requires .on_conflict() with explicit conflict_target and update dict +Database helper functions for PostgreSQL upsert operations. Usage: - from app.db_helpers import upsert_many, DATABASE_TYPE + from app.db_helpers import upsert_many # Instead of: Model.insert_many(batch).on_conflict_replace().execute() @@ -18,13 +11,9 @@ Usage: upsert_many(Model, batch, conflict_fields=['field1', 'field2']) """ -import os -from typing import Any, Dict, List, Type, Union +from typing import Any, Dict, List, Type -from peewee import Model, SQL - -# Re-export DATABASE_TYPE for convenience -DATABASE_TYPE = os.environ.get("DATABASE_TYPE", "sqlite").lower() +from peewee import Model def get_model_fields(model: Type[Model], exclude: List[str] = None) -> List[str]: @@ -54,14 +43,12 @@ def upsert_many( batch_size: int = 100, ) -> int: """ - Insert or update multiple records in a database-agnostic way. - - Works with both SQLite (on_conflict_replace) and PostgreSQL (on_conflict). + Insert or update multiple records using PostgreSQL ON CONFLICT. Args: model: Peewee Model class data: List of dictionaries with field values - conflict_fields: Fields that define uniqueness (for PostgreSQL ON CONFLICT) + conflict_fields: Fields that define uniqueness (for ON CONFLICT target) update_fields: Fields to update on conflict (defaults to all non-conflict fields) batch_size: Number of records per batch @@ -93,33 +80,28 @@ def upsert_many( for i in range(0, len(data), batch_size): batch = data[i : i + batch_size] - if DATABASE_TYPE == "postgresql": - # PostgreSQL: Use ON CONFLICT with explicit target and update - from peewee import EXCLUDED + from peewee import EXCLUDED - # Build conflict target - get actual field objects - conflict_target = [getattr(model, f) for f in conflict_fields] + # Build conflict target - get actual field objects + conflict_target = [getattr(model, f) for f in conflict_fields] - # Build update dict - use column_name for EXCLUDED reference - # (ForeignKeyField column names end in _id, e.g., batter -> batter_id) - update_dict = {} - for f in update_fields: - if hasattr(model, f): - field_obj = getattr(model, f) - # Get the actual column name from the field - col_name = field_obj.column_name - update_dict[field_obj] = EXCLUDED[col_name] + # Build update dict - use column_name for EXCLUDED reference + # (ForeignKeyField column names end in _id, e.g., batter -> batter_id) + update_dict = {} + for f in update_fields: + if hasattr(model, f): + field_obj = getattr(model, f) + # Get the actual column name from the field + col_name = field_obj.column_name + update_dict[field_obj] = EXCLUDED[col_name] - if update_dict: - model.insert_many(batch).on_conflict( - conflict_target=conflict_target, action="update", update=update_dict - ).execute() - else: - # No fields to update, just ignore conflicts - model.insert_many(batch).on_conflict_ignore().execute() + if update_dict: + model.insert_many(batch).on_conflict( + conflict_target=conflict_target, action="update", update=update_dict + ).execute() else: - # SQLite: Use on_conflict_replace (simpler) - model.insert_many(batch).on_conflict_replace().execute() + # No fields to update, just ignore conflicts + model.insert_many(batch).on_conflict_ignore().execute() total += len(batch) diff --git a/app/routers_v2/players.py b/app/routers_v2/players.py index 8dabd83..47d1a9b 100644 --- a/app/routers_v2/players.py +++ b/app/routers_v2/players.py @@ -1336,7 +1336,7 @@ async def put_players(players: PlayerModel, token: str = Depends(oauth2_scheme)) logging.debug(f"new_players: {new_players}") with db.atomic(): - # Use PostgreSQL-compatible upsert helper (preserves SQLite compatibility) + # Use PostgreSQL upsert helper (ON CONFLICT DO UPDATE) upsert_players(new_players, batch_size=15) # sheets.update_all_players(SHEETS_AUTH) diff --git a/tests/conftest.py b/tests/conftest.py index 2e11617..2d1f1e5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,12 +1,9 @@ """ Shared test fixtures for the Paper Dynasty database test suite. -Uses in-memory SQLite with foreign_keys pragma enabled. Each test -gets a fresh set of tables via the setup_test_db fixture (autouse). - -All models are bound to the in-memory database before table creation -so that no connection to the real storage/pd_master.db occurs during -tests. +Provides dummy PostgreSQL credentials so PooledPostgresqlDatabase can be +instantiated during test collection without a real database connection. +Each test module is responsible for binding models to its own test database. """ import os @@ -14,14 +11,6 @@ import pytest import psycopg2 from peewee import SqliteDatabase -# Set DATABASE_TYPE=postgresql so that the module-level SKIP_TABLE_CREATION -# flag is True. This prevents db_engine.py from calling create_tables() -# against the real storage/pd_master.db during import — those calls would -# fail if indexes already exist and would also contaminate the dev database. -# The PooledPostgresqlDatabase object is created but never actually connects -# because our fixture rebinds all models to an in-memory SQLite db before -# any query is executed. -os.environ["DATABASE_TYPE"] = "postgresql" # Provide dummy credentials so PooledPostgresqlDatabase can be instantiated # without raising a configuration error (it will not actually be used). os.environ.setdefault("POSTGRES_PASSWORD", "test-dummy")