From ec04a111c5b8a5041a26401272bc39ce6b284af4 Mon Sep 17 00:00:00 2001 From: Cal Corum Date: Sun, 22 Mar 2026 23:37:02 -0500 Subject: [PATCH] fix: remove legacy SQLite compatibility code (#122) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #122 Both prod and dev environments use PostgreSQL. Removes all SQLite compatibility code that was never exercised in practice. Changes: - db_engine.py: replace SQLite/PostgreSQL branching with direct PooledPostgresqlDatabase init; remove DATABASE_TYPE, SKIP_TABLE_CREATION, all db.create_tables() calls, and commented-out SQLite scout_db code - db_helpers.py: remove DATABASE_TYPE var and SQLite on_conflict_replace branch from upsert_many(); PostgreSQL ON CONFLICT is now the only path - players.py: update stale comment - tests/conftest.py: remove DATABASE_TYPE env var (no longer needed); keep POSTGRES_PASSWORD dummy for instantiation - CLAUDE.md: update SQLite references to PostgreSQL Note: unit tests in test_evolution_seed.py and test_season_stats_model.py use SqliteDatabase(':memory:') for test isolation — this is legitimate test infrastructure, not production SQLite compatibility code. Co-Authored-By: Claude Sonnet 4.6 --- CLAUDE.md | 7 +- app/db_engine.py | 292 ++------------------------------------ app/db_helpers.py | 66 ++++----- app/routers_v2/players.py | 2 +- tests/conftest.py | 8 +- 5 files changed, 44 insertions(+), 331 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 6efe82a..aebe4af 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,6 +1,6 @@ # Paper Dynasty Database API -FastAPI backend for baseball card game data. Peewee ORM with SQLite (WAL mode). +FastAPI backend for baseball card game data. Peewee ORM with PostgreSQL. ## Commands @@ -14,7 +14,7 @@ docker build -t paper-dynasty-db . # Build image ## Architecture - **Routers**: Domain-based in `app/routers_v2/` (cards, players, teams, packs, stats, gauntlets, scouting) -- **ORM**: Peewee with SQLite (`storage/pd_master.db`, WAL journaling) +- **ORM**: Peewee with PostgreSQL (`pd_master` database via PooledPostgresqlDatabase) - **Card images**: Playwright/Chromium renders HTML templates → screenshots (see `routers_v2/players.py`) - **Logging**: Rotating files in `logs/database/{date}.log` @@ -42,12 +42,11 @@ docker build -t paper-dynasty-db . # Build image - **API docs**: `/api/docs` and `/api/redoc` ### Key Env Vars -`API_TOKEN`, `LOG_LEVEL`, `DATABASE_TYPE` (sqlite/postgresql), `POSTGRES_HOST`, `POSTGRES_DB`, `POSTGRES_USER`, `POSTGRES_PASSWORD` +`API_TOKEN`, `LOG_LEVEL`, `POSTGRES_HOST`, `POSTGRES_DB`, `POSTGRES_USER`, `POSTGRES_PASSWORD` ### Common Issues - 502 Bad Gateway → API container crashed; check `docker logs pd_api` - Card image generation failures → Playwright/Chromium issue; check for missing dependencies -- SQLite locking (dev) → WAL mode should prevent, but check for long-running writes - DB connection errors → verify `POSTGRES_HOST` points to correct container name - **CI/CD**: Gitea Actions on PR to `main` — builds Docker image, auto-generates CalVer version (`YYYY.MM.BUILD`) on merge diff --git a/app/db_engine.py b/app/db_engine.py index 4183bb9..088ba3d 100644 --- a/app/db_engine.py +++ b/app/db_engine.py @@ -9,40 +9,20 @@ from peewee import * from peewee import ModelSelect from playhouse.shortcuts import model_to_dict -# Database configuration - supports both SQLite and PostgreSQL -DATABASE_TYPE = os.environ.get("DATABASE_TYPE", "sqlite") -# Skip table creation for PostgreSQL (tables already exist in production) -SKIP_TABLE_CREATION = DATABASE_TYPE.lower() == "postgresql" +from playhouse.pool import PooledPostgresqlDatabase -if DATABASE_TYPE.lower() == "postgresql": - from playhouse.pool import PooledPostgresqlDatabase - - db = PooledPostgresqlDatabase( - os.environ.get("POSTGRES_DB", "pd_master"), - user=os.environ.get("POSTGRES_USER", "pd_admin"), - password=os.environ.get("POSTGRES_PASSWORD"), - host=os.environ.get("POSTGRES_HOST", "localhost"), - port=int(os.environ.get("POSTGRES_PORT", "5432")), - max_connections=20, - stale_timeout=300, # 5 minutes - timeout=0, - autoconnect=True, - autorollback=True, # Automatically rollback failed transactions - ) -else: - # SQLite configuration for local development only. - # Production always uses PostgreSQL (see DATABASE_TYPE env var). - # - # synchronous=0 (OFF): SQLite skips fsync() after every write, maximising - # throughput at the cost of durability — a hard crash could corrupt the DB. - # This is an acceptable trade-off in dev where data loss is tolerable and - # write speed matters. WAL journal mode reduces (but does not eliminate) - # the corruption window by keeping the main database file consistent while - # writes land in the WAL file first. - db = SqliteDatabase( - "storage/pd_master.db", - pragmas={"journal_mode": "wal", "cache_size": -1 * 64000, "synchronous": 0}, - ) +db = PooledPostgresqlDatabase( + os.environ.get("POSTGRES_DB", "pd_master"), + user=os.environ.get("POSTGRES_USER", "pd_admin"), + password=os.environ.get("POSTGRES_PASSWORD"), + host=os.environ.get("POSTGRES_HOST", "localhost"), + port=int(os.environ.get("POSTGRES_PORT", "5432")), + max_connections=20, + stale_timeout=300, # 5 minutes + timeout=0, + autoconnect=True, + autorollback=True, # Automatically rollback failed transactions +) # 2025, 2005 ranked_cardsets = [24, 25, 26, 27, 28, 29] @@ -199,10 +179,6 @@ class Current(BaseModel): return latest_current -if not SKIP_TABLE_CREATION: - db.create_tables([Current], safe=True) - - class Rarity(BaseModel): value = IntegerField() name = CharField(unique=True) @@ -216,9 +192,6 @@ class Rarity(BaseModel): return self.name -if not SKIP_TABLE_CREATION: - db.create_tables([Rarity], safe=True) - class Event(BaseModel): name = CharField() @@ -233,9 +206,6 @@ class Event(BaseModel): table_name = "event" -if not SKIP_TABLE_CREATION: - db.create_tables([Event], safe=True) - class Cardset(BaseModel): name = CharField() @@ -254,9 +224,6 @@ class Cardset(BaseModel): return self.name -if not SKIP_TABLE_CREATION: - db.create_tables([Cardset], safe=True) - class MlbPlayer(BaseModel): first_name = CharField() @@ -272,9 +239,6 @@ class MlbPlayer(BaseModel): table_name = "mlbplayer" -if not SKIP_TABLE_CREATION: - db.create_tables([MlbPlayer], safe=True) - class Player(BaseModel): player_id = IntegerField(primary_key=True) @@ -374,9 +338,6 @@ class Player(BaseModel): table_name = "player" -if not SKIP_TABLE_CREATION: - db.create_tables([Player], safe=True) - class Team(BaseModel): abbrev = CharField(max_length=20) # Gauntlet teams use prefixes like "Gauntlet-NCB" @@ -433,9 +394,6 @@ class Team(BaseModel): table_name = "team" -if not SKIP_TABLE_CREATION: - db.create_tables([Team], safe=True) - class PackType(BaseModel): name = CharField() @@ -449,9 +407,6 @@ class PackType(BaseModel): table_name = "packtype" -if not SKIP_TABLE_CREATION: - db.create_tables([PackType], safe=True) - class Pack(BaseModel): team = ForeignKeyField(Team) @@ -465,9 +420,6 @@ class Pack(BaseModel): table_name = "pack" -if not SKIP_TABLE_CREATION: - db.create_tables([Pack], safe=True) - class Card(BaseModel): player = ForeignKeyField(Player, null=True) @@ -490,9 +442,6 @@ class Card(BaseModel): table_name = "card" -if not SKIP_TABLE_CREATION: - db.create_tables([Card], safe=True) - class Roster(BaseModel): team = ForeignKeyField(Team) @@ -723,25 +672,6 @@ class GauntletRun(BaseModel): table_name = "gauntletrun" -if not SKIP_TABLE_CREATION: - db.create_tables( - [ - Roster, - RosterSlot, - BattingStat, - PitchingStat, - Result, - Award, - Paperdex, - Reward, - GameRewards, - Notification, - GauntletReward, - GauntletRun, - ], - safe=True, - ) - class BattingCard(BaseModel): player = ForeignKeyField(Player) @@ -907,18 +837,6 @@ pos_index = ModelIndex( CardPosition.add_index(pos_index) -if not SKIP_TABLE_CREATION: - db.create_tables( - [ - BattingCard, - BattingCardRatings, - PitchingCard, - PitchingCardRatings, - CardPosition, - ], - safe=True, - ) - class StratGame(BaseModel): season = IntegerField() @@ -1152,12 +1070,6 @@ pitss_player_season_index = ModelIndex( PitchingSeasonStats.add_index(pitss_player_season_index) -if not SKIP_TABLE_CREATION: - db.create_tables( - [StratGame, StratPlay, Decision, BattingSeasonStats, PitchingSeasonStats], - safe=True, - ) - class ScoutOpportunity(BaseModel): pack = ForeignKeyField(Pack, null=True) @@ -1190,182 +1102,4 @@ scout_claim_index = ModelIndex( ScoutClaim.add_index(scout_claim_index) -if not SKIP_TABLE_CREATION: - db.create_tables([ScoutOpportunity, ScoutClaim], safe=True) - - db.close() - -# scout_db = SqliteDatabase( -# 'storage/card_creation.db', -# pragmas={ -# 'journal_mode': 'wal', -# 'cache_size': -1 * 64000, -# 'synchronous': 0 -# } -# ) -# -# -# class BaseModelScout(Model): -# class Meta: -# database = scout_db -# -# -# class ScoutCardset(BaseModelScout): -# set_title = CharField() -# set_subtitle = CharField(null=True) -# -# -# class ScoutPlayer(BaseModelScout): -# sba_id = IntegerField(primary_key=True) -# name = CharField() -# fg_id = IntegerField() -# br_id = CharField() -# offense_col = IntegerField() -# hand = CharField(default='R') -# -# -# scout_db.create_tables([ScoutCardset, ScoutPlayer], safe=True) -# -# -# class BatterRatings(BaseModelScout): -# id = CharField(unique=True, primary_key=True) -# player = ForeignKeyField(ScoutPlayer) -# cardset = ForeignKeyField(ScoutCardset) -# vs_hand = FloatField() -# is_prep = BooleanField() -# homerun = FloatField() -# bp_homerun = FloatField() -# triple = FloatField() -# double_three = FloatField() -# double_two = FloatField() -# double_pull = FloatField() -# single_two = FloatField() -# single_one = FloatField() -# single_center = FloatField() -# bp_single = FloatField() -# hbp = FloatField() -# walk = FloatField() -# strikeout = FloatField() -# lineout = FloatField() -# popout = FloatField() -# flyout_a = FloatField() -# flyout_bq = FloatField() -# flyout_lf_b = FloatField() -# flyout_rf_b = FloatField() -# groundout_a = FloatField() -# groundout_b = FloatField() -# groundout_c = FloatField() -# avg = FloatField(null=True) -# obp = FloatField(null=True) -# slg = FloatField(null=True) -# -# -# class PitcherRatings(BaseModelScout): -# id = CharField(unique=True, primary_key=True) -# player = ForeignKeyField(ScoutPlayer) -# cardset = ForeignKeyField(ScoutCardset) -# vs_hand = CharField() -# is_prep = BooleanField() -# homerun = FloatField() -# bp_homerun = FloatField() -# triple = FloatField() -# double_three = FloatField() -# double_two = FloatField() -# double_cf = FloatField() -# single_two = FloatField() -# single_one = FloatField() -# single_center = FloatField() -# bp_single = FloatField() -# hbp = FloatField() -# walk = FloatField() -# strikeout = FloatField() -# fo_slap = FloatField() -# fo_center = FloatField() -# groundout_a = FloatField() -# groundout_b = FloatField() -# xcheck_p = FloatField() -# xcheck_c = FloatField() -# xcheck_1b = FloatField() -# xcheck_2b = FloatField() -# xcheck_3b = FloatField() -# xcheck_ss = FloatField() -# xcheck_lf = FloatField() -# xcheck_cf = FloatField() -# xcheck_rf = FloatField() -# avg = FloatField(null=True) -# obp = FloatField(null=True) -# slg = FloatField(null=True) -# -# -# # scout_db.create_tables([BatterRatings, PitcherRatings], safe=True) -# -# -# class CardColumns(BaseModelScout): -# id = CharField(unique=True, primary_key=True) -# player = ForeignKeyField(ScoutPlayer) -# hand = CharField() -# b_ratings = ForeignKeyField(BatterRatings, null=True) -# p_ratings = ForeignKeyField(PitcherRatings, null=True) -# one_dice = CharField() -# one_results = CharField() -# one_splits = CharField() -# two_dice = CharField() -# two_results = CharField() -# two_splits = CharField() -# three_dice = CharField() -# three_results = CharField() -# three_splits = CharField() -# -# -# class Position(BaseModelScout): -# player = ForeignKeyField(ScoutPlayer) -# cardset = ForeignKeyField(ScoutCardset) -# position = CharField() -# innings = IntegerField() -# range = IntegerField() -# error = IntegerField() -# arm = CharField(null=True) -# pb = IntegerField(null=True) -# overthrow = IntegerField(null=True) -# -# -# class BatterData(BaseModelScout): -# player = ForeignKeyField(ScoutPlayer) -# cardset = ForeignKeyField(ScoutCardset) -# stealing = CharField() -# st_low = IntegerField() -# st_high = IntegerField() -# st_auto = BooleanField() -# st_jump = FloatField() -# bunting = CharField(null=True) -# hit_and_run = CharField(null=True) -# running = CharField() -# -# -# class PitcherData(BaseModelScout): -# player = ForeignKeyField(ScoutPlayer) -# cardset = ForeignKeyField(ScoutCardset) -# balk = IntegerField(null=True) -# wild_pitch = IntegerField(null=True) -# hold = CharField() -# starter_rating = IntegerField() -# relief_rating = IntegerField() -# closer_rating = IntegerField(null=True) -# batting = CharField(null=True) -# -# -# scout_db.create_tables([CardColumns, Position, BatterData, PitcherData], safe=True) -# -# -# class CardOutput(BaseModelScout): -# name = CharField() -# hand = CharField() -# positions = CharField() -# stealing = CharField() -# bunting = CharField() -# hitandrun = CharField() -# running = CharField() -# -# -# scout_db.close() diff --git a/app/db_helpers.py b/app/db_helpers.py index 9bf6577..12f5390 100644 --- a/app/db_helpers.py +++ b/app/db_helpers.py @@ -1,15 +1,8 @@ """ -Database helper functions for PostgreSQL compatibility. - -This module provides cross-database compatible upsert operations that work -with both SQLite and PostgreSQL. - -The key difference: -- SQLite: .on_conflict_replace() works directly -- PostgreSQL: Requires .on_conflict() with explicit conflict_target and update dict +Database helper functions for PostgreSQL upsert operations. Usage: - from app.db_helpers import upsert_many, DATABASE_TYPE + from app.db_helpers import upsert_many # Instead of: Model.insert_many(batch).on_conflict_replace().execute() @@ -18,13 +11,9 @@ Usage: upsert_many(Model, batch, conflict_fields=['field1', 'field2']) """ -import os -from typing import Any, Dict, List, Type, Union +from typing import Any, Dict, List, Type -from peewee import Model, SQL - -# Re-export DATABASE_TYPE for convenience -DATABASE_TYPE = os.environ.get("DATABASE_TYPE", "sqlite").lower() +from peewee import Model def get_model_fields(model: Type[Model], exclude: List[str] = None) -> List[str]: @@ -54,14 +43,12 @@ def upsert_many( batch_size: int = 100, ) -> int: """ - Insert or update multiple records in a database-agnostic way. - - Works with both SQLite (on_conflict_replace) and PostgreSQL (on_conflict). + Insert or update multiple records using PostgreSQL ON CONFLICT. Args: model: Peewee Model class data: List of dictionaries with field values - conflict_fields: Fields that define uniqueness (for PostgreSQL ON CONFLICT) + conflict_fields: Fields that define uniqueness (for ON CONFLICT target) update_fields: Fields to update on conflict (defaults to all non-conflict fields) batch_size: Number of records per batch @@ -93,33 +80,28 @@ def upsert_many( for i in range(0, len(data), batch_size): batch = data[i : i + batch_size] - if DATABASE_TYPE == "postgresql": - # PostgreSQL: Use ON CONFLICT with explicit target and update - from peewee import EXCLUDED + from peewee import EXCLUDED - # Build conflict target - get actual field objects - conflict_target = [getattr(model, f) for f in conflict_fields] + # Build conflict target - get actual field objects + conflict_target = [getattr(model, f) for f in conflict_fields] - # Build update dict - use column_name for EXCLUDED reference - # (ForeignKeyField column names end in _id, e.g., batter -> batter_id) - update_dict = {} - for f in update_fields: - if hasattr(model, f): - field_obj = getattr(model, f) - # Get the actual column name from the field - col_name = field_obj.column_name - update_dict[field_obj] = EXCLUDED[col_name] + # Build update dict - use column_name for EXCLUDED reference + # (ForeignKeyField column names end in _id, e.g., batter -> batter_id) + update_dict = {} + for f in update_fields: + if hasattr(model, f): + field_obj = getattr(model, f) + # Get the actual column name from the field + col_name = field_obj.column_name + update_dict[field_obj] = EXCLUDED[col_name] - if update_dict: - model.insert_many(batch).on_conflict( - conflict_target=conflict_target, action="update", update=update_dict - ).execute() - else: - # No fields to update, just ignore conflicts - model.insert_many(batch).on_conflict_ignore().execute() + if update_dict: + model.insert_many(batch).on_conflict( + conflict_target=conflict_target, action="update", update=update_dict + ).execute() else: - # SQLite: Use on_conflict_replace (simpler) - model.insert_many(batch).on_conflict_replace().execute() + # No fields to update, just ignore conflicts + model.insert_many(batch).on_conflict_ignore().execute() total += len(batch) diff --git a/app/routers_v2/players.py b/app/routers_v2/players.py index 7ebd1b5..21bb030 100644 --- a/app/routers_v2/players.py +++ b/app/routers_v2/players.py @@ -1098,7 +1098,7 @@ async def put_players(players: PlayerModel, token: str = Depends(oauth2_scheme)) logging.debug(f"new_players: {new_players}") with db.atomic(): - # Use PostgreSQL-compatible upsert helper (preserves SQLite compatibility) + # Use PostgreSQL upsert helper (ON CONFLICT DO UPDATE) upsert_players(new_players, batch_size=15) # sheets.update_all_players(SHEETS_AUTH) diff --git a/tests/conftest.py b/tests/conftest.py index 8d61378..cc65e4a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,14 +1,12 @@ """Pytest configuration for the paper-dynasty-database test suite. -Sets DATABASE_TYPE=postgresql before any app module is imported so that -db_engine.py sets SKIP_TABLE_CREATION=True and does not try to mutate the -production SQLite file during test collection. Each test module is -responsible for binding models to its own in-memory database. +Provides dummy PostgreSQL credentials so PooledPostgresqlDatabase can be +instantiated during test collection without a real database connection. +Each test module is responsible for binding models to its own test database. """ import os -os.environ["DATABASE_TYPE"] = "postgresql" # Provide dummy credentials so PooledPostgresqlDatabase can be instantiated # without raising a configuration error (it will not actually be used). os.environ.setdefault("POSTGRES_PASSWORD", "test-dummy")