cal/paper-dynasty-database
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add PostgreSQL compatibility fixes for query ordering

Browse Source 
- Add explicit ORDER BY id to all queries for consistent results across SQLite and PostgreSQL
- PostgreSQL does not guarantee row order without ORDER BY, unlike SQLite
- Skip table creation when DATABASE_TYPE=postgresql (production tables already exist)
- Fix datetime handling in notifications (PostgreSQL native datetime vs SQLite timestamp)
- Fix grouped query count() calls that don't work in PostgreSQL
- Update .gitignore to include storage/templates/ directory

This completes the PostgreSQL migration compatibility layer while maintaining
backwards compatibility with SQLite for local development.

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Cal Corum 2026-02-03 10:39:14 -06:00
parent 0437eab92a
commit 40c512c665
29 changed files with 102 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -54,6 +54,7 @@ Include/
Lib/
Scripts/
storage/
!storage/templates/
pyenv.cfg
pyvenv.cfg
docker-compose.override.yml

92
app/db_engine.py
View File

@ -11,6 +11,8 @@ from playhouse.shortcuts import model_to_dict
# Database configuration - supports both SQLite and PostgreSQL
DATABASE_TYPE = os.environ.get("DATABASE_TYPE", "sqlite")
# Skip table creation for PostgreSQL (tables already exist in production)
SKIP_TABLE_CREATION = DATABASE_TYPE.lower() == "postgresql"
if DATABASE_TYPE.lower() == "postgresql":
from playhouse.pool import PooledPostgresqlDatabase
@ -178,17 +180,6 @@ class BaseModel(Model):
class Meta:
database = db
@classmethod
def select(cls, *fields):
"""Override select to add default ordering by id for PostgreSQL compatibility.
PostgreSQL does not guarantee row order without ORDER BY, unlike SQLite
which implicitly returned rows by rowid. This ensures consistent ordering
across all queries unless explicitly overridden with .order_by().
"""
query = super().select(*fields)
return query.order_by(cls.id)
class Current(BaseModel):
season = IntegerField()
@ -207,7 +198,8 @@ class Current(BaseModel):
return latest_current
db.create_tables([Current])
if not SKIP_TABLE_CREATION:
db.create_tables([Current], safe=True)
class Rarity(BaseModel):
@ -223,7 +215,8 @@ class Rarity(BaseModel):
return self.name
db.create_tables([Rarity])
if not SKIP_TABLE_CREATION:
db.create_tables([Rarity], safe=True)
class Event(BaseModel):
@ -239,7 +232,8 @@ class Event(BaseModel):
table_name = "event"
db.create_tables([Event])
if not SKIP_TABLE_CREATION:
db.create_tables([Event], safe=True)
class Cardset(BaseModel):
@ -259,7 +253,8 @@ class Cardset(BaseModel):
return self.name
db.create_tables([Cardset])
if not SKIP_TABLE_CREATION:
db.create_tables([Cardset], safe=True)
class MlbPlayer(BaseModel):
@ -276,7 +271,8 @@ class MlbPlayer(BaseModel):
table_name = "mlbplayer"
db.create_tables([MlbPlayer])
if not SKIP_TABLE_CREATION:
db.create_tables([MlbPlayer], safe=True)
class Player(BaseModel):
@ -377,7 +373,8 @@ class Player(BaseModel):
table_name = "player"
db.create_tables([Player])
if not SKIP_TABLE_CREATION:
db.create_tables([Player], safe=True)
class Team(BaseModel):
@ -435,7 +432,8 @@ class Team(BaseModel):
table_name = "team"
db.create_tables([Team])
if not SKIP_TABLE_CREATION:
db.create_tables([Team], safe=True)
class PackType(BaseModel):
@ -450,7 +448,8 @@ class PackType(BaseModel):
table_name = "packtype"
db.create_tables([PackType])
if not SKIP_TABLE_CREATION:
db.create_tables([PackType], safe=True)
class Pack(BaseModel):
@ -465,7 +464,8 @@ class Pack(BaseModel):
table_name = "pack"
db.create_tables([Pack])
if not SKIP_TABLE_CREATION:
db.create_tables([Pack], safe=True)
class Card(BaseModel):
@ -489,7 +489,8 @@ class Card(BaseModel):
table_name = "card"
db.create_tables([Card])
if not SKIP_TABLE_CREATION:
db.create_tables([Card], safe=True)
class Roster(BaseModel):
@ -738,21 +739,23 @@ class GauntletRun(BaseModel):
table_name = "gauntletrun"
db.create_tables(
[
Roster,
BattingStat,
PitchingStat,
Result,
Award,
Paperdex,
Reward,
GameRewards,
Notification,
GauntletReward,
GauntletRun,
]
)
if not SKIP_TABLE_CREATION:
db.create_tables(
[
Roster,
BattingStat,
PitchingStat,
Result,
Award,
Paperdex,
Reward,
GameRewards,
Notification,
GauntletReward,
GauntletRun,
],
safe=True,
)
class BattingCard(BaseModel):
@ -919,9 +922,11 @@ pos_index = ModelIndex(
CardPosition.add_index(pos_index)
db.create_tables(
[BattingCard, BattingCardRatings, PitchingCard, PitchingCardRatings, CardPosition]
)
if not SKIP_TABLE_CREATION:
db.create_tables(
[BattingCard, BattingCardRatings, PitchingCard, PitchingCardRatings, CardPosition],
safe=True,
)
class StratGame(BaseModel):
@ -1054,7 +1059,8 @@ decision_index = ModelIndex(Decision, (Decision.game, Decision.pitcher), unique=
Decision.add_index(decision_index)
db.create_tables([StratGame, StratPlay, Decision])
if not SKIP_TABLE_CREATION:
db.create_tables([StratGame, StratPlay, Decision], safe=True)
db.close()
@ -1088,7 +1094,7 @@ db.close()
# hand = CharField(default='R')
#
#
# scout_db.create_tables([ScoutCardset, ScoutPlayer])
# scout_db.create_tables([ScoutCardset, ScoutPlayer], safe=True)
#
#
# class BatterRatings(BaseModelScout):
@ -1161,7 +1167,7 @@ db.close()
# slg = FloatField(null=True)
#
#
# # scout_db.create_tables([BatterRatings, PitcherRatings])
# # scout_db.create_tables([BatterRatings, PitcherRatings], safe=True)
#
#
# class CardColumns(BaseModelScout):
@ -1218,7 +1224,7 @@ db.close()
# batting = CharField(null=True)
#
#
# scout_db.create_tables([CardColumns, Position, BatterData, PitcherData])
# scout_db.create_tables([CardColumns, Position, BatterData, PitcherData], safe=True)
#
#
# class CardOutput(BaseModelScout):

2
app/routers_v2/awards.py
View File

@ -38,7 +38,7 @@ async def get_awards(
name: Optional[str] = None, season: Optional[int] = None, timing: Optional[str] = None,
card_id: Optional[int] = None, team_id: Optional[int] = None, image: Optional[str] = None,
csv: Optional[bool] = None):
all_awards = Award.select()
all_awards = Award.select().order_by(Award.id)
if all_awards.count() == 0:
db.close()

2
app/routers_v2/batstats.py
View File

@ -72,7 +72,7 @@ class BatStatReturnList(pydantic.BaseModel):
async def get_batstats(
card_id: int = None, player_id: int = None, team_id: int = None, vs_team_id: int = None, week: int = None,
season: int = None, week_start: int = None, week_end: int = None, created: int = None, csv: bool = None):
all_stats = BattingStat.select().join(Card).join(Player)
all_stats = BattingStat.select().join(Card).join(Player).order_by(BattingStat.id)
if season is not None:
all_stats = all_stats.where(BattingStat.season == season)

6
app/routers_v2/battingcardratings.py
View File

@ -170,7 +170,7 @@ async def get_card_ratings(
# detail='You are not authorized to pull card ratings.'
# )
all_ratings = BattingCardRatings.select()
all_ratings = BattingCardRatings.select().order_by(BattingCardRatings.id)
if battingcard_id is not None:
all_ratings = all_ratings.where(
@ -212,7 +212,7 @@ async def get_card_ratings(
def get_scouting_dfs(cardset_id: list = None):
all_ratings = BattingCardRatings.select()
all_ratings = BattingCardRatings.select().order_by(BattingCardRatings.id)
if cardset_id is not None:
set_players = Player.select(Player.player_id).where(
Player.cardset_id << cardset_id
@ -688,7 +688,7 @@ async def get_player_ratings(
all_ratings = BattingCardRatings.select().where(
BattingCardRatings.battingcard << all_cards
)
).order_by(BattingCardRatings.id)
return_val = {
"count": all_ratings.count(),

2
app/routers_v2/battingcards.py
View File

@ -45,7 +45,7 @@ async def get_batting_cards(
limit: Optional[int] = None,
variant: list = Query(default=None),
):
all_cards = BattingCard.select()
all_cards = BattingCard.select().order_by(BattingCard.id)
if player_id is not None:
all_cards = all_cards.where(BattingCard.player_id << player_id)
if cardset_id is not None:

8
app/routers_v2/cardpositions.py
View File

@ -83,13 +83,13 @@ async def get_card_positions(
all_pos = all_pos.where(CardPosition.player << all_players)
if sort == "innings-desc":
all_pos = all_pos.order_by(CardPosition.innings.desc())
all_pos = all_pos.order_by(CardPosition.innings.desc(), CardPosition.id)
elif sort == "innings-asc":
all_pos = all_pos.order_by(CardPosition.innings)
all_pos = all_pos.order_by(CardPosition.innings, CardPosition.id)
elif sort == "range-desc":
all_pos = all_pos.order_by(CardPosition.range.desc())
all_pos = all_pos.order_by(CardPosition.range.desc(), CardPosition.id)
elif sort == "range-asc":
all_pos = all_pos.order_by(CardPosition.range)
all_pos = all_pos.order_by(CardPosition.range, CardPosition.id)
return_val = {
"count": all_pos.count(),

2
app/routers_v2/cards.py
View File

@ -75,6 +75,8 @@ async def get_cards(
if order_by is not None:
if order_by.lower() == 'new':
all_cards = all_cards.order_by(-Card.id)
else:
all_cards = all_cards.order_by(Card.id)
if limit is not None:
all_cards = all_cards.limit(limit)
if dupes:

4
app/routers_v2/cardsets.py
View File

@ -33,7 +33,7 @@ class CardsetModel(pydantic.BaseModel):
async def get_cardsets(
name: Optional[str] = None, in_desc: Optional[str] = None, event_id: Optional[int] = None,
in_packs: Optional[bool] = None, ranked_legal: Optional[bool] = None, csv: Optional[bool] = None):
all_cardsets = Cardset.select()
all_cardsets = Cardset.select().order_by(Cardset.id)
if all_cardsets.count() == 0:
db.close()
@ -97,7 +97,7 @@ async def search_cardsets(
Returns cardsets matching the query with exact matches prioritized over partial matches.
"""
# Start with all cardsets
all_cardsets = Cardset.select()
all_cardsets = Cardset.select().order_by(Cardset.id)
# Apply name filter (partial match)
all_cardsets = all_cardsets.where(fn.Lower(Cardset.name).contains(q.lower()))

3
app/routers_v2/decisions.py
View File

@ -179,8 +179,7 @@ async def get_decisions_for_rest(
.where((StratPlay.game == x.game) & (StratPlay.pitcher == x.pitcher))
.group_by(StratPlay.pitcher, StratPlay.game)
)
logging.info(f"this_line: {this_line[0]}")
if this_line[0].sum_outs is None:
if this_line.count() == 0 or this_line[0].sum_outs is None:
this_val.append(0.0)
else:
this_val.append(

2
app/routers_v2/events.py
View File

@ -32,7 +32,7 @@ class EventModel(pydantic.BaseModel):
async def v1_events_get(
name: Optional[str] = None, in_desc: Optional[str] = None, active: Optional[bool] = None,
csv: Optional[bool] = None):
all_events = Event.select()
all_events = Event.select().order_by(Event.id)
if name is not None:
all_events = all_events.where(fn.Lower(Event.name) == name.lower())

2
app/routers_v2/gamerewards.py
View File

@ -30,7 +30,7 @@ class GameRewardModel(pydantic.BaseModel):
async def v1_gamerewards_get(
name: Optional[str] = None, pack_type_id: Optional[int] = None, player_id: Optional[int] = None,
money: Optional[int] = None, csv: Optional[bool] = None):
all_rewards = GameRewards.select()
all_rewards = GameRewards.select().order_by(GameRewards.id)
# if all_rewards.count() == 0:
# db.close()

2
app/routers_v2/gauntletrewards.py
View File

@ -36,7 +36,7 @@ async def v1_gauntletreward_get(
win_num: Optional[int] = None,
loss_max: Optional[int] = None,
):
all_rewards = GauntletReward.select()
all_rewards = GauntletReward.select().order_by(GauntletReward.id)
if name is not None:
all_rewards = all_rewards.where(GauntletReward.name == name)

2
app/routers_v2/gauntletruns.py
View File

@ -36,7 +36,7 @@ async def get_gauntletruns(
losses_max: Optional[int] = None, gsheet: Optional[str] = None, created_after: Optional[int] = None,
created_before: Optional[int] = None, ended_after: Optional[int] = None, ended_before: Optional[int] = None,
is_active: Optional[bool] = None, gauntlet_id: list = Query(default=None), season: list = Query(default=None)):
all_gauntlets = GauntletRun.select()
all_gauntlets = GauntletRun.select().order_by(GauntletRun.id)
if team_id is not None:
all_gauntlets = all_gauntlets.where(GauntletRun.team_id << team_id)

2
app/routers_v2/mlbplayers.py
View File

@ -82,7 +82,7 @@ async def get_players(
offense_col: list = Query(default=None),
csv: Optional[bool] = False,
):
all_players = MlbPlayer.select()
all_players = MlbPlayer.select().order_by(MlbPlayer.id)
if full_name is not None:
name_list = [x.lower() for x in full_name]

2
app/routers_v2/notifications.py
View File

@ -35,7 +35,7 @@ async def get_notifs(
created_after: Optional[int] = None, title: Optional[str] = None, desc: Optional[str] = None,
field_name: Optional[str] = None, in_desc: Optional[str] = None, about: Optional[str] = None,
ack: Optional[bool] = None, csv: Optional[bool] = None):
all_notif = Notification.select()
all_notif = Notification.select().order_by(Notification.id)
if all_notif.count() == 0:
db.close()

8
app/routers_v2/packs.py
View File

@ -83,8 +83,10 @@ async def get_packs(
all_packs = all_packs.where(Pack.open_time.is_null(not opened))
if limit is not None:
all_packs = all_packs.limit(limit)
if new_to_old is not None:
if new_to_old:
all_packs = all_packs.order_by(-Pack.id)
else:
all_packs = all_packs.order_by(Pack.id)
# if all_packs.count() == 0:
# db.close()
@ -96,7 +98,7 @@ async def get_packs(
data_list.append(
[
line.id, line.team.abbrev, line.pack_type.name,
datetime.fromtimestamp(line.open_time) if line.open_time else None
line.open_time # Already datetime in PostgreSQL
]
)
return_val = DataFrame(data_list).to_csv(header=False, index=False)
@ -125,7 +127,7 @@ async def get_one_pack(pack_id, csv: Optional[bool] = False):
data_list = [
['id', 'team', 'pack_type', 'open_time'],
[this_pack.id, this_pack.team.abbrev, this_pack.pack_type.name,
datetime.fromtimestamp(this_pack.open_time) if this_pack.open_time else None]
this_pack.open_time] # Already datetime in PostgreSQL
]
return_val = DataFrame(data_list).to_csv(header=False, index=False)

2
app/routers_v2/packtypes.py
View File

@ -31,7 +31,7 @@ class PacktypeModel(pydantic.BaseModel):
async def get_packtypes(
name: Optional[str] = None, card_count: Optional[int] = None, in_desc: Optional[str] = None,
available: Optional[bool] = None, csv: Optional[bool] = None):
all_packtypes = PackType.select()
all_packtypes = PackType.select().order_by(PackType.id)
if all_packtypes.count() == 0:
db.close()

2
app/routers_v2/paperdex.py
View File

@ -31,7 +31,7 @@ async def get_paperdex(
team_id: Optional[int] = None, player_id: Optional[int] = None, created_after: Optional[int] = None,
cardset_id: Optional[int] = None, created_before: Optional[int] = None, flat: Optional[bool] = False,
csv: Optional[bool] = None):
all_dex = Paperdex.select().join(Player).join(Cardset)
all_dex = Paperdex.select().join(Player).join(Cardset).order_by(Paperdex.id)
if all_dex.count() == 0:
db.close()

4
app/routers_v2/pitchingcardratings.py
View File

@ -158,7 +158,7 @@ async def get_card_ratings(
status_code=401, detail="You are not authorized to pull card ratings."
)
all_ratings = PitchingCardRatings.select()
all_ratings = PitchingCardRatings.select().order_by(PitchingCardRatings.id)
if pitchingcard_id is not None:
all_ratings = all_ratings.where(
@ -192,7 +192,7 @@ async def get_card_ratings(
def get_scouting_dfs(cardset_id: list = None):
all_ratings = PitchingCardRatings.select()
all_ratings = PitchingCardRatings.select().order_by(PitchingCardRatings.id)
if cardset_id is not None:
set_players = Player.select(Player.player_id).where(
Player.cardset_id << cardset_id

2
app/routers_v2/pitchingcards.py
View File

@ -44,7 +44,7 @@ async def get_pitching_cards(
short_output: bool = False,
limit: Optional[int] = None,
):
all_cards = PitchingCard.select()
all_cards = PitchingCard.select().order_by(PitchingCard.id)
if player_id is not None:
all_cards = all_cards.where(PitchingCard.player_id << player_id)
if cardset_id is not None:

2
app/routers_v2/pitstats.py
View File

@ -58,7 +58,7 @@ async def get_pit_stats(
card_id: int = None, player_id: int = None, team_id: int = None, vs_team_id: int = None, week: int = None,
season: int = None, week_start: int = None, week_end: int = None, created: int = None, gs: bool = None,
csv: bool = None):
all_stats = PitchingStat.select().join(Card).join(Player)
all_stats = PitchingStat.select().join(Card).join(Player).order_by(PitchingStat.id)
logging.debug(f'pit query:\n\n{all_stats}')
if season is not None:

2
app/routers_v2/players.py
View File

@ -217,6 +217,8 @@ async def get_players(
all_players = all_players.order_by(Player.rarity)
elif sort_by == "rarity-asc":
all_players = all_players.order_by(-Player.rarity)
else:
all_players = all_players.order_by(Player.player_id)
final_players = []
# logging.info(f'pos_exclude: {type(pos_exclude)} - {pos_exclude} - is None: {pos_exclude is None}')

2
app/routers_v2/rarity.py
View File

@ -28,7 +28,7 @@ class RarityModel(pydantic.BaseModel):
@router.get('')
async def get_rarities(value: Optional[int] = None, name: Optional[str] = None, min_value: Optional[int] = None,
max_value: Optional[int] = None, csv: Optional[bool] = None):
all_rarities = Rarity.select()
all_rarities = Rarity.select().order_by(Rarity.id)
if all_rarities.count() == 0:
db.close()

2
app/routers_v2/results.py
View File

@ -196,7 +196,7 @@ async def get_one_results(result_id, csv: Optional[bool] = None):
@router.get('/team/{team_id}')
async def get_team_results(
team_id: int, season: Optional[int] = None, week: Optional[int] = None, csv: Optional[bool] = False):
all_results = Result.select().where((Result.away_team_id == team_id) | (Result.home_team_id == team_id))
all_results = Result.select().where((Result.away_team_id == team_id) | (Result.home_team_id == team_id)).order_by(Result.id)
try:
this_team = Team.get_by_id(team_id)
except Exception as e:

2
app/routers_v2/rewards.py
View File

@ -33,7 +33,7 @@ async def get_rewards(
name: Optional[str] = None, in_name: Optional[str] = None, team_id: Optional[int] = None,
season: Optional[int] = None, week: Optional[int] = None, created_after: Optional[int] = None,
flat: Optional[bool] = False, csv: Optional[bool] = None):
all_rewards = Reward.select()
all_rewards = Reward.select().order_by(Reward.id)
if all_rewards.count() == 0:
db.close()

2
app/routers_v2/stratgame.py
View File

@ -48,7 +48,7 @@ async def get_games(
team2_id: list = Query(default=None), game_type: list = Query(default=None), ranked: Optional[bool] = None,
short_game: Optional[bool] = None, csv: Optional[bool] = False, short_output: bool = False,
gauntlet_id: Optional[int] = None):
all_games = StratGame.select()
all_games = StratGame.select().order_by(StratGame.id)
if season is not None:
all_games = all_games.where(StratGame.season << season)

14
app/routers_v2/stratplays.py
View File

@ -177,7 +177,7 @@ async def get_plays(
limit: Optional[int] = 200,
page_num: Optional[int] = 1,
):
all_plays = StratPlay.select()
all_plays = StratPlay.select().order_by(StratPlay.id)
if season is not None:
s_games = StratGame.select().where(StratGame.season << season)
@ -648,9 +648,11 @@ async def get_batting_totals(
logging.debug(f"bat_plays query: {bat_plays}")
logging.debug(f"run_plays query: {run_plays}")
return_stats = {"count": bat_plays.count(), "stats": []}
# Convert to list first - .count() doesn't work on grouped queries in PostgreSQL
bat_plays_list = list(bat_plays)
return_stats = {"count": len(bat_plays_list), "stats": []}
for x in bat_plays:
for x in bat_plays_list:
# NOTE: Removed .order_by(StratPlay.id) - not valid with GROUP BY in PostgreSQL
# and not meaningful for aggregated results anyway
this_run = run_plays
@ -1066,9 +1068,11 @@ async def get_pitching_totals(
limit = 500
pit_plays = pit_plays.paginate(page_num, limit)
return_stats = {"count": pit_plays.count(), "stats": []}
# Convert to list first - .count() doesn't work on grouped queries in PostgreSQL
pit_plays_list = list(pit_plays)
return_stats = {"count": len(pit_plays_list), "stats": []}
for x in pit_plays:
for x in pit_plays_list:
this_dec = all_dec.where(Decision.pitcher == x.pitcher)
if game_type is not None:
all_types = [x.lower() for x in game_type]

7
app/routers_v2/teams.py
View File

@ -155,6 +155,9 @@ async def get_teams(
if event_id is not None:
all_teams = all_teams.where(Team.event_id == event_id)
# Default ordering for PostgreSQL compatibility
all_teams = all_teams.order_by(Team.id)
if limit is not None:
all_teams = all_teams.limit(limit)
@ -1098,7 +1101,7 @@ async def team_buy_players(team_id: int, ids: str, ts: str):
# Post a notification
if this_player.rarity.value >= 2:
new_notif = Notification(
created=int_timestamp(datetime.now()),
created=datetime.now(),
title=f"Price Change",
desc="Modified by buying and selling",
field_name=f"{this_player.description} "
@ -1250,7 +1253,7 @@ async def team_sell_cards(team_id: int, ids: str, ts: str):
# post a notification
if this_player.rarity.value >= 2:
new_notif = Notification(
created=int_timestamp(datetime.now()),
created=datetime.now(),
title=f"Price Change",
desc="Modified by buying and selling",
field_name=f"{this_player.description} "