Add PostgreSQL compatibility fixes for query ordering

- Add explicit ORDER BY id to all queries for consistent results across SQLite and PostgreSQL
- PostgreSQL does not guarantee row order without ORDER BY, unlike SQLite
- Skip table creation when DATABASE_TYPE=postgresql (production tables already exist)
- Fix datetime handling in notifications (PostgreSQL native datetime vs SQLite timestamp)
- Fix grouped query count() calls that don't work in PostgreSQL
- Update .gitignore to include storage/templates/ directory

This completes the PostgreSQL migration compatibility layer while maintaining
backwards compatibility with SQLite for local development.

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

.gitignore

@@ -54,6 +54,7 @@ Include/
 Lib/
 Scripts/
 storage/
+!storage/templates/
 pyenv.cfg
 pyvenv.cfg
 docker-compose.override.yml

app/db_engine.py

@@ -11,6 +11,8 @@ from playhouse.shortcuts import model_to_dict
 
 # Database configuration - supports both SQLite and PostgreSQL
 DATABASE_TYPE = os.environ.get("DATABASE_TYPE", "sqlite")
+# Skip table creation for PostgreSQL (tables already exist in production)
+SKIP_TABLE_CREATION = DATABASE_TYPE.lower() == "postgresql"
 
 if DATABASE_TYPE.lower() == "postgresql":
     from playhouse.pool import PooledPostgresqlDatabase
@@ -178,17 +180,6 @@ class BaseModel(Model):
     class Meta:
         database = db
 
-    @classmethod
-    def select(cls, *fields):
-        """Override select to add default ordering by id for PostgreSQL compatibility.
-
-        PostgreSQL does not guarantee row order without ORDER BY, unlike SQLite
-        which implicitly returned rows by rowid. This ensures consistent ordering
-        across all queries unless explicitly overridden with .order_by().
-        """
-        query = super().select(*fields)
-        return query.order_by(cls.id)
-
 
 class Current(BaseModel):
     season = IntegerField()
@@ -207,7 +198,8 @@ class Current(BaseModel):
         return latest_current
 
 
-db.create_tables([Current])
+if not SKIP_TABLE_CREATION:
+    db.create_tables([Current], safe=True)
 
 
 class Rarity(BaseModel):
@@ -223,7 +215,8 @@ class Rarity(BaseModel):
         return self.name
 
 
-db.create_tables([Rarity])
+if not SKIP_TABLE_CREATION:
+    db.create_tables([Rarity], safe=True)
 
 
 class Event(BaseModel):
@@ -239,7 +232,8 @@ class Event(BaseModel):
         table_name = "event"
 
 
-db.create_tables([Event])
+if not SKIP_TABLE_CREATION:
+    db.create_tables([Event], safe=True)
 
 
 class Cardset(BaseModel):
@@ -259,7 +253,8 @@ class Cardset(BaseModel):
         return self.name
 
 
-db.create_tables([Cardset])
+if not SKIP_TABLE_CREATION:
+    db.create_tables([Cardset], safe=True)
 
 
 class MlbPlayer(BaseModel):
@@ -276,7 +271,8 @@ class MlbPlayer(BaseModel):
         table_name = "mlbplayer"
 
 
-db.create_tables([MlbPlayer])
+if not SKIP_TABLE_CREATION:
+    db.create_tables([MlbPlayer], safe=True)
 
 
 class Player(BaseModel):
@@ -377,7 +373,8 @@ class Player(BaseModel):
         table_name = "player"
 
 
-db.create_tables([Player])
+if not SKIP_TABLE_CREATION:
+    db.create_tables([Player], safe=True)
 
 
 class Team(BaseModel):
@@ -435,7 +432,8 @@ class Team(BaseModel):
         table_name = "team"
 
 
-db.create_tables([Team])
+if not SKIP_TABLE_CREATION:
+    db.create_tables([Team], safe=True)
 
 
 class PackType(BaseModel):
@@ -450,7 +448,8 @@ class PackType(BaseModel):
         table_name = "packtype"
 
 
-db.create_tables([PackType])
+if not SKIP_TABLE_CREATION:
+    db.create_tables([PackType], safe=True)
 
 
 class Pack(BaseModel):
@@ -465,7 +464,8 @@ class Pack(BaseModel):
         table_name = "pack"
 
 
-db.create_tables([Pack])
+if not SKIP_TABLE_CREATION:
+    db.create_tables([Pack], safe=True)
 
 
 class Card(BaseModel):
@@ -489,7 +489,8 @@ class Card(BaseModel):
         table_name = "card"
 
 
-db.create_tables([Card])
+if not SKIP_TABLE_CREATION:
+    db.create_tables([Card], safe=True)
 
 
 class Roster(BaseModel):
@@ -738,7 +739,8 @@ class GauntletRun(BaseModel):
         table_name = "gauntletrun"
 
 
-db.create_tables(
+if not SKIP_TABLE_CREATION:
+    db.create_tables(
         [
             Roster,
             BattingStat,
@@ -751,8 +753,9 @@ db.create_tables(
             Notification,
             GauntletReward,
             GauntletRun,
-    ]
+        ],
-)
+        safe=True,
+    )
 
 
 class BattingCard(BaseModel):
@@ -919,9 +922,11 @@ pos_index = ModelIndex(
 CardPosition.add_index(pos_index)
 
 
-db.create_tables(
+if not SKIP_TABLE_CREATION:
-    [BattingCard, BattingCardRatings, PitchingCard, PitchingCardRatings, CardPosition]
+    db.create_tables(
-)
+        [BattingCard, BattingCardRatings, PitchingCard, PitchingCardRatings, CardPosition],
+        safe=True,
+    )
 
 
 class StratGame(BaseModel):
@@ -1054,7 +1059,8 @@ decision_index = ModelIndex(Decision, (Decision.game, Decision.pitcher), unique=
 Decision.add_index(decision_index)
 
 
-db.create_tables([StratGame, StratPlay, Decision])
+if not SKIP_TABLE_CREATION:
+    db.create_tables([StratGame, StratPlay, Decision], safe=True)
 
 
 db.close()
@@ -1088,7 +1094,7 @@ db.close()
 #     hand = CharField(default='R')
 #
 #
-# scout_db.create_tables([ScoutCardset, ScoutPlayer])
+# scout_db.create_tables([ScoutCardset, ScoutPlayer], safe=True)
 #
 #
 # class BatterRatings(BaseModelScout):
@@ -1161,7 +1167,7 @@ db.close()
 #     slg = FloatField(null=True)
 #
 #
-# # scout_db.create_tables([BatterRatings, PitcherRatings])
+# # scout_db.create_tables([BatterRatings, PitcherRatings], safe=True)
 #
 #
 # class CardColumns(BaseModelScout):
@@ -1218,7 +1224,7 @@ db.close()
 #     batting = CharField(null=True)
 #
 #
-# scout_db.create_tables([CardColumns, Position, BatterData, PitcherData])
+# scout_db.create_tables([CardColumns, Position, BatterData, PitcherData], safe=True)
 #
 #
 # class CardOutput(BaseModelScout):

app/routers_v2/awards.py

@@ -38,7 +38,7 @@ async def get_awards(
         name: Optional[str] = None, season: Optional[int] = None, timing: Optional[str] = None,
         card_id: Optional[int] = None, team_id: Optional[int] = None, image: Optional[str] = None,
         csv: Optional[bool] = None):
-    all_awards = Award.select()
+    all_awards = Award.select().order_by(Award.id)
 
     if all_awards.count() == 0:
         db.close()

app/routers_v2/batstats.py

@@ -72,7 +72,7 @@ class BatStatReturnList(pydantic.BaseModel):
 async def get_batstats(
         card_id: int = None, player_id: int = None, team_id: int = None, vs_team_id: int = None, week: int = None,
         season: int = None, week_start: int = None, week_end: int = None, created: int = None, csv: bool = None):
-    all_stats = BattingStat.select().join(Card).join(Player)
+    all_stats = BattingStat.select().join(Card).join(Player).order_by(BattingStat.id)
 
     if season is not None:
         all_stats = all_stats.where(BattingStat.season == season)

app/routers_v2/battingcardratings.py

@@ -170,7 +170,7 @@ async def get_card_ratings(
     #         detail='You are not authorized to pull card ratings.'
     #     )
 
-    all_ratings = BattingCardRatings.select()
+    all_ratings = BattingCardRatings.select().order_by(BattingCardRatings.id)
 
     if battingcard_id is not None:
         all_ratings = all_ratings.where(
@@ -212,7 +212,7 @@ async def get_card_ratings(
 
 
 def get_scouting_dfs(cardset_id: list = None):
-    all_ratings = BattingCardRatings.select()
+    all_ratings = BattingCardRatings.select().order_by(BattingCardRatings.id)
     if cardset_id is not None:
         set_players = Player.select(Player.player_id).where(
             Player.cardset_id << cardset_id
@@ -688,7 +688,7 @@ async def get_player_ratings(
 
     all_ratings = BattingCardRatings.select().where(
         BattingCardRatings.battingcard << all_cards
-    )
+    ).order_by(BattingCardRatings.id)
 
     return_val = {
         "count": all_ratings.count(),

app/routers_v2/battingcards.py

@@ -45,7 +45,7 @@ async def get_batting_cards(
     limit: Optional[int] = None,
     variant: list = Query(default=None),
 ):
-    all_cards = BattingCard.select()
+    all_cards = BattingCard.select().order_by(BattingCard.id)
     if player_id is not None:
         all_cards = all_cards.where(BattingCard.player_id << player_id)
     if cardset_id is not None:

app/routers_v2/cardpositions.py

@@ -83,13 +83,13 @@ async def get_card_positions(
         all_pos = all_pos.where(CardPosition.player << all_players)
 
     if sort == "innings-desc":
-        all_pos = all_pos.order_by(CardPosition.innings.desc())
+        all_pos = all_pos.order_by(CardPosition.innings.desc(), CardPosition.id)
     elif sort == "innings-asc":
-        all_pos = all_pos.order_by(CardPosition.innings)
+        all_pos = all_pos.order_by(CardPosition.innings, CardPosition.id)
     elif sort == "range-desc":
-        all_pos = all_pos.order_by(CardPosition.range.desc())
+        all_pos = all_pos.order_by(CardPosition.range.desc(), CardPosition.id)
     elif sort == "range-asc":
-        all_pos = all_pos.order_by(CardPosition.range)
+        all_pos = all_pos.order_by(CardPosition.range, CardPosition.id)
 
     return_val = {
         "count": all_pos.count(),

app/routers_v2/cards.py

@@ -75,6 +75,8 @@ async def get_cards(
     if order_by is not None:
         if order_by.lower() == 'new':
             all_cards = all_cards.order_by(-Card.id)
+    else:
+        all_cards = all_cards.order_by(Card.id)
     if limit is not None:
         all_cards = all_cards.limit(limit)
     if dupes:

app/routers_v2/cardsets.py

@@ -33,7 +33,7 @@ class CardsetModel(pydantic.BaseModel):
 async def get_cardsets(
         name: Optional[str] = None, in_desc: Optional[str] = None, event_id: Optional[int] = None,
         in_packs: Optional[bool] = None, ranked_legal: Optional[bool] = None, csv: Optional[bool] = None):
-    all_cardsets = Cardset.select()
+    all_cardsets = Cardset.select().order_by(Cardset.id)
 
     if all_cardsets.count() == 0:
         db.close()
@@ -97,7 +97,7 @@ async def search_cardsets(
     Returns cardsets matching the query with exact matches prioritized over partial matches.
     """
     # Start with all cardsets
-    all_cardsets = Cardset.select()
+    all_cardsets = Cardset.select().order_by(Cardset.id)
 
     # Apply name filter (partial match)
     all_cardsets = all_cardsets.where(fn.Lower(Cardset.name).contains(q.lower()))

app/routers_v2/decisions.py

@@ -179,8 +179,7 @@ async def get_decisions_for_rest(
                 .where((StratPlay.game == x.game) & (StratPlay.pitcher == x.pitcher))
                 .group_by(StratPlay.pitcher, StratPlay.game)
             )
-            logging.info(f"this_line: {this_line[0]}")
+            if this_line.count() == 0 or this_line[0].sum_outs is None:
-            if this_line[0].sum_outs is None:
                 this_val.append(0.0)
             else:
                 this_val.append(

app/routers_v2/events.py

@@ -32,7 +32,7 @@ class EventModel(pydantic.BaseModel):
 async def v1_events_get(
         name: Optional[str] = None, in_desc: Optional[str] = None, active: Optional[bool] = None,
         csv: Optional[bool] = None):
-    all_events = Event.select()
+    all_events = Event.select().order_by(Event.id)
 
     if name is not None:
         all_events = all_events.where(fn.Lower(Event.name) == name.lower())

app/routers_v2/gamerewards.py

@@ -30,7 +30,7 @@ class GameRewardModel(pydantic.BaseModel):
 async def v1_gamerewards_get(
         name: Optional[str] = None, pack_type_id: Optional[int] = None, player_id: Optional[int] = None,
         money: Optional[int] = None, csv: Optional[bool] = None):
-    all_rewards = GameRewards.select()
+    all_rewards = GameRewards.select().order_by(GameRewards.id)
 
     # if all_rewards.count() == 0:
     #     db.close()

app/routers_v2/gauntletrewards.py

@@ -36,7 +36,7 @@ async def v1_gauntletreward_get(
     win_num: Optional[int] = None,
     loss_max: Optional[int] = None,
 ):
-    all_rewards = GauntletReward.select()
+    all_rewards = GauntletReward.select().order_by(GauntletReward.id)
 
     if name is not None:
         all_rewards = all_rewards.where(GauntletReward.name == name)

app/routers_v2/gauntletruns.py

@@ -36,7 +36,7 @@ async def get_gauntletruns(
         losses_max: Optional[int] = None, gsheet: Optional[str] = None, created_after: Optional[int] = None,
         created_before: Optional[int] = None, ended_after: Optional[int] = None, ended_before: Optional[int] = None,
         is_active: Optional[bool] = None, gauntlet_id: list = Query(default=None), season: list = Query(default=None)):
-    all_gauntlets = GauntletRun.select()
+    all_gauntlets = GauntletRun.select().order_by(GauntletRun.id)
 
     if team_id is not None:
         all_gauntlets = all_gauntlets.where(GauntletRun.team_id << team_id)

app/routers_v2/mlbplayers.py

@@ -82,7 +82,7 @@ async def get_players(
     offense_col: list = Query(default=None),
     csv: Optional[bool] = False,
 ):
-    all_players = MlbPlayer.select()
+    all_players = MlbPlayer.select().order_by(MlbPlayer.id)
 
     if full_name is not None:
         name_list = [x.lower() for x in full_name]

app/routers_v2/notifications.py

@@ -35,7 +35,7 @@ async def get_notifs(
         created_after: Optional[int] = None, title: Optional[str] = None, desc: Optional[str] = None,
         field_name: Optional[str] = None, in_desc: Optional[str] = None, about: Optional[str] = None,
         ack: Optional[bool] = None, csv: Optional[bool] = None):
-    all_notif = Notification.select()
+    all_notif = Notification.select().order_by(Notification.id)
 
     if all_notif.count() == 0:
         db.close()

app/routers_v2/packs.py

@@ -83,8 +83,10 @@ async def get_packs(
         all_packs = all_packs.where(Pack.open_time.is_null(not opened))
     if limit is not None:
         all_packs = all_packs.limit(limit)
-    if new_to_old is not None:
+    if new_to_old:
         all_packs = all_packs.order_by(-Pack.id)
+    else:
+        all_packs = all_packs.order_by(Pack.id)
 
     # if all_packs.count() == 0:
     #     db.close()
@@ -96,7 +98,7 @@ async def get_packs(
             data_list.append(
                 [
                     line.id, line.team.abbrev, line.pack_type.name,
-                    datetime.fromtimestamp(line.open_time) if line.open_time else None
+                    line.open_time  # Already datetime in PostgreSQL
                 ]
             )
         return_val = DataFrame(data_list).to_csv(header=False, index=False)
@@ -125,7 +127,7 @@ async def get_one_pack(pack_id, csv: Optional[bool] = False):
         data_list = [
             ['id', 'team', 'pack_type', 'open_time'],
             [this_pack.id, this_pack.team.abbrev, this_pack.pack_type.name,
-             datetime.fromtimestamp(this_pack.open_time) if this_pack.open_time else None]
+             this_pack.open_time]  # Already datetime in PostgreSQL
         ]
         return_val = DataFrame(data_list).to_csv(header=False, index=False)
 

app/routers_v2/packtypes.py

@@ -31,7 +31,7 @@ class PacktypeModel(pydantic.BaseModel):
 async def get_packtypes(
         name: Optional[str] = None, card_count: Optional[int] = None, in_desc: Optional[str] = None,
         available: Optional[bool] = None, csv: Optional[bool] = None):
-    all_packtypes = PackType.select()
+    all_packtypes = PackType.select().order_by(PackType.id)
 
     if all_packtypes.count() == 0:
         db.close()

app/routers_v2/paperdex.py

@@ -31,7 +31,7 @@ async def get_paperdex(
         team_id: Optional[int] = None, player_id: Optional[int] = None, created_after: Optional[int] = None,
         cardset_id: Optional[int] = None, created_before: Optional[int] = None, flat: Optional[bool] = False,
         csv: Optional[bool] = None):
-    all_dex = Paperdex.select().join(Player).join(Cardset)
+    all_dex = Paperdex.select().join(Player).join(Cardset).order_by(Paperdex.id)
 
     if all_dex.count() == 0:
         db.close()

app/routers_v2/pitchingcardratings.py

@@ -158,7 +158,7 @@ async def get_card_ratings(
             status_code=401, detail="You are not authorized to pull card ratings."
         )
 
-    all_ratings = PitchingCardRatings.select()
+    all_ratings = PitchingCardRatings.select().order_by(PitchingCardRatings.id)
 
     if pitchingcard_id is not None:
         all_ratings = all_ratings.where(
@@ -192,7 +192,7 @@ async def get_card_ratings(
 
 
 def get_scouting_dfs(cardset_id: list = None):
-    all_ratings = PitchingCardRatings.select()
+    all_ratings = PitchingCardRatings.select().order_by(PitchingCardRatings.id)
     if cardset_id is not None:
         set_players = Player.select(Player.player_id).where(
             Player.cardset_id << cardset_id

app/routers_v2/pitchingcards.py

@@ -44,7 +44,7 @@ async def get_pitching_cards(
     short_output: bool = False,
     limit: Optional[int] = None,
 ):
-    all_cards = PitchingCard.select()
+    all_cards = PitchingCard.select().order_by(PitchingCard.id)
     if player_id is not None:
         all_cards = all_cards.where(PitchingCard.player_id << player_id)
     if cardset_id is not None:

app/routers_v2/pitstats.py

@@ -58,7 +58,7 @@ async def get_pit_stats(
         card_id: int = None, player_id: int = None, team_id: int = None, vs_team_id: int = None, week: int = None,
         season: int = None, week_start: int = None, week_end: int = None, created: int = None, gs: bool = None,
         csv: bool = None):
-    all_stats = PitchingStat.select().join(Card).join(Player)
+    all_stats = PitchingStat.select().join(Card).join(Player).order_by(PitchingStat.id)
     logging.debug(f'pit query:\n\n{all_stats}')
 
     if season is not None:

app/routers_v2/players.py

@@ -217,6 +217,8 @@ async def get_players(
             all_players = all_players.order_by(Player.rarity)
         elif sort_by == "rarity-asc":
             all_players = all_players.order_by(-Player.rarity)
+    else:
+        all_players = all_players.order_by(Player.player_id)
 
     final_players = []
     # logging.info(f'pos_exclude: {type(pos_exclude)} - {pos_exclude} - is None: {pos_exclude is None}')

app/routers_v2/rarity.py

@@ -28,7 +28,7 @@ class RarityModel(pydantic.BaseModel):
 @router.get('')
 async def get_rarities(value: Optional[int] = None, name: Optional[str] = None, min_value: Optional[int] = None,
                        max_value: Optional[int] = None, csv: Optional[bool] = None):
-    all_rarities = Rarity.select()
+    all_rarities = Rarity.select().order_by(Rarity.id)
 
     if all_rarities.count() == 0:
         db.close()

app/routers_v2/results.py

@@ -196,7 +196,7 @@ async def get_one_results(result_id, csv: Optional[bool] = None):
 @router.get('/team/{team_id}')
 async def get_team_results(
         team_id: int, season: Optional[int] = None, week: Optional[int] = None, csv: Optional[bool] = False):
-    all_results = Result.select().where((Result.away_team_id == team_id) | (Result.home_team_id == team_id))
+    all_results = Result.select().where((Result.away_team_id == team_id) | (Result.home_team_id == team_id)).order_by(Result.id)
     try:
         this_team = Team.get_by_id(team_id)
     except Exception as e:

app/routers_v2/rewards.py

@@ -33,7 +33,7 @@ async def get_rewards(
         name: Optional[str] = None, in_name: Optional[str] = None, team_id: Optional[int] = None,
         season: Optional[int] = None, week: Optional[int] = None, created_after: Optional[int] = None,
         flat: Optional[bool] = False, csv: Optional[bool] = None):
-    all_rewards = Reward.select()
+    all_rewards = Reward.select().order_by(Reward.id)
 
     if all_rewards.count() == 0:
         db.close()

app/routers_v2/stratgame.py

@@ -48,7 +48,7 @@ async def get_games(
         team2_id: list = Query(default=None), game_type: list = Query(default=None), ranked: Optional[bool] = None,
         short_game: Optional[bool] = None, csv: Optional[bool] = False, short_output: bool = False,
         gauntlet_id: Optional[int] = None):
-    all_games = StratGame.select()
+    all_games = StratGame.select().order_by(StratGame.id)
 
     if season is not None:
         all_games = all_games.where(StratGame.season << season)

app/routers_v2/stratplays.py

@@ -177,7 +177,7 @@ async def get_plays(
     limit: Optional[int] = 200,
     page_num: Optional[int] = 1,
 ):
-    all_plays = StratPlay.select()
+    all_plays = StratPlay.select().order_by(StratPlay.id)
 
     if season is not None:
         s_games = StratGame.select().where(StratGame.season << season)
@@ -648,9 +648,11 @@ async def get_batting_totals(
     logging.debug(f"bat_plays query: {bat_plays}")
     logging.debug(f"run_plays query: {run_plays}")
 
-    return_stats = {"count": bat_plays.count(), "stats": []}
+    # Convert to list first - .count() doesn't work on grouped queries in PostgreSQL
+    bat_plays_list = list(bat_plays)
+    return_stats = {"count": len(bat_plays_list), "stats": []}
 
-    for x in bat_plays:
+    for x in bat_plays_list:
         # NOTE: Removed .order_by(StratPlay.id) - not valid with GROUP BY in PostgreSQL
         # and not meaningful for aggregated results anyway
         this_run = run_plays
@@ -1066,9 +1068,11 @@ async def get_pitching_totals(
         limit = 500
     pit_plays = pit_plays.paginate(page_num, limit)
 
-    return_stats = {"count": pit_plays.count(), "stats": []}
+    # Convert to list first - .count() doesn't work on grouped queries in PostgreSQL
+    pit_plays_list = list(pit_plays)
+    return_stats = {"count": len(pit_plays_list), "stats": []}
 
-    for x in pit_plays:
+    for x in pit_plays_list:
         this_dec = all_dec.where(Decision.pitcher == x.pitcher)
         if game_type is not None:
             all_types = [x.lower() for x in game_type]

app/routers_v2/teams.py

@@ -155,6 +155,9 @@ async def get_teams(
     if event_id is not None:
         all_teams = all_teams.where(Team.event_id == event_id)
 
+    # Default ordering for PostgreSQL compatibility
+    all_teams = all_teams.order_by(Team.id)
+
     if limit is not None:
         all_teams = all_teams.limit(limit)
 
@@ -1098,7 +1101,7 @@ async def team_buy_players(team_id: int, ids: str, ts: str):
             # Post a notification
             if this_player.rarity.value >= 2:
                 new_notif = Notification(
-                    created=int_timestamp(datetime.now()),
+                    created=datetime.now(),
                     title=f"Price Change",
                     desc="Modified by buying and selling",
                     field_name=f"{this_player.description} "
@@ -1250,7 +1253,7 @@ async def team_sell_cards(team_id: int, ids: str, ts: str):
             # post a notification
             if this_player.rarity.value >= 2:
                 new_notif = Notification(
-                    created=int_timestamp(datetime.now()),
+                    created=datetime.now(),
                     title=f"Price Change",
                     desc="Modified by buying and selling",
                     field_name=f"{this_player.description} "