From ea36c409020021ec30ae401a2fed6feac3d3cdbd Mon Sep 17 00:00:00 2001 From: Cal Corum Date: Tue, 7 Apr 2026 19:31:48 -0500 Subject: [PATCH 1/4] fix: uncomment variant filter and add variant to CSV export (#197) Restores the commented-out `?variant=` query filter on GET /api/v2/cards so callers can pass variant=0 for base cards or variant=N for a specific refractor variant. Adds variant column to CSV output header and rows. Co-Authored-By: Claude Sonnet 4.6 --- app/routers_v2/cards.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/app/routers_v2/cards.py b/app/routers_v2/cards.py index d56c848..683b33f 100644 --- a/app/routers_v2/cards.py +++ b/app/routers_v2/cards.py @@ -79,8 +79,8 @@ async def get_cards( all_cards = all_cards.where(Card.pack == this_pack) if value is not None: all_cards = all_cards.where(Card.value == value) - # if variant is not None: - # all_cards = all_cards.where(Card.variant == variant) + if variant is not None: + all_cards = all_cards.where(Card.variant == variant) if min_value is not None: all_cards = all_cards.where(Card.value >= min_value) if max_value is not None: @@ -114,8 +114,8 @@ async def get_cards( if csv: data_list = [ - ["id", "player", "cardset", "rarity", "team", "pack", "value"] - ] # , 'variant']] + ["id", "player", "cardset", "rarity", "team", "pack", "value", "variant"] + ] for line in all_cards: data_list.append( [ @@ -125,7 +125,8 @@ async def get_cards( line.player.rarity, line.team.abbrev, line.pack, - line.value, # line.variant + line.value, + line.variant, ] ) return_val = DataFrame(data_list).to_csv(header=False, index=False) From 91a57454f2dd321cee11e0dee6fd9ee57bde7a59 Mon Sep 17 00:00:00 2001 From: Cal Corum Date: Tue, 7 Apr 2026 20:24:51 -0500 Subject: [PATCH 2/4] feat: add template drift check and cache management to deploy tooling deploy.sh now checks local vs remote templates via md5sum on every deploy and warns about drift. Pass --sync-templates to push changed files. Also reports cached card image counts on the target server. New clear-card-cache.sh script inspects or clears cached PNG/APNG card images inside the API container, with --apng-only and --all modes. Added scripts/README.md documenting all operational scripts. Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/README.md | 55 +++++++++++++++++ scripts/clear-card-cache.sh | 89 +++++++++++++++++++++++++++ scripts/deploy.sh | 118 +++++++++++++++++++++++++++++++++++- 3 files changed, 259 insertions(+), 3 deletions(-) create mode 100644 scripts/README.md create mode 100755 scripts/clear-card-cache.sh diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..2edff25 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,55 @@ +# Scripts + +Operational scripts for the Paper Dynasty Database API. + +## deploy.sh + +Deploy the API by tagging a commit and triggering CI/CD. + +```bash +./scripts/deploy.sh dev # Tag HEAD as 'dev', CI builds :dev image +./scripts/deploy.sh prod # Create CalVer tag + 'latest' + 'production' +./scripts/deploy.sh dev abc1234 # Tag a specific commit +./scripts/deploy.sh dev --sync-templates # Deploy + push changed templates to server +``` + +**Template drift check** runs automatically on every deploy. Compares local `storage/templates/*.html` against the target server via md5sum and warns if any files differ. Templates are volume-mounted (not baked into the Docker image), so code deploys alone won't update them. + +**Cached image report** also runs automatically, showing PNG and APNG counts on the target server. + +| Environment | SSH Host | Template Path | +|---|---|---| +| dev | `pd-database` | `/home/cal/container-data/dev-pd-database/storage/templates` | +| prod | `akamai` | `/root/container-data/paper-dynasty/storage/templates` | + +## clear-card-cache.sh + +Inspect or clear cached rendered card images inside the API container. + +```bash +./scripts/clear-card-cache.sh dev # Report cache size (dry run) +./scripts/clear-card-cache.sh dev --apng-only # Delete animated card cache only +./scripts/clear-card-cache.sh dev --all # Delete all cached card images +``` + +Cached images regenerate on demand when next requested. APNG files (T3/T4 animated cards) are the most likely to go stale after template CSS changes. Both destructive modes prompt for confirmation before deleting. + +| Environment | SSH Host | Container | Cache Path | +|---|---|---|---| +| dev | `pd-database` | `dev_pd_database` | `/app/storage/cards/` | +| prod | `akamai` | `pd_api` | `/app/storage/cards/` | + +## Migration Scripts + +| Script | Purpose | +|---|---| +| `migrate_to_postgres.py` | One-time SQLite to PostgreSQL migration | +| `migrate_missing_data.py` | Backfill missing data after migration | +| `db_migrations.py` (in repo root) | Schema migrations | + +## Utility Scripts + +| Script | Purpose | +|---|---| +| `wipe_gauntlet_team.py` | Reset a gauntlet team's state | +| `audit_sqlite.py` | Audit legacy SQLite database | diff --git a/scripts/clear-card-cache.sh b/scripts/clear-card-cache.sh new file mode 100755 index 0000000..3d90152 --- /dev/null +++ b/scripts/clear-card-cache.sh @@ -0,0 +1,89 @@ +#!/bin/bash +# Clear cached card images from the API container +# Usage: ./scripts/clear-card-cache.sh [--apng-only|--all] +# +# With no flags: reports cache size only (dry run) +# --apng-only: delete only .apng files (animated cards) +# --all: delete all cached card images (.png + .apng) +set -euo pipefail + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +NC='\033[0m' + +declare -A DEPLOY_HOST=([dev]="pd-database" [prod]="akamai") +declare -A CONTAINER=([dev]="dev_pd_database" [prod]="pd_api") + +usage() { + echo "Usage: $0 [--apng-only|--all]" + echo "" + echo " No flag Report cache size (dry run)" + echo " --apng-only Delete only .apng files (animated cards)" + echo " --all Delete all cached card images" + exit 1 +} + +[[ $# -lt 1 ]] && usage + +ENV="$1" +ACTION="${2:-report}" + +if [[ "$ENV" != "dev" && "$ENV" != "prod" ]]; then + usage +fi + +HOST="${DEPLOY_HOST[$ENV]}" +CTR="${CONTAINER[$ENV]}" +CACHE_PATH="/app/storage/cards" + +report() { + echo -e "${CYAN}Card image cache on ${HOST} (${CTR}):${NC}" + ssh "$HOST" " + png_count=\$(docker exec $CTR find $CACHE_PATH -name '*.png' 2>/dev/null | wc -l) + apng_count=\$(docker exec $CTR find $CACHE_PATH -name '*.apng' 2>/dev/null | wc -l) + echo \" PNG: \${png_count} files\" + echo \" APNG: \${apng_count} files\" + echo \" Total: \$((\${png_count} + \${apng_count})) files\" + " 2>/dev/null || { + echo -e "${RED}Could not reach ${HOST}.${NC}" + exit 1 + } +} + +report + +case "$ACTION" in + report) + echo -e "${GREEN}Dry run — no files deleted. Pass --apng-only or --all to clear.${NC}" + ;; + + --apng-only) + echo -e "${YELLOW}Deleting all .apng files from ${CTR}...${NC}" + read -rp "Proceed? [y/N] " confirm + [[ "$confirm" =~ ^[Yy]$ ]] || { + echo "Aborted." + exit 0 + } + + deleted=$(ssh "$HOST" "docker exec $CTR find $CACHE_PATH -name '*.apng' -delete -print 2>/dev/null | wc -l") + echo -e "${GREEN}Deleted ${deleted} .apng files.${NC}" + ;; + + --all) + echo -e "${RED}Deleting ALL cached card images from ${CTR}...${NC}" + read -rp "This will clear PNG and APNG caches. Proceed? [y/N] " confirm + [[ "$confirm" =~ ^[Yy]$ ]] || { + echo "Aborted." + exit 0 + } + + deleted=$(ssh "$HOST" "docker exec $CTR find $CACHE_PATH -type f \( -name '*.png' -o -name '*.apng' \) -delete -print 2>/dev/null | wc -l") + echo -e "${GREEN}Deleted ${deleted} cached card images.${NC}" + ;; + + *) + usage + ;; +esac diff --git a/scripts/deploy.sh b/scripts/deploy.sh index ef8ea7d..a3ada84 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -1,31 +1,67 @@ #!/bin/bash # Deploy Paper Dynasty Database API -# Usage: ./scripts/deploy.sh [commit] +# Usage: ./scripts/deploy.sh [--sync-templates] [commit] # # Dev: Force-updates the "dev" git tag → CI builds :dev Docker image # Prod: Creates CalVer tag + force-updates "latest" and "production" git tags # → CI builds :, :latest, :production Docker images +# +# Options: +# --sync-templates Upload changed templates to the target server via scp +# +# Templates are volume-mounted (not in the Docker image). The script always +# checks for template drift and warns if local/remote differ. Pass +# --sync-templates to actually push the changed files. set -euo pipefail RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' +CYAN='\033[0;36m' NC='\033[0m' REMOTE="origin" +SYNC_TEMPLATES=false +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TEMPLATE_DIR="$SCRIPT_DIR/../storage/templates" + +# Server config +declare -A DEPLOY_HOST=([dev]="pd-database" [prod]="akamai") +declare -A TEMPLATE_PATH=( + [dev]="/home/cal/container-data/dev-pd-database/storage/templates" + [prod]="/root/container-data/paper-dynasty/storage/templates" +) usage() { - echo "Usage: $0 [commit]" + echo "Usage: $0 [--sync-templates] [commit]" echo "" echo " dev [commit] Force-update 'dev' tag on HEAD or specified commit" echo " prod [commit] Create CalVer + 'latest' + 'production' tags on HEAD or specified commit" + echo "" + echo "Options:" + echo " --sync-templates Upload changed templates to the target server" exit 1 } [[ $# -lt 1 ]] && usage ENV="$1" -COMMIT="${2:-HEAD}" +shift + +# Parse optional flags +COMMIT="HEAD" +while [[ $# -gt 0 ]]; do + case "$1" in + --sync-templates) + SYNC_TEMPLATES=true + shift + ;; + *) + COMMIT="$1" + shift + ;; + esac +done SHA=$(git rev-parse "$COMMIT" 2>/dev/null) || { echo -e "${RED}Invalid commit: $COMMIT${NC}" @@ -40,6 +76,82 @@ if ! git branch -a --contains "$SHA" 2>/dev/null | grep -qE '(^|\s)(main|remotes exit 1 fi +# --- Template drift check --- +check_templates() { + local host="${DEPLOY_HOST[$ENV]}" + local remote_path="${TEMPLATE_PATH[$ENV]}" + + echo -e "${CYAN}Checking templates against ${host}:${remote_path}...${NC}" + + local local_hashes remote_hashes + local_hashes=$(cd "$TEMPLATE_DIR" && md5sum *.html 2>/dev/null | sort -k2) + remote_hashes=$(ssh "$host" "cd '$remote_path' && md5sum *.html 2>/dev/null | sort -k2" 2>/dev/null) || { + echo -e "${YELLOW} Could not reach ${host} — skipping template check.${NC}" + return 0 + } + + local changed=() + local missing_remote=() + while IFS= read -r line; do + local hash file + hash=$(echo "$line" | awk '{print $1}') + file=$(echo "$line" | awk '{print $2}') + remote_hash=$(echo "$remote_hashes" | awk -v f="$file" '$2 == f {print $1}') + if [[ -z "$remote_hash" ]]; then + missing_remote+=("$file") + elif [[ "$hash" != "$remote_hash" ]]; then + changed+=("$file") + fi + done <<<"$local_hashes" + + if [[ ${#changed[@]} -eq 0 && ${#missing_remote[@]} -eq 0 ]]; then + echo -e "${GREEN} Templates in sync.${NC}" + return 0 + fi + + echo -e "${YELLOW} Template drift detected:${NC}" + for f in "${changed[@]+"${changed[@]}"}"; do + [[ -n "$f" ]] && echo -e " ${YELLOW}CHANGED${NC} $f" + done + for f in "${missing_remote[@]+"${missing_remote[@]}"}"; do + [[ -n "$f" ]] && echo -e " ${YELLOW}MISSING${NC} $f (not on server)" + done + + if [[ "$SYNC_TEMPLATES" == true ]]; then + echo -e "${CYAN} Syncing templates...${NC}" + for f in "${changed[@]+"${changed[@]}"}" "${missing_remote[@]+"${missing_remote[@]}"}"; do + [[ -n "$f" ]] && scp "$TEMPLATE_DIR/$f" "${host}:${remote_path}/$f" + done + echo -e "${GREEN} Templates synced to ${host}.${NC}" + else + echo -e "${YELLOW} Run with --sync-templates to push changes.${NC}" + fi +} + +check_templates + +# --- Cached image report --- +report_cache() { + local host="${DEPLOY_HOST[$ENV]}" + local container + if [[ "$ENV" == "dev" ]]; then + container="dev_pd_database" + else + container="pd_api" + fi + + echo -e "${CYAN}Cached card images on ${host} (${container}):${NC}" + ssh "$host" " + png_count=\$(docker exec $container find /app/storage/cards -name '*.png' 2>/dev/null | wc -l) + apng_count=\$(docker exec $container find /app/storage/cards -name '*.apng' 2>/dev/null | wc -l) + echo \" PNG: \${png_count} files\" + echo \" APNG: \${apng_count} files\" + echo \" Total: \$((\${png_count} + \${apng_count})) files\" + " 2>/dev/null || echo -e "${YELLOW} Could not reach ${host} — skipping cache report.${NC}" +} + +report_cache + case "$ENV" in dev) echo -e "${YELLOW}Deploying to dev...${NC}" From 900f9723e5bf3dc90b342afedac72d025ffbbe15 Mon Sep 17 00:00:00 2001 From: Cal Corum Date: Tue, 7 Apr 2026 20:32:12 -0500 Subject: [PATCH 3/4] =?UTF-8?q?fix:=20address=20PR=20review=20=E2=80=94=20?= =?UTF-8?q?unknown=20flag=20guard,=20local=20var=20scope,=20container=20ma?= =?UTF-8?q?p?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Reject unknown --flags with error instead of silently treating as commit SHA - Declare remote_hash as local to prevent stale values across loop iterations - Use associative array for container names (consistent with DEPLOY_HOST pattern) Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/deploy.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/deploy.sh b/scripts/deploy.sh index a3ada84..509138f 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -56,6 +56,10 @@ while [[ $# -gt 0 ]]; do SYNC_TEMPLATES=true shift ;; + --*) + echo -e "${RED}Unknown option: $1${NC}" + usage + ;; *) COMMIT="$1" shift @@ -93,7 +97,7 @@ check_templates() { local changed=() local missing_remote=() while IFS= read -r line; do - local hash file + local hash file remote_hash hash=$(echo "$line" | awk '{print $1}') file=$(echo "$line" | awk '{print $2}') remote_hash=$(echo "$remote_hashes" | awk -v f="$file" '$2 == f {print $1}') @@ -131,14 +135,11 @@ check_templates() { check_templates # --- Cached image report --- +declare -A API_CONTAINER=([dev]="dev_pd_database" [prod]="pd_api") + report_cache() { local host="${DEPLOY_HOST[$ENV]}" - local container - if [[ "$ENV" == "dev" ]]; then - container="dev_pd_database" - else - container="pd_api" - fi + local container="${API_CONTAINER[$ENV]}" echo -e "${CYAN}Cached card images on ${host} (${container}):${NC}" ssh "$host" " From b7196c1c56e85f8a8bdb0c777ceef4a8d113eb19 Mon Sep 17 00:00:00 2001 From: Cal Corum Date: Tue, 7 Apr 2026 21:04:33 -0500 Subject: [PATCH 4/4] perf: batch image_url prefetch in list_card_states to eliminate N+1 (#199) Replace per-row CardModel.get() in _build_card_state_response with a bulk prefetch in list_card_states: collect variant player IDs, issue at most 2 queries (BattingCard + PitchingCard), build a (player_id, variant) -> image_url map, and pass the resolved value directly to the helper. The single-card get_card_state path is unchanged and still resolves image_url inline (one extra query is acceptable for a single-item response). Closes #199 Co-Authored-By: Claude Sonnet 4.6 --- app/routers_v2/refractor.py | 76 +++++++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 19 deletions(-) diff --git a/app/routers_v2/refractor.py b/app/routers_v2/refractor.py index cb5d06a..d28aba2 100644 --- a/app/routers_v2/refractor.py +++ b/app/routers_v2/refractor.py @@ -23,8 +23,12 @@ _NEXT_THRESHOLD_ATTR = { 4: None, } +# Sentinel used by _build_card_state_response to distinguish "caller did not +# pass image_url" (do the DB lookup) from "caller passed None" (use None). +_UNSET = object() -def _build_card_state_response(state, player_name=None) -> dict: + +def _build_card_state_response(state, player_name=None, image_url=_UNSET) -> dict: """Serialise a RefractorCardState into the standard API response shape. Produces a flat dict with player_id and team_id as plain integers, @@ -67,22 +71,27 @@ def _build_card_state_response(state, player_name=None) -> dict: if player_name is not None: result["player_name"] = player_name - # Resolve image_url from the variant card row - image_url = None - if state.variant and state.variant > 0: - card_type = ( - state.track.card_type if hasattr(state, "track") and state.track else None - ) - if card_type: - CardModel = BattingCard if card_type == "batter" else PitchingCard - try: - variant_card = CardModel.get( - (CardModel.player_id == state.player_id) - & (CardModel.variant == state.variant) - ) - image_url = variant_card.image_url - except CardModel.DoesNotExist: - pass + # Resolve image_url from the variant card row. + # When image_url is pre-fetched by the caller (batch list path), it is + # passed directly and the per-row DB query is skipped entirely. + if image_url is _UNSET: + image_url = None + if state.variant and state.variant > 0: + card_type = ( + state.track.card_type + if hasattr(state, "track") and state.track + else None + ) + if card_type: + CardModel = BattingCard if card_type == "batter" else PitchingCard + try: + variant_card = CardModel.get( + (CardModel.player_id == state.player_id) + & (CardModel.variant == state.variant) + ) + image_url = variant_card.image_url + except CardModel.DoesNotExist: + pass result["image_url"] = image_url return result @@ -230,14 +239,43 @@ async def list_card_states( query = query.where(RefractorCardState.last_evaluated_at.is_null(False)) total = query.count() or 0 + states_page = list(query.offset(offset).limit(limit)) + + # Pre-fetch image_urls in at most 2 bulk queries (one per card table) so + # that _build_card_state_response never issues a per-row CardModel.get(). + batter_pids: set[int] = set() + pitcher_pids: set[int] = set() + for state in states_page: + if state.variant and state.variant > 0: + card_type = state.track.card_type if state.track else None + if card_type == "batter": + batter_pids.add(state.player_id) + elif card_type in ("sp", "rp"): + pitcher_pids.add(state.player_id) + + image_url_map: dict[tuple[int, int], str | None] = {} + if batter_pids: + for card in BattingCard.select().where(BattingCard.player_id.in_(batter_pids)): + image_url_map[(card.player_id, card.variant)] = card.image_url + if pitcher_pids: + for card in PitchingCard.select().where( + PitchingCard.player_id.in_(pitcher_pids) + ): + image_url_map[(card.player_id, card.variant)] = card.image_url + items = [] - for state in query.offset(offset).limit(limit): + for state in states_page: player_name = None try: player_name = state.player.p_name except Exception: pass - items.append(_build_card_state_response(state, player_name=player_name)) + img_url = image_url_map.get((state.player_id, state.variant)) + items.append( + _build_card_state_response( + state, player_name=player_name, image_url=img_url + ) + ) return {"count": total, "items": items}