From 4f2a66b67ea74b4e96e2fa793e363bb3bd1bf305 Mon Sep 17 00:00:00 2001 From: Cal Corum Date: Fri, 13 Mar 2026 00:04:34 -0500 Subject: [PATCH 1/2] feat: add Phase 0 baseline benchmark script and log (WP-00) (#87) Closes #87 Add benchmarks/benchmark_renders.sh to time 10 sequential card renders via curl against any API environment, and benchmarks/BASELINE.md to record methodology and results for pre/post optimization comparison. Co-Authored-By: Claude Sonnet 4.6 --- benchmarks/BASELINE.md | 92 +++++++++++++++++++++++++++++++++ benchmarks/benchmark_renders.sh | 79 ++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100644 benchmarks/BASELINE.md create mode 100755 benchmarks/benchmark_renders.sh diff --git a/benchmarks/BASELINE.md b/benchmarks/BASELINE.md new file mode 100644 index 0000000..1f26b7d --- /dev/null +++ b/benchmarks/BASELINE.md @@ -0,0 +1,92 @@ +# Phase 0 Baseline Benchmarks — WP-00 + +Captured before any Phase 0 render-pipeline optimizations (WP-01 through WP-04). +Run these benchmarks again after each work package lands to measure improvement. + +--- + +## 1. Per-Card Render Time + +**What is measured:** Time from HTTP request to full PNG response for a single card image. +Each render triggers a full Playwright Chromium launch, page load, screenshot, and teardown. + +### Method + +```bash +# Set API_BASE to the environment under test +export API_BASE=http://pddev.manticorum.com:816 + +# Run against 10 batting cards (auto-fetches player IDs) +./benchmarks/benchmark_renders.sh + +# Or supply explicit player IDs: +./benchmarks/benchmark_renders.sh 101 102 103 104 105 106 107 108 109 110 + +# For pitching cards: +CARD_TYPE=pitching ./benchmarks/benchmark_renders.sh +``` + +Prerequisites: `curl`, `jq`, `bc` + +Results are appended to `benchmarks/render_timings.txt`. + +### Baseline Results — 2026-03-13 + +| Environment | Card type | N | Min (s) | Max (s) | Avg (s) | +|-------------|-----------|---|---------|---------|---------| +| dev (pddev.manticorum.com:816) | batting | 10 | _TBD_ | _TBD_ | _TBD_ | +| dev (pddev.manticorum.com:816) | pitching | 10 | _TBD_ | _TBD_ | _TBD_ | + +> **Note:** Run `./benchmarks/benchmark_renders.sh` against the dev API and paste +> the per-render timings from `render_timings.txt` into the table above. + +**Expected baseline (pre-optimization):** ~2.0–3.0s per render +(Chromium spawn ~1.0–1.5s + Google Fonts fetch ~0.3–0.5s + render ~0.3s) + +--- + +## 2. Batch Upload Time + +**What is measured:** Wall-clock time to render and upload N card images to S3 +using the `pd-cards upload` CLI (in the `card-creation` repo). + +### Method + +```bash +# In the card-creation repo: +time pd-cards upload --cardset 24 --limit 20 +``` + +Or to capture more detail: + +```bash +START=$(date +%s%3N) +pd-cards upload --cardset 24 --limit 20 +END=$(date +%s%3N) +echo "Elapsed: $(( (END - START) / 1000 )).$(( (END - START) % 1000 ))s" +``` + +### Baseline Results — 2026-03-13 + +| Environment | Cards | Elapsed (s) | Per-card avg (s) | +|-------------|-------|-------------|-----------------| +| dev | 20 | _TBD_ | _TBD_ | +| dev | 20 | _TBD_ | _TBD_ | + +> **Note:** Run the upload command in the `card-creation` repo and record timings here. + +**Expected baseline (pre-optimization):** ~40–60s for 20 cards (~2–3s each sequential) + +--- + +## 3. Re-run After Each Work Package + +| Milestone | Per-card avg (s) | 20-card upload (s) | Notes | +|-----------|-----------------|-------------------|-------| +| Baseline (pre-WP-01/02) | _TBD_ | _TBD_ | This document | +| After WP-01 (self-hosted fonts) | — | — | | +| After WP-02 (persistent browser) | — | — | | +| After WP-01 + WP-02 combined | — | — | | +| After WP-04 (concurrent upload) | — | — | | + +Target: <1.0s per render, <5 min for 800-card upload (with WP-01 + WP-02 deployed). diff --git a/benchmarks/benchmark_renders.sh b/benchmarks/benchmark_renders.sh new file mode 100755 index 0000000..4046d5b --- /dev/null +++ b/benchmarks/benchmark_renders.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# WP-00: Baseline benchmark — sequential card render timing +# +# Measures per-card render time for 10 cards by calling the card image +# endpoint sequentially and recording curl's time_total for each request. +# +# Usage: +# API_BASE=http://pddev.manticorum.com:816 ./benchmarks/benchmark_renders.sh +# API_BASE=http://localhost:8000 CARD_TYPE=pitching ./benchmarks/benchmark_renders.sh +# API_BASE=http://pddev.manticorum.com:816 ./benchmarks/benchmark_renders.sh 101 102 103 +# +# Arguments (optional): explicit player IDs to render. If omitted, the script +# queries the API for the first 10 players in the live cardset. +# +# Output: results are printed to stdout and appended to benchmarks/render_timings.txt + +set -euo pipefail + +API_BASE="${API_BASE:-http://localhost:8000}" +CARD_TYPE="${CARD_TYPE:-batting}" +OUTFILE="$(dirname "$0")/render_timings.txt" + +echo "=== Card Render Benchmark ===" | tee -a "$OUTFILE" +echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)" | tee -a "$OUTFILE" +echo "API: $API_BASE" | tee -a "$OUTFILE" +echo "Type: $CARD_TYPE" | tee -a "$OUTFILE" + +# --- Resolve player IDs --- +if [ "$#" -gt 0 ]; then + PLAYER_IDS=("$@") + echo "Mode: explicit IDs (${#PLAYER_IDS[@]} players)" | tee -a "$OUTFILE" +else + echo "Mode: auto-fetch first 10 players from live cardset" | tee -a "$OUTFILE" + # Fetch player list and extract IDs; requires jq + RAW=$(curl -sf "$API_BASE/api/v2/players?page_size=10") + PLAYER_IDS=($(echo "$RAW" | jq -r '.players[].id // .[]?.id // .[]' 2>/dev/null | head -10)) + if [ "${#PLAYER_IDS[@]}" -eq 0 ]; then + echo "ERROR: Could not fetch player IDs from $API_BASE/api/v2/players" | tee -a "$OUTFILE" + exit 1 + fi +fi + +echo "Players: ${PLAYER_IDS[*]}" | tee -a "$OUTFILE" +echo "" | tee -a "$OUTFILE" + +# --- Run renders --- +TOTAL=0 +COUNT=0 + +for player_id in "${PLAYER_IDS[@]}"; do + URL="$API_BASE/api/v2/players/$player_id/${CARD_TYPE}card" + # Bypass cached PNG files by adding ?nocache=1 (treated as unknown param, triggers re-render) + # Remove this flag after baseline is captured to test cache-hit performance separately. + HTTP_CODE=$(curl -s -o /dev/null \ + --write-out "%{http_code}" \ + -w " %{time_total}" \ + "$URL" 2>&1) + STATUS=$(echo "$HTTP_CODE" | awk '{print $1}') + TIMING=$(echo "$HTTP_CODE" | awk '{print $2}') + echo " player_id=$player_id http=$STATUS time=${TIMING}s" | tee -a "$OUTFILE" + if [ "$STATUS" = "200" ]; then + TOTAL=$(echo "$TOTAL + $TIMING" | bc -l) + COUNT=$((COUNT + 1)) + fi +done + +# --- Summary --- +echo "" | tee -a "$OUTFILE" +if [ "$COUNT" -gt 0 ]; then + AVG=$(echo "scale=3; $TOTAL / $COUNT" | bc -l) + echo "Successful renders: $COUNT / ${#PLAYER_IDS[@]}" | tee -a "$OUTFILE" + echo "Total time: ${TOTAL}s" | tee -a "$OUTFILE" + echo "Average: ${AVG}s per render" | tee -a "$OUTFILE" +else + echo "No successful renders — check API_BASE and player IDs" | tee -a "$OUTFILE" +fi +echo "---" | tee -a "$OUTFILE" +echo "" | tee -a "$OUTFILE" +echo "Results appended to $OUTFILE" -- 2.25.1 From 2ab6e71735fa6a7e980307ddabdba569a05e0279 Mon Sep 17 00:00:00 2001 From: Cal Corum Date: Fri, 13 Mar 2026 00:31:35 -0500 Subject: [PATCH 2/2] fix: address review feedback (#95) - Fix curl -w override bug: consolidate --write-out/"-w" into single -w "%{http_code} %{time_total}" so STATUS and TIMING are both captured - Add ?nocache=1 to render URL so baseline measures cold render time - Fix duplicate BASELINE.md Section 2 rows (batting vs pitching) - Add benchmarks/render_timings.txt to .gitignore Co-Authored-By: Claude Sonnet 4.6 --- .gitignore | 3 +++ benchmarks/BASELINE.md | 4 ++-- benchmarks/benchmark_renders.sh | 10 +++------- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 2725e46..fa64fac 100644 --- a/.gitignore +++ b/.gitignore @@ -83,3 +83,6 @@ postgres_data/ README_GAUNTLET_CLEANUP.md wipe_gauntlet_team.py SCHEMA.md + +# Benchmark output files +benchmarks/render_timings.txt diff --git a/benchmarks/BASELINE.md b/benchmarks/BASELINE.md index 1f26b7d..2f4bb42 100644 --- a/benchmarks/BASELINE.md +++ b/benchmarks/BASELINE.md @@ -70,8 +70,8 @@ echo "Elapsed: $(( (END - START) / 1000 )).$(( (END - START) % 1000 ))s" | Environment | Cards | Elapsed (s) | Per-card avg (s) | |-------------|-------|-------------|-----------------| -| dev | 20 | _TBD_ | _TBD_ | -| dev | 20 | _TBD_ | _TBD_ | +| dev (batting) | 20 | _TBD_ | _TBD_ | +| dev (pitching) | 20 | _TBD_ | _TBD_ | > **Note:** Run the upload command in the `card-creation` repo and record timings here. diff --git a/benchmarks/benchmark_renders.sh b/benchmarks/benchmark_renders.sh index 4046d5b..a9c5d6e 100755 --- a/benchmarks/benchmark_renders.sh +++ b/benchmarks/benchmark_renders.sh @@ -48,13 +48,9 @@ TOTAL=0 COUNT=0 for player_id in "${PLAYER_IDS[@]}"; do - URL="$API_BASE/api/v2/players/$player_id/${CARD_TYPE}card" - # Bypass cached PNG files by adding ?nocache=1 (treated as unknown param, triggers re-render) - # Remove this flag after baseline is captured to test cache-hit performance separately. - HTTP_CODE=$(curl -s -o /dev/null \ - --write-out "%{http_code}" \ - -w " %{time_total}" \ - "$URL" 2>&1) + # Bypass cached PNG files; remove ?nocache=1 after baseline is captured to test cache-hit performance. + URL="$API_BASE/api/v2/players/$player_id/${CARD_TYPE}card?nocache=1" + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code} %{time_total}" "$URL" 2>&1) STATUS=$(echo "$HTTP_CODE" | awk '{print $1}') TIMING=$(echo "$HTTP_CODE" | awk '{print $2}') echo " player_id=$player_id http=$STATUS time=${TIMING}s" | tee -a "$OUTFILE" -- 2.25.1