8bddf31bf6
Allow upload scripts to target a local API server instead of the remote production server, enabling 32x+ concurrency for dramatically faster full-cardset uploads (~30-45s vs ~2-3min for 800 cards). - pd_cards/core/upload.py: add api_url param to upload_cards_to_s3(), refresh_card_images(), and check_card_images() - pd_cards/commands/upload.py: add --api-url CLI option to upload s3 - check_cards_and_upload.py: read PD_API_URL env var with prod fallback - Update CLAUDE.md, CLI reference, and Phase 0 project plan docs Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
282 lines
8.6 KiB
Python
282 lines
8.6 KiB
Python
"""
|
|
Card image upload commands.
|
|
|
|
Commands for uploading card images to AWS S3.
|
|
"""
|
|
|
|
import asyncio
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
import typer
|
|
from rich.console import Console
|
|
|
|
app = typer.Typer(no_args_is_help=True)
|
|
console = Console()
|
|
|
|
|
|
@app.command()
|
|
def s3(
|
|
cardset: str = typer.Option(..., "--cardset", "-c", help="Cardset name to upload"),
|
|
start_id: Optional[int] = typer.Option(
|
|
None, "--start-id", help="Player ID to start from (for resuming)"
|
|
),
|
|
limit: Optional[int] = typer.Option(
|
|
None, "--limit", "-l", help="Limit number of cards to process"
|
|
),
|
|
html: bool = typer.Option(
|
|
False, "--html", help="Upload HTML preview cards instead of PNG"
|
|
),
|
|
skip_batters: bool = typer.Option(
|
|
False, "--skip-batters", help="Skip batting cards"
|
|
),
|
|
skip_pitchers: bool = typer.Option(
|
|
False, "--skip-pitchers", help="Skip pitching cards"
|
|
),
|
|
upload: bool = typer.Option(True, "--upload/--no-upload", help="Upload to S3"),
|
|
update_urls: bool = typer.Option(
|
|
True, "--update-urls/--no-update-urls", help="Update player URLs in database"
|
|
),
|
|
dry_run: bool = typer.Option(
|
|
False, "--dry-run", "-n", help="Preview without uploading"
|
|
),
|
|
concurrency: int = typer.Option(
|
|
8, "--concurrency", "-j", help="Number of parallel uploads (default: 8)"
|
|
),
|
|
api_url: str = typer.Option(
|
|
"https://pd.manticorum.com/api",
|
|
"--api-url",
|
|
help="API base URL for card rendering (use http://localhost:8000/api for local server)",
|
|
),
|
|
):
|
|
"""
|
|
Upload card images to AWS S3.
|
|
|
|
Fetches card images from Paper Dynasty API and uploads to S3 bucket.
|
|
Cards are processed concurrently; use --concurrency to tune parallelism.
|
|
|
|
For high-concurrency local rendering, start the API server locally and use:
|
|
pd-cards upload s3 --cardset "2005 Live" --api-url http://localhost:8000/api --concurrency 32
|
|
|
|
Example:
|
|
pd-cards upload s3 --cardset "2005 Live" --limit 10
|
|
pd-cards upload s3 --cardset "2005 Live" --concurrency 16
|
|
"""
|
|
console.print()
|
|
console.print("=" * 70)
|
|
console.print(f"[bold]S3 UPLOAD - {cardset}[/bold]")
|
|
console.print("=" * 70)
|
|
|
|
console.print(f"Cardset: {cardset}")
|
|
if start_id:
|
|
console.print(f"Starting from player ID: {start_id}")
|
|
if limit:
|
|
console.print(f"Limit: {limit} cards")
|
|
if html:
|
|
console.print("Mode: HTML preview cards")
|
|
if skip_batters:
|
|
console.print("Skipping: Batting cards")
|
|
if skip_pitchers:
|
|
console.print("Skipping: Pitching cards")
|
|
console.print(f"API URL: {api_url}")
|
|
console.print(f"Upload to S3: {upload and not dry_run}")
|
|
console.print(f"Update URLs: {update_urls and not dry_run}")
|
|
console.print(f"Concurrency: {concurrency} parallel tasks")
|
|
console.print()
|
|
|
|
if dry_run:
|
|
console.print("[yellow]DRY RUN - no uploads will be made[/yellow]")
|
|
console.print()
|
|
console.print("[green]Validation passed - ready to run[/green]")
|
|
raise typer.Exit(0)
|
|
|
|
try:
|
|
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
|
|
|
|
from pd_cards.core.upload import upload_cards_to_s3
|
|
|
|
def progress_callback(_count: int, label: str) -> None:
|
|
console.print(f" Progress: {label}")
|
|
|
|
console.print("[bold]Starting S3 upload...[/bold]")
|
|
console.print()
|
|
|
|
result = asyncio.run(
|
|
upload_cards_to_s3(
|
|
cardset_name=cardset,
|
|
start_id=start_id,
|
|
limit=limit,
|
|
html_cards=html,
|
|
skip_batters=skip_batters,
|
|
skip_pitchers=skip_pitchers,
|
|
upload=upload,
|
|
update_urls=update_urls,
|
|
on_progress=progress_callback,
|
|
concurrency=concurrency,
|
|
api_url=api_url,
|
|
)
|
|
)
|
|
|
|
success_count = len(result["successes"])
|
|
error_count = len(result["errors"])
|
|
upload_count = len(result["uploads"])
|
|
url_update_count = len(result["url_updates"])
|
|
|
|
console.print()
|
|
console.print("=" * 70)
|
|
console.print("[bold green]✓ S3 UPLOAD COMPLETE[/bold green]")
|
|
console.print("=" * 70)
|
|
console.print(f" Successes: {success_count}")
|
|
console.print(f" S3 uploads: {upload_count}")
|
|
console.print(f" URL updates: {url_update_count}")
|
|
if error_count:
|
|
console.print(f" [red]Errors: {error_count}[/red]")
|
|
for player, err in result["errors"][:10]:
|
|
console.print(
|
|
f" - player {player.get('player_id', '?')} "
|
|
f"({player.get('p_name', '?')}): {err}"
|
|
)
|
|
if error_count > 10:
|
|
console.print(f" ... and {error_count - 10} more (see logs)")
|
|
|
|
except ImportError as e:
|
|
console.print(f"[red]Error importing modules: {e}[/red]")
|
|
raise typer.Exit(1)
|
|
except Exception as e:
|
|
console.print(f"[red]Error: {e}[/red]")
|
|
import traceback
|
|
|
|
traceback.print_exc()
|
|
raise typer.Exit(1)
|
|
|
|
|
|
@app.command()
|
|
def refresh(
|
|
cardset: str = typer.Option(..., "--cardset", "-c", help="Cardset name"),
|
|
limit: Optional[int] = typer.Option(
|
|
None, "--limit", "-l", help="Limit number of cards"
|
|
),
|
|
dry_run: bool = typer.Option(
|
|
False, "--dry-run", "-n", help="Preview without refreshing"
|
|
),
|
|
):
|
|
"""
|
|
Refresh card images for a cardset.
|
|
|
|
Re-generates and re-uploads card images.
|
|
|
|
Example:
|
|
pd-cards upload refresh --cardset "2005 Live" --limit 10
|
|
"""
|
|
console.print()
|
|
console.print("=" * 70)
|
|
console.print(f"[bold]CARD REFRESH - {cardset}[/bold]")
|
|
console.print("=" * 70)
|
|
|
|
console.print(f"Cardset: {cardset}")
|
|
if limit:
|
|
console.print(f"Limit: {limit} cards")
|
|
|
|
if dry_run:
|
|
console.print("[yellow]DRY RUN - no changes will be made[/yellow]")
|
|
console.print()
|
|
console.print("[green]Validation passed - ready to run[/green]")
|
|
raise typer.Exit(0)
|
|
|
|
try:
|
|
import sys
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
|
|
|
|
import refresh_cards as rc
|
|
|
|
# Configure the module
|
|
rc.CARDSET_NAME = cardset
|
|
rc.TEST_COUNT = limit if limit else 9999
|
|
|
|
console.print("[bold]Starting card refresh...[/bold]")
|
|
console.print()
|
|
|
|
asyncio.run(rc.main([]))
|
|
|
|
console.print()
|
|
console.print("=" * 70)
|
|
console.print("[bold green]✓ CARD REFRESH COMPLETE[/bold green]")
|
|
console.print("=" * 70)
|
|
|
|
except ImportError as e:
|
|
console.print(f"[red]Error importing modules: {e}[/red]")
|
|
console.print("Try: python refresh_cards.py")
|
|
raise typer.Exit(1)
|
|
except Exception as e:
|
|
console.print(f"[red]Error: {e}[/red]")
|
|
import traceback
|
|
|
|
traceback.print_exc()
|
|
raise typer.Exit(1)
|
|
|
|
|
|
@app.command()
|
|
def check(
|
|
cardset: str = typer.Option(..., "--cardset", "-c", help="Cardset name"),
|
|
limit: Optional[int] = typer.Option(
|
|
None, "--limit", "-l", help="Limit number of cards to check"
|
|
),
|
|
output_dir: Path = typer.Option(
|
|
Path("data-output"), "--output", "-o", help="Output directory"
|
|
),
|
|
):
|
|
"""
|
|
Check and validate card images without uploading.
|
|
|
|
Downloads card images and saves locally for review.
|
|
|
|
Example:
|
|
pd-cards upload check --cardset "2005 Live" --limit 10
|
|
"""
|
|
console.print()
|
|
console.print("=" * 70)
|
|
console.print(f"[bold]CARD CHECK - {cardset}[/bold]")
|
|
console.print("=" * 70)
|
|
|
|
console.print(f"Cardset: {cardset}")
|
|
if limit:
|
|
console.print(f"Limit: {limit} cards")
|
|
console.print(f"Output: {output_dir}")
|
|
console.print()
|
|
|
|
try:
|
|
import sys
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
|
|
|
|
import check_cards_and_upload as ccu
|
|
|
|
# Configure for check-only mode
|
|
ccu.CARDSET_NAME = cardset
|
|
ccu.START_ID = None
|
|
ccu.TEST_COUNT = limit if limit else 9999
|
|
ccu.HTML_CARDS = False
|
|
ccu.UPLOAD_TO_S3 = False
|
|
ccu.UPDATE_PLAYER_URLS = False
|
|
ccu.s3_client = None
|
|
|
|
console.print("[bold]Starting card check...[/bold]")
|
|
console.print()
|
|
|
|
asyncio.run(ccu.main([]))
|
|
|
|
console.print()
|
|
console.print("[green]✓ Card check complete[/green]")
|
|
|
|
except ImportError as e:
|
|
console.print(f"[red]Error importing modules: {e}[/red]")
|
|
raise typer.Exit(1)
|
|
except Exception as e:
|
|
console.print(f"[red]Error: {e}[/red]")
|
|
import traceback
|
|
|
|
traceback.print_exc()
|
|
raise typer.Exit(1)
|