cal/paper-dynasty-database
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PostgreSQL migration: Complete code preparation phase

Browse Source 
- Add db_helpers.py with cross-database upsert functions for SQLite/PostgreSQL
- Replace 12 on_conflict_replace() calls with PostgreSQL-compatible upserts
- Add unique indexes: StratPlay(game, play_num), Decision(game, pitcher)
- Add max_length to Team model fields (abbrev, sname, lname)
- Fix boolean comparison in teams.py (== 0/1 to == False/True)
- Create migrate_to_postgres.py with ID-preserving migration logic
- Create audit_sqlite.py for pre-migration data integrity checks
- Add PROJECT_PLAN.json for migration tracking
- Add .secrets/ to .gitignore for credentials

Audit results: 658,963 records across 29 tables, 2,390 orphaned stats (expected)

Based on Major Domo migration lessons learned (33 issues resolved there)
This commit is contained in:
Cal Corum 2026-01-25 23:05:54 -06:00
parent fbe8623eb4
commit 0cba52cea5
19 changed files with 7987 additions and 3388 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@ -74,4 +74,7 @@ CLAUDE.md
*.backup
# PostgreSQL data directory (local mount)
postgres_data/
postgres_data/
# PostgreSQL credentials
.secrets/

482
PROJECT_PLAN.json Normal file
View File

@ -0,0 +1,482 @@
{
"meta": {
"version": "1.1.0",
"created": "2026-01-25",
"lastUpdated": "2026-01-25",
"planType": "migration",
"description": "SQLite to PostgreSQL migration for Paper Dynasty database API",
"branch": "postgres-migration",
"totalEstimatedHours": 22,
"totalTasks": 16,
"completedTasks": 13
},
"context": {
"sourceDatabase": {
"type": "SQLite",
"file": "storage/pd_master.db",
"size": "110 MB",
"tables": 29,
"totalRecords": 515000,
"largestTable": {
"name": "stratplay",
"records": 332737
}
},
"targetDatabase": {
"type": "PostgreSQL 17",
"server": "sba_postgres (same server as Major Domo)",
"database": "pd_master",
"user": "pd_admin",
"credentialsFile": ".secrets/pd_admin_credentials.txt"
},
"lessonsFromMajorDomo": [
"CRITICAL: Primary key IDs must be explicitly preserved during migration",
"PostgreSQL GROUP BY requires ALL non-aggregated columns",
"Boolean fields cannot be summed directly - cast to integer first",
"Discord snowflake IDs must be strings, not integers (N/A for Paper Dynasty)",
"VARCHAR fields need explicit max_length",
"NULL constraints are stricter in PostgreSQL",
"Foreign key orphaned records need smart fallback handling",
"Reset sequences after ID-preserving inserts"
],
"devServer": {
"access": "ssh sba-db",
"composeLocation": "cd container-data/dev-sba-database/"
}
},
"categories": {
"critical": "Must complete before migration - blocks production",
"high": "Required for successful migration",
"medium": "Improves migration quality/reliability",
"low": "Polish and nice-to-have",
"completed": "Already done on postgres-migration branch"
},
"tasks": [
{
"id": "MIG-001",
"name": "Environment-based database configuration",
"description": "Add PostgreSQL support with environment variable switching between SQLite/PostgreSQL",
"category": "completed",
"priority": 1,
"completed": true,
"tested": true,
"dependencies": [],
"files": [
{
"path": "app/db_engine.py",
"lines": [11, 35],
"issue": "Now supports DATABASE_TYPE env var for SQLite/PostgreSQL switching"
}
],
"suggestedFix": "Already implemented with PooledPostgresqlDatabase",
"estimatedHours": 2,
"notes": "Includes connection pooling (20 max, 5-min stale timeout, autorollback)"
},
{
"id": "MIG-002",
"name": "Add table_name to all models",
"description": "Explicit table naming for PostgreSQL compatibility",
"category": "completed",
"priority": 2,
"completed": true,
"tested": true,
"dependencies": [],
"files": [
{
"path": "app/db_engine.py",
"lines": [],
"issue": "All 29 models now have Meta.table_name defined"
}
],
"suggestedFix": "Already implemented",
"estimatedHours": 1,
"notes": "Prevents Peewee naming inconsistencies"
},
{
"id": "MIG-003",
"name": "Fix GROUP BY queries for PostgreSQL",
"description": "PostgreSQL requires all non-aggregated SELECT fields in GROUP BY clause",
"category": "completed",
"priority": 3,
"completed": true,
"tested": false,
"dependencies": [],
"files": [
{
"path": "app/routers_v2/stratplays.py",
"lines": [342, 456, 645, 733],
"issue": "Conditionally build SELECT fields based on group_by parameter"
}
],
"suggestedFix": "Already implemented - needs testing with all group_by variations",
"estimatedHours": 4,
"notes": "Pattern: only include non-aggregated fields that will be in GROUP BY"
},
{
"id": "MIG-004",
"name": "Add psycopg2-binary dependency",
"description": "PostgreSQL adapter for Python",
"category": "completed",
"priority": 4,
"completed": true,
"tested": true,
"dependencies": [],
"files": [
{
"path": "requirements.txt",
"lines": [],
"issue": "psycopg2-binary added"
}
],
"suggestedFix": "Already implemented",
"estimatedHours": 0.1,
"notes": ""
},
{
"id": "MIG-005",
"name": "Docker Compose for local testing",
"description": "Local PostgreSQL environment for development testing",
"category": "completed",
"priority": 5,
"completed": true,
"tested": true,
"dependencies": [],
"files": [
{
"path": "docker-compose.yml",
"lines": [],
"issue": "PostgreSQL 17 + Adminer configured"
},
{
"path": "QUICK_START.md",
"lines": [],
"issue": "Testing guide created"
}
],
"suggestedFix": "Already implemented",
"estimatedHours": 1,
"notes": "Adminer on port 8081"
},
{
"id": "MIG-006",
"name": "Migration script auto-detection",
"description": "db_migrations.py auto-selects PostgresqlMigrator or SqliteMigrator",
"category": "completed",
"priority": 6,
"completed": true,
"tested": false,
"dependencies": ["MIG-001"],
"files": [
{
"path": "db_migrations.py",
"lines": [],
"issue": "Migrator selection based on DATABASE_TYPE"
}
],
"suggestedFix": "Already implemented",
"estimatedHours": 0.5,
"notes": ""
},
{
"id": "MIG-007",
"name": "Create data migration script with ID preservation",
"description": "CRITICAL: Migrate all data from SQLite to PostgreSQL while preserving primary key IDs exactly",
"category": "critical",
"priority": 1,
"completed": false,
"tested": false,
"dependencies": ["MIG-001", "MIG-002"],
"files": [
{
"path": "scripts/migrate_to_postgres.py",
"lines": [],
"issue": "New file - must explicitly insert IDs and reset sequences"
}
],
"suggestedFix": "1. Read all records from SQLite\n2. Insert into PostgreSQL with explicit ID values\n3. Reset PostgreSQL sequences: SELECT setval('table_id_seq', MAX(id))\n4. Validate record counts match\n5. Smart FK error handling (batch insert with individual fallback)",
"estimatedHours": 3,
"notes": "Major Domo's #1 lesson: Without explicit ID preservation, PostgreSQL auto-assigns sequential IDs starting from 1, causing all FK references to point to wrong records"
},
{
"id": "MIG-008",
"name": "Fix on_conflict_replace() calls (Player model)",
"description": "Convert SQLite on_conflict_replace() to PostgreSQL on_conflict() for Player model",
"category": "critical",
"priority": 2,
"completed": false,
"tested": false,
"dependencies": [],
"files": [
{
"path": "main.py",
"lines": [1696],
"issue": "Player.insert_many(batch).on_conflict_replace()"
},
{
"path": "app/routers_v2/players.py",
"lines": [808],
"issue": "Player.insert_many(batch).on_conflict_replace()"
}
],
"suggestedFix": "Player.insert_many(batch).on_conflict(\n conflict_target=[Player.player_id],\n action='update',\n update={Player.p_name: EXCLUDED.p_name, ...all fields}\n).execute()",
"estimatedHours": 0.5,
"notes": "Player has explicit player_id primary key - straightforward"
},
{
"id": "MIG-009",
"name": "Fix on_conflict_replace() calls (Card models)",
"description": "Convert SQLite on_conflict_replace() for BattingCard, PitchingCard, CardPosition, ratings",
"category": "critical",
"priority": 3,
"completed": false,
"tested": false,
"dependencies": [],
"files": [
{
"path": "app/routers_v2/battingcards.py",
"lines": [134],
"issue": "BattingCard - unique on (player, variant)"
},
{
"path": "app/routers_v2/pitchingcards.py",
"lines": [130],
"issue": "PitchingCard - unique on (player, variant)"
},
{
"path": "app/routers_v2/cardpositions.py",
"lines": [131],
"issue": "CardPosition - unique on (player, variant, position)"
},
{
"path": "app/routers_v2/battingcardratings.py",
"lines": [549],
"issue": "BattingCardRatings - unique on (battingcard, vs_hand)"
},
{
"path": "app/routers_v2/pitchingcardratings.py",
"lines": [432],
"issue": "PitchingCardRatings - unique on (pitchingcard, vs_hand)"
}
],
"suggestedFix": "All have existing unique indexes - use those as conflict_target",
"estimatedHours": 2,
"notes": "These have many fields to update - consider helper function"
},
{
"id": "MIG-010",
"name": "Fix on_conflict_replace() calls (Game models)",
"description": "Convert SQLite on_conflict_replace() for StratPlay, Decision, GauntletReward",
"category": "critical",
"priority": 4,
"completed": false,
"tested": false,
"dependencies": ["MIG-011"],
"files": [
{
"path": "app/routers_v2/stratplays.py",
"lines": [1082],
"issue": "StratPlay - needs unique index on (game, play_num)"
},
{
"path": "app/routers_v2/decisions.py",
"lines": [217],
"issue": "Decision - needs unique index on (game, pitcher)"
},
{
"path": "main.py",
"lines": [4978],
"issue": "GauntletReward - investigate if id provided or needs refactor"
},
{
"path": "app/routers_v2/gauntletrewards.py",
"lines": [127],
"issue": "GauntletReward - same as main.py"
}
],
"suggestedFix": "Add unique indexes first (MIG-011), then implement on_conflict()",
"estimatedHours": 1.5,
"notes": "StratPlay and Decision need new unique indexes to be created first"
},
{
"id": "MIG-011",
"name": "Add missing unique indexes for upserts",
"description": "Create unique indexes needed for PostgreSQL on_conflict() operations",
"category": "high",
"priority": 5,
"completed": false,
"tested": false,
"dependencies": [],
"files": [
{
"path": "app/db_engine.py",
"lines": [779, 848],
"issue": "Add unique indexes for StratPlay and Decision"
}
],
"suggestedFix": "StratPlay: ModelIndex(StratPlay, (StratPlay.game, StratPlay.play_num), unique=True)\nDecision: ModelIndex(Decision, (Decision.game, Decision.pitcher), unique=True)",
"estimatedHours": 1,
"notes": "These are natural business keys - a play number should be unique within a game, and a pitcher should have one decision per game"
},
{
"id": "MIG-012",
"name": "Fix on_conflict_replace() for MlbPlayer",
"description": "Convert or remove on_conflict_replace() for MlbPlayer",
"category": "medium",
"priority": 6,
"completed": false,
"tested": false,
"dependencies": [],
"files": [
{
"path": "app/routers_v2/mlbplayers.py",
"lines": [185],
"issue": "MlbPlayer.insert_many(batch).on_conflict_replace()"
}
],
"suggestedFix": "Code already checks for duplicates before insert (lines 170-179) and raises HTTPException. The on_conflict_replace() may be unnecessary. Option 1: Remove it and use plain insert_many(). Option 2: Use on_conflict with id as target.",
"estimatedHours": 0.25,
"notes": "Low risk - pre-check rejects duplicates"
},
{
"id": "MIG-013",
"name": "Fix boolean comparison in teams.py",
"description": "PostgreSQL requires True/False instead of 1/0 for boolean comparisons",
"category": "low",
"priority": 7,
"completed": false,
"tested": false,
"dependencies": [],
"files": [
{
"path": "app/routers_v2/teams.py",
"lines": [110, 112],
"issue": "Team.has_guide == 0 / Team.has_guide == 1"
}
],
"suggestedFix": "Change to Team.has_guide == False / Team.has_guide == True",
"estimatedHours": 0.25,
"notes": "Peewee may handle this automatically, but explicit is better"
},
{
"id": "MIG-014",
"name": "SQLite data integrity audit",
"description": "Check for NULL values, orphaned FKs, VARCHAR lengths before migration",
"category": "high",
"priority": 8,
"completed": false,
"tested": false,
"dependencies": [],
"files": [
{
"path": "scripts/audit_sqlite.py",
"lines": [],
"issue": "New file - pre-migration data validation"
}
],
"suggestedFix": "Create script to check:\n1. NULL values in NOT NULL fields\n2. Orphaned foreign key records\n3. VARCHAR field max lengths\n4. Table record counts for baseline",
"estimatedHours": 1.5,
"notes": "Major Domo found 206 orphaned decisions and VARCHAR violations"
},
{
"id": "MIG-015",
"name": "Test on dev PostgreSQL server",
"description": "Full migration test on sba-db dev server with production data copy",
"category": "high",
"priority": 9,
"completed": false,
"tested": false,
"dependencies": ["MIG-007", "MIG-008", "MIG-009", "MIG-010", "MIG-014"],
"files": [],
"suggestedFix": "1. ssh sba-db\n2. Create pd_master database with pd_admin user\n3. Copy production SQLite to dev\n4. Run migration script\n5. Verify record counts\n6. Test API endpoints",
"estimatedHours": 3,
"notes": "Dev server access: ssh sba-db, then cd container-data/dev-sba-database/"
},
{
"id": "MIG-016",
"name": "Production migration execution",
"description": "Execute migration on production server within maintenance window",
"category": "critical",
"priority": 10,
"completed": false,
"tested": false,
"dependencies": ["MIG-015"],
"files": [],
"suggestedFix": "1. Notify users of maintenance window\n2. Stop Paper Dynasty API\n3. Create SQLite backup\n4. Create pd_master database and pd_admin user\n5. Run migration script\n6. Verify data integrity\n7. Update docker-compose.yml with PostgreSQL env vars\n8. Start API\n9. Smoke test critical endpoints\n10. Announce migration complete",
"estimatedHours": 3,
"notes": "Downtime window: 1-4 hours. Have rollback plan ready."
}
],
"quickWins": [
{
"taskId": "MIG-013",
"estimatedMinutes": 15,
"impact": "Prevents boolean comparison issues in team queries"
},
{
"taskId": "MIG-012",
"estimatedMinutes": 15,
"impact": "Simplify MlbPlayer insert logic"
}
],
"productionBlockers": [
{
"taskId": "MIG-007",
"reason": "Without ID-preserving migration, all foreign key references will break"
},
{
"taskId": "MIG-008",
"reason": "Player upserts will fail without PostgreSQL-compatible syntax"
},
{
"taskId": "MIG-009",
"reason": "Card data upserts will fail without PostgreSQL-compatible syntax"
},
{
"taskId": "MIG-010",
"reason": "Game data upserts will fail without PostgreSQL-compatible syntax"
}
],
"weeklyRoadmap": {
"week1": {
"theme": "Code Changes - Make PostgreSQL Compatible",
"tasks": ["MIG-007", "MIG-008", "MIG-009", "MIG-010", "MIG-011", "MIG-012", "MIG-013"],
"estimatedHours": 8.5
},
"week2": {
"theme": "Testing & Validation",
"tasks": ["MIG-014", "MIG-015"],
"estimatedHours": 4.5
},
"week3": {
"theme": "Production Migration",
"tasks": ["MIG-016"],
"estimatedHours": 3
}
},
"rollbackPlan": {
"triggers": [
"Data corruption detected",
"More than 5% of endpoints failing",
"Performance more than 5x worse than SQLite",
"Critical functionality broken"
],
"duringTesting": {
"steps": [
"Set DATABASE_TYPE=sqlite",
"API immediately uses SQLite",
"No data loss - PostgreSQL was a copy"
]
},
"afterProduction": {
"steps": [
"Stop API: docker-compose down",
"Update docker-compose.yml: DATABASE_TYPE=sqlite",
"Restore SQLite backup if needed",
"Start API: docker-compose up -d",
"Verify SQLite connectivity",
"Document issues for retry"
],
"timeLimit": "24 hours from migration"
}
}
}

328
app/db_engine.py
View File

@ -10,39 +10,36 @@ from peewee import ModelSelect
from playhouse.shortcuts import model_to_dict
# Database configuration - supports both SQLite and PostgreSQL
DATABASE_TYPE = os.environ.get('DATABASE_TYPE', 'sqlite')
DATABASE_TYPE = os.environ.get("DATABASE_TYPE", "sqlite")
if DATABASE_TYPE.lower() == 'postgresql':
if DATABASE_TYPE.lower() == "postgresql":
from playhouse.pool import PooledPostgresqlDatabase
db = PooledPostgresqlDatabase(
os.environ.get('POSTGRES_DB', 'pd_master'),
user=os.environ.get('POSTGRES_USER', 'pd_admin'),
password=os.environ.get('POSTGRES_PASSWORD'),
host=os.environ.get('POSTGRES_HOST', 'localhost'),
port=int(os.environ.get('POSTGRES_PORT', '5432')),
os.environ.get("POSTGRES_DB", "pd_master"),
user=os.environ.get("POSTGRES_USER", "pd_admin"),
password=os.environ.get("POSTGRES_PASSWORD"),
host=os.environ.get("POSTGRES_HOST", "localhost"),
port=int(os.environ.get("POSTGRES_PORT", "5432")),
max_connections=20,
stale_timeout=300, # 5 minutes
timeout=0,
autoconnect=True,
autorollback=True # Automatically rollback failed transactions
autorollback=True, # Automatically rollback failed transactions
)
else:
# Default SQLite configuration for local development
db = SqliteDatabase(
'storage/pd_master.db',
pragmas={
'journal_mode': 'wal',
'cache_size': -1 * 64000,
'synchronous': 0
}
"storage/pd_master.db",
pragmas={"journal_mode": "wal", "cache_size": -1 * 64000, "synchronous": 0},
)
date = f'{datetime.now().year}-{datetime.now().month}-{datetime.now().day}'
log_level = logging.INFO if os.environ.get('LOG_LEVEL') == 'INFO' else 'WARN'
date = f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}"
log_level = logging.INFO if os.environ.get("LOG_LEVEL") == "INFO" else "WARN"
logging.basicConfig(
filename=f'logs/database/{date}.log',
format='%(asctime)s - database - %(levelname)s - %(message)s',
level=log_level
filename=f"logs/database/{date}.log",
format="%(asctime)s - database - %(levelname)s - %(message)s",
level=log_level,
)
# 2025, 2005
@ -50,63 +47,66 @@ ranked_cardsets = [24, 25, 26, 27, 28, 29]
LIVE_CARDSET_ID = 27
LIVE_PROMO_CARDSET_ID = 28
CARDSETS = {
'ranked': {
'primary': ranked_cardsets,
'human': ranked_cardsets
"ranked": {"primary": ranked_cardsets, "human": ranked_cardsets},
"minor-league": {
"primary": [27, 8], # 2005, Mario
"secondary": [24], # 2025
"human": [x for x in range(1, 30)],
},
'minor-league': {
'primary': [27, 8], # 2005, Mario
'secondary': [24], # 2025
'human': [x for x in range(1, 30)]
"major-league": {
"primary": [
27,
28,
24,
25,
13,
14,
6,
8,
], # 2005 + Promos, 2025 + Promos, 2018 + Promos, 2012, Mario
"secondary": [5, 3], # 2019, 2022
"human": ranked_cardsets,
},
'major-league': {
'primary': [27, 28, 24, 25, 13, 14, 6, 8], # 2005 + Promos, 2025 + Promos, 2018 + Promos, 2012, Mario
'secondary': [5, 3], # 2019, 2022
'human': ranked_cardsets
"hall-of-fame": {"primary": [x for x in range(1, 30)], "human": ranked_cardsets},
"flashback": {
"primary": [13, 5, 1, 3, 8], # 2018, 2019, 2021, 2022, Mario
"secondary": [24], # 2025
"human": [13, 5, 1, 3, 8], # 2018, 2019, 2021, 2022
},
'hall-of-fame': {
'primary': [x for x in range(1, 30)],
'human': ranked_cardsets
"gauntlet-3": {
"primary": [13], # 2018
"secondary": [5, 11, 9], # 2019, 2016, 2023
"human": [x for x in range(1, 30)],
},
'flashback': {
'primary': [13, 5, 1, 3, 8], # 2018, 2019, 2021, 2022, Mario
'secondary': [24], # 2025
'human': [13, 5, 1, 3, 8] # 2018, 2019, 2021, 2022
"gauntlet-4": {
"primary": [3, 6, 16], # 2022, 2013, Backyard Baseball
"secondary": [4, 9], # 2022 Promos, 2023
"human": [3, 4, 6, 9, 15, 16],
},
'gauntlet-3': {
'primary': [13], # 2018
'secondary': [5, 11, 9], # 2019, 2016, 2023
'human': [x for x in range(1, 30)]
"gauntlet-5": {
"primary": [17, 8], # 2024, Mario
"secondary": [13], # 2018
"human": [x for x in range(1, 30)],
},
'gauntlet-4': {
'primary': [3, 6, 16], # 2022, 2013, Backyard Baseball
'secondary': [4, 9], # 2022 Promos, 2023
'human': [3, 4, 6, 9, 15, 16]
"gauntlet-6": {
"primary": [20, 8], # 1998, Mario
"secondary": [12], # 2008
"human": [x for x in range(1, 30)],
},
'gauntlet-5': {
'primary': [17, 8], # 2024, Mario
'secondary': [13], # 2018
'human': [x for x in range(1, 30)]
"gauntlet-7": {
"primary": [5, 23], # 2019, Brilliant Stars
"secondary": [1], # 2021
"human": [x for x in range(1, 30)],
},
'gauntlet-6': {
'primary': [20, 8], # 1998, Mario
'secondary': [12], # 2008
'human': [x for x in range(1, 30)]
"gauntlet-8": {
"primary": [24], # 2025
"secondary": [17],
"human": [24, 25, 22, 23],
},
'gauntlet-7': {
'primary': [5, 23], # 2019, Brilliant Stars
'secondary': [1], # 2021
'human': [x for x in range(1, 30)]
"gauntlet-9": {
"primary": [27], # 2005
"secondary": [24], # 2025
},
'gauntlet-8': {
'primary': [24], # 2025
'secondary': [17],
'human': [24, 25, 22, 23]
},
'gauntlet-9': {
'primary': [27], # 2005
'secondary': [24] # 2025
}
}
@ -122,7 +122,7 @@ def model_to_csv(this_obj, exclude=None) -> List:
def query_to_csv(all_items: ModelSelect, exclude=None):
if all_items.count() == 0:
data_list = [['No data found']]
data_list = [["No data found"]]
else:
data_list = [model_csv_headers(all_items[0], exclude=exclude)]
for x in all_items:
@ -133,29 +133,29 @@ def query_to_csv(all_items: ModelSelect, exclude=None):
def complex_data_to_csv(complex_data: List):
if len(complex_data) == 0:
data_list = [['No data found']]
data_list = [["No data found"]]
else:
data_list = [[x for x in complex_data[0].keys()]]
for line in complex_data:
logging.debug(f'line: {line}')
logging.debug(f"line: {line}")
this_row = []
for key in line:
logging.debug(f'key: {key}')
logging.debug(f"key: {key}")
if line[key] is None:
this_row.append('')
this_row.append("")
elif isinstance(line[key], dict):
if 'name' in line[key]:
this_row.append(line[key]['name'])
elif 'abbrev' in line[key]:
this_row.append(line[key]['abbrev'])
if "name" in line[key]:
this_row.append(line[key]["name"])
elif "abbrev" in line[key]:
this_row.append(line[key]["abbrev"])
else:
this_row.append(line[key]['id'])
this_row.append(line[key]["id"])
elif isinstance(line[key], int) and line[key] > 100000000:
this_row.append(f"'{line[key]}")
elif isinstance(line[key], str) and ',' in line[key]:
elif isinstance(line[key], str) and "," in line[key]:
this_row.append(line[key].replace(",", "-_-"))
else:
@ -180,7 +180,7 @@ class Current(BaseModel):
class Meta:
database = db
table_name = 'current'
table_name = "current"
@staticmethod
def latest():
@ -198,7 +198,7 @@ class Rarity(BaseModel):
class Meta:
database = db
table_name = 'rarity'
table_name = "rarity"
def __str__(self):
return self.name
@ -217,7 +217,7 @@ class Event(BaseModel):
class Meta:
database = db
table_name = 'event'
table_name = "event"
db.create_tables([Event])
@ -234,7 +234,7 @@ class Cardset(BaseModel):
class Meta:
database = db
table_name = 'cardset'
table_name = "cardset"
def __str__(self):
return self.name
@ -254,7 +254,7 @@ class MlbPlayer(BaseModel):
class Meta:
database = db
table_name = 'mlbplayer'
table_name = "mlbplayer"
db.create_tables([MlbPlayer])
@ -289,7 +289,7 @@ class Player(BaseModel):
mlbplayer = ForeignKeyField(MlbPlayer, null=True)
def __str__(self):
return f'{self.cardset} {self.p_name} ({self.rarity.name})'
return f"{self.cardset} {self.p_name} ({self.rarity.name})"
# def __eq__(self, other):
# if self.cardset.id == other.cardset.id and self.name == other.name:
@ -310,21 +310,21 @@ class Player(BaseModel):
def get_all_pos(self):
all_pos = []
if self.pos_1 and self.pos_1 != 'CP':
if self.pos_1 and self.pos_1 != "CP":
all_pos.append(self.pos_1)
if self.pos_2 and self.pos_2 != 'CP':
if self.pos_2 and self.pos_2 != "CP":
all_pos.append(self.pos_2)
if self.pos_3 and self.pos_3 != 'CP':
if self.pos_3 and self.pos_3 != "CP":
all_pos.append(self.pos_3)
if self.pos_4 and self.pos_4 != 'CP':
if self.pos_4 and self.pos_4 != "CP":
all_pos.append(self.pos_4)
if self.pos_5 and self.pos_5 != 'CP':
if self.pos_5 and self.pos_5 != "CP":
all_pos.append(self.pos_5)
if self.pos_6 and self.pos_6 != 'CP':
if self.pos_6 and self.pos_6 != "CP":
all_pos.append(self.pos_6)
if self.pos_7 and self.pos_7 != 'CP':
if self.pos_7 and self.pos_7 != "CP":
all_pos.append(self.pos_7)
if self.pos_8 and self.pos_8 != 'CP':
if self.pos_8 and self.pos_8 != "CP":
all_pos.append(self.pos_8)
return all_pos
@ -338,33 +338,33 @@ class Player(BaseModel):
# 'mvp': 2500,
# 'hof': 999999999
# }
logging.info(f'{self.p_name} cost changing from: {self.cost}')
self.cost = max(math.floor(self.cost * .95), 1)
logging.info(f"{self.p_name} cost changing from: {self.cost}")
self.cost = max(math.floor(self.cost * 0.95), 1)
# if self.quantity != 999:
# self.quantity += 1
logging.info(f'{self.p_name} cost now: {self.cost}')
logging.info(f"{self.p_name} cost now: {self.cost}")
self.save()
def change_on_buy(self):
logging.info(f'{self.p_name} cost changing from: {self.cost}')
logging.info(f"{self.p_name} cost changing from: {self.cost}")
self.cost = math.ceil(self.cost * 1.1)
# if self.quantity != 999:
# self.quantity -= 1
logging.info(f'{self.p_name} cost now: {self.cost}')
logging.info(f"{self.p_name} cost now: {self.cost}")
self.save()
class Meta:
database = db
table_name = 'player'
table_name = "player"
db.create_tables([Player])
class Team(BaseModel):
abbrev = CharField()
sname = CharField()
lname = CharField()
abbrev = CharField(max_length=20) # Gauntlet teams use prefixes like "Gauntlet-NCB"
sname = CharField(max_length=100)
lname = CharField(max_length=255)
gmid = IntegerField()
gmname = CharField()
gsheet = CharField()
@ -381,7 +381,7 @@ class Team(BaseModel):
is_ai = IntegerField(null=True)
def __str__(self):
return f'S{self.season} {self.lname}'
return f"S{self.season} {self.lname}"
@staticmethod
def get_by_owner(gmid, season=None):
@ -407,13 +407,13 @@ class Team(BaseModel):
return Team.get_or_none(Team.season == season, Team.abbrev == abbrev.upper())
def team_hash(self):
hash_string = f'{self.sname[-1]}{self.gmid / 6950123:.0f}{self.sname[-2]}{self.gmid / 42069123:.0f}'
logging.info(f'string: {hash_string}')
hash_string = f"{self.sname[-1]}{self.gmid / 6950123:.0f}{self.sname[-2]}{self.gmid / 42069123:.0f}"
logging.info(f"string: {hash_string}")
return hash_string
class Meta:
database = db
table_name = 'team'
table_name = "team"
db.create_tables([Team])
@ -428,7 +428,7 @@ class PackType(BaseModel):
class Meta:
database = db
table_name = 'packtype'
table_name = "packtype"
db.create_tables([PackType])
@ -443,7 +443,7 @@ class Pack(BaseModel):
class Meta:
database = db
table_name = 'pack'
table_name = "pack"
db.create_tables([Pack])
@ -457,9 +457,9 @@ class Card(BaseModel):
def __str__(self):
if self.player:
return f'{self.player} - {self.team.sname}'
return f"{self.player} - {self.team.sname}"
else:
return f'Blank - {self.team.sname}'
return f"Blank - {self.team.sname}"
@staticmethod
def select_season(season):
@ -467,7 +467,7 @@ class Card(BaseModel):
class Meta:
database = db
table_name = 'card'
table_name = "card"
db.create_tables([Card])
@ -505,7 +505,7 @@ class Roster(BaseModel):
card_26 = ForeignKeyField(Card)
def __str__(self):
return f'{self.team} Roster'
return f"{self.team} Roster"
# def get_cards(self, team):
# all_cards = Card.select().where(Card.roster == self)
@ -519,7 +519,7 @@ class Roster(BaseModel):
class Meta:
database = db
table_name = 'roster'
table_name = "roster"
class Result(BaseModel):
@ -546,7 +546,7 @@ class Result(BaseModel):
class Meta:
database = db
table_name = 'result'
table_name = "result"
class BattingStat(BaseModel):
@ -589,7 +589,7 @@ class BattingStat(BaseModel):
class Meta:
database = db
table_name = 'battingstat'
table_name = "battingstat"
class PitchingStat(BaseModel):
@ -623,7 +623,7 @@ class PitchingStat(BaseModel):
class Meta:
database = db
table_name = 'pitchingstat'
table_name = "pitchingstat"
class Award(BaseModel):
@ -636,17 +636,17 @@ class Award(BaseModel):
class Meta:
database = db
table_name = 'award'
table_name = "award"
class Paperdex(BaseModel):
team = ForeignKeyField(Team)
player = ForeignKeyField(Player)
created = DateTimeField(default=int(datetime.timestamp(datetime.now())*1000))
created = DateTimeField(default=int(datetime.timestamp(datetime.now()) * 1000))
class Meta:
database = db
table_name = 'paperdex'
table_name = "paperdex"
# def add_to_paperdex(self, team, cards: list):
# for x in players:
@ -665,7 +665,7 @@ class Reward(BaseModel):
class Meta:
database = db
table_name = 'reward'
table_name = "reward"
class GameRewards(BaseModel):
@ -676,7 +676,7 @@ class GameRewards(BaseModel):
class Meta:
database = db
table_name = 'gamerewards'
table_name = "gamerewards"
class Notification(BaseModel):
@ -690,7 +690,7 @@ class Notification(BaseModel):
class Meta:
database = db
table_name = 'notification'
table_name = "notification"
class GauntletReward(BaseModel):
@ -702,7 +702,7 @@ class GauntletReward(BaseModel):
class Meta:
database = db
table_name = 'gauntletreward'
table_name = "gauntletreward"
class GauntletRun(BaseModel):
@ -711,18 +711,29 @@ class GauntletRun(BaseModel):
wins = IntegerField(default=0)
losses = IntegerField(default=0)
gsheet = CharField(null=True)
created = DateTimeField(default=int(datetime.timestamp(datetime.now())*1000))
created = DateTimeField(default=int(datetime.timestamp(datetime.now()) * 1000))
ended = DateTimeField(default=0)
class Meta:
database = db
table_name = 'gauntletrun'
table_name = "gauntletrun"
db.create_tables([
Roster, BattingStat, PitchingStat, Result, Award, Paperdex, Reward, GameRewards, Notification, GauntletReward,
GauntletRun
])
db.create_tables(
[
Roster,
BattingStat,
PitchingStat,
Result,
Award,
Paperdex,
Reward,
GameRewards,
Notification,
GauntletReward,
GauntletRun,
]
)
class BattingCard(BaseModel):
@ -736,20 +747,22 @@ class BattingCard(BaseModel):
hit_and_run = CharField()
running = IntegerField()
offense_col = IntegerField()
hand = CharField(default='R')
hand = CharField(default="R")
class Meta:
database = db
table_name = 'battingcard'
table_name = "battingcard"
bc_index = ModelIndex(BattingCard, (BattingCard.player, BattingCard.variant), unique=True)
bc_index = ModelIndex(
BattingCard, (BattingCard.player, BattingCard.variant), unique=True
)
BattingCard.add_index(bc_index)
class BattingCardRatings(BaseModel):
battingcard = ForeignKeyField(BattingCard)
vs_hand = CharField(default='R')
vs_hand = CharField(default="R")
pull_rate = FloatField()
center_rate = FloatField()
slap_rate = FloatField()
@ -781,11 +794,13 @@ class BattingCardRatings(BaseModel):
class Meta:
database = db
table_name = 'battingcardratings'
table_name = "battingcardratings"
bcr_index = ModelIndex(
BattingCardRatings, (BattingCardRatings.battingcard, BattingCardRatings.vs_hand), unique=True
BattingCardRatings,
(BattingCardRatings.battingcard, BattingCardRatings.vs_hand),
unique=True,
)
BattingCardRatings.add_index(bcr_index)
@ -801,20 +816,22 @@ class PitchingCard(BaseModel):
closer_rating = IntegerField(null=True)
batting = CharField(null=True)
offense_col = IntegerField()
hand = CharField(default='R')
hand = CharField(default="R")
class Meta:
database = db
table_name = 'pitchingcard'
table_name = "pitchingcard"
pc_index = ModelIndex(PitchingCard, (PitchingCard.player, PitchingCard.variant), unique=True)
pc_index = ModelIndex(
PitchingCard, (PitchingCard.player, PitchingCard.variant), unique=True
)
PitchingCard.add_index(pc_index)
class PitchingCardRatings(BaseModel):
pitchingcard = ForeignKeyField(PitchingCard)
vs_hand = CharField(default='R')
vs_hand = CharField(default="R")
homerun = FloatField()
bp_homerun = FloatField()
triple = FloatField()
@ -848,11 +865,13 @@ class PitchingCardRatings(BaseModel):
class Meta:
database = db
table_name = 'pitchingcardratings'
table_name = "pitchingcardratings"
pcr_index = ModelIndex(
PitchingCardRatings, (PitchingCardRatings.pitchingcard, PitchingCardRatings.vs_hand), unique=True
PitchingCardRatings,
(PitchingCardRatings.pitchingcard, PitchingCardRatings.vs_hand),
unique=True,
)
PitchingCardRatings.add_index(pcr_index)
@ -870,16 +889,20 @@ class CardPosition(BaseModel):
class Meta:
database = db
table_name = 'cardposition'
table_name = "cardposition"
pos_index = ModelIndex(
CardPosition, (CardPosition.player, CardPosition.variant, CardPosition.position), unique=True
CardPosition,
(CardPosition.player, CardPosition.variant, CardPosition.position),
unique=True,
)
CardPosition.add_index(pos_index)
db.create_tables([BattingCard, BattingCardRatings, PitchingCard, PitchingCardRatings, CardPosition])
db.create_tables(
[BattingCard, BattingCardRatings, PitchingCard, PitchingCardRatings, CardPosition]
)
class StratGame(BaseModel):
@ -900,7 +923,7 @@ class StratGame(BaseModel):
class Meta:
database = db
table_name = 'stratgame'
table_name = "stratgame"
class StratPlay(BaseModel):
@ -973,7 +996,15 @@ class StratPlay(BaseModel):
class Meta:
database = db
table_name = 'stratplay'
table_name = "stratplay"
# Unique index for StratPlay - a play number should be unique within a game
# Required for PostgreSQL on_conflict() upsert operations
stratplay_index = ModelIndex(
StratPlay, (StratPlay.game, StratPlay.play_num), unique=True
)
StratPlay.add_index(stratplay_index)
class Decision(BaseModel):
@ -995,7 +1026,13 @@ class Decision(BaseModel):
class Meta:
database = db
table_name = 'decision'
table_name = "decision"
# Unique index for Decision - one decision per pitcher per game
# Required for PostgreSQL on_conflict() upsert operations
decision_index = ModelIndex(Decision, (Decision.game, Decision.pitcher), unique=True)
Decision.add_index(decision_index)
db.create_tables([StratGame, StratPlay, Decision])
@ -1176,4 +1213,3 @@ db.close()
#
#
# scout_db.close()

284
app/db_helpers.py Normal file
View File

@ -0,0 +1,284 @@
"""
Database helper functions for PostgreSQL compatibility.
This module provides cross-database compatible upsert operations that work
with both SQLite and PostgreSQL.
The key difference:
- SQLite: .on_conflict_replace() works directly
- PostgreSQL: Requires .on_conflict() with explicit conflict_target and update dict
Usage:
from app.db_helpers import upsert_many, DATABASE_TYPE
# Instead of:
Model.insert_many(batch).on_conflict_replace().execute()
# Use:
upsert_many(Model, batch, conflict_fields=['field1', 'field2'])
"""
import os
from typing import Any, Dict, List, Type, Union
from peewee import Model, SQL
# Re-export DATABASE_TYPE for convenience
DATABASE_TYPE = os.environ.get("DATABASE_TYPE", "sqlite").lower()
def get_model_fields(model: Type[Model], exclude: List[str] = None) -> List[str]:
"""
Get all field names for a model, excluding specified fields.
Args:
model: Peewee Model class
exclude: Field names to exclude (e.g., primary key)
Returns:
List of field names
"""
if exclude is None:
exclude = []
return [
field.name for field in model._meta.sorted_fields if field.name not in exclude
]
def upsert_many(
model: Type[Model],
data: List[Dict[str, Any]],
conflict_fields: List[str],
update_fields: List[str] = None,
batch_size: int = 100,
) -> int:
"""
Insert or update multiple records in a database-agnostic way.
Works with both SQLite (on_conflict_replace) and PostgreSQL (on_conflict).
Args:
model: Peewee Model class
data: List of dictionaries with field values
conflict_fields: Fields that define uniqueness (for PostgreSQL ON CONFLICT)
update_fields: Fields to update on conflict (defaults to all non-conflict fields)
batch_size: Number of records per batch
Returns:
Number of records processed
Example:
# For BattingCard with unique constraint on (player, variant)
upsert_many(
BattingCard,
batch_data,
conflict_fields=['player', 'variant']
)
"""
if not data:
return 0
total = 0
# Determine update fields if not specified
if update_fields is None:
# Get primary key name
pk_name = model._meta.primary_key.name if model._meta.primary_key else "id"
# Update all fields except PK and conflict fields
exclude = [pk_name] + conflict_fields
update_fields = get_model_fields(model, exclude=exclude)
# Process in batches
for i in range(0, len(data), batch_size):
batch = data[i : i + batch_size]
if DATABASE_TYPE == "postgresql":
# PostgreSQL: Use ON CONFLICT with explicit target and update
from peewee import EXCLUDED
# Build conflict target - get actual field objects
conflict_target = [getattr(model, f) for f in conflict_fields]
# Build update dict
update_dict = {
getattr(model, f): EXCLUDED[f]
for f in update_fields
if hasattr(model, f)
}
if update_dict:
model.insert_many(batch).on_conflict(
conflict_target=conflict_target, action="update", update=update_dict
).execute()
else:
# No fields to update, just ignore conflicts
model.insert_many(batch).on_conflict_ignore().execute()
else:
# SQLite: Use on_conflict_replace (simpler)
model.insert_many(batch).on_conflict_replace().execute()
total += len(batch)
return total
def upsert_by_pk(
model: Type[Model],
data: List[Dict[str, Any]],
pk_field: str = None,
batch_size: int = 100,
) -> int:
"""
Upsert records using primary key as conflict target.
This is for models where the primary key is explicitly provided in the data
(like Player with player_id).
Args:
model: Peewee Model class
data: List of dictionaries with field values (including PK)
pk_field: Primary key field name (auto-detected if not specified)
batch_size: Number of records per batch
Returns:
Number of records processed
Example:
# For Player with explicit player_id
upsert_by_pk(Player, player_data, pk_field='player_id')
"""
if not data:
return 0
# Auto-detect primary key
if pk_field is None:
pk_field = model._meta.primary_key.name if model._meta.primary_key else "id"
return upsert_many(model, data, conflict_fields=[pk_field], batch_size=batch_size)
# Pre-configured upsert functions for specific models
# These encode the unique constraint knowledge for each model
def upsert_players(data: List[Dict], batch_size: int = 15) -> int:
"""Upsert Player records using player_id as conflict target."""
from app.db_engine import Player
return upsert_by_pk(Player, data, pk_field="player_id", batch_size=batch_size)
def upsert_batting_cards(data: List[Dict], batch_size: int = 30) -> int:
"""Upsert BattingCard records using (player, variant) unique constraint."""
from app.db_engine import BattingCard
return upsert_many(
BattingCard, data, conflict_fields=["player", "variant"], batch_size=batch_size
)
def upsert_pitching_cards(data: List[Dict], batch_size: int = 30) -> int:
"""Upsert PitchingCard records using (player, variant) unique constraint."""
from app.db_engine import PitchingCard
return upsert_many(
PitchingCard, data, conflict_fields=["player", "variant"], batch_size=batch_size
)
def upsert_batting_card_ratings(data: List[Dict], batch_size: int = 30) -> int:
"""Upsert BattingCardRatings using (battingcard, vs_hand) unique constraint."""
from app.db_engine import BattingCardRatings
return upsert_many(
BattingCardRatings,
data,
conflict_fields=["battingcard", "vs_hand"],
batch_size=batch_size,
)
def upsert_pitching_card_ratings(data: List[Dict], batch_size: int = 30) -> int:
"""Upsert PitchingCardRatings using (pitchingcard, vs_hand) unique constraint."""
from app.db_engine import PitchingCardRatings
return upsert_many(
PitchingCardRatings,
data,
conflict_fields=["pitchingcard", "vs_hand"],
batch_size=batch_size,
)
def upsert_card_positions(data: List[Dict], batch_size: int = 30) -> int:
"""Upsert CardPosition using (player, variant, position) unique constraint."""
from app.db_engine import CardPosition
return upsert_many(
CardPosition,
data,
conflict_fields=["player", "variant", "position"],
batch_size=batch_size,
)
def upsert_strat_plays(data: List[Dict], batch_size: int = 20) -> int:
"""Upsert StratPlay using (game, play_num) unique constraint."""
from app.db_engine import StratPlay
return upsert_many(
StratPlay, data, conflict_fields=["game", "play_num"], batch_size=batch_size
)
def upsert_decisions(data: List[Dict], batch_size: int = 10) -> int:
"""Upsert Decision using (game, pitcher) unique constraint."""
from app.db_engine import Decision
return upsert_many(
Decision, data, conflict_fields=["game", "pitcher"], batch_size=batch_size
)
def upsert_gauntlet_rewards(data: List[Dict], batch_size: int = 15) -> int:
"""
Upsert GauntletReward records.
Note: GauntletReward doesn't have a natural unique key defined.
For PostgreSQL, we use id if provided, otherwise insert-only.
"""
from app.db_engine import GauntletReward
# Check if any records have 'id' field
has_ids = any("id" in record for record in data)
if has_ids:
return upsert_by_pk(GauntletReward, data, pk_field="id", batch_size=batch_size)
else:
# No IDs provided - just insert (may fail on duplicates)
total = 0
for i in range(0, len(data), batch_size):
batch = data[i : i + batch_size]
GauntletReward.insert_many(batch).execute()
total += len(batch)
return total
def upsert_mlb_players(data: List[Dict], batch_size: int = 15) -> int:
"""
Upsert MlbPlayer records.
Note: The calling code already checks for duplicates before insert,
so this is effectively just an insert operation.
"""
from app.db_engine import MlbPlayer
# MlbPlayer doesn't have a good unique key other than id
# Since duplicates are already checked, just insert
total = 0
for i in range(0, len(data), batch_size):
batch = data[i : i + batch_size]
MlbPlayer.insert_many(batch).execute()
total += len(batch)
return total

660
app/routers_v2/battingcardratings.py
View File

@ -9,27 +9,34 @@ import pandas as pd
import pydantic
from pydantic import validator, root_validator
from ..db_engine import db, BattingCardRatings, model_to_dict, chunked, BattingCard, Player, query_to_csv, Team, \
CardPosition
from ..db_engine import (
db,
BattingCardRatings,
model_to_dict,
chunked,
BattingCard,
Player,
query_to_csv,
Team,
CardPosition,
)
from ..db_helpers import upsert_batting_card_ratings
from ..dependencies import oauth2_scheme, valid_token, LOG_DATA, PRIVATE_IN_SCHEMA
logging.basicConfig(
filename=LOG_DATA['filename'],
format=LOG_DATA['format'],
level=LOG_DATA['log_level']
filename=LOG_DATA["filename"],
format=LOG_DATA["format"],
level=LOG_DATA["log_level"],
)
router = APIRouter(
prefix='/api/v2/battingcardratings',
tags=['battingcardratings']
)
RATINGS_FILE = 'storage/batting-ratings.csv'
BASIC_FILE = 'storage/batting-basic.csv'
router = APIRouter(prefix="/api/v2/battingcardratings", tags=["battingcardratings"])
RATINGS_FILE = "storage/batting-ratings.csv"
BASIC_FILE = "storage/batting-basic.csv"
class BattingCardRatingsModel(pydantic.BaseModel):
battingcard_id: int
vs_hand: Literal['R', 'L', 'vR', 'vL']
vs_hand: Literal["R", "L", "vR", "vL"]
homerun: float = 0.0
bp_homerun: float = 0.0
triple: float = 0.0
@ -61,33 +68,70 @@ class BattingCardRatingsModel(pydantic.BaseModel):
@validator("avg", always=True)
def avg_validator(cls, v, values, **kwargs):
return (values['homerun'] + values['bp_homerun'] / 2 + values['triple'] + values['double_three'] +
values['double_two'] + values['double_pull'] + values['single_two'] + values['single_one'] +
values['single_center'] + values['bp_single'] / 2) / 108
return (
values["homerun"]
+ values["bp_homerun"] / 2
+ values["triple"]
+ values["double_three"]
+ values["double_two"]
+ values["double_pull"]
+ values["single_two"]
+ values["single_one"]
+ values["single_center"]
+ values["bp_single"] / 2
) / 108
@validator("obp", always=True)
def obp_validator(cls, v, values, **kwargs):
return ((values['hbp'] + values['walk']) / 108) + values['avg']
return ((values["hbp"] + values["walk"]) / 108) + values["avg"]
@validator("slg", always=True)
def slg_validator(cls, v, values, **kwargs):
return (values['homerun'] * 4 + values['bp_homerun'] * 2 + values['triple'] * 3 + values['double_three'] * 2 +
values['double_two'] * 2 + values['double_pull'] * 2 + values['single_two'] + values['single_one'] +
values['single_center'] + values['bp_single'] / 2) / 108
return (
values["homerun"] * 4
+ values["bp_homerun"] * 2
+ values["triple"] * 3
+ values["double_three"] * 2
+ values["double_two"] * 2
+ values["double_pull"] * 2
+ values["single_two"]
+ values["single_one"]
+ values["single_center"]
+ values["bp_single"] / 2
) / 108
@root_validator(skip_on_failure=True)
def validate_chance_total(cls, values):
total_chances = (
values['homerun'] + values['bp_homerun'] + values['triple'] + values['double_three'] +
values['double_two'] + values['double_pull'] + values['single_two'] + values['single_one'] +
values['single_center'] + values['bp_single'] + values['hbp'] + values['walk'] +
values['strikeout'] + values['lineout'] + values['popout'] + values['flyout_a'] +
values['flyout_bq'] + values['flyout_lf_b'] + values['flyout_rf_b'] + values['groundout_a'] +
values['groundout_b'] + values['groundout_c'])
values["homerun"]
+ values["bp_homerun"]
+ values["triple"]
+ values["double_three"]
+ values["double_two"]
+ values["double_pull"]
+ values["single_two"]
+ values["single_one"]
+ values["single_center"]
+ values["bp_single"]
+ values["hbp"]
+ values["walk"]
+ values["strikeout"]
+ values["lineout"]
+ values["popout"]
+ values["flyout_a"]
+ values["flyout_bq"]
+ values["flyout_lf_b"]
+ values["flyout_rf_b"]
+ values["groundout_a"]
+ values["groundout_b"]
+ values["groundout_c"]
)
if round(total_chances) != 108:
raise ValueError(f'BC {values["battingcard_id"]} must have exactly 108 chances on the card '
f'{values["vs_hand"]}; {round(total_chances)} listed')
raise ValueError(
f"BC {values['battingcard_id']} must have exactly 108 chances on the card "
f"{values['vs_hand']}; {round(total_chances)} listed"
)
return values
@ -100,18 +144,23 @@ class RatingsList(pydantic.BaseModel):
ratings: List[BattingCardRatingsModel]
@router.get('')
@router.get("")
async def get_card_ratings(
team_id: int, ts: str, battingcard_id: list = Query(default=None), cardset_id: list = Query(default=None),
vs_hand: Literal['R', 'L', 'vR', 'vL'] = None, short_output: bool = False, csv: bool = False):
team_id: int,
ts: str,
battingcard_id: list = Query(default=None),
cardset_id: list = Query(default=None),
vs_hand: Literal["R", "L", "vR", "vL"] = None,
short_output: bool = False,
csv: bool = False,
):
this_team = Team.get_or_none(Team.id == team_id)
logging.debug(f'Team: {this_team} / has_guide: {this_team.has_guide}')
logging.debug(f"Team: {this_team} / has_guide: {this_team.has_guide}")
if this_team is None or ts != this_team.team_hash() or this_team.has_guide != 1:
logging.warning(f'Team_id {team_id} attempted to pull ratings')
logging.warning(f"Team_id {team_id} attempted to pull ratings")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to pull card ratings.'
status_code=401, detail="You are not authorized to pull card ratings."
)
# elif not valid_token(token):
# logging.warning(f'Bad Token: {token}')
@ -124,29 +173,40 @@ async def get_card_ratings(
all_ratings = BattingCardRatings.select()
if battingcard_id is not None:
all_ratings = all_ratings.where(BattingCardRatings.battingcard_id << battingcard_id)
all_ratings = all_ratings.where(
BattingCardRatings.battingcard_id << battingcard_id
)
if vs_hand is not None:
all_ratings = all_ratings.where(BattingCardRatings.vs_hand == vs_hand[-1])
if cardset_id is not None:
set_players = Player.select(Player.player_id).where(Player.cardset_id << cardset_id)
set_cards = BattingCard.select(BattingCard.id).where(BattingCard.player << set_players)
set_players = Player.select(Player.player_id).where(
Player.cardset_id << cardset_id
)
set_cards = BattingCard.select(BattingCard.id).where(
BattingCard.player << set_players
)
all_ratings = all_ratings.where(BattingCardRatings.battingcard << set_cards)
if csv:
# return_val = query_to_csv(all_ratings)
return_vals = [model_to_dict(x) for x in all_ratings]
for x in return_vals:
x.update(x['battingcard'])
x['player_id'] = x['battingcard']['player']['player_id']
del x['battingcard'], x['player']
x.update(x["battingcard"])
x["player_id"] = x["battingcard"]["player"]["player_id"]
del x["battingcard"], x["player"]
db.close()
return Response(content=pd.DataFrame(return_vals).to_csv(index=False), media_type='text/csv')
return Response(
content=pd.DataFrame(return_vals).to_csv(index=False), media_type="text/csv"
)
else:
return_val = {'count': all_ratings.count(), 'ratings': [
model_to_dict(x, recurse=not short_output) for x in all_ratings
]}
return_val = {
"count": all_ratings.count(),
"ratings": [
model_to_dict(x, recurse=not short_output) for x in all_ratings
],
}
db.close()
return return_val
@ -154,225 +214,298 @@ async def get_card_ratings(
def get_scouting_dfs(cardset_id: list = None):
all_ratings = BattingCardRatings.select()
if cardset_id is not None:
set_players = Player.select(Player.player_id).where(Player.cardset_id << cardset_id)
set_cards = BattingCard.select(BattingCard.id).where(BattingCard.player << set_players)
set_players = Player.select(Player.player_id).where(
Player.cardset_id << cardset_id
)
set_cards = BattingCard.select(BattingCard.id).where(
BattingCard.player << set_players
)
all_ratings = all_ratings.where(BattingCardRatings.battingcard << set_cards)
vl_query = all_ratings.where(BattingCardRatings.vs_hand == 'L')
vr_query = all_ratings.where(BattingCardRatings.vs_hand == 'R')
vl_query = all_ratings.where(BattingCardRatings.vs_hand == "L")
vr_query = all_ratings.where(BattingCardRatings.vs_hand == "R")
vl_vals = [model_to_dict(x) for x in vl_query]
for x in vl_vals:
x.update(x['battingcard'])
x['player_id'] = x['battingcard']['player']['player_id']
x['player_name'] = x['battingcard']['player']['p_name']
x['rarity'] = x['battingcard']['player']['rarity']['name']
x['cardset_id'] = x['battingcard']['player']['cardset']['id']
x['cardset_name'] = x['battingcard']['player']['cardset']['name']
del x['battingcard']
del x['player']
x.update(x["battingcard"])
x["player_id"] = x["battingcard"]["player"]["player_id"]
x["player_name"] = x["battingcard"]["player"]["p_name"]
x["rarity"] = x["battingcard"]["player"]["rarity"]["name"]
x["cardset_id"] = x["battingcard"]["player"]["cardset"]["id"]
x["cardset_name"] = x["battingcard"]["player"]["cardset"]["name"]
del x["battingcard"]
del x["player"]
vr_vals = [model_to_dict(x) for x in vr_query]
for x in vr_vals:
x['player_id'] = x['battingcard']['player']['player_id']
del x['battingcard']
x["player_id"] = x["battingcard"]["player"]["player_id"]
del x["battingcard"]
vl = pd.DataFrame(vl_vals)
vr = pd.DataFrame(vr_vals)
bat_df = pd.merge(vl, vr, on='player_id', suffixes=('_vl', '_vr')).set_index('player_id', drop=False)
bat_df = pd.merge(vl, vr, on="player_id", suffixes=("_vl", "_vr")).set_index(
"player_id", drop=False
)
logging.debug(f'bat_df: {bat_df}')
logging.debug(f"bat_df: {bat_df}")
positions = CardPosition.select()
if cardset_id is not None:
set_players = Player.select(Player.player_id).where(Player.cardset_id << cardset_id)
set_players = Player.select(Player.player_id).where(
Player.cardset_id << cardset_id
)
positions = positions.where(CardPosition.player << set_players)
series_list = []
for pos_code in ['P', 'C', '1B', '2B', '3B', 'SS', 'LF', 'CF', 'RF']:
series_list.append(pd.Series(
dict([(x.player.player_id, x.range) for x in positions.where(CardPosition.position == pos_code)]),
name=f'Range {pos_code}'
))
series_list.append(pd.Series(
dict([(x.player.player_id, x.error) for x in positions.where(CardPosition.position == pos_code)]),
name=f'Error {pos_code}'
))
for pos_code in ["P", "C", "1B", "2B", "3B", "SS", "LF", "CF", "RF"]:
series_list.append(
pd.Series(
dict(
[
(x.player.player_id, x.range)
for x in positions.where(CardPosition.position == pos_code)
]
),
name=f"Range {pos_code}",
)
)
series_list.append(
pd.Series(
dict(
[
(x.player.player_id, x.error)
for x in positions.where(CardPosition.position == pos_code)
]
),
name=f"Error {pos_code}",
)
)
series_list.append(pd.Series(
dict([(x.player.player_id, x.arm) for x in positions.where(CardPosition.position << ['LF', 'CF', 'RF'])]),
name=f'Arm OF'
))
series_list.append(pd.Series(
dict([(x.player.player_id, x.arm) for x in positions.where(CardPosition.position == 'C')]),
name=f'Arm C'
))
series_list.append(pd.Series(
dict([(x.player.player_id, x.pb) for x in positions.where(CardPosition.position == 'C')]),
name=f'PB C'
))
series_list.append(pd.Series(
dict([(x.player.player_id, x.overthrow) for x in positions.where(CardPosition.position == 'C')]),
name=f'Throw C'
))
series_list.append(
pd.Series(
dict(
[
(x.player.player_id, x.arm)
for x in positions.where(
CardPosition.position << ["LF", "CF", "RF"]
)
]
),
name=f"Arm OF",
)
)
series_list.append(
pd.Series(
dict(
[
(x.player.player_id, x.arm)
for x in positions.where(CardPosition.position == "C")
]
),
name=f"Arm C",
)
)
series_list.append(
pd.Series(
dict(
[
(x.player.player_id, x.pb)
for x in positions.where(CardPosition.position == "C")
]
),
name=f"PB C",
)
)
series_list.append(
pd.Series(
dict(
[
(x.player.player_id, x.overthrow)
for x in positions.where(CardPosition.position == "C")
]
),
name=f"Throw C",
)
)
db.close()
logging.debug(f'series_list: {series_list}')
logging.debug(f"series_list: {series_list}")
return bat_df.join(series_list)
@router.get('/scouting')
@router.get("/scouting")
async def get_card_scouting(team_id: int, ts: str):
this_team = Team.get_or_none(Team.id == team_id)
logging.debug(f'Team: {this_team} / has_guide: {this_team.has_guide}')
logging.debug(f"Team: {this_team} / has_guide: {this_team.has_guide}")
if this_team is None or ts != this_team.team_hash() or this_team.has_guide != 1:
logging.warning(f'Team_id {team_id} attempted to pull ratings')
logging.warning(f"Team_id {team_id} attempted to pull ratings")
db.close()
return 'Your team does not have the ratings guide enabled. If you have purchased a copy ping Cal to ' \
'make sure it is enabled on your team. If you are interested you can pick it up here (thank you!): ' \
'https://ko-fi.com/manticorum/shop'
return (
"Your team does not have the ratings guide enabled. If you have purchased a copy ping Cal to "
"make sure it is enabled on your team. If you are interested you can pick it up here (thank you!): "
"https://ko-fi.com/manticorum/shop"
)
if os.path.isfile(f'storage/batting-ratings.csv'):
if os.path.isfile(f"storage/batting-ratings.csv"):
return FileResponse(
path=f'storage/batting-ratings.csv',
media_type='text/csv',
path=f"storage/batting-ratings.csv",
media_type="text/csv",
# headers=headers
)
raise HTTPException(status_code=400, detail='Go pester Cal - the scouting file is missing')
raise HTTPException(
status_code=400, detail="Go pester Cal - the scouting file is missing"
)
@router.post('/calculate/scouting', include_in_schema=PRIVATE_IN_SCHEMA)
@router.post("/calculate/scouting", include_in_schema=PRIVATE_IN_SCHEMA)
async def post_calc_scouting(token: str = Depends(oauth2_scheme)):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to calculate card ratings.'
status_code=401, detail="You are not authorized to calculate card ratings."
)
logging.warning(f'Re-calculating batting ratings\n\n')
logging.warning(f"Re-calculating batting ratings\n\n")
output = get_scouting_dfs()
first = ['player_id', 'player_name', 'cardset_name', 'rarity', 'hand', 'variant']
exclude = first + ['id_vl', 'id_vr', 'vs_hand_vl', 'vs_hand_vr']
first = ["player_id", "player_name", "cardset_name", "rarity", "hand", "variant"]
exclude = first + ["id_vl", "id_vr", "vs_hand_vl", "vs_hand_vr"]
output = output[first + [col for col in output.columns if col not in exclude]]
csv_file = pd.DataFrame(output).to_csv(index=False)
with open(RATINGS_FILE, 'w') as file:
with open(RATINGS_FILE, "w") as file:
file.write(csv_file)
return Response(content=csv_file, media_type='text/csv')
return Response(content=csv_file, media_type="text/csv")
@router.get('/basic')
@router.get("/basic")
async def get_basic_scouting(cardset_id: list = Query(default=None)):
if os.path.isfile(f'storage/batting-basic.csv'):
if os.path.isfile(f"storage/batting-basic.csv"):
return FileResponse(
path=f'storage/batting-basic.csv',
media_type='text/csv',
path=f"storage/batting-basic.csv",
media_type="text/csv",
# headers=headers
)
raise HTTPException(status_code=400, detail='Go pester Cal - the scouting file is missing')
raise HTTPException(
status_code=400, detail="Go pester Cal - the scouting file is missing"
)
@router.post('/calculate/basic', include_in_schema=PRIVATE_IN_SCHEMA)
@router.post("/calculate/basic", include_in_schema=PRIVATE_IN_SCHEMA)
async def post_calc_basic(token: str = Depends(oauth2_scheme)):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to calculate basic ratings.'
status_code=401, detail="You are not authorized to calculate basic ratings."
)
logging.warning(f'Re-calculating basic batting ratings\n\n')
logging.warning(f"Re-calculating basic batting ratings\n\n")
raw_data = get_scouting_dfs()
logging.debug(f'output: {raw_data}')
logging.debug(f"output: {raw_data}")
def get_raw_speed(df_data):
speed_raw = df_data['running'] / 20 + df_data['steal_jump']
if df_data['steal_auto']:
speed_raw = df_data["running"] / 20 + df_data["steal_jump"]
if df_data["steal_auto"]:
speed_raw += 0.5
return speed_raw
raw_series = raw_data.apply(get_raw_speed, axis=1)
rank_series = raw_series.rank(pct=True)
raw_data['Speed'] = round(rank_series * 100)
raw_data["Speed"] = round(rank_series * 100)
def get_raw_steal(df_data):
return (
((df_data['steal_high'] / 20) + (df_data['steal_low'] / 20)) * df_data['steal_jump']
)
return ((df_data["steal_high"] / 20) + (df_data["steal_low"] / 20)) * df_data[
"steal_jump"
]
raw_series = raw_data.apply(get_raw_steal, axis=1)
rank_series = raw_series.rank(pct=True)
raw_data['Steal'] = round(rank_series * 100)
raw_data["Steal"] = round(rank_series * 100)
def get_raw_reaction(df_data):
raw_total = 0
for pos_range in [df_data['Range C'], df_data['Range 1B'], df_data['Range 2B'], df_data['Range 3B'],
df_data['Range SS'], df_data['Range LF'], df_data['Range CF'], df_data['Range RF']]:
for pos_range in [
df_data["Range C"],
df_data["Range 1B"],
df_data["Range 2B"],
df_data["Range 3B"],
df_data["Range SS"],
df_data["Range LF"],
df_data["Range CF"],
df_data["Range RF"],
]:
if pd.notna(pos_range):
raw_total += 10 ** (5 - pos_range)
return raw_total
raw_series = raw_data.apply(get_raw_reaction, axis=1)
rank_series = raw_series.rank(pct=True)
raw_data['Reaction'] = round(rank_series * 100)
raw_data["Reaction"] = round(rank_series * 100)
def get_raw_arm(df_data):
of_arm = None
of_pos = None
if pd.notna(df_data['Range RF']):
of_pos = 'RF'
elif pd.notna(df_data['Range CF']):
of_pos = 'CF'
elif pd.notna(df_data['Range LF']):
of_pos = 'LF'
if pd.notna(df_data["Range RF"]):
of_pos = "RF"
elif pd.notna(df_data["Range CF"]):
of_pos = "CF"
elif pd.notna(df_data["Range LF"]):
of_pos = "LF"
if of_pos is not None:
if df_data['Arm OF'] < 0:
of_raw = df_data['Arm OF'] * -10
if df_data["Arm OF"] < 0:
of_raw = df_data["Arm OF"] * -10
else:
of_raw = (5 - df_data['Arm OF'])
of_raw = 5 - df_data["Arm OF"]
if of_pos == 'RF':
if of_pos == "RF":
of_raw = of_raw * 1.5
of_raw += ((6 - df_data['Range RF']) * 4)
elif of_pos == 'CF':
of_raw += ((6 - df_data['Range CF']) * 3)
elif of_pos == 'LF':
of_raw += (6 - df_data["Range RF"]) * 4
elif of_pos == "CF":
of_raw += (6 - df_data["Range CF"]) * 3
elif of_pos == "LF":
of_raw = of_raw / 2
of_raw += ((6 - df_data['Range LF']) * 2)
of_raw += (6 - df_data["Range LF"]) * 2
of_arm = of_raw
if_arm = None
if pd.notna(df_data['Range 3B']) or pd.notna(df_data['Range 2B']) or pd.notna(df_data['Range 1B']) or \
pd.notna(df_data['Range SS']):
if (
pd.notna(df_data["Range 3B"])
or pd.notna(df_data["Range 2B"])
or pd.notna(df_data["Range 1B"])
or pd.notna(df_data["Range SS"])
):
range_totals = 0
if pd.notna(df_data['Range 3B']):
range_totals += ((6 - df_data['Range 3B']) * 5)
if pd.notna(df_data['Range SS']):
range_totals += ((6 - df_data['Range SS']) * 4)
if pd.notna(df_data['Range 2B']):
range_totals += ((6 - df_data['Range 2B']) * 3)
if pd.notna(df_data['Range 1B']):
range_totals += (6 - df_data['Range 1B'])
if pd.notna(df_data["Range 3B"]):
range_totals += (6 - df_data["Range 3B"]) * 5
if pd.notna(df_data["Range SS"]):
range_totals += (6 - df_data["Range SS"]) * 4
if pd.notna(df_data["Range 2B"]):
range_totals += (6 - df_data["Range 2B"]) * 3
if pd.notna(df_data["Range 1B"]):
range_totals += 6 - df_data["Range 1B"]
if_arm = 100 - (50 - range_totals)
c_arm = None
if pd.notna(df_data['Arm C']):
if df_data['Arm C'] == -5:
if pd.notna(df_data["Arm C"]):
if df_data["Arm C"] == -5:
c_arm = 100
else:
temp_arm = 20 + ((10 - df_data['Arm C']) * 3) + (20 - df_data['PB C']) + (20 - df_data['Throw C']) - \
df_data['Error C']
temp_arm = (
20
+ ((10 - df_data["Arm C"]) * 3)
+ (20 - df_data["PB C"])
+ (20 - df_data["Throw C"])
- df_data["Error C"]
)
c_arm = min(100, temp_arm)
if c_arm is not None:
@ -386,149 +519,192 @@ async def post_calc_basic(token: str = Depends(oauth2_scheme)):
raw_series = raw_data.apply(get_raw_arm, axis=1)
rank_series = raw_series.rank(pct=True)
raw_data['Arm'] = round(rank_series * 100)
raw_data["Arm"] = round(rank_series * 100)
def get_raw_fielding(df_data):
if_error, of_error, c_error = 0, 0, 0
denom = 0
if pd.notna(df_data['Error 3B']) or pd.notna(df_data['Error 2B']) or pd.notna(df_data['Error 1B']) or \
pd.notna(df_data['Error SS']):
if (
pd.notna(df_data["Error 3B"])
or pd.notna(df_data["Error 2B"])
or pd.notna(df_data["Error 1B"])
or pd.notna(df_data["Error SS"])
):
raw_if = 100
if pd.notna(df_data['Error 3B']):
raw_if -= (df_data['Error 3B'] * 2)
if pd.notna(df_data['Error SS']):
raw_if -= (df_data['Error SS'] * .75)
if pd.notna(df_data['Error 2B']):
raw_if -= (df_data['Error 2B'] * 1.25)
if pd.notna(df_data['Error 1B']):
raw_if -= (df_data['Error 1B'] * 2)
if pd.notna(df_data["Error 3B"]):
raw_if -= df_data["Error 3B"] * 2
if pd.notna(df_data["Error SS"]):
raw_if -= df_data["Error SS"] * 0.75
if pd.notna(df_data["Error 2B"]):
raw_if -= df_data["Error 2B"] * 1.25
if pd.notna(df_data["Error 1B"]):
raw_if -= df_data["Error 1B"] * 2
if_error = max(1, raw_if)
denom += 1
if pd.notna(df_data['Error LF']) or pd.notna(df_data['Error CF']) or pd.notna(df_data['Error RF']):
if (
pd.notna(df_data["Error LF"])
or pd.notna(df_data["Error CF"])
or pd.notna(df_data["Error RF"])
):
raw_of = 100
if pd.notna(df_data['Error LF']):
raw_of -= (df_data['Error LF'] * 2)
if pd.notna(df_data['Error CF']):
raw_of -= (df_data['Error CF'] * .75)
if pd.notna(df_data['Error RF']):
raw_of -= (df_data['Error RF'] * 1.25)
if pd.notna(df_data["Error LF"]):
raw_of -= df_data["Error LF"] * 2
if pd.notna(df_data["Error CF"]):
raw_of -= df_data["Error CF"] * 0.75
if pd.notna(df_data["Error RF"]):
raw_of -= df_data["Error RF"] * 1.25
of_error = max(1, raw_of)
denom += 1
if pd.notna(df_data['Error C']):
c_error = max(100 - (df_data['Error C'] * 5) - df_data['Throw C'] - df_data['PB C'], 1)
if pd.notna(df_data["Error C"]):
c_error = max(
100 - (df_data["Error C"] * 5) - df_data["Throw C"] - df_data["PB C"], 1
)
denom += 1
return sum([if_error, of_error, c_error]) / max(denom, 1)
raw_series = raw_data.apply(get_raw_fielding, axis=1)
rank_series = raw_series.rank(pct=True)
raw_data['Fielding'] = round(rank_series * 100)
raw_data["Fielding"] = round(rank_series * 100)
rank_series = raw_data['avg_vl'].rank(pct=True)
raw_data['Contact L'] = round(rank_series * 100)
rank_series = raw_data["avg_vl"].rank(pct=True)
raw_data["Contact L"] = round(rank_series * 100)
rank_series = raw_data['avg_vr'].rank(pct=True)
raw_data['Contact R'] = round(rank_series * 100)
rank_series = raw_data["avg_vr"].rank(pct=True)
raw_data["Contact R"] = round(rank_series * 100)
rank_series = raw_data['slg_vl'].rank(pct=True)
raw_data['Power L'] = round(rank_series * 100)
rank_series = raw_data["slg_vl"].rank(pct=True)
raw_data["Power L"] = round(rank_series * 100)
rank_series = raw_data['slg_vr'].rank(pct=True)
raw_data['Power R'] = round(rank_series * 100)
rank_series = raw_data["slg_vr"].rank(pct=True)
raw_data["Power R"] = round(rank_series * 100)
def get_raw_vision(df_data):
return (
((((df_data['obp_vr'] * 0.67) + (df_data['obp_vl'] * 0.33)) -
((df_data['avg_vr'] * 0.67) + (df_data['avg_vl'] * 0.33))) * 5) -
(((df_data['strikeout_vl'] * 0.33) + (df_data['strikeout_vr'] * 0.67)) / 208)
(
((df_data["obp_vr"] * 0.67) + (df_data["obp_vl"] * 0.33))
- ((df_data["avg_vr"] * 0.67) + (df_data["avg_vl"] * 0.33))
)
* 5
) - (
((df_data["strikeout_vl"] * 0.33) + (df_data["strikeout_vr"] * 0.67)) / 208
)
raw_series = raw_data.apply(get_raw_vision, axis=1)
rank_series = raw_series.rank(pct=True)
raw_data['Vision'] = round(rank_series * 100)
raw_data["Vision"] = round(rank_series * 100)
def get_raw_rating(df_data):
return (
((df_data['Reaction'] + df_data['Arm'] + df_data['Fielding']) * 2) +
(df_data['Speed'] + df_data['Steal']) +
((((df_data['Contact R'] + df_data['Power R']) * 0.67) +
((df_data['Contact L'] + df_data['Power L']) * 0.33) + df_data['Vision'] ) * 6
)
((df_data["Reaction"] + df_data["Arm"] + df_data["Fielding"]) * 2)
+ (df_data["Speed"] + df_data["Steal"])
+ (
(
((df_data["Contact R"] + df_data["Power R"]) * 0.67)
+ ((df_data["Contact L"] + df_data["Power L"]) * 0.33)
+ df_data["Vision"]
)
* 6
)
)
raw_series = raw_data.apply(get_raw_rating, axis=1)
rank_series = raw_series.rank(pct=True)
raw_data['Rating'] = round(rank_series * 100)
raw_data["Rating"] = round(rank_series * 100)
output = raw_data[[
'player_id', 'player_name', 'Rating', 'Contact R', 'Contact L', 'Power R', 'Power L', 'Vision', 'Speed',
'Steal', 'Reaction', 'Arm', 'Fielding', 'hand', 'cardset_name'
]]
output = raw_data[
[
"player_id",
"player_name",
"Rating",
"Contact R",
"Contact L",
"Power R",
"Power L",
"Vision",
"Speed",
"Steal",
"Reaction",
"Arm",
"Fielding",
"hand",
"cardset_name",
]
]
csv_file = pd.DataFrame(output).to_csv(index=False)
with open(BASIC_FILE, 'w') as file:
with open(BASIC_FILE, "w") as file:
file.write(csv_file)
return Response(content=csv_file, media_type='text/csv')
return Response(content=csv_file, media_type="text/csv")
@router.get('/{ratings_id}')
@router.get("/{ratings_id}")
async def get_one_rating(ratings_id: int, token: str = Depends(oauth2_scheme)):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to pull card ratings.'
status_code=401, detail="You are not authorized to pull card ratings."
)
this_rating = BattingCardRatings.get_or_none(BattingCardRatings.id == ratings_id)
if this_rating is None:
db.close()
raise HTTPException(status_code=404, detail=f'BattingCardRating id {ratings_id} not found')
raise HTTPException(
status_code=404, detail=f"BattingCardRating id {ratings_id} not found"
)
r_data = model_to_dict(this_rating)
db.close()
return r_data
@router.get('/player/{player_id}')
@router.get("/player/{player_id}")
async def get_player_ratings(
player_id: int, variant: list = Query(default=None), short_output: bool = False,
token: str = Depends(oauth2_scheme)):
player_id: int,
variant: list = Query(default=None),
short_output: bool = False,
token: str = Depends(oauth2_scheme),
):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to pull card ratings.'
status_code=401, detail="You are not authorized to pull card ratings."
)
all_cards = BattingCard.select().where(BattingCard.player_id == player_id).order_by(BattingCard.variant)
all_cards = (
BattingCard.select()
.where(BattingCard.player_id == player_id)
.order_by(BattingCard.variant)
)
if variant is not None:
all_cards = all_cards.where(BattingCard.variant << variant)
all_ratings = BattingCardRatings.select().where(BattingCardRatings.battingcard << all_cards)
all_ratings = BattingCardRatings.select().where(
BattingCardRatings.battingcard << all_cards
)
return_val = {'count': all_ratings.count(), 'ratings': [
model_to_dict(x, recurse=not short_output) for x in all_ratings
]}
return_val = {
"count": all_ratings.count(),
"ratings": [model_to_dict(x, recurse=not short_output) for x in all_ratings],
}
db.close()
return return_val
@router.put('', include_in_schema=PRIVATE_IN_SCHEMA)
@router.put("", include_in_schema=PRIVATE_IN_SCHEMA)
async def put_ratings(ratings: RatingsList, token: str = Depends(oauth2_scheme)):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to post card ratings.'
status_code=401, detail="You are not authorized to post card ratings."
)
new_ratings = []
@ -536,44 +712,50 @@ async def put_ratings(ratings: RatingsList, token: str = Depends(oauth2_scheme))
for x in ratings.ratings:
try:
BattingCardRatings.get(
(BattingCardRatings.battingcard_id == x.battingcard_id) & (BattingCardRatings.vs_hand == x.vs_hand)
(BattingCardRatings.battingcard_id == x.battingcard_id)
& (BattingCardRatings.vs_hand == x.vs_hand)
)
updates += (
BattingCardRatings.update(x.dict())
.where(
(BattingCardRatings.battingcard_id == x.battingcard_id)
& (BattingCardRatings.vs_hand == x.vs_hand)
)
.execute()
)
updates += BattingCardRatings.update(x.dict()).where(
(BattingCardRatings.battingcard_id == x.battingcard_id) & (BattingCardRatings.vs_hand == x.vs_hand)
).execute()
except BattingCardRatings.DoesNotExist:
new_ratings.append(x.dict())
with db.atomic():
for batch in chunked(new_ratings, 30):
BattingCardRatings.insert_many(batch).on_conflict_replace().execute()
# Use PostgreSQL-compatible upsert helper
upsert_batting_card_ratings(new_ratings, batch_size=30)
db.close()
return f'Updated ratings: {updates}; new ratings: {len(new_ratings)}'
return f"Updated ratings: {updates}; new ratings: {len(new_ratings)}"
@router.delete('/{ratings_id}', include_in_schema=PRIVATE_IN_SCHEMA)
async def delete_rating(
ratings_id: int, token: str = Depends(oauth2_scheme)):
@router.delete("/{ratings_id}", include_in_schema=PRIVATE_IN_SCHEMA)
async def delete_rating(ratings_id: int, token: str = Depends(oauth2_scheme)):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to post card ratings.'
status_code=401, detail="You are not authorized to post card ratings."
)
this_rating = BattingCardRatings.get_or_none(BattingCardRatings.id == ratings_id)
if this_rating is None:
db.close()
raise HTTPException(status_code=404, detail=f'BattingCardRating id {ratings_id} not found')
raise HTTPException(
status_code=404, detail=f"BattingCardRating id {ratings_id} not found"
)
count = this_rating.delete_instance()
db.close()
if count == 1:
return f'Rating {this_rating} has been deleted'
return f"Rating {this_rating} has been deleted"
else:
raise HTTPException(status_code=500, detail=f'Rating {this_rating} could not be deleted')
raise HTTPException(
status_code=500, detail=f"Rating {this_rating} could not be deleted"
)

156
app/routers_v2/battingcards.py
View File

@ -6,18 +6,16 @@ import logging
import pydantic
from ..db_engine import db, BattingCard, model_to_dict, fn, chunked, Player, MlbPlayer
from ..db_helpers import upsert_batting_cards
from ..dependencies import oauth2_scheme, valid_token, LOG_DATA
logging.basicConfig(
filename=LOG_DATA['filename'],
format=LOG_DATA['format'],
level=LOG_DATA['log_level']
filename=LOG_DATA["filename"],
format=LOG_DATA["format"],
level=LOG_DATA["log_level"],
)
router = APIRouter(
prefix='/api/v2/battingcards',
tags=['battingcards']
)
router = APIRouter(prefix="/api/v2/battingcards", tags=["battingcards"])
class BattingCardModel(pydantic.BaseModel):
@ -27,22 +25,26 @@ class BattingCardModel(pydantic.BaseModel):
steal_high: int = 20
steal_auto: bool = False
steal_jump: float = 0
bunting: str = 'C'
hit_and_run: str = 'C'
bunting: str = "C"
hit_and_run: str = "C"
running: int = 10
offense_col: int = None
hand: Literal['R', 'L', 'S'] = 'R'
hand: Literal["R", "L", "S"] = "R"
class BattingCardList(pydantic.BaseModel):
cards: List[BattingCardModel]
@router.get('')
@router.get("")
async def get_batting_cards(
player_id: list = Query(default=None), player_name: list = Query(default=None),
cardset_id: list = Query(default=None), short_output: bool = False, limit: Optional[int] = None,
variant: list = Query(default=None)):
player_id: list = Query(default=None),
player_name: list = Query(default=None),
cardset_id: list = Query(default=None),
short_output: bool = False,
limit: Optional[int] = None,
variant: list = Query(default=None),
):
all_cards = BattingCard.select()
if player_id is not None:
all_cards = all_cards.where(BattingCard.player_id << player_id)
@ -59,102 +61,134 @@ async def get_batting_cards(
if limit is not None:
all_cards = all_cards.limit(limit)
return_val = {'count': all_cards.count(), 'cards': [
model_to_dict(x, recurse=not short_output) for x in all_cards
]}
return_val = {
"count": all_cards.count(),
"cards": [model_to_dict(x, recurse=not short_output) for x in all_cards],
}
db.close()
return return_val
@router.get('/{card_id}')
@router.get("/{card_id}")
async def get_one_card(card_id: int):
this_card = BattingCard.get_or_none(BattingCard.id == card_id)
if this_card is None:
db.close()
raise HTTPException(status_code=404, detail=f'BattingCard id {card_id} not found')
raise HTTPException(
status_code=404, detail=f"BattingCard id {card_id} not found"
)
r_card = model_to_dict(this_card)
db.close()
return r_card
@router.get('/player/{player_id}')
async def get_player_cards(player_id: int, variant: list = Query(default=None), short_output: bool = False):
all_cards = BattingCard.select().where(BattingCard.player_id == player_id).order_by(BattingCard.variant)
@router.get("/player/{player_id}")
async def get_player_cards(
player_id: int, variant: list = Query(default=None), short_output: bool = False
):
all_cards = (
BattingCard.select()
.where(BattingCard.player_id == player_id)
.order_by(BattingCard.variant)
)
if variant is not None:
all_cards = all_cards.where(BattingCard.variant << variant)
return_val = {'count': all_cards.count(), 'cards': [
model_to_dict(x, recurse=not short_output) for x in all_cards
]}
return_val = {
"count": all_cards.count(),
"cards": [model_to_dict(x, recurse=not short_output) for x in all_cards],
}
db.close()
return return_val
@router.put('')
@router.put("")
async def put_cards(cards: BattingCardList, token: str = Depends(oauth2_scheme)):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to post batting cards. This event has been logged.'
detail="You are not authorized to post batting cards. This event has been logged.",
)
new_cards = []
updates = 0
logging.info(f'here!')
logging.info(f"here!")
for x in cards.cards:
try:
old = BattingCard.get(
(BattingCard.player_id == x.player_id) & (BattingCard.variant == x.variant)
(BattingCard.player_id == x.player_id)
& (BattingCard.variant == x.variant)
)
if x.offense_col is None:
x.offense_col = old.offense_col
updates += BattingCard.update(x.dict()).where(
(BattingCard.player_id == x.player_id) & (BattingCard.variant == x.variant)
).execute()
updates += (
BattingCard.update(x.dict())
.where(
(BattingCard.player_id == x.player_id)
& (BattingCard.variant == x.variant)
)
.execute()
)
except BattingCard.DoesNotExist:
if x.offense_col is None:
this_player = Player.get_or_none(Player.player_id == x.player_id)
mlb_player = MlbPlayer.get_or_none(MlbPlayer.key_bbref == this_player.bbref_id)
mlb_player = MlbPlayer.get_or_none(
MlbPlayer.key_bbref == this_player.bbref_id
)
if mlb_player is not None:
logging.info(f'setting offense_col to {mlb_player.offense_col} for {this_player.p_name}')
logging.info(
f"setting offense_col to {mlb_player.offense_col} for {this_player.p_name}"
)
x.offense_col = mlb_player.offense_col
else:
logging.info(f'randomly setting offense_col for {this_player.p_name}')
logging.info(
f"randomly setting offense_col for {this_player.p_name}"
)
x.offense_col = random.randint(1, 3)
logging.debug(f'x.dict(): {x.dict()}')
logging.debug(f"x.dict(): {x.dict()}")
new_cards.append(x.dict())
with db.atomic():
for batch in chunked(new_cards, 30):
BattingCard.insert_many(batch).on_conflict_replace().execute()
# Use PostgreSQL-compatible upsert helper
upsert_batting_cards(new_cards, batch_size=30)
db.close()
return f'Updated cards: {updates}; new cards: {len(new_cards)}'
return f"Updated cards: {updates}; new cards: {len(new_cards)}"
@router.patch('/{card_id}')
@router.patch("/{card_id}")
async def patch_card(
card_id: int, steal_low: Optional[int] = None, steal_high: Optional[int] = None,
steal_auto: Optional[bool] = None, steal_jump: Optional[float] = None, bunting: Optional[str] = None,
hit_and_run: Optional[str] = None, running: Optional[int] = None, offense_col: Optional[int] = None,
hand: Literal['R', 'L', 'S'] = None, token: str = Depends(oauth2_scheme)):
card_id: int,
steal_low: Optional[int] = None,
steal_high: Optional[int] = None,
steal_auto: Optional[bool] = None,
steal_jump: Optional[float] = None,
bunting: Optional[str] = None,
hit_and_run: Optional[str] = None,
running: Optional[int] = None,
offense_col: Optional[int] = None,
hand: Literal["R", "L", "S"] = None,
token: str = Depends(oauth2_scheme),
):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to patch batting cards. This event has been logged.'
detail="You are not authorized to patch batting cards. This event has been logged.",
)
this_card = BattingCard.get_or_none(BattingCard.id == card_id)
if this_card is None:
db.close()
raise HTTPException(status_code=404, detail=f'BattingCard id {card_id} not found')
raise HTTPException(
status_code=404, detail=f"BattingCard id {card_id} not found"
)
if steal_low is not None:
this_card.steal_low = steal_low
@ -183,45 +217,49 @@ async def patch_card(
db.close()
raise HTTPException(
status_code=418,
detail='Well slap my ass and call me a teapot; I could not save that card'
detail="Well slap my ass and call me a teapot; I could not save that card",
)
@router.delete('/{card_id}')
@router.delete("/{card_id}")
async def delete_card(card_id: int, token: str = Depends(oauth2_scheme)):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to delete batting cards. This event has been logged.'
detail="You are not authorized to delete batting cards. This event has been logged.",
)
this_card = BattingCard.get_or_none(BattingCard.id == card_id)
if this_card is None:
db.close()
raise HTTPException(status_code=404, detail=f'BattingCard id {card_id} not found')
raise HTTPException(
status_code=404, detail=f"BattingCard id {card_id} not found"
)
count = this_card.delete_instance()
db.close()
if count == 1:
return f'Card {this_card} has been deleted'
return f"Card {this_card} has been deleted"
else:
raise HTTPException(status_code=500, detail=f'Card {this_card} could not be deleted')
raise HTTPException(
status_code=500, detail=f"Card {this_card} could not be deleted"
)
@router.delete('')
@router.delete("")
async def delete_all_cards(token: str = Depends(oauth2_scheme)):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to delete batting cards. This event has been logged.'
detail="You are not authorized to delete batting cards. This event has been logged.",
)
d_query = BattingCard.delete()
d_query.execute()
return f'Deleted {d_query.count()} batting cards'
return f"Deleted {d_query.count()} batting cards"

115
app/routers_v2/cardpositions.py
View File

@ -5,24 +5,22 @@ import pydantic
from pydantic import root_validator
from ..db_engine import db, CardPosition, model_to_dict, chunked, Player, fn
from ..db_helpers import upsert_card_positions
from ..dependencies import oauth2_scheme, valid_token, LOG_DATA
logging.basicConfig(
filename=LOG_DATA['filename'],
format=LOG_DATA['format'],
level=LOG_DATA['log_level']
filename=LOG_DATA["filename"],
format=LOG_DATA["format"],
level=LOG_DATA["log_level"],
)
router = APIRouter(
prefix='/api/v2/cardpositions',
tags=['cardpositions']
)
router = APIRouter(prefix="/api/v2/cardpositions", tags=["cardpositions"])
class CardPositionModel(pydantic.BaseModel):
player_id: int
variant: int = 0
position: Literal['P', 'C', '1B', '2B', '3B', 'SS', 'LF', 'CF', 'RF', 'DH']
position: Literal["P", "C", "1B", "2B", "3B", "SS", "LF", "CF", "RF", "DH"]
innings: int = 1
range: int = 5
error: int = 0
@ -32,10 +30,12 @@ class CardPositionModel(pydantic.BaseModel):
@root_validator(skip_on_failure=True)
def position_validator(cls, values):
if values['position'] in ['C', 'LF', 'CF', 'RF'] and values['arm'] is None:
raise ValueError(f'{values["position"]} must have an arm rating')
if values['position'] == 'C' and (values['pb'] is None or values['overthrow'] is None):
raise ValueError('Catchers must have a pb and overthrow rating')
if values["position"] in ["C", "LF", "CF", "RF"] and values["arm"] is None:
raise ValueError(f"{values['position']} must have an arm rating")
if values["position"] == "C" and (
values["pb"] is None or values["overthrow"] is None
):
raise ValueError("Catchers must have a pb and overthrow rating")
return values
@ -43,14 +43,24 @@ class PositionList(pydantic.BaseModel):
positions: List[CardPositionModel]
@router.get('')
@router.get("")
async def get_card_positions(
player_id: list = Query(default=None), position: list = Query(default=None), min_innings: Optional[int] = 1,
r: list = Query(default=None), e: list = Query(default=None), arm: list = Query(default=None),
pb: list = Query(default=None), overthrow: list = Query(default=None), cardset_id: list = Query(default=None),
short_output: Optional[bool] = False, sort: Optional[str] = 'innings-desc'):
all_pos = CardPosition.select().where(CardPosition.innings >= min_innings).order_by(
CardPosition.player, CardPosition.position, CardPosition.variant
player_id: list = Query(default=None),
position: list = Query(default=None),
min_innings: Optional[int] = 1,
r: list = Query(default=None),
e: list = Query(default=None),
arm: list = Query(default=None),
pb: list = Query(default=None),
overthrow: list = Query(default=None),
cardset_id: list = Query(default=None),
short_output: Optional[bool] = False,
sort: Optional[str] = "innings-desc",
):
all_pos = (
CardPosition.select()
.where(CardPosition.innings >= min_innings)
.order_by(CardPosition.player, CardPosition.position, CardPosition.variant)
)
if player_id is not None:
@ -72,42 +82,45 @@ async def get_card_positions(
all_players = Player.select().where(Player.cardset_id << cardset_id)
all_pos = all_pos.where(CardPosition.player << all_players)
if sort == 'innings-desc':
if sort == "innings-desc":
all_pos = all_pos.order_by(CardPosition.innings.desc())
elif sort == 'innings-asc':
elif sort == "innings-asc":
all_pos = all_pos.order_by(CardPosition.innings)
elif sort == 'range-desc':
elif sort == "range-desc":
all_pos = all_pos.order_by(CardPosition.range.desc())
elif sort == 'range-asc':
elif sort == "range-asc":
all_pos = all_pos.order_by(CardPosition.range)
return_val = {'count': all_pos.count(), 'positions': [
model_to_dict(x, recurse=not short_output) for x in all_pos
]}
return_val = {
"count": all_pos.count(),
"positions": [model_to_dict(x, recurse=not short_output) for x in all_pos],
}
db.close()
return return_val
@router.get('/{position_id}')
@router.get("/{position_id}")
async def get_one_position(position_id: int):
this_pos = CardPosition.get_or_none(CardPosition.id == position_id)
if this_pos is None:
db.close()
raise HTTPException(status_code=404, detail=f'CardPosition id {position_id} not found')
raise HTTPException(
status_code=404, detail=f"CardPosition id {position_id} not found"
)
r_data = model_to_dict(this_pos)
db.close()
return r_data
@router.put('')
@router.put("")
async def put_positions(positions: PositionList, token: str = Depends(oauth2_scheme)):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to post card positions. This event has been logged.'
detail="You are not authorized to post card positions. This event has been logged.",
)
new_cards = []
@ -116,43 +129,53 @@ async def put_positions(positions: PositionList, token: str = Depends(oauth2_sch
for x in positions.positions:
try:
CardPosition.get(
(CardPosition.player_id == x.player_id) & (CardPosition.variant == x.variant) &
(CardPosition.position == x.position)
(CardPosition.player_id == x.player_id)
& (CardPosition.variant == x.variant)
& (CardPosition.position == x.position)
)
updates += (
CardPosition.update(x.dict())
.where(
(CardPosition.player_id == x.player_id)
& (CardPosition.variant == x.variant)
& (CardPosition.position == x.position)
)
.execute()
)
updates += CardPosition.update(x.dict()).where(
(CardPosition.player_id == x.player_id) & (CardPosition.variant == x.variant) &
(CardPosition.position == x.position)
).execute()
except CardPosition.DoesNotExist:
new_cards.append(x.dict())
with db.atomic():
for batch in chunked(new_cards, 30):
CardPosition.insert_many(batch).on_conflict_replace().execute()
# Use PostgreSQL-compatible upsert helper
upsert_card_positions(new_cards, batch_size=30)
db.close()
return f'Updated cards: {updates}; new cards: {len(new_cards)}'
return f"Updated cards: {updates}; new cards: {len(new_cards)}"
@router.delete('/{position_id}')
@router.delete("/{position_id}")
async def delete_position(position_id: int, token: str = Depends(oauth2_scheme)):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to delete card positions. This event has been logged.'
detail="You are not authorized to delete card positions. This event has been logged.",
)
this_pos = CardPosition.get_or_none(CardPosition.id == position_id)
if this_pos is None:
db.close()
raise HTTPException(status_code=404, detail=f'CardPosition id {position_id} not found')
raise HTTPException(
status_code=404, detail=f"CardPosition id {position_id} not found"
)
count = this_pos.delete_instance()
db.close()
if count == 1:
return f'Card Position {this_pos} has been deleted'
return f"Card Position {this_pos} has been deleted"
else:
raise HTTPException(status_code=500, detail=f'Card Position {this_pos} could not be deleted')
raise HTTPException(
status_code=500, detail=f"Card Position {this_pos} could not be deleted"
)

205
app/routers_v2/decisions.py
View File

@ -5,19 +5,28 @@ import logging
import pandas as pd
import pydantic
from ..db_engine import db, Decision, StratGame, Player, model_to_dict, chunked, fn, Team, Card, StratPlay
from ..db_engine import (
db,
Decision,
StratGame,
Player,
model_to_dict,
chunked,
fn,
Team,
Card,
StratPlay,
)
from ..db_helpers import upsert_decisions
from ..dependencies import oauth2_scheme, valid_token, LOG_DATA
logging.basicConfig(
filename=LOG_DATA['filename'],
format=LOG_DATA['format'],
level=LOG_DATA['log_level']
filename=LOG_DATA["filename"],
format=LOG_DATA["format"],
level=LOG_DATA["log_level"],
)
router = APIRouter(
prefix='/api/v2/decisions',
tags=['decisions']
)
router = APIRouter(prefix="/api/v2/decisions", tags=["decisions"])
class DecisionModel(pydantic.BaseModel):
@ -42,14 +51,26 @@ class DecisionList(pydantic.BaseModel):
decisions: List[DecisionModel]
@router.get('')
@router.get("")
async def get_decisions(
season: list = Query(default=None), week: list = Query(default=None), team_id: list = Query(default=None),
win: Optional[int] = None, loss: Optional[int] = None, hold: Optional[int] = None, save: Optional[int] = None,
b_save: Optional[int] = None, irunners: list = Query(default=None), irunners_scored: list = Query(default=None),
game_type: list = Query(default=None),
game_id: list = Query(default=None), player_id: list = Query(default=None), csv: Optional[bool] = False,
limit: Optional[int] = 100, page_num: Optional[int] = 1, short_output: Optional[bool] = False):
season: list = Query(default=None),
week: list = Query(default=None),
team_id: list = Query(default=None),
win: Optional[int] = None,
loss: Optional[int] = None,
hold: Optional[int] = None,
save: Optional[int] = None,
b_save: Optional[int] = None,
irunners: list = Query(default=None),
irunners_scored: list = Query(default=None),
game_type: list = Query(default=None),
game_id: list = Query(default=None),
player_id: list = Query(default=None),
csv: Optional[bool] = False,
limit: Optional[int] = 100,
page_num: Optional[int] = 1,
short_output: Optional[bool] = False,
):
all_dec = Decision.select().order_by(-Decision.season, -Decision.week, -Decision.id)
if season is not None:
@ -88,40 +109,50 @@ async def get_decisions(
all_dec = all_dec.paginate(page_num, limit)
return_dec = {
'count': all_dec.count(),
'decisions': [model_to_dict(x, recurse=not short_output) for x in all_dec]
"count": all_dec.count(),
"decisions": [model_to_dict(x, recurse=not short_output) for x in all_dec],
}
db.close()
if csv:
return_vals = return_dec['decisions']
return_vals = return_dec["decisions"]
if len(return_vals) == 0:
return Response(content=pd.DataFrame().to_csv(index=False), media_type='text/csv')
return Response(
content=pd.DataFrame().to_csv(index=False), media_type="text/csv"
)
for x in return_vals:
x['game_id'] = x['game']['id']
x['game_type'] = x['game']['game_type']
x['player_id'] = x['pitcher']['player_id']
x['player_name'] = x['pitcher']['p_name']
x['player_cardset'] = x['pitcher']['cardset']['name']
x['team_id'] = x['pitcher_team']['id']
x['team_abbrev'] = x['pitcher_team']['abbrev']
del x['pitcher'], x['pitcher_team'], x['game']
x["game_id"] = x["game"]["id"]
x["game_type"] = x["game"]["game_type"]
x["player_id"] = x["pitcher"]["player_id"]
x["player_name"] = x["pitcher"]["p_name"]
x["player_cardset"] = x["pitcher"]["cardset"]["name"]
x["team_id"] = x["pitcher_team"]["id"]
x["team_abbrev"] = x["pitcher_team"]["abbrev"]
del x["pitcher"], x["pitcher_team"], x["game"]
output = pd.DataFrame(return_vals)
first = ['player_id', 'player_name', 'player_cardset', 'team_id', 'team_abbrev']
exclude = first + ['lob_all', 'lob_all_rate', 'lob_2outs', 'rbi%']
first = ["player_id", "player_name", "player_cardset", "team_id", "team_abbrev"]
exclude = first + ["lob_all", "lob_all_rate", "lob_2outs", "rbi%"]
output = output[first + [col for col in output.columns if col not in exclude]]
db.close()
return Response(content=pd.DataFrame(output).to_csv(index=False), media_type='text/csv')
return Response(
content=pd.DataFrame(output).to_csv(index=False), media_type="text/csv"
)
return return_dec
@router.get('/rest')
async def get_decisions_for_rest(team_id: int, season: int = None, limit: int = 80, native_rest: bool = False):
all_dec = Decision.select().order_by(-Decision.season, -Decision.week, -Decision.id).paginate(1, limit)
@router.get("/rest")
async def get_decisions_for_rest(
team_id: int, season: int = None, limit: int = 80, native_rest: bool = False
):
all_dec = (
Decision.select()
.order_by(-Decision.season, -Decision.week, -Decision.id)
.paginate(1, limit)
)
if season is not None:
all_dec = all_dec.where(Decision.season == season)
@ -131,41 +162,61 @@ async def get_decisions_for_rest(team_id: int, season: int = None, limit: int =
return_dec = []
for x in all_dec:
this_val = []
this_card = Card.get_or_none(Card.player_id == x.pitcher.player_id, Card.team_id == x.pitcher_team.id)
this_card = Card.get_or_none(
Card.player_id == x.pitcher.player_id, Card.team_id == x.pitcher_team.id
)
this_val.append(x.game.id)
this_val.append(x.pitcher.player_id)
this_val.append(this_card.id if this_card is not None else -1)
this_val.append(1 if x.is_start else 0)
if not native_rest:
this_line = StratPlay.select(
StratPlay.pitcher, StratPlay.game, fn.SUM(StratPlay.outs).alias('sum_outs')
StratPlay.pitcher,
StratPlay.game,
fn.SUM(StratPlay.outs).alias("sum_outs"),
).where((StratPlay.game == x.game) & (StratPlay.pitcher == x.pitcher))
logging.info(f'this_line: {this_line[0]}')
logging.info(f"this_line: {this_line[0]}")
if this_line[0].sum_outs is None:
this_val.append(0.0)
else:
this_val.append(float(this_line[0].sum_outs // 3) + (float(this_line[0].sum_outs % 3) * .1))
this_val.append(
float(this_line[0].sum_outs // 3)
+ (float(this_line[0].sum_outs % 3) * 0.1)
)
return_dec.append(this_val)
db.close()
return Response(content=pd.DataFrame(return_dec).to_csv(index=False, header=False), media_type='text/csv')
return Response(
content=pd.DataFrame(return_dec).to_csv(index=False, header=False),
media_type="text/csv",
)
@router.patch('/{decision_id}')
@router.patch("/{decision_id}")
async def patch_decision(
decision_id: int, win: Optional[int] = None, loss: Optional[int] = None, hold: Optional[int] = None,
save: Optional[int] = None, b_save: Optional[int] = None, irunners: Optional[int] = None,
irunners_scored: Optional[int] = None, rest_ip: Optional[int] = None, rest_required: Optional[int] = None,
token: str = Depends(oauth2_scheme)):
decision_id: int,
win: Optional[int] = None,
loss: Optional[int] = None,
hold: Optional[int] = None,
save: Optional[int] = None,
b_save: Optional[int] = None,
irunners: Optional[int] = None,
irunners_scored: Optional[int] = None,
rest_ip: Optional[int] = None,
rest_required: Optional[int] = None,
token: str = Depends(oauth2_scheme),
):
if not valid_token(token):
logging.warning(f'patch_decision - Bad Token: {token}')
raise HTTPException(status_code=401, detail='Unauthorized')
logging.warning(f"patch_decision - Bad Token: {token}")
raise HTTPException(status_code=401, detail="Unauthorized")
this_dec = Decision.get_or_none(Decision.id == decision_id)
if this_dec is None:
db.close()
raise HTTPException(status_code=404, detail=f'Decision ID {decision_id} not found')
raise HTTPException(
status_code=404, detail=f"Decision ID {decision_id} not found"
)
if win is not None:
this_dec.win = win
@ -192,72 +243,84 @@ async def patch_decision(
return d_result
else:
db.close()
raise HTTPException(status_code=500, detail=f'Unable to patch decision {decision_id}')
raise HTTPException(
status_code=500, detail=f"Unable to patch decision {decision_id}"
)
@router.post('')
@router.post("")
async def post_decisions(dec_list: DecisionList, token: str = Depends(oauth2_scheme)):
if not valid_token(token):
logging.warning(f'post_decisions - Bad Token: {token}')
raise HTTPException(status_code=401, detail='Unauthorized')
logging.warning(f"post_decisions - Bad Token: {token}")
raise HTTPException(status_code=401, detail="Unauthorized")
new_dec = []
for x in dec_list.decisions:
if StratGame.get_or_none(StratGame.id == x.game_id) is None:
raise HTTPException(status_code=404, detail=f'Game ID {x.game_id} not found')
raise HTTPException(
status_code=404, detail=f"Game ID {x.game_id} not found"
)
if Player.get_or_none(Player.player_id == x.pitcher_id) is None:
raise HTTPException(status_code=404, detail=f'Player ID {x.pitcher_id} not found')
raise HTTPException(
status_code=404, detail=f"Player ID {x.pitcher_id} not found"
)
if Team.get_or_none(Team.id == x.pitcher_team_id) is None:
raise HTTPException(status_code=404, detail=f'Team ID {x.pitcher_team_id} not found')
raise HTTPException(
status_code=404, detail=f"Team ID {x.pitcher_team_id} not found"
)
new_dec.append(x.dict())
with db.atomic():
for batch in chunked(new_dec, 10):
Decision.insert_many(batch).on_conflict_replace().execute()
# Use PostgreSQL-compatible upsert helper
upsert_decisions(new_dec, batch_size=10)
db.close()
return f'Inserted {len(new_dec)} decisions'
return f"Inserted {len(new_dec)} decisions"
@router.delete('/{decision_id}')
@router.delete("/{decision_id}")
async def delete_decision(decision_id: int, token: str = Depends(oauth2_scheme)):
if not valid_token(token):
logging.warning(f'delete_decision - Bad Token: {token}')
raise HTTPException(status_code=401, detail='Unauthorized')
logging.warning(f"delete_decision - Bad Token: {token}")
raise HTTPException(status_code=401, detail="Unauthorized")
this_dec = Decision.get_or_none(Decision.id == decision_id)
if this_dec is None:
db.close()
raise HTTPException(status_code=404, detail=f'Decision ID {decision_id} not found')
raise HTTPException(
status_code=404, detail=f"Decision ID {decision_id} not found"
)
count = this_dec.delete_instance()
db.close()
if count == 1:
return f'Decision {decision_id} has been deleted'
return f"Decision {decision_id} has been deleted"
else:
raise HTTPException(status_code=500, detail=f'Decision {decision_id} could not be deleted')
raise HTTPException(
status_code=500, detail=f"Decision {decision_id} could not be deleted"
)
@router.delete('/game/{game_id}')
@router.delete("/game/{game_id}")
async def delete_decisions_game(game_id: int, token: str = Depends(oauth2_scheme)):
if not valid_token(token):
logging.warning(f'delete_decisions_game - Bad Token: {token}')
raise HTTPException(status_code=401, detail='Unauthorized')
logging.warning(f"delete_decisions_game - Bad Token: {token}")
raise HTTPException(status_code=401, detail="Unauthorized")
this_game = StratGame.get_or_none(StratGame.id == game_id)
if not this_game:
db.close()
raise HTTPException(status_code=404, detail=f'Game ID {game_id} not found')
raise HTTPException(status_code=404, detail=f"Game ID {game_id} not found")
count = Decision.delete().where(Decision.game == this_game).execute()
db.close()
if count > 0:
return f'Deleted {count} decisions matching Game ID {game_id}'
return f"Deleted {count} decisions matching Game ID {game_id}"
else:
raise HTTPException(status_code=500, detail=f'No decisions matching Game ID {game_id} were deleted')
raise HTTPException(
status_code=500,
detail=f"No decisions matching Game ID {game_id} were deleted",
)

77
app/routers_v2/gauntletrewards.py
View File

@ -4,18 +4,16 @@ import logging
import pydantic
from ..db_engine import db, GauntletReward, model_to_dict, chunked, DatabaseError
from ..db_helpers import upsert_gauntlet_rewards
from ..dependencies import oauth2_scheme, valid_token, LOG_DATA
logging.basicConfig(
filename=LOG_DATA['filename'],
format=LOG_DATA['format'],
level=LOG_DATA['log_level']
filename=LOG_DATA["filename"],
format=LOG_DATA["format"],
level=LOG_DATA["log_level"],
)
router = APIRouter(
prefix='/api/v2/gauntletrewards',
tags=['gauntletrewards']
)
router = APIRouter(prefix="/api/v2/gauntletrewards", tags=["gauntletrewards"])
class GauntletRewardModel(pydantic.BaseModel):
@ -30,10 +28,14 @@ class GauntletRewardList(pydantic.BaseModel):
rewards: List[GauntletRewardModel]
@router.get('')
@router.get("")
async def v1_gauntletreward_get(
name: Optional[str] = None, gauntlet_id: Optional[int] = None, reward_id: list = Query(default=None),
win_num: Optional[int] = None, loss_max: Optional[int] = None):
name: Optional[str] = None,
gauntlet_id: Optional[int] = None,
reward_id: list = Query(default=None),
win_num: Optional[int] = None,
loss_max: Optional[int] = None,
):
all_rewards = GauntletReward.select()
if name is not None:
@ -49,44 +51,52 @@ async def v1_gauntletreward_get(
all_rewards = all_rewards.order_by(-GauntletReward.loss_max, GauntletReward.win_num)
return_val = {'count': all_rewards.count(), 'rewards': []}
return_val = {"count": all_rewards.count(), "rewards": []}
for x in all_rewards:
return_val['rewards'].append(model_to_dict(x))
return_val["rewards"].append(model_to_dict(x))
db.close()
return return_val
@router.get('/{gauntletreward_id}')
@router.get("/{gauntletreward_id}")
async def v1_gauntletreward_get_one(gauntletreward_id):
try:
this_reward = GauntletReward.get_by_id(gauntletreward_id)
except Exception:
db.close()
raise HTTPException(status_code=404, detail=f'No gauntlet reward found with id {gauntletreward_id}')
raise HTTPException(
status_code=404,
detail=f"No gauntlet reward found with id {gauntletreward_id}",
)
return_val = model_to_dict(this_reward)
db.close()
return return_val
@router.patch('/{gauntletreward_id}')
@router.patch("/{gauntletreward_id}")
async def v1_gauntletreward_patch(
gauntletreward_id, name: Optional[str] = None, gauntlet_id: Optional[int] = None,
reward_id: Optional[int] = None, win_num: Optional[int] = None, loss_max: Optional[int] = None,
token: str = Depends(oauth2_scheme)):
gauntletreward_id,
name: Optional[str] = None,
gauntlet_id: Optional[int] = None,
reward_id: Optional[int] = None,
win_num: Optional[int] = None,
loss_max: Optional[int] = None,
token: str = Depends(oauth2_scheme),
):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to patch gauntlet rewards. This event has been logged.'
detail="You are not authorized to patch gauntlet rewards. This event has been logged.",
)
this_reward = GauntletReward.get_or_none(GauntletReward.id == gauntletreward_id)
if this_reward is None:
db.close()
raise KeyError(f'Gauntlet Reward ID {gauntletreward_id} not found')
raise KeyError(f"Gauntlet Reward ID {gauntletreward_id} not found")
if gauntlet_id is not None:
this_reward.gauntlet_id = gauntlet_id
@ -105,17 +115,19 @@ async def v1_gauntletreward_patch(
return r_curr
else:
db.close()
raise DatabaseError(f'Unable to patch gauntlet reward {gauntletreward_id}')
raise DatabaseError(f"Unable to patch gauntlet reward {gauntletreward_id}")
@router.post('')
async def v1_gauntletreward_post(gauntletreward: GauntletRewardList, token: str = Depends(oauth2_scheme)):
@router.post("")
async def v1_gauntletreward_post(
gauntletreward: GauntletRewardList, token: str = Depends(oauth2_scheme)
):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to post gauntlets. This event has been logged.'
detail="You are not authorized to post gauntlets. This event has been logged.",
)
all_rewards = []
@ -123,17 +135,16 @@ async def v1_gauntletreward_post(gauntletreward: GauntletRewardList, token: str
all_rewards.append(x.dict())
with db.atomic():
for batch in chunked(all_rewards, 15):
GauntletReward.insert_many(batch).on_conflict_replace().execute()
# Use PostgreSQL-compatible upsert helper
upsert_gauntlet_rewards(all_rewards, batch_size=15)
db.close()
return f'Inserted {len(all_rewards)} gauntlet rewards'
return f"Inserted {len(all_rewards)} gauntlet rewards"
@router.delete('/{gauntletreward_id}')
@router.delete("/{gauntletreward_id}")
async def v1_gauntletreward_delete(gauntletreward_id):
if GauntletReward.delete_by_id(gauntletreward_id) == 1:
return f'Deleted gauntlet reward ID {gauntletreward_id}'
raise DatabaseError(f'Unable to delete gauntlet run {gauntletreward_id}')
return f"Deleted gauntlet reward ID {gauntletreward_id}"
raise DatabaseError(f"Unable to delete gauntlet run {gauntletreward_id}")

218
app/routers_v2/mlbplayers.py
View File

@ -7,19 +7,27 @@ import logging
import pydantic
from pandas import DataFrame
from ..db_engine import db, MlbPlayer, Player, BattingCard, PitchingCard, model_to_dict, fn, chunked, query_to_csv
from ..db_engine import (
db,
MlbPlayer,
Player,
BattingCard,
PitchingCard,
model_to_dict,
fn,
chunked,
query_to_csv,
)
from ..db_helpers import upsert_mlb_players
from ..dependencies import oauth2_scheme, valid_token, LOG_DATA
logging.basicConfig(
filename=LOG_DATA['filename'],
format=LOG_DATA['format'],
level=LOG_DATA['log_level']
filename=LOG_DATA["filename"],
format=LOG_DATA["format"],
level=LOG_DATA["log_level"],
)
router = APIRouter(
prefix='/api/v2/mlbplayers',
tags=['mlbplayers']
)
router = APIRouter(prefix="/api/v2/mlbplayers", tags=["mlbplayers"])
class PlayerModel(pydantic.BaseModel):
@ -37,35 +45,50 @@ class PlayerList(pydantic.BaseModel):
def update_card_urls(mlbplayer: MlbPlayer):
logging.info(f'Updating cards for mlbplayer: {mlbplayer.first_name} {mlbplayer.last_name} ({mlbplayer.key_bbref})')
logging.info(
f"Updating cards for mlbplayer: {mlbplayer.first_name} {mlbplayer.last_name} ({mlbplayer.key_bbref})"
)
now = datetime.datetime.now()
c1_update = Player.update({
Player.image: Player.image.name.split('?d=')[0] + f'?d={now.year}-{now.month}-{now.day}'
}).where(Player.mlbplayer_id == mlbplayer.id)
c1_update = Player.update(
{
Player.image: Player.image.name.split("?d=")[0]
+ f"?d={now.year}-{now.month}-{now.day}"
}
).where(Player.mlbplayer_id == mlbplayer.id)
count = c1_update.execute()
logging.info(f'Updated {count} image1s')
logging.info(f"Updated {count} image1s")
c2_update = Player.update({
Player.image2: Player.image2.name.split('?d=')[0] + f'{now.year}-{now.month}-{now.day}'
}).where((Player.mlbplayer_id == mlbplayer.id) & (Player.image2.is_null(False)))
c2_update = Player.update(
{
Player.image2: Player.image2.name.split("?d=")[0]
+ f"{now.year}-{now.month}-{now.day}"
}
).where((Player.mlbplayer_id == mlbplayer.id) & (Player.image2.is_null(False)))
count2 = c2_update.execute()
logging.info(f'Updated {count2} image2s')
logging.info(f"Updated {count2} image2s")
return count + count2
@router.get('')
@router.get("")
async def get_players(
full_name: list = Query(default=None), first_name: list = Query(default=None),
last_name: list = Query(default=None), key_fangraphs: list = Query(default=None),
key_bbref: list = Query(default=None), key_retro: list = Query(default=None),
key_mlbam: list = Query(default=None), offense_col: list = Query(default=None), csv: Optional[bool] = False):
full_name: list = Query(default=None),
first_name: list = Query(default=None),
last_name: list = Query(default=None),
key_fangraphs: list = Query(default=None),
key_bbref: list = Query(default=None),
key_retro: list = Query(default=None),
key_mlbam: list = Query(default=None),
offense_col: list = Query(default=None),
csv: Optional[bool] = False,
):
all_players = MlbPlayer.select()
if full_name is not None:
name_list = [x.lower() for x in full_name]
all_players = all_players.where(
fn.lower(MlbPlayer.first_name) + ' ' + fn.lower(MlbPlayer.last_name) << name_list
fn.lower(MlbPlayer.first_name) + " " + fn.lower(MlbPlayer.last_name)
<< name_list
)
if first_name is not None:
name_list = [x.lower() for x in first_name]
@ -89,44 +112,56 @@ async def get_players(
if csv:
return_val = query_to_csv(all_players)
db.close()
return Response(content=return_val, media_type='text/csv')
return Response(content=return_val, media_type="text/csv")
return_val = {'count': all_players.count(), 'players': [
model_to_dict(x) for x in all_players
]}
return_val = {
"count": all_players.count(),
"players": [model_to_dict(x) for x in all_players],
}
db.close()
return return_val
@router.get('/{player_id}')
@router.get("/{player_id}")
async def get_one_player(player_id: int):
this_player = MlbPlayer.get_or_none(MlbPlayer.id == player_id)
if this_player is None:
db.close()
raise HTTPException(status_code=404, detail=f'MlbPlayer id {player_id} not found')
raise HTTPException(
status_code=404, detail=f"MlbPlayer id {player_id} not found"
)
r_data = model_to_dict(this_player)
db.close()
return r_data
@router.patch('/{player_id}')
@router.patch("/{player_id}")
async def patch_player(
player_id: int, first_name: Optional[str] = None, last_name: Optional[str] = None,
key_fangraphs: Optional[str] = None, key_bbref: Optional[str] = None, key_retro: Optional[str] = None,
key_mlbam: Optional[str] = None, offense_col: Optional[str] = None, token: str = Depends(oauth2_scheme)):
player_id: int,
first_name: Optional[str] = None,
last_name: Optional[str] = None,
key_fangraphs: Optional[str] = None,
key_bbref: Optional[str] = None,
key_retro: Optional[str] = None,
key_mlbam: Optional[str] = None,
offense_col: Optional[str] = None,
token: str = Depends(oauth2_scheme),
):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to patch mlb players. This event has been logged.'
detail="You are not authorized to patch mlb players. This event has been logged.",
)
this_player = MlbPlayer.get_or_none(MlbPlayer.id == player_id)
if this_player is None:
db.close()
raise HTTPException(status_code=404, detail=f'MlbPlayer id {player_id} not found')
raise HTTPException(
status_code=404, detail=f"MlbPlayer id {player_id} not found"
)
if first_name is not None:
this_player.first_name = first_name
@ -151,65 +186,69 @@ async def patch_player(
db.close()
raise HTTPException(
status_code=418,
detail='Well slap my ass and call me a teapot; I could not save that player'
detail="Well slap my ass and call me a teapot; I could not save that player",
)
@router.post('')
@router.post("")
async def post_players(players: PlayerList, token: str = Depends(oauth2_scheme)):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to post mlb players. This event has been logged.'
detail="You are not authorized to post mlb players. This event has been logged.",
)
new_players = []
for x in players.players:
dupes = MlbPlayer.select().where(
(MlbPlayer.key_fangraphs == x.key_fangraphs) | (MlbPlayer.key_mlbam == x.key_mlbam) |
(MlbPlayer.key_retro == x.key_retro) | (MlbPlayer.key_bbref == x.key_bbref)
(MlbPlayer.key_fangraphs == x.key_fangraphs)
| (MlbPlayer.key_mlbam == x.key_mlbam)
| (MlbPlayer.key_retro == x.key_retro)
| (MlbPlayer.key_bbref == x.key_bbref)
)
if dupes.count() > 0:
db.close()
raise HTTPException(
status_code=400,
detail=f'{x.first_name} {x.last_name} has a key already in the database'
detail=f"{x.first_name} {x.last_name} has a key already in the database",
)
new_players.append(x.dict())
with db.atomic():
for batch in chunked(new_players, 15):
MlbPlayer.insert_many(batch).on_conflict_replace().execute()
# Use PostgreSQL-compatible upsert helper
# Note: Duplicate check is already done above, so this is effectively just insert
upsert_mlb_players(new_players, batch_size=15)
db.close()
return f'Inserted {len(new_players)} new MLB players'
return f"Inserted {len(new_players)} new MLB players"
@router.post('/one')
@router.post("/one")
async def post_one_player(player: PlayerModel, token: str = Depends(oauth2_scheme)):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to post mlb players. This event has been logged.'
detail="You are not authorized to post mlb players. This event has been logged.",
)
dupes = MlbPlayer.select().where(
(MlbPlayer.key_fangraphs == player.key_fangraphs) | (MlbPlayer.key_mlbam == player.key_mlbam) |
(MlbPlayer.key_bbref == player.key_bbref)
(MlbPlayer.key_fangraphs == player.key_fangraphs)
| (MlbPlayer.key_mlbam == player.key_mlbam)
| (MlbPlayer.key_bbref == player.key_bbref)
)
if dupes.count() > 0:
logging.info(f'POST /mlbplayers/one - dupes found:')
logging.info(f"POST /mlbplayers/one - dupes found:")
for x in dupes:
logging.info(f'{x}')
logging.info(f"{x}")
db.close()
raise HTTPException(
status_code=400,
detail=f'{player.first_name} {player.last_name} has a key already in the database'
detail=f"{player.first_name} {player.last_name} has a key already in the database",
)
new_player = MlbPlayer(**player.dict())
@ -221,43 +260,51 @@ async def post_one_player(player: PlayerModel, token: str = Depends(oauth2_schem
else:
raise HTTPException(
status_code=418,
detail='Well slap my ass and call me a teapot; I could not save that player'
detail="Well slap my ass and call me a teapot; I could not save that player",
)
@router.delete('/{player_id}')
@router.delete("/{player_id}")
async def delete_player(player_id: int, token: str = Depends(oauth2_scheme)):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to delete mlb players. This event has been logged.'
detail="You are not authorized to delete mlb players. This event has been logged.",
)
this_player = MlbPlayer.get_or_none(MlbPlayer.id == player_id)
if this_player is None:
db.close()
raise HTTPException(status_code=404, detail=f'MlbPlayer id {player_id} not found')
raise HTTPException(
status_code=404, detail=f"MlbPlayer id {player_id} not found"
)
count = this_player.delete_instance()
db.close()
if count == 1:
raise HTTPException(status_code=200, detail=f'Player {player_id} has been deleted')
raise HTTPException(
status_code=200, detail=f"Player {player_id} has been deleted"
)
else:
raise HTTPException(status_code=500, detail=f'Player {player_id} was not deleted')
raise HTTPException(
status_code=500, detail=f"Player {player_id} was not deleted"
)
@router.post('/update-cols')
@router.post('/update-cols/{mlbplayer_id}')
async def update_columns(mlbplayer_id: Optional[int] = None, token: str = Depends(oauth2_scheme)):
@router.post("/update-cols")
@router.post("/update-cols/{mlbplayer_id}")
async def update_columns(
mlbplayer_id: Optional[int] = None, token: str = Depends(oauth2_scheme)
):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to update mlb players. This event has been logged.'
detail="You are not authorized to update mlb players. This event has been logged.",
)
p_query = MlbPlayer.select()
@ -267,27 +314,34 @@ async def update_columns(mlbplayer_id: Optional[int] = None, token: str = Depend
total_count = 0
for x in p_query:
all_players = Player.select().where(Player.mlbplayer == x)
bc_update = BattingCard.update({
BattingCard.offense_col: x.offense_col,
}).where((BattingCard.player << all_players) & (BattingCard.offense_col != x.offense_col))
bc_update = BattingCard.update(
{
BattingCard.offense_col: x.offense_col,
}
).where(
(BattingCard.player << all_players)
& (BattingCard.offense_col != x.offense_col)
)
count = bc_update.execute()
total_count += count
logging.info(f'Updated {count} batting cards for {x.first_name} {x.last_name}')
logging.info(f"Updated {count} batting cards for {x.first_name} {x.last_name}")
update_card_urls(x)
db.close()
return f'Updated {total_count} batting cards'
return f"Updated {total_count} batting cards"
@router.post('/update-names')
@router.post('/update-names/{mlbplayer_id}')
async def update_names(mlbplayer_id: Optional[int] = None, token: str = Depends(oauth2_scheme)):
@router.post("/update-names")
@router.post("/update-names/{mlbplayer_id}")
async def update_names(
mlbplayer_id: Optional[int] = None, token: str = Depends(oauth2_scheme)
):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to update mlb players. This event has been logged.'
detail="You are not authorized to update mlb players. This event has been logged.",
)
p_query = MlbPlayer.select()
@ -296,16 +350,18 @@ async def update_names(mlbplayer_id: Optional[int] = None, token: str = Depends(
total_count = 0
for x in p_query:
p_update = Player.update({
Player.p_name: f'{x.first_name} {x.last_name}'
}).where((Player.mlbplayer == x) & (Player.p_name != f'{x.first_name} {x.last_name}'))
p_update = Player.update(
{Player.p_name: f"{x.first_name} {x.last_name}"}
).where(
(Player.mlbplayer == x) & (Player.p_name != f"{x.first_name} {x.last_name}")
)
count = p_update.execute()
total_count += count
logging.info(f'Update {count} player records for {x.first_name} {x.last_name}')
logging.info(f"Update {count} player records for {x.first_name} {x.last_name}")
update_card_urls(x)
db.close()
return f'Updated {total_count} names'
return f"Updated {total_count} names"
# @router.post('/link-players')

477
app/routers_v2/pitchingcardratings.py
View File

@ -8,27 +8,34 @@ import pandas as pd
import pydantic
from pydantic import validator, root_validator
from ..db_engine import db, PitchingCardRatings, model_to_dict, chunked, PitchingCard, Player, query_to_csv, Team, \
CardPosition
from ..db_engine import (
db,
PitchingCardRatings,
model_to_dict,
chunked,
PitchingCard,
Player,
query_to_csv,
Team,
CardPosition,
)
from ..db_helpers import upsert_pitching_card_ratings
from ..dependencies import oauth2_scheme, valid_token, LOG_DATA
logging.basicConfig(
filename=LOG_DATA['filename'],
format=LOG_DATA['format'],
level=LOG_DATA['log_level']
filename=LOG_DATA["filename"],
format=LOG_DATA["format"],
level=LOG_DATA["log_level"],
)
router = APIRouter(
prefix='/api/v2/pitchingcardratings',
tags=['pitchingcardratings']
)
RATINGS_FILE = 'storage/pitching-ratings.csv'
BASIC_FILE = 'storage/pitching-basic.csv'
router = APIRouter(prefix="/api/v2/pitchingcardratings", tags=["pitchingcardratings"])
RATINGS_FILE = "storage/pitching-ratings.csv"
BASIC_FILE = "storage/pitching-basic.csv"
class PitchingCardRatingsModel(pydantic.BaseModel):
pitchingcard_id: int
vs_hand: Literal['R', 'L', 'vR', 'vL']
vs_hand: Literal["R", "L", "vR", "vL"]
homerun: float = 0.0
bp_homerun: float = 0.0
triple: float = 0.0
@ -62,30 +69,69 @@ class PitchingCardRatingsModel(pydantic.BaseModel):
@validator("avg", always=True)
def avg_validator(cls, v, values, **kwargs):
return (values['homerun'] + values['bp_homerun'] / 2 + values['triple'] + values['double_three'] +
values['double_two'] + values['double_cf'] + values['single_two'] + values['single_one'] +
values['single_center'] + values['bp_single'] / 2) / 108
return (
values["homerun"]
+ values["bp_homerun"] / 2
+ values["triple"]
+ values["double_three"]
+ values["double_two"]
+ values["double_cf"]
+ values["single_two"]
+ values["single_one"]
+ values["single_center"]
+ values["bp_single"] / 2
) / 108
@validator("obp", always=True)
def obp_validator(cls, v, values, **kwargs):
return ((values['hbp'] + values['walk']) / 108) + values['avg']
return ((values["hbp"] + values["walk"]) / 108) + values["avg"]
@validator("slg", always=True)
def slg_validator(cls, v, values, **kwargs):
return (values['homerun'] * 4 + values['bp_homerun'] * 2 + values['triple'] * 3 + values['double_three'] * 2 +
values['double_two'] * 2 + values['double_cf'] * 2 + values['single_two'] + values['single_one'] +
values['single_center'] + values['bp_single'] / 2) / 108
return (
values["homerun"] * 4
+ values["bp_homerun"] * 2
+ values["triple"] * 3
+ values["double_three"] * 2
+ values["double_two"] * 2
+ values["double_cf"] * 2
+ values["single_two"]
+ values["single_one"]
+ values["single_center"]
+ values["bp_single"] / 2
) / 108
@root_validator(skip_on_failure=True)
def validate_chance_total(cls, values):
total_chances = (
values['homerun'] + values['bp_homerun'] + values['triple'] + values['double_three'] +
values['double_two'] + values['double_cf'] + values['single_two'] + values['single_one'] +
values['single_center'] + values['bp_single'] + values['hbp'] + values['walk'] +
values['strikeout'] + values['flyout_lf_b'] + values['flyout_cf_b'] + values['flyout_rf_b'] +
values['groundout_a'] + values['groundout_b'] + values['xcheck_p'] + values['xcheck_c'] +
values['xcheck_1b'] + values['xcheck_2b'] + values['xcheck_3b'] + values['xcheck_ss'] +
values['xcheck_lf'] + values['xcheck_cf'] + values['xcheck_rf'])
values["homerun"]
+ values["bp_homerun"]
+ values["triple"]
+ values["double_three"]
+ values["double_two"]
+ values["double_cf"]
+ values["single_two"]
+ values["single_one"]
+ values["single_center"]
+ values["bp_single"]
+ values["hbp"]
+ values["walk"]
+ values["strikeout"]
+ values["flyout_lf_b"]
+ values["flyout_cf_b"]
+ values["flyout_rf_b"]
+ values["groundout_a"]
+ values["groundout_b"]
+ values["xcheck_p"]
+ values["xcheck_c"]
+ values["xcheck_1b"]
+ values["xcheck_2b"]
+ values["xcheck_3b"]
+ values["xcheck_ss"]
+ values["xcheck_lf"]
+ values["xcheck_cf"]
+ values["xcheck_rf"]
)
if round(total_chances) != 108:
raise ValueError("Must have exactly 108 chances on the card")
@ -96,39 +142,51 @@ class RatingsList(pydantic.BaseModel):
ratings: List[PitchingCardRatingsModel]
@router.get('')
@router.get("")
async def get_card_ratings(
pitchingcard_id: list = Query(default=None), vs_hand: Literal['R', 'L', 'vR', 'vL'] = None,
short_output: bool = False, csv: bool = False, cardset_id: list = Query(default=None),
token: str = Depends(oauth2_scheme)):
pitchingcard_id: list = Query(default=None),
vs_hand: Literal["R", "L", "vR", "vL"] = None,
short_output: bool = False,
csv: bool = False,
cardset_id: list = Query(default=None),
token: str = Depends(oauth2_scheme),
):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to pull card ratings.'
status_code=401, detail="You are not authorized to pull card ratings."
)
all_ratings = PitchingCardRatings.select()
if pitchingcard_id is not None:
all_ratings = all_ratings.where(PitchingCardRatings.pitchingcard_id << pitchingcard_id)
all_ratings = all_ratings.where(
PitchingCardRatings.pitchingcard_id << pitchingcard_id
)
if vs_hand is not None:
all_ratings = all_ratings.where(PitchingCardRatings.vs_hand == vs_hand[-1])
if cardset_id is not None:
set_players = Player.select(Player.player_id).where(Player.cardset_id << cardset_id)
set_cards = PitchingCard.select(PitchingCard.id).where(PitchingCard.player << set_players)
set_players = Player.select(Player.player_id).where(
Player.cardset_id << cardset_id
)
set_cards = PitchingCard.select(PitchingCard.id).where(
PitchingCard.player << set_players
)
all_ratings = all_ratings.where(PitchingCardRatings.pitchingcard << set_cards)
if csv:
return_val = query_to_csv(all_ratings)
db.close()
return Response(content=return_val, media_type='text/csv')
return Response(content=return_val, media_type="text/csv")
else:
return_val = {'count': all_ratings.count(), 'ratings': [
model_to_dict(x, recurse=not short_output) for x in all_ratings
]}
return_val = {
"count": all_ratings.count(),
"ratings": [
model_to_dict(x, recurse=not short_output) for x in all_ratings
],
}
db.close()
return return_val
@ -136,282 +194,358 @@ async def get_card_ratings(
def get_scouting_dfs(cardset_id: list = None):
all_ratings = PitchingCardRatings.select()
if cardset_id is not None:
set_players = Player.select(Player.player_id).where(Player.cardset_id << cardset_id)
set_cards = PitchingCard.select(PitchingCard.id).where(PitchingCard.player << set_players)
set_players = Player.select(Player.player_id).where(
Player.cardset_id << cardset_id
)
set_cards = PitchingCard.select(PitchingCard.id).where(
PitchingCard.player << set_players
)
all_ratings = all_ratings.where(PitchingCardRatings.pitchingcard << set_cards)
vl_query = all_ratings.where(PitchingCardRatings.vs_hand == 'L')
vr_query = all_ratings.where(PitchingCardRatings.vs_hand == 'R')
vl_query = all_ratings.where(PitchingCardRatings.vs_hand == "L")
vr_query = all_ratings.where(PitchingCardRatings.vs_hand == "R")
vl_vals = [model_to_dict(x) for x in vl_query]
for x in vl_vals:
x.update(x['pitchingcard'])
x['player_id'] = x['pitchingcard']['player']['player_id']
x['player_name'] = x['pitchingcard']['player']['p_name']
x['rarity'] = x['pitchingcard']['player']['rarity']['name']
x['cardset_id'] = x['pitchingcard']['player']['cardset']['id']
x['cardset_name'] = x['pitchingcard']['player']['cardset']['name']
x['starter_rating'] = x['pitchingcard']['starter_rating']
x['relief_rating'] = x['pitchingcard']['relief_rating']
x['closer_rating'] = x['pitchingcard']['closer_rating']
del x['pitchingcard'], x['player']
x.update(x["pitchingcard"])
x["player_id"] = x["pitchingcard"]["player"]["player_id"]
x["player_name"] = x["pitchingcard"]["player"]["p_name"]
x["rarity"] = x["pitchingcard"]["player"]["rarity"]["name"]
x["cardset_id"] = x["pitchingcard"]["player"]["cardset"]["id"]
x["cardset_name"] = x["pitchingcard"]["player"]["cardset"]["name"]
x["starter_rating"] = x["pitchingcard"]["starter_rating"]
x["relief_rating"] = x["pitchingcard"]["relief_rating"]
x["closer_rating"] = x["pitchingcard"]["closer_rating"]
del x["pitchingcard"], x["player"]
vr_vals = [model_to_dict(x) for x in vr_query]
for x in vr_vals:
x['player_id'] = x['pitchingcard']['player']['player_id']
del x['pitchingcard']
x["player_id"] = x["pitchingcard"]["player"]["player_id"]
del x["pitchingcard"]
vl = pd.DataFrame(vl_vals)
vr = pd.DataFrame(vr_vals)
pit_df = pd.merge(vl, vr, on='player_id', suffixes=('_vl', '_vr')).set_index('player_id', drop=False)
logging.debug(f'pit_df: {pit_df}')
pit_df = pd.merge(vl, vr, on="player_id", suffixes=("_vl", "_vr")).set_index(
"player_id", drop=False
)
logging.debug(f"pit_df: {pit_df}")
positions = CardPosition.select().where(CardPosition.position == 'P')
positions = CardPosition.select().where(CardPosition.position == "P")
if cardset_id is not None:
set_players = Player.select(Player.player_id).where(Player.cardset_id << cardset_id)
set_players = Player.select(Player.player_id).where(
Player.cardset_id << cardset_id
)
positions = positions.where(CardPosition.player << set_players)
series_list = [pd.Series(
dict([(x.player.player_id, x.range) for x in positions]),
name=f'Range P'
), pd.Series(
dict([(x.player.player_id, x.error) for x in positions]),
name=f'Error P'
)]
series_list = [
pd.Series(
dict([(x.player.player_id, x.range) for x in positions]), name=f"Range P"
),
pd.Series(
dict([(x.player.player_id, x.error) for x in positions]), name=f"Error P"
),
]
db.close()
logging.debug(f'series_list: {series_list}')
logging.debug(f"series_list: {series_list}")
return pit_df.join(series_list)
@router.get('/scouting')
@router.get("/scouting")
async def get_card_scouting(team_id: int, ts: str):
this_team = Team.get_or_none(Team.id == team_id)
logging.debug(f'Team: {this_team} / has_guide: {this_team.has_guide}')
logging.debug(f"Team: {this_team} / has_guide: {this_team.has_guide}")
if this_team is None or ts != this_team.team_hash() or this_team.has_guide != 1:
logging.warning(f'Team_id {team_id} attempted to pull ratings')
logging.warning(f"Team_id {team_id} attempted to pull ratings")
db.close()
return 'Your team does not have the ratings guide enabled. If you have purchased a copy ping Cal to ' \
'make sure it is enabled on your team. If you are interested you can pick it up here (thank you!): ' \
'https://ko-fi.com/manticorum/shop'
return (
"Your team does not have the ratings guide enabled. If you have purchased a copy ping Cal to "
"make sure it is enabled on your team. If you are interested you can pick it up here (thank you!): "
"https://ko-fi.com/manticorum/shop"
)
if os.path.isfile(RATINGS_FILE):
return FileResponse(
path=RATINGS_FILE,
media_type='text/csv',
media_type="text/csv",
# headers=headers
)
raise HTTPException(status_code=400, detail='Go pester Cal - the scouting file is missing')
raise HTTPException(
status_code=400, detail="Go pester Cal - the scouting file is missing"
)
@router.post('/calculate/scouting')
@router.post("/calculate/scouting")
async def post_calc_scouting(token: str = Depends(oauth2_scheme)):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to calculate card ratings.'
status_code=401, detail="You are not authorized to calculate card ratings."
)
logging.warning(f'Re-calculating pitching ratings\n\n')
logging.warning(f"Re-calculating pitching ratings\n\n")
output = get_scouting_dfs()
first = ['player_id', 'player_name', 'cardset_name', 'rarity', 'hand', 'variant']
exclude = first + ['id_vl', 'id_vr', 'vs_hand_vl', 'vs_hand_vr']
first = ["player_id", "player_name", "cardset_name", "rarity", "hand", "variant"]
exclude = first + ["id_vl", "id_vr", "vs_hand_vl", "vs_hand_vr"]
output = output[first + [col for col in output.columns if col not in exclude]]
csv_file = pd.DataFrame(output).to_csv(index=False)
with open(RATINGS_FILE, 'w') as file:
with open(RATINGS_FILE, "w") as file:
file.write(csv_file)
return Response(content=csv_file, media_type='text/csv')
return Response(content=csv_file, media_type="text/csv")
@router.get('/basic')
@router.get("/basic")
async def get_basic_scouting():
if os.path.isfile(BASIC_FILE):
return FileResponse(
path=BASIC_FILE,
media_type='text/csv',
media_type="text/csv",
# headers=headers
)
raise HTTPException(status_code=400, detail='Go pester Cal - the scouting file is missing')
raise HTTPException(
status_code=400, detail="Go pester Cal - the scouting file is missing"
)
@router.post('/calculate/basic')
@router.post("/calculate/basic")
async def post_calc_basic(token: str = Depends(oauth2_scheme)):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to calculate basic ratings.'
status_code=401, detail="You are not authorized to calculate basic ratings."
)
logging.warning(f'Re-calculating basic pitching ratings\n\n')
logging.warning(f"Re-calculating basic pitching ratings\n\n")
raw_data = get_scouting_dfs()
logging.debug(f'output: {raw_data}')
logging.debug(f"output: {raw_data}")
def get_raw_leftcontrol(df_data):
return ((1 - (df_data['obp_vl'] - df_data['avg_vl'])) * 100) + (1 - (df_data['wild_pitch'] / 20))
return ((1 - (df_data["obp_vl"] - df_data["avg_vl"])) * 100) + (
1 - (df_data["wild_pitch"] / 20)
)
raw_series = raw_data.apply(get_raw_leftcontrol, axis=1)
rank_series = raw_series.rank(pct=True)
raw_data['Control L'] = round(rank_series * 100)
raw_data["Control L"] = round(rank_series * 100)
def get_raw_rightcontrol(df_data):
return ((1 - (df_data['obp_vr'] - df_data['avg_vr'])) * 100) + (1 - (df_data['wild_pitch'] / 20))
return ((1 - (df_data["obp_vr"] - df_data["avg_vr"])) * 100) + (
1 - (df_data["wild_pitch"] / 20)
)
raw_series = raw_data.apply(get_raw_rightcontrol, axis=1)
rank_series = raw_series.rank(pct=True)
raw_data['Control R'] = round(rank_series * 100)
raw_data["Control R"] = round(rank_series * 100)
def get_raw_leftstuff(df_data):
return 10 - (df_data['slg_vl'] + df_data['slg_vl'] + ((df_data['homerun_vl'] + df_data['bp_homerun_vl']) / 108))
return 10 - (
df_data["slg_vl"]
+ df_data["slg_vl"]
+ ((df_data["homerun_vl"] + df_data["bp_homerun_vl"]) / 108)
)
raw_series = raw_data.apply(get_raw_leftstuff, axis=1)
rank_series = raw_series.rank(pct=True)
raw_data['Stuff L'] = round(rank_series * 100)
raw_data["Stuff L"] = round(rank_series * 100)
def get_raw_rightstuff(df_data):
return 10 - (df_data['slg_vr'] + df_data['slg_vr'] + ((df_data['homerun_vr'] + df_data['bp_homerun_vr']) / 108))
return 10 - (
df_data["slg_vr"]
+ df_data["slg_vr"]
+ ((df_data["homerun_vr"] + df_data["bp_homerun_vr"]) / 108)
)
raw_series = raw_data.apply(get_raw_rightstuff, axis=1)
rank_series = raw_series.rank(pct=True)
raw_data['Stuff R'] = round(rank_series * 100)
raw_data["Stuff R"] = round(rank_series * 100)
def get_raw_fielding(df_data):
return ((6 - df_data['Range P']) * 10) + (50 - df_data['Error P'])
return ((6 - df_data["Range P"]) * 10) + (50 - df_data["Error P"])
raw_series = raw_data.apply(get_raw_fielding, axis=1)
rank_series = raw_series.rank(pct=True)
logging.debug(f'max fld: {raw_series.max()} / min fld: {raw_series.min()}')
raw_data['Fielding'] = round(rank_series * 100)
logging.debug(f"max fld: {raw_series.max()} / min fld: {raw_series.min()}")
raw_data["Fielding"] = round(rank_series * 100)
def get_raw_stamina(df_data):
spow = df_data['starter_rating'] if pd.isna(df_data['starter_rating']) else -1
rpow = df_data['relief_rating'] if pd.isna(df_data['relief_rating']) else -1
spow = df_data["starter_rating"] if pd.isna(df_data["starter_rating"]) else -1
rpow = df_data["relief_rating"] if pd.isna(df_data["relief_rating"]) else -1
this_pow = spow if spow > rpow else rpow
return (((this_pow * (df_data['obp_vr'] * (2 / 3))) + (this_pow * (df_data['obp_vl'] / 3))) * 4.5) + this_pow
return (
(
(this_pow * (df_data["obp_vr"] * (2 / 3)))
+ (this_pow * (df_data["obp_vl"] / 3))
)
* 4.5
) + this_pow
raw_series = raw_data.apply(get_raw_stamina, axis=1)
rank_series = raw_series.rank(pct=True)
raw_data['Stamina'] = round(rank_series * 100)
raw_data["Stamina"] = round(rank_series * 100)
def get_raw_hit(df_data):
return 1 - (df_data['avg_vr'] * (2 / 3)) + (df_data['avg_vl'] / 3)
return 1 - (df_data["avg_vr"] * (2 / 3)) + (df_data["avg_vl"] / 3)
raw_series = raw_data.apply(get_raw_hit, axis=1)
rank_series = raw_series.rank(pct=True)
raw_data['H/9'] = round(rank_series * 100)
raw_data["H/9"] = round(rank_series * 100)
def get_raw_k(df_data):
return ((df_data['strikeout_vr'] / 108) * (2 / 3)) + ((df_data['strikeout_vl'] / 108) / 3)
return ((df_data["strikeout_vr"] / 108) * (2 / 3)) + (
(df_data["strikeout_vl"] / 108) / 3
)
raw_series = raw_data.apply(get_raw_k, axis=1)
rank_series = raw_series.rank(pct=True)
raw_data['K/9'] = round(rank_series * 100)
raw_data["K/9"] = round(rank_series * 100)
def get_raw_bb(df_data):
return ((df_data['walk_vr'] / 108) * (2 / 3)) + ((df_data['walk_vl'] / 108) / 3)
return ((df_data["walk_vr"] / 108) * (2 / 3)) + ((df_data["walk_vl"] / 108) / 3)
raw_series = raw_data.apply(get_raw_bb, axis=1)
rank_series = raw_series.rank(pct=True, ascending=False)
raw_data['BB/9'] = round(rank_series * 100)
raw_data["BB/9"] = round(rank_series * 100)
def get_raw_hr(df_data):
return 1 - (
(((df_data['homerun_vr'] + df_data['bp_homerun_vr']) / 108) * (2 / 3)) +
(((df_data['homerun_vl'] + df_data['bp_homerun_vl']) / 108) / 3))
(((df_data["homerun_vr"] + df_data["bp_homerun_vr"]) / 108) * (2 / 3))
+ (((df_data["homerun_vl"] + df_data["bp_homerun_vl"]) / 108) / 3)
)
raw_series = raw_data.apply(get_raw_hr, axis=1)
rank_series = raw_series.rank(pct=True)
raw_data['HR/9'] = round(rank_series * 100)
raw_data["HR/9"] = round(rank_series * 100)
def get_raw_rating(df_data):
spow = df_data['starter_rating'] if pd.isna(df_data['starter_rating']) else -1
rpow = df_data['relief_rating'] if pd.isna(df_data['relief_rating']) else -1
spow = df_data["starter_rating"] if pd.isna(df_data["starter_rating"]) else -1
rpow = df_data["relief_rating"] if pd.isna(df_data["relief_rating"]) else -1
if spow > rpow and spow >= 4:
return (
((df_data['H/9'] + df_data['K/9'] + df_data['BB/9'] + df_data['HR/9']) * 5) +
(df_data['Fielding']) + (df_data['Stamina'] * 5) +
(((df_data['Stuff L'] / 3) + (df_data['Stuff R'] * (2 / 3))) * 4) +
(((df_data['Control L'] / 3) + (df_data['Control R'] * (2 / 3))) * 2)
(
(
df_data["H/9"]
+ df_data["K/9"]
+ df_data["BB/9"]
+ df_data["HR/9"]
)
* 5
)
+ (df_data["Fielding"])
+ (df_data["Stamina"] * 5)
+ (((df_data["Stuff L"] / 3) + (df_data["Stuff R"] * (2 / 3))) * 4)
+ (((df_data["Control L"] / 3) + (df_data["Control R"] * (2 / 3))) * 2)
)
else:
return (
((df_data['H/9'] + df_data['K/9'] + df_data['BB/9'] + df_data['HR/9']) * 5) +
(df_data['Fielding']) + (df_data['Stamina'] * 5) +
(((df_data['Stuff L'] / 3) + (df_data['Stuff R'] * (2 / 3))) * 4) +
(((df_data['Control L'] / 3) + (df_data['Control R'] * (2 / 3))) * 2)
(
(
df_data["H/9"]
+ df_data["K/9"]
+ df_data["BB/9"]
+ df_data["HR/9"]
)
* 5
)
+ (df_data["Fielding"])
+ (df_data["Stamina"] * 5)
+ (((df_data["Stuff L"] / 3) + (df_data["Stuff R"] * (2 / 3))) * 4)
+ (((df_data["Control L"] / 3) + (df_data["Control R"] * (2 / 3))) * 2)
)
raw_series = raw_data.apply(get_raw_rating, axis=1)
rank_series = raw_series.rank(pct=True)
raw_data['Rating'] = round(rank_series * 100)
raw_data["Rating"] = round(rank_series * 100)
output = raw_data[[
'player_id', 'player_name', 'Rating', 'Control R', 'Control L', 'Stuff R', 'Stuff L', 'Stamina', 'Fielding',
'H/9', 'K/9', 'BB/9', 'HR/9', 'hand', 'cardset_name'
]]
output = raw_data[
[
"player_id",
"player_name",
"Rating",
"Control R",
"Control L",
"Stuff R",
"Stuff L",
"Stamina",
"Fielding",
"H/9",
"K/9",
"BB/9",
"HR/9",
"hand",
"cardset_name",
]
]
csv_file = pd.DataFrame(output).to_csv(index=False)
with open(BASIC_FILE, 'w') as file:
with open(BASIC_FILE, "w") as file:
file.write(csv_file)
return Response(content=csv_file, media_type='text/csv')
return Response(content=csv_file, media_type="text/csv")
@router.get('/{ratings_id}')
@router.get("/{ratings_id}")
async def get_one_rating(ratings_id: int, token: str = Depends(oauth2_scheme)):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to pull card ratings.'
status_code=401, detail="You are not authorized to pull card ratings."
)
this_rating = PitchingCardRatings.get_or_none(PitchingCardRatings.id == ratings_id)
if this_rating is None:
db.close()
raise HTTPException(status_code=404, detail=f'PitchingCardRating id {ratings_id} not found')
raise HTTPException(
status_code=404, detail=f"PitchingCardRating id {ratings_id} not found"
)
r_data = model_to_dict(this_rating)
db.close()
return r_data
@router.get('/player/{player_id}')
async def get_player_ratings(player_id: int, variant: list = Query(default=None), short_output: bool = False):
all_cards = PitchingCard.select().where(PitchingCard.player_id == player_id).order_by(PitchingCard.variant)
@router.get("/player/{player_id}")
async def get_player_ratings(
player_id: int, variant: list = Query(default=None), short_output: bool = False
):
all_cards = (
PitchingCard.select()
.where(PitchingCard.player_id == player_id)
.order_by(PitchingCard.variant)
)
if variant is not None:
all_cards = all_cards.where(PitchingCard.variant << variant)
all_ratings = PitchingCardRatings.select().where(PitchingCardRatings.pitchingcard << all_cards)
all_ratings = PitchingCardRatings.select().where(
PitchingCardRatings.pitchingcard << all_cards
)
return_val = {'count': all_ratings.count(), 'ratings': [
model_to_dict(x, recurse=not short_output) for x in all_ratings
]}
return_val = {
"count": all_ratings.count(),
"ratings": [model_to_dict(x, recurse=not short_output) for x in all_ratings],
}
db.close()
return return_val
@router.put('')
@router.put("")
async def put_ratings(ratings: RatingsList, token: str = Depends(oauth2_scheme)):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to post card ratings.'
status_code=401, detail="You are not authorized to post card ratings."
)
new_ratings = []
@ -419,43 +553,50 @@ async def put_ratings(ratings: RatingsList, token: str = Depends(oauth2_scheme))
for x in ratings.ratings:
try:
PitchingCardRatings.get(
(PitchingCardRatings.pitchingcard_id == x.pitchingcard_id) & (PitchingCardRatings.vs_hand == x.vs_hand)
(PitchingCardRatings.pitchingcard_id == x.pitchingcard_id)
& (PitchingCardRatings.vs_hand == x.vs_hand)
)
updates += (
PitchingCardRatings.update(x.dict())
.where(
(PitchingCardRatings.pitchingcard_id == x.pitchingcard_id)
& (PitchingCardRatings.vs_hand == x.vs_hand)
)
.execute()
)
updates += PitchingCardRatings.update(x.dict()).where(
(PitchingCardRatings.pitchingcard_id == x.pitchingcard_id) & (PitchingCardRatings.vs_hand == x.vs_hand)
).execute()
except PitchingCardRatings.DoesNotExist:
new_ratings.append(x.dict())
with db.atomic():
for batch in chunked(new_ratings, 30):
PitchingCardRatings.insert_many(batch).on_conflict_replace().execute()
# Use PostgreSQL-compatible upsert helper
upsert_pitching_card_ratings(new_ratings, batch_size=30)
db.close()
return f'Updated ratings: {updates}; new ratings: {len(new_ratings)}'
return f"Updated ratings: {updates}; new ratings: {len(new_ratings)}"
@router.delete('/{ratings_id}')
async def delete_rating(
ratings_id: int, token: str = Depends(oauth2_scheme)):
@router.delete("/{ratings_id}")
async def delete_rating(ratings_id: int, token: str = Depends(oauth2_scheme)):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to post card ratings.'
status_code=401, detail="You are not authorized to post card ratings."
)
this_rating = PitchingCardRatings.get_or_none(PitchingCardRatings.id == ratings_id)
if this_rating is None:
db.close()
raise HTTPException(status_code=404, detail=f'PitchingCardRating id {ratings_id} not found')
raise HTTPException(
status_code=404, detail=f"PitchingCardRating id {ratings_id} not found"
)
count = this_rating.delete_instance()
db.close()
if count == 1:
return f'Rating {this_rating} has been deleted'
return f"Rating {this_rating} has been deleted"
else:
raise HTTPException(status_code=500, detail=f'Rating {this_rating} could not be deleted')
raise HTTPException(
status_code=500, detail=f"Rating {this_rating} could not be deleted"
)

143
app/routers_v2/pitchingcards.py
View File

@ -6,18 +6,16 @@ import logging
import pydantic
from ..db_engine import db, PitchingCard, model_to_dict, chunked, Player, fn, MlbPlayer
from ..db_helpers import upsert_pitching_cards
from ..dependencies import oauth2_scheme, valid_token, LOG_DATA
logging.basicConfig(
filename=LOG_DATA['filename'],
format=LOG_DATA['format'],
level=LOG_DATA['log_level']
filename=LOG_DATA["filename"],
format=LOG_DATA["format"],
level=LOG_DATA["log_level"],
)
router = APIRouter(
prefix='/api/v2/pitchingcards',
tags=['pitchingcards']
)
router = APIRouter(prefix="/api/v2/pitchingcards", tags=["pitchingcards"])
class PitchingCardModel(pydantic.BaseModel):
@ -31,17 +29,21 @@ class PitchingCardModel(pydantic.BaseModel):
closer_rating: int = None
batting: str = "#1WR-C"
offense_col: int = None
hand: Literal['R', 'L', 'S'] = 'R'
hand: Literal["R", "L", "S"] = "R"
class PitchingCardList(pydantic.BaseModel):
cards: List[PitchingCardModel]
@router.get('')
@router.get("")
async def get_pitching_cards(
player_id: list = Query(default=None), player_name: list = Query(default=None),
cardset_id: list = Query(default=None), short_output: bool = False, limit: Optional[int] = None):
player_id: list = Query(default=None),
player_name: list = Query(default=None),
cardset_id: list = Query(default=None),
short_output: bool = False,
limit: Optional[int] = None,
):
all_cards = PitchingCard.select()
if player_id is not None:
all_cards = all_cards.where(PitchingCard.player_id << player_id)
@ -56,46 +58,56 @@ async def get_pitching_cards(
if limit is not None:
all_cards = all_cards.limit(limit)
return_val = {'count': all_cards.count(), 'cards': [
model_to_dict(x, recurse=not short_output) for x in all_cards
]}
return_val = {
"count": all_cards.count(),
"cards": [model_to_dict(x, recurse=not short_output) for x in all_cards],
}
db.close()
return return_val
@router.get('/{card_id}')
@router.get("/{card_id}")
async def get_one_card(card_id: int):
this_card = PitchingCard.get_or_none(PitchingCard.id == card_id)
if this_card is None:
db.close()
raise HTTPException(status_code=404, detail=f'PitchingCard id {card_id} not found')
raise HTTPException(
status_code=404, detail=f"PitchingCard id {card_id} not found"
)
r_card = model_to_dict(this_card)
db.close()
return r_card
@router.get('/player/{player_id}')
async def get_player_cards(player_id: int, variant: list = Query(default=None), short_output: bool = False):
all_cards = PitchingCard.select().where(PitchingCard.player_id == player_id).order_by(PitchingCard.variant)
@router.get("/player/{player_id}")
async def get_player_cards(
player_id: int, variant: list = Query(default=None), short_output: bool = False
):
all_cards = (
PitchingCard.select()
.where(PitchingCard.player_id == player_id)
.order_by(PitchingCard.variant)
)
if variant is not None:
all_cards = all_cards.where(PitchingCard.variant << variant)
return_val = {'count': all_cards.count(), 'cards': [
model_to_dict(x, recurse=not short_output) for x in all_cards
]}
return_val = {
"count": all_cards.count(),
"cards": [model_to_dict(x, recurse=not short_output) for x in all_cards],
}
db.close()
return return_val
@router.put('')
@router.put("")
async def put_cards(cards: PitchingCardList, token: str = Depends(oauth2_scheme)):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to post pitching cards. This event has been logged.'
detail="You are not authorized to post pitching cards. This event has been logged.",
)
new_cards = []
@ -104,52 +116,73 @@ async def put_cards(cards: PitchingCardList, token: str = Depends(oauth2_scheme)
for x in cards.cards:
try:
old = PitchingCard.get(
(PitchingCard.player_id == x.player_id) & (PitchingCard.variant == x.variant)
(PitchingCard.player_id == x.player_id)
& (PitchingCard.variant == x.variant)
)
if x.offense_col is None:
x.offense_col = old.offense_col
updates += PitchingCard.update(x.dict()).where(
(PitchingCard.player_id == x.player_id) & (PitchingCard.variant == x.variant)
).execute()
updates += (
PitchingCard.update(x.dict())
.where(
(PitchingCard.player_id == x.player_id)
& (PitchingCard.variant == x.variant)
)
.execute()
)
except PitchingCard.DoesNotExist:
if x.offense_col is None:
this_player = Player.get_or_none(Player.player_id == x.player_id)
mlb_player = MlbPlayer.get_or_none(MlbPlayer.key_bbref == this_player.bbref_id)
mlb_player = MlbPlayer.get_or_none(
MlbPlayer.key_bbref == this_player.bbref_id
)
if mlb_player is not None:
logging.info(f'setting offense_col to {mlb_player.offense_col} for {this_player.p_name}')
logging.info(
f"setting offense_col to {mlb_player.offense_col} for {this_player.p_name}"
)
x.offense_col = mlb_player.offense_col
else:
logging.info(f'randomly setting offense_col for {this_player.p_name}')
logging.info(
f"randomly setting offense_col for {this_player.p_name}"
)
x.offense_col = random.randint(1, 3)
logging.debug(f'x.dict(): {x.dict()}')
logging.debug(f"x.dict(): {x.dict()}")
new_cards.append(x.dict())
with db.atomic():
for batch in chunked(new_cards, 30):
PitchingCard.insert_many(batch).on_conflict_replace().execute()
# Use PostgreSQL-compatible upsert helper
upsert_pitching_cards(new_cards, batch_size=30)
db.close()
return f'Updated cards: {updates}; new cards: {len(new_cards)}'
return f"Updated cards: {updates}; new cards: {len(new_cards)}"
@router.patch('/{card_id}')
@router.patch("/{card_id}")
async def patch_card(
card_id: int, balk: Optional[int] = None, wild_pitch: Optional[int] = None, hold: Optional[int] = None,
starter_rating: Optional[int] = None, relief_rating: Optional[int] = None, closer_rating: Optional[int] = None,
batting: Optional[int] = None, token: str = Depends(oauth2_scheme)):
card_id: int,
balk: Optional[int] = None,
wild_pitch: Optional[int] = None,
hold: Optional[int] = None,
starter_rating: Optional[int] = None,
relief_rating: Optional[int] = None,
closer_rating: Optional[int] = None,
batting: Optional[int] = None,
token: str = Depends(oauth2_scheme),
):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to patch pitching cards. This event has been logged.'
detail="You are not authorized to patch pitching cards. This event has been logged.",
)
this_card = PitchingCard.get_or_none(PitchingCard.id == card_id)
if this_card is None:
db.close()
raise HTTPException(status_code=404, detail=f'PitchingCard id {card_id} not found')
raise HTTPException(
status_code=404, detail=f"PitchingCard id {card_id} not found"
)
if balk is not None:
this_card.balk = balk
@ -174,45 +207,47 @@ async def patch_card(
db.close()
raise HTTPException(
status_code=418,
detail='Well slap my ass and call me a teapot; I could not save that card'
detail="Well slap my ass and call me a teapot; I could not save that card",
)
@router.delete('/{card_id}')
@router.delete("/{card_id}")
async def delete_card(card_id: int, token: str = Depends(oauth2_scheme)):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to delete pitching cards. This event has been logged.'
detail="You are not authorized to delete pitching cards. This event has been logged.",
)
this_card = PitchingCard.get_or_none(PitchingCard.id == card_id)
if this_card is None:
db.close()
raise HTTPException(status_code=404, detail=f'Pitching id {card_id} not found')
raise HTTPException(status_code=404, detail=f"Pitching id {card_id} not found")
count = this_card.delete_instance()
db.close()
if count == 1:
return f'Card {this_card} has been deleted'
return f"Card {this_card} has been deleted"
else:
raise HTTPException(status_code=500, detail=f'Card {this_card} could not be deleted')
raise HTTPException(
status_code=500, detail=f"Card {this_card} could not be deleted"
)
@router.delete('')
@router.delete("")
async def delete_all_cards(token: str = Depends(oauth2_scheme)):
if not valid_token(token):
logging.warning(f'Bad Token: {token}')
logging.warning(f"Bad Token: {token}")
db.close()
raise HTTPException(
status_code=401,
detail='You are not authorized to delete pitching cards. This event has been logged.'
detail="You are not authorized to delete pitching cards. This event has been logged.",
)
d_query = PitchingCard.delete()
d_query.execute()
return f'Deleted {d_query.count()} pitching cards'
return f"Deleted {d_query.count()} pitching cards"

831
app/routers_v2/players.py
View File

File diff suppressed because it is too large Load Diff

1465
app/routers_v2/stratplays.py
View File

File diff suppressed because it is too large Load Diff

1125
app/routers_v2/teams.py
View File

File diff suppressed because it is too large Load Diff

3638
main.py
View File

File diff suppressed because it is too large Load Diff

92
scripts/audit_results.json Normal file
View File

@ -0,0 +1,92 @@
{
"generated_at": "2026-01-25T23:04:23.049271",
"summary": {
"total_tables": 29,
"total_records": 658963,
"total_issues": 3,
"critical_issues": 0,
"high_issues": 3,
"medium_issues": 0
},
"table_counts": {
"award": 0,
"battingcard": 6036,
"battingcardratings": 12072,
"battingstat": 50228,
"card": 61746,
"cardposition": 18654,
"cardset": 29,
"current": 1,
"decision": 29694,
"event": 9,
"gamerewards": 10,
"gauntletreward": 45,
"gauntletrun": 458,
"mlbplayer": 4781,
"notification": 14362,
"pack": 20595,
"packtype": 9,
"paperdex": 46121,
"pitchingcard": 6887,
"pitchingcardratings": 13774,
"pitchingstat": 13135,
"player": 12964,
"rarity": 6,
"result": 2235,
"reward": 8043,
"roster": 23,
"stratgame": 4208,
"stratplay": 332737,
"team": 101
},
"issues": [
{
"type": "ORPHANED_FK",
"severity": "HIGH",
"child_table": "battingstat",
"child_field": "card_id",
"parent_table": "card",
"parent_field": "id",
"description": "Batting stats referencing non-existent cards",
"orphan_count": 1953,
"sample_orphan_ids": [
1419,
1419,
1419,
1419,
1433
]
},
{
"type": "ORPHANED_FK",
"severity": "HIGH",
"child_table": "pitchingstat",
"child_field": "card_id",
"parent_table": "card",
"parent_field": "id",
"description": "Pitching stats referencing non-existent cards",
"orphan_count": 437,
"sample_orphan_ids": [
1412,
1660,
2045,
2046,
2061
]
},
{
"type": "VARCHAR_TOO_LONG",
"severity": "HIGH",
"table": "team",
"field": "abbrev",
"description": "Team abbreviation",
"max_found": 13,
"expected_max": 10,
"sample_values": [
"Gauntlet-KC...",
"Gauntlet-NCB...",
"Gauntlet-SLV..."
]
}
]
}

564
scripts/audit_sqlite.py Executable file
View File

@ -0,0 +1,564 @@
#!/usr/bin/env python3
"""
Paper Dynasty SQLite Data Integrity Audit
Pre-migration script to identify potential issues before migrating to PostgreSQL.
Based on issues discovered during Major Domo migration (August 2025).
Checks for:
1. NULL values in fields that will be NOT NULL in PostgreSQL
2. Orphaned foreign key records
3. VARCHAR field max lengths (PostgreSQL is stricter)
4. Record counts for baseline comparison
5. Primary key gaps or duplicates
Usage:
python scripts/audit_sqlite.py
python scripts/audit_sqlite.py --fix # Apply safe fixes
"""
import argparse
import json
import sqlite3
import sys
from datetime import datetime
from pathlib import Path
def connect_db(db_path: str) -> sqlite3.Connection:
"""Connect to SQLite database."""
conn = sqlite3.connect(db_path)
conn.row_factory = sqlite3.Row
return conn
def get_table_record_counts(conn: sqlite3.Connection) -> dict:
"""Get record counts for all tables."""
counts = {}
cursor = conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
)
for row in cursor:
table_name = row["name"]
count_cursor = conn.execute(f"SELECT COUNT(*) FROM {table_name}")
counts[table_name] = count_cursor.fetchone()[0]
return counts
def check_null_values(conn: sqlite3.Connection) -> list:
"""
Check for NULL values in fields that should not be null.
These are the fields that Major Domo found issues with.
"""
issues = []
# Fields to check - based on Major Domo experience
null_checks = [
# (table, field, description)
("team", "abbrev", "Team abbreviation"),
("team", "sname", "Team short name"),
("team", "lname", "Team long name"),
("player", "p_name", "Player name"),
("player", "image", "Player image URL"),
("card", "player_id", "Card player reference"),
("stratplay", "game_id", "Play game reference"),
("stratplay", "pitcher_id", "Play pitcher reference"),
("decision", "game_id", "Decision game reference"),
("decision", "pitcher_id", "Decision pitcher reference"),
]
for table, field, description in null_checks:
try:
cursor = conn.execute(f"SELECT COUNT(*) FROM {table} WHERE {field} IS NULL")
null_count = cursor.fetchone()[0]
if null_count > 0:
cursor = conn.execute(f"SELECT COUNT(*) FROM {table}")
total_count = cursor.fetchone()[0]
issues.append(
{
"type": "NULL_VALUE",
"severity": "HIGH"
if null_count > total_count * 0.1
else "MEDIUM",
"table": table,
"field": field,
"description": description,
"null_count": null_count,
"total_count": total_count,
"percentage": round(null_count / total_count * 100, 2)
if total_count > 0
else 0,
}
)
except sqlite3.OperationalError:
# Table or column doesn't exist
pass
return issues
def check_orphaned_foreign_keys(conn: sqlite3.Connection) -> list:
"""
Check for orphaned foreign key records.
These will fail with foreign key constraint violations in PostgreSQL.
"""
issues = []
# Foreign key relationships to check
fk_checks = [
# (child_table, child_field, parent_table, parent_field, description)
(
"card",
"player_id",
"player",
"player_id",
"Cards referencing non-existent players",
),
("card", "team_id", "team", "id", "Cards referencing non-existent teams"),
(
"stratplay",
"game_id",
"stratgame",
"id",
"Plays referencing non-existent games",
),
(
"stratplay",
"batter_id",
"player",
"player_id",
"Plays referencing non-existent batters",
),
(
"stratplay",
"pitcher_id",
"player",
"player_id",
"Plays referencing non-existent pitchers",
),
(
"decision",
"game_id",
"stratgame",
"id",
"Decisions referencing non-existent games",
),
(
"decision",
"pitcher_id",
"player",
"player_id",
"Decisions referencing non-existent pitchers",
),
(
"battingstat",
"card_id",
"card",
"id",
"Batting stats referencing non-existent cards",
),
(
"pitchingstat",
"card_id",
"card",
"id",
"Pitching stats referencing non-existent cards",
),
(
"battingcard",
"player_id",
"player",
"player_id",
"Batting cards referencing non-existent players",
),
(
"pitchingcard",
"player_id",
"player",
"player_id",
"Pitching cards referencing non-existent players",
),
(
"cardposition",
"player_id",
"player",
"player_id",
"Card positions referencing non-existent players",
),
(
"paperdex",
"player_id",
"player",
"player_id",
"Paperdex entries referencing non-existent players",
),
(
"paperdex",
"team_id",
"team",
"id",
"Paperdex entries referencing non-existent teams",
),
(
"gauntletrun",
"team_id",
"team",
"id",
"Gauntlet runs referencing non-existent teams",
),
]
for child_table, child_field, parent_table, parent_field, description in fk_checks:
try:
# Use explicit column names to avoid ambiguity
query = f"""
SELECT COUNT(*)
FROM {child_table} c
LEFT JOIN {parent_table} p ON c.{child_field} = p.{parent_field}
WHERE c.{child_field} IS NOT NULL AND p.{parent_field} IS NULL
"""
cursor = conn.execute(query)
orphan_count = cursor.fetchone()[0]
if orphan_count > 0:
# Get sample orphaned IDs
sample_query = f"""
SELECT c.{child_field}
FROM {child_table} c
LEFT JOIN {parent_table} p ON c.{child_field} = p.{parent_field}
WHERE c.{child_field} IS NOT NULL AND p.{parent_field} IS NULL
LIMIT 5
"""
sample_cursor = conn.execute(sample_query)
sample_ids = [row[0] for row in sample_cursor.fetchall()]
issues.append(
{
"type": "ORPHANED_FK",
"severity": "HIGH",
"child_table": child_table,
"child_field": child_field,
"parent_table": parent_table,
"parent_field": parent_field,
"description": description,
"orphan_count": orphan_count,
"sample_orphan_ids": sample_ids,
}
)
except sqlite3.OperationalError as e:
# Table or column doesn't exist
print(
f"Warning: Could not check {child_table}.{child_field} -> {parent_table}.{parent_field}: {e}"
)
return issues
def check_varchar_lengths(conn: sqlite3.Connection) -> list:
"""
Check max lengths of string fields.
PostgreSQL VARCHAR fields have stricter length limits than SQLite.
"""
issues = []
# Fields to check with expected max lengths
varchar_checks = [
# (table, field, expected_max_length, description)
("player", "p_name", 255, "Player name"),
("player", "image", 1000, "Player image URL"),
("player", "image2", 1000, "Player image2 URL"),
("player", "headshot", 500, "Player headshot URL"),
("player", "vanity_card", 500, "Player vanity card"),
("player", "strat_code", 100, "Strat code"),
("player", "bbref_id", 50, "Baseball Reference ID"),
("player", "description", 1000, "Player description"),
("team", "abbrev", 10, "Team abbreviation"),
("team", "sname", 100, "Team short name"),
("team", "lname", 255, "Team long name"),
("notification", "title", 255, "Notification title"),
("notification", "message", 2000, "Notification message"),
]
for table, field, expected_max, description in varchar_checks:
try:
cursor = conn.execute(f"SELECT MAX(LENGTH({field})) FROM {table}")
max_length = cursor.fetchone()[0]
if max_length and max_length > expected_max:
# Get sample of long values
sample_cursor = conn.execute(
f"SELECT {field} FROM {table} WHERE LENGTH({field}) > {expected_max} LIMIT 3"
)
samples = [
row[0][:100] + "..." if row[0] else None
for row in sample_cursor.fetchall()
]
issues.append(
{
"type": "VARCHAR_TOO_LONG",
"severity": "HIGH",
"table": table,
"field": field,
"description": description,
"max_found": max_length,
"expected_max": expected_max,
"sample_values": samples,
}
)
elif max_length:
# Info: report actual max for reference
pass
except sqlite3.OperationalError:
pass
return issues
def check_duplicate_primary_keys(conn: sqlite3.Connection) -> list:
"""
Check for duplicate primary keys (shouldn't happen but good to verify).
"""
issues = []
pk_checks = [
("player", "player_id"),
("team", "id"),
("card", "id"),
("stratgame", "id"),
("stratplay", "id"),
]
for table, pk_field in pk_checks:
try:
cursor = conn.execute(f"""
SELECT {pk_field}, COUNT(*) as cnt
FROM {table}
GROUP BY {pk_field}
HAVING COUNT(*) > 1
""")
duplicates = cursor.fetchall()
if duplicates:
issues.append(
{
"type": "DUPLICATE_PK",
"severity": "CRITICAL",
"table": table,
"pk_field": pk_field,
"duplicate_ids": [row[0] for row in duplicates[:10]],
"duplicate_count": len(duplicates),
}
)
except sqlite3.OperationalError:
pass
return issues
def check_unique_constraints(conn: sqlite3.Connection) -> list:
"""
Check that composite unique constraints would be satisfied.
These are the indexes that on_conflict_replace() depends on.
"""
issues = []
unique_checks = [
# (table, fields, description)
("battingcard", ["player_id", "variant"], "Batting card unique constraint"),
("pitchingcard", ["player_id", "variant"], "Pitching card unique constraint"),
(
"cardposition",
["player_id", "variant", "position"],
"Card position unique constraint",
),
(
"battingcardratings",
["battingcard_id", "vs_hand"],
"Batting card ratings unique constraint",
),
(
"pitchingcardratings",
["pitchingcard_id", "vs_hand"],
"Pitching card ratings unique constraint",
),
]
for table, fields, description in unique_checks:
try:
fields_str = ", ".join(fields)
cursor = conn.execute(f"""
SELECT {fields_str}, COUNT(*) as cnt
FROM {table}
GROUP BY {fields_str}
HAVING COUNT(*) > 1
""")
duplicates = cursor.fetchall()
if duplicates:
issues.append(
{
"type": "DUPLICATE_UNIQUE",
"severity": "HIGH",
"table": table,
"fields": fields,
"description": description,
"duplicate_count": len(duplicates),
"sample_duplicates": [
dict(zip(fields + ["count"], row)) for row in duplicates[:5]
],
}
)
except sqlite3.OperationalError as e:
print(f"Warning: Could not check unique constraint on {table}: {e}")
return issues
def generate_report(counts: dict, issues: list, output_path: str = None) -> str:
"""Generate audit report."""
report = {
"generated_at": datetime.now().isoformat(),
"summary": {
"total_tables": len(counts),
"total_records": sum(counts.values()),
"total_issues": len(issues),
"critical_issues": len(
[i for i in issues if i.get("severity") == "CRITICAL"]
),
"high_issues": len([i for i in issues if i.get("severity") == "HIGH"]),
"medium_issues": len([i for i in issues if i.get("severity") == "MEDIUM"]),
},
"table_counts": counts,
"issues": issues,
}
if output_path:
with open(output_path, "w") as f:
json.dump(report, f, indent=2)
return json.dumps(report, indent=2)
def main():
parser = argparse.ArgumentParser(
description="Audit SQLite database before PostgreSQL migration"
)
parser.add_argument(
"--db-path",
type=str,
default="storage/pd_master.db",
help="Path to SQLite database",
)
parser.add_argument("--output", type=str, help="Output JSON file for report")
parser.add_argument(
"--fix", action="store_true", help="Apply safe fixes (not implemented)"
)
args = parser.parse_args()
print("=" * 60)
print("Paper Dynasty SQLite Data Integrity Audit")
print("=" * 60)
if not Path(args.db_path).exists():
print(f"ERROR: Database not found: {args.db_path}")
sys.exit(1)
conn = connect_db(args.db_path)
# Run checks
print("\n1. Getting table record counts...")
counts = get_table_record_counts(conn)
print(f" Found {len(counts)} tables with {sum(counts.values()):,} total records")
print("\n2. Checking for NULL values...")
null_issues = check_null_values(conn)
print(f" Found {len(null_issues)} NULL value issues")
print("\n3. Checking for orphaned foreign keys...")
fk_issues = check_orphaned_foreign_keys(conn)
print(f" Found {len(fk_issues)} orphaned FK issues")
print("\n4. Checking VARCHAR lengths...")
varchar_issues = check_varchar_lengths(conn)
print(f" Found {len(varchar_issues)} VARCHAR length issues")
print("\n5. Checking for duplicate primary keys...")
pk_issues = check_duplicate_primary_keys(conn)
print(f" Found {len(pk_issues)} duplicate PK issues")
print("\n6. Checking unique constraints...")
unique_issues = check_unique_constraints(conn)
print(f" Found {len(unique_issues)} unique constraint issues")
# Combine all issues
all_issues = null_issues + fk_issues + varchar_issues + pk_issues + unique_issues
# Generate report
print("\n" + "=" * 60)
print("AUDIT RESULTS")
print("=" * 60)
if args.output:
report = generate_report(counts, all_issues, args.output)
print(f"Full report saved to: {args.output}")
else:
report = generate_report(counts, all_issues)
# Print summary
print(f"\nTotal Issues: {len(all_issues)}")
critical = [i for i in all_issues if i.get("severity") == "CRITICAL"]
high = [i for i in all_issues if i.get("severity") == "HIGH"]
medium = [i for i in all_issues if i.get("severity") == "MEDIUM"]
if critical:
print(f"\n CRITICAL ({len(critical)}):")
for issue in critical:
print(
f" - {issue['type']}: {issue.get('description', issue.get('table', 'Unknown'))}"
)
if high:
print(f"\n HIGH ({len(high)}):")
for issue in high:
desc = issue.get(
"description",
f"{issue.get('table', 'Unknown')}.{issue.get('field', 'Unknown')}",
)
print(f" - {issue['type']}: {desc}")
if medium:
print(f"\n MEDIUM ({len(medium)}):")
for issue in medium:
desc = issue.get(
"description",
f"{issue.get('table', 'Unknown')}.{issue.get('field', 'Unknown')}",
)
print(f" - {issue['type']}: {desc}")
# Table counts
print("\n" + "-" * 60)
print("TABLE RECORD COUNTS (for baseline comparison)")
print("-" * 60)
for table, count in sorted(counts.items()):
print(f" {table:30} {count:>10,}")
conn.close()
# Exit code based on issues
if critical:
print("\n CRITICAL ISSUES FOUND - Migration may fail!")
sys.exit(2)
elif high:
print("\n HIGH PRIORITY ISSUES FOUND - Review before migration")
sys.exit(1)
else:
print("\n No critical issues found - Ready for migration")
sys.exit(0)
if __name__ == "__main__":
main()

510
scripts/migrate_to_postgres.py Executable file
View File

@ -0,0 +1,510 @@
#!/usr/bin/env python3
"""
Paper Dynasty SQLite to PostgreSQL Migration Script
CRITICAL: This script preserves primary key IDs exactly as they exist in SQLite.
Failing to preserve IDs will cause all foreign key references to break.
Usage:
# Dry run (validate only, no changes)
python scripts/migrate_to_postgres.py --dry-run
# Full migration
python scripts/migrate_to_postgres.py
# Migrate specific table only
python scripts/migrate_to_postgres.py --table player
Environment Variables Required:
POSTGRES_HOST, POSTGRES_DB, POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_PORT
Based on lessons learned from Major Domo PostgreSQL migration (August 2025).
"""
import argparse
import logging
import os
import sqlite3
import sys
from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple
import psycopg2
from psycopg2.extras import execute_values
# Configure logging
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(levelname)s - %(message)s",
handlers=[
logging.StreamHandler(sys.stdout),
logging.FileHandler(
f"logs/migration_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log"
),
],
)
logger = logging.getLogger(__name__)
# Migration order - tables with no FK dependencies first, then dependent tables
# This ensures parent records exist before children are inserted
MIGRATION_ORDER = [
# Tier 1: No foreign key dependencies
"current",
"rarity",
"event",
"packtype",
"notification",
# Tier 2: Simple FK dependencies (single level)
"cardset", # -> event
"mlbplayer", # no FKs
"gamerewards", # -> packtype, player (but player not created yet, so nullable)
# Tier 3: Core entity tables
"team", # -> event
"player", # -> cardset, rarity, mlbplayer
# Tier 4: Dependent on core entities
"pack", # -> team, packtype, cardset
"card", # -> player, team, pack
"roster", # -> team, card (x26)
"result", # -> team (x2)
"stratgame", # -> team (x2)
# Tier 5: Statistics and game data
"battingstat", # -> card, team, result
"pitchingstat", # -> card, team, result
"stratplay", # -> stratgame, player (many), team (many)
"decision", # -> stratgame, player, team
# Tier 6: Card detail tables
"battingcard", # -> player
"battingcardratings", # -> battingcard
"pitchingcard", # -> player
"pitchingcardratings", # -> pitchingcard
"cardposition", # -> player
# Tier 7: Other dependent tables
"award", # -> card, team
"paperdex", # -> team, player
"reward", # -> team
"gauntletreward", # -> event, gamerewards
"gauntletrun", # -> team, event
]
# Tables with explicit primary keys (not auto-increment)
EXPLICIT_PK_TABLES = {
"player": "player_id", # Uses player_id as explicit PK
}
# All other tables use 'id' as auto-increment PK
def get_sqlite_connection(db_path: str) -> sqlite3.Connection:
"""Connect to SQLite database."""
if not os.path.exists(db_path):
raise FileNotFoundError(f"SQLite database not found: {db_path}")
conn = sqlite3.connect(db_path)
conn.row_factory = sqlite3.Row
return conn
def get_postgres_connection() -> psycopg2.extensions.connection:
"""Connect to PostgreSQL database using environment variables."""
required_vars = [
"POSTGRES_HOST",
"POSTGRES_DB",
"POSTGRES_USER",
"POSTGRES_PASSWORD",
]
missing = [v for v in required_vars if not os.environ.get(v)]
if missing:
raise EnvironmentError(f"Missing required environment variables: {missing}")
return psycopg2.connect(
host=os.environ["POSTGRES_HOST"],
database=os.environ["POSTGRES_DB"],
user=os.environ["POSTGRES_USER"],
password=os.environ["POSTGRES_PASSWORD"],
port=int(os.environ.get("POSTGRES_PORT", "5432")),
)
def get_table_columns(sqlite_conn: sqlite3.Connection, table_name: str) -> List[str]:
"""Get column names for a table from SQLite."""
cursor = sqlite_conn.execute(f"PRAGMA table_info({table_name})")
return [row["name"] for row in cursor.fetchall()]
def get_primary_key_column(table_name: str) -> str:
"""Get the primary key column name for a table."""
return EXPLICIT_PK_TABLES.get(table_name, "id")
def get_sequence_name(table_name: str, pk_column: str) -> str:
"""Get the PostgreSQL sequence name for a table's primary key."""
return f"{table_name}_{pk_column}_seq"
def get_record_count(conn, table_name: str, is_sqlite: bool = True) -> int:
"""Get record count for a table."""
if is_sqlite:
cursor = conn.execute(f"SELECT COUNT(*) FROM {table_name}")
return cursor.fetchone()[0]
else:
cursor = conn.cursor()
cursor.execute(f"SELECT COUNT(*) FROM {table_name}")
return cursor.fetchone()[0]
def migrate_table(
sqlite_conn: sqlite3.Connection,
pg_conn: psycopg2.extensions.connection,
table_name: str,
batch_size: int = 500,
dry_run: bool = False,
) -> Dict[str, Any]:
"""
Migrate a single table from SQLite to PostgreSQL.
CRITICAL: Preserves primary key IDs exactly.
Returns:
Dict with migration statistics
"""
stats = {
"table": table_name,
"sqlite_count": 0,
"postgres_count": 0,
"inserted": 0,
"skipped": 0,
"errors": [],
"success": False,
}
try:
# Get column info
columns = get_table_columns(sqlite_conn, table_name)
pk_column = get_primary_key_column(table_name)
# Count source records
stats["sqlite_count"] = get_record_count(
sqlite_conn, table_name, is_sqlite=True
)
logger.info(f"Table {table_name}: {stats['sqlite_count']} records to migrate")
if stats["sqlite_count"] == 0:
logger.info(f"Table {table_name}: No records to migrate")
stats["success"] = True
return stats
if dry_run:
logger.info(
f"[DRY RUN] Would migrate {stats['sqlite_count']} records from {table_name}"
)
stats["success"] = True
return stats
# Read all records from SQLite
cursor = sqlite_conn.execute(f"SELECT * FROM {table_name}")
rows = cursor.fetchall()
# Prepare PostgreSQL insert
pg_cursor = pg_conn.cursor()
# Build column list string
columns_str = ", ".join(columns)
placeholders = ", ".join(["%s"] * len(columns))
# Process in batches
for i in range(0, len(rows), batch_size):
batch = rows[i : i + batch_size]
batch_values = []
for row in batch:
# Convert sqlite3.Row to tuple, preserving all values including ID
values = tuple(row[col] for col in columns)
batch_values.append(values)
try:
# Use execute_values for efficient batch insert
insert_sql = f"INSERT INTO {table_name} ({columns_str}) VALUES %s"
execute_values(pg_cursor, insert_sql, batch_values)
stats["inserted"] += len(batch)
except psycopg2.errors.ForeignKeyViolation as e:
# Foreign key error - fall back to individual inserts
logger.warning(
f"FK violation in batch, falling back to individual inserts: {e}"
)
pg_conn.rollback()
for values in batch_values:
try:
pg_cursor.execute(
f"INSERT INTO {table_name} ({columns_str}) VALUES ({placeholders})",
values,
)
stats["inserted"] += 1
except psycopg2.errors.ForeignKeyViolation as e:
stats["skipped"] += 1
# Extract ID for logging
pk_idx = columns.index(pk_column) if pk_column in columns else 0
record_id = values[pk_idx]
stats["errors"].append(
{
"id": record_id,
"error": "ForeignKeyViolation",
"message": str(e),
}
)
logger.warning(
f"Skipped orphaned record {table_name}.{pk_column}={record_id}"
)
pg_conn.rollback()
except Exception as e:
stats["skipped"] += 1
pk_idx = columns.index(pk_column) if pk_column in columns else 0
record_id = values[pk_idx]
stats["errors"].append(
{
"id": record_id,
"error": type(e).__name__,
"message": str(e),
}
)
logger.error(
f"Error inserting {table_name}.{pk_column}={record_id}: {e}"
)
pg_conn.rollback()
logger.info(
f"Table {table_name}: Processed {min(i + batch_size, len(rows))}/{len(rows)} records"
)
# Commit the transaction
pg_conn.commit()
# CRITICAL: Reset the PostgreSQL sequence to MAX(id) + 1
# Without this, new inserts will fail with duplicate key errors
sequence_name = get_sequence_name(table_name, pk_column)
try:
pg_cursor.execute(f"""
SELECT setval('{sequence_name}', COALESCE((SELECT MAX({pk_column}) FROM {table_name}), 1), true)
""")
pg_conn.commit()
logger.info(f"Table {table_name}: Reset sequence {sequence_name}")
except psycopg2.errors.UndefinedTable as e:
# Sequence might not exist for explicit PK tables
logger.warning(f"Could not reset sequence {sequence_name}: {e}")
pg_conn.rollback()
# Verify counts
stats["postgres_count"] = get_record_count(pg_conn, table_name, is_sqlite=False)
if stats["postgres_count"] == stats["sqlite_count"]:
logger.info(
f"Table {table_name}: SUCCESS - {stats['postgres_count']} records migrated"
)
stats["success"] = True
elif stats["postgres_count"] == stats["inserted"]:
logger.warning(
f"Table {table_name}: PARTIAL - {stats['inserted']} inserted, "
f"{stats['skipped']} skipped (orphaned FK records)"
)
stats["success"] = (
True # Partial success is acceptable for orphaned records
)
else:
logger.error(
f"Table {table_name}: MISMATCH - SQLite: {stats['sqlite_count']}, "
f"PostgreSQL: {stats['postgres_count']}"
)
stats["success"] = False
except Exception as e:
logger.error(f"Table {table_name}: FAILED - {e}")
stats["errors"].append({"error": type(e).__name__, "message": str(e)})
stats["success"] = False
pg_conn.rollback()
return stats
def verify_id_preservation(
sqlite_conn: sqlite3.Connection,
pg_conn: psycopg2.extensions.connection,
sample_tables: List[str] = None,
) -> bool:
"""
Verify that primary key IDs were preserved correctly.
This is a CRITICAL check - if IDs don't match, the migration has failed.
"""
if sample_tables is None:
sample_tables = ["player", "team", "card", "stratgame"]
all_match = True
for table_name in sample_tables:
pk_column = get_primary_key_column(table_name)
# Get first and last 5 IDs from SQLite
sqlite_cursor = sqlite_conn.execute(
f"SELECT {pk_column} FROM {table_name} ORDER BY {pk_column} LIMIT 5"
)
sqlite_first = [row[0] for row in sqlite_cursor.fetchall()]
sqlite_cursor = sqlite_conn.execute(
f"SELECT {pk_column} FROM {table_name} ORDER BY {pk_column} DESC LIMIT 5"
)
sqlite_last = [row[0] for row in sqlite_cursor.fetchall()]
# Get same IDs from PostgreSQL
pg_cursor = pg_conn.cursor()
pg_cursor.execute(
f"SELECT {pk_column} FROM {table_name} ORDER BY {pk_column} LIMIT 5"
)
pg_first = [row[0] for row in pg_cursor.fetchall()]
pg_cursor.execute(
f"SELECT {pk_column} FROM {table_name} ORDER BY {pk_column} DESC LIMIT 5"
)
pg_last = [row[0] for row in pg_cursor.fetchall()]
if sqlite_first == pg_first and sqlite_last == pg_last:
logger.info(f"ID Verification {table_name}: PASS - IDs match")
else:
logger.error(
f"ID Verification {table_name}: FAIL - "
f"SQLite first: {sqlite_first}, PG first: {pg_first}, "
f"SQLite last: {sqlite_last}, PG last: {pg_last}"
)
all_match = False
return all_match
def main():
parser = argparse.ArgumentParser(
description="Migrate Paper Dynasty from SQLite to PostgreSQL"
)
parser.add_argument(
"--dry-run", action="store_true", help="Validate without making changes"
)
parser.add_argument("--table", type=str, help="Migrate only this table")
parser.add_argument(
"--sqlite-path",
type=str,
default="storage/pd_master.db",
help="Path to SQLite database",
)
parser.add_argument(
"--batch-size", type=int, default=500, help="Batch size for inserts"
)
parser.add_argument(
"--skip-verification", action="store_true", help="Skip ID verification"
)
args = parser.parse_args()
logger.info("=" * 60)
logger.info("Paper Dynasty SQLite to PostgreSQL Migration")
logger.info("=" * 60)
if args.dry_run:
logger.info("DRY RUN MODE - No changes will be made")
# Connect to databases
try:
sqlite_conn = get_sqlite_connection(args.sqlite_path)
logger.info(f"Connected to SQLite: {args.sqlite_path}")
except FileNotFoundError as e:
logger.error(str(e))
sys.exit(1)
try:
pg_conn = get_postgres_connection()
logger.info(
f"Connected to PostgreSQL: {os.environ['POSTGRES_HOST']}/{os.environ['POSTGRES_DB']}"
)
except EnvironmentError as e:
logger.error(str(e))
sys.exit(1)
except psycopg2.Error as e:
logger.error(f"PostgreSQL connection failed: {e}")
sys.exit(1)
# Determine tables to migrate
tables_to_migrate = [args.table] if args.table else MIGRATION_ORDER
# Validate tables exist
available_tables = set()
cursor = sqlite_conn.execute("SELECT name FROM sqlite_master WHERE type='table'")
for row in cursor:
available_tables.add(row[0])
for table in tables_to_migrate:
if table not in available_tables:
logger.warning(f"Table {table} not found in SQLite, skipping")
tables_to_migrate.remove(table)
# Migration summary
results = []
start_time = datetime.now()
logger.info(f"Migrating {len(tables_to_migrate)} tables...")
logger.info("-" * 60)
for table_name in tables_to_migrate:
stats = migrate_table(
sqlite_conn,
pg_conn,
table_name,
batch_size=args.batch_size,
dry_run=args.dry_run,
)
results.append(stats)
logger.info("-" * 60)
# Summary
elapsed = datetime.now() - start_time
successful = sum(1 for r in results if r["success"])
total_records = sum(r["inserted"] for r in results)
total_skipped = sum(r["skipped"] for r in results)
logger.info("=" * 60)
logger.info("MIGRATION SUMMARY")
logger.info("=" * 60)
logger.info(f"Tables: {successful}/{len(results)} successful")
logger.info(f"Records: {total_records} inserted, {total_skipped} skipped")
logger.info(f"Duration: {elapsed}")
# Failed tables
failed = [r for r in results if not r["success"]]
if failed:
logger.error("FAILED TABLES:")
for r in failed:
logger.error(f" - {r['table']}: {r['errors']}")
# ID Verification (CRITICAL)
if not args.dry_run and not args.skip_verification:
logger.info("-" * 60)
logger.info("VERIFYING ID PRESERVATION...")
if verify_id_preservation(sqlite_conn, pg_conn):
logger.info("ID VERIFICATION: PASS - All IDs preserved correctly")
else:
logger.error("ID VERIFICATION: FAIL - IDs do not match!")
logger.error(
"THIS IS A CRITICAL FAILURE - Foreign key references may be broken"
)
sys.exit(1)
# Close connections
sqlite_conn.close()
pg_conn.close()
if all(r["success"] for r in results):
logger.info("MIGRATION COMPLETE - SUCCESS")
sys.exit(0)
else:
logger.warning("MIGRATION COMPLETE - PARTIAL SUCCESS (some tables failed)")
sys.exit(1)
if __name__ == "__main__":
main()