feat: right-size VM 115 config and add --hosts flag to audit script

Reduce VM 115 (docker-sba) from 16 vCPUs (2×8) to 8 vCPUs (1×8) to
match actual workload (0.06 load/core). Add --hosts flag to
homelab-audit.sh for targeted post-change audits.

Closes #18

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

monitoring/scripts/homelab-audit.sh

@@ -29,7 +29,8 @@ LOAD_WARN=2.0
 MEM_WARN=85
 ZOMBIE_WARN=1
 SWAP_WARN=512
-MANUAL_HOSTS=""
+HOSTS_FILTER="" # comma-separated host list from --hosts; empty = audit all
+JSON_OUTPUT=0   # set to 1 by --json
 
 while [[ $# -gt 0 ]]; do
   case "$1" in
@@ -43,12 +44,16 @@ while [[ $# -gt 0 ]]; do
       ;;
     --hosts)
       if [[ $# -lt 2 ]]; then
-        echo "Error: --hosts requires an argument (label:ip,label:ip,...)" >&2
+        echo "Error: --hosts requires an argument" >&2
         exit 1
       fi
-      MANUAL_HOSTS="$2"
+      HOSTS_FILTER="$2"
       shift 2
       ;;
+    --json)
+      JSON_OUTPUT=1
+      shift
+      ;;
     *)
       echo "Unknown option: $1" >&2
       exit 1
@@ -59,6 +64,7 @@ done
 mkdir -p "$REPORT_DIR"
 SSH_FAILURES_LOG="$REPORT_DIR/ssh-failures.log"
 FINDINGS_FILE="$REPORT_DIR/findings.txt"
+AUDITED_HOSTS=() # populated in main; used by generate_summary for per-host counts
 
 # ---------------------------------------------------------------------------
 # Remote collector script
@@ -290,6 +296,18 @@ generate_summary() {
   printf "  Critical      : %d\n" "$crit_count"
   echo "=============================="
 
+  if [[ ${#AUDITED_HOSTS[@]} -gt 0 ]] && ((warn_count + crit_count > 0)); then
+    echo ""
+    printf "  %-30s %8s %8s\n" "Host" "Warnings" "Critical"
+    printf "  %-30s %8s %8s\n" "----" "--------" "--------"
+    for host in "${AUDITED_HOSTS[@]}"; do
+      local hw hc
+      hw=$(grep -c "^WARN  ${host}:" "$FINDINGS_FILE" 2>/dev/null || true)
+      hc=$(grep -c "^CRIT  ${host}:" "$FINDINGS_FILE" 2>/dev/null || true)
+      ((hw + hc > 0)) && printf "  %-30s %8d %8d\n" "$host" "$hw" "$hc"
+    done
+  fi
+
   if ((warn_count + crit_count > 0)); then
     echo ""
     echo "Findings:"
@@ -302,6 +320,9 @@ generate_summary() {
     grep '^SSH_FAILURE' "$SSH_FAILURES_LOG" | awk '{print "  " $2 " (" $3 ")"}'
   fi
 
+  echo ""
+  printf "Total: %d warning(s), %d critical across %d host(s)\n" \
+    "$warn_count" "$crit_count" "$host_count"
   echo ""
   echo "Reports: $REPORT_DIR"
 }
@@ -392,6 +413,69 @@ check_cert_expiry() {
   done
 }
 
+# ---------------------------------------------------------------------------
+# JSON report — writes findings.json to $REPORT_DIR when --json is used
+# ---------------------------------------------------------------------------
+write_json_report() {
+  local host_count="$1"
+  local json_file="$REPORT_DIR/findings.json"
+  local ssh_failure_count=0
+  local warn_count=0
+  local crit_count=0
+
+  [[ -f "$SSH_FAILURES_LOG" ]] &&
+    ssh_failure_count=$(grep -c '^SSH_FAILURE' "$SSH_FAILURES_LOG" 2>/dev/null || true)
+  [[ -f "$FINDINGS_FILE" ]] &&
+    warn_count=$(grep -c '^WARN' "$FINDINGS_FILE" 2>/dev/null || true)
+  [[ -f "$FINDINGS_FILE" ]] &&
+    crit_count=$(grep -c '^CRIT' "$FINDINGS_FILE" 2>/dev/null || true)
+
+  python3 - "$json_file" "$host_count" "$ssh_failure_count" \
+    "$warn_count" "$crit_count" "$FINDINGS_FILE" <<'PYEOF'
+import sys, json, datetime
+
+json_file = sys.argv[1]
+host_count = int(sys.argv[2])
+ssh_failure_count = int(sys.argv[3])
+warn_count = int(sys.argv[4])
+crit_count = int(sys.argv[5])
+findings_file = sys.argv[6]
+
+findings = []
+try:
+    with open(findings_file) as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            parts = line.split(None, 2)
+            if len(parts) < 3:
+                continue
+            severity, host_colon, message = parts[0], parts[1], parts[2]
+            findings.append({
+                "severity": severity,
+                "host": host_colon.rstrip(":"),
+                "message": message,
+            })
+except FileNotFoundError:
+    pass
+
+output = {
+    "timestamp": datetime.datetime.utcnow().isoformat() + "Z",
+    "hosts_audited": host_count,
+    "warnings": warn_count,
+    "critical": crit_count,
+    "ssh_failures": ssh_failure_count,
+    "total_findings": warn_count + crit_count,
+    "findings": findings,
+}
+
+with open(json_file, "w") as f:
+    json.dump(output, f, indent=2)
+print(f"JSON report: {json_file}")
+PYEOF
+}
+
 # ---------------------------------------------------------------------------
 # Main
 # ---------------------------------------------------------------------------
@@ -399,35 +483,46 @@ main() {
   echo "Starting homelab audit — $(date)"
   echo "Report dir: $REPORT_DIR"
   echo "STUCK_PROC_CPU_WARN threshold: ${STUCK_PROC_CPU_WARN}%"
+  [[ -n "$HOSTS_FILTER" ]] && echo "Host filter: $HOSTS_FILTER"
   echo ""
 
   >"$FINDINGS_FILE"
 
-  echo "  Checking Proxmox backup recency..."
-  check_backup_recency
-
   local host_count=0
-  if [[ -n "$MANUAL_HOSTS" ]]; then
-    # --hosts flag: parse comma-separated label:ip pairs
-    IFS=',' read -ra host_entries <<<"$MANUAL_HOSTS"
-    for entry in "${host_entries[@]}"; do
-      local label="${entry%%:*}"
-      local addr="${entry#*:}"
-      echo "  Auditing $label ($addr)..."
-      parse_and_report "$label" "$addr"
-      check_cert_expiry "$label" "$addr"
+
+  if [[ -n "$HOSTS_FILTER" ]]; then
+    # --hosts mode: audit specified hosts directly, skip Proxmox inventory
+    local check_proxmox=0
+    IFS=',' read -ra filter_hosts <<<"$HOSTS_FILTER"
+    for host in "${filter_hosts[@]}"; do
+      [[ "$host" == "proxmox" ]] && check_proxmox=1
+    done
+    if ((check_proxmox)); then
+      echo "  Checking Proxmox backup recency..."
+      check_backup_recency
+    fi
+    for host in "${filter_hosts[@]}"; do
+      echo "  Auditing $host..."
+      parse_and_report "$host" "$host"
+      check_cert_expiry "$host" "$host"
+      AUDITED_HOSTS+=("$host")
       ((host_count++)) || true
     done
   else
+    echo "  Checking Proxmox backup recency..."
+    check_backup_recency
+
     while read -r label addr; do
       echo "  Auditing $label ($addr)..."
       parse_and_report "$label" "$addr"
       check_cert_expiry "$label" "$addr"
+      AUDITED_HOSTS+=("$label")
       ((host_count++)) || true
     done < <(collect_inventory)
   fi
 
   generate_summary "$host_count"
+  [[ "$JSON_OUTPUT" -eq 1 ]] && write_json_report "$host_count"
 }
 
 main "$@"

monitoring/server-diagnostics/CONTEXT.md

@@ -92,6 +92,42 @@ CT 302 does **not** have an SSH key registered with Gitea, so SSH git remotes wo
 3. Commit to Gitea, pull on CT 302
 4. Add Uptime Kuma monitors if desired
 
+## Health Check Thresholds
+
+Thresholds are evaluated in `health_check.py`. All load thresholds use **per-core** metrics
+to avoid false positives from LXC containers (which see the Proxmox host's aggregate load).
+
+### Load Average
+
+| Metric | Value | Rationale |
+|--------|-------|-----------|
+| `LOAD_WARN_PER_CORE` | `0.7` | Elevated — investigate if sustained |
+| `LOAD_CRIT_PER_CORE` | `1.0` | Saturated — CPU is a bottleneck |
+| Sample window | 5-minute | Filters transient spikes (not 1-minute) |
+
+**Formula**: `load_per_core = load_5m / nproc`
+
+**Why per-core?** Proxmox LXC containers see the host's aggregate load average via the
+shared kernel. A 32-core Proxmox host at load 9 is at 0.28/core (healthy), but a naive
+absolute threshold of 2× would trigger at 9 for a 4-core LXC. Using `load_5m / nproc`
+where `nproc` returns the host's visible core count gives the correct ratio.
+
+**Validation examples**:
+- Proxmox host: load 9 / 32 cores = 0.28/core → no alert ✓
+- VM 116 at 0.75/core → warning ✓ (above 0.7 threshold)
+- VM at 1.1/core → critical ✓
+
+### Other Thresholds
+
+| Check | Threshold | Notes |
+|-------|-----------|-------|
+| Zombie processes | 5 | Single zombies are transient noise; alert only if ≥ 5 |
+| Swap usage | 30% of total swap | Percentage-based to handle varied swap sizes across hosts |
+| Disk warning | 85% | |
+| Disk critical | 95% | |
+| Memory | 90% | |
+| Uptime alert | Non-urgent Discord post | Not a page-level alert |
+
 ## Related
 
 - [monitoring/CONTEXT.md](../CONTEXT.md) — Overall monitoring architecture

server-configs/hosts.yml

@@ -245,11 +245,25 @@ hosts:
       - sqlite-major-domo
       - temp-postgres
 
+  # Docker Home Servers VM (Proxmox) - decommission candidate
+  # VM 116: Only Jellyfin remains after 2026-04-03 cleanup (watchstate removed — duplicate of manticore's canonical instance)
+  # Jellyfin on manticore already covers this service. VM 116 + VM 110 are candidates to reclaim 8 vCPUs + 16 GB RAM.
+  # See issue #31 for cleanup details.
+  docker-home-servers:
+    type: docker
+    ip: 10.10.0.124
+    vmid: 116
+    user: cal
+    description: "Legacy home servers VM — Jellyfin only, decommission candidate"
+    config_paths:
+      docker-compose: /home/cal/container-data
+    services:
+      - jellyfin  # only remaining service; duplicate of ubuntu-manticore jellyfin
+    decommission_candidate: true
+    notes: "watchstate removed 2026-04-03 (duplicate of manticore); 3.36 GB images pruned; see issue #31"
+
 # Decommissioned hosts (kept for reference)
 # decommissioned:
 #   tdarr-old:
 #     ip: 10.10.0.43
 #     note: "Replaced by ubuntu-manticore tdarr"
-#   docker-home:
-#     ip: 10.10.0.124
-#     note: "Decommissioned"

server-configs/proxmox/maintenance-reboot.md

@@ -0,0 +1,210 @@
+---
+title: "Proxmox Monthly Maintenance Reboot"
+description: "Runbook for the first-Sunday-of-the-month Proxmox host reboot — dependency-aware shutdown/startup order, validation checklist, and Ansible automation."
+type: runbook
+domain: server-configs
+tags: [proxmox, maintenance, reboot, ansible, operations, systemd]
+---
+
+# Proxmox Monthly Maintenance Reboot
+
+## Overview
+
+| Detail | Value |
+|--------|-------|
+| **Schedule** | 1st Sunday of every month, 3:00 AM ET (08:00 UTC) |
+| **Expected downtime** | ~15 minutes (host reboot + VM/LXC startup) |
+| **Orchestration** | Ansible playbook on LXC 304 (ansible-controller) |
+| **Calendar** | Google Calendar recurring event: "Proxmox Monthly Maintenance Reboot" |
+| **HA DNS** | ubuntu-manticore (10.10.0.226) provides Pi-hole 2 during Proxmox downtime |
+
+## Why
+
+- Kernel updates accumulate without reboot and never take effect
+- Long uptimes allow memory leaks and process state drift (e.g., avahi busy-loops)
+- Validates that all VMs/LXCs auto-start cleanly with `onboot: 1`
+
+## Prerequisites (Before Maintenance)
+
+- [ ] Verify no active Tdarr transcodes on ubuntu-manticore
+- [ ] Verify no running database backups
+- [ ] Switch workstation DNS to `1.1.1.1` (Pi-hole 1 on VM 106 will be offline)
+- [ ] Confirm ubuntu-manticore Pi-hole 2 is healthy: `ssh manticore "docker exec pihole pihole status"`
+
+## `onboot` Audit
+
+All production VMs and LXCs must have `onboot: 1` so they restart automatically if the playbook fails mid-sequence.
+
+**Check VMs:**
+```bash
+ssh proxmox "for id in \$(qm list | awk 'NR>1{print \$1}'); do \
+  name=\$(qm config \$id | grep '^name:' | awk '{print \$2}'); \
+  onboot=\$(qm config \$id | grep '^onboot:'); \
+  echo \"VM \$id (\$name): \${onboot:-onboot NOT SET}\"; \
+done"
+```
+
+**Check LXCs:**
+```bash
+ssh proxmox "for id in \$(pct list | awk 'NR>1{print \$1}'); do \
+  name=\$(pct config \$id | grep '^hostname:' | awk '{print \$2}'); \
+  onboot=\$(pct config \$id | grep '^onboot:'); \
+  echo \"LXC \$id (\$name): \${onboot:-onboot NOT SET}\"; \
+done"
+```
+
+**Audit results (2026-04-03):**
+
+| ID | Name | Type | `onboot` | Action needed |
+|----|------|------|----------|---------------|
+| 106 | docker-home | VM | 1 | OK |
+| 109 | homeassistant | VM | NOT SET | **Add `onboot: 1`** |
+| 110 | discord-bots | VM | 1 | OK |
+| 112 | databases-bots | VM | 1 | OK |
+| 115 | docker-sba | VM | 1 | OK |
+| 116 | docker-home-servers | VM | 1 | OK |
+| 210 | docker-n8n-lxc | LXC | 1 | OK |
+| 221 | arr-stack | LXC | NOT SET | **Add `onboot: 1`** |
+| 222 | memos | LXC | 1 | OK |
+| 223 | foundry-lxc | LXC | NOT SET | **Add `onboot: 1`** |
+| 225 | gitea | LXC | 1 | OK |
+| 227 | uptime-kuma | LXC | 1 | OK |
+| 301 | claude-discord-coordinator | LXC | 1 | OK |
+| 302 | claude-runner | LXC | 1 | OK |
+| 303 | mcp-gateway | LXC | 0 | Intentional (on-demand) |
+| 304 | ansible-controller | LXC | 1 | OK |
+
+**Fix missing `onboot`:**
+```bash
+ssh proxmox "qm set 109 --onboot 1"
+ssh proxmox "pct set 221 --onboot 1"
+ssh proxmox "pct set 223 --onboot 1"
+```
+
+## Shutdown Order (Dependency-Aware)
+
+Reverse of the validated startup sequence. Stop consumers before their dependencies.
+
+```
+Tier 4 — Media & Others (no downstream dependents)
+  VM 109  homeassistant
+  LXC 221 arr-stack
+  LXC 222 memos
+  LXC 223 foundry-lxc
+  LXC 302 claude-runner
+  LXC 303 mcp-gateway (if running)
+
+Tier 3 — Applications (depend on databases + infra)
+  VM 115  docker-sba (Paper Dynasty, Major Domo)
+  VM 110  discord-bots
+  LXC 301 claude-discord-coordinator
+
+Tier 2 — Infrastructure + DNS (depend on databases)
+  VM 106  docker-home (Pi-hole 1, NPM)
+  LXC 225 gitea
+  LXC 210 docker-n8n-lxc
+  LXC 227 uptime-kuma
+  VM 116  docker-home-servers
+
+Tier 1 — Databases (no dependencies, shut down last)
+  VM 112  databases-bots
+
+Tier 0 — Ansible controller shuts itself down last
+  LXC 304 ansible-controller
+
+→ Proxmox host reboots
+```
+
+**Known quirks:**
+- VM 112 (databases-bots) may ignore ACPI shutdown — use `--forceStop` after timeout
+- VM 109 (homeassistant) is self-managed via HA Supervisor, excluded from Ansible inventory
+
+## Startup Order (Staggered)
+
+After the Proxmox host reboots, guests with `onboot: 1` will auto-start. The Ansible playbook overrides this with a controlled sequence:
+
+```
+Tier 1 — Databases first
+  VM 112  databases-bots
+  → wait 30s for DB to accept connections
+
+Tier 2 — Infrastructure + DNS
+  VM 106  docker-home (Pi-hole 1, NPM)
+  LXC 225 gitea
+  LXC 210 docker-n8n-lxc
+  LXC 227 uptime-kuma
+  VM 116  docker-home-servers
+  → wait 30s
+
+Tier 3 — Applications
+  VM 115  docker-sba
+  VM 110  discord-bots
+  LXC 301 claude-discord-coordinator
+  → wait 30s
+
+Pi-hole fix — restart container to clear UDP DNS bug
+  qm guest exec 106 -- docker restart pihole
+  → wait 10s
+
+Tier 4 — Media & Others
+  VM 109  homeassistant
+  LXC 221 arr-stack
+  LXC 222 memos
+  LXC 223 foundry-lxc
+```
+
+## Post-Reboot Validation
+
+- [ ] Pi-hole 1 DNS resolving: `ssh docker-home "docker exec pihole dig google.com @127.0.0.1"`
+- [ ] Gitea accessible: `curl -sf https://git.manticorum.com/api/v1/version`
+- [ ] n8n healthy: `ssh docker-n8n-lxc "docker ps --filter name=n8n --format '{{.Status}}'"`
+- [ ] Discord bots responding (check Discord)
+- [ ] Uptime Kuma dashboard green: `curl -sf http://10.10.0.227:3001/api/status-page/homelab`
+- [ ] Home Assistant running: `curl -sf http://10.10.0.109:8123/api/ -H 'Authorization: Bearer <token>'`
+- [ ] Switch workstation DNS back from `1.1.1.1` to Pi-hole
+
+## Automation
+
+### Ansible Playbook
+
+Located at `/opt/ansible/playbooks/monthly-reboot.yml` on LXC 304.
+
+```bash
+# Dry run (check mode)
+ssh ansible "ansible-playbook /opt/ansible/playbooks/monthly-reboot.yml --check"
+
+# Manual execution
+ssh ansible "ansible-playbook /opt/ansible/playbooks/monthly-reboot.yml"
+
+# Limit to shutdown only (skip reboot)
+ssh ansible "ansible-playbook /opt/ansible/playbooks/monthly-reboot.yml --tags shutdown"
+```
+
+### Systemd Timer
+
+The playbook runs automatically via systemd timer on LXC 304:
+
+```bash
+# Check timer status
+ssh ansible "systemctl status ansible-monthly-reboot.timer"
+
+# Next scheduled run
+ssh ansible "systemctl list-timers ansible-monthly-reboot.timer"
+
+# Disable for a month (e.g., during an incident)
+ssh ansible "systemctl stop ansible-monthly-reboot.timer"
+```
+
+## Rollback
+
+If a guest fails to start after reboot:
+1. Check Proxmox web UI or `pvesh get /nodes/proxmox/qemu/<VMID>/status/current`
+2. Review guest logs: `ssh proxmox "journalctl -u pve-guests -n 50"`
+3. Manual start: `ssh proxmox "pvesh create /nodes/proxmox/qemu/<VMID>/status/start"`
+4. If guest is corrupted, restore from the pre-reboot Proxmox snapshot
+
+## Related Documentation
+
+- [Ansible Controller Setup](../../vm-management/ansible-controller-setup.md) — LXC 304 details and inventory
+- [Proxmox 7→9 Upgrade Plan](../../vm-management/proxmox-upgrades/proxmox-7-to-9-upgrade-plan.md) — original startup order and Phase 1 lessons
+- [VM Decommission Runbook](../../vm-management/vm-decommission-runbook.md) — removing VMs from the rotation

vm-management/proxmox-upgrades/proxmox-7-to-9-upgrade-plan.md

@@ -28,7 +28,7 @@ tags: [proxmox, upgrade, pve, backup, rollback, infrastructure]
 **Production Services** (7 LXC + 7 VMs) — cleaned up 2026-02-19:
 - **Critical**: Paper Dynasty/Major Domo (VM 115), Discord bots (VM 110), Gitea (LXC 225), n8n (LXC 210), Home Assistant (VM 109), Databases (VM 112), docker-home/Pi-hole 1 (VM 106)
 - **Important**: Claude Discord Coordinator (LXC 301), arr-stack (LXC 221), Uptime Kuma (LXC 227), Foundry VTT (LXC 223), Memos (LXC 222)
-- **Stopped/Investigate**: docker-home-servers (VM 116, needs investigation)
+- **Decommission Candidate**: docker-home-servers (VM 116) — Jellyfin-only after 2026-04-03 cleanup; watchstate removed (duplicate of manticore); see issue #31
 - **Removed (2026-02-19)**: 108 (ansible), 224 (openclaw), 300 (openclaw-migrated), 101/102/104/111/211 (game servers), 107 (plex), 113 (tdarr - moved to .226), 114 (duplicate arr-stack), 117 (unused), 100/103 (old templates), 105 (docker-vpn - decommissioned 2026-04)
 
 **Key Constraints**:

workstation/troubleshooting.md

@@ -0,0 +1,33 @@
+---
+title: "Workstation Troubleshooting"
+description: "Troubleshooting notes for Nobara/KDE Wayland workstation issues."
+type: troubleshooting
+domain: workstation
+tags: [troubleshooting, wayland, kde]
+---
+
+# Workstation Troubleshooting
+
+## Discord screen sharing shows no windows on KDE Wayland (2026-04-03)
+
+**Severity:** Medium — cannot share screen via Discord desktop app
+
+**Problem:** Clicking "Share Your Screen" in Discord desktop app (v0.0.131, Electron 37) opens the Discord picker but shows zero windows/screens. Same behavior in both the desktop app and the web app when using Discord's own picker. Affects both native Wayland and XWayland modes.
+
+**Root Cause:** Discord's built-in screen picker uses Electron's `desktopCapturer.getSources()` which relies on X11 window enumeration. On KDE Wayland:
+- In native Wayland mode: no X11 windows exist, so the picker is empty
+- In forced X11/XWayland mode (`ELECTRON_OZONE_PLATFORM_HINT=x11`): Discord can only see other XWayland windows (itself, Android emulator), not native Wayland apps
+- Discord ignores `--use-fake-ui-for-media-stream` and other Chromium flags that should force portal usage
+- The `discord-flags.conf` file is **not read** by the Nobara/RPM Discord package — flags must go in the `.desktop` file `Exec=` line
+
+**Fix:** Use **Discord web app in Firefox** for screen sharing. Firefox natively delegates to the XDG Desktop Portal via PipeWire, which shows the KDE screen picker with all windows. The desktop app's own picker remains broken on Wayland as of v0.0.131.
+
+Configuration applied (for general Discord Wayland support):
+- `~/.local/share/applications/discord.desktop` — overrides system `.desktop` with Wayland flags
+- `~/.config/discord-flags.conf` — created but not read by this Discord build
+
+**Lesson:**
+- Discord desktop on Linux Wayland cannot do screen sharing through its own picker — always use the web app in Firefox for this
+- Electron's `desktopCapturer` API is fundamentally X11-only; the PipeWire/portal path requires the app to use `getDisplayMedia()` instead, which Discord's desktop app does not do
+- `discord-flags.conf` is unreliable across distros — always verify flags landed in `/proc/<pid>/cmdline`
+- Vesktop (community client) is an alternative that properly implements portal-based screen sharing, if the web app is insufficient