f28dfeb4bf
All checks were successful
Auto-merge docs-only PRs / auto-merge-docs (pull_request) Successful in 3s
Extend homelab-audit.sh collector with zombie_parents(), swap_mb(), and oom_events() functions so the audit identifies which process spawns zombies, flags high swap usage, and reports recent OOM kills. Add init: true to both Tdarr docker-compose services so tini reaps orphaned ffmpeg children, and cap tdarr-node at 28g RAM / 30g total to prevent unbounded memory use. Closes #30 Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
311 lines
9.7 KiB
Bash
Executable File
311 lines
9.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# homelab-audit.sh — SSH-based homelab health audit
|
|
#
|
|
# Runs on the Proxmox host. Discovers running LXCs and VMs, SSHes into each
|
|
# to collect system metrics, then generates a summary report.
|
|
#
|
|
# Usage:
|
|
# homelab-audit.sh [--output-dir DIR]
|
|
#
|
|
# Environment overrides:
|
|
# STUCK_PROC_CPU_WARN CPU% at which a D-state process is flagged (default: 10)
|
|
# REPORT_DIR Output directory for per-host reports and logs
|
|
# SSH_USER Remote user (default: root)
|
|
|
|
# -e omitted intentionally — unreachable hosts should not abort the full audit
|
|
set -uo pipefail
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Configuration
|
|
# ---------------------------------------------------------------------------
|
|
STUCK_PROC_CPU_WARN="${STUCK_PROC_CPU_WARN:-10}"
|
|
REPORT_DIR="${REPORT_DIR:-/tmp/homelab-audit-$(date +%Y%m%d-%H%M%S)}"
|
|
SSH_USER="${SSH_USER:-root}"
|
|
SSH_OPTS="-o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o BatchMode=yes"
|
|
|
|
DISK_WARN=80
|
|
DISK_CRIT=90
|
|
LOAD_WARN=2.0
|
|
MEM_WARN=85
|
|
ZOMBIE_WARN=1
|
|
SWAP_WARN=512
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--output-dir)
|
|
if [[ $# -lt 2 ]]; then
|
|
echo "Error: --output-dir requires an argument" >&2
|
|
exit 1
|
|
fi
|
|
REPORT_DIR="$2"
|
|
shift 2
|
|
;;
|
|
*)
|
|
echo "Unknown option: $1" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
mkdir -p "$REPORT_DIR"
|
|
SSH_FAILURES_LOG="$REPORT_DIR/ssh-failures.log"
|
|
FINDINGS_FILE="$REPORT_DIR/findings.txt"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Remote collector script
|
|
#
|
|
# Kept single-quoted so no local variables are interpolated into the heredoc.
|
|
# STUCK_PROC_CPU_WARN is passed as $1 when invoking the remote bash session,
|
|
# so the configurable threshold reaches the collector without escaping issues.
|
|
# ---------------------------------------------------------------------------
|
|
COLLECTOR_SCRIPT='#!/usr/bin/env bash
|
|
STUCK_PROC_CPU_WARN="${1:-10}"
|
|
|
|
cpu_load() {
|
|
uptime | awk -F"load average:" "{print \$2}" | awk -F"[, ]+" "{print \$2}"
|
|
}
|
|
|
|
mem_pct() {
|
|
free | awk "/^Mem:/ {printf \"%.0f\", \$3/\$2*100}"
|
|
}
|
|
|
|
disk_usage() {
|
|
df --output=pcent,target -x tmpfs -x devtmpfs 2>/dev/null | tail -n +2 | \
|
|
while read -r pct mnt; do echo "${pct%%%} $mnt"; done
|
|
}
|
|
|
|
zombie_count() {
|
|
ps -eo stat= | grep -c "^Z" || true
|
|
}
|
|
|
|
stuck_procs() {
|
|
ps -eo stat=,pcpu=,comm= | \
|
|
awk -v t="$STUCK_PROC_CPU_WARN" '\''$1 ~ /^D/ && $2+0 >= t+0 {print $3}'\'' | \
|
|
paste -sd,
|
|
}
|
|
|
|
zombie_parents() {
|
|
ps -eo pid=,ppid=,stat= | awk '\''$3 ~ /^Z/ {print $2}'\'' | sort -u | \
|
|
xargs -I{} ps -o comm= -p {} 2>/dev/null | paste -sd,
|
|
}
|
|
|
|
swap_mb() {
|
|
free | awk '\''/^Swap:/ {printf "%.0f", $3/1024}'\''
|
|
}
|
|
|
|
oom_events() {
|
|
local count
|
|
count=$(journalctl -k --since "7 days ago" 2>/dev/null | grep -ci "out of memory") || true
|
|
echo "${count:-0}"
|
|
}
|
|
|
|
echo "CPU_LOAD=$(cpu_load)"
|
|
echo "MEM_PCT=$(mem_pct)"
|
|
echo "ZOMBIES=$(zombie_count)"
|
|
echo "STUCK_PROCS=$(stuck_procs)"
|
|
echo "ZOMBIE_PARENTS=$(zombie_parents)"
|
|
echo "SWAP_MB=$(swap_mb)"
|
|
echo "OOM_EVENTS=$(oom_events)"
|
|
disk_usage | while read -r pct mnt; do
|
|
echo "DISK $pct $mnt"
|
|
done
|
|
'
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# SSH helper — logs stderr to ssh-failures.log instead of silently discarding
|
|
# ---------------------------------------------------------------------------
|
|
ssh_cmd() {
|
|
local host="$1"
|
|
shift
|
|
# shellcheck disable=SC2086
|
|
ssh $SSH_OPTS "${SSH_USER}@${host}" "$@" 2>>"$SSH_FAILURES_LOG"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# LXC IP discovery
|
|
#
|
|
# lxc-info only returns IPs for containers using Proxmox-managed DHCP bridges.
|
|
# Containers with static IPs defined inside the container (not via Proxmox
|
|
# network config) return nothing. Fall back to parsing `pct config` in that
|
|
# case to find the ip= field from the container's network interface config.
|
|
# ---------------------------------------------------------------------------
|
|
get_lxc_ip() {
|
|
local ctid="$1"
|
|
local ip
|
|
ip=$(lxc-info -n "$ctid" -iH 2>/dev/null | head -1)
|
|
if [[ -z "$ip" ]]; then
|
|
ip=$(pct config "$ctid" 2>/dev/null | grep -oP '(?<=ip=)[^/,]+' | head -1)
|
|
fi
|
|
echo "$ip"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Inventory: running LXCs and VMs
|
|
# Returns lines of "label ip"
|
|
# ---------------------------------------------------------------------------
|
|
collect_inventory() {
|
|
# LXCs
|
|
pct list 2>/dev/null | tail -n +2 | while read -r ctid status _name; do
|
|
[[ "$status" != "running" ]] && continue
|
|
local ip
|
|
ip=$(get_lxc_ip "$ctid")
|
|
[[ -n "$ip" ]] && echo "lxc-${ctid} $ip"
|
|
done
|
|
|
|
# VMs — use agent network info if available, fall back to qm config
|
|
qm list 2>/dev/null | tail -n +2 | while read -r vmid _name status _mem _bootdisk _pid; do
|
|
[[ "$status" != "running" ]] && continue
|
|
local ip
|
|
ip=$(qm guest cmd "$vmid" network-get-interfaces 2>/dev/null |
|
|
python3 -c "
|
|
import sys, json
|
|
try:
|
|
data = json.load(sys.stdin)
|
|
for iface in data:
|
|
for addr in iface.get('ip-addresses', []):
|
|
if addr['ip-address-type'] == 'ipv4' and not addr['ip-address'].startswith('127.'):
|
|
print(addr['ip-address'])
|
|
raise SystemExit
|
|
except Exception:
|
|
pass
|
|
" 2>/dev/null)
|
|
[[ -n "$ip" ]] && echo "vm-${vmid} $ip"
|
|
done
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Collect metrics from one host and record findings
|
|
# ---------------------------------------------------------------------------
|
|
parse_and_report() {
|
|
local label="$1"
|
|
local addr="$2"
|
|
local raw
|
|
|
|
if ! raw=$(echo "$COLLECTOR_SCRIPT" | ssh_cmd "$addr" bash -s -- "$STUCK_PROC_CPU_WARN"); then
|
|
echo "SSH_FAILURE $label $addr" >>"$SSH_FAILURES_LOG"
|
|
echo "WARN $label: SSH connection failed" >>"$FINDINGS_FILE"
|
|
return
|
|
fi
|
|
|
|
while IFS= read -r line; do
|
|
case "$line" in
|
|
CPU_LOAD=*)
|
|
local load="${line#CPU_LOAD=}"
|
|
if [[ -n "$load" ]] && awk "BEGIN{exit !($load > $LOAD_WARN)}"; then
|
|
echo "WARN $label: load average ${load} > ${LOAD_WARN}" >>"$FINDINGS_FILE"
|
|
fi
|
|
;;
|
|
MEM_PCT=*)
|
|
local mem="${line#MEM_PCT=}"
|
|
if [[ -n "$mem" ]] && ((mem >= MEM_WARN)); then
|
|
echo "WARN $label: memory ${mem}% >= ${MEM_WARN}%" >>"$FINDINGS_FILE"
|
|
fi
|
|
;;
|
|
ZOMBIES=*)
|
|
local zombies="${line#ZOMBIES=}"
|
|
if [[ -n "$zombies" ]] && ((zombies >= ZOMBIE_WARN)); then
|
|
echo "WARN $label: ${zombies} zombie process(es)" >>"$FINDINGS_FILE"
|
|
fi
|
|
;;
|
|
STUCK_PROCS=*)
|
|
local procs="${line#STUCK_PROCS=}"
|
|
if [[ -n "$procs" ]]; then
|
|
echo "WARN $label: D-state procs with CPU>=${STUCK_PROC_CPU_WARN}%: ${procs}" >>"$FINDINGS_FILE"
|
|
fi
|
|
;;
|
|
ZOMBIE_PARENTS=*)
|
|
local zparents="${line#ZOMBIE_PARENTS=}"
|
|
if [[ -n "$zparents" ]]; then
|
|
echo "INFO $label: zombie parent process(es): ${zparents}" >>"$FINDINGS_FILE"
|
|
fi
|
|
;;
|
|
SWAP_MB=*)
|
|
local swap="${line#SWAP_MB=}"
|
|
if [[ -n "$swap" ]] && ((swap >= SWAP_WARN)); then
|
|
echo "WARN $label: swap usage ${swap} MB >= ${SWAP_WARN} MB" >>"$FINDINGS_FILE"
|
|
fi
|
|
;;
|
|
OOM_EVENTS=*)
|
|
local ooms="${line#OOM_EVENTS=}"
|
|
if [[ -n "$ooms" ]] && ((ooms > 0)); then
|
|
echo "WARN $label: ${ooms} OOM kill event(s) in last 7 days" >>"$FINDINGS_FILE"
|
|
fi
|
|
;;
|
|
DISK\ *)
|
|
local pct mnt
|
|
read -r _ pct mnt <<<"$line"
|
|
if ((pct >= DISK_CRIT)); then
|
|
echo "CRIT $label: disk ${mnt} at ${pct}% >= ${DISK_CRIT}%" >>"$FINDINGS_FILE"
|
|
elif ((pct >= DISK_WARN)); then
|
|
echo "WARN $label: disk ${mnt} at ${pct}% >= ${DISK_WARN}%" >>"$FINDINGS_FILE"
|
|
fi
|
|
;;
|
|
esac
|
|
done <<<"$raw"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Summary — driven by actual findings in findings.txt and ssh-failures.log
|
|
# ---------------------------------------------------------------------------
|
|
generate_summary() {
|
|
local host_count="$1"
|
|
local ssh_failure_count=0
|
|
local warn_count=0
|
|
local crit_count=0
|
|
|
|
[[ -f "$SSH_FAILURES_LOG" ]] &&
|
|
ssh_failure_count=$(grep -c '^SSH_FAILURE' "$SSH_FAILURES_LOG" 2>/dev/null || true)
|
|
[[ -f "$FINDINGS_FILE" ]] &&
|
|
warn_count=$(grep -c '^WARN' "$FINDINGS_FILE" 2>/dev/null || true)
|
|
[[ -f "$FINDINGS_FILE" ]] &&
|
|
crit_count=$(grep -c '^CRIT' "$FINDINGS_FILE" 2>/dev/null || true)
|
|
|
|
echo ""
|
|
echo "=============================="
|
|
echo " HOMELAB AUDIT SUMMARY"
|
|
echo "=============================="
|
|
printf " Hosts audited : %d\n" "$host_count"
|
|
printf " SSH failures : %d\n" "$ssh_failure_count"
|
|
printf " Warnings : %d\n" "$warn_count"
|
|
printf " Critical : %d\n" "$crit_count"
|
|
echo "=============================="
|
|
|
|
if ((warn_count + crit_count > 0)); then
|
|
echo ""
|
|
echo "Findings:"
|
|
sort "$FINDINGS_FILE"
|
|
fi
|
|
|
|
if ((ssh_failure_count > 0)); then
|
|
echo ""
|
|
echo "SSH failures (see $SSH_FAILURES_LOG for details):"
|
|
grep '^SSH_FAILURE' "$SSH_FAILURES_LOG" | awk '{print " " $2 " (" $3 ")"}'
|
|
fi
|
|
|
|
echo ""
|
|
echo "Reports: $REPORT_DIR"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
main() {
|
|
echo "Starting homelab audit — $(date)"
|
|
echo "Report dir: $REPORT_DIR"
|
|
echo "STUCK_PROC_CPU_WARN threshold: ${STUCK_PROC_CPU_WARN}%"
|
|
echo ""
|
|
|
|
>"$FINDINGS_FILE"
|
|
|
|
local host_count=0
|
|
while read -r label addr; do
|
|
echo " Auditing $label ($addr)..."
|
|
parse_and_report "$label" "$addr"
|
|
((host_count++)) || true
|
|
done < <(collect_inventory)
|
|
|
|
generate_summary "$host_count"
|
|
}
|
|
|
|
main "$@"
|