401 lines
14 KiB
Bash
Executable File
401 lines
14 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
LAB_DIR="${REPO_ROOT}/.lab"
|
|
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/jeannie-unit.XXXXXX")"
|
|
BACKUP_DIR="${TMP_DIR}/backup"
|
|
FAILURES=0
|
|
LAB_STATE_FILES="pimox-workers.tsv cluster-workers.auto.tfvars.json manual-workers.tsv"
|
|
|
|
mkdir -p "${BACKUP_DIR}" "${LAB_DIR}" "${TMP_DIR}/bin"
|
|
|
|
cleanup() {
|
|
for file in ${LAB_STATE_FILES}; do
|
|
rm -f "${LAB_DIR}/${file}"
|
|
if [[ -f "${BACKUP_DIR}/${file}" ]]; then
|
|
cp "${BACKUP_DIR}/${file}" "${LAB_DIR}/${file}"
|
|
fi
|
|
done
|
|
rm -rf "${TMP_DIR}"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
for file in ${LAB_STATE_FILES}; do
|
|
if [[ -f "${LAB_DIR}/${file}" ]]; then
|
|
cp "${LAB_DIR}/${file}" "${BACKUP_DIR}/${file}"
|
|
fi
|
|
done
|
|
|
|
cat >"${TMP_DIR}/bin/tofu" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
exit 1
|
|
EOF
|
|
chmod +x "${TMP_DIR}/bin/tofu"
|
|
export PATH="${TMP_DIR}/bin:${PATH}"
|
|
|
|
cat >"${LAB_DIR}/pimox-workers.tsv" <<'EOF'
|
|
pimox01 192.168.100.66 jv pimox-worker-01 /home/jv/.ssh/id_ed25519
|
|
pimox02 192.168.100.67 jv pimox-worker-02 /home/jv/.ssh/id_ed25519
|
|
pimox03 192.168.100.76 jv pimox-worker-03 /home/jv/.ssh/id_ed25519
|
|
EOF
|
|
|
|
cat >"${LAB_DIR}/manual-workers.tsv" <<'EOF'
|
|
manual01 192.168.100.99 jv manual-worker-01 /home/jv/.ssh/id_ed25519
|
|
EOF
|
|
|
|
cat >"${LAB_DIR}/cluster-workers.auto.tfvars.json" <<'EOF'
|
|
{
|
|
"worker_nodes": {
|
|
"pimox01": {
|
|
"host": "192.168.100.66",
|
|
"user": "jv",
|
|
"node_name": "pimox-worker-01",
|
|
"ssh_private_key_path": "/home/jv/.ssh/id_ed25519"
|
|
},
|
|
"pimox02": {
|
|
"host": "192.168.100.67",
|
|
"user": "jv",
|
|
"node_name": "pimox-worker-02",
|
|
"ssh_private_key_path": "/home/jv/.ssh/id_ed25519"
|
|
},
|
|
"pimox03": {
|
|
"host": "192.168.100.76",
|
|
"user": "jv",
|
|
"node_name": "pimox-worker-03",
|
|
"ssh_private_key_path": "/home/jv/.ssh/id_ed25519"
|
|
},
|
|
"manual01": {
|
|
"host": "192.168.100.99",
|
|
"user": "jv",
|
|
"node_name": "manual-worker-01",
|
|
"ssh_private_key_path": "/home/jv/.ssh/id_ed25519"
|
|
}
|
|
}
|
|
}
|
|
EOF
|
|
|
|
export JEANNIE_LIBRARY_MODE=true
|
|
# shellcheck disable=SC1091
|
|
source "${REPO_ROOT}/jeannie"
|
|
|
|
pass() {
|
|
printf 'ok %s\n' "$1"
|
|
}
|
|
|
|
fail() {
|
|
printf 'fail %s\n' "$1"
|
|
FAILURES=$((FAILURES + 1))
|
|
}
|
|
|
|
assert_eq() {
|
|
local label="$1"
|
|
local expected="$2"
|
|
local actual="$3"
|
|
|
|
if [[ "${actual}" == "${expected}" ]]; then
|
|
pass "${label}"
|
|
else
|
|
fail "${label}: expected '${expected}', got '${actual}'"
|
|
fi
|
|
}
|
|
|
|
assert_contains() {
|
|
local label="$1"
|
|
local needle="$2"
|
|
local haystack="$3"
|
|
|
|
if grep -Fq "${needle}" <<<"${haystack}"; then
|
|
pass "${label}"
|
|
else
|
|
fail "${label}: expected output to contain '${needle}'"
|
|
fi
|
|
}
|
|
|
|
assert_command_success() {
|
|
local label="$1"
|
|
shift
|
|
|
|
if "$@" >/dev/null 2>&1; then
|
|
pass "${label}"
|
|
else
|
|
fail "${label}: command failed"
|
|
fi
|
|
}
|
|
|
|
assert_command_failure_contains() {
|
|
local label="$1"
|
|
local needle="$2"
|
|
shift 2
|
|
local output
|
|
local status
|
|
|
|
set +e
|
|
output="$("$@" 2>&1)"
|
|
status=$?
|
|
set -e
|
|
if [[ "${status}" -eq 0 ]]; then
|
|
fail "${label}: command unexpectedly passed"
|
|
return 0
|
|
fi
|
|
assert_contains "${label}" "${needle}" "${output}"
|
|
}
|
|
|
|
test_pimox_worker_count_detects_generated_topology() {
|
|
unset LAB_PIMOX_WORKER_COUNT
|
|
assert_eq "detects 3 generated Pimox workers" "3" "$(pimox_worker_count_default)"
|
|
}
|
|
|
|
test_configured_worker_count_cannot_hide_existing_workers() {
|
|
LAB_PIMOX_WORKER_COUNT=1
|
|
assert_eq "effective count keeps detected 3 workers" "3" "$(pimox_worker_count_effective)"
|
|
unset LAB_PIMOX_WORKER_COUNT
|
|
}
|
|
|
|
test_configured_worker_count_can_expand_target_set() {
|
|
LAB_PIMOX_WORKER_COUNT=4
|
|
assert_eq "effective count allows explicit expansion" "4" "$(pimox_worker_count_effective)"
|
|
unset LAB_PIMOX_WORKER_COUNT
|
|
}
|
|
|
|
test_cluster_worker_targets_include_pimox_and_manual_workers() {
|
|
CLUSTER_WORKER_TARGETS=()
|
|
cluster_worker_targets
|
|
local targets
|
|
targets="$(printf '%s\n' "${CLUSTER_WORKER_TARGETS[@]}" | sort | paste -sd ',' -)"
|
|
assert_eq "cluster targets include generated Pimox workers" "jv@192.168.100.66,jv@192.168.100.67,jv@192.168.100.76,jv@192.168.100.99" "${targets}"
|
|
}
|
|
|
|
test_prepare_cluster_worker_var_file_merges_manual_and_pimox_workers() {
|
|
unset LAB_INCLUDE_RASPBERRY_WORKER
|
|
LAB_PIMOX_WORKER_NODE_LABELS_JSON='{"homelab.dev/node-role":"app","homelab.dev/storage":"ssd"}'
|
|
LAB_MANUAL_WORKER_NODE_LABELS_JSON='{"homelab.dev/node-role":"manual","homelab.dev/storage":"custom"}'
|
|
prepare_cluster_worker_var_file false
|
|
python3 - "${LAB_DIR}/cluster-workers.auto.tfvars.json" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
with open(sys.argv[1], encoding="utf-8") as handle:
|
|
document = json.load(handle)
|
|
nodes = document["worker_nodes"]
|
|
labels = document["worker_node_labels"]
|
|
expected = {"pimox01", "pimox02", "pimox03", "manual01"}
|
|
if set(nodes) != expected:
|
|
raise SystemExit(f"worker keys mismatch: {sorted(nodes)}")
|
|
if nodes["manual01"]["node_name"] != "manual-worker-01":
|
|
raise SystemExit("manual worker node name was not preserved")
|
|
if labels["pimox01"]["homelab.dev/storage"] != "ssd":
|
|
raise SystemExit("pimox labels were not applied")
|
|
if labels["manual01"]["homelab.dev/node-role"] != "manual":
|
|
raise SystemExit("manual labels were not applied")
|
|
PY
|
|
pass "worker var file merges manual and Pimox workers"
|
|
unset LAB_PIMOX_WORKER_NODE_LABELS_JSON LAB_MANUAL_WORKER_NODE_LABELS_JSON LAB_CLUSTER_VAR_FILE LAB_INCLUDE_RASPBERRY_WORKER
|
|
}
|
|
|
|
test_prepare_cluster_worker_var_file_can_include_raspberry_worker() {
|
|
LAB_INCLUDE_RASPBERRY_WORKER=true
|
|
LAB_RASPBERRY_HOST=192.168.100.89
|
|
LAB_RASPBERRY_NODE_NAME=raspberry
|
|
prepare_cluster_worker_var_file false
|
|
python3 - "${LAB_DIR}/cluster-workers.auto.tfvars.json" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
with open(sys.argv[1], encoding="utf-8") as handle:
|
|
document = json.load(handle)
|
|
node = document["worker_nodes"].get("raspberrypi")
|
|
if not node:
|
|
raise SystemExit("raspberrypi worker was not included")
|
|
if node["host"] != "192.168.100.89" or node["node_name"] != "raspberry":
|
|
raise SystemExit(f"unexpected raspberry worker: {node}")
|
|
PY
|
|
pass "worker var file can include Raspberry Pi worker"
|
|
unset LAB_INCLUDE_RASPBERRY_WORKER LAB_RASPBERRY_HOST LAB_RASPBERRY_NODE_NAME LAB_CLUSTER_VAR_FILE
|
|
}
|
|
|
|
test_prepare_cluster_worker_var_file_rebuilds_static_pimox_spec() {
|
|
rm -f "${LAB_DIR}/pimox-workers.tsv" "${LAB_DIR}/manual-workers.tsv" "${LAB_DIR}/cluster-workers.auto.tfvars.json"
|
|
LAB_PIMOX_WORKER_NETWORK_MODE=static
|
|
LAB_PIMOX_WORKER_COUNT=2
|
|
LAB_PIMOX_WORKER_STATIC_IPS="192.168.100.66 192.168.100.67"
|
|
LAB_PIMOX_WORKER_USER=jv
|
|
LAB_PIMOX_WORKER_SSH_KEY_PATH=/home/jv/.ssh/id_ed25519
|
|
unset LAB_INCLUDE_RASPBERRY_WORKER
|
|
prepare_cluster_worker_var_file false
|
|
python3 - "${LAB_DIR}/pimox-workers.tsv" "${LAB_DIR}/cluster-workers.auto.tfvars.json" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
spec_file, var_file = sys.argv[1:3]
|
|
with open(spec_file, encoding="utf-8") as handle:
|
|
specs = [line.rstrip("\n").split("\t") for line in handle if line.strip()]
|
|
if specs != [
|
|
["pimox01", "192.168.100.66", "jv", "pimox-worker-01", "/home/jv/.ssh/id_ed25519"],
|
|
["pimox02", "192.168.100.67", "jv", "pimox-worker-02", "/home/jv/.ssh/id_ed25519"],
|
|
]:
|
|
raise SystemExit(f"unexpected static worker specs: {specs}")
|
|
with open(var_file, encoding="utf-8") as handle:
|
|
document = json.load(handle)
|
|
nodes = document.get("worker_nodes") or {}
|
|
if set(nodes) != {"pimox01", "pimox02"}:
|
|
raise SystemExit(f"worker_nodes was not rebuilt from static spec: {nodes}")
|
|
PY
|
|
pass "worker var file rebuilds missing static Pimox spec"
|
|
unset LAB_PIMOX_WORKER_NETWORK_MODE LAB_PIMOX_WORKER_COUNT LAB_PIMOX_WORKER_STATIC_IPS LAB_PIMOX_WORKER_USER LAB_PIMOX_WORKER_SSH_KEY_PATH LAB_CLUSTER_VAR_FILE LAB_INCLUDE_RASPBERRY_WORKER
|
|
}
|
|
|
|
test_report_renderer_hides_ok_links() {
|
|
local output
|
|
output="$(
|
|
printf 'ok\tArea\tHealthy\tok\tdetail\tfix\tgraph\tquery\nwarn\tArea\tWarning\tneeds attention\tdetail\tfix\tgraph\tquery\n' |
|
|
"${REPO_ROOT}/scripts/report-render" --title "Unit Report" --only all
|
|
)"
|
|
if grep -A4 'Healthy' <<<"${output}" | grep -Eq 'graph:|query:'; then
|
|
fail "report renderer hides graph/query for ok rows"
|
|
else
|
|
pass "report renderer hides graph/query for ok rows"
|
|
fi
|
|
}
|
|
|
|
test_report_renderer_problems_only_hides_ok_rows() {
|
|
local output
|
|
output="$(
|
|
printf 'ok\tArea\tHealthy\tok\t\t\t\t\nwarn\tArea\tWarning\tneeds attention\tdetail\tfix\tgraph\tquery\n' |
|
|
"${REPO_ROOT}/scripts/report-render" --title "Unit Report" --only problems
|
|
)"
|
|
if grep -Fq 'Healthy' <<<"${output}"; then
|
|
fail "report renderer problems-only hides ok rows"
|
|
else
|
|
pass "report renderer problems-only hides ok rows"
|
|
fi
|
|
assert_contains "report renderer problems-only keeps warning" "Warning" "${output}"
|
|
assert_contains "report renderer problems-only prints next fix" "Next Fix" "${output}"
|
|
}
|
|
|
|
test_report_renderer_empty_problems_message() {
|
|
local output
|
|
output="$(
|
|
printf 'ok\tArea\tHealthy\tok\t\t\t\t\n' |
|
|
"${REPO_ROOT}/scripts/report-render" --title "Unit Report" --only problems
|
|
)"
|
|
assert_contains "report renderer empty problems message" "No failing or warning items." "${output}"
|
|
}
|
|
|
|
test_inventory_validator_accepts_current_inventory() {
|
|
assert_command_success "inventory validator accepts repo inventory" "${REPO_ROOT}/scripts/validate-homelab-inventory" "${REPO_ROOT}/homelab.yml"
|
|
}
|
|
|
|
test_inventory_validator_rejects_ip_drift() {
|
|
local inventory="${TMP_DIR}/invalid-homelab.yml"
|
|
python3 - "${REPO_ROOT}/homelab.yml" "${inventory}" <<'PY'
|
|
import sys
|
|
|
|
source, target = sys.argv[1:3]
|
|
text = open(source, encoding="utf-8").read()
|
|
text = text.replace("lan_ip: 192.168.100.73", "lan_ip: not-an-ip", 1)
|
|
with open(target, "w", encoding="utf-8") as handle:
|
|
handle.write(text)
|
|
PY
|
|
assert_command_failure_contains "inventory validator rejects invalid Debian LAN IP" "hosts.debian.lan_ip is not a valid IP" "${REPO_ROOT}/scripts/validate-homelab-inventory" "${inventory}"
|
|
}
|
|
|
|
test_heal_cooldown_escalates_to_diagnostics() {
|
|
local status_json="${TMP_DIR}/heal-status.json"
|
|
local state_json="${TMP_DIR}/heal-state.json"
|
|
local output
|
|
|
|
cat >"${status_json}" <<'EOF'
|
|
{
|
|
"rows": [
|
|
{
|
|
"area": "Pimox And Cluster",
|
|
"check": "Kubernetes nodes Ready",
|
|
"status": "fail",
|
|
"summary": "pimox-worker-01 NotReady",
|
|
"detail": "pimox-worker-01 NotReady"
|
|
},
|
|
{
|
|
"area": "Platform And Apps",
|
|
"check": "Traefik deployment",
|
|
"status": "fail",
|
|
"summary": "deployment unavailable"
|
|
}
|
|
]
|
|
}
|
|
EOF
|
|
python3 - "${state_json}" <<'PY'
|
|
import json
|
|
import sys
|
|
import time
|
|
|
|
with open(sys.argv[1], "w", encoding="utf-8") as handle:
|
|
json.dump(
|
|
{
|
|
"failed_actions": {
|
|
"./jeannie workers restart 1": {
|
|
"failed_at": int(time.time()),
|
|
"finding": "Pimox And Cluster / Kubernetes nodes Ready",
|
|
}
|
|
}
|
|
},
|
|
handle,
|
|
)
|
|
PY
|
|
|
|
output="$(
|
|
LAB_HEAL_STATE_FILE="${state_json}" LAB_HEAL_COOLDOWN_SECONDS=3600 \
|
|
"${REPO_ROOT}/scripts/heal" plan --status-json "${status_json}"
|
|
)"
|
|
assert_contains "heal cooldown removes auto target" "Next heal target: none" "${output}"
|
|
assert_contains "heal cooldown escalates diagnostics" "next diagnostic command: ./jeannie doctor-cluster --details" "${output}"
|
|
}
|
|
|
|
test_pimox_worker_static_ip_selection() {
|
|
assert_eq "pimox worker static IP index 1" "192.168.100.66" "$(pimox_worker_static_ip 1 "192.168.100.66 192.168.100.67")"
|
|
assert_eq "pimox worker static IP index 2" "192.168.100.67" "$(pimox_worker_static_ip 2 "192.168.100.66,192.168.100.67")"
|
|
if pimox_worker_static_ip 3 "192.168.100.66 192.168.100.67" >/dev/null; then
|
|
fail "pimox worker static IP missing index should fail"
|
|
else
|
|
pass "pimox worker static IP missing index should fail"
|
|
fi
|
|
}
|
|
|
|
test_pimox_guest_exec_exitcode_parser() {
|
|
assert_eq "pimox guest exec parses success exitcode" "0" "$(pimox_guest_exec_exitcode '{"exitcode":0,"exited":1}')"
|
|
assert_eq "pimox guest exec parses failed exitcode" "1" "$(pimox_guest_exec_exitcode '{"exitcode":1,"exited":1}')"
|
|
assert_command_failure_contains "pimox guest exec rejects invalid JSON" "" pimox_guest_exec_exitcode 'not json'
|
|
}
|
|
|
|
test_pimox_worker_net0_config_defaults_to_deterministic_mac() {
|
|
unset LAB_PIMOX_WORKER_MAC_MODE
|
|
assert_eq "pimox worker net0 defaults to deterministic MAC" "virtio=02:68:10:00:23:32,bridge=vmbr0" "$(pimox_worker_net0_config 9010 vmbr0)"
|
|
LAB_PIMOX_WORKER_MAC_MODE=auto
|
|
assert_eq "pimox worker net0 auto MAC opt-in" "virtio,bridge=vmbr0" "$(pimox_worker_net0_config 9010 vmbr0)"
|
|
unset LAB_PIMOX_WORKER_MAC_MODE
|
|
}
|
|
|
|
test_pimox_worker_count_detects_generated_topology
|
|
test_configured_worker_count_cannot_hide_existing_workers
|
|
test_configured_worker_count_can_expand_target_set
|
|
test_cluster_worker_targets_include_pimox_and_manual_workers
|
|
test_prepare_cluster_worker_var_file_merges_manual_and_pimox_workers
|
|
test_prepare_cluster_worker_var_file_can_include_raspberry_worker
|
|
test_prepare_cluster_worker_var_file_rebuilds_static_pimox_spec
|
|
test_report_renderer_hides_ok_links
|
|
test_report_renderer_problems_only_hides_ok_rows
|
|
test_report_renderer_empty_problems_message
|
|
test_inventory_validator_accepts_current_inventory
|
|
test_inventory_validator_rejects_ip_drift
|
|
test_heal_cooldown_escalates_to_diagnostics
|
|
test_pimox_worker_static_ip_selection
|
|
test_pimox_guest_exec_exitcode_parser
|
|
test_pimox_worker_net0_config_defaults_to_deterministic_mac
|
|
|
|
if ((FAILURES > 0)); then
|
|
printf '\n%s unit test(s) failed.\n' "${FAILURES}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
printf '\nAll jeannie unit tests passed.\n'
|