Compare commits
No commits in common. "0cda6573ad9f74e3a7dafdadeed88e9366619a6f" and "1bdde9c128f053666574fe19d9e7ed7e9b1007d5" have entirely different histories.
0cda6573ad
...
1bdde9c128
|
|
@ -734,9 +734,6 @@ qemu-guest-agent. Static mode uses
|
||||||
`LAB_PIMOX_WORKER_STATIC_IPS="192.168.100.66 192.168.100.67 192.168.100.76"`,
|
`LAB_PIMOX_WORKER_STATIC_IPS="192.168.100.66 192.168.100.67 192.168.100.76"`,
|
||||||
`LAB_PIMOX_WORKER_GATEWAY=192.168.100.1`, and
|
`LAB_PIMOX_WORKER_GATEWAY=192.168.100.1`, and
|
||||||
`LAB_PIMOX_WORKER_DNS_SERVERS="192.168.100.89 1.1.1.1"`.
|
`LAB_PIMOX_WORKER_DNS_SERVERS="192.168.100.89 1.1.1.1"`.
|
||||||
Worker NIC MAC addresses are also auto-generated by Pimox by default. Set
|
|
||||||
`LAB_PIMOX_WORKER_MAC_MODE=deterministic` only when you explicitly need stable
|
|
||||||
MAC addresses derived from VM IDs.
|
|
||||||
Worker creation fails faster than template creation: static guest-network
|
Worker creation fails faster than template creation: static guest-network
|
||||||
configuration waits up to `LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`,
|
configuration waits up to `LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`,
|
||||||
worker IP/SSH readiness waits up to `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`,
|
worker IP/SSH readiness waits up to `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`,
|
||||||
|
|
@ -753,11 +750,6 @@ Long-running `./jeannie up` steps keep compact output by default, but
|
||||||
the active step line refreshes with elapsed time and the latest log line. Set
|
the active step line refreshes with elapsed time and the latest log line. Set
|
||||||
`JEANNIE_PROGRESS_INTERVAL_SECONDS=5` for faster heartbeats, or
|
`JEANNIE_PROGRESS_INTERVAL_SECONDS=5` for faster heartbeats, or
|
||||||
`JEANNIE_VERBOSE=true` to stream full command output.
|
`JEANNIE_VERBOSE=true` to stream full command output.
|
||||||
Independent host services run through the Go DAG runner during `up`: Gitea
|
|
||||||
deploy/bootstrap, RPi services, Pimox workers, OpenWrt, and OCI edge host checks
|
|
||||||
can progress concurrently. Set `JEANNIE_UP_PARALLELISM=1` to force sequential
|
|
||||||
execution while troubleshooting, or raise it above the default `4` if the lab
|
|
||||||
can absorb more concurrent work.
|
|
||||||
|
|
||||||
Run a full cluster rebuild from the Debian server with:
|
Run a full cluster rebuild from the Debian server with:
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -734,9 +734,6 @@ qemu-guest-agent. Static mode uses
|
||||||
`LAB_PIMOX_WORKER_STATIC_IPS="192.168.100.66 192.168.100.67 192.168.100.76"`,
|
`LAB_PIMOX_WORKER_STATIC_IPS="192.168.100.66 192.168.100.67 192.168.100.76"`,
|
||||||
`LAB_PIMOX_WORKER_GATEWAY=192.168.100.1`, and
|
`LAB_PIMOX_WORKER_GATEWAY=192.168.100.1`, and
|
||||||
`LAB_PIMOX_WORKER_DNS_SERVERS="192.168.100.89 1.1.1.1"`.
|
`LAB_PIMOX_WORKER_DNS_SERVERS="192.168.100.89 1.1.1.1"`.
|
||||||
Worker NIC MAC addresses are also auto-generated by Pimox by default. Set
|
|
||||||
`LAB_PIMOX_WORKER_MAC_MODE=deterministic` only when you explicitly need stable
|
|
||||||
MAC addresses derived from VM IDs.
|
|
||||||
Worker creation fails faster than template creation: static guest-network
|
Worker creation fails faster than template creation: static guest-network
|
||||||
configuration waits up to `LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`,
|
configuration waits up to `LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`,
|
||||||
worker IP/SSH readiness waits up to `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`,
|
worker IP/SSH readiness waits up to `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`,
|
||||||
|
|
@ -753,11 +750,6 @@ Long-running `./{{ main_script }} up` steps keep compact output by default, but
|
||||||
the active step line refreshes with elapsed time and the latest log line. Set
|
the active step line refreshes with elapsed time and the latest log line. Set
|
||||||
`JEANNIE_PROGRESS_INTERVAL_SECONDS=5` for faster heartbeats, or
|
`JEANNIE_PROGRESS_INTERVAL_SECONDS=5` for faster heartbeats, or
|
||||||
`JEANNIE_VERBOSE=true` to stream full command output.
|
`JEANNIE_VERBOSE=true` to stream full command output.
|
||||||
Independent host services run through the Go DAG runner during `up`: Gitea
|
|
||||||
deploy/bootstrap, RPi services, Pimox workers, OpenWrt, and OCI edge host checks
|
|
||||||
can progress concurrently. Set `JEANNIE_UP_PARALLELISM=1` to force sequential
|
|
||||||
execution while troubleshooting, or raise it above the default `4` if the lab
|
|
||||||
can absorb more concurrent work.
|
|
||||||
|
|
||||||
Run a full cluster rebuild from the Debian server with:
|
Run a full cluster rebuild from the Debian server with:
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -148,9 +148,6 @@ reservations are not available: set `LAB_PIMOX_WORKER_NETWORK_MODE=static`,
|
||||||
`LAB_PIMOX_WORKER_STATIC_IPS="192.168.100.66 192.168.100.67 192.168.100.76"`,
|
`LAB_PIMOX_WORKER_STATIC_IPS="192.168.100.66 192.168.100.67 192.168.100.76"`,
|
||||||
`LAB_PIMOX_WORKER_GATEWAY=192.168.100.1`, `LAB_PIMOX_WORKER_INTERFACE=enp0s18`,
|
`LAB_PIMOX_WORKER_GATEWAY=192.168.100.1`, `LAB_PIMOX_WORKER_INTERFACE=enp0s18`,
|
||||||
and `LAB_PIMOX_WORKER_DNS_SERVERS="192.168.100.89 1.1.1.1"`.
|
and `LAB_PIMOX_WORKER_DNS_SERVERS="192.168.100.89 1.1.1.1"`.
|
||||||
Worker NIC MAC addresses are auto-generated by Pimox by default. Set
|
|
||||||
`LAB_PIMOX_WORKER_MAC_MODE=deterministic` only when VMID-derived MAC addresses
|
|
||||||
are required.
|
|
||||||
Worker clone failures are intentionally faster than template build failures:
|
Worker clone failures are intentionally faster than template build failures:
|
||||||
`LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`,
|
`LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`,
|
||||||
`LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`, and
|
`LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`, and
|
||||||
|
|
|
||||||
|
|
@ -1,18 +1,13 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type reportRow struct {
|
type reportRow struct {
|
||||||
|
|
@ -38,11 +33,6 @@ func main() {
|
||||||
fmt.Fprintln(os.Stderr, err)
|
fmt.Fprintln(os.Stderr, err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
case "run-dag":
|
|
||||||
if err := runDAG(os.Args[2:], os.Stdout, os.Stderr); err != nil {
|
|
||||||
fmt.Fprintln(os.Stderr, err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
case "help", "-h", "--help":
|
case "help", "-h", "--help":
|
||||||
usage(os.Stdout)
|
usage(os.Stdout)
|
||||||
default:
|
default:
|
||||||
|
|
@ -52,184 +42,7 @@ func main() {
|
||||||
}
|
}
|
||||||
|
|
||||||
func usage(out io.Writer) {
|
func usage(out io.Writer) {
|
||||||
fmt.Fprintln(out, "Usage: jeannie-core {report-render|run-dag}")
|
fmt.Fprintln(out, "Usage: jeannie-core report-render [--title TITLE] [--details] [--only failures|warnings|problems|all] [--json]")
|
||||||
}
|
|
||||||
|
|
||||||
type dagSpec struct {
|
|
||||||
Tasks []dagTask `json:"tasks"`
|
|
||||||
Parallelism int `json:"parallelism"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type dagTask struct {
|
|
||||||
ID string `json:"id"`
|
|
||||||
Title string `json:"title"`
|
|
||||||
Command []string `json:"command"`
|
|
||||||
Needs []string `json:"needs"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type dagTaskResult struct {
|
|
||||||
id string
|
|
||||||
exitCode int
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func runDAG(args []string, out io.Writer, errOut io.Writer) error {
|
|
||||||
flags := flag.NewFlagSet("run-dag", flag.ContinueOnError)
|
|
||||||
flags.SetOutput(io.Discard)
|
|
||||||
specPath := flags.String("spec", "", "")
|
|
||||||
if err := flags.Parse(args); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if *specPath == "" {
|
|
||||||
return fmt.Errorf("run-dag requires --spec")
|
|
||||||
}
|
|
||||||
|
|
||||||
specData, err := os.ReadFile(*specPath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
var spec dagSpec
|
|
||||||
if err := json.Unmarshal(specData, &spec); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return executeDAG(context.Background(), spec, out, errOut)
|
|
||||||
}
|
|
||||||
|
|
||||||
func executeDAG(ctx context.Context, spec dagSpec, out io.Writer, errOut io.Writer) error {
|
|
||||||
if spec.Parallelism <= 0 {
|
|
||||||
spec.Parallelism = 4
|
|
||||||
}
|
|
||||||
|
|
||||||
taskByID := map[string]dagTask{}
|
|
||||||
dependents := map[string][]string{}
|
|
||||||
remainingNeeds := map[string]int{}
|
|
||||||
for _, task := range spec.Tasks {
|
|
||||||
if task.ID == "" {
|
|
||||||
return fmt.Errorf("dag task has empty id")
|
|
||||||
}
|
|
||||||
if _, exists := taskByID[task.ID]; exists {
|
|
||||||
return fmt.Errorf("duplicate dag task id %q", task.ID)
|
|
||||||
}
|
|
||||||
if len(task.Command) == 0 {
|
|
||||||
return fmt.Errorf("dag task %q has empty command", task.ID)
|
|
||||||
}
|
|
||||||
taskByID[task.ID] = task
|
|
||||||
remainingNeeds[task.ID] = len(task.Needs)
|
|
||||||
for _, need := range task.Needs {
|
|
||||||
dependents[need] = append(dependents[need], task.ID)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, task := range spec.Tasks {
|
|
||||||
for _, need := range task.Needs {
|
|
||||||
if _, exists := taskByID[need]; !exists {
|
|
||||||
return fmt.Errorf("dag task %q depends on unknown task %q", task.ID, need)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(ctx)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
ready := make(chan string, len(spec.Tasks))
|
|
||||||
results := make(chan dagTaskResult, len(spec.Tasks))
|
|
||||||
sem := make(chan struct{}, spec.Parallelism)
|
|
||||||
var writersMu sync.Mutex
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
started := map[string]bool{}
|
|
||||||
completed := map[string]bool{}
|
|
||||||
|
|
||||||
for id, count := range remainingNeeds {
|
|
||||||
if count == 0 {
|
|
||||||
ready <- id
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
failed := false
|
|
||||||
for len(completed) < len(spec.Tasks) {
|
|
||||||
select {
|
|
||||||
case id := <-ready:
|
|
||||||
if started[id] || failed {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
started[id] = true
|
|
||||||
task := taskByID[id]
|
|
||||||
wg.Add(1)
|
|
||||||
go func() {
|
|
||||||
defer wg.Done()
|
|
||||||
sem <- struct{}{}
|
|
||||||
defer func() { <-sem }()
|
|
||||||
results <- runDAGTask(ctx, task, out, errOut, &writersMu)
|
|
||||||
}()
|
|
||||||
case result := <-results:
|
|
||||||
completed[result.id] = true
|
|
||||||
if result.err != nil || result.exitCode != 0 {
|
|
||||||
failed = true
|
|
||||||
cancel()
|
|
||||||
wg.Wait()
|
|
||||||
if result.err != nil {
|
|
||||||
return result.err
|
|
||||||
}
|
|
||||||
return fmt.Errorf("dag task %q failed with exit code %d", result.id, result.exitCode)
|
|
||||||
}
|
|
||||||
for _, dependent := range dependents[result.id] {
|
|
||||||
remainingNeeds[dependent]--
|
|
||||||
if remainingNeeds[dependent] == 0 {
|
|
||||||
ready <- dependent
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func runDAGTask(ctx context.Context, task dagTask, out io.Writer, errOut io.Writer, writersMu *sync.Mutex) dagTaskResult {
|
|
||||||
title := task.Title
|
|
||||||
if title == "" {
|
|
||||||
title = task.ID
|
|
||||||
}
|
|
||||||
started := time.Now()
|
|
||||||
writeDAGLine(out, writersMu, task.ID, "started "+title)
|
|
||||||
|
|
||||||
cmd := exec.CommandContext(ctx, task.Command[0], task.Command[1:]...)
|
|
||||||
cmd.Env = os.Environ()
|
|
||||||
stdout, err := cmd.StdoutPipe()
|
|
||||||
if err != nil {
|
|
||||||
return dagTaskResult{id: task.ID, exitCode: 1, err: err}
|
|
||||||
}
|
|
||||||
cmd.Stderr = cmd.Stdout
|
|
||||||
if err := cmd.Start(); err != nil {
|
|
||||||
return dagTaskResult{id: task.ID, exitCode: 1, err: err}
|
|
||||||
}
|
|
||||||
|
|
||||||
scanner := bufio.NewScanner(stdout)
|
|
||||||
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
|
||||||
for scanner.Scan() {
|
|
||||||
writeDAGLine(out, writersMu, task.ID, scanner.Text())
|
|
||||||
}
|
|
||||||
if scanErr := scanner.Err(); scanErr != nil {
|
|
||||||
writeDAGLine(errOut, writersMu, task.ID, "output read error: "+scanErr.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
err = cmd.Wait()
|
|
||||||
elapsed := time.Since(started).Round(time.Second)
|
|
||||||
exitCode := 0
|
|
||||||
if err != nil {
|
|
||||||
exitCode = 1
|
|
||||||
if exitErr, ok := err.(*exec.ExitError); ok {
|
|
||||||
exitCode = exitErr.ExitCode()
|
|
||||||
}
|
|
||||||
writeDAGLine(out, writersMu, task.ID, fmt.Sprintf("failed after %s", elapsed))
|
|
||||||
return dagTaskResult{id: task.ID, exitCode: exitCode, err: nil}
|
|
||||||
}
|
|
||||||
writeDAGLine(out, writersMu, task.ID, fmt.Sprintf("completed in %s", elapsed))
|
|
||||||
return dagTaskResult{id: task.ID, exitCode: 0, err: nil}
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeDAGLine(out io.Writer, writersMu *sync.Mutex, id string, line string) {
|
|
||||||
writersMu.Lock()
|
|
||||||
defer writersMu.Unlock()
|
|
||||||
fmt.Fprintf(out, "[%s] %s\n", id, line)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func runReportRender(args []string, in io.Reader, out io.Writer) error {
|
func runReportRender(args []string, in io.Reader, out io.Writer) error {
|
||||||
|
|
|
||||||
|
|
@ -2,9 +2,6 @@ package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
@ -45,46 +42,3 @@ func TestReportRenderHidesOKGraphAndQuery(t *testing.T) {
|
||||||
t.Fatalf("ok row leaked graph/query links:\n%s", got)
|
t.Fatalf("ok row leaked graph/query links:\n%s", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExecuteDAGRespectsDependencies(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
first := filepath.Join(dir, "first")
|
|
||||||
second := filepath.Join(dir, "second")
|
|
||||||
spec := dagSpec{
|
|
||||||
Parallelism: 2,
|
|
||||||
Tasks: []dagTask{
|
|
||||||
{
|
|
||||||
ID: "first",
|
|
||||||
Command: []string{"sh", "-c", "printf first > \"$1\"", "sh", first},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
ID: "second",
|
|
||||||
Needs: []string{"first"},
|
|
||||||
Command: []string{"sh", "-c", "test -f \"$1\" && printf second > \"$2\"", "sh", first, second},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
var output bytes.Buffer
|
|
||||||
if err := executeDAG(context.Background(), spec, &output, &output); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(second); err != nil {
|
|
||||||
t.Fatalf("dependent task did not run after dependency: %v\n%s", err, output.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExecuteDAGReturnsFailedTask(t *testing.T) {
|
|
||||||
spec := dagSpec{
|
|
||||||
Parallelism: 2,
|
|
||||||
Tasks: []dagTask{
|
|
||||||
{ID: "bad", Command: []string{"sh", "-c", "exit 7"}},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
var output bytes.Buffer
|
|
||||||
err := executeDAG(context.Background(), spec, &output, &output)
|
|
||||||
if err == nil || !strings.Contains(err.Error(), `dag task "bad" failed`) {
|
|
||||||
t.Fatalf("expected failed task error, got %v\n%s", err, output.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -535,10 +535,6 @@ falls back to the first Lynis warning, then the first suggestion.
|
||||||
`LAB_AUTO_APPROVE=false`
|
`LAB_AUTO_APPROVE=false`
|
||||||
: Disable automatic OpenTofu approval for apply paths that support confirmation.
|
: Disable automatic OpenTofu approval for apply paths that support confirmation.
|
||||||
|
|
||||||
`JEANNIE_UP_PARALLELISM`
|
|
||||||
: Maximum number of independent `up` branches the Go DAG runner may execute at
|
|
||||||
once. Defaults to `4`; set to `1` for sequential troubleshooting.
|
|
||||||
|
|
||||||
`LAB_PIMOX_WORKER_COUNT`
|
`LAB_PIMOX_WORKER_COUNT`
|
||||||
: Desired Pimox worker count when not supplied by generated cluster topology
|
: Desired Pimox worker count when not supplied by generated cluster topology
|
||||||
state.
|
state.
|
||||||
|
|
@ -547,10 +543,6 @@ state.
|
||||||
: Worker guest networking mode. Defaults to `dhcp`. Set to `static` only when
|
: Worker guest networking mode. Defaults to `dhcp`. Set to `static` only when
|
||||||
Jeannie should configure guest static IPs through qemu-guest-agent.
|
Jeannie should configure guest static IPs through qemu-guest-agent.
|
||||||
|
|
||||||
`LAB_PIMOX_WORKER_MAC_MODE`
|
|
||||||
: Worker NIC MAC mode. Defaults to `auto`, letting Pimox generate the MAC. Set
|
|
||||||
to `deterministic` to derive MAC addresses from VM IDs.
|
|
||||||
|
|
||||||
`LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE`
|
`LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE`
|
||||||
: Require the worker guest to ping the LAN gateway and probe host after static
|
: Require the worker guest to ping the LAN gateway and probe host after static
|
||||||
IP assignment. Defaults to `true`.
|
IP assignment. Defaults to `true`.
|
||||||
|
|
|
||||||
138
jeannie
138
jeannie
|
|
@ -876,14 +876,14 @@ run_step() {
|
||||||
|
|
||||||
if jeannie_verbose_enabled; then
|
if jeannie_verbose_enabled; then
|
||||||
set +e
|
set +e
|
||||||
"$@" 2>&1 | redact_sensitive_output | tee "${step_log}" | tee -a "${JEANNIE_LOG_FILE}"
|
"$@" 2>&1 | redact_sensitive_output | tee "${step_log}"
|
||||||
status=${PIPESTATUS[0]}
|
status=${PIPESTATUS[0]}
|
||||||
set -e
|
set -e
|
||||||
else
|
else
|
||||||
set +e
|
set +e
|
||||||
(
|
(
|
||||||
set +e
|
set +e
|
||||||
"$@" 2>&1 | redact_sensitive_output | tee "${step_log}" >>"${JEANNIE_LOG_FILE}"
|
"$@" 2>&1 | redact_sensitive_output >"${step_log}"
|
||||||
printf '%s\n' "${PIPESTATUS[0]}" >"${status_file}"
|
printf '%s\n' "${PIPESTATUS[0]}" >"${status_file}"
|
||||||
) &
|
) &
|
||||||
progress_pid=$!
|
progress_pid=$!
|
||||||
|
|
@ -902,6 +902,7 @@ run_step() {
|
||||||
set -e
|
set -e
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
cat "${step_log}" >>"${JEANNIE_LOG_FILE}"
|
||||||
printf 'Finished: %s\n' "$(date -Is)" >>"${JEANNIE_LOG_FILE}"
|
printf 'Finished: %s\n' "$(date -Is)" >>"${JEANNIE_LOG_FILE}"
|
||||||
|
|
||||||
if ((status == 0)); then
|
if ((status == 0)); then
|
||||||
|
|
@ -1212,27 +1213,6 @@ pimox_worker_static_ip() {
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
pimox_worker_net0_config() {
|
|
||||||
local vmid="$1"
|
|
||||||
local bridge="$2"
|
|
||||||
local mac_mode="${LAB_PIMOX_WORKER_MAC_MODE:-auto}"
|
|
||||||
local mac
|
|
||||||
|
|
||||||
case "${mac_mode}" in
|
|
||||||
auto)
|
|
||||||
printf 'virtio,bridge=%s\n' "${bridge}"
|
|
||||||
;;
|
|
||||||
deterministic)
|
|
||||||
mac="$(pimox_generated_mac "${vmid}")"
|
|
||||||
printf 'virtio=%s,bridge=%s\n' "${mac}" "${bridge}"
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
echo "LAB_PIMOX_WORKER_MAC_MODE must be 'auto' or 'deterministic'." >&2
|
|
||||||
return 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
}
|
|
||||||
|
|
||||||
validate_ipv4_cidr_or_host() {
|
validate_ipv4_cidr_or_host() {
|
||||||
local value="$1"
|
local value="$1"
|
||||||
|
|
||||||
|
|
@ -1455,7 +1435,7 @@ ensure_pimox_worker_node() {
|
||||||
local worker_key
|
local worker_key
|
||||||
local worker_name
|
local worker_name
|
||||||
local node_name
|
local node_name
|
||||||
local net0_config
|
local mac
|
||||||
local guest_ip
|
local guest_ip
|
||||||
local static_ip
|
local static_ip
|
||||||
|
|
||||||
|
|
@ -1464,9 +1444,7 @@ ensure_pimox_worker_node() {
|
||||||
worker_key="${worker_key_prefix}${padded}"
|
worker_key="${worker_key_prefix}${padded}"
|
||||||
worker_name="${worker_name_prefix}-${padded}"
|
worker_name="${worker_name_prefix}-${padded}"
|
||||||
node_name="${worker_node_prefix}-${padded}"
|
node_name="${worker_node_prefix}-${padded}"
|
||||||
if ! net0_config="$(pimox_worker_net0_config "${vmid}" "${bridge}")"; then
|
mac="$(pimox_generated_mac "${vmid}")"
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1"; then
|
if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1"; then
|
||||||
if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${vmid}' | grep -q '^template: 1$'"; then
|
if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${vmid}' | grep -q '^template: 1$'"; then
|
||||||
|
|
@ -1545,7 +1523,7 @@ if [ -n '${worker_cpu_affinity}' ]; then
|
||||||
esac
|
esac
|
||||||
}
|
}
|
||||||
fi
|
fi
|
||||||
sudo '${qm_bin}' set '${vmid}' --net0 '${net0_config}'
|
sudo '${qm_bin}' set '${vmid}' --net0 'virtio=${mac},bridge=${bridge}'
|
||||||
sudo '${qm_bin}' set '${vmid}' --boot 'order=scsi0;net0'
|
sudo '${qm_bin}' set '${vmid}' --boot 'order=scsi0;net0'
|
||||||
sudo '${qm_bin}' set '${vmid}' --onboot 1
|
sudo '${qm_bin}' set '${vmid}' --onboot 1
|
||||||
sudo '${qm_bin}' start '${vmid}'"
|
sudo '${qm_bin}' start '${vmid}'"
|
||||||
|
|
@ -4569,111 +4547,22 @@ ensure_cluster_worker_var_file() {
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
run_up_parallel_stage() {
|
|
||||||
local spec_file
|
|
||||||
local parallelism="${JEANNIE_UP_PARALLELISM:-4}"
|
|
||||||
local status
|
|
||||||
|
|
||||||
if ! [[ "${parallelism}" =~ ^[0-9]+$ ]] || ((parallelism < 1)); then
|
|
||||||
echo "JEANNIE_UP_PARALLELISM must be a positive integer." >&2
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
spec_file="$(mktemp)"
|
|
||||||
python3 - "${spec_file}" "${REPO_ROOT}/jeannie" "${parallelism}" <<'PY'
|
|
||||||
import json
|
|
||||||
import sys
|
|
||||||
|
|
||||||
spec_file, jeannie, parallelism = sys.argv[1:4]
|
|
||||||
spec = {
|
|
||||||
"parallelism": int(parallelism),
|
|
||||||
"tasks": [
|
|
||||||
{
|
|
||||||
"id": "gitea-deploy",
|
|
||||||
"title": "Gitea deploy",
|
|
||||||
"command": [jeannie, "__up-task", "gitea-deploy"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "gitea-bootstrap",
|
|
||||||
"title": "Gitea repo bootstrap",
|
|
||||||
"needs": ["gitea-deploy"],
|
|
||||||
"command": [jeannie, "__up-task", "gitea-bootstrap"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "rpi-services",
|
|
||||||
"title": "RPi services",
|
|
||||||
"command": [jeannie, "__up-task", "rpi-services"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "pimox-workers",
|
|
||||||
"title": "Pimox provisioning and workers",
|
|
||||||
"command": [jeannie, "__up-task", "pimox-workers"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "openwrt",
|
|
||||||
"title": "OpenWrt VM",
|
|
||||||
"command": [jeannie, "__up-task", "openwrt"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "edge-host",
|
|
||||||
"title": "OCI edge host check",
|
|
||||||
"command": [jeannie, "__up-task", "edge-host"],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
}
|
|
||||||
with open(spec_file, "w", encoding="utf-8") as handle:
|
|
||||||
json.dump(spec, handle, indent=2)
|
|
||||||
handle.write("\n")
|
|
||||||
PY
|
|
||||||
set +e
|
|
||||||
"${REPO_ROOT}/scripts/jeannie-core" run-dag --spec "${spec_file}"
|
|
||||||
status=$?
|
|
||||||
set -e
|
|
||||||
rm -f "${spec_file}"
|
|
||||||
return "${status}"
|
|
||||||
}
|
|
||||||
|
|
||||||
up_task() {
|
|
||||||
require_debian_server "__up-task"
|
|
||||||
|
|
||||||
case "${1:-}" in
|
|
||||||
gitea-deploy)
|
|
||||||
deploy_gitea
|
|
||||||
;;
|
|
||||||
gitea-bootstrap)
|
|
||||||
bootstrap_gitea_repo
|
|
||||||
;;
|
|
||||||
rpi-services)
|
|
||||||
deploy_rpi_services
|
|
||||||
;;
|
|
||||||
pimox-workers)
|
|
||||||
run_pimox_pipeline
|
|
||||||
;;
|
|
||||||
openwrt)
|
|
||||||
run_openwrt_pipeline
|
|
||||||
;;
|
|
||||||
edge-host)
|
|
||||||
check_edge_ssh
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
echo "Unknown __up-task '${1:-}'." >&2
|
|
||||||
return 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
}
|
|
||||||
|
|
||||||
up() {
|
up() {
|
||||||
require_debian_server "up"
|
require_debian_server "up"
|
||||||
|
|
||||||
echo "Deploying the homelab infrastructure..."
|
echo "Deploying the homelab infrastructure..."
|
||||||
jeannie_log_start "up"
|
jeannie_log_start "up"
|
||||||
jeannie_step_plan 12
|
jeannie_step_plan 16
|
||||||
|
|
||||||
run_step "Early preflight" homelab_preflight early
|
run_step "Early preflight" homelab_preflight early
|
||||||
run_step "Go toolchain" ensure_go_toolchain
|
run_step "Go toolchain" ensure_go_toolchain
|
||||||
run_step "Independent host services" run_up_parallel_stage
|
run_step "Gitea deploy" deploy_gitea
|
||||||
|
run_step "Gitea repo bootstrap" bootstrap_gitea_repo
|
||||||
|
run_step "RPi services" deploy_rpi_services
|
||||||
run_step "Full preflight" homelab_preflight full
|
run_step "Full preflight" homelab_preflight full
|
||||||
run_step "Pre-apply doctor" doctor_preapply
|
run_step "Pre-apply doctor" doctor_preapply
|
||||||
|
run_step "Pimox provisioning and workers" run_pimox_pipeline
|
||||||
|
run_step "OpenWrt VM" run_openwrt_pipeline
|
||||||
run_step "Worker var file" ensure_cluster_worker_var_file
|
run_step "Worker var file" ensure_cluster_worker_var_file
|
||||||
run_step "Start existing cluster if stopped" ensure_existing_cluster_started_for_up
|
run_step "Start existing cluster if stopped" ensure_existing_cluster_started_for_up
|
||||||
run_step "Cluster OpenTofu apply" run_tofu_stack "bootstrap/cluster"
|
run_step "Cluster OpenTofu apply" run_tofu_stack "bootstrap/cluster"
|
||||||
|
|
@ -7128,9 +7017,6 @@ if [[ "${JEANNIE_LIBRARY_MODE:-false}" == "true" ]]; then
|
||||||
fi
|
fi
|
||||||
|
|
||||||
case "${1:-}" in
|
case "${1:-}" in
|
||||||
__up-task)
|
|
||||||
up_task "${2:-}"
|
|
||||||
;;
|
|
||||||
"" | help | -h | --help)
|
"" | help | -h | --help)
|
||||||
print_usage
|
print_usage
|
||||||
;;
|
;;
|
||||||
|
|
|
||||||
|
|
@ -1,27 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
||||||
CORE_BIN="${JEANNIE_CORE_BIN:-${REPO_ROOT}/.lab/bin/jeannie-core}"
|
|
||||||
|
|
||||||
core_sources_newer() {
|
|
||||||
local source
|
|
||||||
|
|
||||||
while IFS= read -r source; do
|
|
||||||
if [[ "${source}" -nt "${CORE_BIN}" ]]; then
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
done < <(find "${REPO_ROOT}/cmd/jeannie-core" -name '*.go' -type f)
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
if [[ ! -x "${CORE_BIN}" || "${REPO_ROOT}/go.mod" -nt "${CORE_BIN}" ]] || core_sources_newer; then
|
|
||||||
if ! command -v go >/dev/null 2>&1; then
|
|
||||||
echo "go is required to build ${CORE_BIN}; run ./jeannie up or install golang-go." >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
mkdir -p "$(dirname "${CORE_BIN}")"
|
|
||||||
GOCACHE="${GOCACHE:-${REPO_ROOT}/.lab/go-build-cache}" go build -o "${CORE_BIN}" "${REPO_ROOT}/cmd/jeannie-core"
|
|
||||||
fi
|
|
||||||
|
|
||||||
exec "${CORE_BIN}" "$@"
|
|
||||||
|
|
@ -2,6 +2,19 @@
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
CORE_BIN="${JEANNIE_CORE_BIN:-${REPO_ROOT}/.lab/bin/jeannie-core}"
|
||||||
|
|
||||||
|
core_sources_newer() {
|
||||||
|
local source
|
||||||
|
|
||||||
|
while IFS= read -r source; do
|
||||||
|
if [[ "${source}" -nt "${CORE_BIN}" ]]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
done < <(find "${REPO_ROOT}/cmd/jeannie-core" -name '*.go' -type f)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
usage() {
|
usage() {
|
||||||
cat <<'EOF'
|
cat <<'EOF'
|
||||||
Usage: report-render [--title TITLE] [--details] [--only failures|warnings|problems|all] [--json]
|
Usage: report-render [--title TITLE] [--details] [--only failures|warnings|problems|all] [--json]
|
||||||
|
|
@ -22,4 +35,13 @@ if (($# > 0)); then
|
||||||
esac
|
esac
|
||||||
fi
|
fi
|
||||||
|
|
||||||
exec "${REPO_ROOT}/scripts/jeannie-core" report-render "$@"
|
if [[ ! -x "${CORE_BIN}" || "${REPO_ROOT}/go.mod" -nt "${CORE_BIN}" ]] || core_sources_newer; then
|
||||||
|
if ! command -v go >/dev/null 2>&1; then
|
||||||
|
echo "go is required to build ${CORE_BIN}; run ./jeannie up or install golang-go." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
mkdir -p "$(dirname "${CORE_BIN}")"
|
||||||
|
GOCACHE="${GOCACHE:-${REPO_ROOT}/.lab/go-build-cache}" go build -o "${CORE_BIN}" "${REPO_ROOT}/cmd/jeannie-core"
|
||||||
|
fi
|
||||||
|
|
||||||
|
exec "${CORE_BIN}" report-render "$@"
|
||||||
|
|
|
||||||
|
|
@ -336,14 +336,6 @@ test_pimox_guest_exec_exitcode_parser() {
|
||||||
assert_command_failure_contains "pimox guest exec rejects invalid JSON" "" pimox_guest_exec_exitcode 'not json'
|
assert_command_failure_contains "pimox guest exec rejects invalid JSON" "" pimox_guest_exec_exitcode 'not json'
|
||||||
}
|
}
|
||||||
|
|
||||||
test_pimox_worker_net0_config_defaults_to_auto_mac() {
|
|
||||||
unset LAB_PIMOX_WORKER_MAC_MODE
|
|
||||||
assert_eq "pimox worker net0 defaults to auto MAC" "virtio,bridge=vmbr0" "$(pimox_worker_net0_config 9010 vmbr0)"
|
|
||||||
LAB_PIMOX_WORKER_MAC_MODE=deterministic
|
|
||||||
assert_eq "pimox worker net0 deterministic MAC opt-in" "virtio=02:68:10:00:23:32,bridge=vmbr0" "$(pimox_worker_net0_config 9010 vmbr0)"
|
|
||||||
unset LAB_PIMOX_WORKER_MAC_MODE
|
|
||||||
}
|
|
||||||
|
|
||||||
test_pimox_worker_count_detects_generated_topology
|
test_pimox_worker_count_detects_generated_topology
|
||||||
test_configured_worker_count_cannot_hide_existing_workers
|
test_configured_worker_count_cannot_hide_existing_workers
|
||||||
test_configured_worker_count_can_expand_target_set
|
test_configured_worker_count_can_expand_target_set
|
||||||
|
|
@ -358,7 +350,6 @@ test_inventory_validator_rejects_ip_drift
|
||||||
test_heal_cooldown_escalates_to_diagnostics
|
test_heal_cooldown_escalates_to_diagnostics
|
||||||
test_pimox_worker_static_ip_selection
|
test_pimox_worker_static_ip_selection
|
||||||
test_pimox_guest_exec_exitcode_parser
|
test_pimox_guest_exec_exitcode_parser
|
||||||
test_pimox_worker_net0_config_defaults_to_auto_mac
|
|
||||||
|
|
||||||
if ((FAILURES > 0)); then
|
if ((FAILURES > 0)); then
|
||||||
printf '\n%s unit test(s) failed.\n' "${FAILURES}" >&2
|
printf '\n%s unit test(s) failed.\n' "${FAILURES}" >&2
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue