Compare commits
3 Commits
1bdde9c128
...
0cda6573ad
| Author | SHA1 | Date |
|---|---|---|
|
|
0cda6573ad | |
|
|
ed0983d09a | |
|
|
ea4576c707 |
|
|
@ -734,6 +734,9 @@ qemu-guest-agent. Static mode uses
|
||||||
`LAB_PIMOX_WORKER_STATIC_IPS="192.168.100.66 192.168.100.67 192.168.100.76"`,
|
`LAB_PIMOX_WORKER_STATIC_IPS="192.168.100.66 192.168.100.67 192.168.100.76"`,
|
||||||
`LAB_PIMOX_WORKER_GATEWAY=192.168.100.1`, and
|
`LAB_PIMOX_WORKER_GATEWAY=192.168.100.1`, and
|
||||||
`LAB_PIMOX_WORKER_DNS_SERVERS="192.168.100.89 1.1.1.1"`.
|
`LAB_PIMOX_WORKER_DNS_SERVERS="192.168.100.89 1.1.1.1"`.
|
||||||
|
Worker NIC MAC addresses are also auto-generated by Pimox by default. Set
|
||||||
|
`LAB_PIMOX_WORKER_MAC_MODE=deterministic` only when you explicitly need stable
|
||||||
|
MAC addresses derived from VM IDs.
|
||||||
Worker creation fails faster than template creation: static guest-network
|
Worker creation fails faster than template creation: static guest-network
|
||||||
configuration waits up to `LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`,
|
configuration waits up to `LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`,
|
||||||
worker IP/SSH readiness waits up to `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`,
|
worker IP/SSH readiness waits up to `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`,
|
||||||
|
|
@ -750,6 +753,11 @@ Long-running `./jeannie up` steps keep compact output by default, but
|
||||||
the active step line refreshes with elapsed time and the latest log line. Set
|
the active step line refreshes with elapsed time and the latest log line. Set
|
||||||
`JEANNIE_PROGRESS_INTERVAL_SECONDS=5` for faster heartbeats, or
|
`JEANNIE_PROGRESS_INTERVAL_SECONDS=5` for faster heartbeats, or
|
||||||
`JEANNIE_VERBOSE=true` to stream full command output.
|
`JEANNIE_VERBOSE=true` to stream full command output.
|
||||||
|
Independent host services run through the Go DAG runner during `up`: Gitea
|
||||||
|
deploy/bootstrap, RPi services, Pimox workers, OpenWrt, and OCI edge host checks
|
||||||
|
can progress concurrently. Set `JEANNIE_UP_PARALLELISM=1` to force sequential
|
||||||
|
execution while troubleshooting, or raise it above the default `4` if the lab
|
||||||
|
can absorb more concurrent work.
|
||||||
|
|
||||||
Run a full cluster rebuild from the Debian server with:
|
Run a full cluster rebuild from the Debian server with:
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -734,6 +734,9 @@ qemu-guest-agent. Static mode uses
|
||||||
`LAB_PIMOX_WORKER_STATIC_IPS="192.168.100.66 192.168.100.67 192.168.100.76"`,
|
`LAB_PIMOX_WORKER_STATIC_IPS="192.168.100.66 192.168.100.67 192.168.100.76"`,
|
||||||
`LAB_PIMOX_WORKER_GATEWAY=192.168.100.1`, and
|
`LAB_PIMOX_WORKER_GATEWAY=192.168.100.1`, and
|
||||||
`LAB_PIMOX_WORKER_DNS_SERVERS="192.168.100.89 1.1.1.1"`.
|
`LAB_PIMOX_WORKER_DNS_SERVERS="192.168.100.89 1.1.1.1"`.
|
||||||
|
Worker NIC MAC addresses are also auto-generated by Pimox by default. Set
|
||||||
|
`LAB_PIMOX_WORKER_MAC_MODE=deterministic` only when you explicitly need stable
|
||||||
|
MAC addresses derived from VM IDs.
|
||||||
Worker creation fails faster than template creation: static guest-network
|
Worker creation fails faster than template creation: static guest-network
|
||||||
configuration waits up to `LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`,
|
configuration waits up to `LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`,
|
||||||
worker IP/SSH readiness waits up to `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`,
|
worker IP/SSH readiness waits up to `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`,
|
||||||
|
|
@ -750,6 +753,11 @@ Long-running `./{{ main_script }} up` steps keep compact output by default, but
|
||||||
the active step line refreshes with elapsed time and the latest log line. Set
|
the active step line refreshes with elapsed time and the latest log line. Set
|
||||||
`JEANNIE_PROGRESS_INTERVAL_SECONDS=5` for faster heartbeats, or
|
`JEANNIE_PROGRESS_INTERVAL_SECONDS=5` for faster heartbeats, or
|
||||||
`JEANNIE_VERBOSE=true` to stream full command output.
|
`JEANNIE_VERBOSE=true` to stream full command output.
|
||||||
|
Independent host services run through the Go DAG runner during `up`: Gitea
|
||||||
|
deploy/bootstrap, RPi services, Pimox workers, OpenWrt, and OCI edge host checks
|
||||||
|
can progress concurrently. Set `JEANNIE_UP_PARALLELISM=1` to force sequential
|
||||||
|
execution while troubleshooting, or raise it above the default `4` if the lab
|
||||||
|
can absorb more concurrent work.
|
||||||
|
|
||||||
Run a full cluster rebuild from the Debian server with:
|
Run a full cluster rebuild from the Debian server with:
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -148,6 +148,9 @@ reservations are not available: set `LAB_PIMOX_WORKER_NETWORK_MODE=static`,
|
||||||
`LAB_PIMOX_WORKER_STATIC_IPS="192.168.100.66 192.168.100.67 192.168.100.76"`,
|
`LAB_PIMOX_WORKER_STATIC_IPS="192.168.100.66 192.168.100.67 192.168.100.76"`,
|
||||||
`LAB_PIMOX_WORKER_GATEWAY=192.168.100.1`, `LAB_PIMOX_WORKER_INTERFACE=enp0s18`,
|
`LAB_PIMOX_WORKER_GATEWAY=192.168.100.1`, `LAB_PIMOX_WORKER_INTERFACE=enp0s18`,
|
||||||
and `LAB_PIMOX_WORKER_DNS_SERVERS="192.168.100.89 1.1.1.1"`.
|
and `LAB_PIMOX_WORKER_DNS_SERVERS="192.168.100.89 1.1.1.1"`.
|
||||||
|
Worker NIC MAC addresses are auto-generated by Pimox by default. Set
|
||||||
|
`LAB_PIMOX_WORKER_MAC_MODE=deterministic` only when VMID-derived MAC addresses
|
||||||
|
are required.
|
||||||
Worker clone failures are intentionally faster than template build failures:
|
Worker clone failures are intentionally faster than template build failures:
|
||||||
`LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`,
|
`LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`,
|
||||||
`LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`, and
|
`LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`, and
|
||||||
|
|
|
||||||
|
|
@ -1,13 +1,18 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bufio"
|
||||||
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"os/exec"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
type reportRow struct {
|
type reportRow struct {
|
||||||
|
|
@ -33,6 +38,11 @@ func main() {
|
||||||
fmt.Fprintln(os.Stderr, err)
|
fmt.Fprintln(os.Stderr, err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
case "run-dag":
|
||||||
|
if err := runDAG(os.Args[2:], os.Stdout, os.Stderr); err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
case "help", "-h", "--help":
|
case "help", "-h", "--help":
|
||||||
usage(os.Stdout)
|
usage(os.Stdout)
|
||||||
default:
|
default:
|
||||||
|
|
@ -42,7 +52,184 @@ func main() {
|
||||||
}
|
}
|
||||||
|
|
||||||
func usage(out io.Writer) {
|
func usage(out io.Writer) {
|
||||||
fmt.Fprintln(out, "Usage: jeannie-core report-render [--title TITLE] [--details] [--only failures|warnings|problems|all] [--json]")
|
fmt.Fprintln(out, "Usage: jeannie-core {report-render|run-dag}")
|
||||||
|
}
|
||||||
|
|
||||||
|
type dagSpec struct {
|
||||||
|
Tasks []dagTask `json:"tasks"`
|
||||||
|
Parallelism int `json:"parallelism"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type dagTask struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Title string `json:"title"`
|
||||||
|
Command []string `json:"command"`
|
||||||
|
Needs []string `json:"needs"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type dagTaskResult struct {
|
||||||
|
id string
|
||||||
|
exitCode int
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func runDAG(args []string, out io.Writer, errOut io.Writer) error {
|
||||||
|
flags := flag.NewFlagSet("run-dag", flag.ContinueOnError)
|
||||||
|
flags.SetOutput(io.Discard)
|
||||||
|
specPath := flags.String("spec", "", "")
|
||||||
|
if err := flags.Parse(args); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *specPath == "" {
|
||||||
|
return fmt.Errorf("run-dag requires --spec")
|
||||||
|
}
|
||||||
|
|
||||||
|
specData, err := os.ReadFile(*specPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var spec dagSpec
|
||||||
|
if err := json.Unmarshal(specData, &spec); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return executeDAG(context.Background(), spec, out, errOut)
|
||||||
|
}
|
||||||
|
|
||||||
|
func executeDAG(ctx context.Context, spec dagSpec, out io.Writer, errOut io.Writer) error {
|
||||||
|
if spec.Parallelism <= 0 {
|
||||||
|
spec.Parallelism = 4
|
||||||
|
}
|
||||||
|
|
||||||
|
taskByID := map[string]dagTask{}
|
||||||
|
dependents := map[string][]string{}
|
||||||
|
remainingNeeds := map[string]int{}
|
||||||
|
for _, task := range spec.Tasks {
|
||||||
|
if task.ID == "" {
|
||||||
|
return fmt.Errorf("dag task has empty id")
|
||||||
|
}
|
||||||
|
if _, exists := taskByID[task.ID]; exists {
|
||||||
|
return fmt.Errorf("duplicate dag task id %q", task.ID)
|
||||||
|
}
|
||||||
|
if len(task.Command) == 0 {
|
||||||
|
return fmt.Errorf("dag task %q has empty command", task.ID)
|
||||||
|
}
|
||||||
|
taskByID[task.ID] = task
|
||||||
|
remainingNeeds[task.ID] = len(task.Needs)
|
||||||
|
for _, need := range task.Needs {
|
||||||
|
dependents[need] = append(dependents[need], task.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, task := range spec.Tasks {
|
||||||
|
for _, need := range task.Needs {
|
||||||
|
if _, exists := taskByID[need]; !exists {
|
||||||
|
return fmt.Errorf("dag task %q depends on unknown task %q", task.ID, need)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(ctx)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
ready := make(chan string, len(spec.Tasks))
|
||||||
|
results := make(chan dagTaskResult, len(spec.Tasks))
|
||||||
|
sem := make(chan struct{}, spec.Parallelism)
|
||||||
|
var writersMu sync.Mutex
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
started := map[string]bool{}
|
||||||
|
completed := map[string]bool{}
|
||||||
|
|
||||||
|
for id, count := range remainingNeeds {
|
||||||
|
if count == 0 {
|
||||||
|
ready <- id
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
failed := false
|
||||||
|
for len(completed) < len(spec.Tasks) {
|
||||||
|
select {
|
||||||
|
case id := <-ready:
|
||||||
|
if started[id] || failed {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
started[id] = true
|
||||||
|
task := taskByID[id]
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
sem <- struct{}{}
|
||||||
|
defer func() { <-sem }()
|
||||||
|
results <- runDAGTask(ctx, task, out, errOut, &writersMu)
|
||||||
|
}()
|
||||||
|
case result := <-results:
|
||||||
|
completed[result.id] = true
|
||||||
|
if result.err != nil || result.exitCode != 0 {
|
||||||
|
failed = true
|
||||||
|
cancel()
|
||||||
|
wg.Wait()
|
||||||
|
if result.err != nil {
|
||||||
|
return result.err
|
||||||
|
}
|
||||||
|
return fmt.Errorf("dag task %q failed with exit code %d", result.id, result.exitCode)
|
||||||
|
}
|
||||||
|
for _, dependent := range dependents[result.id] {
|
||||||
|
remainingNeeds[dependent]--
|
||||||
|
if remainingNeeds[dependent] == 0 {
|
||||||
|
ready <- dependent
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func runDAGTask(ctx context.Context, task dagTask, out io.Writer, errOut io.Writer, writersMu *sync.Mutex) dagTaskResult {
|
||||||
|
title := task.Title
|
||||||
|
if title == "" {
|
||||||
|
title = task.ID
|
||||||
|
}
|
||||||
|
started := time.Now()
|
||||||
|
writeDAGLine(out, writersMu, task.ID, "started "+title)
|
||||||
|
|
||||||
|
cmd := exec.CommandContext(ctx, task.Command[0], task.Command[1:]...)
|
||||||
|
cmd.Env = os.Environ()
|
||||||
|
stdout, err := cmd.StdoutPipe()
|
||||||
|
if err != nil {
|
||||||
|
return dagTaskResult{id: task.ID, exitCode: 1, err: err}
|
||||||
|
}
|
||||||
|
cmd.Stderr = cmd.Stdout
|
||||||
|
if err := cmd.Start(); err != nil {
|
||||||
|
return dagTaskResult{id: task.ID, exitCode: 1, err: err}
|
||||||
|
}
|
||||||
|
|
||||||
|
scanner := bufio.NewScanner(stdout)
|
||||||
|
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||||
|
for scanner.Scan() {
|
||||||
|
writeDAGLine(out, writersMu, task.ID, scanner.Text())
|
||||||
|
}
|
||||||
|
if scanErr := scanner.Err(); scanErr != nil {
|
||||||
|
writeDAGLine(errOut, writersMu, task.ID, "output read error: "+scanErr.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
err = cmd.Wait()
|
||||||
|
elapsed := time.Since(started).Round(time.Second)
|
||||||
|
exitCode := 0
|
||||||
|
if err != nil {
|
||||||
|
exitCode = 1
|
||||||
|
if exitErr, ok := err.(*exec.ExitError); ok {
|
||||||
|
exitCode = exitErr.ExitCode()
|
||||||
|
}
|
||||||
|
writeDAGLine(out, writersMu, task.ID, fmt.Sprintf("failed after %s", elapsed))
|
||||||
|
return dagTaskResult{id: task.ID, exitCode: exitCode, err: nil}
|
||||||
|
}
|
||||||
|
writeDAGLine(out, writersMu, task.ID, fmt.Sprintf("completed in %s", elapsed))
|
||||||
|
return dagTaskResult{id: task.ID, exitCode: 0, err: nil}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeDAGLine(out io.Writer, writersMu *sync.Mutex, id string, line string) {
|
||||||
|
writersMu.Lock()
|
||||||
|
defer writersMu.Unlock()
|
||||||
|
fmt.Fprintf(out, "[%s] %s\n", id, line)
|
||||||
}
|
}
|
||||||
|
|
||||||
func runReportRender(args []string, in io.Reader, out io.Writer) error {
|
func runReportRender(args []string, in io.Reader, out io.Writer) error {
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,9 @@ package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
@ -42,3 +45,46 @@ func TestReportRenderHidesOKGraphAndQuery(t *testing.T) {
|
||||||
t.Fatalf("ok row leaked graph/query links:\n%s", got)
|
t.Fatalf("ok row leaked graph/query links:\n%s", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteDAGRespectsDependencies(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
first := filepath.Join(dir, "first")
|
||||||
|
second := filepath.Join(dir, "second")
|
||||||
|
spec := dagSpec{
|
||||||
|
Parallelism: 2,
|
||||||
|
Tasks: []dagTask{
|
||||||
|
{
|
||||||
|
ID: "first",
|
||||||
|
Command: []string{"sh", "-c", "printf first > \"$1\"", "sh", first},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "second",
|
||||||
|
Needs: []string{"first"},
|
||||||
|
Command: []string{"sh", "-c", "test -f \"$1\" && printf second > \"$2\"", "sh", first, second},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
var output bytes.Buffer
|
||||||
|
if err := executeDAG(context.Background(), spec, &output, &output); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(second); err != nil {
|
||||||
|
t.Fatalf("dependent task did not run after dependency: %v\n%s", err, output.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteDAGReturnsFailedTask(t *testing.T) {
|
||||||
|
spec := dagSpec{
|
||||||
|
Parallelism: 2,
|
||||||
|
Tasks: []dagTask{
|
||||||
|
{ID: "bad", Command: []string{"sh", "-c", "exit 7"}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
var output bytes.Buffer
|
||||||
|
err := executeDAG(context.Background(), spec, &output, &output)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), `dag task "bad" failed`) {
|
||||||
|
t.Fatalf("expected failed task error, got %v\n%s", err, output.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -535,6 +535,10 @@ falls back to the first Lynis warning, then the first suggestion.
|
||||||
`LAB_AUTO_APPROVE=false`
|
`LAB_AUTO_APPROVE=false`
|
||||||
: Disable automatic OpenTofu approval for apply paths that support confirmation.
|
: Disable automatic OpenTofu approval for apply paths that support confirmation.
|
||||||
|
|
||||||
|
`JEANNIE_UP_PARALLELISM`
|
||||||
|
: Maximum number of independent `up` branches the Go DAG runner may execute at
|
||||||
|
once. Defaults to `4`; set to `1` for sequential troubleshooting.
|
||||||
|
|
||||||
`LAB_PIMOX_WORKER_COUNT`
|
`LAB_PIMOX_WORKER_COUNT`
|
||||||
: Desired Pimox worker count when not supplied by generated cluster topology
|
: Desired Pimox worker count when not supplied by generated cluster topology
|
||||||
state.
|
state.
|
||||||
|
|
@ -543,6 +547,10 @@ state.
|
||||||
: Worker guest networking mode. Defaults to `dhcp`. Set to `static` only when
|
: Worker guest networking mode. Defaults to `dhcp`. Set to `static` only when
|
||||||
Jeannie should configure guest static IPs through qemu-guest-agent.
|
Jeannie should configure guest static IPs through qemu-guest-agent.
|
||||||
|
|
||||||
|
`LAB_PIMOX_WORKER_MAC_MODE`
|
||||||
|
: Worker NIC MAC mode. Defaults to `auto`, letting Pimox generate the MAC. Set
|
||||||
|
to `deterministic` to derive MAC addresses from VM IDs.
|
||||||
|
|
||||||
`LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE`
|
`LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE`
|
||||||
: Require the worker guest to ping the LAN gateway and probe host after static
|
: Require the worker guest to ping the LAN gateway and probe host after static
|
||||||
IP assignment. Defaults to `true`.
|
IP assignment. Defaults to `true`.
|
||||||
|
|
|
||||||
138
jeannie
138
jeannie
|
|
@ -876,14 +876,14 @@ run_step() {
|
||||||
|
|
||||||
if jeannie_verbose_enabled; then
|
if jeannie_verbose_enabled; then
|
||||||
set +e
|
set +e
|
||||||
"$@" 2>&1 | redact_sensitive_output | tee "${step_log}"
|
"$@" 2>&1 | redact_sensitive_output | tee "${step_log}" | tee -a "${JEANNIE_LOG_FILE}"
|
||||||
status=${PIPESTATUS[0]}
|
status=${PIPESTATUS[0]}
|
||||||
set -e
|
set -e
|
||||||
else
|
else
|
||||||
set +e
|
set +e
|
||||||
(
|
(
|
||||||
set +e
|
set +e
|
||||||
"$@" 2>&1 | redact_sensitive_output >"${step_log}"
|
"$@" 2>&1 | redact_sensitive_output | tee "${step_log}" >>"${JEANNIE_LOG_FILE}"
|
||||||
printf '%s\n' "${PIPESTATUS[0]}" >"${status_file}"
|
printf '%s\n' "${PIPESTATUS[0]}" >"${status_file}"
|
||||||
) &
|
) &
|
||||||
progress_pid=$!
|
progress_pid=$!
|
||||||
|
|
@ -902,7 +902,6 @@ run_step() {
|
||||||
set -e
|
set -e
|
||||||
fi
|
fi
|
||||||
|
|
||||||
cat "${step_log}" >>"${JEANNIE_LOG_FILE}"
|
|
||||||
printf 'Finished: %s\n' "$(date -Is)" >>"${JEANNIE_LOG_FILE}"
|
printf 'Finished: %s\n' "$(date -Is)" >>"${JEANNIE_LOG_FILE}"
|
||||||
|
|
||||||
if ((status == 0)); then
|
if ((status == 0)); then
|
||||||
|
|
@ -1213,6 +1212,27 @@ pimox_worker_static_ip() {
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pimox_worker_net0_config() {
|
||||||
|
local vmid="$1"
|
||||||
|
local bridge="$2"
|
||||||
|
local mac_mode="${LAB_PIMOX_WORKER_MAC_MODE:-auto}"
|
||||||
|
local mac
|
||||||
|
|
||||||
|
case "${mac_mode}" in
|
||||||
|
auto)
|
||||||
|
printf 'virtio,bridge=%s\n' "${bridge}"
|
||||||
|
;;
|
||||||
|
deterministic)
|
||||||
|
mac="$(pimox_generated_mac "${vmid}")"
|
||||||
|
printf 'virtio=%s,bridge=%s\n' "${mac}" "${bridge}"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "LAB_PIMOX_WORKER_MAC_MODE must be 'auto' or 'deterministic'." >&2
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
validate_ipv4_cidr_or_host() {
|
validate_ipv4_cidr_or_host() {
|
||||||
local value="$1"
|
local value="$1"
|
||||||
|
|
||||||
|
|
@ -1435,7 +1455,7 @@ ensure_pimox_worker_node() {
|
||||||
local worker_key
|
local worker_key
|
||||||
local worker_name
|
local worker_name
|
||||||
local node_name
|
local node_name
|
||||||
local mac
|
local net0_config
|
||||||
local guest_ip
|
local guest_ip
|
||||||
local static_ip
|
local static_ip
|
||||||
|
|
||||||
|
|
@ -1444,7 +1464,9 @@ ensure_pimox_worker_node() {
|
||||||
worker_key="${worker_key_prefix}${padded}"
|
worker_key="${worker_key_prefix}${padded}"
|
||||||
worker_name="${worker_name_prefix}-${padded}"
|
worker_name="${worker_name_prefix}-${padded}"
|
||||||
node_name="${worker_node_prefix}-${padded}"
|
node_name="${worker_node_prefix}-${padded}"
|
||||||
mac="$(pimox_generated_mac "${vmid}")"
|
if ! net0_config="$(pimox_worker_net0_config "${vmid}" "${bridge}")"; then
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1"; then
|
if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1"; then
|
||||||
if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${vmid}' | grep -q '^template: 1$'"; then
|
if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${vmid}' | grep -q '^template: 1$'"; then
|
||||||
|
|
@ -1523,7 +1545,7 @@ if [ -n '${worker_cpu_affinity}' ]; then
|
||||||
esac
|
esac
|
||||||
}
|
}
|
||||||
fi
|
fi
|
||||||
sudo '${qm_bin}' set '${vmid}' --net0 'virtio=${mac},bridge=${bridge}'
|
sudo '${qm_bin}' set '${vmid}' --net0 '${net0_config}'
|
||||||
sudo '${qm_bin}' set '${vmid}' --boot 'order=scsi0;net0'
|
sudo '${qm_bin}' set '${vmid}' --boot 'order=scsi0;net0'
|
||||||
sudo '${qm_bin}' set '${vmid}' --onboot 1
|
sudo '${qm_bin}' set '${vmid}' --onboot 1
|
||||||
sudo '${qm_bin}' start '${vmid}'"
|
sudo '${qm_bin}' start '${vmid}'"
|
||||||
|
|
@ -4547,22 +4569,111 @@ ensure_cluster_worker_var_file() {
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
run_up_parallel_stage() {
|
||||||
|
local spec_file
|
||||||
|
local parallelism="${JEANNIE_UP_PARALLELISM:-4}"
|
||||||
|
local status
|
||||||
|
|
||||||
|
if ! [[ "${parallelism}" =~ ^[0-9]+$ ]] || ((parallelism < 1)); then
|
||||||
|
echo "JEANNIE_UP_PARALLELISM must be a positive integer." >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
spec_file="$(mktemp)"
|
||||||
|
python3 - "${spec_file}" "${REPO_ROOT}/jeannie" "${parallelism}" <<'PY'
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
spec_file, jeannie, parallelism = sys.argv[1:4]
|
||||||
|
spec = {
|
||||||
|
"parallelism": int(parallelism),
|
||||||
|
"tasks": [
|
||||||
|
{
|
||||||
|
"id": "gitea-deploy",
|
||||||
|
"title": "Gitea deploy",
|
||||||
|
"command": [jeannie, "__up-task", "gitea-deploy"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "gitea-bootstrap",
|
||||||
|
"title": "Gitea repo bootstrap",
|
||||||
|
"needs": ["gitea-deploy"],
|
||||||
|
"command": [jeannie, "__up-task", "gitea-bootstrap"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "rpi-services",
|
||||||
|
"title": "RPi services",
|
||||||
|
"command": [jeannie, "__up-task", "rpi-services"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "pimox-workers",
|
||||||
|
"title": "Pimox provisioning and workers",
|
||||||
|
"command": [jeannie, "__up-task", "pimox-workers"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "openwrt",
|
||||||
|
"title": "OpenWrt VM",
|
||||||
|
"command": [jeannie, "__up-task", "openwrt"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "edge-host",
|
||||||
|
"title": "OCI edge host check",
|
||||||
|
"command": [jeannie, "__up-task", "edge-host"],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
with open(spec_file, "w", encoding="utf-8") as handle:
|
||||||
|
json.dump(spec, handle, indent=2)
|
||||||
|
handle.write("\n")
|
||||||
|
PY
|
||||||
|
set +e
|
||||||
|
"${REPO_ROOT}/scripts/jeannie-core" run-dag --spec "${spec_file}"
|
||||||
|
status=$?
|
||||||
|
set -e
|
||||||
|
rm -f "${spec_file}"
|
||||||
|
return "${status}"
|
||||||
|
}
|
||||||
|
|
||||||
|
up_task() {
|
||||||
|
require_debian_server "__up-task"
|
||||||
|
|
||||||
|
case "${1:-}" in
|
||||||
|
gitea-deploy)
|
||||||
|
deploy_gitea
|
||||||
|
;;
|
||||||
|
gitea-bootstrap)
|
||||||
|
bootstrap_gitea_repo
|
||||||
|
;;
|
||||||
|
rpi-services)
|
||||||
|
deploy_rpi_services
|
||||||
|
;;
|
||||||
|
pimox-workers)
|
||||||
|
run_pimox_pipeline
|
||||||
|
;;
|
||||||
|
openwrt)
|
||||||
|
run_openwrt_pipeline
|
||||||
|
;;
|
||||||
|
edge-host)
|
||||||
|
check_edge_ssh
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Unknown __up-task '${1:-}'." >&2
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
up() {
|
up() {
|
||||||
require_debian_server "up"
|
require_debian_server "up"
|
||||||
|
|
||||||
echo "Deploying the homelab infrastructure..."
|
echo "Deploying the homelab infrastructure..."
|
||||||
jeannie_log_start "up"
|
jeannie_log_start "up"
|
||||||
jeannie_step_plan 16
|
jeannie_step_plan 12
|
||||||
|
|
||||||
run_step "Early preflight" homelab_preflight early
|
run_step "Early preflight" homelab_preflight early
|
||||||
run_step "Go toolchain" ensure_go_toolchain
|
run_step "Go toolchain" ensure_go_toolchain
|
||||||
run_step "Gitea deploy" deploy_gitea
|
run_step "Independent host services" run_up_parallel_stage
|
||||||
run_step "Gitea repo bootstrap" bootstrap_gitea_repo
|
|
||||||
run_step "RPi services" deploy_rpi_services
|
|
||||||
run_step "Full preflight" homelab_preflight full
|
run_step "Full preflight" homelab_preflight full
|
||||||
run_step "Pre-apply doctor" doctor_preapply
|
run_step "Pre-apply doctor" doctor_preapply
|
||||||
run_step "Pimox provisioning and workers" run_pimox_pipeline
|
|
||||||
run_step "OpenWrt VM" run_openwrt_pipeline
|
|
||||||
run_step "Worker var file" ensure_cluster_worker_var_file
|
run_step "Worker var file" ensure_cluster_worker_var_file
|
||||||
run_step "Start existing cluster if stopped" ensure_existing_cluster_started_for_up
|
run_step "Start existing cluster if stopped" ensure_existing_cluster_started_for_up
|
||||||
run_step "Cluster OpenTofu apply" run_tofu_stack "bootstrap/cluster"
|
run_step "Cluster OpenTofu apply" run_tofu_stack "bootstrap/cluster"
|
||||||
|
|
@ -7017,6 +7128,9 @@ if [[ "${JEANNIE_LIBRARY_MODE:-false}" == "true" ]]; then
|
||||||
fi
|
fi
|
||||||
|
|
||||||
case "${1:-}" in
|
case "${1:-}" in
|
||||||
|
__up-task)
|
||||||
|
up_task "${2:-}"
|
||||||
|
;;
|
||||||
"" | help | -h | --help)
|
"" | help | -h | --help)
|
||||||
print_usage
|
print_usage
|
||||||
;;
|
;;
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,27 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
CORE_BIN="${JEANNIE_CORE_BIN:-${REPO_ROOT}/.lab/bin/jeannie-core}"
|
||||||
|
|
||||||
|
core_sources_newer() {
|
||||||
|
local source
|
||||||
|
|
||||||
|
while IFS= read -r source; do
|
||||||
|
if [[ "${source}" -nt "${CORE_BIN}" ]]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
done < <(find "${REPO_ROOT}/cmd/jeannie-core" -name '*.go' -type f)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
if [[ ! -x "${CORE_BIN}" || "${REPO_ROOT}/go.mod" -nt "${CORE_BIN}" ]] || core_sources_newer; then
|
||||||
|
if ! command -v go >/dev/null 2>&1; then
|
||||||
|
echo "go is required to build ${CORE_BIN}; run ./jeannie up or install golang-go." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
mkdir -p "$(dirname "${CORE_BIN}")"
|
||||||
|
GOCACHE="${GOCACHE:-${REPO_ROOT}/.lab/go-build-cache}" go build -o "${CORE_BIN}" "${REPO_ROOT}/cmd/jeannie-core"
|
||||||
|
fi
|
||||||
|
|
||||||
|
exec "${CORE_BIN}" "$@"
|
||||||
|
|
@ -2,19 +2,6 @@
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
CORE_BIN="${JEANNIE_CORE_BIN:-${REPO_ROOT}/.lab/bin/jeannie-core}"
|
|
||||||
|
|
||||||
core_sources_newer() {
|
|
||||||
local source
|
|
||||||
|
|
||||||
while IFS= read -r source; do
|
|
||||||
if [[ "${source}" -nt "${CORE_BIN}" ]]; then
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
done < <(find "${REPO_ROOT}/cmd/jeannie-core" -name '*.go' -type f)
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
usage() {
|
usage() {
|
||||||
cat <<'EOF'
|
cat <<'EOF'
|
||||||
Usage: report-render [--title TITLE] [--details] [--only failures|warnings|problems|all] [--json]
|
Usage: report-render [--title TITLE] [--details] [--only failures|warnings|problems|all] [--json]
|
||||||
|
|
@ -35,13 +22,4 @@ if (($# > 0)); then
|
||||||
esac
|
esac
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ ! -x "${CORE_BIN}" || "${REPO_ROOT}/go.mod" -nt "${CORE_BIN}" ]] || core_sources_newer; then
|
exec "${REPO_ROOT}/scripts/jeannie-core" report-render "$@"
|
||||||
if ! command -v go >/dev/null 2>&1; then
|
|
||||||
echo "go is required to build ${CORE_BIN}; run ./jeannie up or install golang-go." >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
mkdir -p "$(dirname "${CORE_BIN}")"
|
|
||||||
GOCACHE="${GOCACHE:-${REPO_ROOT}/.lab/go-build-cache}" go build -o "${CORE_BIN}" "${REPO_ROOT}/cmd/jeannie-core"
|
|
||||||
fi
|
|
||||||
|
|
||||||
exec "${CORE_BIN}" report-render "$@"
|
|
||||||
|
|
|
||||||
|
|
@ -336,6 +336,14 @@ test_pimox_guest_exec_exitcode_parser() {
|
||||||
assert_command_failure_contains "pimox guest exec rejects invalid JSON" "" pimox_guest_exec_exitcode 'not json'
|
assert_command_failure_contains "pimox guest exec rejects invalid JSON" "" pimox_guest_exec_exitcode 'not json'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
test_pimox_worker_net0_config_defaults_to_auto_mac() {
|
||||||
|
unset LAB_PIMOX_WORKER_MAC_MODE
|
||||||
|
assert_eq "pimox worker net0 defaults to auto MAC" "virtio,bridge=vmbr0" "$(pimox_worker_net0_config 9010 vmbr0)"
|
||||||
|
LAB_PIMOX_WORKER_MAC_MODE=deterministic
|
||||||
|
assert_eq "pimox worker net0 deterministic MAC opt-in" "virtio=02:68:10:00:23:32,bridge=vmbr0" "$(pimox_worker_net0_config 9010 vmbr0)"
|
||||||
|
unset LAB_PIMOX_WORKER_MAC_MODE
|
||||||
|
}
|
||||||
|
|
||||||
test_pimox_worker_count_detects_generated_topology
|
test_pimox_worker_count_detects_generated_topology
|
||||||
test_configured_worker_count_cannot_hide_existing_workers
|
test_configured_worker_count_cannot_hide_existing_workers
|
||||||
test_configured_worker_count_can_expand_target_set
|
test_configured_worker_count_can_expand_target_set
|
||||||
|
|
@ -350,6 +358,7 @@ test_inventory_validator_rejects_ip_drift
|
||||||
test_heal_cooldown_escalates_to_diagnostics
|
test_heal_cooldown_escalates_to_diagnostics
|
||||||
test_pimox_worker_static_ip_selection
|
test_pimox_worker_static_ip_selection
|
||||||
test_pimox_guest_exec_exitcode_parser
|
test_pimox_guest_exec_exitcode_parser
|
||||||
|
test_pimox_worker_net0_config_defaults_to_auto_mac
|
||||||
|
|
||||||
if ((FAILURES > 0)); then
|
if ((FAILURES > 0)); then
|
||||||
printf '\n%s unit test(s) failed.\n' "${FAILURES}" >&2
|
printf '\n%s unit test(s) failed.\n' "${FAILURES}" >&2
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue