Add canary promotion gates

This commit is contained in:
juvdiaz 2026-06-30 13:22:50 -06:00
parent 7f537c4025
commit ed3402bb71
8 changed files with 160 additions and 1 deletions

View File

@ -256,6 +256,20 @@ Safe red-team/blue-team checks are grouped under one command:
Local mode runs deterministic prompt-injection and eval checks. Live scans are
listed but gated so they can run from the Debian host when intended.
## Canary Promotion
The promotion helper defines release gates before production changes:
```bash
./jeannie promote plan
./jeannie promote validate
./jeannie promote rollback website-production
```
The current version is intentionally conservative: it validates local gates and
prints rollback steps while dev namespace automation remains explicit future
work.
## Deploying
From the Debian server:

View File

@ -256,6 +256,20 @@ Safe red-team/blue-team checks are grouped under one command:
Local mode runs deterministic prompt-injection and eval checks. Live scans are
listed but gated so they can run from the Debian host when intended.
## Canary Promotion
The promotion helper defines release gates before production changes:
```bash
./{{ main_script }} promote plan
./{{ main_script }} promote validate
./{{ main_script }} promote rollback website-production
```
The current version is intentionally conservative: it validates local gates and
prints rollback steps while dev namespace automation remains explicit future
work.
## Deploying
From the Debian server:

View File

@ -118,6 +118,10 @@ context is needed.
or degraded apps, repository secret presence, recent events, and recent
repo-server/application-controller errors.
`promote {plan|validate|rollback APP}`
: Print the canary promotion path, run local required promotion gates, or print
rollback steps for an app.
`grafana-dashboards {list|apply}`
: List or rebuild repo-provisioned Grafana dashboard ConfigMaps from
`bootstrap/platform/grafana-dashboards`.

18
infra/promotion/README.md Normal file
View File

@ -0,0 +1,18 @@
# Canary Promotion And Rollback
The promotion flow is a guarded release model for app changes:
1. validate repository and generated docs
2. deploy/check dev namespace
3. run synthetic, eval, and security checks
4. promote main/prod only after checks pass
5. keep rollback instructions visible
Run:
```bash
./jeannie promote plan
./jeannie promote validate
./jeannie promote rollback website-production
```

View File

@ -0,0 +1,8 @@
stage command required
repo ./jeannie validate true
evals ./jeannie ai-evals run true
prompt-injection ./jeannie prompt-injection-lab run true
red-blue-local ./jeannie red-blue run --local true
synthetic ./jeannie synthetic-checks false
security-web ./jeannie security-web false
1 stage command required
2 repo ./jeannie validate true
3 evals ./jeannie ai-evals run true
4 prompt-injection ./jeannie prompt-injection-lab run true
5 red-blue-local ./jeannie red-blue run --local true
6 synthetic ./jeannie synthetic-checks false
7 security-web ./jeannie security-web false

View File

@ -6350,6 +6350,10 @@ red_blue_loop() {
"${REPO_ROOT}/scripts/red-blue-loop" "${@:2}"
}
promote() {
"${REPO_ROOT}/scripts/promote" "${@:2}"
}
validate_homelab() {
"${REPO_ROOT}/scripts/validate-homelab"
}
@ -6555,6 +6559,7 @@ Operator View And Reports
Observability And GitOps
grafana-dashboards {list|apply} List or apply repo-managed Grafana dashboards.
gitops-status Print focused Argo CD health and sync status.
promote {plan|validate|rollback APP} Run canary promotion gates and rollback plan.
cert-check Check public DNS, TLS, and edge URL health.
backup-status Check backup and restore-drill freshness.
synthetic-checks Run end-to-end service probes.
@ -6671,6 +6676,9 @@ case "${1:-}" in
gitops-status)
gitops_status
;;
promote)
promote "$@"
;;
cert-check)
cert_check
;;

View File

@ -34,7 +34,7 @@ EOF
is_read_only_command() {
case "$1" in
status|capacity|capacity-advisor|capacity-limits|recover-plan|incident|safety-case|agent-sandbox|scorecard|gitops-status|cert-check|backup-status|synthetic-checks|resource-budget|route-inventory|golden-ledger|workers|change-journal|map|validate|access-audit|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|secrets-check|tailnet-policy-check|ai-check|ask|ai-evals|ai-scheduler|ai-memory|impact|review-last-change|security-scan|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-attack-path|prompt-injection-lab|red-blue|control-plane)
status|capacity|capacity-advisor|capacity-limits|recover-plan|incident|safety-case|agent-sandbox|scorecard|gitops-status|promote|cert-check|backup-status|synthetic-checks|resource-budget|route-inventory|golden-ledger|workers|change-journal|map|validate|access-audit|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|secrets-check|tailnet-policy-check|ai-check|ask|ai-evals|ai-scheduler|ai-memory|impact|review-last-change|security-scan|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-attack-path|prompt-injection-lab|red-blue|control-plane)
return 0
;;
*)

93
scripts/promote Executable file
View File

@ -0,0 +1,93 @@
#!/usr/bin/env python3
"""Plan and validate canary promotion gates."""
from __future__ import annotations
import argparse
import csv
import subprocess
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
CHECKS_FILE = REPO_ROOT / "infra" / "promotion" / "checks.tsv"
def checks() -> list[dict[str, str]]:
with CHECKS_FILE.open(encoding="utf-8", newline="") as handle:
return list(csv.DictReader(handle, delimiter="\t"))
def plan() -> int:
print("Promotion Plan")
print("==============")
print("dev branch -> validation -> main -> Argo CD sync")
print()
for row in checks():
marker = "required" if row["required"] == "true" else "optional-live"
print(f"{marker:13} {row['stage']:18} {row['command']}")
print()
print("Prod promotion remains manual until dev namespace/app wiring is added.")
return 0
def validate(local_only: bool) -> int:
failures = 0
print("Promotion Validation")
print("====================")
for row in checks():
is_required = row["required"] == "true"
if local_only and not is_required:
print(f"skip {row['stage']} - live check gated")
continue
print(f"run {row['stage']} - {row['command']}")
process = subprocess.run(
row["command"].split(),
cwd=REPO_ROOT,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
check=False,
)
if process.returncode == 0:
print("ok")
else:
failures += 1
print(process.stdout.rstrip())
print(f"failures={failures}")
return 1 if failures else 0
def rollback(app: str) -> int:
print("Rollback Plan")
print("=============")
print(f"app: {app}")
print("1. identify last known good Git commit or image digest")
print("2. git revert <bad-commit>")
print("3. ./jeannie safety-case --since HEAD~1")
print("4. ./jeannie validate")
print("5. push to main and let Argo CD reconcile")
print("6. ./jeannie synthetic-checks")
return 0
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
subparsers = parser.add_subparsers(dest="command", required=True)
subparsers.add_parser("plan")
validate_parser = subparsers.add_parser("validate")
validate_parser.add_argument("--all", action="store_true", help="Include optional live checks.")
rollback_parser = subparsers.add_parser("rollback")
rollback_parser.add_argument("app")
args = parser.parse_args()
if args.command == "plan":
return plan()
if args.command == "validate":
return validate(local_only=not args.all)
if args.command == "rollback":
return rollback(args.app)
return 2
if __name__ == "__main__":
raise SystemExit(main())