Add canary promotion gates
This commit is contained in:
parent
7f537c4025
commit
ed3402bb71
14
README.md
14
README.md
|
|
@ -256,6 +256,20 @@ Safe red-team/blue-team checks are grouped under one command:
|
|||
Local mode runs deterministic prompt-injection and eval checks. Live scans are
|
||||
listed but gated so they can run from the Debian host when intended.
|
||||
|
||||
## Canary Promotion
|
||||
|
||||
The promotion helper defines release gates before production changes:
|
||||
|
||||
```bash
|
||||
./jeannie promote plan
|
||||
./jeannie promote validate
|
||||
./jeannie promote rollback website-production
|
||||
```
|
||||
|
||||
The current version is intentionally conservative: it validates local gates and
|
||||
prints rollback steps while dev namespace automation remains explicit future
|
||||
work.
|
||||
|
||||
## Deploying
|
||||
|
||||
From the Debian server:
|
||||
|
|
|
|||
|
|
@ -256,6 +256,20 @@ Safe red-team/blue-team checks are grouped under one command:
|
|||
Local mode runs deterministic prompt-injection and eval checks. Live scans are
|
||||
listed but gated so they can run from the Debian host when intended.
|
||||
|
||||
## Canary Promotion
|
||||
|
||||
The promotion helper defines release gates before production changes:
|
||||
|
||||
```bash
|
||||
./{{ main_script }} promote plan
|
||||
./{{ main_script }} promote validate
|
||||
./{{ main_script }} promote rollback website-production
|
||||
```
|
||||
|
||||
The current version is intentionally conservative: it validates local gates and
|
||||
prints rollback steps while dev namespace automation remains explicit future
|
||||
work.
|
||||
|
||||
## Deploying
|
||||
|
||||
From the Debian server:
|
||||
|
|
|
|||
|
|
@ -118,6 +118,10 @@ context is needed.
|
|||
or degraded apps, repository secret presence, recent events, and recent
|
||||
repo-server/application-controller errors.
|
||||
|
||||
`promote {plan|validate|rollback APP}`
|
||||
: Print the canary promotion path, run local required promotion gates, or print
|
||||
rollback steps for an app.
|
||||
|
||||
`grafana-dashboards {list|apply}`
|
||||
: List or rebuild repo-provisioned Grafana dashboard ConfigMaps from
|
||||
`bootstrap/platform/grafana-dashboards`.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,18 @@
|
|||
# Canary Promotion And Rollback
|
||||
|
||||
The promotion flow is a guarded release model for app changes:
|
||||
|
||||
1. validate repository and generated docs
|
||||
2. deploy/check dev namespace
|
||||
3. run synthetic, eval, and security checks
|
||||
4. promote main/prod only after checks pass
|
||||
5. keep rollback instructions visible
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
./jeannie promote plan
|
||||
./jeannie promote validate
|
||||
./jeannie promote rollback website-production
|
||||
```
|
||||
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
stage command required
|
||||
repo ./jeannie validate true
|
||||
evals ./jeannie ai-evals run true
|
||||
prompt-injection ./jeannie prompt-injection-lab run true
|
||||
red-blue-local ./jeannie red-blue run --local true
|
||||
synthetic ./jeannie synthetic-checks false
|
||||
security-web ./jeannie security-web false
|
||||
|
||||
|
8
jeannie
8
jeannie
|
|
@ -6350,6 +6350,10 @@ red_blue_loop() {
|
|||
"${REPO_ROOT}/scripts/red-blue-loop" "${@:2}"
|
||||
}
|
||||
|
||||
promote() {
|
||||
"${REPO_ROOT}/scripts/promote" "${@:2}"
|
||||
}
|
||||
|
||||
validate_homelab() {
|
||||
"${REPO_ROOT}/scripts/validate-homelab"
|
||||
}
|
||||
|
|
@ -6555,6 +6559,7 @@ Operator View And Reports
|
|||
Observability And GitOps
|
||||
grafana-dashboards {list|apply} List or apply repo-managed Grafana dashboards.
|
||||
gitops-status Print focused Argo CD health and sync status.
|
||||
promote {plan|validate|rollback APP} Run canary promotion gates and rollback plan.
|
||||
cert-check Check public DNS, TLS, and edge URL health.
|
||||
backup-status Check backup and restore-drill freshness.
|
||||
synthetic-checks Run end-to-end service probes.
|
||||
|
|
@ -6671,6 +6676,9 @@ case "${1:-}" in
|
|||
gitops-status)
|
||||
gitops_status
|
||||
;;
|
||||
promote)
|
||||
promote "$@"
|
||||
;;
|
||||
cert-check)
|
||||
cert_check
|
||||
;;
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ EOF
|
|||
|
||||
is_read_only_command() {
|
||||
case "$1" in
|
||||
status|capacity|capacity-advisor|capacity-limits|recover-plan|incident|safety-case|agent-sandbox|scorecard|gitops-status|cert-check|backup-status|synthetic-checks|resource-budget|route-inventory|golden-ledger|workers|change-journal|map|validate|access-audit|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|secrets-check|tailnet-policy-check|ai-check|ask|ai-evals|ai-scheduler|ai-memory|impact|review-last-change|security-scan|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-attack-path|prompt-injection-lab|red-blue|control-plane)
|
||||
status|capacity|capacity-advisor|capacity-limits|recover-plan|incident|safety-case|agent-sandbox|scorecard|gitops-status|promote|cert-check|backup-status|synthetic-checks|resource-budget|route-inventory|golden-ledger|workers|change-journal|map|validate|access-audit|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|secrets-check|tailnet-policy-check|ai-check|ask|ai-evals|ai-scheduler|ai-memory|impact|review-last-change|security-scan|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-attack-path|prompt-injection-lab|red-blue|control-plane)
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,93 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Plan and validate canary promotion gates."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
CHECKS_FILE = REPO_ROOT / "infra" / "promotion" / "checks.tsv"
|
||||
|
||||
|
||||
def checks() -> list[dict[str, str]]:
|
||||
with CHECKS_FILE.open(encoding="utf-8", newline="") as handle:
|
||||
return list(csv.DictReader(handle, delimiter="\t"))
|
||||
|
||||
|
||||
def plan() -> int:
|
||||
print("Promotion Plan")
|
||||
print("==============")
|
||||
print("dev branch -> validation -> main -> Argo CD sync")
|
||||
print()
|
||||
for row in checks():
|
||||
marker = "required" if row["required"] == "true" else "optional-live"
|
||||
print(f"{marker:13} {row['stage']:18} {row['command']}")
|
||||
print()
|
||||
print("Prod promotion remains manual until dev namespace/app wiring is added.")
|
||||
return 0
|
||||
|
||||
|
||||
def validate(local_only: bool) -> int:
|
||||
failures = 0
|
||||
print("Promotion Validation")
|
||||
print("====================")
|
||||
for row in checks():
|
||||
is_required = row["required"] == "true"
|
||||
if local_only and not is_required:
|
||||
print(f"skip {row['stage']} - live check gated")
|
||||
continue
|
||||
print(f"run {row['stage']} - {row['command']}")
|
||||
process = subprocess.run(
|
||||
row["command"].split(),
|
||||
cwd=REPO_ROOT,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
check=False,
|
||||
)
|
||||
if process.returncode == 0:
|
||||
print("ok")
|
||||
else:
|
||||
failures += 1
|
||||
print(process.stdout.rstrip())
|
||||
print(f"failures={failures}")
|
||||
return 1 if failures else 0
|
||||
|
||||
|
||||
def rollback(app: str) -> int:
|
||||
print("Rollback Plan")
|
||||
print("=============")
|
||||
print(f"app: {app}")
|
||||
print("1. identify last known good Git commit or image digest")
|
||||
print("2. git revert <bad-commit>")
|
||||
print("3. ./jeannie safety-case --since HEAD~1")
|
||||
print("4. ./jeannie validate")
|
||||
print("5. push to main and let Argo CD reconcile")
|
||||
print("6. ./jeannie synthetic-checks")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
subparsers.add_parser("plan")
|
||||
validate_parser = subparsers.add_parser("validate")
|
||||
validate_parser.add_argument("--all", action="store_true", help="Include optional live checks.")
|
||||
rollback_parser = subparsers.add_parser("rollback")
|
||||
rollback_parser.add_argument("app")
|
||||
args = parser.parse_args()
|
||||
if args.command == "plan":
|
||||
return plan()
|
||||
if args.command == "validate":
|
||||
return validate(local_only=not args.all)
|
||||
if args.command == "rollback":
|
||||
return rollback(args.app)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Loading…
Reference in New Issue