Add canary promotion gates
This commit is contained in:
parent
7f537c4025
commit
ed3402bb71
14
README.md
14
README.md
|
|
@ -256,6 +256,20 @@ Safe red-team/blue-team checks are grouped under one command:
|
||||||
Local mode runs deterministic prompt-injection and eval checks. Live scans are
|
Local mode runs deterministic prompt-injection and eval checks. Live scans are
|
||||||
listed but gated so they can run from the Debian host when intended.
|
listed but gated so they can run from the Debian host when intended.
|
||||||
|
|
||||||
|
## Canary Promotion
|
||||||
|
|
||||||
|
The promotion helper defines release gates before production changes:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./jeannie promote plan
|
||||||
|
./jeannie promote validate
|
||||||
|
./jeannie promote rollback website-production
|
||||||
|
```
|
||||||
|
|
||||||
|
The current version is intentionally conservative: it validates local gates and
|
||||||
|
prints rollback steps while dev namespace automation remains explicit future
|
||||||
|
work.
|
||||||
|
|
||||||
## Deploying
|
## Deploying
|
||||||
|
|
||||||
From the Debian server:
|
From the Debian server:
|
||||||
|
|
|
||||||
|
|
@ -256,6 +256,20 @@ Safe red-team/blue-team checks are grouped under one command:
|
||||||
Local mode runs deterministic prompt-injection and eval checks. Live scans are
|
Local mode runs deterministic prompt-injection and eval checks. Live scans are
|
||||||
listed but gated so they can run from the Debian host when intended.
|
listed but gated so they can run from the Debian host when intended.
|
||||||
|
|
||||||
|
## Canary Promotion
|
||||||
|
|
||||||
|
The promotion helper defines release gates before production changes:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./{{ main_script }} promote plan
|
||||||
|
./{{ main_script }} promote validate
|
||||||
|
./{{ main_script }} promote rollback website-production
|
||||||
|
```
|
||||||
|
|
||||||
|
The current version is intentionally conservative: it validates local gates and
|
||||||
|
prints rollback steps while dev namespace automation remains explicit future
|
||||||
|
work.
|
||||||
|
|
||||||
## Deploying
|
## Deploying
|
||||||
|
|
||||||
From the Debian server:
|
From the Debian server:
|
||||||
|
|
|
||||||
|
|
@ -118,6 +118,10 @@ context is needed.
|
||||||
or degraded apps, repository secret presence, recent events, and recent
|
or degraded apps, repository secret presence, recent events, and recent
|
||||||
repo-server/application-controller errors.
|
repo-server/application-controller errors.
|
||||||
|
|
||||||
|
`promote {plan|validate|rollback APP}`
|
||||||
|
: Print the canary promotion path, run local required promotion gates, or print
|
||||||
|
rollback steps for an app.
|
||||||
|
|
||||||
`grafana-dashboards {list|apply}`
|
`grafana-dashboards {list|apply}`
|
||||||
: List or rebuild repo-provisioned Grafana dashboard ConfigMaps from
|
: List or rebuild repo-provisioned Grafana dashboard ConfigMaps from
|
||||||
`bootstrap/platform/grafana-dashboards`.
|
`bootstrap/platform/grafana-dashboards`.
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,18 @@
|
||||||
|
# Canary Promotion And Rollback
|
||||||
|
|
||||||
|
The promotion flow is a guarded release model for app changes:
|
||||||
|
|
||||||
|
1. validate repository and generated docs
|
||||||
|
2. deploy/check dev namespace
|
||||||
|
3. run synthetic, eval, and security checks
|
||||||
|
4. promote main/prod only after checks pass
|
||||||
|
5. keep rollback instructions visible
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./jeannie promote plan
|
||||||
|
./jeannie promote validate
|
||||||
|
./jeannie promote rollback website-production
|
||||||
|
```
|
||||||
|
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
stage command required
|
||||||
|
repo ./jeannie validate true
|
||||||
|
evals ./jeannie ai-evals run true
|
||||||
|
prompt-injection ./jeannie prompt-injection-lab run true
|
||||||
|
red-blue-local ./jeannie red-blue run --local true
|
||||||
|
synthetic ./jeannie synthetic-checks false
|
||||||
|
security-web ./jeannie security-web false
|
||||||
|
|
||||||
|
8
jeannie
8
jeannie
|
|
@ -6350,6 +6350,10 @@ red_blue_loop() {
|
||||||
"${REPO_ROOT}/scripts/red-blue-loop" "${@:2}"
|
"${REPO_ROOT}/scripts/red-blue-loop" "${@:2}"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
promote() {
|
||||||
|
"${REPO_ROOT}/scripts/promote" "${@:2}"
|
||||||
|
}
|
||||||
|
|
||||||
validate_homelab() {
|
validate_homelab() {
|
||||||
"${REPO_ROOT}/scripts/validate-homelab"
|
"${REPO_ROOT}/scripts/validate-homelab"
|
||||||
}
|
}
|
||||||
|
|
@ -6555,6 +6559,7 @@ Operator View And Reports
|
||||||
Observability And GitOps
|
Observability And GitOps
|
||||||
grafana-dashboards {list|apply} List or apply repo-managed Grafana dashboards.
|
grafana-dashboards {list|apply} List or apply repo-managed Grafana dashboards.
|
||||||
gitops-status Print focused Argo CD health and sync status.
|
gitops-status Print focused Argo CD health and sync status.
|
||||||
|
promote {plan|validate|rollback APP} Run canary promotion gates and rollback plan.
|
||||||
cert-check Check public DNS, TLS, and edge URL health.
|
cert-check Check public DNS, TLS, and edge URL health.
|
||||||
backup-status Check backup and restore-drill freshness.
|
backup-status Check backup and restore-drill freshness.
|
||||||
synthetic-checks Run end-to-end service probes.
|
synthetic-checks Run end-to-end service probes.
|
||||||
|
|
@ -6671,6 +6676,9 @@ case "${1:-}" in
|
||||||
gitops-status)
|
gitops-status)
|
||||||
gitops_status
|
gitops_status
|
||||||
;;
|
;;
|
||||||
|
promote)
|
||||||
|
promote "$@"
|
||||||
|
;;
|
||||||
cert-check)
|
cert-check)
|
||||||
cert_check
|
cert_check
|
||||||
;;
|
;;
|
||||||
|
|
|
||||||
|
|
@ -34,7 +34,7 @@ EOF
|
||||||
|
|
||||||
is_read_only_command() {
|
is_read_only_command() {
|
||||||
case "$1" in
|
case "$1" in
|
||||||
status|capacity|capacity-advisor|capacity-limits|recover-plan|incident|safety-case|agent-sandbox|scorecard|gitops-status|cert-check|backup-status|synthetic-checks|resource-budget|route-inventory|golden-ledger|workers|change-journal|map|validate|access-audit|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|secrets-check|tailnet-policy-check|ai-check|ask|ai-evals|ai-scheduler|ai-memory|impact|review-last-change|security-scan|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-attack-path|prompt-injection-lab|red-blue|control-plane)
|
status|capacity|capacity-advisor|capacity-limits|recover-plan|incident|safety-case|agent-sandbox|scorecard|gitops-status|promote|cert-check|backup-status|synthetic-checks|resource-budget|route-inventory|golden-ledger|workers|change-journal|map|validate|access-audit|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|secrets-check|tailnet-policy-check|ai-check|ask|ai-evals|ai-scheduler|ai-memory|impact|review-last-change|security-scan|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-attack-path|prompt-injection-lab|red-blue|control-plane)
|
||||||
return 0
|
return 0
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,93 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Plan and validate canary promotion gates."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import csv
|
||||||
|
import subprocess
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
CHECKS_FILE = REPO_ROOT / "infra" / "promotion" / "checks.tsv"
|
||||||
|
|
||||||
|
|
||||||
|
def checks() -> list[dict[str, str]]:
|
||||||
|
with CHECKS_FILE.open(encoding="utf-8", newline="") as handle:
|
||||||
|
return list(csv.DictReader(handle, delimiter="\t"))
|
||||||
|
|
||||||
|
|
||||||
|
def plan() -> int:
|
||||||
|
print("Promotion Plan")
|
||||||
|
print("==============")
|
||||||
|
print("dev branch -> validation -> main -> Argo CD sync")
|
||||||
|
print()
|
||||||
|
for row in checks():
|
||||||
|
marker = "required" if row["required"] == "true" else "optional-live"
|
||||||
|
print(f"{marker:13} {row['stage']:18} {row['command']}")
|
||||||
|
print()
|
||||||
|
print("Prod promotion remains manual until dev namespace/app wiring is added.")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def validate(local_only: bool) -> int:
|
||||||
|
failures = 0
|
||||||
|
print("Promotion Validation")
|
||||||
|
print("====================")
|
||||||
|
for row in checks():
|
||||||
|
is_required = row["required"] == "true"
|
||||||
|
if local_only and not is_required:
|
||||||
|
print(f"skip {row['stage']} - live check gated")
|
||||||
|
continue
|
||||||
|
print(f"run {row['stage']} - {row['command']}")
|
||||||
|
process = subprocess.run(
|
||||||
|
row["command"].split(),
|
||||||
|
cwd=REPO_ROOT,
|
||||||
|
text=True,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
if process.returncode == 0:
|
||||||
|
print("ok")
|
||||||
|
else:
|
||||||
|
failures += 1
|
||||||
|
print(process.stdout.rstrip())
|
||||||
|
print(f"failures={failures}")
|
||||||
|
return 1 if failures else 0
|
||||||
|
|
||||||
|
|
||||||
|
def rollback(app: str) -> int:
|
||||||
|
print("Rollback Plan")
|
||||||
|
print("=============")
|
||||||
|
print(f"app: {app}")
|
||||||
|
print("1. identify last known good Git commit or image digest")
|
||||||
|
print("2. git revert <bad-commit>")
|
||||||
|
print("3. ./jeannie safety-case --since HEAD~1")
|
||||||
|
print("4. ./jeannie validate")
|
||||||
|
print("5. push to main and let Argo CD reconcile")
|
||||||
|
print("6. ./jeannie synthetic-checks")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||||
|
subparsers.add_parser("plan")
|
||||||
|
validate_parser = subparsers.add_parser("validate")
|
||||||
|
validate_parser.add_argument("--all", action="store_true", help="Include optional live checks.")
|
||||||
|
rollback_parser = subparsers.add_parser("rollback")
|
||||||
|
rollback_parser.add_argument("app")
|
||||||
|
args = parser.parse_args()
|
||||||
|
if args.command == "plan":
|
||||||
|
return plan()
|
||||||
|
if args.command == "validate":
|
||||||
|
return validate(local_only=not args.all)
|
||||||
|
if args.command == "rollback":
|
||||||
|
return rollback(args.app)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
Loading…
Reference in New Issue