From 5d23388ef58a79e8b0f05289fdc6fad29b9e0b99 Mon Sep 17 00:00:00 2001 From: Johannes Date: Thu, 2 Jul 2026 07:55:49 -0400 Subject: [PATCH] docs(guide): add platform.production-checks + new DEVOPS section (backups, security) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - platform.production-checks: the daily prod health sweep (deep actuator/critical set, Loki ERROR+5xx top-5, backup freshness, ClamAV per-VM) -> 06:30 email report. - NEW DevOps nav section in the reader (SPECIAL + section block + expandFor). - devops.backups: pg_dumpall->Swift nightly job, verify/restore, gaps (#13/#14/#15). - devops.security: ClamAV/firewall posture from the 2026-07-02 sweep — FAILING, remediation in #16 (Critical, PCI CDE) / #17. Reconciler: devops = concept docs. Co-Authored-By: Claude Opus 4.8 --- .../content/devops/backups/overview.md | 55 ++++++++++++ .../content/devops/security/overview.md | 49 +++++++++++ .../platform/production-checks/overview.md | 87 +++++++++++++++++++ frontend/src/components/GuideReader.svelte | 19 +++- scripts/coverage-reconcile.py | 2 +- 5 files changed, 210 insertions(+), 2 deletions(-) create mode 100644 backend/src/main/resources/content/devops/backups/overview.md create mode 100644 backend/src/main/resources/content/devops/security/overview.md create mode 100644 backend/src/main/resources/content/platform/production-checks/overview.md diff --git a/backend/src/main/resources/content/devops/backups/overview.md b/backend/src/main/resources/content/devops/backups/overview.md new file mode 100644 index 0000000..5dcb8f7 --- /dev/null +++ b/backend/src/main/resources/content/devops/backups/overview.md @@ -0,0 +1,55 @@ +--- +module: devops.backups +title: Database Backups +tab: Overview +order: 10 +audience: dev +--- + +> **DRAFT · internal.** Written 2026-07-02 from the live backup job on `hiveiq-database`. Verify +> against the script before relying on specifics. + +## What runs + +Production databases are backed up by **`/opt/hiveiq/backup-db-swift.sh`** on the **database VM** +(`10.10.10.188`, `hiveiq-database`), via **cron daily at 01:00** (logs to `/var/log/hiveiq-backup.log`). + +- **Method:** `docker exec hiveiq-postgres pg_dumpall -U hiveiq | gzip` — one logical dump of **all** + databases (`hiveiq_auth`, `hiveiq_mgmt`, `hiveiq_incident`, `hiveiq_journal`, …). +- **Destination:** OpenStack **Swift**, container `hiveiq-backups`, object `db/db_.sql.gz`. +- **Retention:** **30 days** (Swift `X-Delete-After`, set at upload). +- **Auth:** Swift creds in `/opt/hiveiq/.backup-env` (sourced by the script). +- Postgres image: `pgvector/pgvector:pg16`, container `hiveiq-postgres`. + +Verified healthy 2026-07-02: dump **1.4 GB**, uploaded OK, **31 backups** in Swift. + +## Verify / operate + +```bash +# on hiveiq-database (10.10.10.188, via ProxyJump) +sudo tail -20 /var/log/hiveiq-backup.log # last run — expect "=== DB backup done ===" +source /opt/hiveiq/.backup-env +swift --insecure list hiveiq-backups --prefix db/ # newest object should be today +sudo /opt/hiveiq/backup-db-swift.sh # run on-demand +``` + +The **daily production health check** (platform.production-checks §3) asserts: log done < 24h, +dump size ~1.3–1.4 GB, Swift count ≈ 30. + +## Restore + +```bash +source /opt/hiveiq/.backup-env +swift --insecure download hiveiq-backups db/db_.sql.gz -o restore.sql.gz +gunzip -c restore.sql.gz | docker exec -i psql -U hiveiq # pg_dumpall = full cluster +``` + +## Known limitations (tracked) + +- **No PITR / WAL archiving** — nightly logical snapshot only, so worst-case data loss is **up to + ~24h**. → **hiveiq-devops#13** +- **No restore test** — backups are never verified by an actual restore. → **hiveiq-devops#14** +- **DR boundary** — Swift is offsite from the VM but likely on the **same OpenMetal cluster**; a + full-cluster loss could take the backups too. → **hiveiq-devops#15** + +See **platform.production-checks**, **devops.security**, **platform.topology**. diff --git a/backend/src/main/resources/content/devops/security/overview.md b/backend/src/main/resources/content/devops/security/overview.md new file mode 100644 index 0000000..629b47d --- /dev/null +++ b/backend/src/main/resources/content/devops/security/overview.md @@ -0,0 +1,49 @@ +--- +module: devops.security +title: Host Security & Anti-Virus +tab: Overview +order: 20 +audience: dev +--- + +> **DRAFT · internal.** Written 2026-07-02 from a live sweep of all prod VMs. Security posture +> below was **failing** at time of writing — see the tracked issues. Re-verify before citing. + +## What the daily check verifies (platform.production-checks §4) + +Lightweight "are the controls up" across **all prod VMs** — services (`10.10.10.164`), database +(`10.10.10.188`, **PCI CDE**), cdn (`10.10.10.103`), proxy (`10.10.10.12`), share (`10.10.10.170`), +ollama (`10.10.10.7`). Per VM: + +```bash +dpkg -l | grep -c clamav # ClamAV installed? +systemctl is-active clamav-freshclam clamav-daemon # updater + daemon running? +ls -l --time-style=+%F /var/lib/clamav/daily.c[vl]d # signature DB age (< 2 days) +sudo ufw status | head -1 ; systemctl is-active fail2ban # host firewall / brute-force guard +``` + +**RED** if AV is missing/stale on any in-scope VM — loudest on the **database VM (PCI CDE)**. + +## Posture as of 2026-07-02 (FAILING — being remediated) + +| VM | ClamAV | freshclam | DB age | clamd | ufw | fail2ban | +|----|--------|-----------|--------|-------|-----|----------| +| services `.250` | installed | active | current | **failed** | off | off | +| **database `.188` (CDE)** | installed | **stopped** | **stale 06-09** | — | off | off | +| cdn `.252` | installed | **stopped** | **stale 06-09** | off | off | off | +| proxy `.253` | **none** | — | — | — | off | off | +| share `.254` | **none** | — | — | — | off | off | +| ollama `.7` | **none** | — | — | — | off | off | + +- **Anti-virus (PCI Req 5):** 3 VMs have no AV; db + cdn have a ~3-week-stale signature DB with the + updater stopped; services' `clamd` daemon is failed. Gap on the **CDE** host. → **hiveiq-devops#16 (Critical)** +- **Firewall / brute-force (PCI Req 1):** no host `ufw` and no `fail2ban` anywhere. **Caveat:** + network access is very likely gated by **OpenStack security groups** at the cloud layer — confirm + the SG rules before concluding "no firewall." → **hiveiq-devops#17 (High)** + +## PCI context + +The A-LIGN engagement puts the prod stack (esp. the CDE DB VM) in scope. Anti-malware on the CDE is +a hard requirement (Req 5); this must be closed and evidenced. See the PCI notes in project memory. + +See **platform.production-checks**, **devops.backups**, **platform.topology**. diff --git a/backend/src/main/resources/content/platform/production-checks/overview.md b/backend/src/main/resources/content/platform/production-checks/overview.md new file mode 100644 index 0000000..83eff95 --- /dev/null +++ b/backend/src/main/resources/content/platform/production-checks/overview.md @@ -0,0 +1,87 @@ +--- +module: platform.production-checks +title: Production Health Checks +tab: Overview +order: 40 +audience: dev +--- + +> **DRAFT · internal.** Written 2026-07-02. Defines the daily production health sweep that emails a +> report. Verify commands against the live hosts before relying on them. + +## What this is + +A **daily automated sweep** (cron on the services VM, ~06:30) that runs five checks and emails a +single report to `johannes@bcos.cloud` + `tristan@bcos.cloud` from `hiveiq@bcos.cloud` +(`mail.bcos.cloud:587`). **Sent every day** (silence must never hide a dead cron); the **verdict is +in the subject** — e.g. `HiveIQ Prod Health — 🔴 2 issues (2026-07-02)`. + +Script: `/opt/hiveiq/prod-health-report.sh` on `173.231.195.250` (next to `backup-db-swift.sh`). +Section detail for backups + anti-virus lives under **devops.backups** and **devops.security**. + +## Report shape + +- **Subject** carries the overall verdict (🟢 all clear / 🔴 N issues). +- **Summary block** — one line per section with 🟢 / 🟡 / 🔴. +- **Details** — only non-green items expanded (which service, top errors, backup age, AV table). +- HTML with a plaintext fallback. + +## 1. HiveIQ Platform health (critical set = auth, mgmt, incident, agent-proxy → RED) + +**Deep actuator** — parse the `components` block, not just top-level `status`, so a degraded +dependency (DB/Kafka/disk) shows even when the app reports UP. **Active blue/green slot only.** + +```bash +# per service (host ports on 173.231.195.250): auth 8001, mgmt 8003, incident 8005, journal 8006, +# remote 8008, ai 8009, messaging 8010, reports 8011, analytics 8012, recon 8013, claims 8014, agent-proxy 8015 +curl -fsS localhost:/actuator/health | jq -e '.status=="UP" and (all(.components[]?; .status=="UP"))' +docker compose ps --format '{{.Name}} {{.State}} {{.Health}}' # flag not-running / unhealthy / restarting +``` + +- **RED** if any **critical-set** service (auth/mgmt/incident/agent-proxy) is not UP or has a + non-UP component; **YELLOW** for the rest. +- Confirm the **active** slot is the one nginx serves (`00-backends.conf` map) and it's healthy. +- Edge: NGINX 200 through `api.bcos.cloud` + TLS cert days-to-expiry. + +## 2. Loki error check (last 24h) + +Query **Loki** (`localhost:3100` on the services VM — never SSH+cat logs). Count **`level=ERROR` ++ HTTP 5xx**, **highlight stacktraces**, and surface the **top-5 error signatures per service** +(normalise out timestamps/ids so the same error groups). Apply a **suppression list** for +known-benign noise (e.g. transient agent reconnects, AGENT_OFFLINE churn). + +```bash +logcli --addr=http://localhost:3100 query --since=24h --quiet \ + 'sum by (service_name) (count_over_time({service_name=~".+"} | level="ERROR" [24h]))' +# 5xx from the nginx access stream if present; group + rank signatures; drop suppressed patterns +``` + +- **YELLOW/RED** when a service crosses the flag threshold (tuned once real volume is known). +- Note: confirm Loki's actual label scheme (`service_name` vs `container`/`job`) on the box. + +## 3. Backup check → **devops.backups** + +The nightly `pg_dumpall` → Swift job. Verify: +- `/var/log/hiveiq-backup.log` ends in `=== DB backup done ===` **within the last 24h** +- last dump size sane (**~1.3–1.4 GB**, not near-zero → catches truncated dumps) +- Swift object count ≈ 30 (`swift --insecure list hiveiq-backups --prefix db/` — newest object today) +- **RED** if last run failed or is >24h old. (Restore-test verification tracked in hiveiq-devops#14.) + +## 4. Security check → **devops.security** + +Lightweight "are the controls up" across **all prod VMs** (services, database, cdn, proxy, share, +ollama). Per VM: +- **ClamAV** installed, `clamav-freshclam` **active**, signature DB **< 2 days old** + (`/var/lib/clamav/daily.c[vl]d`), `clamav-daemon` running. +- Report `ufw` / `fail2ban` state (network firewall may be OpenStack security groups — see devops.security). +- **RED** if AV is missing/stale on any in-scope VM — loudest on the **database VM (PCI CDE)**. + *(As of 2026-07-02 this is RED — remediation in hiveiq-devops#16/#17.)* + +## 5. Email delivery + +`hiveiq@bcos.cloud` via `mail.bcos.cloud:587` (STARTTLS). Recipients: johannes + tristan (so +Tristan knows the state when Johannes is out). Always send; verdict in subject. + +## Related +Gaps found while building this: backups hiveiq-devops#13/#14/#15; security hiveiq-devops#16/#17. +See **devops.backups**, **devops.security**, **platform.gotchas**, **platform.ci-cd**. diff --git a/frontend/src/components/GuideReader.svelte b/frontend/src/components/GuideReader.svelte index 1c258fd..f3f38f4 100644 --- a/frontend/src/components/GuideReader.svelte +++ b/frontend/src/components/GuideReader.svelte @@ -22,6 +22,7 @@ function expandFor(key: string) { const app = key.split('.')[0]; if (app === 'platform') open = { ...open, platform: true }; + else if (app === 'devops') open = { ...open, devops: true }; else if (app === 'technical') open = { ...open, technical: true }; else if (app === 'processes') open = { ...open, processes: true }; else open = { ...open, modules: true, ['app:' + app]: true }; @@ -81,9 +82,10 @@ // group nav: product-app "Modules" section (all except platform/technical) + platform + technical $: searching = !!search.trim(); - const SPECIAL = ['platform', 'technical', 'processes']; + const SPECIAL = ['platform', 'devops', 'technical', 'processes']; $: productApps = filtered.filter(a => !SPECIAL.includes(a.app)); $: platformApp = filtered.find(a => a.app === 'platform') ?? null; + $: devopsApp = filtered.find(a => a.app === 'devops') ?? null; $: technicalApp = filtered.find(a => a.app === 'technical') ?? null; $: processesApp = filtered.find(a => a.app === 'processes') ?? null; @@ -178,6 +180,21 @@ {/if} + + {#if devopsApp} +
+ + {#if (searching || open['devops'])} + {#each devopsApp.modules as m (m.module)} + + {/each} + {/if} +
+ {/if} + {#if technicalApp}
diff --git a/scripts/coverage-reconcile.py b/scripts/coverage-reconcile.py index e17da02..290ac05 100644 --- a/scripts/coverage-reconcile.py +++ b/scripts/coverage-reconcile.py @@ -17,7 +17,7 @@ PRODUCT_APPS = { "profile","recon","reports","transactions","vault", } # guide app-prefixes that are NOT code-derived (concept / process docs) — skip in the code diff -NON_CODE_APPS = {"platform","technical","processes"} +NON_CODE_APPS = {"platform","devops","technical","processes"} # the 5 role-tabs expected on a product module PRODUCT_TABS = {"Customer","Internal","Architect","Testing","Claude"} # dir-name (minus hiveops-) → guide app/service key, where they differ