openova/platform/cilium/chart/values.yaml

# Catalyst Blueprint umbrella metadata — the upstream chart is now resolved
# as a Helm subchart via Chart.yaml `dependencies:`. This values.yaml carries
# both:
#   1. The catalystBlueprint metadata block (provenance + version) so
#      observability/audit pipelines can inspect the artifact and report
#      which upstream chart + version is bundled.
#   2. The upstream subchart values overlay under the `cilium:` key
#      (umbrella-chart convention — the dependency name from Chart.yaml is
#      the values namespace).

global:
  # When set, ALL image pulls in this chart route through this registry.
  # Used post-handover when the Sovereign's own Harbor takes over the
  # proxy_cache role from contabo's central Harbor. Empty = no rewrite
  # (image references use upstream defaults). The upstream cilium subchart
  # does not expose a single `image.registry` knob; per-Sovereign overlays
  # should set specific image paths (e.g. `cilium.image.repository`) in
  # conjunction with this value. Tracked under #560.
  imageRegistry: ""

catalystBlueprint:
  # Pinned to match Chart.lock (truth-on-disk). Aligned with Chart.yaml's
  # dependencies version. The previous "1.19.3" was provenance drift —
  # Chart.lock had been resolving 1.16.5 the whole time (EPIC-0 #1095 audit).
  upstream: { chart: cilium, version: "1.16.5", repo: "https://helm.cilium.io" }

# Catalyst-curated Cilium values per platform/cilium/README.md.
#
# Defaults assume a Catalyst Sovereign on bare-metal/VM Kubernetes (k3s on
# Hetzner is the canonical target). Cilium replaces both flannel and
# kube-proxy; k3s must be started with --flannel-backend=none and
# --disable-network-policy.

cilium:
  # kube-proxy replacement (faster + simpler than iptables). MUST be true
  # because we also set `bpf.masquerade: true` (line 54) — Cilium rejects
  # the BPF masquerade datapath without NodePort enabled, and Cilium only
  # auto-enables NodePort when kubeProxyReplacement is true:
  #   "BPF masquerade requires NodePort (--enable-node-port=\"true\")"
  # The Sovereign cloud-init pre-Flux Cilium install (infra/hetzner/
  # cloudinit-control-plane.tftpl `/var/lib/catalyst/cilium-values.yaml`)
  # already uses kubeProxyReplacement: true, so this aligns the Flux HR
  # overlay with the pre-Flux bootstrap. Symptom when divergent: CP
  # cilium-agent kept its old in-memory state and stayed Ready, but every
  # WORKER node that joined AFTER the Flux upgrade saw the new ConfigMap
  # with enable-bpf-masquerade=true + enable-node-port=false → fatal
  # "BPF masquerade requires NodePort" → cilium-agent CrashLoopBackOff →
  # node.cilium.io/agent-not-ready taint never lifts → every keycloak/
  # gitea/catalyst-platform/openbao/powerdns/mimir post-install Job pod
  # stays Pending → whole bootstrap-kit chain stalls.
  # Caught on prov #55 (8d85a64cb8807cdc, 2026-05-12). The DaemonSet
  # is started with kube-proxy-replacement off → NodePort off → masq off
  # incompat. Both single-node and multi-node Sovereigns hit this once
  # cluster-autoscaler scales workers.
  kubeProxyReplacement: true
  # k8sServiceHost: CP's stable private IP on the Sovereign's 10.0.1.0/24
  # subnet (cp1=10.0.1.2 per infra/hetzner/main.tf). Multi-node-aware:
  # works on the CP itself (10.0.1.2 IS its own private IP) AND on
  # workers (which need the CP's IP because k3s agents do NOT expose the
  # apiserver on localhost — only the supervisor on :6444). Previously
  # `127.0.0.1` here, which crashloop'd Cilium on every worker for
  # multi-node Sovereigns (issue #733). For air-gap / multi-CP / non-
  # Hetzner Sovereigns the canonical fix is to point this at the in-
  # cluster apiserver Service (kubernetes.default.svc.cluster.local) —
  # but that requires DNS up before Cilium, which is the chicken-egg
  # this whole config exists to solve. The Sovereign template subnet
  # is fixed at 10.0.1.0/24 so 10.0.1.2 is a stable load-bearing value.
  k8sServiceHost: 10.0.1.2
  k8sServicePort: 6443

  # eBPF-host-routing — bypasses iptables stack
  bpf:
    masquerade: true
    # qa-loop iter-12 Fix #54 Workstream 2 — fresh worker pods on
    # catalyst-omantel-biz-w2/w3 could not resolve github.com (BPF map
    # capacity exhaustion when a node first joins). Pre-allocate the
    # NAT/connection BPF map slots at agent startup so DNS lookups from
    # newly-scheduled pods succeed on the first eBPF socketLB hit instead
    # of falling through to iptables fallback (which on k3s with
    # --disable-network-policy is no-op — the lookup just fails).
    #
    # Cost: ~12 MiB extra RSS per cilium-agent. Benefit: DNS lookups from
    # pods on first-join workers don't transient-fail for the 30-90s
    # window the lazy-allocate path otherwise takes (see Cilium issue
    # cilium/cilium#28456 — "DNS races during node bring-up when BPF maps
    # allocate on-demand"). Worth the RSS cost; canonical fix per upstream.
    preallocateMaps: true

  # qa-loop iter-12 Fix #54 Workstream 2 — socket-level load-balancing
  # MUST stay in the host network namespace (default true is correct, but
  # we pin the value explicitly so a future upstream default-flip doesn't
  # silently re-introduce the per-pod-netns kube-proxy-replacement DNS
  # race). When socketLB runs in the pod netns (hostNamespaceOnly: false),
  # a brand-new pod's first DNS query can race with cilium-agent's
  # service-map sync and miss kube-dns; the lookup then falls back to
  # the iptables stack which on a kube-proxy-replacement cluster is
  # absent → DNS fails. Keeping socketLB host-only ensures every
  # pod-egress lookup goes through the cilium-agent's already-synced
  # service map.
  socketLB:
    hostNamespaceOnly: true

  # IPAM — k8s mode uses the k8s pod CIDR allocator (works with k3s defaults)
  ipam:
    mode: kubernetes

  # WireGuard mTLS for transparent encryption between every pod (per
  # platform/cilium/README.md "mTLS via WireGuard"). This is the
  # CANONICAL east-west mTLS layer for OpenOva Sovereign clusters —
  # 100% mesh enforcement at the kernel transport level. SPIFFE-based
  # workload identity is intentionally NOT enabled here; bp-spire was
  # dropped from the bootstrap-kit (founder direction 2026-05-03)
  # because (a) Cilium WireGuard already gives every connection
  # transparent encryption + node-level identity, and (b) ESO→OpenBao
  # auth uses K8s ServiceAccount auth method (TokenReview), not
  # JWT-SVID. Federation across Sovereigns or per-workload-fingerprint
  # attestation can re-introduce SPIRE later as an opt-in blueprint.
  encryption:
    enabled: true
    type: wireguard
    nodeEncryption: true

  # Hubble — network observability (L3/L4/L7 flows, DNS, drop, http metrics).
  #
  # DEFAULT OFF per docs/BLUEPRINT-AUTHORING.md §11.2 (Observability toggles
  # must default false) + INVIOLABLE-PRINCIPLES.md #4 (never hardcode).
  #
  # Hubble relay/ui depend on the kube-prometheus-stack
  # `monitoring.coreos.com/v1` ServiceMonitor CRD AND on Hubble metrics
  # being scraped — neither exists on a fresh Sovereign before the
  # observability tier (Application Blueprint bp-kube-prometheus-stack)
  # is reconciled. Worse: enabling `hubble.metrics.enabled` with values
  # like `dns:query;ignoreAAAA` causes the upstream Cilium chart to render
  # a ServiceMonitor unconditionally (verified on omantel.omani.works
  # 2026-04-29 — install fails with "no matches for kind ServiceMonitor in
  # version monitoring.coreos.com/v1").
  #
  # `metrics.enabled: null` (NOT [] and NOT a populated list) is the
  # exact value that makes the upstream Cilium chart skip the metrics
  # ServiceMonitor template branch — verified by reading the upstream
  # cilium 1.19.3 chart's _hubble.tpl.
  #
  # Operator opts in via clusters/<sovereign>/bootstrap-kit/01-cilium.yaml
  # values overlay once bp-kube-prometheus-stack reconciles (issue #182).
  hubble:
    enabled: true
    relay:
      # Default false — every Sovereign in production runs without the relay.
      # EPIC-5 (#1100) flips this on per-Sovereign via clusters/<sov>/bootstrap-kit/01-cilium.yaml
      # values overlay alongside the catalystOverlay.hubbleUI block below.
      enabled: false
    ui:
      enabled: false
    metrics:
      # `null` — not a list. Disables the upstream chart's metrics
      # ServiceMonitor template entirely.
      enabled: null
      serviceMonitor:
        enabled: false

  # Gateway API — replaces traditional ingress controllers
  gatewayAPI:
    enabled: true
    gatewayClass:
      # Force GatewayClass creation regardless of CRD presence at Helm
      # render time. Upstream default is "auto" which skips creation when
      # the gateway.networking.k8s.io CRDs are absent (Capabilities check
      # at helm install time). During bootstrap, bp-gateway-api runs AFTER
      # bp-cilium, so "auto" silently skips the GatewayClass — leaving
      # every HTTPRoute orphaned until the operator is manually restarted.
      # Forcing "true" ensures GatewayClass/cilium is always created.
      # Permanent fix for the cilium-gateway race (issue #503).
      create: "true"
    # hostNetwork mode — bind gateway listeners directly to the host's
    # 80/443 ports (instead of Service-of-type-LoadBalancer with random
    # nodePort allocation). Required for the Hetzner Cloud Load Balancer
    # forwarding chain: HCLB listens on public 80/443 and forwards via
    # the private network to the CP node's 80/443 (NOT to a random
    # nodePort). With nodePort mode, Cilium's BPF L7LB Proxy Port doesn't
    # match what cilium-envoy actually binds, breaking TLS termination
    # entirely (cilium service list shows the redirect, but envoy isn't
    # listening on the proxy port). Caught live on otech45 — TCP connects
    # but TLS handshake never completes.
    hostNetwork:
      enabled: true
      # nodes.matchLabels MUST be non-empty for Cilium hostNetwork
      # mode to activate. An empty selector becomes the configmap key
      # `gateway-api-hostnetwork-nodelabelselector =` (empty value),
      # which Cilium docs explicitly call out as DISABLED state:
      # https://docs.cilium.io/en/stable/network/servicemesh/gateway-api/host-network/
      # Without this, the Gateway listener config has empty bind
      # address, eBPF never programs a redirect for the gateway
      # NodePorts, and incoming traffic on 30443/30080 never reaches
      # envoy. Hetzner LB health checks against those NodePorts
      # report `unhealthy` indefinitely.
      #
      # Caught live on prov #76 (2026-05-14): TLS handshake initiates
      # at the Hetzner LB, then SSL_ERROR_SYSCALL because no backend
      # answers — envoy is running with the listener config but not
      # bound to a socket because hostNetwork mode is silently OFF.
      #
      # Match labels are conservative — `kubernetes.io/os: linux` is
      # universally present on every k3s node we ship, so the gateway
      # listener is exposed on every node in single- and multi-CP
      # Sovereign topologies. Operators that want gateway-only-on-
      # primary-region can tighten this in their per-cluster overlay.
      nodes:
        matchLabels:
          kubernetes.io/os: linux

  # L7 proxy via Envoy — for HTTPRoute, gRPCRoute, L7 NetworkPolicy
  envoy:
    enabled: true

  # L2 LoadBalancer announcements (alternative to MetalLB for bare-metal)
  l2announcements:
    enabled: false  # Catalyst uses Hetzner LB on cloud Sovereigns; flip true for bare-metal

  # Operator HA — single replica is fine for solo, bump to 2-3 for HA
  operator:
    replicas: 1

  # Pod resource requests/limits — tuned for k3s on cx32 (8 vCPU / 16 GB)
  resources:
    requests:
      cpu: 100m
      memory: 256Mi
    limits:
      memory: 1Gi

  # Prometheus scraping + ServiceMonitor — DEFAULT OFF.
  #
  # Per docs/INVIOLABLE-PRINCIPLES.md #4 (never hardcode) and
  # docs/BLUEPRINT-AUTHORING.md §11.2 (Observability toggles must default
  # false): observability toggles MUST default `false` because the
  # `monitoring.coreos.com/v1` CRDs that back ServiceMonitor / PrometheusRule
  # ship with kube-prometheus-stack — a second-tier Application Blueprint
  # that itself depends on the bootstrap-kit (cilium first). Defaulting
  # `serviceMonitor.enabled: true` here creates a circular CRD dependency:
  # bp-cilium wants the CRD that bp-kube-prometheus-stack provides, but
  # bp-kube-prometheus-stack cannot install before bp-cilium. The
  # `trustCRDsExist=true` mitigation only suppresses Helm's render-time
  # gate; at install time the apiserver still rejects ServiceMonitor with
  # "no matches for kind ServiceMonitor in version monitoring.coreos.com/v1
  # — ensure CRDs are installed first" (verified on omantel.omani.works
  # 2026-04-29, issue #182).
  #
  # Operator opts in once the observability tier is reconciled. The opt-in
  # is a per-cluster values overlay at clusters/<sovereign>/bootstrap-kit/
  # 01-cilium.yaml `spec.values.cilium.prometheus.serviceMonitor.enabled:
  # true` — no rebuild of the Blueprint OCI artifact is required.
  prometheus:
    enabled: false
    serviceMonitor:
      enabled: false

# ─── Catalyst overlay templates (chart/templates/) ─────────────────────
#
# Templates that ride alongside the upstream cilium subchart. Every block
# is gated by an explicit `enabled: false` so installing the chart never
# regresses production behaviour — operator opts in via per-Sovereign
# overlay at clusters/<sov>/bootstrap-kit/01-cilium.yaml.

catalystOverlay:
  # Hubble UI exposed via Cilium Gateway with Keycloak OIDC. Activated
  # by EPIC-5 (#1100) as part of the zero-trust observability roll-out.
  #
  # Prerequisite to enable:
  #   1. cilium.hubble.relay.enabled = true
  #   2. cilium.hubble.ui.enabled    = true
  #   3. Keycloak realm has a `hubble-ui` OIDC client (ConfigMap
  #      configmap-sovereign-realm.yaml in bp-keycloak — wired by
  #      slice D1).
  #   4. Cilium Gateway exists with the wildcard cert (every Sovereign
  #      has this from clusters/_template/sovereign-tls/).
  hubbleUI:
    enabled: false
    # Hostname under the Sovereign domain. Per docs/NAMING-CONVENTION.md
    # §5.1 the canonical control-plane DNS pattern is
    # `{component}.{location-code}.{sovereign-domain}`, but every existing
    # bootstrap-kit HTTPRoute (gitea, auth/keycloak, grafana, harbor,
    # openbao, powerdns, console/catalyst-platform) uses the flat
    # `{component}.{sovereign-domain}` form because the Cilium Gateway's
    # wildcard cert + listener is `*.${SOVEREIGN_FQDN}` — a single-level
    # wildcard. Multi-level forms like `hubble.console.<fqdn>` would NOT
    # match `*.<fqdn>` (TLS wildcards are single-label). For consistency
    # this overlay uses the flat form.
    #
    # Two ways to populate this:
    #   1. Set hostname directly (full FQDN, e.g. "hubble.omantel.biz").
    #   2. Set sovereignFQDN (e.g. "omantel.biz") and the chart derives
    #      `hubble.<sovereignFQDN>` automatically — preferred for Flux
    #      envsubst flows where the Sovereign FQDN is the single source.
    # Explicit hostname wins when both are set.
    hostname: ""
    # Optional: when set AND hostname is empty, the chart auto-derives
    # `hubble.<sovereignFQDN>` so per-Sovereign overlays only need to
    # forward ${SOVEREIGN_FQDN} once (matches the gitea/auth/grafana
    # bootstrap-kit pattern). Empty = no auto-derivation.
    sovereignFQDN: ""
    # Reference to the Cilium Gateway that fronts this Sovereign's wildcard
    # cert. Defaults match clusters/_template/sovereign-tls/cilium-gateway.yaml,
    # which installs the Gateway into kube-system (NOT a separate
    # cilium-gateway namespace — the previous default was a stale leftover
    # from an earlier design that never landed).
    gatewayRef:
      name: cilium-gateway
      namespace: kube-system
    # Authentication mode at the gateway boundary. `oidc` requires the
    # Keycloak realm to have an `hubble-ui` client (slice D1). `none` is
    # only acceptable for fully air-gapped lab Sovereigns; per-Sovereign
    # overlays flip to `oidc` once the realm-config wires the client.
    auth: oidc
    # Hubble UI Service ref. The bp-cilium HelmRelease installs Cilium
    # into kube-system (clusters/_template/bootstrap-kit/01-cilium.yaml
    # `targetNamespace: kube-system`), so the upstream subchart renders
    # the hubble-ui Service in kube-system. The previous default
    # `namespace: cilium` would have routed traffic to a non-existent
    # Service on every Sovereign.
    serviceRef:
      name: hubble-ui
      namespace: kube-system
      port: 80