Add Apache status monitoring to Helm chart

.gitea/workflows/container.yml

@@ -57,6 +57,22 @@ jobs:
         run: |
           set -euo pipefail
           docker run --rm --entrypoint httpd "${IMAGE}:${COMMIT_SHA}" -t
+          docker run --rm \
+            --env APACHE_SERVER_ADMIN=ci@example.invalid \
+            --entrypoint /bin/bash \
+            "${IMAGE}:${COMMIT_SHA}" \
+            -euc '
+              httpd
+              php -r '\''
+                $status = file_get_contents("http://127.0.0.1:8080/server-status?auto");
+                if ($status === false || !str_contains($status, "ServerVersion:")) {
+                    fwrite(STDERR, "Apache server-status smoke test failed.\n");
+                    exit(1);
+                }
+                echo "Apache server-status smoke test passed.\n";
+              '\''
+              httpd -k stop
+            '
           docker run --rm --entrypoint php "${IMAGE}:${COMMIT_SHA}" -r '
             foreach (["exif", "gd", "intl", "Zend OPcache", "pgsql", "pdo_pgsql"] as $extension) {
                 if (!extension_loaded($extension)) {

Dockerfile

@@ -1,5 +1,8 @@
 FROM archlinux:latest
 
+ENV APACHE_SERVER_ADMIN=webmaster@localhost \
+    APACHE_STATUS_PORT=8080
+
 RUN pacman -Syu --noconfirm apache php-apache php php-gd php-pgsql \
  && pacman -Sc --noconfirm \
  && rm -v /var/log/pacman.log /var/lib/pacman/sync/*
@@ -10,9 +13,11 @@ RUN sed -i '/mod_mpm_event/ s/^/#/' /etc/httpd/conf/httpd.conf \
  && sed -i -E 's#^(\s*ErrorLog\s+)(["'\'']?)[^"'\''[:space:]]+\2#\1/dev/null#' /etc/httpd/conf/httpd.conf \
  && sed -i '/<Directory[[:space:]]\+"\/srv\/http">/,/<\/Directory>/ s/^\([[:space:]]*AllowOverride[[:space:]]*\)None/\1All/' /etc/httpd/conf/httpd.conf \
  && sed -i '/mod_rewrite/ s/^#//' /etc/httpd/conf/httpd.conf \
+ && sed -i '/status_module/ s/^#//' /etc/httpd/conf/httpd.conf \
  && echo LoadModule php_module modules/libphp.so >> /etc/httpd/conf/httpd.conf \
  && echo AddHandler php-script .php >> /etc/httpd/conf/httpd.conf \
  && echo Include conf/extra/php_module.conf >> /etc/httpd/conf/httpd.conf \
+ && echo Include conf/extra/apache-status.conf >> /etc/httpd/conf/httpd.conf \
  && install -d /etc/php/conf.d \
  && sed -i -e '/extension=exif/ s/^;//' \
            -e '/extension=gd/ s/^;//' \
@@ -22,5 +27,6 @@ RUN sed -i '/mod_mpm_event/ s/^/#/' /etc/httpd/conf/httpd.conf \
            /etc/php/php.ini
 
 COPY opcache.ini /etc/php/conf.d/opcache.ini
+COPY apache-status.conf /etc/httpd/conf/extra/apache-status.conf
 
 ENTRYPOINT ["httpd", "-DFOREGROUND"]

README.md

@@ -9,6 +9,10 @@ and a production-sized PHP OPcache. Timestamp checks remain enabled so edits
 made through the browser-based development container and WordPress updates on
 the shared CephFS volume become visible without restarting Apache.
 
+Apache exposes extended `mod_status` data only on pod-local loopback port 8080.
+The Helm chart can add an Apache exporter sidecar, a `ServiceMonitor`, and
+portable alert rules without exposing the detailed status page to the cluster.
+
 ## Continuous integration
 
 The Gitea Actions workflow builds and tests the image on every push. It also
@@ -46,7 +50,7 @@ updating the Helm repository and upgrading a release selects the image from the
 new chart:
 
 ```sh
-helm repo add brunner https://code.brunner.ninja/api/packages/feedc0de/helm
+helm repo add brunner https://brunner.ninja/charts
 helm repo update brunner
 helm upgrade --install example-site brunner/php-homepage -f values.yaml
 ```

apache-status.conf

@@ -0,0 +1,18 @@
+# Values are supplied as environment variables so Kubernetes can configure
+# each site without replacing Apache's complete configuration.
+ServerName localhost
+ServerAdmin ${APACHE_SERVER_ADMIN}
+
+# Keep the detailed status page inside the pod. The Prometheus exporter
+# sidecar shares the pod network namespace and is the only intended client.
+Listen 127.0.0.1:${APACHE_STATUS_PORT}
+ExtendedStatus On
+
+<VirtualHost 127.0.0.1:${APACHE_STATUS_PORT}>
+    ServerName localhost
+
+    <Location "/server-status">
+        SetHandler server-status
+        Require local
+    </Location>
+</VirtualHost>

helm/php-homepage/README.md

@@ -13,7 +13,7 @@ upgrade also upgrades the runtime image.
 Add and refresh the Gitea chart repository with:
 
 ```sh
-helm repo add brunner https://code.brunner.ninja/api/packages/feedc0de/helm
+helm repo add brunner https://brunner.ninja/charts
 helm repo update brunner
 ```
 
@@ -40,6 +40,7 @@ helm upgrade --install example-site brunner/php-homepage \
   --namespace default \
   --values /path/to/example-site/values.yaml \
   --take-ownership \
+  --force-conflicts \
   --wait
 ```
 
@@ -55,7 +56,21 @@ With `persistence.existingClaim` empty, the chart creates an RWX PVC using
 uninstall does not discard site content. Set `persistence.retain: false` only
 when deleting the release should also delete its dynamically created claim.
 
-Ingress, TLS, probes, scheduling, storage size, and the image tag are all
-configurable through `values.yaml`. TCP probes are used by default because
-individual websites may redirect `/`, require authentication, or select their
-own HTTP status behavior.
+Ingress, TLS, probes, scheduling, storage size, Apache's `ServerAdmin`, and the
+image tag are all configurable through `values.yaml`. TCP startup, readiness,
+and liveness probes are enabled by default because individual websites may
+redirect `/`, require authentication, or select their own HTTP status behavior.
+
+## Apache metrics
+
+The image provides an extended `mod_status` endpoint on pod-local loopback port
+8080. It is deliberately not exposed directly by the Service because extended
+status can contain current client and request details.
+
+Set `metrics.enabled` to add an Apache exporter sidecar and expose its sanitized
+Prometheus metrics on the Service. `metrics.serviceMonitor.enabled` and
+`metrics.prometheusRule.enabled` render the corresponding Prometheus Operator
+resources when their CRDs exist. The rules cover exporter scrape failures,
+Apache status failures, and worker saturation. No Prometheus namespace, release
+label, or Alertmanager is hard-coded; selection and alert routing remain the
+responsibility of the cluster's Prometheus installation.

helm/php-homepage/templates/deployment.yaml

@@ -36,10 +36,18 @@ spec:
         - name: {{ include "php-homepage.fullname" . }}
           image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
+          env:
+            - name: APACHE_SERVER_ADMIN
+              value: {{ .Values.apache.serverAdmin | quote }}
+            - name: APACHE_STATUS_PORT
+              value: {{ .Values.apache.statusPort | quote }}
           ports:
             - name: http
               containerPort: 80
               protocol: TCP
+            - name: apache-status
+              containerPort: {{ .Values.apache.statusPort }}
+              protocol: TCP
           {{- if .Values.probes.startup.enabled }}
           startupProbe:
             tcpSocket:
@@ -73,6 +81,36 @@ spec:
             - name: webroot
               mountPath: {{ .Values.persistence.mountPath }}
           {{- end }}
+        {{- if .Values.metrics.enabled }}
+        - name: apache-exporter
+          image: "{{ .Values.metrics.image.repository }}:{{ .Values.metrics.image.tag }}"
+          imagePullPolicy: {{ .Values.metrics.image.pullPolicy }}
+          args:
+            - "--scrape_uri=http://127.0.0.1:{{ .Values.apache.statusPort }}/server-status?auto"
+            - "--web.listen-address=:{{ .Values.metrics.port }}"
+          ports:
+            - name: metrics
+              containerPort: {{ .Values.metrics.port }}
+              protocol: TCP
+          readinessProbe:
+            httpGet:
+              path: /metrics
+              port: metrics
+            periodSeconds: 10
+            timeoutSeconds: 2
+            failureThreshold: 3
+          livenessProbe:
+            httpGet:
+              path: /metrics
+              port: metrics
+            periodSeconds: 30
+            timeoutSeconds: 2
+            failureThreshold: 3
+          {{- with .Values.metrics.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        {{- end }}
       {{- if .Values.persistence.enabled }}
       volumes:
         - name: webroot

helm/php-homepage/templates/prometheusrule.yaml

@@ -0,0 +1,49 @@
+{{- if and .Values.metrics.enabled .Values.metrics.prometheusRule.enabled (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/PrometheusRule") }}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: {{ include "php-homepage.fullname" . }}
+  labels:
+    {{- include "php-homepage.labels" . | nindent 4 }}
+    {{- with .Values.metrics.prometheusRule.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- with .Values.metrics.prometheusRule.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  groups:
+    - name: {{ include "php-homepage.fullname" . }}.apache
+      rules:
+        {{- if .Values.metrics.prometheusRule.exporterDown.enabled }}
+        - alert: ApacheExporterDown
+          expr: up{namespace={{ .Release.Namespace | quote }},service={{ include "php-homepage.fullname" . | quote }},endpoint="metrics"} == 0
+          for: {{ .Values.metrics.prometheusRule.exporterDown.for }}
+          labels:
+            severity: {{ .Values.metrics.prometheusRule.exporterDown.severity | quote }}
+          annotations:
+            summary: {{ printf "Apache exporter for %s is unavailable" (include "php-homepage.fullname" .) | quote }}
+            description: {{ printf "Prometheus has not been able to scrape the Apache exporter in namespace %s." .Release.Namespace | quote }}
+        {{- end }}
+        {{- if .Values.metrics.prometheusRule.apacheDown.enabled }}
+        - alert: ApacheDown
+          expr: apache_up{namespace={{ .Release.Namespace | quote }},service={{ include "php-homepage.fullname" . | quote }}} == 0
+          for: {{ .Values.metrics.prometheusRule.apacheDown.for }}
+          labels:
+            severity: {{ .Values.metrics.prometheusRule.apacheDown.severity | quote }}
+          annotations:
+            summary: {{ printf "Apache for %s is unavailable" (include "php-homepage.fullname" .) | quote }}
+            description: {{ printf "The exporter cannot read Apache server-status in namespace %s." .Release.Namespace | quote }}
+        {{- end }}
+        {{- if .Values.metrics.prometheusRule.workerSaturation.enabled }}
+        - alert: ApacheWorkerSaturation
+          expr: (sum(apache_workers{namespace={{ .Release.Namespace | quote }},service={{ include "php-homepage.fullname" . | quote }},state="busy"}) / clamp_min(sum(apache_workers{namespace={{ .Release.Namespace | quote }},service={{ include "php-homepage.fullname" . | quote }}}), 1)) > {{ .Values.metrics.prometheusRule.workerSaturation.threshold }}
+          for: {{ .Values.metrics.prometheusRule.workerSaturation.for }}
+          labels:
+            severity: {{ .Values.metrics.prometheusRule.workerSaturation.severity | quote }}
+          annotations:
+            summary: {{ printf "Apache workers for %s are nearly exhausted" (include "php-homepage.fullname" .) | quote }}
+            description: {{ printf "The ratio of busy to available Apache workers has exceeded the configured threshold (%v) for the configured duration." .Values.metrics.prometheusRule.workerSaturation.threshold | quote }}
+        {{- end }}
+{{- end }}

helm/php-homepage/templates/service.yaml

@@ -11,5 +11,11 @@ spec:
       port: {{ .Values.service.port }}
       protocol: TCP
       targetPort: http
+    {{- if .Values.metrics.enabled }}
+    - name: metrics
+      port: {{ .Values.metrics.port }}
+      protocol: TCP
+      targetPort: metrics
+    {{- end }}
   selector:
     {{- include "php-homepage.selectorLabels" . | nindent 4 }}

helm/php-homepage/templates/servicemonitor.yaml

@@ -0,0 +1,24 @@
+{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/ServiceMonitor") }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "php-homepage.fullname" . }}
+  labels:
+    {{- include "php-homepage.labels" . | nindent 4 }}
+    {{- with .Values.metrics.serviceMonitor.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- with .Values.metrics.serviceMonitor.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "php-homepage.selectorLabels" . | nindent 6 }}
+  endpoints:
+    - port: metrics
+      path: /metrics
+      interval: {{ .Values.metrics.serviceMonitor.interval }}
+      scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }}
+{{- end }}

helm/php-homepage/values.schema.json

@@ -1,10 +1,18 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
   "type": "object",
-  "required": ["image", "persistence", "service", "ingress"],
+  "required": ["apache", "image", "metrics", "persistence", "service", "ingress"],
   "properties": {
     "replicaCount": { "type": "integer", "minimum": 1 },
     "fullnameOverride": { "type": "string" },
+    "apache": {
+      "type": "object",
+      "required": ["serverAdmin", "statusPort"],
+      "properties": {
+        "serverAdmin": { "type": "string", "minLength": 3 },
+        "statusPort": { "type": "integer", "minimum": 1, "maximum": 65535 }
+      }
+    },
     "image": {
       "type": "object",
       "required": ["repository", "tag", "pullPolicy"],
@@ -14,6 +22,39 @@
         "pullPolicy": { "enum": ["Always", "IfNotPresent", "Never"] }
       }
     },
+    "metrics": {
+      "type": "object",
+      "required": ["enabled", "image", "port", "serviceMonitor", "prometheusRule"],
+      "properties": {
+        "enabled": { "type": "boolean" },
+        "port": { "type": "integer", "minimum": 1, "maximum": 65535 },
+        "image": {
+          "type": "object",
+          "required": ["repository", "tag", "pullPolicy"],
+          "properties": {
+            "repository": { "type": "string", "minLength": 1 },
+            "tag": { "type": "string", "minLength": 1 },
+            "pullPolicy": { "enum": ["Always", "IfNotPresent", "Never"] }
+          }
+        },
+        "serviceMonitor": {
+          "type": "object",
+          "required": ["enabled", "interval", "scrapeTimeout"],
+          "properties": {
+            "enabled": { "type": "boolean" },
+            "interval": { "type": "string", "minLength": 2 },
+            "scrapeTimeout": { "type": "string", "minLength": 2 }
+          }
+        },
+        "prometheusRule": {
+          "type": "object",
+          "required": ["enabled"],
+          "properties": {
+            "enabled": { "type": "boolean" }
+          }
+        }
+      }
+    },
     "persistence": {
       "type": "object",
       "required": ["enabled", "existingClaim", "mountPath"],

helm/php-homepage/values.yaml

@@ -16,6 +16,10 @@ fullnameOverride: ""
 deploymentAnnotations: {}
 podAnnotations: {}
 
+apache:
+  serverAdmin: webmaster@localhost
+  statusPort: 8080
+
 strategy:
   type: RollingUpdate
   rollingUpdate:
@@ -26,6 +30,38 @@ service:
   type: ClusterIP
   port: 80
 
+metrics:
+  enabled: false
+  image:
+    repository: quay.io/lusitaniae/apache-exporter
+    tag: v1.1.1
+    pullPolicy: IfNotPresent
+  port: 9117
+  resources: {}
+  serviceMonitor:
+    enabled: false
+    interval: 30s
+    scrapeTimeout: 10s
+    labels: {}
+    annotations: {}
+  prometheusRule:
+    enabled: false
+    labels: {}
+    annotations: {}
+    exporterDown:
+      enabled: true
+      for: 5m
+      severity: warning
+    apacheDown:
+      enabled: true
+      for: 5m
+      severity: critical
+    workerSaturation:
+      enabled: true
+      threshold: 0.9
+      for: 10m
+      severity: warning
+
 persistence:
   enabled: true
   existingClaim: ""