From da96c83f00a257ac6797632e8fa10018df9f4a33 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 31 Jan 2026 19:51:07 +0200 Subject: Update content for html --- ...5-07-14-f3s-kubernetes-with-freebsd-part-6.html | 12 +- gemfeed/DRAFT-ipv6test-deployment.html | 322 +++++++++++++++++++++ gemfeed/atom.xml | 14 +- 3 files changed, 337 insertions(+), 11 deletions(-) create mode 100644 gemfeed/DRAFT-ipv6test-deployment.html (limited to 'gemfeed') diff --git a/gemfeed/2025-07-14-f3s-kubernetes-with-freebsd-part-6.html b/gemfeed/2025-07-14-f3s-kubernetes-with-freebsd-part-6.html index 7e293951..9bb66851 100644 --- a/gemfeed/2025-07-14-f3s-kubernetes-with-freebsd-part-6.html +++ b/gemfeed/2025-07-14-f3s-kubernetes-with-freebsd-part-6.html @@ -1225,6 +1225,8 @@ paul@f0:~ % doas sh -c 'for client in r0 r1 r2 earth; do < -subj "/C=US/ST=State/L=City/O=F3S Storage/CN=${client}.lan.buetow.org" openssl x509 -req -days 3650 -in ${client}.csr -CA ca-cert.pem \ -CAkey ca-key.pem -CAcreateserial -out ${client}-cert.pem + # Combine cert and key into a single file for stunnel client + cat ${client}-cert.pem ${client}-key.pem > ${client}-stunnel.pem done'
@@ -1713,12 +1715,12 @@ http://www.gnu.org/software/src-highlite --> [root@r0 ~]# dnf install -y stunnel nfs-utils # Copy client certificate and CA certificate from f0 -[root@r0 ~]# scp f0:/usr/local/etc/stunnel/ca/r0-key.pem /etc/stunnel/ +[root@r0 ~]# scp f0:/usr/local/etc/stunnel/ca/r0-stunnel.pem /etc/stunnel/ [root@r0 ~]# scp f0:/usr/local/etc/stunnel/ca/ca-cert.pem /etc/stunnel/ # Configure stunnel client with certificate authentication [root@r0 ~]# tee /etc/stunnel/stunnel.conf <<'EOF' -cert = /etc/stunnel/r0-key.pem +cert = /etc/stunnel/r0-stunnel.pem CAfile = /etc/stunnel/ca-cert.pem client = yes verify = 2 @@ -1734,7 +1736,7 @@ EOF # Repeat for r1 and r2 with their respective certificates
-Note: Each client must use its certificate file (r0-key.pem, r1-key.pem, r2-key.pem, or earth-key.pem - the latter is for my Laptop, which can also mount the NFS shares).
+Note: Each client must use its certificate file (r0-stunnel.pem, r1-stunnel.pem, r2-stunnel.pem, or earth-stunnel.pem - the latter is for my Laptop, which can also mount the NFS shares).

NFSv4 user mapping config on Rocky

@@ -1785,11 +1787,11 @@ http://www.gnu.org/software/src-highlite --> [root@r0 ~]# mkdir -p /data/nfs/k3svolumes # Mount through stunnel (using localhost and NFSv4) -[root@r0 ~]# mount -t nfs4 -o port=2323 127.0.0.1:/data/nfs/k3svolumes /data/nfs/k3svolumes +[root@r0 ~]# mount -t nfs4 -o port=2323 127.0.0.1:/k3svolumes /data/nfs/k3svolumes # Verify mount [root@r0 ~]# mount | grep k3svolumes -127.0.0.1:/data/nfs/k3svolumes on /data/nfs/k3svolumes +127.0.0.1:/k3svolumes on /data/nfs/k3svolumes type nfs4 (rw,relatime,vers=4.2,rsize=131072,wsize=131072, namlen=255,hard,proto=tcp,port=2323,timeo=600,retrans=2,sec=sys, clientaddr=127.0.0.1,local_lock=none,addr=127.0.0.1) diff --git a/gemfeed/DRAFT-ipv6test-deployment.html b/gemfeed/DRAFT-ipv6test-deployment.html new file mode 100644 index 00000000..9bc33ffb --- /dev/null +++ b/gemfeed/DRAFT-ipv6test-deployment.html @@ -0,0 +1,322 @@ + + + + +Deploying an IPv6 Test Service on Kubernetes + + + + + + +

Deploying an IPv6 Test Service on Kubernetes

+
+

Introduction

+
+This post covers deploying a simple IPv6/IPv4 connectivity test application to the f3s Kubernetes cluster. The application displays visitors' IP addresses and determines whether they're connecting via IPv6 or IPv4—useful for testing dual-stack connectivity.
+
+The interesting technical challenge was preserving the original client IP address through multiple reverse proxies: from the OpenBSD relayd frontends, through Traefik ingress, to the Apache CGI backend.
+
+f3s series
+
+

Architecture Overview

+
+The request flow looks like this:
+
+

+Client → relayd (OpenBSD) → Traefik (k3s) → Apache + CGI (Pod)
+

+
+Each hop needs to preserve the client's real IP address via the X-Forwarded-For header.
+
+

The Application

+
+The application is a simple Perl CGI script that:
+
+1. Detects whether the client is using IPv4 or IPv6
+2. Performs DNS lookups on client and server addresses
+3. Displays diagnostic information
+
+ +

#!/usr/bin/perl
+use strict;
+use warnings;
+
+print "Content-type: text/html\n\n";
+
+my $is_ipv4 = ($ENV{REMOTE_ADDR} =~ /(?:\d+\.){3}\d/);
+print "You are using: " . ($is_ipv4 ? "IPv4" : "IPv6") . "\n";
+print "Client address: $ENV{REMOTE_ADDR}\n";
+

+
+

Docker Image

+
+The Docker image uses Apache httpd with CGI and mod_remoteip enabled:
+
+

+FROM httpd:2.4-alpine
+
+RUN apk add --no-cache perl bind-tools
+
+# Enable CGI and remoteip modules
+RUN sed -i 's/#LoadModule cgid_module/LoadModule cgid_module/' \
+      /usr/local/apache2/conf/httpd.conf && \
+    sed -i 's/#LoadModule remoteip_module/LoadModule remoteip_module/' \
+      /usr/local/apache2/conf/httpd.conf && \
+    echo 'RemoteIPHeader X-Forwarded-For' >> /usr/local/apache2/conf/httpd.conf && \
+    echo 'RemoteIPInternalProxy 10.0.0.0/8' >> /usr/local/apache2/conf/httpd.conf && \
+    echo 'RemoteIPInternalProxy 192.168.0.0/16' >> /usr/local/apache2/conf/httpd.conf
+
+COPY index.pl /usr/local/apache2/cgi-bin/index.pl
+

+
+The key is mod_remoteip: it reads the X-Forwarded-For header and sets REMOTE_ADDR to the original client IP. The RemoteIPInternalProxy directives tell Apache which upstream proxies to trust.
+
+

Kubernetes Deployment

+
+The Helm chart is straightforward:
+
+

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ipv6test
+  namespace: services
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: ipv6test
+  template:
+    spec:
+      containers:
+      - name: ipv6test
+        image: registry.lan.buetow.org:30001/ipv6test:1.1.0
+        ports:
+        - containerPort: 80
+

+
+

Configuring Traefik to Trust Forwarded Headers

+
+By default, Traefik overwrites X-Forwarded-For with its own view of the client IP (which is the upstream proxy, not the real client). To preserve the original header, Traefik needs to trust the upstream proxies.
+
+In k3s, this is configured via a HelmChartConfig:
+
+

+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+  name: traefik
+  namespace: kube-system
+spec:
+  valuesContent: |-
+    additionalArguments:
+      - "--entryPoints.web.forwardedHeaders.trustedIPs=192.168.0.0/16,10.0.0.0/8"
+      - "--entryPoints.websecure.forwardedHeaders.trustedIPs=192.168.0.0/16,10.0.0.0/8"
+

+
+This tells Traefik to trust X-Forwarded-For headers from the WireGuard tunnel IPs (where relayd connects from) and internal pod networks.
+
+

Relayd Configuration

+
+The OpenBSD relayd proxy already sets the X-Forwarded-For header:
+
+

+http protocol "https" {
+    match request header set "X-Forwarded-For" value "$REMOTE_ADDR"
+    match request header set "X-Forwarded-Proto" value "https"
+}
+

+
+

IPv4-Only and IPv6-Only Subdomains

+
+To properly test IPv4 and IPv6 connectivity separately, three hostnames are configured:
+
+

ipv6test.f3s.buetow.org - Dual stack (A + AAAA records)
ipv4.ipv6test.f3s.buetow.org - IPv4 only (A record only)
ipv6.ipv6test.f3s.buetow.org - IPv6 only (AAAA record only)

+The NSD zone template dynamically generates the correct record types:
+
+ +

<% for my $host (@$f3s_hosts) {
+     my $is_ipv6_only = $host =~ /^ipv6\./;
+     my $is_ipv4_only = $host =~ /^ipv4\./;
+-%>
+<% unless ($is_ipv6_only) { -%>
+<%= $host %>.         300 IN A <%= $ips->{current_master}{ipv4} %>
+<% } -%>
+<% unless ($is_ipv4_only) { -%>
+<%= $host %>.         300 IN AAAA <%= $ips->{current_master}{ipv6} %>
+<% } -%>
+<% } -%>
+

+
+This ensures:
+

Hosts starting with ipv6. get only AAAA records
Hosts starting with ipv4. get only A records
All other hosts get both A and AAAA records

+The Kubernetes ingress handles all three hostnames, routing to the same backend service.
+
+

TLS Certificates with Subject Alternative Names

+
+Since Let's Encrypt validates domains via HTTP, the IPv6-only subdomain (ipv6.ipv6test.f3s.buetow.org) cannot be validated directly—Let's Encrypt's validation servers use IPv4. The solution is to include all subdomains as Subject Alternative Names (SANs) in the parent certificate.
+
+The ACME client configuration template dynamically builds the SAN list:
+
+ +

<% for my $host (@$acme_hosts) {
+     # Skip ipv4/ipv6 subdomains - they're included as SANs in parent cert
+     next if $host =~ /^(ipv4|ipv6)\./;
+-%>
+<%   my @alt_names = ("www.$host");
+     for my $sub_host (@$acme_hosts) {
+         if ($sub_host =~ /^(ipv4|ipv6)\.\Q$host\E$/) {
+             push @alt_names, $sub_host;
+         }
+     }
+-%>
+domain <%= $host %> {
+    alternative names { <%= join(' ', @alt_names) %> }
+    ...
+}
+<% } -%>
+

+
+This generates a single certificate for ipv6test.f3s.buetow.org that includes:
+

www.ipv6test.f3s.buetow.org
ipv4.ipv6test.f3s.buetow.org
ipv6.ipv6test.f3s.buetow.org

DNS and TLS Deployment

+
+The DNS records and ACME certificates are managed via Rex automation:
+
+ +

our @f3s_hosts = qw/
+    ...
+    ipv6test.f3s.buetow.org
+    ipv4.ipv6test.f3s.buetow.org
+    ipv6.ipv6test.f3s.buetow.org
+/;
+
+our @acme_hosts = qw/
+    ...
+    ipv6test.f3s.buetow.org
+    ipv4.ipv6test.f3s.buetow.org
+    ipv6.ipv6test.f3s.buetow.org
+/;
+

+
+Running rex nsd httpd acme acme_invoke relayd deploys the DNS zone, configures httpd for ACME challenges, obtains the certificates, and reloads relayd.
+
+

Testing

+
+Verify DNS records are correct:
+
+ +

$ dig ipv4.ipv6test.f3s.buetow.org A +short
+46.23.94.99
+
+$ dig ipv4.ipv6test.f3s.buetow.org AAAA +short
+(no output - IPv4 only)
+
+$ dig ipv6.ipv6test.f3s.buetow.org AAAA +short
+2a03:6000:6f67:624::99
+
+$ dig ipv6.ipv6test.f3s.buetow.org A +short
+(no output - IPv6 only)
+

+
+Verify the application shows the correct test type:
+
+ +

$ curl -s https://ipv4.ipv6test.f3s.buetow.org/cgi-bin/index.pl | grep "Test Results"
+<h3>IPv4 Only Test Results:</h3>
+

+
+The displayed IP should be the real client IP, not an internal cluster address.
+
+

W3C Compliant HTML

+
+The CGI script generates valid HTML5 that passes W3C validation. Key considerations:
+
+

Proper DOCTYPE, charset, and lang attributes
HTML-escaping command outputs (dig output contains <<>> characters)

+ +

sub html_escape {
+    my $str = shift;
+    $str =~ s/&/&amp;/g;
+    $str =~ s/</&lt;/g;
+    $str =~ s/>/&gt;/g;
+    return $str;
+}
+
+my $digremote = html_escape(`dig -x $ENV{REMOTE_ADDR}`);
+

+
+You can verify the output passes validation:
+
+W3C Validator
+
+

Summary

+
+Preserving client IP addresses through multiple reverse proxies requires configuration at each layer:
+
+1. **relayd**: Sets X-Forwarded-For header
+2. **Traefik**: Trusts headers from known proxy IPs via forwardedHeaders.trustedIPs
+3. **Apache**: Uses mod_remoteip to set REMOTE_ADDR from the header
+
+Additional challenges solved:
+
+

**TLS for IPv6-only hosts**: Use SANs to include all subdomains in a single certificate validated via the dual-stack parent domain
**W3C compliance**: HTML-escape all command outputs to handle special characters

+The configuration is managed via GitOps with ArgoCD, including the Traefik HelmChartConfig.
+
+Source code
+Kubernetes manifests
+Traefik configuration
+
+E-Mail your comments to paul@paulbias.net :-)
+
+← Back to the index
+ + + diff --git a/gemfeed/atom.xml b/gemfeed/atom.xml index 5a7541e1..d8063be1 100644 --- a/gemfeed/atom.xml +++ b/gemfeed/atom.xml @@ -1,6 +1,6 @@ - 2026-01-27T10:09:14+02:00 + 2026-01-31T19:49:46+02:00 foo.zone feed To be in the .zone! @@ -7671,6 +7671,8 @@ paul@f0:~ % doas sh -c 'for client in r0 r1 r2 earth; do < -subj "/C=US/ST=State/L=City/O=F3S Storage/CN=${client}.lan.buetow.org" openssl x509 -req -days 3650 -in ${client}.csr -CA ca-cert.pem \ -CAkey ca-key.pem -CAcreateserial -out ${client}-cert.pem + # Combine cert and key into a single file for stunnel client + cat ${client}-cert.pem ${client}-key.pem > ${client}-stunnel.pem done'
@@ -8159,12 +8161,12 @@ http://www.gnu.org/software/src-highlite --> [root@r0 ~]# dnf install -y stunnel nfs-utils # Copy client certificate and CA certificate from f0 -[root@r0 ~]# scp f0:/usr/local/etc/stunnel/ca/r0-key.pem /etc/stunnel/ +[root@r0 ~]# scp f0:/usr/local/etc/stunnel/ca/r0-stunnel.pem /etc/stunnel/ [root@r0 ~]# scp f0:/usr/local/etc/stunnel/ca/ca-cert.pem /etc/stunnel/ # Configure stunnel client with certificate authentication [root@r0 ~]# tee /etc/stunnel/stunnel.conf <<'EOF' -cert = /etc/stunnel/r0-key.pem +cert = /etc/stunnel/r0-stunnel.pem CAfile = /etc/stunnel/ca-cert.pem client = yes verify = 2 @@ -8180,7 +8182,7 @@ EOF # Repeat for r1 and r2 with their respective certificates
-Note: Each client must use its certificate file (r0-key.pem, r1-key.pem, r2-key.pem, or earth-key.pem - the latter is for my Laptop, which can also mount the NFS shares).
+Note: Each client must use its certificate file (r0-stunnel.pem, r1-stunnel.pem, r2-stunnel.pem, or earth-stunnel.pem - the latter is for my Laptop, which can also mount the NFS shares).

NFSv4 user mapping config on Rocky

@@ -8231,11 +8233,11 @@ http://www.gnu.org/software/src-highlite --> [root@r0 ~]# mkdir -p /data/nfs/k3svolumes # Mount through stunnel (using localhost and NFSv4) -[root@r0 ~]# mount -t nfs4 -o port=2323 127.0.0.1:/data/nfs/k3svolumes /data/nfs/k3svolumes +[root@r0 ~]# mount -t nfs4 -o port=2323 127.0.0.1:/k3svolumes /data/nfs/k3svolumes # Verify mount [root@r0 ~]# mount | grep k3svolumes -127.0.0.1:/data/nfs/k3svolumes on /data/nfs/k3svolumes +127.0.0.1:/k3svolumes on /data/nfs/k3svolumes type nfs4 (rw,relatime,vers=4.2,rsize=131072,wsize=131072, namlen=255,hard,proto=tcp,port=2323,timeo=600,retrans=2,sec=sys, clientaddr=127.0.0.1,local_lock=none,addr=127.0.0.1) -- cgit v1.2.3