1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
|
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>f3s: Kubernetes with FreeBSD - Part 7: k3s and first pod deployments</title>
<link rel="shortcut icon" type="image/gif" href="/favicon.ico" />
<link rel="stylesheet" href="../style.css" />
<link rel="stylesheet" href="style-override.css" />
</head>
<body>
<p class="header">
<a href="https://foo.zone">Home</a> | <a href="https://codeberg.org/snonux/foo.zone/src/branch/content-md/gemfeed/2025-10-02-f3s-kubernetes-with-freebsd-part-7.md">Markdown</a> | <a href="gemini://foo.zone/gemfeed/2025-10-02-f3s-kubernetes-with-freebsd-part-7.gmi">Gemini</a>
</p>
<h1 style='display: inline' id='f3s-kubernetes-with-freebsd---part-7-k3s-and-first-pod-deployments'>f3s: Kubernetes with FreeBSD - Part 7: k3s and first pod deployments</h1><br />
<br />
<span class='quote'>Published at 2025-10-02T11:27:19+03:00, last updated Tue 30 Dec 10:11:58 EET 2025</span><br />
<br />
<span>This is the seventh blog post about the f3s series for my self-hosting demands in a home lab. f3s? The "f" stands for FreeBSD, and the "3s" stands for k3s, the Kubernetes distribution I use on FreeBSD-based physical machines.</span><br />
<br />
<a class='textlink' href='./2024-11-17-f3s-kubernetes-with-freebsd-part-1.html'>2024-11-17 f3s: Kubernetes with FreeBSD - Part 1: Setting the stage</a><br />
<a class='textlink' href='./2024-12-03-f3s-kubernetes-with-freebsd-part-2.html'>2024-12-03 f3s: Kubernetes with FreeBSD - Part 2: Hardware and base installation</a><br />
<a class='textlink' href='./2025-02-01-f3s-kubernetes-with-freebsd-part-3.html'>2025-02-01 f3s: Kubernetes with FreeBSD - Part 3: Protecting from power cuts</a><br />
<a class='textlink' href='./2025-04-05-f3s-kubernetes-with-freebsd-part-4.html'>2025-04-05 f3s: Kubernetes with FreeBSD - Part 4: Rocky Linux Bhyve VMs</a><br />
<a class='textlink' href='./2025-05-11-f3s-kubernetes-with-freebsd-part-5.html'>2025-05-11 f3s: Kubernetes with FreeBSD - Part 5: WireGuard mesh network</a><br />
<a class='textlink' href='./2025-07-14-f3s-kubernetes-with-freebsd-part-6.html'>2025-07-14 f3s: Kubernetes with FreeBSD - Part 6: Storage</a><br />
<a class='textlink' href='./2025-10-02-f3s-kubernetes-with-freebsd-part-7.html'>2025-10-02 f3s: Kubernetes with FreeBSD - Part 7: k3s and first pod deployments (You are currently reading this)</a><br />
<a class='textlink' href='./2025-12-07-f3s-kubernetes-with-freebsd-part-8.html'>2025-12-07 f3s: Kubernetes with FreeBSD - Part 8: Observability</a><br />
<br />
<a href='./f3s-kubernetes-with-freebsd-part-1/f3slogo.png'><img alt='f3s logo' title='f3s logo' src='./f3s-kubernetes-with-freebsd-part-1/f3slogo.png' /></a><br />
<br />
<h2 style='display: inline' id='table-of-contents'>Table of Contents</h2><br />
<br />
<ul>
<li><a href='#f3s-kubernetes-with-freebsd---part-7-k3s-and-first-pod-deployments'>f3s: Kubernetes with FreeBSD - Part 7: k3s and first pod deployments</a></li>
<li>⇢ <a href='#introduction'>Introduction</a></li>
<li>⇢ <a href='#important-note-gitops-migration'>Important Note: GitOps Migration</a></li>
<li>⇢ <a href='#updating'>Updating</a></li>
<li>⇢ <a href='#installing-k3s'>Installing k3s</a></li>
<li>⇢ ⇢ <a href='#generating-k3stoken-and-starting-the-first-k3s-node'>Generating <span class='inlinecode'>K3S_TOKEN</span> and starting the first k3s node</a></li>
<li>⇢ ⇢ <a href='#adding-the-remaining-nodes-to-the-cluster'>Adding the remaining nodes to the cluster</a></li>
<li>⇢ <a href='#test-deployments'>Test deployments</a></li>
<li>⇢ ⇢ <a href='#test-deployment-to-kubernetes'>Test deployment to Kubernetes</a></li>
<li>⇢ ⇢ <a href='#test-deployment-with-persistent-volume-claim'>Test deployment with persistent volume claim</a></li>
<li>⇢ ⇢ <a href='#scaling-traefik-for-faster-failover'>Scaling Traefik for faster failover</a></li>
<li>⇢ <a href='#make-it-accessible-from-the-public-internet'>Make it accessible from the public internet</a></li>
<li>⇢ ⇢ <a href='#openbsd-relayd-configuration'>OpenBSD relayd configuration</a></li>
<li>⇢ ⇢ <a href='#automatic-failover-when-f3s-cluster-is-down'>Automatic failover when f3s cluster is down</a></li>
<li>⇢ ⇢ <a href='#openbsd-httpd-fallback-configuration'>OpenBSD httpd fallback configuration</a></li>
<li>⇢ <a href='#exposing-services-via-lan-ingress'>Exposing services via LAN ingress</a></li>
<li>⇢ ⇢ <a href='#architecture-overview'>Architecture overview</a></li>
<li>⇢ ⇢ <a href='#installing-cert-manager'>Installing cert-manager</a></li>
<li>⇢ ⇢ <a href='#configuring-freebsd-relayd-for-lan-access'>Configuring FreeBSD relayd for LAN access</a></li>
<li>⇢ ⇢ <a href='#adding-lan-ingress-to-services'>Adding LAN ingress to services</a></li>
<li>⇢ ⇢ <a href='#client-side-dns-and-ca-setup'>Client-side DNS and CA setup</a></li>
<li>⇢ ⇢ <a href='#scaling-to-other-services'>Scaling to other services</a></li>
<li>⇢ ⇢ <a href='#tls-offloaders-summary'>TLS offloaders summary</a></li>
<li>⇢ <a href='#deploying-the-private-docker-image-registry'>Deploying the private Docker image registry</a></li>
<li>⇢ ⇢ <a href='#prepare-the-nfs-backed-storage'>Prepare the NFS-backed storage</a></li>
<li>⇢ ⇢ <a href='#install-or-upgrade-the-chart'>Install (or upgrade) the chart</a></li>
<li>⇢ ⇢ <a href='#allow-nodes-and-workstations-to-trust-the-registry'>Allow nodes and workstations to trust the registry</a></li>
<li>⇢ ⇢ <a href='#pushing-and-pulling-images'>Pushing and pulling images</a></li>
<li>⇢ <a href='#example-anki-sync-server-from-the-private-registry'>Example: Anki Sync Server from the private registry</a></li>
<li>⇢ ⇢ <a href='#build-and-push-the-image'>Build and push the image</a></li>
<li>⇢ ⇢ <a href='#create-the-anki-secret-and-storage-on-the-cluster'>Create the Anki secret and storage on the cluster</a></li>
<li>⇢ ⇢ <a href='#deploy-the-chart'>Deploy the chart</a></li>
<li>⇢ <a href='#nfsv4-uid-mapping-for-postgres-backed-and-other-apps'>NFSv4 UID mapping for Postgres-backed (and other) apps</a></li>
<li>⇢ ⇢ <a href='#helm-charts-currently-in-service'>Helm charts currently in service</a></li>
</ul><br />
<h2 style='display: inline' id='introduction'>Introduction</h2><br />
<br />
<span>In this blog post, I am finally going to install k3s (the Kubernetes distribution I use) to the whole setup and deploy the first workloads (helm charts, and a private registry) to it.</span><br />
<br />
<a class='textlink' href='https://k3s.io'>https://k3s.io</a><br />
<br />
<h2 style='display: inline' id='important-note-gitops-migration'>Important Note: GitOps Migration</h2><br />
<br />
<span>**Note:** After publishing this blog post, the f3s cluster was migrated from imperative Helm deployments to declarative GitOps using ArgoCD. The Kubernetes manifests and Helm charts in the repository have been reorganized for ArgoCD-based continuous deployment.</span><br />
<br />
<span>**To view the exact manifests and charts as they existed when this blog post was written** (before the ArgoCD migration), check out the pre-ArgoCD revision:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>$ git clone https://codeberg.org/snonux/conf.git
$ cd conf
$ git checkout 15a86f3 <i><font color="silver"># Last commit before ArgoCD migration</font></i>
$ cd f3s/
</pre>
<br />
<span>**Current master branch** contains the ArgoCD-managed versions with:</span><br />
<ul>
<li>Application manifests organized under <span class='inlinecode'>argocd-apps/{monitoring,services,infra,test}/</span></li>
<li>Additional resources under <span class='inlinecode'>*/manifests/</span> directories (e.g., <span class='inlinecode'>prometheus/manifests/</span>)</li>
<li>Justfiles updated to trigger ArgoCD syncs instead of direct Helm commands</li>
</ul><br />
<span>The deployment concepts and architecture remain the same—only the deployment method changed from imperative (<span class='inlinecode'>helm install/upgrade</span>) to declarative (GitOps with ArgoCD).</span><br />
<br />
<h2 style='display: inline' id='updating'>Updating</h2><br />
<br />
<span>Before proceeding, I bring all systems involved up-to-date. On all three Rocky Linux 9 boxes <span class='inlinecode'>r0</span>, <span class='inlinecode'>r1</span>, and <span class='inlinecode'>r2</span>:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>dnf update -y
reboot
</pre>
<br />
<span>On the FreeBSD hosts, I upgraded from FreeBSD 14.2 to 14.3-RELEASE, running this on all three hosts <span class='inlinecode'>f0</span>, <span class='inlinecode'>f1</span> and <span class='inlinecode'>f2</span>:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>paul@f0:~ % doas freebsd-update fetch
paul@f0:~ % doas freebsd-update install
paul@f0:~ % doas reboot
.
.
.
paul@f0:~ % doas freebsd-update -r <font color="#000000">14.3</font>-RELEASE upgrade
paul@f0:~ % doas freebsd-update install
paul@f0:~ % doas freebsd-update install
paul@f0:~ % doas reboot
.
.
.
paul@f0:~ % doas freebsd-update install
paul@f0:~ % doas pkg update
paul@f0:~ % doas pkg upgrade
paul@f0:~ % doas reboot
.
.
.
paul@f0:~ % uname -a
FreeBSD f0.lan.buetow.org <font color="#000000">14.3</font>-RELEASE FreeBSD <font color="#000000">14.3</font>-RELEASE
releng/<font color="#000000">14.3</font>-n<font color="#000000">271432</font>-8c9ce319fef7 GENERIC amd64
</pre>
<br />
<h2 style='display: inline' id='installing-k3s'>Installing k3s</h2><br />
<br />
<h3 style='display: inline' id='generating-k3stoken-and-starting-the-first-k3s-node'>Generating <span class='inlinecode'>K3S_TOKEN</span> and starting the first k3s node</h3><br />
<br />
<span>I generated the k3s token on my Fedora laptop with <span class='inlinecode'>pwgen -n 32</span> and selected one of the results. Then, on all three <span class='inlinecode'>r</span> hosts, I ran the following (replace SECRET_TOKEN with the actual secret):</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>[root@r0 ~]<i><font color="silver"># echo -n SECRET_TOKEN > ~/.k3s_token</font></i>
</pre>
<br />
<span>The following steps are also documented on the k3s website:</span><br />
<br />
<a class='textlink' href='https://docs.k3s.io/datastore/ha-embedded'>https://docs.k3s.io/datastore/ha-embedded</a><br />
<br />
<span>To bootstrap k3s on the first node, I ran this on <span class='inlinecode'>r0</span>:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>[root@r0 ~]<i><font color="silver"># curl -sfL https://get.k3s.io | K3S_TOKEN=$(cat ~/.k3s_token) \</font></i>
sh -s - server --cluster-init \
--node-ip=<font color="#000000">192.168</font>.<font color="#000000">2.120</font> \
--advertise-address=<font color="#000000">192.168</font>.<font color="#000000">2.120</font> \
--tls-san=r0.wg0.wan.buetow.org
[INFO] Finding release <b><u><font color="#000000">for</font></u></b> channel stable
[INFO] Using v1.<font color="#000000">32.6</font>+k3s1 as release
.
.
.
[INFO] systemd: Starting k3s
</pre>
<br />
<span>Note: The <span class='inlinecode'>--node-ip</span> and <span class='inlinecode'>--advertise-address</span> flags are important to ensure that the embedded etcd cluster communicates over the WireGuard interface (192.168.2.x) rather than the LAN interface (192.168.1.x). This ensures that all control plane traffic is encrypted via WireGuard.</span><br />
<br />
<h3 style='display: inline' id='adding-the-remaining-nodes-to-the-cluster'>Adding the remaining nodes to the cluster</h3><br />
<br />
<span>Then I ran on the other two nodes <span class='inlinecode'>r1</span> and <span class='inlinecode'>r2</span>:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>[root@r1 ~]<i><font color="silver"># curl -sfL https://get.k3s.io | K3S_TOKEN=$(cat ~/.k3s_token) \</font></i>
sh -s - server --server https://r<font color="#000000">0</font>.wg0.wan.buetow.org:<font color="#000000">6443</font> \
--node-ip=<font color="#000000">192.168</font>.<font color="#000000">2.121</font> \
--advertise-address=<font color="#000000">192.168</font>.<font color="#000000">2.121</font> \
--tls-san=r1.wg0.wan.buetow.org
[root@r2 ~]<i><font color="silver"># curl -sfL https://get.k3s.io | K3S_TOKEN=$(cat ~/.k3s_token) \</font></i>
sh -s - server --server https://r<font color="#000000">0</font>.wg0.wan.buetow.org:<font color="#000000">6443</font> \
--node-ip=<font color="#000000">192.168</font>.<font color="#000000">2.122</font> \
--advertise-address=<font color="#000000">192.168</font>.<font color="#000000">2.122</font> \
--tls-san=r2.wg0.wan.buetow.org
.
.
.
</pre>
<br />
<span>Once done, I had a three-node Kubernetes cluster control plane:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>[root@r0 ~]<i><font color="silver"># kubectl get nodes</font></i>
NAME STATUS ROLES AGE VERSION
r0.lan.buetow.org Ready control-plane,etcd,master 4m44s v1.<font color="#000000">32.6</font>+k3s1
r1.lan.buetow.org Ready control-plane,etcd,master 3m13s v1.<font color="#000000">32.6</font>+k3s1
r2.lan.buetow.org Ready control-plane,etcd,master 30s v1.<font color="#000000">32.6</font>+k3s1
[root@r0 ~]<i><font color="silver"># kubectl get pods --all-namespaces</font></i>
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system coredns-5688667fd4-fs2jj <font color="#000000">1</font>/<font color="#000000">1</font> Running <font color="#000000">0</font> 5m27s
kube-system helm-install-traefik-crd-f9hgd <font color="#000000">0</font>/<font color="#000000">1</font> Completed <font color="#000000">0</font> 5m27s
kube-system helm-install-traefik-zqqqk <font color="#000000">0</font>/<font color="#000000">1</font> Completed <font color="#000000">2</font> 5m27s
kube-system local-path-provisioner-774c6665dc-jqlnc <font color="#000000">1</font>/<font color="#000000">1</font> Running <font color="#000000">0</font> 5m27s
kube-system metrics-server-6f4c6675d5-5xpmp <font color="#000000">1</font>/<font color="#000000">1</font> Running <font color="#000000">0</font> 5m27s
kube-system svclb-traefik-411cec5b-cdp2l <font color="#000000">2</font>/<font color="#000000">2</font> Running <font color="#000000">0</font> 78s
kube-system svclb-traefik-411cec5b-f625r <font color="#000000">2</font>/<font color="#000000">2</font> Running <font color="#000000">0</font> 4m58s
kube-system svclb-traefik-411cec5b-twrd<font color="#000000">7</font> <font color="#000000">2</font>/<font color="#000000">2</font> Running <font color="#000000">0</font> 4m2s
kube-system traefik-c98fdf6fb-lt6fx <font color="#000000">1</font>/<font color="#000000">1</font> Running <font color="#000000">0</font> 4m58s
</pre>
<br />
<span>In order to connect with <span class='inlinecode'>kubectl</span> from my Fedora laptop, I had to copy <span class='inlinecode'>/etc/rancher/k3s/k3s.yaml</span> from <span class='inlinecode'>r0</span> to <span class='inlinecode'>~/.kube/config</span> and then replace the value of the server field with <span class='inlinecode'>r0.lan.buetow.org</span>. kubectl can now manage the cluster. Note that this step has to be repeated when I want to connect to another node of the cluster (e.g. when <span class='inlinecode'>r0</span> is down).</span><br />
<br />
<h2 style='display: inline' id='test-deployments'>Test deployments</h2><br />
<br />
<h3 style='display: inline' id='test-deployment-to-kubernetes'>Test deployment to Kubernetes</h3><br />
<br />
<span>Let's create a test namespace:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>> ~ kubectl create namespace <b><u><font color="#000000">test</font></u></b>
namespace/test created
> ~ kubectl get namespaces
NAME STATUS AGE
default Active 6h11m
kube-node-lease Active 6h11m
kube-public Active 6h11m
kube-system Active 6h11m
<b><u><font color="#000000">test</font></u></b> Active 5s
> ~ kubectl config set-context --current --namespace=<b><u><font color="#000000">test</font></u></b>
Context <font color="#808080">"default"</font> modified.
</pre>
<br />
<span>And let's also create an Apache test pod:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>> ~ cat <<END > apache-deployment.yaml
<i><font color="silver"># Apache HTTP Server Deployment</font></i>
apiVersion: apps/v<font color="#000000">1</font>
kind: Deployment
metadata:
name: apache-deployment
spec:
replicas: <font color="#000000">1</font>
selector:
matchLabels:
app: apache
template:
metadata:
labels:
app: apache
spec:
containers:
- name: apache
image: httpd:latest
ports:
<i><font color="silver"># Container port where Apache listens</font></i>
- containerPort: <font color="#000000">80</font>
END
> ~ kubectl apply -f apache-deployment.yaml
deployment.apps/apache-deployment created
> ~ kubectl get all
NAME READY STATUS RESTARTS AGE
pod/apache-deployment-5fd955856f-4pjmf <font color="#000000">1</font>/<font color="#000000">1</font> Running <font color="#000000">0</font> 7s
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/apache-deployment <font color="#000000">1</font>/<font color="#000000">1</font> <font color="#000000">1</font> <font color="#000000">1</font> 7s
NAME DESIRED CURRENT READY AGE
replicaset.apps/apache-deployment-5fd955856f <font color="#000000">1</font> <font color="#000000">1</font> <font color="#000000">1</font> 7s
</pre>
<br />
<span>Let's also create a service: </span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>> ~ cat <<END > apache-service.yaml
apiVersion: v1
kind: Service
metadata:
labels:
app: apache
name: apache-service
spec:
ports:
- name: web
port: <font color="#000000">80</font>
protocol: TCP
<i><font color="silver"># Expose port 80 on the service</font></i>
targetPort: <font color="#000000">80</font>
selector:
<i><font color="silver"># Link this service to pods with the label app=apache</font></i>
app: apache
END
> ~ kubectl apply -f apache-service.yaml
service/apache-service created
> ~ kubectl get service
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
apache-service ClusterIP <font color="#000000">10.43</font>.<font color="#000000">249.165</font> <none> <font color="#000000">80</font>/TCP 4s
</pre>
<br />
<span>Now let's create an ingress:</span><br />
<br />
<span class='quote'>Note: I've modified the hosts listed in this example after I published this blog post to ensure that there aren't any bots scraping it.</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>> ~ cat <<END > apache-ingress.yaml
apiVersion: networking.k8s.io/v<font color="#000000">1</font>
kind: Ingress
metadata:
name: apache-ingress
namespace: <b><u><font color="#000000">test</font></u></b>
annotations:
spec.ingressClassName: traefik
traefik.ingress.kubernetes.io/router.entrypoints: web
spec:
rules:
- host: f3s.foo.zone
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: apache-service
port:
number: <font color="#000000">80</font>
- host: standby.f3s.foo.zone
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: apache-service
port:
number: <font color="#000000">80</font>
- host: www.f3s.foo.zone
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: apache-service
port:
number: <font color="#000000">80</font>
END
> ~ kubectl apply -f apache-ingress.yaml
ingress.networking.k8s.io/apache-ingress created
> ~ kubectl describe ingress
Name: apache-ingress
Labels: <none>
Namespace: <b><u><font color="#000000">test</font></u></b>
Address: <font color="#000000">192.168</font>.<font color="#000000">2.120</font>,<font color="#000000">192.168</font>.<font color="#000000">2.121</font>,<font color="#000000">192.168</font>.<font color="#000000">2.122</font>
Ingress Class: traefik
Default backend: <default>
Rules:
Host Path Backends
---- ---- --------
f3s.foo.zone
/ apache-service:<font color="#000000">80</font> (<font color="#000000">10.42</font>.<font color="#000000">1.11</font>:<font color="#000000">80</font>)
standby.f3s.foo.zone
/ apache-service:<font color="#000000">80</font> (<font color="#000000">10.42</font>.<font color="#000000">1.11</font>:<font color="#000000">80</font>)
www.f3s.foo.zone
/ apache-service:<font color="#000000">80</font> (<font color="#000000">10.42</font>.<font color="#000000">1.11</font>:<font color="#000000">80</font>)
Annotations: spec.ingressClassName: traefik
traefik.ingress.kubernetes.io/router.entrypoints: web
Events: <none>
</pre>
<br />
<span>Notes: </span><br />
<br />
<ul>
<li>In the ingress, I use plain HTTP (web) for the Traefik rule, as all the "production" traffic will be routed through a WireGuard tunnel anyway, as I will show later.</li>
</ul><br />
<span>So I tested the Apache web server through the ingress rule:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>> ~ curl -H <font color="#808080">"Host: www.f3s.foo.zone"</font> http://r<font color="#000000">0</font>.lan.buetow.org:<font color="#000000">80</font>
<html><body><h1>It works!</h<font color="#000000">1</font>></body></html>
</pre>
<br />
<h3 style='display: inline' id='test-deployment-with-persistent-volume-claim'>Test deployment with persistent volume claim</h3><br />
<br />
<span>Next, I modified the Apache example to serve the <span class='inlinecode'>htdocs</span> directory from the NFS share I created in the previous blog post. I used the following manifests. Most of them are the same as before, except for the persistent volume claim and the volume mount in the Apache deployment.</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>> ~ cat <<END > apache-deployment.yaml
<i><font color="silver"># Apache HTTP Server Deployment</font></i>
apiVersion: apps/v<font color="#000000">1</font>
kind: Deployment
metadata:
name: apache-deployment
namespace: <b><u><font color="#000000">test</font></u></b>
spec:
replicas: <font color="#000000">2</font>
selector:
matchLabels:
app: apache
template:
metadata:
labels:
app: apache
spec:
containers:
- name: apache
image: httpd:latest
ports:
<i><font color="silver"># Container port where Apache listens</font></i>
- containerPort: <font color="#000000">80</font>
readinessProbe:
httpGet:
path: /
port: <font color="#000000">80</font>
initialDelaySeconds: <font color="#000000">5</font>
periodSeconds: <font color="#000000">10</font>
livenessProbe:
httpGet:
path: /
port: <font color="#000000">80</font>
initialDelaySeconds: <font color="#000000">15</font>
periodSeconds: <font color="#000000">10</font>
volumeMounts:
- name: apache-htdocs
mountPath: /usr/local/apache<font color="#000000">2</font>/htdocs/
volumes:
- name: apache-htdocs
persistentVolumeClaim:
claimName: example-apache-pvc
END
> ~ cat <<END > apache-ingress.yaml
apiVersion: networking.k8s.io/v<font color="#000000">1</font>
kind: Ingress
metadata:
name: apache-ingress
namespace: <b><u><font color="#000000">test</font></u></b>
annotations:
spec.ingressClassName: traefik
traefik.ingress.kubernetes.io/router.entrypoints: web
spec:
rules:
- host: f3s.foo.zone
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: apache-service
port:
number: <font color="#000000">80</font>
- host: standby.f3s.foo.zone
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: apache-service
port:
number: <font color="#000000">80</font>
- host: www.f3s.foo.zone
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: apache-service
port:
number: <font color="#000000">80</font>
END
> ~ cat <<END > apache-persistent-volume.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: example-apache-pv
spec:
capacity:
storage: 1Gi
volumeMode: Filesystem
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Retain
hostPath:
path: /data/nfs/k3svolumes/example-apache-volume-claim
<b><u><font color="#000000">type</font></u></b>: Directory
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: example-apache-pvc
namespace: <b><u><font color="#000000">test</font></u></b>
spec:
storageClassName: <font color="#808080">""</font>
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
END
> ~ cat <<END > apache-service.yaml
apiVersion: v1
kind: Service
metadata:
labels:
app: apache
name: apache-service
namespace: <b><u><font color="#000000">test</font></u></b>
spec:
ports:
- name: web
port: <font color="#000000">80</font>
protocol: TCP
<i><font color="silver"># Expose port 80 on the service</font></i>
targetPort: <font color="#000000">80</font>
selector:
<i><font color="silver"># Link this service to pods with the label app=apache</font></i>
app: apache
END
</pre>
<br />
<span>I applied the manifests:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>> ~ kubectl apply -f apache-persistent-volume.yaml
> ~ kubectl apply -f apache-service.yaml
> ~ kubectl apply -f apache-deployment.yaml
> ~ kubectl apply -f apache-ingress.yaml
</pre>
<br />
<span>Looking at the deployment, I could see it failed because the directory didn't exist yet on the NFS share (note that I also increased the replica count to 2 so if one node goes down there's already a replica running on another node for faster failover):</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>> ~ kubectl get pods
NAME READY STATUS RESTARTS AGE
apache-deployment-5b96bd6b6b-fv2jx <font color="#000000">0</font>/<font color="#000000">1</font> ContainerCreating <font color="#000000">0</font> 9m15s
apache-deployment-5b96bd6b6b-ax2ji <font color="#000000">0</font>/<font color="#000000">1</font> ContainerCreating <font color="#000000">0</font> 9m15s
> ~ kubectl describe pod apache-deployment-5b96bd6b6b-fv2jx | tail -n <font color="#000000">5</font>
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Scheduled 9m34s default-scheduler Successfully
assigned test/apache-deployment-5b96bd6b6b-fv2jx to r2.lan.buetow.org
Warning FailedMount 80s (x12 over 9m34s) kubelet MountVolume.SetUp
failed <b><u><font color="#000000">for</font></u></b> volume <font color="#808080">"example-apache-pv"</font> : hostPath <b><u><font color="#000000">type</font></u></b> check failed:
/data/nfs/k3svolumes/example-apache is not a directory
</pre>
<br />
<span>That's intentional—I needed to create the directory on the NFS share first, so I did that (e.g. on <span class='inlinecode'>r0</span>):</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>[root@r0 ~]<i><font color="silver"># mkdir /data/nfs/k3svolumes/example-apache-volume-claim/</font></i>
[root@r0 ~]<i><font color="silver"># cat <<END > /data/nfs/k3svolumes/example-apache-volume-claim/index.html</font></i>
<!DOCTYPE html>
<html>
<head>
<title>Hello, it works</title>
</head>
<body>
<h1>Hello, it works!</h<font color="#000000">1</font>>
<p>This site is served via a PVC!</p>
</body>
</html>
END
</pre>
<br />
<span>The <span class='inlinecode'>index.html</span> file gives us some actual content to serve. After deleting the pod, it recreates itself and the volume mounts correctly:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>> ~ kubectl delete pod apache-deployment-5b96bd6b6b-fv2jx
> ~ curl -H <font color="#808080">"Host: www.f3s.foo.zone"</font> http://r<font color="#000000">0</font>.lan.buetow.org:<font color="#000000">80</font>
<!DOCTYPE html>
<html>
<head>
<title>Hello, it works</title>
</head>
<body>
<h1>Hello, it works!</h<font color="#000000">1</font>>
<p>This site is served via a PVC!</p>
</body>
</html>
</pre>
<br />
<h3 style='display: inline' id='scaling-traefik-for-faster-failover'>Scaling Traefik for faster failover</h3><br />
<br />
<span>Traefik (used for ingress on k3s) ships with a single replica by default, but for faster failover I bumped it to two replicas so each worker node runs one pod. That way, if a node disappears, the service stays up while Kubernetes schedules a replacement. Here's the command I used:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>> ~ kubectl -n kube-system scale deployment traefik --replicas=<font color="#000000">2</font>
</pre>
<br />
<span>And the result:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>> ~ kubectl -n kube-system get pods -l app.kubernetes.io/name=traefik
kube-system traefik-c98fdf6fb-97kqk <font color="#000000">1</font>/<font color="#000000">1</font> Running <font color="#000000">19</font> (53d ago) 64d
kube-system traefik-c98fdf6fb-9npg2 <font color="#000000">1</font>/<font color="#000000">1</font> Running <font color="#000000">11</font> (53d ago) 61d
</pre>
<br />
<h2 style='display: inline' id='make-it-accessible-from-the-public-internet'>Make it accessible from the public internet</h2><br />
<br />
<span>Next, I made this accessible through the public internet via the <span class='inlinecode'>www.f3s.foo.zone</span> hosts. As a reminder from part 1 of this series, I reviewed the section titled "OpenBSD/relayd to the rescue for external connectivity":</span><br />
<br />
<a class='textlink' href='./2024-11-17-f3s-kubernetes-with-freebsd-part-1.html'>f3s: Kubernetes with FreeBSD - Part 1: Setting the stage</a><br />
<br />
<span class='quote'>All apps should be reachable through the internet (e.g., from my phone or computer when travelling). For external connectivity and TLS management, I've got two OpenBSD VMs (one hosted by OpenBSD Amsterdam and another hosted by Hetzner) handling public-facing services like DNS, relaying traffic, and automating Let's Encrypt certificates.</span><br />
<br />
<span class='quote'>All of this (every Linux VM to every OpenBSD box) will be connected via WireGuard tunnels, keeping everything private and secure. There will be 6 WireGuard tunnels (3 k3s nodes times two OpenBSD VMs).</span><br />
<br />
<span class='quote'>So, when I want to access a service running in k3s, I will hit an external DNS endpoint (with the authoritative DNS servers being the OpenBSD boxes). The DNS will resolve to the master OpenBSD VM (see my KISS highly-available with OpenBSD blog post), and from there, the relayd process (with a Let's Encrypt certificate—see my Let's Encrypt with OpenBSD and Rex blog post) will accept the TCP connection and forward it through the WireGuard tunnel to a reachable node port of one of the k3s nodes, thus serving the traffic.</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>> ~ curl https://f3s.foo.zone
<html><body><h1>It works!</h<font color="#000000">1</font>></body></html>
> ~ curl https://www.f3s.foo.zone
<html><body><h1>It works!</h<font color="#000000">1</font>></body></html>
> ~ curl https://standby.f3s.foo.zone
<html><body><h1>It works!</h<font color="#000000">1</font>></body></html>
</pre>
<br />
<span>This is how it works in <span class='inlinecode'>relayd.conf</span> on OpenBSD:</span><br />
<br />
<h3 style='display: inline' id='openbsd-relayd-configuration'>OpenBSD relayd configuration</h3><br />
<br />
<span>The OpenBSD edge relays keep the Kubernetes-facing addresses for the f3s ingress endpoints in a shared backend table so TLS traffic for every <span class='inlinecode'>f3s</span> hostname lands on the same pool of k3s nodes (pointing to the WireGuard IP addresses of those nodes - remember, they are running locally in my LAN, wheras the OpenBSD edge relays operate in the public internet):</span><br />
<br />
<pre>
table <f3s> {
192.168.2.120
192.168.2.121
192.168.2.122
}
</pre>
<br />
<span>Inside the <span class='inlinecode'>http protocol "https"</span> block each public hostname gets its Let's Encrypt certificate. The protocol configures TLS keypairs for all f3s services and other public endpoints. For f3s hosts specifically, there are no explicit <span class='inlinecode'>forward to</span> rules in the protocol—they use the relay-level failover mechanism described later. Non-f3s hosts get explicit localhost routing to prevent them from trying the f3s backends:</span><br />
<br />
<pre>
http protocol "https" {
# TLS certificates for all f3s services
tls keypair f3s.foo.zone
tls keypair www.f3s.foo.zone
tls keypair standby.f3s.foo.zone
tls keypair anki.f3s.foo.zone
tls keypair www.anki.f3s.foo.zone
tls keypair standby.anki.f3s.foo.zone
tls keypair bag.f3s.foo.zone
tls keypair www.bag.f3s.foo.zone
tls keypair standby.bag.f3s.foo.zone
tls keypair flux.f3s.foo.zone
tls keypair www.flux.f3s.foo.zone
tls keypair standby.flux.f3s.foo.zone
tls keypair audiobookshelf.f3s.foo.zone
tls keypair www.audiobookshelf.f3s.foo.zone
tls keypair standby.audiobookshelf.f3s.foo.zone
tls keypair gpodder.f3s.foo.zone
tls keypair www.gpodder.f3s.foo.zone
tls keypair standby.gpodder.f3s.foo.zone
tls keypair radicale.f3s.foo.zone
tls keypair www.radicale.f3s.foo.zone
tls keypair standby.radicale.f3s.foo.zone
tls keypair vault.f3s.foo.zone
tls keypair www.vault.f3s.foo.zone
tls keypair standby.vault.f3s.foo.zone
tls keypair syncthing.f3s.foo.zone
tls keypair www.syncthing.f3s.foo.zone
tls keypair standby.syncthing.f3s.foo.zone
tls keypair uprecords.f3s.foo.zone
tls keypair www.uprecords.f3s.foo.zone
tls keypair standby.uprecords.f3s.foo.zone
# Explicitly route non-f3s hosts to localhost
match request header "Host" value "foo.zone" forward to <localhost>
match request header "Host" value "www.foo.zone" forward to <localhost>
match request header "Host" value "dtail.dev" forward to <localhost>
# ... other non-f3s hosts ...
# NOTE: f3s hosts have NO match rules here!
# They use relay-level failover (f3s -> localhost backup)
# See the relay configuration below for automatic failover details
}
</pre>
<br />
<span>Both IPv4 and IPv6 listeners reuse the same protocol definition, making the relay transparent for dual-stack clients while still health checking every k3s backend before forwarding traffic over WireGuard:</span><br />
<br />
<pre>
relay "https4" {
listen on 46.23.94.99 port 443 tls
protocol "https"
# Primary: f3s cluster (with health checks) - Falls back to localhost when all hosts down
forward to <f3s> port 80 check tcp
forward to <localhost> port 8080
}
relay "https6" {
listen on 2a03:6000:6f67:624::99 port 443 tls
protocol "https"
# Primary: f3s cluster (with health checks) - Falls back to localhost when all hosts down
forward to <f3s> port 80 check tcp
forward to <localhost> port 8080
}
</pre>
<br />
<span>In practice, that means relayd terminates TLS with the correct certificate, keeps the three WireGuard-connected backends in rotation, and ships each request to whichever bhyve VM answers first.</span><br />
<br />
<h3 style='display: inline' id='automatic-failover-when-f3s-cluster-is-down'>Automatic failover when f3s cluster is down</h3><br />
<br />
<span class='quote'>Update: This section was added at Tue 30 Dec 10:11:44 EET 2025</span><br />
<br />
<span>One important aspect of this setup is graceful degradation: when all three f3s nodes are unreachable (e.g., during maintenance or a power outage in my LAN), users should see a friendly status page instead of an error message.</span><br />
<br />
<span>OpenBSD's relayd supports automatic failover through its health check mechanism. According to the relayd.conf manual:</span><br />
<br />
<span class='quote'>This directive can be specified multiple times - subsequent entries will be used as the backup table if all hosts in the previous table are down.</span><br />
<br />
<span>The key is the order of <span class='inlinecode'>forward to</span> statements in the relay configuration. By placing the f3s table first with <span class='inlinecode'>check tcp</span> health checks, followed by localhost as a backup, relayd automatically routes traffic based on backend availability:</span><br />
<br />
<span>When f3s cluster is UP:</span><br />
<br />
<ul>
<li>Health checks on port 80 succeed for f3s nodes</li>
<li>All f3s traffic routes to the Kubernetes cluster</li>
<li>Localhost backup remains idle</li>
</ul><br />
<span>When f3s cluster is DOWN:</span><br />
<br />
<ul>
<li>All health checks fail (nodes unreachable)</li>
<li>The <span class='inlinecode'><f3s></span> table becomes unavailable</li>
<li>Traffic automatically falls back to <span class='inlinecode'><localhost></span> on port 8080</li>
<li>OpenBSD's httpd serves a static fallback page</li>
</ul><br />
<pre>
# NEW configuration - supports automatic failover
http protocol "https" {
# Explicitly route non-f3s hosts to localhost
match request header "Host" value "foo.zone" forward to <localhost>
match request header "Host" value "dtail.dev" forward to <localhost>
# ... other non-f3s hosts ...
# f3s hosts have NO protocol rules - they use relay-level failover
# (no match rules for f3s.foo.zone, anki.f3s.foo.zone, etc.)
}
relay "https4" {
# f3s FIRST (with health checks), localhost as BACKUP
forward to <f3s> port 80 check tcp
forward to <localhost> port 8080
}
</pre>
<br />
<span>This way, f3s traffic uses the relay's default behavior: try the first table, fall back to the second when health checks fail.</span><br />
<br />
<h3 style='display: inline' id='openbsd-httpd-fallback-configuration'>OpenBSD httpd fallback configuration</h3><br />
<br />
<span>The localhost httpd service on port 8080 serves the fallback content from <span class='inlinecode'>/var/www/htdocs/f3s_fallback/</span>. This directory contains a simple HTML page explaining the situation.</span><br />
<br />
<span>The key configuration detail is using <span class='inlinecode'>request rewrite</span> to ensure the fallback page is served for ALL paths, not just the root. Without this, accessing paths like <span class='inlinecode'>/login?redirect=/files/</span> would return 404 instead of the fallback page:</span><br />
<br />
<pre>
# OpenBSD httpd.conf
# Fallback for f3s hosts - serve fallback page for ALL paths
server "f3s.foo.zone" {
listen on * port 8080
log style forwarded
location * {
# Rewrite all requests to /index.html to show fallback page regardless of path
request rewrite "/index.html"
root "/htdocs/f3s_fallback"
}
}
server "anki.f3s.foo.zone" {
listen on * port 8080
log style forwarded
location * {
request rewrite "/index.html"
root "/htdocs/f3s_fallback"
}
}
# ... similar blocks for all f3s hostnames ...
</pre>
<br />
<span>The <span class='inlinecode'>request rewrite "/index.html"</span> directive ensures that whether someone accesses <span class='inlinecode'>/</span>, <span class='inlinecode'>/login</span>, <span class='inlinecode'>/api/status</span>, or any other path, they all receive the same fallback page. This prevents confusing 404 errors when users have bookmarked specific URLs or follow deep links while the cluster is down.</span><br />
<br />
<span>The fallback page itself is straightforward:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><b><u><font color="#000000"><!DOCTYPE</font></u></b> <b><font color="#000000">html</font></b><b><u><font color="#000000">></font></u></b>
<b><u><font color="#000000"><html></font></u></b>
<b><u><font color="#000000"><head></font></u></b>
<b><u><font color="#000000"><title></font></u></b>Server turned off<b><u><font color="#000000"></title></font></u></b>
<b><u><font color="#000000"><style></font></u></b>
body {
font-family: <font color="#808080">sans-serif</font>;
text-align: <font color="#808080">center</font>;
padding-top: <font color="#808080">50px</font>;
}
.container {
max-width: <font color="#808080">600px</font>;
margin: <font color="#808080">0</font> <font color="#808080">auto</font>;
}
<b><u><font color="#000000"></style></font></u></b>
<b><u><font color="#000000"></head></font></u></b>
<b><u><font color="#000000"><body></font></u></b>
<b><u><font color="#000000"><div</font></u></b> <b><font color="#000000">class</font></b>=<font color="#808080">"container"</font><b><u><font color="#000000">></font></u></b>
<b><u><font color="#000000"><h1></font></u></b>Server turned off<b><u><font color="#000000"></h1></font></u></b>
<b><u><font color="#000000"><p></font></u></b>The servers are all currently turned off.<b><u><font color="#000000"></p></font></u></b>
<b><u><font color="#000000"><p></font></u></b>Please try again later.<b><u><font color="#000000"></p></font></u></b>
<b><u><font color="#000000"><p></font></u></b>Or email <b><u><font color="#000000"><a</font></u></b> <b><font color="#000000">href</font></b>=<font color="#808080">"mailto:paul@nospam.buetow.org"</font><b><u><font color="#000000">></font></u></b>paul@nospam.buetow.org<b><u><font color="#000000"></a></font></u></b>
- so I can turn them back on for you!<b><u><font color="#000000"></p></font></u></b>
<b><u><font color="#000000"></div></font></u></b>
<b><u><font color="#000000"></body></font></u></b>
<b><u><font color="#000000"></html></font></u></b>
</pre>
<br />
<span>This approach provides several benefits:</span><br />
<br />
<ul>
<li>Automatic detection: Health checks run continuously; no manual intervention needed</li>
<li>Instant fallback: When all f3s nodes go down, the next request automatically routes to localhost</li>
<li>Transparent recovery: When f3s comes back online, health checks pass and traffic resumes automatically</li>
<li>User experience: Visitors see a helpful message instead of connection errors</li>
<li>No DNS changes: The same hostnames work whether f3s is up or down</li>
</ul><br />
<span>This fallback mechanism has proven invaluable during maintenance windows and unexpected outages, ensuring that users always get a response even when the home lab is offline.</span><br />
<br />
<h2 style='display: inline' id='exposing-services-via-lan-ingress'>Exposing services via LAN ingress</h2><br />
<br />
<span>In addition to external access through the OpenBSD relays, services can also be exposed on the local network using LAN-specific ingresses. This is useful for accessing services from within the home network without going through the internet, reducing latency and providing an alternative path if the external relays are unavailable.</span><br />
<br />
<span>The LAN ingress architecture leverages the existing FreeBSD CARP (Common Address Redundancy Protocol) failover infrastructure that's already in place for NFS-over-TLS (see Part 5). Instead of deploying MetalLB or another LoadBalancer implementation, we reuse the CARP virtual IP (<span class='inlinecode'>192.168.1.138</span>) by adding HTTP/HTTPS forwarding alongside the existing stunnel service on port 2323.</span><br />
<br />
<h3 style='display: inline' id='architecture-overview'>Architecture overview</h3><br />
<br />
<span>The LAN access path differs from external access:</span><br />
<br />
<span>**External access (*.f3s.foo.zone):**</span><br />
<pre>
Internet → OpenBSD relayd (TLS termination, Let's Encrypt)
→ WireGuard tunnel
→ k3s Traefik :80 (HTTP)
→ Service
</pre>
<br />
<span>**LAN access (*.f3s.lan.foo.zone):**</span><br />
<pre>
LAN → FreeBSD CARP VIP (192.168.1.138)
→ FreeBSD relayd (TCP forwarding)
→ k3s Traefik :443 (TLS termination, cert-manager)
→ Service
</pre>
<br />
<span>The key architectural decisions:</span><br />
<br />
<ul>
<li>FreeBSD <span class='inlinecode'>relayd</span> performs pure TCP forwarding (Layer 4) for ports 80 and 443, not TLS termination</li>
<li>Traefik inside k3s handles TLS offloading using certificates from cert-manager</li>
<li>Self-signed CA for LAN domains (no external dependencies)</li>
<li>CARP provides automatic failover between f0 and f1</li>
<li>No code changes to applications—just add a LAN ingress resource</li>
</ul><br />
<h3 style='display: inline' id='installing-cert-manager'>Installing cert-manager</h3><br />
<br />
<span>First, install cert-manager to handle certificate lifecycle management for LAN services. The installation is automated with a Justfile:</span><br />
<br />
<a class='textlink' href='https://codeberg.org/snonux/conf/src/branch/master/f3s/cert-manager'>codeberg.org/snonux/conf/f3s/cert-manager</a><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>$ cd conf/f3s/cert-manager
$ just install
kubectl apply -f cert-manager.yaml
<i><font color="silver"># ... cert-manager CRDs and resources created ...</font></i>
kubectl apply -f self-signed-issuer.yaml
clusterissuer.cert-manager.io/selfsigned-issuer created
clusterissuer.cert-manager.io/selfsigned-ca-issuer created
kubectl apply -f ca-certificate.yaml
certificate.cert-manager.io/selfsigned-ca created
kubectl apply -f wildcard-certificate.yaml
certificate.cert-manager.io/f3s-lan-wildcard created
</pre>
<br />
<span>This creates:</span><br />
<br />
<ul>
<li>A self-signed ClusterIssuer</li>
<li>A CA certificate (<span class='inlinecode'>f3s-lan-ca</span>) valid for 10 years</li>
<li>A CA-signed ClusterIssuer</li>
<li>A wildcard certificate (<span class='inlinecode'>*.f3s.lan.foo.zone</span>) valid for 90 days with automatic renewal</li>
</ul><br />
<span>Verify the certificates:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>$ kubectl get certificate -n cert-manager
NAME READY SECRET AGE
f3s-lan-wildcard True f3s-lan-tls 5m
selfsigned-ca True selfsigned-ca-secret 5m
</pre>
<br />
<span>The wildcard certificate (<span class='inlinecode'>f3s-lan-tls</span>) needs to be copied to any namespace that uses it:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>$ kubectl get secret f3s-lan-tls -n cert-manager -o yaml | \
sed <font color="#808080">'s/namespace: cert-manager/namespace: services/'</font> | \
kubectl apply -f -
</pre>
<br />
<h3 style='display: inline' id='configuring-freebsd-relayd-for-lan-access'>Configuring FreeBSD relayd for LAN access</h3><br />
<br />
<span>On both FreeBSD hosts (f0, f1), install and configure <span class='inlinecode'>relayd</span> for TCP forwarding:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>paul@f0:~ % doas pkg install -y relayd
</pre>
<br />
<span>Create <span class='inlinecode'>/usr/local/etc/relayd.conf</span>:</span><br />
<br />
<pre>
# k3s nodes backend table
table <k3s_nodes> { 192.168.1.120 192.168.1.121 192.168.1.122 }
# TCP forwarding to Traefik (no TLS termination)
relay "lan_http" {
listen on 192.168.1.138 port 80
forward to <k3s_nodes> port 80 check tcp
}
relay "lan_https" {
listen on 192.168.1.138 port 443
forward to <k3s_nodes> port 443 check tcp
}
</pre>
<br />
<span class='quote'>Note: The IP addresses <span class='inlinecode'>192.168.1.120-122</span> are the LAN IPs of the k3s nodes (r0, r1, r2), not their WireGuard IPs. FreeBSD <span class='inlinecode'>relayd</span> requires PF (Packet Filter) to be enabled. Create a minimal <span class='inlinecode'>/etc/pf.conf</span>:</span><br />
<br />
<pre>
# Basic PF rules for relayd
set skip on lo0
pass in quick
pass out quick
</pre>
<br />
<span>Enable PF and relayd:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>paul@f0:~ % doas sysrc pf_enable=YES pflog_enable=YES relayd_enable=YES
paul@f0:~ % doas service pf start
paul@f0:~ % doas service pflog start
paul@f0:~ % doas service relayd start
</pre>
<br />
<span>Verify <span class='inlinecode'>relayd</span> is listening on the CARP VIP:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>paul@f0:~ % doas sockstat -<font color="#000000">4</font> -l | grep <font color="#000000">192.168</font>.<font color="#000000">1.138</font>
_relayd relayd <font color="#000000">2903</font> <font color="#000000">11</font> tcp4 <font color="#000000">192.168</font>.<font color="#000000">1.138</font>:<font color="#000000">80</font> *:*
_relayd relayd <font color="#000000">2903</font> <font color="#000000">12</font> tcp4 <font color="#000000">192.168</font>.<font color="#000000">1.138</font>:<font color="#000000">443</font> *:*
</pre>
<br />
<span>Repeat the same configuration on f1. Both hosts will run <span class='inlinecode'>relayd</span> listening on the CARP VIP, but only the CARP MASTER will respond to traffic. When failover occurs, the new MASTER takes over seamlessly.</span><br />
<br />
<h3 style='display: inline' id='adding-lan-ingress-to-services'>Adding LAN ingress to services</h3><br />
<br />
<span>To expose a service on the LAN, add a second Ingress resource to its Helm chart. Here's an example:</span><br />
<br />
<pre>
---
# LAN Ingress for f3s.lan.foo.zone
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress-lan
namespace: services
annotations:
spec.ingressClassName: traefik
traefik.ingress.kubernetes.io/router.entrypoints: web,websecure
spec:
tls:
- hosts:
- f3s.lan.foo.zone
secretName: f3s-lan-tls
rules:
- host: f3s.lan.foo.zone
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: service
port:
number: 4533
</pre>
<br />
<span>Key points:</span><br />
<br />
<ul>
<li>Use <span class='inlinecode'>web,websecure</span> entrypoints (both HTTP and HTTPS)</li>
<li>Reference the <span class='inlinecode'>f3s-lan-tls</span> secret in the <span class='inlinecode'>tls</span> section</li>
<li>Use <span class='inlinecode'>.f3s.lan.foo.zone</span> subdomain pattern</li>
<li>Same backend service as the external ingress</li>
</ul><br />
<span>Apply the ingress and test:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>$ kubectl apply -f ingress-lan.yaml
ingress.networking.k8s.io/ingress-lan created
$ curl -k https://f3s.lan.foo.zone
HTTP/<font color="#000000">2</font> <font color="#000000">302</font>
location: /app/
</pre>
<br />
<h3 style='display: inline' id='client-side-dns-and-ca-setup'>Client-side DNS and CA setup</h3><br />
<br />
<span>To access LAN services, clients need DNS entries and must trust the self-signed CA.</span><br />
<br />
<span>Add DNS entries to <span class='inlinecode'>/etc/hosts</span> on your laptop:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>$ sudo tee -a /etc/hosts << <font color="#808080">'EOF'</font>
<i><font color="silver"># f3s LAN services</font></i>
<font color="#000000">192.168</font>.<font color="#000000">1.138</font> f3s.lan.foo.zone
EOF
</pre>
<br />
<span>The CARP VIP <span class='inlinecode'>192.168.1.138</span> provides high availability—traffic automatically fails over to the backup host if the master goes down.</span><br />
<br />
<span>Export the self-signed CA certificate:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>$ kubectl get secret selfsigned-ca-secret -n cert-manager -o jsonpath=<font color="#808080">'{.data.ca</font>\.<font color="#808080">crt}'</font> | \
base64 -d > f3s-lan-ca.crt
</pre>
<br />
<span>Install the CA certificate on Linux (Fedora/Rocky):</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>$ sudo cp f3s-lan-ca.crt /etc/pki/ca-trust/source/anchors/
$ sudo update-ca-trust
</pre>
<br />
<span>After trusting the CA, browsers will accept the LAN certificates without warnings.</span><br />
<br />
<h3 style='display: inline' id='scaling-to-other-services'>Scaling to other services</h3><br />
<br />
<span>The same pattern can be applied to any service. To add LAN access:</span><br />
<br />
<span>1. Copy the <span class='inlinecode'>f3s-lan-tls</span> secret to the service's namespace (if not already there)</span><br />
<span>2. Add a LAN Ingress resource using the pattern above</span><br />
<span>3. Configure DNS: <span class='inlinecode'>192.168.1.138 service.f3s.lan.foo.zone</span></span><br />
<br />
<span>No changes needed to:</span><br />
<br />
<ul>
<li>relayd configuration (forwards all traffic)</li>
<li>cert-manager (wildcard cert covers all <span class='inlinecode'>*.f3s.lan.foo.zone</span>)</li>
<li>CARP configuration (VIP shared by all services)</li>
</ul><br />
<h3 style='display: inline' id='tls-offloaders-summary'>TLS offloaders summary</h3><br />
<br />
<span>The f3s infrastructure now has three distinct TLS offloaders:</span><br />
<br />
<ul>
<li>**OpenBSD relayd**: External internet traffic (<span class='inlinecode'>*.f3s.foo.zone</span>) using Let's Encrypt</li>
<li>**Traefik (k3s)**: LAN HTTPS traffic (<span class='inlinecode'>*.f3s.lan.foo.zone</span>) using cert-manager</li>
<li>**stunnel**: NFS-over-TLS (port 2323) using custom PKI</li>
</ul><br />
<span>Each serves a different purpose with appropriate certificate management for its use case.</span><br />
<br />
<h2 style='display: inline' id='deploying-the-private-docker-image-registry'>Deploying the private Docker image registry</h2><br />
<br />
<span>As not all Docker images I want to deploy are available on public Docker registries and as I also build some of them by myself, there is the need of a private registry.</span><br />
<br />
<span>All manifests for the f3s stack live in my configuration repository:</span><br />
<br />
<a class='textlink' href='https://codeberg.org/snonux/conf/src/branch/master/f3s'>codeberg.org/snonux/conf/f3s</a><br />
<br />
<span>Within that repo, the <span class='inlinecode'>f3s/registry/</span> directory contains the Helm chart, a <span class='inlinecode'>Justfile</span>, and a detailed <span class='inlinecode'>README</span>. Here's the condensed walkthrough I used to roll out the registry with Helm.</span><br />
<br />
<h3 style='display: inline' id='prepare-the-nfs-backed-storage'>Prepare the NFS-backed storage</h3><br />
<br />
<span>Create the directory that will hold the registry blobs on the NFS share (I ran this on <span class='inlinecode'>r0</span>, but any node that exports <span class='inlinecode'>/data/nfs/k3svolumes</span> works):</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>[root@r0 ~]<i><font color="silver"># mkdir -p /data/nfs/k3svolumes/registry</font></i>
</pre>
<br />
<h3 style='display: inline' id='install-or-upgrade-the-chart'>Install (or upgrade) the chart</h3><br />
<br />
<span>Clone the repo (or pull the latest changes) on a workstation that has <span class='inlinecode'>helm</span> configured for the cluster, then deploy the chart. The Justfile wraps the commands, but the raw Helm invocation looks like this:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>$ git clone https://codeberg.org/snonux/conf/f3s.git
$ cd conf/f3s/examples/conf/f3s/registry
$ helm upgrade --install registry ./helm-chart --namespace infra --create-namespace
</pre>
<br />
<span>Helm creates the <span class='inlinecode'>infra</span> namespace if it does not exist, provisions a <span class='inlinecode'>PersistentVolume</span>/<span class='inlinecode'>PersistentVolumeClaim</span> pair that points at <span class='inlinecode'>/data/nfs/k3svolumes/registry</span>, and spins up a single registry pod exposed via the <span class='inlinecode'>docker-registry-service</span> NodePort (<span class='inlinecode'>30001</span>). Verify everything is up before continuing:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>$ kubectl get pods --namespace infra
NAME READY STATUS RESTARTS AGE
docker-registry-6bc9bb46bb-6grkr <font color="#000000">1</font>/<font color="#000000">1</font> Running <font color="#000000">6</font> (53d ago) 54d
$ kubectl get svc docker-registry-service -n infra
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
docker-registry-service NodePort <font color="#000000">10.43</font>.<font color="#000000">141.56</font> <none> <font color="#000000">5000</font>:<font color="#000000">30001</font>/TCP 54d
</pre>
<br />
<h3 style='display: inline' id='allow-nodes-and-workstations-to-trust-the-registry'>Allow nodes and workstations to trust the registry</h3><br />
<br />
<span>The registry listens on plain HTTP, so both Docker daemons on workstations and the k3s nodes need to treat it as an insecure registry. That's fine for my personal needs, as:</span><br />
<br />
<ul>
<li>I don't store any secrets in the images</li>
<li>I access the registry this way only via my LAN</li>
<li>I may will change it later on...</li>
</ul><br />
<span>On my Fedora workstation where I build images:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>$ cat <<<font color="#808080">"EOF"</font> | sudo tee /etc/docker/daemon.json >/dev/null
{
<font color="#808080">"insecure-registries"</font>: [
<font color="#808080">"r0.lan.buetow.org:30001"</font>,
<font color="#808080">"r1.lan.buetow.org:30001"</font>,
<font color="#808080">"r2.lan.buetow.org:30001"</font>
]
}
EOF
$ sudo systemctl restart docker
</pre>
<br />
<span>On each k3s node, make <span class='inlinecode'>registry.lan.buetow.org</span> resolve locally and point k3s at the NodePort:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>$ <b><u><font color="#000000">for</font></u></b> node <b><u><font color="#000000">in</font></u></b> r0 r1 r2; <b><u><font color="#000000">do</font></u></b>
> ssh root@$node <font color="#808080">"echo '127.0.0.1 registry.lan.buetow.org' >> /etc/hosts"</font>
> <b><u><font color="#000000">done</font></u></b>
$ <b><u><font color="#000000">for</font></u></b> node <b><u><font color="#000000">in</font></u></b> r0 r1 r2; <b><u><font color="#000000">do</font></u></b>
> ssh root@$node <font color="#808080">"cat <<'EOF' > /etc/rancher/k3s/registries.yaml</font>
<font color="#808080">mirrors:</font>
<font color="#808080"> "</font>registry.lan.buetow.org:<font color="#000000">30001</font><font color="#808080">":</font>
<font color="#808080"> endpoint:</font>
<font color="#808080"> - "</font>http://localhost:<font color="#000000">30001</font><font color="#808080">"</font>
<font color="#808080">EOF</font>
<font color="#808080">systemctl restart k3s"</font>
> <b><u><font color="#000000">done</font></u></b>
</pre>
<br />
<span>Thanks to the relayd configuration earlier in the post, the external hostnames (<span class='inlinecode'>f3s.foo.zone</span>, etc.) can already reach NodePort <span class='inlinecode'>30001</span>, so publishing the registry later to the outside world is just a matter of wiring the DNS the same way as the ingress hosts. But by default, that's not enabled for now due to security reasons.</span><br />
<br />
<h3 style='display: inline' id='pushing-and-pulling-images'>Pushing and pulling images</h3><br />
<br />
<span>Tag any locally built image with one of the node IPs on port <span class='inlinecode'>30001</span>, then push it. I usually target whichever node is closest to me, but any of the three will do:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>$ docker tag my-app:latest r0.lan.buetow.org:<font color="#000000">30001</font>/my-app:latest
$ docker push r0.lan.buetow.org:<font color="#000000">30001</font>/my-app:latest
</pre>
<br />
<span>Inside the cluster (or from other nodes), reference the image via the service name that Helm created:</span><br />
<br />
<pre>
image: docker-registry-service:5000/my-app:latest
</pre>
<br />
<span>You can test the pull path straight away:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>$ kubectl run registry-test \
> --image=docker-registry-service:<font color="#000000">5000</font>/my-app:latest \
> --restart=Never -n <b><u><font color="#000000">test</font></u></b> --command -- sleep <font color="#000000">300</font>
</pre>
<br />
<span>If the pod pulls successfully, the private registry is ready for use by the rest of the workloads. Note, that the commands above actually don't work, they are only for illustration purpose mentioned here.</span><br />
<br />
<h2 style='display: inline' id='example-anki-sync-server-from-the-private-registry'>Example: Anki Sync Server from the private registry</h2><br />
<br />
<span>One of the first workloads I migrated onto the k3s cluster after standing up the registry was my Anki sync server. The configuration repo ships everything in <span class='inlinecode'>examples/conf/f3s/anki-sync-server/</span>: a Docker build context plus a Helm chart that references the freshly built image.</span><br />
<br />
<h3 style='display: inline' id='build-and-push-the-image'>Build and push the image</h3><br />
<br />
<span>The Dockerfile lives under <span class='inlinecode'>docker-image/</span> and takes the Anki release to compile as an <span class='inlinecode'>ANKI_VERSION</span> build argument. The accompanying <span class='inlinecode'>Justfile</span> wraps the steps, but the raw commands look like this:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>$ cd conf/f3s/examples/conf/f3s/anki-sync-server/docker-image
$ docker build -t anki-sync-server:<font color="#000000">25.07</font>.5b --build-arg ANKI_VERSION=<font color="#000000">25.07</font>.<font color="#000000">5</font> .
$ docker tag anki-sync-server:<font color="#000000">25.07</font>.5b \
r0.lan.buetow.org:<font color="#000000">30001</font>/anki-sync-server:<font color="#000000">25.07</font>.5b
$ docker push r0.lan.buetow.org:<font color="#000000">30001</font>/anki-sync-server:<font color="#000000">25.07</font>.5b
</pre>
<br />
<span>Because every k3s node treats <span class='inlinecode'>registry.lan.buetow.org:30001</span> as an insecure mirror (see above), the push succeeds regardless of which node answers. If you prefer the shortcut, <span class='inlinecode'>just f3s</span> in that directory performs the same build/tag/push sequence.</span><br />
<br />
<h3 style='display: inline' id='create-the-anki-secret-and-storage-on-the-cluster'>Create the Anki secret and storage on the cluster</h3><br />
<br />
<span>The Helm chart expects the <span class='inlinecode'>services</span> namespace, a pre-created NFS directory, and a Kubernetes secret that holds the credentials the upstream container understands:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>$ ssh root@r0 <font color="#808080">"mkdir -p /data/nfs/k3svolumes/anki-sync-server/anki_data"</font>
$ kubectl create namespace services
$ kubectl create secret generic anki-sync-server-secret \
--from-literal=SYNC_USER1=<font color="#808080">'paul:SECRETPASSWORD'</font> \
-n services
</pre>
<br />
<span>If the <span class='inlinecode'>services</span> namespace already exists, you can skip that line or let Kubernetes tell you the namespace is unchanged.</span><br />
<br />
<h3 style='display: inline' id='deploy-the-chart'>Deploy the chart</h3><br />
<br />
<span>With the prerequisites in place, install (or upgrade) the chart. It pins the container image to the tag we just pushed and mounts the NFS export via a <span class='inlinecode'>PersistentVolume/PersistentVolumeClaim</span> pair:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>$ cd ../helm-chart
$ helm upgrade --install anki-sync-server . -n services
</pre>
<br />
<span>Helm provisions everything referenced in the templates:</span><br />
<br />
<pre>
containers:
- name: anki-sync-server image: registry.lan.buetow.org:30001/anki-sync-server:25.07.5b
volumeMounts:
- name: anki-data
mountPath: /anki_data
</pre>
<br />
<span>Once the release comes up, verify that the pod pulled the freshly pushed image and that the ingress we configured earlier resolves through relayd just like the Apache example.</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>$ kubectl get pods -n services
$ kubectl get ingress anki-sync-server-ingress -n services
$ curl https://anki.f3s.foo.zone/health
</pre>
<br />
<span>All of this runs solely on first-party images that now live in the private registry, proving the full flow from local bild to WireGuard-exposed service.</span><br />
<br />
<h2 style='display: inline' id='nfsv4-uid-mapping-for-postgres-backed-and-other-apps'>NFSv4 UID mapping for Postgres-backed (and other) apps</h2><br />
<br />
<span>NFSv4 only sees numeric user and group IDs, so the <span class='inlinecode'>postgres</span> account created inside the container must exist with the same UID/GID on the Kubernetes worker and on the FreeBSD NFS servers. Otherwise the pod starts with UID 999, the export sees it as an unknown anonymous user, and Postgres fails to initialise its data directory.</span><br />
<br />
<span>To verify things line up end-to-end I run <span class='inlinecode'>id</span> in the container and on the hosts:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>> ~ kubectl <b><u><font color="#000000">exec</font></u></b> -n services deploy/miniflux-postgres -- id postgres
uid=<font color="#000000">999</font>(postgres) gid=<font color="#000000">999</font>(postgres) groups=<font color="#000000">999</font>(postgres)
[root@r0 ~]<i><font color="silver"># id postgres</font></i>
uid=<font color="#000000">999</font>(postgres) gid=<font color="#000000">999</font>(postgres) groups=<font color="#000000">999</font>(postgres)
paul@f0:~ % doas id postgres
uid=<font color="#000000">999</font>(postgres) gid=<font color="#000000">99</font>(postgres) groups=<font color="#000000">999</font>(postgres)
</pre>
<br />
<span>The Rocky Linux workers get their matching user with plain <span class='inlinecode'>useradd</span>/<span class='inlinecode'>groupadd</span> (repeat on <span class='inlinecode'>r0</span>, <span class='inlinecode'>r1</span>, and <span class='inlinecode'>r2</span>):</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>[root@r0 ~]<i><font color="silver"># groupadd --gid 999 postgres</font></i>
[root@r0 ~]<i><font color="silver"># useradd --uid 999 --gid 999 \</font></i>
--home-dir /var/lib/pgsql \
--shell /sbin/nologin postgres
</pre>
<br />
<span>FreeBSD uses <span class='inlinecode'>pw</span>, so on each NFS server (<span class='inlinecode'>f0</span>, <span class='inlinecode'>f1</span>, <span class='inlinecode'>f2</span>) I created the same account and disabled shell access:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>paul@f0:~ % doas pw groupadd postgres -g <font color="#000000">999</font>
paul@f0:~ % doas pw useradd postgres -u <font color="#000000">999</font> -g postgres \
-d /var/db/postgres -s /usr/sbin/nologin
</pre>
<br />
<span>Once the UID/GID exist everywhere, the Miniflux chart in <span class='inlinecode'>examples/conf/f3s/miniflux</span> deploys cleanly. The chart provisions both the application and its bundled Postgres database, mounts the exported directory, and builds the DSN at runtime. The important bits live in <span class='inlinecode'>helm-chart/templates/persistent-volumes.yaml</span> and <span class='inlinecode'>deployment.yaml</span>:</span><br />
<br />
<pre>
# Persistent volume lives on the NFS export
hostPath:
path: /data/nfs/k3svolumes/miniflux/data
type: Directory
...
containers:
- name: miniflux-postgres
image: postgres:17
volumeMounts:
- name: miniflux-postgres-data
mountPath: /var/lib/postgresql/data
</pre>
<br />
<span>Follow the <span class='inlinecode'>README</span> beside the chart to create the secrets and the target directory:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre>$ cd examples/conf/f3s/miniflux/helm-chart
$ mkdir -p /data/nfs/k3svolumes/miniflux/data
$ kubectl create secret generic miniflux-db-password \
--from-literal=fluxdb_password=<font color="#808080">'YOUR_PASSWORD'</font> -n services
$ kubectl create secret generic miniflux-admin-password \
--from-literal=admin_password=<font color="#808080">'YOUR_ADMIN_PASSWORD'</font> -n services
$ helm upgrade --install miniflux . -n services --create-namespace
</pre>
<br />
<span>And to verify it's all up:</span><br />
<br />
<pre>
$ kubectl get all --namespace=services | grep mini
pod/miniflux-postgres-556444cb8d-xvv2p 1/1 Running 0 54d
pod/miniflux-server-85d7c64664-stmt9 1/1 Running 0 54d
service/miniflux ClusterIP 10.43.47.80 <none> 8080/TCP 54d
service/miniflux-postgres ClusterIP 10.43.139.50 <none> 5432/TCP 54d
deployment.apps/miniflux-postgres 1/1 1 1 54d
deployment.apps/miniflux-server 1/1 1 1 54d
replicaset.apps/miniflux-postgres-556444cb8d 1 1 1 54d
replicaset.apps/miniflux-server-85d7c64664 1 1 1 54d
</pre>
<br />
<span>Or from the repository root I simply run:</span><br />
<br />
<h3 style='display: inline' id='helm-charts-currently-in-service'>Helm charts currently in service</h3><br />
<br />
<span>These are the charts that already live under <span class='inlinecode'>examples/conf/f3s</span> and run on the cluster today (and I'll keep adding more as new services graduate into production):</span><br />
<br />
<ul>
<li><span class='inlinecode'>anki-sync-server</span> — custom-built image served from the private registry, stores decks on <span class='inlinecode'>/data/nfs/k3svolumes/anki-sync-server/anki_data</span>, and authenticates through the <span class='inlinecode'>anki-sync-server-secret</span>.</li>
<li><span class='inlinecode'>koreade-sync-server</span> — Sync server for KOReader.</li>
<li><span class='inlinecode'>audiobookshelf</span> — media streaming stack with three hostPath mounts (<span class='inlinecode'>config</span>, <span class='inlinecode'>audiobooks</span>, <span class='inlinecode'>podcasts</span>) so the library survives node rebuilds.</li>
<li><span class='inlinecode'>example-apache</span> — minimal HTTP service I use for smoke-testing ingress and relayd rules.</li>
<li><span class='inlinecode'>example-apache-volume-claim</span> — Apache plus PVC variant that exercises NFS-backed storage for walkthroughs like the one earlier in this post.</li>
<li><span class='inlinecode'>miniflux</span> — the Postgres-backed feed reader described above, wired for NFSv4 UID mapping and per-release secrets.</li>
<li><span class='inlinecode'>opodsync</span> — podsync deployment with its data directory under <span class='inlinecode'>/data/nfs/k3svolumes/opodsync/data</span>.</li>
<li><span class='inlinecode'>radicale</span> — CalDAV/CardDAV (and gpodder) backend with separate <span class='inlinecode'>collections</span> and <span class='inlinecode'>auth</span> volumes.</li>
<li><span class='inlinecode'>registry</span> — the plain-HTTP Docker registry exposed on NodePort 30001 and mirrored internally as <span class='inlinecode'>registry.lan.buetow.org:30001</span>.</li>
<li><span class='inlinecode'>syncthing</span> — two-volume setup for config and shared data, fronted by the <span class='inlinecode'>syncthing.f3s.foo.zone</span> ingress.</li>
<li><span class='inlinecode'>wallabag</span> — read-it-later service with persistent <span class='inlinecode'>data</span> and <span class='inlinecode'>images</span> directories on the NFS export.</li>
</ul><br />
<span>I hope you enjoyed this walkthrough. Read the next post of this series:</span><br />
<br />
<a class='textlink' href='./2025-12-07-f3s-kubernetes-with-freebsd-part-8.html'>f3s: Kubernetes with FreeBSD - Part 8: Observability</a><br />
<br />
<span>Other *BSD-related posts:</span><br />
<br />
<a class='textlink' href='./2025-12-07-f3s-kubernetes-with-freebsd-part-8.html'>2025-12-07 f3s: Kubernetes with FreeBSD - Part 8: Observability</a><br />
<a class='textlink' href='./2025-10-02-f3s-kubernetes-with-freebsd-part-7.html'>2025-10-02 f3s: Kubernetes with FreeBSD - Part 7: k3s and first pod deployments (You are currently reading this)</a><br />
<a class='textlink' href='./2025-07-14-f3s-kubernetes-with-freebsd-part-6.html'>2025-07-14 f3s: Kubernetes with FreeBSD - Part 6: Storage</a><br />
<a class='textlink' href='./2025-05-11-f3s-kubernetes-with-freebsd-part-5.html'>2025-05-11 f3s: Kubernetes with FreeBSD - Part 5: WireGuard mesh network</a><br />
<a class='textlink' href='./2025-04-05-f3s-kubernetes-with-freebsd-part-4.html'>2025-04-05 f3s: Kubernetes with FreeBSD - Part 4: Rocky Linux Bhyve VMs</a><br />
<a class='textlink' href='./2025-02-01-f3s-kubernetes-with-freebsd-part-3.html'>2025-02-01 f3s: Kubernetes with FreeBSD - Part 3: Protecting from power cuts</a><br />
<a class='textlink' href='./2024-12-03-f3s-kubernetes-with-freebsd-part-2.html'>2024-12-03 f3s: Kubernetes with FreeBSD - Part 2: Hardware and base installation</a><br />
<a class='textlink' href='./2024-11-17-f3s-kubernetes-with-freebsd-part-1.html'>2024-11-17 f3s: Kubernetes with FreeBSD - Part 1: Setting the stage</a><br />
<a class='textlink' href='./2024-04-01-KISS-high-availability-with-OpenBSD.html'>2024-04-01 KISS high-availability with OpenBSD</a><br />
<a class='textlink' href='./2024-01-13-one-reason-why-i-love-openbsd.html'>2024-01-13 One reason why I love OpenBSD</a><br />
<a class='textlink' href='./2022-10-30-installing-dtail-on-openbsd.html'>2022-10-30 Installing DTail on OpenBSD</a><br />
<a class='textlink' href='./2022-07-30-lets-encrypt-with-openbsd-and-rex.html'>2022-07-30 Let's Encrypt with OpenBSD and Rex</a><br />
<a class='textlink' href='./2016-04-09-jails-and-zfs-on-freebsd-with-puppet.html'>2016-04-09 Jails and ZFS with Puppet on FreeBSD</a><br />
<br />
<span>E-Mail your comments to <span class='inlinecode'>paul@nospam.buetow.org</span></span><br />
<br />
<a class='textlink' href='../'>Back to the main site</a><br />
<p class="footer">
Generated with <a href="https://codeberg.org/snonux/gemtexter">Gemtexter 3.0.1-develop</a> |
served by <a href="https://www.OpenBSD.org">OpenBSD</a>/<a href="https://man.openbsd.org/relayd.8">relayd(8)</a>+<a href="https://man.openbsd.org/httpd.8">httpd(8)</a> |
<a href="https://foo.zone/site-mirrors.html">Site Mirrors</a>
<br />
Webring: <a href="https://shring.sh/foo.zone/previous">previous</a> | <a href="https://shring.sh">shring</a> | <a href="https://shring.sh/foo.zone/next">next</a>
</p>
</body>
</html>
|
