1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
|
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>f3s: Kubernetes with FreeBSD - Part 6: Storage</title>
<link rel="shortcut icon" type="image/gif" href="/favicon.ico" />
<link rel="stylesheet" href="../style.css" />
<link rel="stylesheet" href="style-override.css" />
</head>
<body>
<div class="rfx-overlay-grid"></div>
<div class="rfx-overlay-scanlines"></div>
<div id="rfx-stars"></div>
<div class="rfx-vignette"></div>
<p class="header">
<a href="https://foo.zone">Home</a> | <a href="https://codeberg.org/snonux/foo.zone/src/branch/content-md/gemfeed/2025-07-14-f3s-kubernetes-with-freebsd-part-6.md">Markdown</a> | <a href="gemini://foo.zone/gemfeed/2025-07-14-f3s-kubernetes-with-freebsd-part-6.gmi">Gemini</a>
</p>
<h1 style='display: inline' id='f3s-kubernetes-with-freebsd---part-6-storage'>f3s: Kubernetes with FreeBSD - Part 6: Storage</h1><br />
<br />
<span class='quote'>Published at 2025-07-13T16:44:29+03:00, last updated Tue 27 Jan 10:09:08 EET 2026</span><br />
<br />
<span>This is the sixth blog post about the f3s series for self-hosting demands in a home lab. f3s? The "f" stands for FreeBSD, and the "3s" stands for k3s, the Kubernetes distribution used on FreeBSD-based physical machines.</span><br />
<br />
<a class='textlink' href='./2024-11-17-f3s-kubernetes-with-freebsd-part-1.html'>2024-11-17 f3s: Kubernetes with FreeBSD - Part 1: Setting the stage</a><br />
<a class='textlink' href='./2024-12-03-f3s-kubernetes-with-freebsd-part-2.html'>2024-12-03 f3s: Kubernetes with FreeBSD - Part 2: Hardware and base installation</a><br />
<a class='textlink' href='./2025-02-01-f3s-kubernetes-with-freebsd-part-3.html'>2025-02-01 f3s: Kubernetes with FreeBSD - Part 3: Protecting from power cuts</a><br />
<a class='textlink' href='./2025-04-05-f3s-kubernetes-with-freebsd-part-4.html'>2025-04-05 f3s: Kubernetes with FreeBSD - Part 4: Rocky Linux Bhyve VMs</a><br />
<a class='textlink' href='./2025-05-11-f3s-kubernetes-with-freebsd-part-5.html'>2025-05-11 f3s: Kubernetes with FreeBSD - Part 5: WireGuard mesh network</a><br />
<a class='textlink' href='./2025-07-14-f3s-kubernetes-with-freebsd-part-6.html'>2025-07-14 f3s: Kubernetes with FreeBSD - Part 6: Storage (You are currently reading this)</a><br />
<a class='textlink' href='./2025-10-02-f3s-kubernetes-with-freebsd-part-7.html'>2025-10-02 f3s: Kubernetes with FreeBSD - Part 7: k3s and first pod deployments</a><br />
<a class='textlink' href='./2025-12-07-f3s-kubernetes-with-freebsd-part-8.html'>2025-12-07 f3s: Kubernetes with FreeBSD - Part 8: Observability</a><br />
<br />
<a href='./f3s-kubernetes-with-freebsd-part-1/f3slogo.png'><img alt='f3s logo' title='f3s logo' src='./f3s-kubernetes-with-freebsd-part-1/f3slogo.png' /></a><br />
<br />
<h2 style='display: inline' id='table-of-contents'>Table of Contents</h2><br />
<br />
<ul>
<li><a href='#f3s-kubernetes-with-freebsd---part-6-storage'>f3s: Kubernetes with FreeBSD - Part 6: Storage</a></li>
<li>⇢ <a href='#introduction'>Introduction</a></li>
<li>⇢ <a href='#additional-storage-capacity'>Additional storage capacity</a></li>
<li>⇢ <a href='#zfs-encryption-keys'>ZFS encryption keys</a></li>
<li>⇢ ⇢ <a href='#ufs-on-usb-keys'>UFS on USB keys</a></li>
<li>⇢ ⇢ <a href='#generating-encryption-keys'>Generating encryption keys</a></li>
<li>⇢ ⇢ <a href='#configuring-zdata-zfs-pool-encryption'>Configuring <span class='inlinecode'>zdata</span> ZFS pool encryption</a></li>
<li>⇢ ⇢ <a href='#migrating-bhyve-vms-to-an-encrypted-bhyve-zfs-volume'>Migrating Bhyve VMs to an encrypted <span class='inlinecode'>bhyve</span> ZFS volume</a></li>
<li>⇢ <a href='#zfs-replication-with-zrepl'>ZFS Replication with <span class='inlinecode'>zrepl</span></a></li>
<li>⇢ ⇢ <a href='#understanding-replication-requirements'>Understanding Replication Requirements</a></li>
<li>⇢ ⇢ <a href='#installing-zrepl'>Installing <span class='inlinecode'>zrepl</span></a></li>
<li>⇢ ⇢ <a href='#configuring-zrepl-on-f1-sink'>Configuring <span class='inlinecode'>zrepl</span> on <span class='inlinecode'>f1</span> (sink)</a></li>
<li>⇢ ⇢ <a href='#enabling-and-starting-zrepl-services'>Enabling and starting <span class='inlinecode'>zrepl</span> services</a></li>
<li>⇢ ⇢ <a href='#monitoring-replication'>Monitoring replication</a></li>
<li>⇢ ⇢ <a href='#verifying-replication-after-reboot'>Verifying replication after reboot</a></li>
<li>⇢ ⇢ <a href='#understanding-failover-limitations-and-design-decisions'>Understanding Failover Limitations and Design Decisions</a></li>
<li>⇢ ⇢ <a href='#mounting-the-nfs-datasets'>Mounting the NFS datasets</a></li>
<li>⇢ <a href='#troubleshooting-files-not-appearing-in-replication'>Troubleshooting: Files not appearing in replication</a></li>
<li>⇢ ⇢ <a href='#configuring-automatic-key-loading-on-boot'>Configuring automatic key loading on boot</a></li>
<li>⇢ ⇢ <a href='#troubleshooting-zrepl-replication-not-working'>Troubleshooting: zrepl Replication Not Working</a></li>
<li>⇢ ⇢ <a href='#check-if-zrepl-services-are-running'>Check if zrepl Services are Running</a></li>
<li>⇢ ⇢ <a href='#check-zrepl-status-for-errors'>Check zrepl Status for Errors</a></li>
<li>⇢ ⇢ <a href='#fixing-no-common-snapshot-errors'>Fixing "No Common Snapshot" Errors</a></li>
<li>⇢ ⇢ <a href='#network-connectivity-issues'>Network Connectivity Issues</a></li>
<li>⇢ ⇢ <a href='#encryption-key-issues'>Encryption Key Issues</a></li>
<li>⇢ ⇢ <a href='#monitoring-ongoing-replication'>Monitoring Ongoing Replication</a></li>
<li>⇢ <a href='#carp-common-address-redundancy-protocol'>CARP (Common Address Redundancy Protocol)</a></li>
<li>⇢ ⇢ <a href='#how-carp-works'>How CARP Works</a></li>
<li>⇢ ⇢ <a href='#configuring-carp'>Configuring CARP</a></li>
<li>⇢ ⇢ <a href='#carp-state-change-notifications'>CARP State Change Notifications</a></li>
<li>⇢ <a href='#nfs-server-configuration'>NFS Server Configuration</a></li>
<li>⇢ ⇢ <a href='#setting-up-nfs-on-f0-primary'>Setting up NFS on <span class='inlinecode'>f0</span> (Primary)</a></li>
<li>⇢ ⇢ <a href='#configuring-stunnel-for-nfs-encryption-with-carp-failover'>Configuring Stunnel for NFS Encryption with CARP Failover</a></li>
<li>⇢ ⇢ <a href='#creating-a-certificate-authority-for-client-authentication'>Creating a Certificate Authority for Client Authentication</a></li>
<li>⇢ ⇢ <a href='#install-and-configure-stunnel-on-f0'>Install and Configure Stunnel on <span class='inlinecode'>f0</span></a></li>
<li>⇢ ⇢ <a href='#setting-up-nfs-on-f1-standby'>Setting up NFS on <span class='inlinecode'>f1</span> (Standby)</a></li>
<li>⇢ ⇢ <a href='#carp-control-script-for-clean-failover'>CARP Control Script for Clean Failover</a></li>
<li>⇢ ⇢ <a href='#carp-management-script'>CARP Management Script</a></li>
<li>⇢ ⇢ <a href='#automatic-failback-after-reboot'>Automatic Failback After Reboot</a></li>
<li>⇢ <a href='#client-configuration-for-nfs-via-stunnel'>Client Configuration for NFS via Stunnel</a></li>
<li>⇢ ⇢ <a href='#configuring-rocky-linux-clients-r0-r1-r2'>Configuring Rocky Linux Clients (<span class='inlinecode'>r0</span>, <span class='inlinecode'>r1</span>, <span class='inlinecode'>r2</span>)</a></li>
<li>⇢ ⇢ <a href='#nfsv4-user-mapping-config-on-rocky'>NFSv4 user mapping config on Rocky</a></li>
<li>⇢ ⇢ <a href='#testing-nfs-mount-with-stunnel'>Testing NFS Mount with Stunnel</a></li>
<li>⇢ ⇢ <a href='#testing-carp-failover-with-mounted-clients-and-stale-file-handles'>Testing CARP Failover with mounted clients and stale file handles:</a></li>
<li>⇢ ⇢ <a href='#complete-failover-test'>Complete Failover Test</a></li>
<li>⇢ <a href='#update-upgrade-to-4tb-drives'>Update: Upgrade to 4TB drives</a></li>
<li>⇢ ⇢ <a href='#upgrading-f1-simpler-approach'>Upgrading f1 (simpler approach)</a></li>
<li>⇢ ⇢ <a href='#upgrading-f0-using-zfs-resilvering'>Upgrading f0 (using ZFS resilvering)</a></li>
<li>⇢ <a href='#conclusion'>Conclusion</a></li>
<li>⇢ <a href='#future-storage-explorations'>Future Storage Explorations</a></li>
<li>⇢ ⇢ <a href='#minio-for-s3-compatible-object-storage'>MinIO for S3-Compatible Object Storage</a></li>
<li>⇢ ⇢ <a href='#moosefs-for-distributed-high-availability'>MooseFS for Distributed High Availability</a></li>
</ul><br />
<h2 style='display: inline' id='introduction'>Introduction</h2><br />
<br />
<span>In the previous posts, we set up a WireGuard mesh network. In the future, we will also set up a Kubernetes cluster. Kubernetes workloads often require persistent storage for databases, configuration files, and application data. Local storage on each node has significant limitations:</span><br />
<br />
<ul>
<li>No data sharing: Pods (once we run Kubernetes) on different nodes can't access the same data</li>
<li>Pod mobility: If a pod moves to another node, it loses access to its data</li>
<li>No redundancy: Hardware failure means data loss</li>
</ul><br />
<span>This post implements a robust storage solution using:</span><br />
<br />
<ul>
<li>CARP: For high availability with automatic IP failover</li>
<li>NFS over stunnel: For secure, encrypted network storage</li>
<li>ZFS: For data integrity, encryption, and efficient snapshots</li>
<li><span class='inlinecode'>zrepl</span>: For continuous ZFS replication between nodes</li>
</ul><br />
<span>The result is a highly available, encrypted storage system that survives node failures while providing shared storage to all Kubernetes pods.</span><br />
<br />
<span>Other than what was mentioned in the first post of this blog series, we aren't using HAST, but <span class='inlinecode'>zrepl</span> for data replication. Read more about it later in this blog post.</span><br />
<br />
<h2 style='display: inline' id='additional-storage-capacity'>Additional storage capacity</h2><br />
<br />
<span>We add 1 TB of additional storage to each of the nodes (<span class='inlinecode'>f0</span>, <span class='inlinecode'>f1</span>, <span class='inlinecode'>f2</span>) in the form of an SSD drive. The Beelink mini PCs have enough space in the chassis for the extra space.</span><br />
<br />
<a href='./f3s-kubernetes-with-freebsd-part-6/drives.jpg'><img src='./f3s-kubernetes-with-freebsd-part-6/drives.jpg' /></a><br />
<br />
<span>Upgrading the storage was as easy as unscrewing, plugging the drive in, and then screwing it back together again. The procedure was uneventful! We're using two different SSD models (Samsung 870 EVO and Crucial BX500) to avoid simultaneous failures from the same manufacturing batch.</span><br />
<br />
<span>We then create the <span class='inlinecode'>zdata</span> ZFS pool on all three nodes:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zpool create -m /data zdata /dev/ada</font><font color="#bb00ff">1</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> zpool list</font>
<font color="#ff0000">NAME SIZE ALLOC FREE CKPOINT EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT</font>
<font color="#ff0000">zdata 928G </font><font color="#bb00ff">12</font><font color="#F3E651">.</font><font color="#ff0000">1M 928G - - </font><font color="#bb00ff">0</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">0</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">1</font><font color="#F3E651">.</font><font color="#ff0000">00x ONLINE -</font>
<font color="#ff0000">zroot 472G </font><font color="#bb00ff">29</font><font color="#F3E651">.</font><font color="#ff0000">0G 443G - - </font><font color="#bb00ff">0</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">6</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">1</font><font color="#F3E651">.</font><font color="#ff0000">00x ONLINE -</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:/</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas camcontrol devlist</font>
<font color="#F3E651"><</font><font color="#ff0000">512GB SSD D910R170</font><font color="#F3E651">></font><font color="#ff0000"> at scbus0 target </font><font color="#bb00ff">0</font><font color="#ff0000"> lun </font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#F3E651">(</font><font color="#ff0000">pass0</font><font color="#F3E651">,</font><font color="#ff0000">ada0</font><font color="#F3E651">)</font>
<font color="#F3E651"><</font><font color="#ff0000">Samsung SSD </font><font color="#bb00ff">870</font><font color="#ff0000"> EVO 1TB SVT03B6Q</font><font color="#F3E651">></font><font color="#ff0000"> at scbus1 target </font><font color="#bb00ff">0</font><font color="#ff0000"> lun </font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#F3E651">(</font><font color="#ff0000">pass1</font><font color="#F3E651">,</font><font color="#ff0000">ada1</font><font color="#F3E651">)</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:/</font><font color="#ff0000"> </font><font color="#F3E651">%</font>
</pre>
<br />
<span>To verify that we have a different SSD on the second node (the third node has the same drive as the first):</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f1</font><font color="#F3E651">:/</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas camcontrol devlist</font>
<font color="#F3E651"><</font><font color="#ff0000">512GB SSD D910R170</font><font color="#F3E651">></font><font color="#ff0000"> at scbus0 target </font><font color="#bb00ff">0</font><font color="#ff0000"> lun </font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#F3E651">(</font><font color="#ff0000">pass0</font><font color="#F3E651">,</font><font color="#ff0000">ada0</font><font color="#F3E651">)</font>
<font color="#F3E651"><</font><font color="#ff0000">CT1000BX500SSD1 M6CR072</font><font color="#F3E651">></font><font color="#ff0000"> at scbus1 target </font><font color="#bb00ff">0</font><font color="#ff0000"> lun </font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#F3E651">(</font><font color="#ff0000">pass1</font><font color="#F3E651">,</font><font color="#ff0000">ada1</font><font color="#F3E651">)</font>
</pre>
<br />
<h2 style='display: inline' id='zfs-encryption-keys'>ZFS encryption keys</h2><br />
<br />
<span>ZFS native encryption requires encryption keys to unlock datasets. We need a secure method to store these keys that balances security with operational needs:</span><br />
<br />
<ul>
<li>Security: Keys must not be stored on the same disks they encrypt</li>
<li>Availability: Keys must be available at boot for automatic mounting</li>
<li>Portability: Keys should be easily moved between systems for recovery</li>
</ul><br />
<span>Using USB flash drives as hardware key storage provides a convenient and elegant solution. The encrypted data is unreadable without physical access to the USB key, protecting against disk theft or improper disposal. In production environments, you may use enterprise key management systems; however, for a home lab, USB keys offer good security with minimal complexity.</span><br />
<br />
<h3 style='display: inline' id='ufs-on-usb-keys'>UFS on USB keys</h3><br />
<br />
<span>We'll format the USB drives with UFS (Unix File System) rather than ZFS for simplicity. There is no need to use ZFS.</span><br />
<br />
<span>Let's see the USB keys:</span><br />
<br />
<a href='./f3s-kubernetes-with-freebsd-part-6/usbkeys1.jpg'><img alt='USB keys' title='USB keys' src='./f3s-kubernetes-with-freebsd-part-6/usbkeys1.jpg' /></a><br />
<br />
<span>To verify that the USB key (flash disk) is there:</span><br />
<br />
<pre>
paul@f0:/ % doas camcontrol devlist
<512GB SSD D910R170> at scbus0 target 0 lun 0 (pass0,ada0)
<Samsung SSD 870 EVO 1TB SVT03B6Q> at scbus1 target 0 lun 0 (pass1,ada1)
<Generic Flash Disk 8.07> at scbus2 target 0 lun 0 (da0,pass2)
paul@f0:/ %
</pre>
<br />
<span>Let's create the UFS file system and mount it (done on all three nodes <span class='inlinecode'>f0</span>, <span class='inlinecode'>f1</span> and <span class='inlinecode'>f2</span>):</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:/</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas newfs /dev/da</font><font color="#bb00ff">0</font>
<font color="#ff0000">/dev/da</font><font color="#bb00ff">0</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">15000</font><font color="#F3E651">.</font><font color="#ff0000">0MB </font><font color="#F3E651">(</font><font color="#bb00ff">30720000</font><font color="#ff0000"> sectors</font><font color="#F3E651">)</font><font color="#ff0000"> block size </font><font color="#bb00ff">32768</font><font color="#F3E651">,</font><font color="#ff0000"> fragment size </font><font color="#bb00ff">4096</font>
<font color="#ff0000"> using </font><font color="#bb00ff">24</font><font color="#ff0000"> cylinder groups of </font><font color="#bb00ff">625</font><font color="#F3E651">.</font><font color="#ff0000">22MB</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">20007</font><font color="#ff0000"> blks</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">80128</font><font color="#ff0000"> inodes</font><font color="#F3E651">.</font>
<font color="#ff0000"> with soft updates</font>
<font color="#ff0000">super-block backups </font><font color="#F3E651">(</font><b><font color="#ffffff">for</font></b><font color="#ff0000"> fsck_ffs -b </font><i><font color="#ababab">#) at:</font></i>
<font color="#ff0000"> </font><font color="#bb00ff">192</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">1280640</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">2561088</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">3841536</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">5121984</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">6402432</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">7682880</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">8963328</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">10243776</font><font color="#F3E651">,</font>
<font color="#bb00ff">11524224</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">12804672</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">14085120</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">15365568</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">16646016</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">17926464</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">19206912</font><font color="#F3E651">,</font><font color="#ff0000">k </font><font color="#bb00ff">20487360</font><font color="#F3E651">,</font>
<font color="#F3E651">...</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:/</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> echo </font><font color="#bb00ff">'/dev/da0 /keys ufs rw 0 2'</font><font color="#ff0000"> </font><font color="#F3E651">|</font><font color="#ff0000"> doas tee -a /etc/fstab</font>
<font color="#ff0000">/dev/da</font><font color="#bb00ff">0</font><font color="#ff0000"> /keys ufs rw </font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#bb00ff">2</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:/</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas mkdir /keys</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:/</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas mount /keys</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:/</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> df </font><font color="#F3E651">|</font><font color="#ff0000"> grep keys</font>
<font color="#ff0000">/dev/da</font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#bb00ff">14877596</font><font color="#ff0000"> </font><font color="#bb00ff">8</font><font color="#ff0000"> </font><font color="#bb00ff">13687384</font><font color="#ff0000"> </font><font color="#bb00ff">0</font><font color="#F3E651">%</font><font color="#ff0000"> /keys</font>
</pre>
<br />
<a href='./f3s-kubernetes-with-freebsd-part-6/usbkeys2.jpg'><img alt='USB keys stuck in' title='USB keys stuck in' src='./f3s-kubernetes-with-freebsd-part-6/usbkeys2.jpg' /></a><br />
<br />
<h3 style='display: inline' id='generating-encryption-keys'>Generating encryption keys</h3><br />
<br />
<span>The following keys will later be used to encrypt the ZFS file systems. They will be stored on all three nodes, serving as a backup in case one of the keys is lost or corrupted. When we later replicate encrypted ZFS volumes from one node to another, the keys must also be available on the destination node.</span><br />
<br />
<pre>
paul@f0:/keys % doas openssl rand -out /keys/f0.lan.buetow.org:bhyve.key 32
paul@f0:/keys % doas openssl rand -out /keys/f1.lan.buetow.org:bhyve.key 32
paul@f0:/keys % doas openssl rand -out /keys/f2.lan.buetow.org:bhyve.key 32
paul@f0:/keys % doas openssl rand -out /keys/f0.lan.buetow.org:zdata.key 32
paul@f0:/keys % doas openssl rand -out /keys/f1.lan.buetow.org:zdata.key 32
paul@f0:/keys % doas openssl rand -out /keys/f2.lan.buetow.org:zdata.key 32
paul@f0:/keys % doas chown root *
paul@f0:/keys % doas chmod 400 *
paul@f0:/keys % ls -l
total 20
*r-------- 1 root wheel 32 May 25 13:07 f0.lan.buetow.org:bhyve.key
*r-------- 1 root wheel 32 May 25 13:07 f1.lan.buetow.org:bhyve.key
*r-------- 1 root wheel 32 May 25 13:07 f2.lan.buetow.org:bhyve.key
*r-------- 1 root wheel 32 May 25 13:07 f0.lan.buetow.org:zdata.key
*r-------- 1 root wheel 32 May 25 13:07 f1.lan.buetow.org:zdata.key
*r-------- 1 root wheel 32 May 25 13:07 f2.lan.buetow.org:zdata.key
</pre>
<br />
<span>After creation, these are copied to the other two nodes, <span class='inlinecode'>f1</span> and <span class='inlinecode'>f2</span>, into the <span class='inlinecode'>/keys</span> partition (I won't provide the commands here; create a tarball, copy it over, and extract it on the destination nodes).</span><br />
<br />
<h3 style='display: inline' id='configuring-zdata-zfs-pool-encryption'>Configuring <span class='inlinecode'>zdata</span> ZFS pool encryption</h3><br />
<br />
<span>Let's encrypt our <span class='inlinecode'>zdata</span> ZFS pool. We are not encrypting the whole pool, but everything within the <span class='inlinecode'>zdata/enc</span> data set:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs create -o </font><font color="#ff0000">encryption</font><font color="#F3E651">=</font><font color="#ff0000">on -o </font><font color="#ff0000">keyformat</font><font color="#F3E651">=</font><font color="#ff0000">raw -o </font><font color="#F3E651">\</font>
<font color="#ff0000"> </font><font color="#ff0000">keylocation</font><font color="#F3E651">=</font><font color="#ff0000">file</font><font color="#F3E651">:</font><font color="#ff0000">///keys</font><font color="#F3E651">/</font><font color="#ff0000">`hostname`</font><font color="#F3E651">:</font><font color="#ff0000">zdata</font><font color="#F3E651">.</font><font color="#ff0000">key zdata/enc</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:/</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> zfs list </font><font color="#F3E651">|</font><font color="#ff0000"> grep zdata</font>
<font color="#ff0000">zdata 836K 899G 96K /data</font>
<font color="#ff0000">zdata/enc 200K 899G 200K /data/enc</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> zfs get all zdata/enc </font><font color="#F3E651">|</font><font color="#ff0000"> grep -E -i </font><font color="#bb00ff">'(encryption|key)'</font>
<font color="#ff0000">zdata/enc encryption aes-</font><font color="#bb00ff">256</font><font color="#ff0000">-gcm -</font>
<font color="#ff0000">zdata/enc keylocation file</font><font color="#F3E651">:</font><font color="#ff0000">///keys/f</font><font color="#bb00ff">0</font><font color="#F3E651">.</font><font color="#ff0000">lan</font><font color="#F3E651">.</font><font color="#ff0000">buetow</font><font color="#F3E651">.</font><font color="#ff0000">org</font><font color="#F3E651">:</font><font color="#ff0000">zdata</font><font color="#F3E651">.</font><font color="#ff0000">key </font><b><font color="#ffffff">local</font></b>
<font color="#ff0000">zdata/enc keyformat raw -</font>
<font color="#ff0000">zdata/enc encryptionroot zdata/enc -</font>
<font color="#ff0000">zdata/enc keystatus available -</font>
</pre>
<br />
<span>All future data sets within <span class='inlinecode'>zdata/enc</span> will inherit the same encryption key.</span><br />
<br />
<h3 style='display: inline' id='migrating-bhyve-vms-to-an-encrypted-bhyve-zfs-volume'>Migrating Bhyve VMs to an encrypted <span class='inlinecode'>bhyve</span> ZFS volume</h3><br />
<br />
<span>We set up Bhyve VMs in a previous blog post. Their ZFS data sets rely on <span class='inlinecode'>zroot</span>, which is the default ZFS pool on the internal 512GB NVME drive. They aren't encrypted yet, so we encrypt the VM data sets as well now. To do so, we first shut down the VMs on all three nodes:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> doas vm stop rocky</font>
<font color="#ff0000">Sending ACPI shutdown to rocky</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> doas vm list</font>
<font color="#ff0000">NAME DATASTORE LOADER CPU MEMORY VNC AUTO STATE</font>
<font color="#ff0000">rocky default uefi </font><font color="#bb00ff">4</font><font color="#ff0000"> 14G - Yes </font><font color="#F3E651">[</font><font color="#bb00ff">1</font><font color="#F3E651">]</font><font color="#ff0000"> Stopped</font>
</pre>
<br />
<span>After this, we rename the unencrypted data set to <span class='inlinecode'>_old</span>, create a new encrypted data set, and also snapshot it as <span class='inlinecode'>@hamburger</span>.</span><br />
<span> </span><br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs rename zroot/bhyve zroot/bhyve_old</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs </font><b><font color="#ffffff">set</font></b><font color="#ff0000"> </font><font color="#ff0000">mountpoint</font><font color="#F3E651">=</font><font color="#ff0000">/mnt zroot/bhyve_old</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs snapshot zroot/bhyve_old/rocky@hamburger</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs create -o </font><font color="#ff0000">encryption</font><font color="#F3E651">=</font><font color="#ff0000">on -o </font><font color="#ff0000">keyformat</font><font color="#F3E651">=</font><font color="#ff0000">raw -o </font><font color="#F3E651">\</font>
<font color="#ff0000"> </font><font color="#ff0000">keylocation</font><font color="#F3E651">=</font><font color="#ff0000">file</font><font color="#F3E651">:</font><font color="#ff0000">///keys</font><font color="#F3E651">/</font><font color="#ff0000">`hostname`</font><font color="#F3E651">:</font><font color="#ff0000">bhyve</font><font color="#F3E651">.</font><font color="#ff0000">key zroot/bhyve</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs </font><b><font color="#ffffff">set</font></b><font color="#ff0000"> </font><font color="#ff0000">mountpoint</font><font color="#F3E651">=</font><font color="#ff0000">/zroot/bhyve zroot/bhyve</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs </font><b><font color="#ffffff">set</font></b><font color="#ff0000"> </font><font color="#ff0000">mountpoint</font><font color="#F3E651">=</font><font color="#ff0000">/zroot/bhyve/rocky zroot/bhyve/rocky</font>
</pre>
<br />
<span>Once done, we import the snapshot into the encrypted dataset and also copy some other metadata files from <span class='inlinecode'>vm-bhyve</span> back over.</span><br />
<br />
<pre>
paul@f0:/keys % doas zfs send zroot/bhyve_old/rocky@hamburger | \
doas zfs recv zroot/bhyve/rocky
paul@f0:/keys % doas cp -Rp /mnt/.config /zroot/bhyve/
paul@f0:/keys % doas cp -Rp /mnt/.img /zroot/bhyve/
paul@f0:/keys % doas cp -Rp /mnt/.templates /zroot/bhyve/
paul@f0:/keys % doas cp -Rp /mnt/.iso /zroot/bhyve/
</pre>
<br />
<span>We also have to make encrypted ZFS data sets mount automatically on boot:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">zfskeys_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font>
<font color="#ff0000">zfskeys_enable</font><font color="#F3E651">:</font><font color="#ff0000"> -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> doas vm init</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> doas reboot</font>
<font color="#F3E651">.</font>
<font color="#F3E651">.</font>
<font color="#F3E651">.</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas vm list</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas vm list</font>
<font color="#ff0000">NAME DATASTORE LOADER CPU MEMORY VNC AUTO STATE</font>
<font color="#ff0000">rocky default uefi </font><font color="#bb00ff">4</font><font color="#ff0000"> 14G </font><font color="#bb00ff">0.0</font><font color="#F3E651">.</font><font color="#bb00ff">0.0</font><font color="#F3E651">:</font><font color="#bb00ff">5900</font><font color="#ff0000"> Yes </font><font color="#F3E651">[</font><font color="#bb00ff">1</font><font color="#F3E651">]</font><font color="#ff0000"> Running </font><font color="#F3E651">(</font><font color="#bb00ff">2265</font><font color="#F3E651">)</font>
</pre>
<br />
<span>As you can see, the VM is running. This means the encrypted <span class='inlinecode'>zroot/bhyve</span> was mounted successfully after the reboot! Now we can destroy the old, unencrypted, and now unused bhyve dataset:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs destroy -R zroot/bhyve_old</font>
</pre>
<br />
<span>To verify once again that <span class='inlinecode'>zroot/bhyve</span> and <span class='inlinecode'>zroot/bhyve/rocky</span> are now both encrypted, we run:</span><br />
<span> </span><br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> zfs get all zroot/bhyve </font><font color="#F3E651">|</font><font color="#ff0000"> grep -E </font><font color="#bb00ff">'(encryption|key)'</font>
<font color="#ff0000">zroot/bhyve encryption aes-</font><font color="#bb00ff">256</font><font color="#ff0000">-gcm -</font>
<font color="#ff0000">zroot/bhyve keylocation file</font><font color="#F3E651">:</font><font color="#ff0000">///keys/f</font><font color="#bb00ff">0</font><font color="#F3E651">.</font><font color="#ff0000">lan</font><font color="#F3E651">.</font><font color="#ff0000">buetow</font><font color="#F3E651">.</font><font color="#ff0000">org</font><font color="#F3E651">:</font><font color="#ff0000">bhyve</font><font color="#F3E651">.</font><font color="#ff0000">key </font><b><font color="#ffffff">local</font></b>
<font color="#ff0000">zroot/bhyve keyformat raw -</font>
<font color="#ff0000">zroot/bhyve encryptionroot zroot/bhyve -</font>
<font color="#ff0000">zroot/bhyve keystatus available -</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> zfs get all zroot/bhyve/rocky </font><font color="#F3E651">|</font><font color="#ff0000"> grep -E </font><font color="#bb00ff">'(encryption|key)'</font>
<font color="#ff0000">zroot/bhyve/rocky encryption aes-</font><font color="#bb00ff">256</font><font color="#ff0000">-gcm -</font>
<font color="#ff0000">zroot/bhyve/rocky keylocation none default</font>
<font color="#ff0000">zroot/bhyve/rocky keyformat raw -</font>
<font color="#ff0000">zroot/bhyve/rocky encryptionroot zroot/bhyve -</font>
<font color="#ff0000">zroot/bhyve/rocky keystatus available -</font>
</pre>
<br />
<h2 style='display: inline' id='zfs-replication-with-zrepl'>ZFS Replication with <span class='inlinecode'>zrepl</span></h2><br />
<br />
<span>Data replication is the cornerstone of high availability. While CARP handles IP failover (see later in this post), we need continuous data replication to ensure the backup server has current data when it becomes active. Without replication, failover would result in data loss or require shared storage (like iSCSI), which introduces a single point of failure.</span><br />
<br />
<h3 style='display: inline' id='understanding-replication-requirements'>Understanding Replication Requirements</h3><br />
<br />
<span>Our storage system has different replication needs:</span><br />
<br />
<ul>
<li>NFS data (<span class='inlinecode'>/data/nfs/k3svolumes</span>): Soon, it will contain active Kubernetes persistent volumes. Needs frequent replication (every minute) to minimise data loss during failover.</li>
<li>VM data (<span class='inlinecode'>/zroot/bhyve/freebsd</span>): Contains VM images that change less frequently. Can tolerate longer replication intervals (every 10 minutes).</li>
</ul><br />
<span>The 1-minute replication window is perfectly acceptable for my personal use cases. This isn't a high-frequency trading system or a real-time database—it's storage for personal projects, development work, and home lab experiments. Losing at most 1 minute of work in a disaster scenario is a reasonable trade-off for the reliability and simplicity of snapshot-based replication. Additionally, in the case of a "1 minute of data loss," I would likely still have the data available on the client side.</span><br />
<br />
<span>Why use <span class='inlinecode'>zrepl</span> instead of HAST? While HAST (Highly Available Storage) is FreeBSD's native solution for high-availability storage and supports synchronous replication—thus eliminating the mentioned 1-minute window—I've chosen <span class='inlinecode'>zrepl</span> for several important reasons:</span><br />
<br />
<ul>
<li>HAST can cause ZFS corruption: HAST operates at the block level and doesn't understand ZFS's transactional semantics. During failover, in-flight transactions can lead to corrupted zpools. I've experienced this firsthand (I am confident I have configured something wrong) - the automatic failover would trigger while ZFS was still writing, resulting in an unmountable pool.</li>
<li>ZFS-aware replication: <span class='inlinecode'>zrepl</span> understands ZFS datasets and snapshots. It replicates at the dataset level, ensuring each snapshot is a consistent point-in-time copy. This is fundamentally safer than block-level replication.</li>
<li>Snapshot history: With <span class='inlinecode'>zrepl</span>, you get multiple recovery points (every minute for NFS data in our setup). If corruption occurs, you can roll back to any previous snapshot. HAST only gives you the current state.</li>
<li>Easier recovery: When something goes wrong with <span class='inlinecode'>zrepl</span>, you still have intact snapshots on both sides. With HAST, a corrupted primary often means a corrupted secondary as well.</li>
</ul><br />
<a class='textlink' href='https://wiki.freebsd.org/HighlyAvailableStorage'>FreeBSD HAST</a><br />
<br />
<h3 style='display: inline' id='installing-zrepl'>Installing <span class='inlinecode'>zrepl</span></h3><br />
<br />
<span>First, install <span class='inlinecode'>zrepl</span> on both hosts involved (we will replicate data from <span class='inlinecode'>f0</span> to <span class='inlinecode'>f1</span>):</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas pkg install -y zrepl</font>
</pre>
<br />
<span>Then, we verify the pools and datasets on both hosts:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># On f0</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zpool list</font>
<font color="#ff0000">NAME SIZE ALLOC FREE CKPOINT EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT</font>
<font color="#ff0000">zdata 928G </font><font color="#bb00ff">1</font><font color="#F3E651">.</font><font color="#ff0000">03M 928G - - </font><font color="#bb00ff">0</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">0</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">1</font><font color="#F3E651">.</font><font color="#ff0000">00x ONLINE -</font>
<font color="#ff0000">zroot 472G </font><font color="#bb00ff">26</font><font color="#F3E651">.</font><font color="#ff0000">7G 445G - - </font><font color="#bb00ff">0</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">5</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">1</font><font color="#F3E651">.</font><font color="#ff0000">00x ONLINE -</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs list -r zdata/enc</font>
<font color="#ff0000">NAME USED AVAIL REFER MOUNTPOINT</font>
<font color="#ff0000">zdata/enc 200K 899G 200K /data/enc</font>
<i><font color="#ababab"># On f1</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zpool list</font>
<font color="#ff0000">NAME SIZE ALLOC FREE CKPOINT EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT</font>
<font color="#ff0000">zdata 928G 956K 928G - - </font><font color="#bb00ff">0</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">0</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">1</font><font color="#F3E651">.</font><font color="#ff0000">00x ONLINE -</font>
<font color="#ff0000">zroot 472G </font><font color="#bb00ff">11</font><font color="#F3E651">.</font><font color="#ff0000">7G 460G - - </font><font color="#bb00ff">0</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">2</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">1</font><font color="#F3E651">.</font><font color="#ff0000">00x ONLINE -</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs list -r zdata/enc</font>
<font color="#ff0000">NAME USED AVAIL REFER MOUNTPOINT</font>
<font color="#ff0000">zdata/enc 200K 899G 200K /data/enc</font>
</pre>
<br />
<span>Since we have a WireGuard tunnel between <span class='inlinecode'>f0</span> and f1, we'll use TCP transport over the secure tunnel instead of SSH. First, check the WireGuard IP addresses:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># Check WireGuard interface IPs</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> ifconfig wg0 </font><font color="#F3E651">|</font><font color="#ff0000"> grep inet</font>
<font color="#ff0000"> inet </font><font color="#bb00ff">192.168</font><font color="#F3E651">.</font><font color="#bb00ff">2.130</font><font color="#ff0000"> netmask </font><font color="#bb00ff">0xffffff00</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> ifconfig wg0 </font><font color="#F3E651">|</font><font color="#ff0000"> grep inet</font>
<font color="#ff0000"> inet </font><font color="#bb00ff">192.168</font><font color="#F3E651">.</font><font color="#bb00ff">2.131</font><font color="#ff0000"> netmask </font><font color="#bb00ff">0xffffff00</font>
</pre>
<br />
<span>Let's create a dedicated dataset for NFS data that will be replicated:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># Create the nfsdata dataset that will hold all data exposed via NFS</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs create zdata/enc/nfsdata</font>
</pre>
<br />
<span>Afterwards, we create the <span class='inlinecode'>zrepl</span> configuration on <span class='inlinecode'>f0</span>:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas tee /usr/local/etc/zrepl/zrepl</font><font color="#F3E651">.</font><font color="#ff0000">yml </font><font color="#F3E651"><<</font><font color="#bb00ff">'EOF'</font>
<font color="#ff0000">global</font><font color="#F3E651">:</font>
<font color="#ff0000"> logging</font><font color="#F3E651">:</font>
<font color="#ff0000"> - </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> stdout</font>
<font color="#ff0000"> level</font><font color="#F3E651">:</font><font color="#ff0000"> info</font>
<font color="#ff0000"> format</font><font color="#F3E651">:</font><font color="#ff0000"> human</font>
<b><font color="#ffffff">jobs</font></b><font color="#F3E651">:</font>
<font color="#ff0000"> - name</font><font color="#F3E651">:</font><font color="#ff0000"> f0_to_f1_nfsdata</font>
<font color="#ff0000"> </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> push</font>
<font color="#ff0000"> connect</font><font color="#F3E651">:</font>
<font color="#ff0000"> </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> tcp</font>
<font color="#ff0000"> address</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">"192.168.2.131:8888"</font>
<font color="#ff0000"> filesystems</font><font color="#F3E651">:</font>
<font color="#ff0000"> </font><font color="#bb00ff">"zdata/enc/nfsdata"</font><font color="#F3E651">:</font><font color="#ff0000"> </font><b><font color="#ffffff">true</font></b>
<font color="#ff0000"> send</font><font color="#F3E651">:</font>
<font color="#ff0000"> encrypted</font><font color="#F3E651">:</font><font color="#ff0000"> </font><b><font color="#ffffff">true</font></b>
<font color="#ff0000"> snapshotting</font><font color="#F3E651">:</font>
<font color="#ff0000"> </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> periodic</font>
<font color="#ff0000"> prefix</font><font color="#F3E651">:</font><font color="#ff0000"> zrepl_</font>
<font color="#ff0000"> interval</font><font color="#F3E651">:</font><font color="#ff0000"> 1m</font>
<font color="#ff0000"> pruning</font><font color="#F3E651">:</font>
<font color="#ff0000"> keep_sender</font><font color="#F3E651">:</font>
<font color="#ff0000"> - </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> last_n</font>
<font color="#ff0000"> count</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">10</font>
<font color="#ff0000"> - </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> grid</font>
<font color="#ff0000"> grid</font><font color="#F3E651">:</font><font color="#ff0000"> 4x7d </font><font color="#F3E651">|</font><font color="#ff0000"> 6x30d</font>
<font color="#ff0000"> regex</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">"^zrepl_.*"</font>
<font color="#ff0000"> keep_receiver</font><font color="#F3E651">:</font>
<font color="#ff0000"> - </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> last_n</font>
<font color="#ff0000"> count</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">10</font>
<font color="#ff0000"> - </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> grid</font>
<font color="#ff0000"> grid</font><font color="#F3E651">:</font><font color="#ff0000"> 4x7d </font><font color="#F3E651">|</font><font color="#ff0000"> 6x30d</font>
<font color="#ff0000"> regex</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">"^zrepl_.*"</font>
<font color="#ff0000"> - name</font><font color="#F3E651">:</font><font color="#ff0000"> f0_to_f1_freebsd</font>
<font color="#ff0000"> </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> push</font>
<font color="#ff0000"> connect</font><font color="#F3E651">:</font>
<font color="#ff0000"> </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> tcp</font>
<font color="#ff0000"> address</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">"192.168.2.131:8888"</font>
<font color="#ff0000"> filesystems</font><font color="#F3E651">:</font>
<font color="#ff0000"> </font><font color="#bb00ff">"zroot/bhyve/freebsd"</font><font color="#F3E651">:</font><font color="#ff0000"> </font><b><font color="#ffffff">true</font></b>
<font color="#ff0000"> send</font><font color="#F3E651">:</font>
<font color="#ff0000"> encrypted</font><font color="#F3E651">:</font><font color="#ff0000"> </font><b><font color="#ffffff">true</font></b>
<font color="#ff0000"> snapshotting</font><font color="#F3E651">:</font>
<font color="#ff0000"> </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> periodic</font>
<font color="#ff0000"> prefix</font><font color="#F3E651">:</font><font color="#ff0000"> zrepl_</font>
<font color="#ff0000"> interval</font><font color="#F3E651">:</font><font color="#ff0000"> 10m</font>
<font color="#ff0000"> pruning</font><font color="#F3E651">:</font>
<font color="#ff0000"> keep_sender</font><font color="#F3E651">:</font>
<font color="#ff0000"> - </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> last_n</font>
<font color="#ff0000"> count</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">10</font>
<font color="#ff0000"> - </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> grid</font>
<font color="#ff0000"> grid</font><font color="#F3E651">:</font><font color="#ff0000"> 4x7d</font>
<font color="#ff0000"> regex</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">"^zrepl_.*"</font>
<font color="#ff0000"> keep_receiver</font><font color="#F3E651">:</font>
<font color="#ff0000"> - </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> last_n</font>
<font color="#ff0000"> count</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">10</font>
<font color="#ff0000"> - </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> grid</font>
<font color="#ff0000"> grid</font><font color="#F3E651">:</font><font color="#ff0000"> 4x7d</font>
<font color="#ff0000"> regex</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">"^zrepl_.*"</font>
<font color="#ff0000">EOF</font>
</pre>
<br />
<span> We're using two separate replication jobs with different intervals:</span><br />
<br />
<ul>
<li><span class='inlinecode'>f0_to_f1_nfsdata</span>: Replicates NFS data every minute for faster failover recovery</li>
<li><span class='inlinecode'>f0_to_f1_freebsd</span>: Replicates FreeBSD VM every ten minutes (less critical)</li>
</ul><br />
<span>The FreeBSD VM is only used for development purposes, so it doesn't require as frequent replication as the NFS data. It's off-topic to this blog series, but it showcases how <span class='inlinecode'>zrepl</span>'s flexibility in handling different datasets with varying replication needs.</span><br />
<br />
<span>Furthermore:</span><br />
<br />
<ul>
<li>We're specifically replicating <span class='inlinecode'>zdata/enc/nfsdata</span> instead of the entire <span class='inlinecode'>zdata/enc</span> dataset. This dedicated dataset will contain all the data we later want to expose via NFS, keeping a clear separation between replicated NFS data and other local encrypted data.</li>
<li>We use <span class='inlinecode'>send: encrypted: true</span> to keep the replication stream encrypted. While WireGuard already encrypts in transit, this provides additional protection. For reduced CPU overhead, you could set <span class='inlinecode'>encrypted: false</span> since the tunnel is secure.</li>
</ul><br />
<h3 style='display: inline' id='configuring-zrepl-on-f1-sink'>Configuring <span class='inlinecode'>zrepl</span> on <span class='inlinecode'>f1</span> (sink)</h3><br />
<br />
<span>On <span class='inlinecode'>f1</span> (the sink, meaning it's the node receiving the replication data), we configure <span class='inlinecode'>zrepl</span> to receive the data as follows:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># First, create a dedicated sink dataset</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs create zdata/sink</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas tee /usr/local/etc/zrepl/zrepl</font><font color="#F3E651">.</font><font color="#ff0000">yml </font><font color="#F3E651"><<</font><font color="#bb00ff">'EOF'</font>
<font color="#ff0000">global</font><font color="#F3E651">:</font>
<font color="#ff0000"> logging</font><font color="#F3E651">:</font>
<font color="#ff0000"> - </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> stdout</font>
<font color="#ff0000"> level</font><font color="#F3E651">:</font><font color="#ff0000"> info</font>
<font color="#ff0000"> format</font><font color="#F3E651">:</font><font color="#ff0000"> human</font>
<b><font color="#ffffff">jobs</font></b><font color="#F3E651">:</font>
<font color="#ff0000"> - name</font><font color="#F3E651">:</font><font color="#ff0000"> sink</font>
<font color="#ff0000"> </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> sink</font>
<font color="#ff0000"> serve</font><font color="#F3E651">:</font>
<font color="#ff0000"> </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> tcp</font>
<font color="#ff0000"> listen</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">"192.168.2.131:8888"</font>
<font color="#ff0000"> clients</font><font color="#F3E651">:</font>
<font color="#ff0000"> </font><font color="#bb00ff">"192.168.2.130"</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">"f0"</font>
<font color="#ff0000"> recv</font><font color="#F3E651">:</font>
<font color="#ff0000"> placeholder</font><font color="#F3E651">:</font>
<font color="#ff0000"> encryption</font><font color="#F3E651">:</font><font color="#ff0000"> inherit</font>
<font color="#ff0000"> root_fs</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">"zdata/sink"</font>
<font color="#ff0000">EOF</font>
</pre>
<br />
<h3 style='display: inline' id='enabling-and-starting-zrepl-services'>Enabling and starting <span class='inlinecode'>zrepl</span> services</h3><br />
<br />
<span>We then enable and start <span class='inlinecode'>zrepl</span> on both hosts via:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># On f0</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">zrepl_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font>
<font color="#ff0000">zrepl_enable</font><font color="#F3E651">:</font><font color="#ff0000"> -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service `zrepl` start</font>
<font color="#ff0000">Starting zrepl</font><font color="#F3E651">.</font>
<i><font color="#ababab"># On f1</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">zrepl_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font>
<font color="#ff0000">zrepl_enable</font><font color="#F3E651">:</font><font color="#ff0000"> -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service `zrepl` start</font>
<font color="#ff0000">Starting zrepl</font><font color="#F3E651">.</font>
</pre>
<br />
<span>To check the replication status, we run:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># On f0, check `zrepl` status (use raw mode for non-tty)</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas pkg install jq</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zrepl status --mode raw </font><font color="#F3E651">|</font><font color="#ff0000"> grep -A</font><font color="#bb00ff">2</font><font color="#ff0000"> </font><font color="#bb00ff">"Replication"</font><font color="#ff0000"> </font><font color="#F3E651">|</font><font color="#ff0000"> jq </font><font color="#F3E651">.</font>
<font color="#bb00ff">"Replication"</font><font color="#F3E651">:</font><font color="#ff0000">{</font><font color="#bb00ff">"StartAt"</font><font color="#F3E651">:</font><font color="#bb00ff">"2025-07-01T22:31:48.712143123+03:00"</font><font color="#F3E651">...</font>
<i><font color="#ababab"># Check if services are running</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service zrepl status</font>
<font color="#ff0000">zrepl is running as pid </font><font color="#bb00ff">2649</font><font color="#F3E651">.</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service zrepl status</font>
<font color="#ff0000">zrepl is running as pid </font><font color="#bb00ff">2574</font><font color="#F3E651">.</font>
<i><font color="#ababab"># Check for `zrepl` snapshots on source</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs list -t snapshot -r zdata/enc </font><font color="#F3E651">|</font><font color="#ff0000"> grep zrepl</font>
<font color="#ff0000">zdata/enc@zrepl_20250701_193148_000 0B - 176K -</font>
<i><font color="#ababab"># On f1, verify the replicated datasets </font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs list -r zdata </font><font color="#F3E651">|</font><font color="#ff0000"> grep f0</font>
<font color="#ff0000">zdata/f</font><font color="#bb00ff">0</font><font color="#ff0000"> 576K 899G 200K none</font>
<font color="#ff0000">zdata/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata 376K 899G 200K none</font>
<font color="#ff0000">zdata/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc 176K 899G 176K none</font>
<i><font color="#ababab"># Check replicated snapshots on f1</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs list -t snapshot -r zdata </font><font color="#F3E651">|</font><font color="#ff0000"> grep zrepl</font>
<font color="#ff0000">zdata/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc@zrepl_20250701_193148_000 0B - 176K -</font>
<font color="#ff0000">zdata/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc@zrepl_20250701_194148_000 0B - 176K -</font>
<font color="#F3E651">.</font>
<font color="#F3E651">.</font>
<font color="#F3E651">.</font>
</pre>
<br />
<h3 style='display: inline' id='monitoring-replication'>Monitoring replication</h3><br />
<br />
<span>You can monitor the replication progress with:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zrepl status</font>
</pre>
<br />
<a href='./f3s-kubernetes-with-freebsd-part-6/zrepl.png'><img alt='zrepl status' title='zrepl status' src='./f3s-kubernetes-with-freebsd-part-6/zrepl.png' /></a><br />
<br />
<span>With this setup, both <span class='inlinecode'>zdata/enc/nfsdata</span> and <span class='inlinecode'>zroot/bhyve/freebsd</span> on <span class='inlinecode'>f0</span> will be automatically replicated to <span class='inlinecode'>f1</span> every 1 minute (or 10 minutes in the case of the FreeBSD VM), with encrypted snapshots preserved on both sides. The pruning policy ensures that we keep the last 10 snapshots while managing disk space efficiently.</span><br />
<br />
<span>The replicated data appears on <span class='inlinecode'>f1</span> under <span class='inlinecode'>zdata/sink/</span> with the source host and dataset hierarchy preserved:</span><br />
<br />
<ul>
<li><span class='inlinecode'>zdata/enc/nfsdata</span> → <span class='inlinecode'>zdata/sink/f0/zdata/enc/nfsdata</span></li>
<li><span class='inlinecode'>zroot/bhyve/freebsd</span> → <span class='inlinecode'>zdata/sink/f0/zroot/bhyve/freebsd</span></li>
</ul><br />
<span>This is by design - <span class='inlinecode'>zrepl</span> preserves the complete path from the source to ensure there are no conflicts when replicating from multiple sources.</span><br />
<br />
<h3 style='display: inline' id='verifying-replication-after-reboot'>Verifying replication after reboot</h3><br />
<br />
<span>The <span class='inlinecode'>zrepl</span> service is configured to start automatically at boot. After rebooting both hosts:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> uptime</font>
<font color="#bb00ff">11</font><font color="#F3E651">:</font><font color="#ff0000">17PM up </font><font color="#bb00ff">1</font><font color="#ff0000"> min</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">0</font><font color="#ff0000"> users</font><font color="#F3E651">,</font><font color="#ff0000"> load averages</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">0.16</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">0.06</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">0.02</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service `zrepl` status</font>
<font color="#ff0000">zrepl is running as pid </font><font color="#bb00ff">2366</font><font color="#F3E651">.</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service `zrepl` status</font>
<font color="#ff0000">zrepl is running as pid </font><font color="#bb00ff">2309</font><font color="#F3E651">.</font>
<i><font color="#ababab"># Check that new snapshots are being created and replicated</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs list -t snapshot </font><font color="#F3E651">|</font><font color="#ff0000"> grep `zrepl` </font><font color="#F3E651">|</font><font color="#ff0000"> tail -</font><font color="#bb00ff">2</font>
<font color="#ff0000">zdata/enc/nfsdata@zrepl_20250701_202530_000 0B - 200K -</font>
<font color="#ff0000">zroot/bhyve/freebsd@zrepl_20250701_202530_000 0B - </font><font color="#bb00ff">2</font><font color="#F3E651">.</font><font color="#ff0000">97G -</font>
<font color="#F3E651">.</font>
<font color="#F3E651">.</font>
<font color="#F3E651">.</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs list -t snapshot -r zdata/sink </font><font color="#F3E651">|</font><font color="#ff0000"> grep </font><font color="#bb00ff">202530</font>
<font color="#ff0000">zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata@zrepl_20250701_202530_000 0B - 176K -</font>
<font color="#ff0000">zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zroot/bhyve/freebsd@zrepl_20250701_202530_000 0B - </font><font color="#bb00ff">2</font><font color="#F3E651">.</font><font color="#ff0000">97G -</font>
<font color="#F3E651">.</font>
<font color="#F3E651">.</font>
<font color="#F3E651">.</font>
</pre>
<br />
<span>The timestamps confirm that replication resumed automatically after the reboot, ensuring continuous data protection. We can also write a test file to the NFS data directory on <span class='inlinecode'>f0</span> and verify whether it appears on <span class='inlinecode'>f1</span> after a minute.</span><br />
<br />
<h3 style='display: inline' id='understanding-failover-limitations-and-design-decisions'>Understanding Failover Limitations and Design Decisions</h3><br />
<br />
<span>Our system intentionally fails over to a read-only copy of the replica in the event of the primary's failure. This is due to the nature of <span class='inlinecode'>zrepl</span>, which only replicates data in one direction. If we mount the data set on the sink node in read-write mode, it would cause the ZFS dataset to diverge from the original, and the replication would break. It can still be mounted read-write on the sink node in case of a genuine issue on the primary node, but that step is left intentionally manual. Therefore, we don't need to fix the replication later on manually.</span><br />
<br />
<span>So in summary:</span><br />
<br />
<ul>
<li>Split-brain prevention: Automatic failover to a read-write copy can cause both nodes to become active simultaneously if network communication fails. This leads to data divergence that's extremely difficult to resolve.</li>
<li>False positive protection: Temporary network issues or high load can trigger unwanted failovers. Manual intervention ensures that failovers occur only when truly necessary.</li>
<li>Data integrity over availability: For storage systems, data consistency is paramount. A few minutes of downtime is preferable to data corruption in this specific use case.</li>
<li>Simplified recovery: With manual failover, you always know which dataset is authoritative, making recovery more straightforward.</li>
</ul><br />
<h3 style='display: inline' id='mounting-the-nfs-datasets'>Mounting the NFS datasets</h3><br />
<br />
<span>To make the NFS data accessible on both nodes, we need to mount it. On <span class='inlinecode'>f0</span>, this is straightforward:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># On f0 - set mountpoint for the primary nfsdata</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs </font><b><font color="#ffffff">set</font></b><font color="#ff0000"> </font><font color="#ff0000">mountpoint</font><font color="#F3E651">=</font><font color="#ff0000">/data/nfs zdata/enc/nfsdata</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas mkdir -p /data/nfs</font>
<i><font color="#ababab"># Verify it's mounted</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> df -h /data/nfs</font>
<font color="#ff0000">Filesystem Size Used Avail Capacity Mounted on</font>
<font color="#ff0000">zdata/enc/nfsdata 899G 204K 899G </font><font color="#bb00ff">0</font><font color="#F3E651">%</font><font color="#ff0000"> /data/nfs</font>
</pre>
<br />
<span>On <span class='inlinecode'>f1</span>, we need to handle the encryption key and mount the standby copy:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># On f1 - first check encryption status</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs get keystatus zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font>
<font color="#ff0000">NAME PROPERTY VALUE SOURCE</font>
<font color="#ff0000">zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata keystatus unavailable -</font>
<i><font color="#ababab"># Load the encryption key (using f0's key stored on the USB)</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs load-key -L file</font><font color="#F3E651">:</font><font color="#ff0000">///keys/f</font><font color="#bb00ff">0</font><font color="#F3E651">.</font><font color="#ff0000">lan</font><font color="#F3E651">.</font><font color="#ff0000">buetow</font><font color="#F3E651">.</font><font color="#ff0000">org</font><font color="#F3E651">:</font><font color="#ff0000">zdata</font><font color="#F3E651">.</font><font color="#ff0000">key </font><font color="#F3E651">\</font>
<font color="#ff0000"> zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font>
<i><font color="#ababab"># Set mountpoint and mount (same path as f0 for easier failover)</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas mkdir -p /data/nfs</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs </font><b><font color="#ffffff">set</font></b><font color="#ff0000"> </font><font color="#ff0000">mountpoint</font><font color="#F3E651">=</font><font color="#ff0000">/data/nfs zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs mount zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font>
<i><font color="#ababab"># Make it read-only to prevent accidental writes that would break replication</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs </font><b><font color="#ffffff">set</font></b><font color="#ff0000"> </font><b><font color="#ffffff">readonly</font></b><font color="#F3E651">=</font><font color="#ff0000">on zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font>
<i><font color="#ababab"># Verify</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> df -h /data/nfs</font>
<font color="#ff0000">Filesystem Size Used Avail Capacity Mounted on</font>
<font color="#ff0000">zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata 896G 204K 896G </font><font color="#bb00ff">0</font><font color="#F3E651">%</font><font color="#ff0000"> /data/nfs</font>
</pre>
<br />
<span>Note: The dataset is mounted at the same path (<span class='inlinecode'>/data/nfs</span>) on both hosts to simplify failover procedures. The dataset on <span class='inlinecode'>f1</span> is set to <span class='inlinecode'>readonly=on</span> to prevent accidental modifications, which, as mentioned earlier, would break replication. If we did, replication from <span class='inlinecode'>f0</span> to <span class='inlinecode'>f1</span> would fail like this:</span><br />
<br />
<span class='quote'>cannot receive incremental stream: destination zdata/sink/f0/zdata/enc/nfsdata has been modified since most recent snapshot </span><br />
<br />
<span>To fix a broken replication after accidental writes, we can do:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># Option 1: Rollback to the last common snapshot (loses local changes)</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs rollback zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata@zrepl_20250701_204054_000</font>
<i><font color="#ababab"># Option 2: Make it read-only to prevent accidents again</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs </font><b><font color="#ffffff">set</font></b><font color="#ff0000"> </font><b><font color="#ffffff">readonly</font></b><font color="#F3E651">=</font><font color="#ff0000">on zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font>
</pre>
<br />
<span>And replication should work again!</span><br />
<br />
<h2 style='display: inline' id='troubleshooting-files-not-appearing-in-replication'>Troubleshooting: Files not appearing in replication</h2><br />
<br />
<span>If you write files to <span class='inlinecode'>/data/nfs/</span> on <span class='inlinecode'>f0</span> but they don't appear on <span class='inlinecode'>f1</span>, check if the dataset is mounted on <span class='inlinecode'>f0</span>?</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs list -o name</font><font color="#F3E651">,</font><font color="#ff0000">mountpoint</font><font color="#F3E651">,</font><font color="#ff0000">mounted </font><font color="#F3E651">|</font><font color="#ff0000"> grep nfsdata</font>
<font color="#ff0000">zdata/enc/nfsdata /data/nfs yes</font>
</pre>
<br />
<span>If it shows <span class='inlinecode'>no</span>, the dataset isn't mounted! This means files are being written to the root filesystem, not ZFS. Next, we should check whether the encryption key is loaded:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs get keystatus zdata/enc/nfsdata</font>
<font color="#ff0000">NAME PROPERTY VALUE SOURCE</font>
<font color="#ff0000">zdata/enc/nfsdata keystatus available -</font>
<i><font color="#ababab"># If "unavailable", load the key:</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs load-key -L file</font><font color="#F3E651">:</font><font color="#ff0000">///keys/f</font><font color="#bb00ff">0</font><font color="#F3E651">.</font><font color="#ff0000">lan</font><font color="#F3E651">.</font><font color="#ff0000">buetow</font><font color="#F3E651">.</font><font color="#ff0000">org</font><font color="#F3E651">:</font><font color="#ff0000">zdata</font><font color="#F3E651">.</font><font color="#ff0000">key zdata/enc/nfsdata</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs mount zdata/enc/nfsdata</font>
</pre>
<br />
<span>You can also verify that files are in the snapshot (not just the directory):</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> ls -la /data/nfs</font><font color="#F3E651">/.</font><font color="#ff0000">zfs/snapshot/zrepl_</font><font color="#F3E651">*/</font>
</pre>
<br />
<span>This issue commonly occurs after a reboot if the encryption keys aren't configured to load automatically.</span><br />
<br />
<h3 style='display: inline' id='configuring-automatic-key-loading-on-boot'>Configuring automatic key loading on boot</h3><br />
<br />
<span>To ensure all additional encrypted datasets are mounted automatically after reboot as well, we do:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># On f0 - configure all encrypted datasets</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">zfskeys_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font>
<font color="#ff0000">zfskeys_enable</font><font color="#F3E651">:</font><font color="#ff0000"> YES -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">zfskeys_datasets</font><font color="#F3E651">=</font><font color="#bb00ff">"zdata/enc zdata/enc/nfsdata zroot/bhyve"</font>
<font color="#ff0000">zfskeys_datasets</font><font color="#F3E651">:</font><font color="#ff0000"> -</font><font color="#F3E651">></font><font color="#ff0000"> zdata/enc zdata/enc/nfsdata zroot/bhyve</font>
<i><font color="#ababab"># Set correct key locations for all datasets</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs </font><b><font color="#ffffff">set</font></b><font color="#ff0000"> </font><font color="#F3E651">\</font>
<font color="#ff0000"> </font><font color="#ff0000">keylocation</font><font color="#F3E651">=</font><font color="#ff0000">file</font><font color="#F3E651">:</font><font color="#ff0000">///keys/f</font><font color="#bb00ff">0</font><font color="#F3E651">.</font><font color="#ff0000">lan</font><font color="#F3E651">.</font><font color="#ff0000">buetow</font><font color="#F3E651">.</font><font color="#ff0000">org</font><font color="#F3E651">:</font><font color="#ff0000">zdata</font><font color="#F3E651">.</font><font color="#ff0000">key zdata/enc/nfsdata</font>
<i><font color="#ababab"># On f1 - include the replicated dataset</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">zfskeys_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font>
<font color="#ff0000">zfskeys_enable</font><font color="#F3E651">:</font><font color="#ff0000"> YES -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#F3E651">\</font>
<font color="#ff0000"> </font><font color="#ff0000">zfskeys_datasets</font><font color="#F3E651">=</font><font color="#bb00ff">"zdata/enc zroot/bhyve zdata/sink/f0/zdata/enc/nfsdata"</font>
<font color="#ff0000">zfskeys_datasets</font><font color="#F3E651">:</font><font color="#ff0000"> -</font><font color="#F3E651">></font><font color="#ff0000"> zdata/enc zroot/bhyve zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font>
<i><font color="#ababab"># Set key location for replicated dataset</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs </font><b><font color="#ffffff">set</font></b><font color="#ff0000"> </font><font color="#F3E651">\</font>
<font color="#ff0000"> </font><font color="#ff0000">keylocation</font><font color="#F3E651">=</font><font color="#ff0000">file</font><font color="#F3E651">:</font><font color="#ff0000">///keys/f</font><font color="#bb00ff">0</font><font color="#F3E651">.</font><font color="#ff0000">lan</font><font color="#F3E651">.</font><font color="#ff0000">buetow</font><font color="#F3E651">.</font><font color="#ff0000">org</font><font color="#F3E651">:</font><font color="#ff0000">zdata</font><font color="#F3E651">.</font><font color="#ff0000">key zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font>
</pre>
<br />
<span>Important notes:</span><br />
<br />
<ul>
<li>Each encryption root needs its own key load entry</li>
<li>The replicated dataset on <span class='inlinecode'>f1</span> uses the same encryption key as the source on <span class='inlinecode'>f0</span></li>
<li>Always verify datasets are mounted after reboot with <span class='inlinecode'>zfs list -o name,mounted</span></li>
<li>Critical: Always ensure the replicated dataset on <span class='inlinecode'>f1</span> remains read-only with <span class='inlinecode'>doas zfs set readonly=on zdata/sink/f0/zdata/enc/nfsdata</span></li>
</ul><br />
<h3 style='display: inline' id='troubleshooting-zrepl-replication-not-working'>Troubleshooting: zrepl Replication Not Working</h3><br />
<br />
<span>If <span class='inlinecode'>zrepl</span> replication is not working, here's a systematic approach to diagnose and fix common issues:</span><br />
<br />
<h3 style='display: inline' id='check-if-zrepl-services-are-running'>Check if zrepl Services are Running</h3><br />
<br />
<span>First, verify that <span class='inlinecode'>zrepl</span> is running on both nodes:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># Check service status on both f0 and f1</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service zrepl status</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service zrepl status</font>
<i><font color="#ababab"># If not running, start the service</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service zrepl start</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service zrepl start</font>
</pre>
<br />
<h3 style='display: inline' id='check-zrepl-status-for-errors'>Check zrepl Status for Errors</h3><br />
<br />
<span>Use the status command to see detailed error information:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># Check detailed status (use --mode raw for non-tty environments)</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zrepl status --mode raw</font>
<i><font color="#ababab"># Look for error messages in the replication section</font></i>
<i><font color="#ababab"># Common errors include "no common snapshot" or connection failures</font></i>
</pre>
<br />
<h3 style='display: inline' id='fixing-no-common-snapshot-errors'>Fixing "No Common Snapshot" Errors</h3><br />
<br />
<span>This is the most common replication issue, typically occurring when:</span><br />
<br />
<ul>
<li>The receiver has existing snapshots that don't match the sender</li>
<li>Different snapshot naming schemes are in use</li>
<li>The receiver dataset was created independently</li>
</ul><br />
<span>**Error message example:**</span><br />
<pre>
no common snapshot or suitable bookmark between sender and receiver
</pre>
<br />
<span>**Solution: Clean up conflicting snapshots on receiver**</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># First, identify the destination dataset on f1</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs list </font><font color="#F3E651">|</font><font color="#ff0000"> grep sink</font>
<i><font color="#ababab"># Check existing snapshots on the problematic dataset</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs list -t snapshot </font><font color="#F3E651">|</font><font color="#ff0000"> grep nfsdata</font>
<i><font color="#ababab"># If you see snapshots with different naming (e.g., @daily-*, @weekly-*)</font></i>
<i><font color="#ababab"># these conflict with zrepl's @zrepl_* snapshots</font></i>
<i><font color="#ababab"># Destroy the entire destination dataset to allow clean replication</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs destroy -r zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font>
<i><font color="#ababab"># For VM replication, do the same for the freebsd dataset</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs destroy -r zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zroot/bhyve/freebsd</font>
<i><font color="#ababab"># Wake up zrepl to start fresh replication</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zrepl signal wakeup f0_to_f1_nfsdata</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zrepl signal wakeup f0_to_f1_freebsd</font>
<i><font color="#ababab"># Check replication status</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zrepl status --mode raw</font>
</pre>
<br />
<span>**Verification that replication is working:**</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># Look for "stepping" state and active zfs send processes</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zrepl status --mode raw </font><font color="#F3E651">|</font><font color="#ff0000"> grep -A</font><font color="#bb00ff">5</font><font color="#ff0000"> </font><font color="#bb00ff">"State.*stepping"</font>
<i><font color="#ababab"># Check for active ZFS commands</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zrepl status --mode raw </font><font color="#F3E651">|</font><font color="#ff0000"> grep -A</font><font color="#bb00ff">10</font><font color="#ff0000"> </font><font color="#bb00ff">"ZFSCmds.*Active"</font>
<i><font color="#ababab"># Monitor progress - bytes replicated should be increasing</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zrepl status --mode raw </font><font color="#F3E651">|</font><font color="#ff0000"> grep BytesReplicated</font>
</pre>
<br />
<h3 style='display: inline' id='network-connectivity-issues'>Network Connectivity Issues</h3><br />
<br />
<span>If replication fails to connect:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># Test connectivity between nodes</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> nc -zv </font><font color="#bb00ff">192.168</font><font color="#F3E651">.</font><font color="#bb00ff">2.131</font><font color="#ff0000"> </font><font color="#bb00ff">8888</font>
<i><font color="#ababab"># Check if zrepl is listening on f1</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas netstat -an </font><font color="#F3E651">|</font><font color="#ff0000"> grep </font><font color="#bb00ff">8888</font>
<i><font color="#ababab"># Verify WireGuard tunnel is working</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> ping </font><font color="#bb00ff">192.168</font><font color="#F3E651">.</font><font color="#bb00ff">2.131</font>
</pre>
<br />
<h3 style='display: inline' id='encryption-key-issues'>Encryption Key Issues</h3><br />
<br />
<span>If encrypted replication fails:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># Verify encryption keys are available on both nodes</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs get keystatus zdata/enc/nfsdata</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs get keystatus zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font>
<i><font color="#ababab"># Load keys if unavailable</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs load-key -L file</font><font color="#F3E651">:</font><font color="#ff0000">///keys/f</font><font color="#bb00ff">0</font><font color="#F3E651">.</font><font color="#ff0000">lan</font><font color="#F3E651">.</font><font color="#ff0000">buetow</font><font color="#F3E651">.</font><font color="#ff0000">org</font><font color="#F3E651">:</font><font color="#ff0000">zdata</font><font color="#F3E651">.</font><font color="#ff0000">key </font><font color="#F3E651">\</font>
<font color="#ff0000"> zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font>
</pre>
<br />
<h3 style='display: inline' id='monitoring-ongoing-replication'>Monitoring Ongoing Replication</h3><br />
<br />
<span>After fixing issues, monitor replication health:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># Monitor replication progress (run repeatedly to check status)</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zrepl status --mode raw </font><font color="#F3E651">|</font><font color="#ff0000"> grep -A</font><font color="#bb00ff">10</font><font color="#ff0000"> BytesReplicated</font>
<i><font color="#ababab"># Or install watch from ports and use it</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas pkg install watch</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> watch -n </font><font color="#bb00ff">5</font><font color="#ff0000"> </font><font color="#bb00ff">'doas zrepl status --mode raw | grep -A10 BytesReplicated'</font>
<i><font color="#ababab"># Check for new snapshots being created</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs list -t snapshot </font><font color="#F3E651">|</font><font color="#ff0000"> grep zrepl </font><font color="#F3E651">|</font><font color="#ff0000"> tail -</font><font color="#bb00ff">5</font>
<i><font color="#ababab"># Verify snapshots appear on receiver</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs list -t snapshot -r zdata/sink </font><font color="#F3E651">|</font><font color="#ff0000"> grep zrepl </font><font color="#F3E651">|</font><font color="#ff0000"> tail -</font><font color="#bb00ff">5</font>
</pre>
<br />
<span>This troubleshooting process resolves the most common <span class='inlinecode'>zrepl</span> issues and ensures continuous data replication between your storage nodes.</span><br />
<br />
<h2 style='display: inline' id='carp-common-address-redundancy-protocol'>CARP (Common Address Redundancy Protocol)</h2><br />
<br />
<span>High availability is crucial for storage systems. If the storage server goes down, all NFS clients (which will also be Kubernetes pods later on in this series) lose access to their persistent data. CARP provides a solution by creating a virtual IP address that automatically migrates to a different server during failures. This means that clients point to that VIP for NFS mounts and are always contacting the current primary node.</span><br />
<br />
<h3 style='display: inline' id='how-carp-works'>How CARP Works</h3><br />
<br />
<span>In our case, CARP allows two hosts (<span class='inlinecode'>f0</span> and <span class='inlinecode'>f1</span>) to share a virtual IP address (VIP). The hosts communicate using multicast to elect a MASTER, while the other remain as BACKUP. When the MASTER fails, the BACKUP automatically promotes itself, and the VIP is reassigned to the new MASTER. This happens within seconds.</span><br />
<br />
<span>Key benefits for our storage system:</span><br />
<br />
<ul>
<li>Automatic failover: No manual intervention is required for basic failures, although there are a few limitations. The backup will have read-only access to the available data by default, as we have already learned.</li>
<li>Transparent to clients: Pods continue using the same IP address</li>
<li>Works with <span class='inlinecode'>stunnel</span>: Behind the VIP, there will be a <span class='inlinecode'>stunnel</span> process running, which ensures encrypted connections follow the active server.</li>
</ul><br />
<a class='textlink' href='https://docs-archive.freebsd.org/doc/13.0-RELEASE/usr/local/share/doc/freebsd/en/books/handbook/carp.html'>FreeBSD CARP</a><br />
<a class='textlink' href='https://www.stunnel.org/'>Stunnel</a><br />
<br />
<h3 style='display: inline' id='configuring-carp'>Configuring CARP</h3><br />
<br />
<span>First, we add the CARP configuration to <span class='inlinecode'>/etc/rc.conf</span> on both <span class='inlinecode'>f0</span> and <span class='inlinecode'>f1</span>:</span><br />
<br />
<span class='quote'>Update: Sun 4 Jan 00:17:00 EET 2026 - Added <span class='inlinecode'>advskew 100</span> to f1 so f0 always wins CARP elections when it comes back online after a reboot.</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># On f0 - The virtual IP 192.168.1.138 will float between f0 and f1</font></i>
<font color="#ff0000">ifconfig_re0_alias0</font><font color="#F3E651">=</font><font color="#bb00ff">"inet vhid 1 pass testpass alias 192.168.1.138/32"</font>
<i><font color="#ababab"># On f1 - Higher advskew means lower priority, so f0 wins elections</font></i>
<font color="#ff0000">ifconfig_re0_alias0</font><font color="#F3E651">=</font><font color="#bb00ff">"inet vhid 1 advskew 100 pass testpass alias 192.168.1.138/32"</font>
</pre>
<br />
<span>Whereas:</span><br />
<br />
<ul>
<li><span class='inlinecode'>vhid 1</span>: Virtual Host ID - must match on all CARP members</li>
<li><span class='inlinecode'>advskew</span>: Advertisement skew - higher value means lower priority (f1 uses 100, f0 uses default 0)</li>
<li><span class='inlinecode'>pass testpass</span>: Password for CARP authentication (if you follow this, use a different password!)</li>
<li><span class='inlinecode'>alias 192.168.1.138/32</span>: The virtual IP address with a /32 netmask</li>
</ul><br />
<span>Next, update <span class='inlinecode'>/etc/hosts</span> on all nodes (<span class='inlinecode'>f0</span>, <span class='inlinecode'>f1</span>, <span class='inlinecode'>f2</span>, <span class='inlinecode'>r0</span>, <span class='inlinecode'>r1</span>, <span class='inlinecode'>r2</span>) to resolve the VIP hostname:</span><br />
<br />
<pre>
192.168.2.138 f3s-storage-ha f3s-storage-ha.wg0 f3s-storage-ha.wg0.wan.buetow.org
fd42:beef:cafe:2::138 f3s-storage-ha f3s-storage-ha.wg0 f3s-storage-ha.wg0.wan.buetow.org
</pre>
<br />
<span>This allows clients to connect to <span class='inlinecode'>f3s-storage-ha</span> regardless of which physical server is currently the MASTER.</span><br />
<br />
<h3 style='display: inline' id='carp-state-change-notifications'>CARP State Change Notifications</h3><br />
<br />
<span>To correctly manage services during failover, we need to detect CARP state changes. FreeBSD's devd system can notify us when CARP transitions between MASTER and BACKUP states.</span><br />
<br />
<span>Add this to <span class='inlinecode'>/etc/devd.conf</span> on both <span class='inlinecode'>f0</span> and <span class='inlinecode'>f1</span>:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> cat </font><font color="#F3E651"><<</font><font color="#ff0000">END </font><font color="#F3E651">|</font><font color="#ff0000"> doas tee -a /etc/devd</font><font color="#F3E651">.</font><font color="#ff0000">conf</font>
<font color="#ff0000">notify </font><font color="#bb00ff">0</font><font color="#ff0000"> {</font>
<font color="#ff0000"> match </font><font color="#bb00ff">"system"</font><font color="#ff0000"> </font><font color="#bb00ff">"CARP"</font><font color="#F3E651">;</font>
<font color="#ff0000"> match </font><font color="#bb00ff">"subsystem"</font><font color="#ff0000"> </font><font color="#bb00ff">"[0-9]+@[0-9a-z.]+"</font><font color="#F3E651">;</font>
<font color="#ff0000"> match </font><font color="#bb00ff">"type"</font><font color="#ff0000"> </font><font color="#bb00ff">"(MASTER|BACKUP)"</font><font color="#F3E651">;</font>
<font color="#ff0000"> action </font><font color="#bb00ff">"/usr/local/bin/carpcontrol.sh $subsystem $type"</font><font color="#F3E651">;</font>
<font color="#ff0000">}</font><font color="#F3E651">;</font>
<font color="#ff0000">END</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service devd restart</font>
</pre>
<br />
<span>Next, we create the CARP control script that will restart stunnel when the CARP state changes:</span><br />
<br />
<span class='quote'>Update: Fixed the script at Sat 3 Jan 23:55:11 EET 2026 - changed <span class='inlinecode'>$1</span> to <span class='inlinecode'>$2</span> because devd passes <span class='inlinecode'>$subsystem $type</span>, so the state is in the second argument.</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas tee /usr/local/bin/carpcontrol</font><font color="#F3E651">.</font><font color="#ff0000">sh </font><font color="#F3E651"><<</font><font color="#bb00ff">'EOF'</font>
<i><font color="#ababab">#!/bin/sh</font></i>
<i><font color="#ababab"># CARP state change control script</font></i>
<b><font color="#ffffff">case</font></b><font color="#ff0000"> </font><font color="#bb00ff">"$2"</font><font color="#ff0000"> </font><b><font color="#ffffff">in</font></b>
<font color="#ff0000"> MASTER</font><font color="#F3E651">)</font>
<font color="#ff0000"> logger </font><font color="#bb00ff">"CARP state changed to MASTER, starting services"</font>
<font color="#ff0000"> </font><font color="#F3E651">;;</font>
<font color="#ff0000"> BACKUP</font><font color="#F3E651">)</font>
<font color="#ff0000"> logger </font><font color="#bb00ff">"CARP state changed to BACKUP, stopping services"</font>
<font color="#ff0000"> </font><font color="#F3E651">;;</font>
<font color="#ff0000"> </font><font color="#F3E651">*)</font>
<font color="#ff0000"> logger </font><font color="#bb00ff">"CARP state changed to $2 (unhandled)"</font>
<font color="#ff0000"> </font><font color="#F3E651">;;</font>
<b><font color="#ffffff">esac</font></b>
<font color="#ff0000">EOF</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas chmod </font><font color="#F3E651">+</font><font color="#ff0000">x /usr/local/bin/carpcontrol</font><font color="#F3E651">.</font><font color="#ff0000">sh</font>
<i><font color="#ababab"># Copy the same script to f1</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> scp /usr/local/bin/carpcontrol</font><font color="#F3E651">.</font><font color="#ff0000">sh f1</font><font color="#F3E651">:</font><font color="#ff0000">/tmp</font><font color="#F3E651">/</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas mv /tmp/carpcontrol</font><font color="#F3E651">.</font><font color="#ff0000">sh /usr/local/bin</font><font color="#F3E651">/</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas chmod </font><font color="#F3E651">+</font><font color="#ff0000">x /usr/local/bin/carpcontrol</font><font color="#F3E651">.</font><font color="#ff0000">sh</font>
</pre>
<br />
<span>Note that <span class='inlinecode'>carpcontrol.sh</span> doesn't do anything useful yet. We will provide more details (including starting and stopping services upon failover) later in this blog post.</span><br />
<br />
<span>To enable CARP in <span class='inlinecode'>/boot/loader.conf</span>, run:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> echo </font><font color="#bb00ff">'carp_load="YES"'</font><font color="#ff0000"> </font><font color="#F3E651">|</font><font color="#ff0000"> doas tee -a /boot/loader</font><font color="#F3E651">.</font><font color="#ff0000">conf</font>
<font color="#ff0000">carp_load</font><font color="#F3E651">=</font><font color="#bb00ff">"YES"</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> echo </font><font color="#bb00ff">'carp_load="YES"'</font><font color="#ff0000"> </font><font color="#F3E651">|</font><font color="#ff0000"> doas tee -a /boot/loader</font><font color="#F3E651">.</font><font color="#ff0000">conf </font>
<font color="#ff0000">carp_load</font><font color="#F3E651">=</font><font color="#bb00ff">"YES"</font>
</pre>
<br />
<span>Then reboot both hosts or run <span class='inlinecode'>doas kldload carp</span> to load the module immediately. </span><br />
<br />
<h2 style='display: inline' id='nfs-server-configuration'>NFS Server Configuration</h2><br />
<br />
<span>With ZFS replication in place, we can now set up NFS servers on both <span class='inlinecode'>f0</span> and <span class='inlinecode'>f1</span> to export the replicated data. Since native NFS over TLS (RFC 9289) has compatibility issues between Linux and FreeBSD (not digging into the details here, but I couldn't get it to work), we'll use stunnel to provide encryption.</span><br />
<br />
<h3 style='display: inline' id='setting-up-nfs-on-f0-primary'>Setting up NFS on <span class='inlinecode'>f0</span> (Primary)</h3><br />
<br />
<span>First, enable the NFS services in rc.conf:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">nfs_server_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font>
<font color="#ff0000">nfs_server_enable</font><font color="#F3E651">:</font><font color="#ff0000"> YES -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">nfsv4_server_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font>
<font color="#ff0000">nfsv4_server_enable</font><font color="#F3E651">:</font><font color="#ff0000"> YES -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">nfsuserd_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font>
<font color="#ff0000">nfsuserd_enable</font><font color="#F3E651">:</font><font color="#ff0000"> YES -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">nfsuserd_flags</font><font color="#F3E651">=</font><font color="#bb00ff">"-domain lan.buetow.org"</font>
<font color="#ff0000">nfsuserd_flags</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">""</font><font color="#ff0000"> -</font><font color="#F3E651">></font><font color="#ff0000"> </font><font color="#bb00ff">"-domain lan.buetow.org"</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">mountd_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font>
<font color="#ff0000">mountd_enable</font><font color="#F3E651">:</font><font color="#ff0000"> NO -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">rpcbind_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font>
<font color="#ff0000">rpcbind_enable</font><font color="#F3E651">:</font><font color="#ff0000"> NO -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font>
</pre>
<br />
<span class='quote'>Update: 08.08.2025: I've added the domain to <span class='inlinecode'>nfsuserd_flags</span></span><br />
<br />
<span>And we also create a dedicated directory for Kubernetes volumes:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># First, ensure the dataset is mounted</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs get mounted zdata/enc/nfsdata</font>
<font color="#ff0000">NAME PROPERTY VALUE SOURCE</font>
<font color="#ff0000">zdata/enc/nfsdata mounted yes -</font>
<i><font color="#ababab"># Create the k3svolumes directory</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas mkdir -p /data/nfs/k3svolumes</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas chmod </font><font color="#bb00ff">755</font><font color="#ff0000"> /data/nfs/k3svolumes</font>
</pre>
<br />
<span>We also create the <span class='inlinecode'>/etc/exports</span> file. Since we're using stunnel for encryption, ALL clients must connect through stunnel, which appears as localhost (<span class='inlinecode'>127.0.0.1</span>) to the NFS server:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas tee /etc/exports </font><font color="#F3E651"><<</font><font color="#bb00ff">'EOF'</font>
<font color="#ff0000">V4</font><font color="#F3E651">:</font><font color="#ff0000"> /data/nfs -sec</font><font color="#F3E651">=</font><font color="#ff0000">sys</font>
<font color="#ff0000">/data/nfs -alldirs -maproot</font><font color="#F3E651">=</font><font color="#ff0000">root -network </font><font color="#bb00ff">127.0</font><font color="#F3E651">.</font><font color="#bb00ff">0.1</font><font color="#ff0000"> -mask </font><font color="#bb00ff">255.255</font><font color="#F3E651">.</font><font color="#bb00ff">255.255</font>
<font color="#ff0000">EOF</font>
</pre>
<br />
<span>The exports configuration:</span><br />
<br />
<ul>
<li><span class='inlinecode'>V4: /data/nfs -sec=sys</span>: Sets the NFSv4 root directory to /data/nfs</li>
<li><span class='inlinecode'>-maproot=root</span>: Maps root user from client to root on server</li>
<li><span class='inlinecode'>-network 127.0.0.1</span>: Only accepts connections from localhost (<span class='inlinecode'>stunnel</span>)</li>
</ul><br />
<span>To start the NFS services, we run:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service rpcbind start</font>
<font color="#ff0000">Starting rpcbind</font><font color="#F3E651">.</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service mountd start</font>
<font color="#ff0000">Starting mountd</font><font color="#F3E651">.</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service nfsd start</font>
<font color="#ff0000">Starting nfsd</font><font color="#F3E651">.</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service nfsuserd start</font>
<font color="#ff0000">Starting nfsuserd</font><font color="#F3E651">.</font>
</pre>
<br />
<h3 style='display: inline' id='configuring-stunnel-for-nfs-encryption-with-carp-failover'>Configuring Stunnel for NFS Encryption with CARP Failover</h3><br />
<br />
<span>Using stunnel with client certificate authentication for NFS encryption provides several advantages:</span><br />
<br />
<ul>
<li>Compatibility: Works with any NFS version and between different operating systems</li>
<li>Strong encryption: Uses TLS/SSL with configurable cipher suites</li>
<li>Transparent: Applications don't need modification, encryption happens at the transport layer</li>
<li>Performance: Minimal overhead (~2% in benchmarks)</li>
<li>Flexibility: Can encrypt any TCP-based protocol, not just NFS</li>
<li>Strong Authentication: Client certificates provide cryptographic proof of identity</li>
<li>Access Control: Only clients with valid certificates signed by your CA can connect</li>
<li>Certificate Revocation: You can revoke access by removing certificates from the CA</li>
</ul><br />
<span>Stunnel integrates seamlessly with our CARP setup:</span><br />
<br />
<pre>
CARP VIP (192.168.1.138)
|
f0 (MASTER) ←---------→|←---------→ f1 (BACKUP)
stunnel:2323 | stunnel:stopped
nfsd:2049 | nfsd:stopped
|
Clients connect here
</pre>
<br />
<span>The key insight is that stunnel binds to the CARP VIP. When CARP fails over, the VIP is moved to the new master, and stunnel starts there automatically. Clients maintain their connection to the same IP throughout.</span><br />
<br />
<h3 style='display: inline' id='creating-a-certificate-authority-for-client-authentication'>Creating a Certificate Authority for Client Authentication</h3><br />
<br />
<span>First, create a CA to sign both server and client certificates:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># On f0 - Create CA</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas mkdir -p /usr/local/etc/stunnel/ca</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> cd /usr/local/etc/stunnel/ca</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas openssl genrsa -out ca-key</font><font color="#F3E651">.</font><font color="#ff0000">pem </font><font color="#bb00ff">4096</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas openssl req -new -x</font><font color="#bb00ff">509</font><font color="#ff0000"> -days </font><font color="#bb00ff">3650</font><font color="#ff0000"> -key ca-key</font><font color="#F3E651">.</font><font color="#ff0000">pem -out ca-cert</font><font color="#F3E651">.</font><font color="#ff0000">pem </font><font color="#F3E651">\</font>
<font color="#ff0000"> -subj </font><font color="#bb00ff">'/C=US/ST=State/L=City/O=F3S Storage/CN=F3S Stunnel CA'</font>
<i><font color="#ababab"># Create server certificate</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> cd /usr/local/etc/stunnel</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas openssl genrsa -out server-key</font><font color="#F3E651">.</font><font color="#ff0000">pem </font><font color="#bb00ff">4096</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas openssl req -new -key server-key</font><font color="#F3E651">.</font><font color="#ff0000">pem -out server</font><font color="#F3E651">.</font><font color="#ff0000">csr </font><font color="#F3E651">\</font>
<font color="#ff0000"> -subj </font><font color="#bb00ff">'/C=US/ST=State/L=City/O=F3S Storage/CN=f3s-storage-ha.lan'</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas openssl x509 -req -days </font><font color="#bb00ff">3650</font><font color="#ff0000"> -in server</font><font color="#F3E651">.</font><font color="#ff0000">csr -CA ca/ca-cert</font><font color="#F3E651">.</font><font color="#ff0000">pem </font><font color="#F3E651">\</font>
<font color="#ff0000"> -CAkey ca/ca-key</font><font color="#F3E651">.</font><font color="#ff0000">pem -CAcreateserial -out server-cert</font><font color="#F3E651">.</font><font color="#ff0000">pem</font>
<i><font color="#ababab"># Create client certificates for authorised clients</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> cd /usr/local/etc/stunnel/ca</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sh -c </font><font color="#bb00ff">'for client in r0 r1 r2 earth; do </font>
<font color="#bb00ff"> openssl genrsa -out ${client}-key.pem 4096</font>
<font color="#bb00ff"> openssl req -new -key ${client}-key.pem -out ${client}.csr \</font>
<font color="#bb00ff"> -subj "/C=US/ST=State/L=City/O=F3S Storage/CN=${client}.lan.buetow.org"</font>
<font color="#bb00ff"> openssl x509 -req -days 3650 -in ${client}.csr -CA ca-cert.pem \</font>
<font color="#bb00ff"> -CAkey ca-key.pem -CAcreateserial -out ${client}-cert.pem</font>
<font color="#bb00ff"> # Combine cert and key into a single file for stunnel client</font>
<font color="#bb00ff"> cat ${client}-cert.pem ${client}-key.pem > ${client}-stunnel.pem</font>
<font color="#bb00ff">done'</font>
</pre>
<br />
<h3 style='display: inline' id='install-and-configure-stunnel-on-f0'>Install and Configure Stunnel on <span class='inlinecode'>f0</span></h3><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># Install stunnel</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas pkg install -y stunnel</font>
<i><font color="#ababab"># Configure stunnel server with client certificate authentication</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas tee /usr/local/etc/stunnel/stunnel</font><font color="#F3E651">.</font><font color="#ff0000">conf </font><font color="#F3E651"><<</font><font color="#bb00ff">'EOF'</font>
<font color="#ff0000">cert </font><font color="#F3E651">=</font><font color="#ff0000"> /usr/local/etc/stunnel/server-cert</font><font color="#F3E651">.</font><font color="#ff0000">pem</font>
<font color="#ff0000">key </font><font color="#F3E651">=</font><font color="#ff0000"> /usr/local/etc/stunnel/server-key</font><font color="#F3E651">.</font><font color="#ff0000">pem</font>
<font color="#ff0000">setuid </font><font color="#F3E651">=</font><font color="#ff0000"> stunnel</font>
<font color="#ff0000">setgid </font><font color="#F3E651">=</font><font color="#ff0000"> stunnel</font>
<font color="#F3E651">[</font><font color="#ff0000">nfs-tls</font><font color="#F3E651">]</font>
<font color="#ff0000">accept </font><font color="#F3E651">=</font><font color="#ff0000"> </font><font color="#bb00ff">192.168</font><font color="#F3E651">.</font><font color="#bb00ff">1.138</font><font color="#F3E651">:</font><font color="#bb00ff">2323</font>
<font color="#ff0000">connect </font><font color="#F3E651">=</font><font color="#ff0000"> </font><font color="#bb00ff">127.0</font><font color="#F3E651">.</font><font color="#bb00ff">0.1</font><font color="#F3E651">:</font><font color="#bb00ff">2049</font>
<font color="#ff0000">CAfile </font><font color="#F3E651">=</font><font color="#ff0000"> /usr/local/etc/stunnel/ca/ca-cert</font><font color="#F3E651">.</font><font color="#ff0000">pem</font>
<font color="#ff0000">verify </font><font color="#F3E651">=</font><font color="#ff0000"> </font><font color="#bb00ff">2</font>
<font color="#ff0000">requireCert </font><font color="#F3E651">=</font><font color="#ff0000"> yes</font>
<font color="#ff0000">EOF</font>
<i><font color="#ababab"># Enable and start stunnel</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">stunnel_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font>
<font color="#ff0000">stunnel_enable</font><font color="#F3E651">:</font><font color="#ff0000"> -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service stunnel start</font>
<font color="#ff0000">Starting stunnel</font><font color="#F3E651">.</font>
<i><font color="#ababab"># Restart stunnel to apply the CARP VIP binding</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service stunnel restart</font>
<font color="#ff0000">Stopping stunnel</font><font color="#F3E651">.</font>
<font color="#ff0000">Starting stunnel</font><font color="#F3E651">.</font>
</pre>
<br />
<span>The configuration includes:</span><br />
<br />
<ul>
<li><span class='inlinecode'>verify = 2</span>: Verify client certificate and fail if not provided</li>
<li><span class='inlinecode'>requireCert = yes</span>: Client must present a valid certificate</li>
<li><span class='inlinecode'>CAfile</span>: Path to the CA certificate that signed the client certificates</li>
</ul><br />
<h3 style='display: inline' id='setting-up-nfs-on-f1-standby'>Setting up NFS on <span class='inlinecode'>f1</span> (Standby)</h3><br />
<br />
<span>Repeat the same configuration on <span class='inlinecode'>f1</span>:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">nfs_server_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font>
<font color="#ff0000">nfs_server_enable</font><font color="#F3E651">:</font><font color="#ff0000"> NO -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">nfsv4_server_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font>
<font color="#ff0000">nfsv4_server_enable</font><font color="#F3E651">:</font><font color="#ff0000"> NO -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">nfsuserd_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font>
<font color="#ff0000">nfsuserd_enable</font><font color="#F3E651">:</font><font color="#ff0000"> NO -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">mountd_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font>
<font color="#ff0000">mountd_enable</font><font color="#F3E651">:</font><font color="#ff0000"> NO -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">rpcbind_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font>
<font color="#ff0000">rpcbind_enable</font><font color="#F3E651">:</font><font color="#ff0000"> NO -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas tee /etc/exports </font><font color="#F3E651"><<</font><font color="#bb00ff">'EOF'</font>
<font color="#ff0000">V4</font><font color="#F3E651">:</font><font color="#ff0000"> /data/nfs -sec</font><font color="#F3E651">=</font><font color="#ff0000">sys</font>
<font color="#ff0000">/data/nfs -alldirs -maproot</font><font color="#F3E651">=</font><font color="#ff0000">root -network </font><font color="#bb00ff">127.0</font><font color="#F3E651">.</font><font color="#bb00ff">0.1</font><font color="#ff0000"> -mask </font><font color="#bb00ff">255.255</font><font color="#F3E651">.</font><font color="#bb00ff">255.255</font>
<font color="#ff0000">EOF</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service rpcbind start</font>
<font color="#ff0000">Starting rpcbind</font><font color="#F3E651">.</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service mountd start</font>
<font color="#ff0000">Starting mountd</font><font color="#F3E651">.</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service nfsd start</font>
<font color="#ff0000">Starting nfsd</font><font color="#F3E651">.</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service nfsuserd start</font>
<font color="#ff0000">Starting nfsuserd</font><font color="#F3E651">.</font>
</pre>
<br />
<span>And to configure stunnel on <span class='inlinecode'>f1</span>, we run:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># Install stunnel</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas pkg install -y stunnel</font>
<i><font color="#ababab"># Copy certificates from f0</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas tar -cf /tmp/stunnel-certs</font><font color="#F3E651">.</font><font color="#ff0000">tar </font><font color="#F3E651">\</font>
<font color="#ff0000"> -C /usr/local/etc/stunnel server-cert</font><font color="#F3E651">.</font><font color="#ff0000">pem server-key</font><font color="#F3E651">.</font><font color="#ff0000">pem ca</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> scp /tmp/stunnel-certs</font><font color="#F3E651">.</font><font color="#ff0000">tar f1</font><font color="#F3E651">:</font><font color="#ff0000">/tmp</font><font color="#F3E651">/</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> cd /usr/local/etc/stunnel </font><font color="#F3E651">&&</font><font color="#ff0000"> doas tar -xf /tmp/stunnel-certs</font><font color="#F3E651">.</font><font color="#ff0000">tar</font>
<i><font color="#ababab"># Configure stunnel server on f1 with client certificate authentication</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas tee /usr/local/etc/stunnel/stunnel</font><font color="#F3E651">.</font><font color="#ff0000">conf </font><font color="#F3E651"><<</font><font color="#bb00ff">'EOF'</font>
<font color="#ff0000">cert </font><font color="#F3E651">=</font><font color="#ff0000"> /usr/local/etc/stunnel/server-cert</font><font color="#F3E651">.</font><font color="#ff0000">pem</font>
<font color="#ff0000">key </font><font color="#F3E651">=</font><font color="#ff0000"> /usr/local/etc/stunnel/server-key</font><font color="#F3E651">.</font><font color="#ff0000">pem</font>
<font color="#ff0000">setuid </font><font color="#F3E651">=</font><font color="#ff0000"> stunnel</font>
<font color="#ff0000">setgid </font><font color="#F3E651">=</font><font color="#ff0000"> stunnel</font>
<font color="#F3E651">[</font><font color="#ff0000">nfs-tls</font><font color="#F3E651">]</font>
<font color="#ff0000">accept </font><font color="#F3E651">=</font><font color="#ff0000"> </font><font color="#bb00ff">192.168</font><font color="#F3E651">.</font><font color="#bb00ff">1.138</font><font color="#F3E651">:</font><font color="#bb00ff">2323</font>
<font color="#ff0000">connect </font><font color="#F3E651">=</font><font color="#ff0000"> </font><font color="#bb00ff">127.0</font><font color="#F3E651">.</font><font color="#bb00ff">0.1</font><font color="#F3E651">:</font><font color="#bb00ff">2049</font>
<font color="#ff0000">CAfile </font><font color="#F3E651">=</font><font color="#ff0000"> /usr/local/etc/stunnel/ca/ca-cert</font><font color="#F3E651">.</font><font color="#ff0000">pem</font>
<font color="#ff0000">verify </font><font color="#F3E651">=</font><font color="#ff0000"> </font><font color="#bb00ff">2</font>
<font color="#ff0000">requireCert </font><font color="#F3E651">=</font><font color="#ff0000"> yes</font>
<font color="#ff0000">EOF</font>
<i><font color="#ababab"># Enable and start stunnel</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">stunnel_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font>
<font color="#ff0000">stunnel_enable</font><font color="#F3E651">:</font><font color="#ff0000"> -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service stunnel start</font>
<font color="#ff0000">Starting stunnel</font><font color="#F3E651">.</font>
<i><font color="#ababab"># Restart stunnel to apply the CARP VIP binding</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service stunnel restart</font>
<font color="#ff0000">Stopping stunnel</font><font color="#F3E651">.</font>
<font color="#ff0000">Starting stunnel</font><font color="#F3E651">.</font>
</pre>
<br />
<h3 style='display: inline' id='carp-control-script-for-clean-failover'>CARP Control Script for Clean Failover</h3><br />
<br />
<span>With stunnel configured to bind to the CARP VIP (192.168.1.138), only the server that is currently the CARP MASTER will accept stunnel connections. This provides automatic failover for encrypted NFS:</span><br />
<br />
<ul>
<li>When <span class='inlinecode'>f0</span> is CARP MASTER: stunnel on <span class='inlinecode'>f0</span> accepts connections on <span class='inlinecode'>192.168.1.138:2323</span></li>
<li>When <span class='inlinecode'>f1</span> becomes CARP MASTER: stunnel on <span class='inlinecode'>f1</span> starts accepting connections on <span class='inlinecode'>192.168.1.138:2323</span></li>
<li>The backup server's stunnel process will fail to bind to the VIP and won't accept connections</li>
</ul><br />
<span>This ensures that clients always connect to the active NFS server through the CARP VIP. To ensure clean failover behaviour and prevent stale file handles, we'll update our <span class='inlinecode'>carpcontrol.sh</span> script so that:</span><br />
<br />
<ul>
<li>Stops NFS services on BACKUP nodes (preventing split-brain scenarios)</li>
<li>Starts NFS services only on the MASTER node</li>
<li>Manages stunnel binding to the CARP VIP</li>
</ul><br />
<span>This approach ensures clients can only connect to the active server, eliminating stale handles from the inactive server:</span><br />
<br />
<span class='quote'>Update: Fixed the script at Sat 3 Jan 23:55:11 EET 2026 - changed <span class='inlinecode'>$1</span> to <span class='inlinecode'>$2</span> because devd passes <span class='inlinecode'>$subsystem $type</span>, so the state is in the second argument.</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># Create CARP control script on both f0 and f1</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas tee /usr/local/bin/carpcontrol</font><font color="#F3E651">.</font><font color="#ff0000">sh </font><font color="#F3E651"><<</font><font color="#bb00ff">'EOF'</font>
<i><font color="#ababab">#!/bin/sh</font></i>
<i><font color="#ababab"># CARP state change control script</font></i>
<font color="#ff0000">HOSTNAME</font><font color="#F3E651">=</font><font color="#ff0000">`hostname`</font>
<b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">[</font><font color="#ff0000"> </font><font color="#F3E651">!</font><font color="#ff0000"> -f /data/nfs/nfs</font><font color="#F3E651">.</font><font color="#ff0000">DO_NOT_REMOVE </font><font color="#F3E651">];</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b>
<font color="#ff0000"> logger </font><font color="#bb00ff">'/data/nfs not mounted, mounting it now!'</font>
<font color="#ff0000"> </font><b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">[</font><font color="#ff0000"> </font><font color="#bb00ff">"$HOSTNAME"</font><font color="#ff0000"> </font><font color="#F3E651">=</font><font color="#ff0000"> </font><font color="#bb00ff">'f0.lan.buetow.org'</font><font color="#ff0000"> </font><font color="#F3E651">];</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b>
<font color="#ff0000"> zfs load-key -L file</font><font color="#F3E651">:</font><font color="#ff0000">///keys/f</font><font color="#bb00ff">0</font><font color="#F3E651">.</font><font color="#ff0000">lan</font><font color="#F3E651">.</font><font color="#ff0000">buetow</font><font color="#F3E651">.</font><font color="#ff0000">org</font><font color="#F3E651">:</font><font color="#ff0000">zdata</font><font color="#F3E651">.</font><font color="#ff0000">key zdata/enc/nfsdata</font>
<font color="#ff0000"> zfs </font><b><font color="#ffffff">set</font></b><font color="#ff0000"> </font><font color="#ff0000">mountpoint</font><font color="#F3E651">=</font><font color="#ff0000">/data/nfs zdata/enc/nfsdata</font>
<font color="#ff0000"> </font><b><font color="#ffffff">else</font></b>
<font color="#ff0000"> zfs load-key -L file</font><font color="#F3E651">:</font><font color="#ff0000">///keys/f</font><font color="#bb00ff">0</font><font color="#F3E651">.</font><font color="#ff0000">lan</font><font color="#F3E651">.</font><font color="#ff0000">buetow</font><font color="#F3E651">.</font><font color="#ff0000">org</font><font color="#F3E651">:</font><font color="#ff0000">zdata</font><font color="#F3E651">.</font><font color="#ff0000">key zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font>
<font color="#ff0000"> zfs </font><b><font color="#ffffff">set</font></b><font color="#ff0000"> </font><font color="#ff0000">mountpoint</font><font color="#F3E651">=</font><font color="#ff0000">/data/nfs zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font>
<font color="#ff0000"> zfs mount zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font>
<font color="#ff0000"> zfs </font><b><font color="#ffffff">set</font></b><font color="#ff0000"> </font><b><font color="#ffffff">readonly</font></b><font color="#F3E651">=</font><font color="#ff0000">on zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font>
<font color="#ff0000"> </font><b><font color="#ffffff">fi</font></b>
<font color="#ff0000"> service nfsd stop </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font>
<font color="#ff0000"> service mountd stop </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font>
<b><font color="#ffffff">fi</font></b>
<b><font color="#ffffff">case</font></b><font color="#ff0000"> </font><font color="#bb00ff">"$2"</font><font color="#ff0000"> </font><b><font color="#ffffff">in</font></b>
<font color="#ff0000"> MASTER</font><font color="#F3E651">)</font>
<font color="#ff0000"> logger </font><font color="#bb00ff">"CARP state changed to MASTER, starting services"</font>
<font color="#ff0000"> service rpcbind start </font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font>
<font color="#ff0000"> service mountd start </font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font>
<font color="#ff0000"> service nfsd start </font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font>
<font color="#ff0000"> service nfsuserd start </font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font>
<font color="#ff0000"> service stunnel restart </font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font>
<font color="#ff0000"> logger </font><font color="#bb00ff">"CARP MASTER: NFS and stunnel services started"</font>
<font color="#ff0000"> </font><font color="#F3E651">;;</font>
<font color="#ff0000"> BACKUP</font><font color="#F3E651">)</font>
<font color="#ff0000"> logger </font><font color="#bb00ff">"CARP state changed to BACKUP, stopping services"</font>
<font color="#ff0000"> service stunnel stop </font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font>
<font color="#ff0000"> service nfsd stop </font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font>
<font color="#ff0000"> service mountd stop </font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font>
<font color="#ff0000"> service nfsuserd stop </font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font>
<font color="#ff0000"> logger </font><font color="#bb00ff">"CARP BACKUP: NFS and stunnel services stopped"</font>
<font color="#ff0000"> </font><font color="#F3E651">;;</font>
<font color="#ff0000"> </font><font color="#F3E651">*)</font>
<font color="#ff0000"> logger </font><font color="#bb00ff">"CARP state changed to $2 (unhandled)"</font>
<font color="#ff0000"> </font><font color="#F3E651">;;</font>
<b><font color="#ffffff">esac</font></b>
<font color="#ff0000">EOF</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas chmod </font><font color="#F3E651">+</font><font color="#ff0000">x /usr/local/bin/carpcontrol</font><font color="#F3E651">.</font><font color="#ff0000">sh</font>
</pre>
<br />
<h3 style='display: inline' id='carp-management-script'>CARP Management Script</h3><br />
<br />
<span>To simplify CARP state management and failover testing, create this helper script on both <span class='inlinecode'>f0</span> and <span class='inlinecode'>f1</span>:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># Create the CARP management script</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas tee /usr/local/bin/carp </font><font color="#F3E651"><<</font><font color="#bb00ff">'EOF'</font>
<i><font color="#ababab">#!/bin/sh</font></i>
<i><font color="#ababab"># CARP state management script</font></i>
<i><font color="#ababab"># Usage: carp [master|backup|auto-failback enable|auto-failback disable]</font></i>
<i><font color="#ababab"># Without arguments: shows current state</font></i>
<i><font color="#ababab"># Find the interface with CARP configured</font></i>
<font color="#ff0000">CARP_IF</font><font color="#F3E651">=</font><font color="#ff0000">$(</font><font color="#ff0000">ifconfig -l </font><font color="#F3E651">|</font><font color="#ff0000"> xargs -n</font><font color="#bb00ff">1</font><font color="#ff0000"> </font><font color="#F3E651">|</font><font color="#ff0000"> </font><b><font color="#ffffff">while</font></b><font color="#ff0000"> </font><b><font color="#ffffff">read</font></b><font color="#ff0000"> </font><b><font color="#ffffff">if</font></b><font color="#F3E651">;</font><font color="#ff0000"> </font><b><font color="#ffffff">do</font></b>
<font color="#ff0000"> ifconfig </font><font color="#bb00ff">"$if"</font><font color="#ff0000"> </font><font color="#bb00ff">2</font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#F3E651">|</font><font color="#ff0000"> grep -q </font><font color="#bb00ff">"carp:"</font><font color="#ff0000"> </font><font color="#F3E651">&&</font><font color="#ff0000"> echo </font><font color="#bb00ff">"$if"</font><font color="#ff0000"> </font><font color="#F3E651">&&</font><font color="#ff0000"> </font><b><font color="#ffffff">break</font></b>
<b><font color="#ffffff">done</font></b><font color="#F3E651">)</font>
<b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">[</font><font color="#ff0000"> -z </font><font color="#bb00ff">"$CARP_IF"</font><font color="#ff0000"> </font><font color="#F3E651">];</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b>
<font color="#ff0000"> echo </font><font color="#bb00ff">"Error: No CARP interface found"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">exit</font></b><font color="#ff0000"> </font><font color="#bb00ff">1</font>
<b><font color="#ffffff">fi</font></b>
<i><font color="#ababab"># Get CARP VHID</font></i>
<font color="#ff0000">VHID</font><font color="#F3E651">=</font><font color="#ff0000">$(</font><font color="#ff0000">ifconfig </font><font color="#bb00ff">"$CARP_IF"</font><font color="#ff0000"> </font><font color="#F3E651">|</font><font color="#ff0000"> grep </font><font color="#bb00ff">"carp:"</font><font color="#ff0000"> </font><font color="#F3E651">|</font><font color="#ff0000"> sed -n </font><font color="#bb00ff">'s/.*vhid </font><font color="#ffffff">\(</font><font color="#bb00ff">[0-9]*</font><font color="#ffffff">\)</font><font color="#bb00ff">.*/</font><font color="#ffffff">\1</font><font color="#bb00ff">/p'</font><font color="#F3E651">)</font>
<b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">[</font><font color="#ff0000"> -z </font><font color="#bb00ff">"$VHID"</font><font color="#ff0000"> </font><font color="#F3E651">];</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b>
<font color="#ff0000"> echo </font><font color="#bb00ff">"Error: Could not determine CARP VHID"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">exit</font></b><font color="#ff0000"> </font><font color="#bb00ff">1</font>
<b><font color="#ffffff">fi</font></b>
<i><font color="#ababab"># Function to get the current state</font></i>
<font color="#7bc710">get_state()</font><font color="#ff0000"> {</font>
<font color="#ff0000"> ifconfig </font><font color="#bb00ff">"$CARP_IF"</font><font color="#ff0000"> </font><font color="#F3E651">|</font><font color="#ff0000"> grep </font><font color="#bb00ff">"carp:"</font><font color="#ff0000"> </font><font color="#F3E651">|</font><font color="#ff0000"> awk </font><font color="#bb00ff">'{print $2}'</font>
<font color="#ff0000">}</font>
<i><font color="#ababab"># Check for auto-failback block file</font></i>
<font color="#ff0000">BLOCK_FILE</font><font color="#F3E651">=</font><font color="#bb00ff">"/data/nfs/nfs.NO_AUTO_FAILBACK"</font>
<font color="#7bc710">check_auto_failback()</font><font color="#ff0000"> {</font>
<font color="#ff0000"> </font><b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">[</font><font color="#ff0000"> -f </font><font color="#bb00ff">"$BLOCK_FILE"</font><font color="#ff0000"> </font><font color="#F3E651">];</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b>
<font color="#ff0000"> echo </font><font color="#bb00ff">"WARNING: Auto-failback is DISABLED (file exists: $BLOCK_FILE)"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">fi</font></b>
<font color="#ff0000">}</font>
<i><font color="#ababab"># Main logic</font></i>
<b><font color="#ffffff">case</font></b><font color="#ff0000"> </font><font color="#bb00ff">"$1"</font><font color="#ff0000"> </font><b><font color="#ffffff">in</font></b>
<font color="#ff0000"> </font><font color="#bb00ff">""</font><font color="#F3E651">)</font>
<font color="#ff0000"> </font><i><font color="#ababab"># No argument - show current state</font></i>
<font color="#ff0000"> </font><font color="#ff0000">STATE</font><font color="#F3E651">=</font><font color="#ff0000">$(get_state)</font>
<font color="#ff0000"> echo </font><font color="#bb00ff">"CARP state on $CARP_IF (vhid $VHID): $STATE"</font>
<font color="#ff0000"> check_auto_failback</font>
<font color="#ff0000"> </font><font color="#F3E651">;;</font>
<font color="#ff0000"> master</font><font color="#F3E651">)</font>
<font color="#ff0000"> </font><i><font color="#ababab"># Force to MASTER state</font></i>
<font color="#ff0000"> echo </font><font color="#bb00ff">"Setting CARP to MASTER state..."</font>
<font color="#ff0000"> ifconfig </font><font color="#bb00ff">"$CARP_IF"</font><font color="#ff0000"> vhid </font><font color="#bb00ff">"$VHID"</font><font color="#ff0000"> state master</font>
<font color="#ff0000"> sleep </font><font color="#bb00ff">1</font>
<font color="#ff0000"> </font><font color="#ff0000">STATE</font><font color="#F3E651">=</font><font color="#ff0000">$(get_state)</font>
<font color="#ff0000"> echo </font><font color="#bb00ff">"CARP state on $CARP_IF (vhid $VHID): $STATE"</font>
<font color="#ff0000"> check_auto_failback</font>
<font color="#ff0000"> </font><font color="#F3E651">;;</font>
<font color="#ff0000"> backup</font><font color="#F3E651">)</font>
<font color="#ff0000"> </font><i><font color="#ababab"># Force to BACKUP state</font></i>
<font color="#ff0000"> echo </font><font color="#bb00ff">"Setting CARP to BACKUP state..."</font>
<font color="#ff0000"> ifconfig </font><font color="#bb00ff">"$CARP_IF"</font><font color="#ff0000"> vhid </font><font color="#bb00ff">"$VHID"</font><font color="#ff0000"> state backup</font>
<font color="#ff0000"> sleep </font><font color="#bb00ff">1</font>
<font color="#ff0000"> </font><font color="#ff0000">STATE</font><font color="#F3E651">=</font><font color="#ff0000">$(get_state)</font>
<font color="#ff0000"> echo </font><font color="#bb00ff">"CARP state on $CARP_IF (vhid $VHID): $STATE"</font>
<font color="#ff0000"> check_auto_failback</font>
<font color="#ff0000"> </font><font color="#F3E651">;;</font>
<font color="#ff0000"> auto-failback</font><font color="#F3E651">)</font>
<font color="#ff0000"> </font><b><font color="#ffffff">case</font></b><font color="#ff0000"> </font><font color="#bb00ff">"$2"</font><font color="#ff0000"> </font><b><font color="#ffffff">in</font></b>
<font color="#ff0000"> </font><b><font color="#ffffff">enable</font></b><font color="#F3E651">)</font>
<font color="#ff0000"> </font><b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">[</font><font color="#ff0000"> -f </font><font color="#bb00ff">"$BLOCK_FILE"</font><font color="#ff0000"> </font><font color="#F3E651">];</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b>
<font color="#ff0000"> rm </font><font color="#bb00ff">"$BLOCK_FILE"</font>
<font color="#ff0000"> echo </font><font color="#bb00ff">"Auto-failback ENABLED (removed $BLOCK_FILE)"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">else</font></b>
<font color="#ff0000"> echo </font><font color="#bb00ff">"Auto-failback was already enabled"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">fi</font></b>
<font color="#ff0000"> </font><font color="#F3E651">;;</font>
<font color="#ff0000"> disable</font><font color="#F3E651">)</font>
<font color="#ff0000"> </font><b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">[</font><font color="#ff0000"> </font><font color="#F3E651">!</font><font color="#ff0000"> -f </font><font color="#bb00ff">"$BLOCK_FILE"</font><font color="#ff0000"> </font><font color="#F3E651">];</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b>
<font color="#ff0000"> touch </font><font color="#bb00ff">"$BLOCK_FILE"</font>
<font color="#ff0000"> echo </font><font color="#bb00ff">"Auto-failback DISABLED (created $BLOCK_FILE)"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">else</font></b>
<font color="#ff0000"> echo </font><font color="#bb00ff">"Auto-failback was already disabled"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">fi</font></b>
<font color="#ff0000"> </font><font color="#F3E651">;;</font>
<font color="#ff0000"> </font><font color="#F3E651">*)</font>
<font color="#ff0000"> echo </font><font color="#bb00ff">"Usage: $0 auto-failback [enable|disable]"</font>
<font color="#ff0000"> echo </font><font color="#bb00ff">" enable: Remove block file to allow automatic failback"</font>
<font color="#ff0000"> echo </font><font color="#bb00ff">" disable: Create block file to prevent automatic failback"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">exit</font></b><font color="#ff0000"> </font><font color="#bb00ff">1</font>
<font color="#ff0000"> </font><font color="#F3E651">;;</font>
<font color="#ff0000"> </font><b><font color="#ffffff">esac</font></b>
<font color="#ff0000"> </font><font color="#F3E651">;;</font>
<font color="#ff0000"> </font><font color="#F3E651">*)</font>
<font color="#ff0000"> echo </font><font color="#bb00ff">"Usage: $0 [master|backup|auto-failback enable|auto-failback disable]"</font>
<font color="#ff0000"> echo </font><font color="#bb00ff">" Without arguments: show current CARP state"</font>
<font color="#ff0000"> echo </font><font color="#bb00ff">" master: force this node to become CARP MASTER"</font>
<font color="#ff0000"> echo </font><font color="#bb00ff">" backup: force this node to become CARP BACKUP"</font>
<font color="#ff0000"> echo </font><font color="#bb00ff">" auto-failback enable: allow automatic failback to f0"</font>
<font color="#ff0000"> echo </font><font color="#bb00ff">" auto-failback disable: prevent automatic failback to f0"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">exit</font></b><font color="#ff0000"> </font><font color="#bb00ff">1</font>
<font color="#ff0000"> </font><font color="#F3E651">;;</font>
<b><font color="#ffffff">esac</font></b>
<font color="#ff0000">EOF</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas chmod </font><font color="#F3E651">+</font><font color="#ff0000">x /usr/local/bin/carp</font>
<i><font color="#ababab"># Copy to f1 as well</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> scp /usr/local/bin/carp f1</font><font color="#F3E651">:</font><font color="#ff0000">/tmp</font><font color="#F3E651">/</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas cp /tmp/carp /usr/local/bin/carp </font><font color="#F3E651">&&</font><font color="#ff0000"> doas chmod </font><font color="#F3E651">+</font><font color="#ff0000">x /usr/local/bin/carp</font>
</pre>
<br />
<span>Now you can easily manage CARP states and auto-failback:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># Check current CARP state</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas carp</font>
<font color="#ff0000">CARP state on re0 </font><font color="#F3E651">(</font><font color="#ff0000">vhid </font><font color="#bb00ff">1</font><font color="#F3E651">):</font><font color="#ff0000"> MASTER</font>
<i><font color="#ababab"># If auto-failback is disabled, you'll see a warning</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas carp</font>
<font color="#ff0000">CARP state on re0 </font><font color="#F3E651">(</font><font color="#ff0000">vhid </font><font color="#bb00ff">1</font><font color="#F3E651">):</font><font color="#ff0000"> MASTER</font>
<font color="#ff0000">WARNING</font><font color="#F3E651">:</font><font color="#ff0000"> Auto-failback is DISABLED </font><font color="#F3E651">(</font><font color="#ff0000">file exists</font><font color="#F3E651">:</font><font color="#ff0000"> /data/nfs/nfs</font><font color="#F3E651">.</font><font color="#ff0000">NO_AUTO_FAILBACK</font><font color="#F3E651">)</font>
<i><font color="#ababab"># Force f0 to become BACKUP (triggers failover to f1)</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas carp backup</font>
<font color="#ff0000">Setting CARP to BACKUP state</font><font color="#F3E651">...</font>
<font color="#ff0000">CARP state on re0 </font><font color="#F3E651">(</font><font color="#ff0000">vhid </font><font color="#bb00ff">1</font><font color="#F3E651">):</font><font color="#ff0000"> BACKUP</font>
<i><font color="#ababab"># Disable auto-failback (useful for maintenance)</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas carp auto-failback disable</font>
<font color="#ff0000">Auto-failback DISABLED </font><font color="#F3E651">(</font><font color="#ff0000">created /data/nfs/nfs</font><font color="#F3E651">.</font><font color="#ff0000">NO_AUTO_FAILBACK</font><font color="#F3E651">)</font>
<i><font color="#ababab"># Enable auto-failback</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas carp auto-failback </font><b><font color="#ffffff">enable</font></b>
<font color="#ff0000">Auto-failback ENABLED </font><font color="#F3E651">(</font><font color="#ff0000">removed /data/nfs/nfs</font><font color="#F3E651">.</font><font color="#ff0000">NO_AUTO_FAILBACK</font><font color="#F3E651">)</font>
</pre>
<br />
<h3 style='display: inline' id='automatic-failback-after-reboot'>Automatic Failback After Reboot</h3><br />
<br />
<span>When <span class='inlinecode'>f0</span> reboots (planned or unplanned), <span class='inlinecode'>f1</span> takes over as CARP MASTER. To ensure <span class='inlinecode'>f0</span> automatically reclaims its primary role once it's fully operational, we'll implement an automatic failback mechanism. With:</span><br />
<br />
<span class='quote'>Update: Fixed the script at Sun 4 Jan 00:04:28 EET 2026 - removed the NFS service check because when f0 is BACKUP, NFS services are intentionally stopped by carpcontrol.sh, which would prevent auto-failback from ever triggering.</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas tee /usr/local/bin/carp-auto-failback</font><font color="#F3E651">.</font><font color="#ff0000">sh </font><font color="#F3E651"><<</font><font color="#bb00ff">'EOF'</font>
<i><font color="#ababab">#!/bin/sh</font></i>
<i><font color="#ababab"># CARP automatic failback script for f0</font></i>
<i><font color="#ababab"># Ensures f0 reclaims MASTER role after reboot when storage is ready</font></i>
<font color="#ff0000">LOGFILE</font><font color="#F3E651">=</font><font color="#bb00ff">"/var/log/carp-auto-failback.log"</font>
<font color="#ff0000">MARKER_FILE</font><font color="#F3E651">=</font><font color="#bb00ff">"/data/nfs/nfs.DO_NOT_REMOVE"</font>
<font color="#ff0000">BLOCK_FILE</font><font color="#F3E651">=</font><font color="#bb00ff">"/data/nfs/nfs.NO_AUTO_FAILBACK"</font>
<font color="#7bc710">log_message()</font><font color="#ff0000"> {</font>
<font color="#ff0000"> echo </font><font color="#bb00ff">"$(date '+%Y-%m-%d %H:%M:%S') - $1"</font><font color="#ff0000"> </font><font color="#F3E651">>></font><font color="#ff0000"> </font><font color="#bb00ff">"$LOGFILE"</font>
<font color="#ff0000">}</font>
<i><font color="#ababab"># Check if we're already MASTER</font></i>
<font color="#ff0000">CURRENT_STATE</font><font color="#F3E651">=</font><font color="#ff0000">$(</font><font color="#ff0000">/usr/local/bin/carp </font><font color="#F3E651">|</font><font color="#ff0000"> awk </font><font color="#bb00ff">'{print $NF}'</font><font color="#F3E651">)</font>
<b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">[</font><font color="#ff0000"> </font><font color="#bb00ff">"$CURRENT_STATE"</font><font color="#ff0000"> </font><font color="#F3E651">=</font><font color="#ff0000"> </font><font color="#bb00ff">"MASTER"</font><font color="#ff0000"> </font><font color="#F3E651">];</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b>
<font color="#ff0000"> </font><b><font color="#ffffff">exit</font></b><font color="#ff0000"> </font><font color="#bb00ff">0</font>
<b><font color="#ffffff">fi</font></b>
<i><font color="#ababab"># Check if /data/nfs is mounted</font></i>
<b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">!</font><font color="#ff0000"> mount </font><font color="#F3E651">|</font><font color="#ff0000"> grep -q </font><font color="#bb00ff">"on /data/nfs "</font><font color="#F3E651">;</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b>
<font color="#ff0000"> log_message </font><font color="#bb00ff">"SKIP: /data/nfs not mounted"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">exit</font></b><font color="#ff0000"> </font><font color="#bb00ff">0</font>
<b><font color="#ffffff">fi</font></b>
<i><font color="#ababab"># Check if the marker file exists</font></i>
<i><font color="#ababab"># (identifies that the ZFS data set is properly mounted)</font></i>
<b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">[</font><font color="#ff0000"> </font><font color="#F3E651">!</font><font color="#ff0000"> -f </font><font color="#bb00ff">"$MARKER_FILE"</font><font color="#ff0000"> </font><font color="#F3E651">];</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b>
<font color="#ff0000"> log_message </font><font color="#bb00ff">"SKIP: Marker file $MARKER_FILE not found"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">exit</font></b><font color="#ff0000"> </font><font color="#bb00ff">0</font>
<b><font color="#ffffff">fi</font></b>
<i><font color="#ababab"># Check if failback is blocked (for maintenance)</font></i>
<b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">[</font><font color="#ff0000"> -f </font><font color="#bb00ff">"$BLOCK_FILE"</font><font color="#ff0000"> </font><font color="#F3E651">];</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b>
<font color="#ff0000"> log_message </font><font color="#bb00ff">"SKIP: Failback blocked by $BLOCK_FILE"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">exit</font></b><font color="#ff0000"> </font><font color="#bb00ff">0</font>
<b><font color="#ffffff">fi</font></b>
<i><font color="#ababab"># All conditions met - promote to MASTER</font></i>
<font color="#ff0000">log_message </font><font color="#bb00ff">"CONDITIONS MET: Promoting to MASTER (was $CURRENT_STATE)"</font>
<font color="#ff0000">/usr/local/bin/carp master</font>
<i><font color="#ababab"># Log result</font></i>
<font color="#ff0000">sleep </font><font color="#bb00ff">2</font>
<font color="#ff0000">NEW_STATE</font><font color="#F3E651">=</font><font color="#ff0000">$(</font><font color="#ff0000">/usr/local/bin/carp </font><font color="#F3E651">|</font><font color="#ff0000"> awk </font><font color="#bb00ff">'{print $NF}'</font><font color="#F3E651">)</font>
<font color="#ff0000">log_message </font><font color="#bb00ff">"Failback complete: State is now $NEW_STATE"</font>
<i><font color="#ababab"># If successful, log to the system log too</font></i>
<b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">[</font><font color="#ff0000"> </font><font color="#bb00ff">"$NEW_STATE"</font><font color="#ff0000"> </font><font color="#F3E651">=</font><font color="#ff0000"> </font><font color="#bb00ff">"MASTER"</font><font color="#ff0000"> </font><font color="#F3E651">];</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b>
<font color="#ff0000"> logger </font><font color="#bb00ff">"CARP: f0 automatically reclaimed MASTER role"</font>
<b><font color="#ffffff">fi</font></b>
<font color="#ff0000">EOF</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas chmod </font><font color="#F3E651">+</font><font color="#ff0000">x /usr/local/bin/carp-auto-failback</font><font color="#F3E651">.</font><font color="#ff0000">sh</font>
</pre>
<br />
<span>The marker file identifies that the ZFS data set is mounted correctly. We create it with:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas touch /data/nfs/nfs</font><font color="#F3E651">.</font><font color="#ff0000">DO_NOT_REMOVE</font>
</pre>
<br />
<span>We add a cron job to check every minute:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> echo </font><font color="#bb00ff">"* * * * * /usr/local/bin/carp-auto-failback.sh"</font><font color="#ff0000"> </font><font color="#F3E651">|</font><font color="#ff0000"> doas crontab -</font>
</pre>
<br />
<span>The enhanced CARP script provides integrated control over auto-failback. To temporarily turn off automatic failback (e.g., for <span class='inlinecode'>f0</span> maintenance), we run:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas carp auto-failback disable</font>
<font color="#ff0000">Auto-failback DISABLED </font><font color="#F3E651">(</font><font color="#ff0000">created /data/nfs/nfs</font><font color="#F3E651">.</font><font color="#ff0000">NO_AUTO_FAILBACK</font><font color="#F3E651">)</font>
</pre>
<br />
<span>And to re-enable it:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas carp auto-failback </font><b><font color="#ffffff">enable</font></b>
<font color="#ff0000">Auto-failback ENABLED </font><font color="#F3E651">(</font><font color="#ff0000">removed /data/nfs/nfs</font><font color="#F3E651">.</font><font color="#ff0000">NO_AUTO_FAILBACK</font><font color="#F3E651">)</font>
</pre>
<br />
<span>To check whether auto-failback is enabled, we run:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas carp</font>
<font color="#ff0000">CARP state on re0 </font><font color="#F3E651">(</font><font color="#ff0000">vhid </font><font color="#bb00ff">1</font><font color="#F3E651">):</font><font color="#ff0000"> MASTER</font>
<i><font color="#ababab"># If disabled, you'll see: WARNING: Auto-failback is DISABLED</font></i>
</pre>
<br />
<span>The failback attempts are logged to <span class='inlinecode'>/var/log/carp-auto-failback.log</span>!</span><br />
<br />
<span>So, in summary:</span><br />
<br />
<ul>
<li>After f<span class='inlinecode'>0 </span>reboots: <span class='inlinecode'>f1</span> is MASTER, f<span class='inlinecode'>0 </span>boots as BACKUP</li>
<li>Cron runs every minute: Checks if conditions are met (Is <span class='inlinecode'>f0</span> currently BACKUP? (don't run if already MASTER)), (Is /data/nfs mounted? (ZFS datasets are ready)), (Does marker file exist? (confirms this is primary storage)), (Is failback blocked? (admin can prevent failback)), (Are NFS services running? (system is fully ready))</li>
<li>Failback occurs: Typically 2-3 minutes after boot completes</li>
<li>Logging: All attempts logged for troubleshooting</li>
</ul><br />
<span>This ensures <span class='inlinecode'>f0</span> automatically resumes its role as primary storage server after any reboot, while providing administrative control when needed.</span><br />
<br />
<h2 style='display: inline' id='client-configuration-for-nfs-via-stunnel'>Client Configuration for NFS via Stunnel</h2><br />
<br />
<span>To mount NFS shares with stunnel encryption, clients must install and configure stunnel using their client certificates.</span><br />
<br />
<h3 style='display: inline' id='configuring-rocky-linux-clients-r0-r1-r2'>Configuring Rocky Linux Clients (<span class='inlinecode'>r0</span>, <span class='inlinecode'>r1</span>, <span class='inlinecode'>r2</span>)</h3><br />
<br />
<span>On the Rocky Linux VMs, we run:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># Install stunnel on client (example for `r0`)</font></i>
<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># dnf install -y stunnel nfs-utils</font></i>
<i><font color="#ababab"># Copy client certificate and CA certificate from f0</font></i>
<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># scp f0:/usr/local/etc/stunnel/ca/r0-stunnel.pem /etc/stunnel/</font></i>
<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># scp f0:/usr/local/etc/stunnel/ca/ca-cert.pem /etc/stunnel/</font></i>
<i><font color="#ababab"># Configure stunnel client with certificate authentication</font></i>
<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># tee /etc/stunnel/stunnel.conf <<'EOF'</font></i>
<font color="#ff0000">cert </font><font color="#F3E651">=</font><font color="#ff0000"> /etc/stunnel/r</font><font color="#bb00ff">0</font><font color="#ff0000">-stunnel</font><font color="#F3E651">.</font><font color="#ff0000">pem</font>
<font color="#ff0000">CAfile </font><font color="#F3E651">=</font><font color="#ff0000"> /etc/stunnel/ca-cert</font><font color="#F3E651">.</font><font color="#ff0000">pem</font>
<font color="#ff0000">client </font><font color="#F3E651">=</font><font color="#ff0000"> yes</font>
<font color="#ff0000">verify </font><font color="#F3E651">=</font><font color="#ff0000"> </font><font color="#bb00ff">2</font>
<font color="#F3E651">[</font><font color="#ff0000">nfs-ha</font><font color="#F3E651">]</font>
<font color="#ff0000">accept </font><font color="#F3E651">=</font><font color="#ff0000"> </font><font color="#bb00ff">127.0</font><font color="#F3E651">.</font><font color="#bb00ff">0.1</font><font color="#F3E651">:</font><font color="#bb00ff">2323</font>
<font color="#ff0000">connect </font><font color="#F3E651">=</font><font color="#ff0000"> </font><font color="#bb00ff">192.168</font><font color="#F3E651">.</font><font color="#bb00ff">1.138</font><font color="#F3E651">:</font><font color="#bb00ff">2323</font>
<font color="#ff0000">EOF</font>
<i><font color="#ababab"># Enable and start stunnel</font></i>
<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># systemctl enable --now stunnel</font></i>
<i><font color="#ababab"># Repeat for r1 and r2 with their respective certificates</font></i>
</pre>
<br />
<span>Note: Each client must use its certificate file (<span class='inlinecode'>r0-stunnel.pem</span>, <span class='inlinecode'>r1-stunnel.pem</span>, <span class='inlinecode'>r2-stunnel.pem</span>, or <span class='inlinecode'>earth-stunnel.pem</span> - the latter is for my Laptop, which can also mount the NFS shares).</span><br />
<br />
<h3 style='display: inline' id='nfsv4-user-mapping-config-on-rocky'>NFSv4 user mapping config on Rocky</h3><br />
<br />
<span class='quote'>Update: This section was added 08.08.2025!</span><br />
<br />
<span>For this, we need to set the <span class='inlinecode'>Domain</span> in <span class='inlinecode'>/etc/idmapd.conf</span> on all 3 Rocky hosts to <span class='inlinecode'>lan.buetow.org</span> (remember, earlier in this blog post we set the <span class='inlinecode'>nfsuserd</span> domain on the NFS server side to <span class='inlinecode'>lan.buetow.org</span> as well!)</span><br />
<br />
<pre>
[General]
Domain = lan.buetow.org
.
.
.
</pre>
<br />
<span>We also need to increase the inotify limit, otherwise nfs-idmapd may fail to start with "Too many open files":</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># echo 'fs.inotify.max_user_instances = 512' > /etc/sysctl.d/99-inotify.conf</font></i>
<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># sysctl -w fs.inotify.max_user_instances=512</font></i>
</pre>
<br />
<span>And afterwards, we need to run the following on all 3 Rocky hosts:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># systemctl start nfs-idmapd</font></i>
<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># systemctl enable --now nfs-client.target</font></i>
</pre>
<br />
<span>and then, safest, reboot those.</span><br />
<br />
<h3 style='display: inline' id='testing-nfs-mount-with-stunnel'>Testing NFS Mount with Stunnel</h3><br />
<br />
<span>To mount NFS through the stunnel encrypted tunnel, we run:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># Create a mount point</font></i>
<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># mkdir -p /data/nfs/k3svolumes</font></i>
<i><font color="#ababab"># Mount through stunnel (using localhost and NFSv4)</font></i>
<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># mount -t nfs4 -o port=2323 127.0.0.1:/k3svolumes /data/nfs/k3svolumes</font></i>
<i><font color="#ababab"># Verify mount</font></i>
<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># mount | grep k3svolumes</font></i>
<font color="#bb00ff">127.0</font><font color="#F3E651">.</font><font color="#bb00ff">0.1</font><font color="#F3E651">:</font><font color="#ff0000">/k3svolumes on /data/nfs/k3svolumes </font>
<font color="#ff0000"> </font><b><font color="#ffffff">type</font></b><font color="#ff0000"> nfs4 </font><font color="#F3E651">(</font><font color="#ff0000">rw</font><font color="#F3E651">,</font><font color="#ff0000">relatime</font><font color="#F3E651">,</font><font color="#ff0000">vers</font><font color="#F3E651">=</font><font color="#bb00ff">4.2</font><font color="#F3E651">,</font><font color="#ff0000">rsize</font><font color="#F3E651">=</font><font color="#bb00ff">131072</font><font color="#F3E651">,</font><font color="#ff0000">wsize</font><font color="#F3E651">=</font><font color="#bb00ff">131072</font><font color="#F3E651">,</font>
<font color="#ff0000"> </font><font color="#ff0000">namlen</font><font color="#F3E651">=</font><font color="#bb00ff">255</font><font color="#F3E651">,</font><font color="#ff0000">hard</font><font color="#F3E651">,</font><font color="#ff0000">proto</font><font color="#F3E651">=</font><font color="#ff0000">tcp</font><font color="#F3E651">,</font><font color="#ff0000">port</font><font color="#F3E651">=</font><font color="#bb00ff">2323</font><font color="#F3E651">,</font><font color="#ff0000">timeo</font><font color="#F3E651">=</font><font color="#bb00ff">600</font><font color="#F3E651">,</font><font color="#ff0000">retrans</font><font color="#F3E651">=</font><font color="#bb00ff">2</font><font color="#F3E651">,</font><font color="#ff0000">sec</font><font color="#F3E651">=</font><font color="#ff0000">sys</font><font color="#F3E651">,</font>
<font color="#ff0000"> </font><font color="#ff0000">clientaddr</font><font color="#F3E651">=</font><font color="#bb00ff">127.0</font><font color="#F3E651">.</font><font color="#bb00ff">0.1</font><font color="#F3E651">,</font><font color="#ff0000">local_lock</font><font color="#F3E651">=</font><font color="#ff0000">none</font><font color="#F3E651">,</font><font color="#ff0000">addr</font><font color="#F3E651">=</font><font color="#bb00ff">127.0</font><font color="#F3E651">.</font><font color="#bb00ff">0.1</font><font color="#F3E651">)</font>
<i><font color="#ababab"># For persistent mount, add to /etc/fstab:</font></i>
<font color="#bb00ff">127.0</font><font color="#F3E651">.</font><font color="#bb00ff">0.1</font><font color="#F3E651">:</font><font color="#ff0000">/k3svolumes /data/nfs/k3svolumes nfs4 </font><font color="#ff0000">port</font><font color="#F3E651">=</font><font color="#bb00ff">2323</font><font color="#F3E651">,</font><font color="#ff0000">_netdev</font><font color="#F3E651">,</font><font color="#ff0000">soft</font><font color="#F3E651">,</font><font color="#ff0000">timeo</font><font color="#F3E651">=</font><font color="#bb00ff">10</font><font color="#F3E651">,</font><font color="#ff0000">retrans</font><font color="#F3E651">=</font><font color="#bb00ff">2</font><font color="#F3E651">,</font><font color="#ff0000">intr </font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#bb00ff">0</font>
</pre>
<br />
<span>Note: The mount uses localhost (<span class='inlinecode'>127.0.0.1</span>) because stunnel is listening locally and forwarding the encrypted traffic to the remote server.</span><br />
<br />
<h3 style='display: inline' id='testing-carp-failover-with-mounted-clients-and-stale-file-handles'>Testing CARP Failover with mounted clients and stale file handles:</h3><br />
<br />
<span>To test the failover process:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># On f0 (current MASTER) - trigger failover</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas ifconfig re0 vhid </font><font color="#bb00ff">1</font><font color="#ff0000"> state backup</font>
<i><font color="#ababab"># On f1 - verify it becomes MASTER</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> ifconfig re0 </font><font color="#F3E651">|</font><font color="#ff0000"> grep carp</font>
<font color="#ff0000"> inet </font><font color="#bb00ff">192.168</font><font color="#F3E651">.</font><font color="#bb00ff">1.138</font><font color="#ff0000"> netmask </font><font color="#bb00ff">0xffffffff</font><font color="#ff0000"> broadcast </font><font color="#bb00ff">192.168</font><font color="#F3E651">.</font><font color="#bb00ff">1.138</font><font color="#ff0000"> vhid </font><font color="#bb00ff">1</font>
<i><font color="#ababab"># Check stunnel is now listening on f1</font></i>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sockstat -l </font><font color="#F3E651">|</font><font color="#ff0000"> grep </font><font color="#bb00ff">2323</font>
<font color="#ff0000">stunnel stunnel </font><font color="#bb00ff">4567</font><font color="#ff0000"> </font><font color="#bb00ff">3</font><font color="#ff0000"> tcp4 </font><font color="#bb00ff">192.168</font><font color="#F3E651">.</font><font color="#bb00ff">1.138</font><font color="#F3E651">:</font><font color="#bb00ff">2323</font><font color="#ff0000"> </font><font color="#F3E651">*:*</font>
<i><font color="#ababab"># On client - verify NFS mount still works</font></i>
<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># ls /data/nfs/k3svolumes/</font></i>
<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># echo "Test after failover" > /data/nfs/k3svolumes/failover-test.txt</font></i>
</pre>
<br />
<span>After a CARP failover, NFS clients may experience "Stale file handle" errors because they cached file handles from the previous server. To resolve this manually, we can run:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># Force unmount and remount</font></i>
<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># umount -f /data/nfs/k3svolumes</font></i>
<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># mount /data/nfs/k3svolumes</font></i>
</pre>
<br />
<span>For the automatic recovery, we create a script:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># cat > /usr/local/bin/check-nfs-mount.sh << 'EOF'</font></i>
<i><font color="#ababab">#!/bin/bash</font></i>
<i><font color="#ababab"># Fast NFS mount health monitor - runs every 10 seconds via systemd timer</font></i>
<font color="#ff0000">MOUNT_POINT</font><font color="#F3E651">=</font><font color="#bb00ff">"/data/nfs/k3svolumes"</font>
<font color="#ff0000">LOCK_FILE</font><font color="#F3E651">=</font><font color="#bb00ff">"/var/run/nfs-mount-check.lock"</font>
<i><font color="#ababab"># Use a lock file to prevent concurrent runs</font></i>
<b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">[</font><font color="#ff0000"> -f </font><font color="#bb00ff">"$LOCK_FILE"</font><font color="#ff0000"> </font><font color="#F3E651">];</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b>
<font color="#ff0000"> </font><b><font color="#ffffff">exit</font></b><font color="#ff0000"> </font><font color="#bb00ff">0</font>
<b><font color="#ffffff">fi</font></b>
<font color="#ff0000">touch </font><font color="#bb00ff">"$LOCK_FILE"</font>
<b><font color="#ffffff">trap</font></b><font color="#ff0000"> </font><font color="#bb00ff">"rm -f $LOCK_FILE"</font><font color="#ff0000"> EXIT</font>
<font color="#7bc710">fix_mount ()</font><font color="#ff0000"> {</font>
<font color="#ff0000"> echo </font><font color="#bb00ff">"Attempting to remount NFS mount $MOUNT_POINT"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">if</font></b><font color="#ff0000"> mount -o remount -f </font><font color="#bb00ff">"$MOUNT_POINT"</font><font color="#ff0000"> </font><font color="#bb00ff">2</font><font color="#F3E651">></font><font color="#ff0000">/dev/null</font><font color="#F3E651">;</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b>
<font color="#ff0000"> echo </font><font color="#bb00ff">"Remount command issued for $MOUNT_POINT"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">else</font></b>
<font color="#ff0000"> echo </font><font color="#bb00ff">"Failed to remount NFS mount $MOUNT_POINT"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">fi</font></b>
<font color="#ff0000"> echo </font><font color="#bb00ff">"Checking if $MOUNT_POINT is a mountpoint"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">if</font></b><font color="#ff0000"> mountpoint </font><font color="#bb00ff">"$MOUNT_POINT"</font><font color="#ff0000"> </font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font><font color="#F3E651">;</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b>
<font color="#ff0000"> echo </font><font color="#bb00ff">"$MOUNT_POINT is a valid mountpoint"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">else</font></b>
<font color="#ff0000"> echo </font><font color="#bb00ff">"$MOUNT_POINT is not a valid mountpoint, attempting mount"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">if</font></b><font color="#ff0000"> mount </font><font color="#bb00ff">"$MOUNT_POINT"</font><font color="#F3E651">;</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b>
<font color="#ff0000"> echo </font><font color="#bb00ff">"Successfully mounted $MOUNT_POINT"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">return</font></b>
<font color="#ff0000"> </font><b><font color="#ffffff">else</font></b>
<font color="#ff0000"> echo </font><font color="#bb00ff">"Failed to mount $MOUNT_POINT"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">fi</font></b>
<font color="#ff0000"> </font><b><font color="#ffffff">fi</font></b>
<font color="#ff0000"> echo </font><font color="#bb00ff">"Attempting to unmount $MOUNT_POINT"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">if</font></b><font color="#ff0000"> umount -f </font><font color="#bb00ff">"$MOUNT_POINT"</font><font color="#ff0000"> </font><font color="#bb00ff">2</font><font color="#F3E651">></font><font color="#ff0000">/dev/null</font><font color="#F3E651">;</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b>
<font color="#ff0000"> echo </font><font color="#bb00ff">"Successfully unmounted $MOUNT_POINT"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">else</font></b>
<font color="#ff0000"> echo </font><font color="#bb00ff">"Failed to unmount $MOUNT_POINT (it might not be mounted)"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">fi</font></b>
<font color="#ff0000"> echo </font><font color="#bb00ff">"Attempting to mount $MOUNT_POINT"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">if</font></b><font color="#ff0000"> mount </font><font color="#bb00ff">"$MOUNT_POINT"</font><font color="#F3E651">;</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b>
<font color="#ff0000"> echo </font><font color="#bb00ff">"NFS mount $MOUNT_POINT mounted successfully"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">return</font></b>
<font color="#ff0000"> </font><b><font color="#ffffff">else</font></b>
<font color="#ff0000"> echo </font><font color="#bb00ff">"Failed to mount NFS mount $MOUNT_POINT"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">fi</font></b>
<font color="#ff0000"> echo </font><font color="#bb00ff">"Failed to fix NFS mount $MOUNT_POINT"</font>
<font color="#ff0000"> </font><b><font color="#ffffff">exit</font></b><font color="#ff0000"> </font><font color="#bb00ff">1</font>
<font color="#ff0000">}</font>
<b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">!</font><font color="#ff0000"> mountpoint </font><font color="#bb00ff">"$MOUNT_POINT"</font><font color="#ff0000"> </font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font><font color="#F3E651">;</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b>
<font color="#ff0000"> echo </font><font color="#bb00ff">"NFS mount $MOUNT_POINT not found"</font>
<font color="#ff0000"> fix_mount</font>
<b><font color="#ffffff">fi</font></b>
<b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">!</font><font color="#ff0000"> timeout 2s stat </font><font color="#bb00ff">"$MOUNT_POINT"</font><font color="#ff0000"> </font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font><font color="#F3E651">;</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b>
<font color="#ff0000"> echo </font><font color="#bb00ff">"NFS mount $MOUNT_POINT appears to be unresponsive"</font>
<font color="#ff0000"> fix_mount</font>
<b><font color="#ffffff">fi</font></b>
<font color="#ff0000">EOF</font>
<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># chmod +x /usr/local/bin/check-nfs-mount.sh</font></i>
</pre>
<br />
<span>And we create the systemd service as follows:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># cat > /etc/systemd/system/nfs-mount-monitor.service << 'EOF'</font></i>
<font color="#F3E651">[</font><font color="#ff0000">Unit</font><font color="#F3E651">]</font>
<font color="#ff0000">Description</font><font color="#F3E651">=</font><font color="#ff0000">NFS Mount Health Monitor</font>
<font color="#ff0000">After</font><font color="#F3E651">=</font><font color="#ff0000">network-online</font><font color="#F3E651">.</font><font color="#ff0000">target</font>
<font color="#F3E651">[</font><font color="#ff0000">Service</font><font color="#F3E651">]</font>
<font color="#ff0000">Type</font><font color="#F3E651">=</font><font color="#ff0000">oneshot</font>
<font color="#ff0000">ExecStart</font><font color="#F3E651">=</font><font color="#ff0000">/usr/local/bin/check-nfs-mount</font><font color="#F3E651">.</font><font color="#ff0000">sh</font>
<font color="#ff0000">StandardOutput</font><font color="#F3E651">=</font><font color="#ff0000">journal</font>
<font color="#ff0000">StandardError</font><font color="#F3E651">=</font><font color="#ff0000">journal</font>
<font color="#ff0000">EOF</font>
</pre>
<br />
<span>And we also create the systemd timer (runs every 10 seconds):</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># cat > /etc/systemd/system/nfs-mount-monitor.timer << 'EOF'</font></i>
<font color="#F3E651">[</font><font color="#ff0000">Unit</font><font color="#F3E651">]</font>
<font color="#ff0000">Description</font><font color="#F3E651">=</font><font color="#ff0000">Run NFS Mount Health Monitor every </font><font color="#bb00ff">10</font><font color="#ff0000"> seconds</font>
<font color="#ff0000">Requires</font><font color="#F3E651">=</font><font color="#ff0000">nfs-mount-monitor</font><font color="#F3E651">.</font><font color="#ff0000">service</font>
<font color="#F3E651">[</font><font color="#ff0000">Timer</font><font color="#F3E651">]</font>
<font color="#ff0000">OnBootSec</font><font color="#F3E651">=</font><font color="#ff0000">30s</font>
<font color="#ff0000">OnUnitActiveSec</font><font color="#F3E651">=</font><font color="#ff0000">10s</font>
<font color="#ff0000">AccuracySec</font><font color="#F3E651">=</font><font color="#ff0000">1s</font>
<font color="#F3E651">[</font><font color="#ff0000">Install</font><font color="#F3E651">]</font>
<font color="#ff0000">WantedBy</font><font color="#F3E651">=</font><font color="#ff0000">timers</font><font color="#F3E651">.</font><font color="#ff0000">target</font>
<font color="#ff0000">EOF</font>
</pre>
<br />
<span>To enable and start the timer, we run:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># systemctl daemon-reload</font></i>
<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># systemctl enable nfs-mount-monitor.timer</font></i>
<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># systemctl start nfs-mount-monitor.timer</font></i>
<i><font color="#ababab"># Check status</font></i>
<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># systemctl status nfs-mount-monitor.timer</font></i>
<font color="#ff0000">● nfs-mount-monitor</font><font color="#F3E651">.</font><font color="#ff0000">timer - Run NFS Mount Health Monitor every </font><font color="#bb00ff">10</font><font color="#ff0000"> seconds</font>
<font color="#ff0000"> Loaded</font><font color="#F3E651">:</font><font color="#ff0000"> loaded </font><font color="#F3E651">(</font><font color="#ff0000">/etc/systemd/system/nfs-mount-monitor</font><font color="#F3E651">.</font><font color="#ff0000">timer</font><font color="#F3E651">;</font><font color="#ff0000"> enabled</font><font color="#F3E651">)</font>
<font color="#ff0000"> Active</font><font color="#F3E651">:</font><font color="#ff0000"> active </font><font color="#F3E651">(</font><font color="#ff0000">waiting</font><font color="#F3E651">)</font><font color="#ff0000"> since Sat </font><font color="#bb00ff">2025</font><font color="#ff0000">-</font><font color="#bb00ff">07</font><font color="#ff0000">-</font><font color="#bb00ff">06</font><font color="#ff0000"> </font><font color="#bb00ff">10</font><font color="#F3E651">:</font><font color="#bb00ff">00</font><font color="#F3E651">:</font><font color="#bb00ff">00</font><font color="#ff0000"> EEST</font>
<font color="#ff0000"> Trigger</font><font color="#F3E651">:</font><font color="#ff0000"> Sat </font><font color="#bb00ff">2025</font><font color="#ff0000">-</font><font color="#bb00ff">07</font><font color="#ff0000">-</font><font color="#bb00ff">06</font><font color="#ff0000"> </font><font color="#bb00ff">10</font><font color="#F3E651">:</font><font color="#bb00ff">00</font><font color="#F3E651">:</font><font color="#bb00ff">10</font><font color="#ff0000"> EEST</font><font color="#F3E651">;</font><font color="#ff0000"> 8s left</font>
<i><font color="#ababab"># Monitor logs</font></i>
<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># journalctl -u nfs-mount-monitor -f</font></i>
</pre>
<br />
<span>Note: Stale file handles are inherent to NFS failover because file handles are server-specific. The best approach depends on your application's tolerance for brief disruptions. Of course, all the changes made to <span class='inlinecode'>r0</span> above must also be applied to <span class='inlinecode'>r1</span> and <span class='inlinecode'>r2</span>.</span><br />
<br />
<h3 style='display: inline' id='complete-failover-test'>Complete Failover Test</h3><br />
<br />
<span>Here's a comprehensive test of the failover behaviour with all optimisations in place:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><i><font color="#ababab"># 1. Check the initial state</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> ifconfig re0 </font><font color="#F3E651">|</font><font color="#ff0000"> grep carp</font>
<font color="#ff0000"> carp</font><font color="#F3E651">:</font><font color="#ff0000"> MASTER vhid </font><font color="#bb00ff">1</font><font color="#ff0000"> advbase </font><font color="#bb00ff">1</font><font color="#ff0000"> advskew </font><font color="#bb00ff">0</font>
<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> ifconfig re0 </font><font color="#F3E651">|</font><font color="#ff0000"> grep carp</font>
<font color="#ff0000"> carp</font><font color="#F3E651">:</font><font color="#ff0000"> BACKUP vhid </font><font color="#bb00ff">1</font><font color="#ff0000"> advbase </font><font color="#bb00ff">1</font><font color="#ff0000"> advskew </font><font color="#bb00ff">100</font>
<i><font color="#ababab"># 2. Create a test file from a client</font></i>
<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># echo "test before failover" > /data/nfs/k3svolumes/test-before.txt</font></i>
<i><font color="#ababab"># 3. Trigger failover (f0 → f1)</font></i>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas ifconfig re0 vhid </font><font color="#bb00ff">1</font><font color="#ff0000"> state backup</font>
<i><font color="#ababab"># 4. Monitor client behaviour</font></i>
<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># ls /data/nfs/k3svolumes/</font></i>
<font color="#ff0000">ls</font><font color="#F3E651">:</font><font color="#ff0000"> cannot access </font><font color="#bb00ff">'/data/nfs/k3svolumes/'</font><font color="#F3E651">:</font><font color="#ff0000"> Stale file handle</font>
<i><font color="#ababab"># 5. Check automatic recovery (within 10 seconds)</font></i>
<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># journalctl -u nfs-mount-monitor -f</font></i>
<font color="#ff0000">Jul </font><font color="#bb00ff">06</font><font color="#ff0000"> </font><font color="#bb00ff">10</font><font color="#F3E651">:</font><font color="#bb00ff">15</font><font color="#F3E651">:</font><font color="#bb00ff">32</font><font color="#ff0000"> r0 nfs-monitor</font><font color="#F3E651">[</font><font color="#bb00ff">1234</font><font color="#F3E651">]:</font><font color="#ff0000"> NFS mount unhealthy detected at </font><font color="#F3E651">\</font>
<font color="#ff0000"> Sun Jul </font><font color="#bb00ff">6</font><font color="#ff0000"> </font><font color="#bb00ff">10</font><font color="#F3E651">:</font><font color="#bb00ff">15</font><font color="#F3E651">:</font><font color="#bb00ff">32</font><font color="#ff0000"> EEST </font><font color="#bb00ff">2025</font>
<font color="#ff0000">Jul </font><font color="#bb00ff">06</font><font color="#ff0000"> </font><font color="#bb00ff">10</font><font color="#F3E651">:</font><font color="#bb00ff">15</font><font color="#F3E651">:</font><font color="#bb00ff">32</font><font color="#ff0000"> r0 nfs-monitor</font><font color="#F3E651">[</font><font color="#bb00ff">1234</font><font color="#F3E651">]:</font><font color="#ff0000"> Attempting to fix stale NFS mount at </font><font color="#F3E651">\</font>
<font color="#ff0000"> Sun Jul </font><font color="#bb00ff">6</font><font color="#ff0000"> </font><font color="#bb00ff">10</font><font color="#F3E651">:</font><font color="#bb00ff">15</font><font color="#F3E651">:</font><font color="#bb00ff">32</font><font color="#ff0000"> EEST </font><font color="#bb00ff">2025</font>
<font color="#ff0000">Jul </font><font color="#bb00ff">06</font><font color="#ff0000"> </font><font color="#bb00ff">10</font><font color="#F3E651">:</font><font color="#bb00ff">15</font><font color="#F3E651">:</font><font color="#bb00ff">33</font><font color="#ff0000"> r0 nfs-monitor</font><font color="#F3E651">[</font><font color="#bb00ff">1234</font><font color="#F3E651">]:</font><font color="#ff0000"> NFS mount fixed at </font><font color="#F3E651">\</font>
<font color="#ff0000"> Sun Jul </font><font color="#bb00ff">6</font><font color="#ff0000"> </font><font color="#bb00ff">10</font><font color="#F3E651">:</font><font color="#bb00ff">15</font><font color="#F3E651">:</font><font color="#bb00ff">33</font><font color="#ff0000"> EEST </font><font color="#bb00ff">2025</font>
</pre>
<br />
<span>Failover Timeline:</span><br />
<br />
<ul>
<li>0 seconds: CARP failover triggered</li>
<li>0-2 seconds: Clients get "Stale file handle" errors (not hanging)</li>
<li>3-10 seconds: Soft mounts ensure quick failure of operations</li>
<li>Within 10 seconds: Automatic recovery via systemd timer</li>
</ul><br />
<span>Benefits of the Optimised Setup:</span><br />
<br />
<ul>
<li>No hanging processes - Soft mounts fail quickly</li>
<li>Clean failover - Old server stops serving immediately</li>
<li>Fast automatic recovery - No manual intervention needed</li>
<li>Predictable timing - Recovery within 10 seconds with systemd timer</li>
<li>Better visibility - systemd journal provides detailed logs</li>
</ul><br />
<span>Important Considerations:</span><br />
<br />
<ul>
<li>Recent writes (within 1 minute) may not be visible after failover due to replication lag</li>
<li>Applications should handle brief NFS errors gracefully</li>
<li>For zero-downtime requirements, consider synchronous replication or distributed storage (see "Future storage explorations" section later in this blog post)</li>
</ul><br />
<h2 style='display: inline' id='update-upgrade-to-4tb-drives'>Update: Upgrade to 4TB drives</h2><br />
<br />
<span class='quote'>Update: 27.01.2026 I have since replaced the 1TB drives with 4TB drives for more storage capacity. The upgrade procedure was different for each node!</span><br />
<br />
<h3 style='display: inline' id='upgrading-f1-simpler-approach'>Upgrading f1 (simpler approach)</h3><br />
<br />
<span>Since f1 is the replication sink, the upgrade was straightforward:</span><br />
<br />
<ul>
<li>1. Physically replaced the 1TB drive with the 4TB drive</li>
<li>2. Re-setup the drive as described earlier in this blog post</li>
<li>3. Re-replicated all data from f0 to f1 via zrepl</li>
<li>4. Reloaded the encryption keys as described in this blog post</li>
<li>5. Set the mount point again for the encrypted dataset, explicitly as read-only (since f1 is the replication sink)</li>
</ul><br />
<h3 style='display: inline' id='upgrading-f0-using-zfs-resilvering'>Upgrading f0 (using ZFS resilvering)</h3><br />
<br />
<span>For f0, which is the primary storage node, I used ZFS resilvering to avoid data loss:</span><br />
<br />
<ul>
<li>1. Plugged the new 4TB drive into an external USB SSD drive reader</li>
<li>2. Attached the 4TB drive to the zdata pool for resilvering</li>
<li>3. Once resilvering completed, detached the 1TB drive from the zdata pool</li>
<li>4. Shutdown f0 and physically replaced the internal drive</li>
<li>5. Booted with the new drive in place</li>
<li>6. Expanded the pool to use the full 4TB capacity:</li>
</ul><br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zpool online -e /dev/ada</font><font color="#bb00ff">1</font>
</pre>
<br />
<ul>
<li>7. Reloaded the encryption keys as described in this blog post</li>
<li>8. Set the mount point again for the encrypted dataset</li>
</ul><br />
<span>This was a one-time effort on both nodes - after a reboot, everything was remembered and came up normally. Here are the updated outputs:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zpool list</font>
<font color="#ff0000">NAME SIZE ALLOC FREE CKPOINT EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT</font>
<font color="#ff0000">zdata </font><font color="#bb00ff">3</font><font color="#F3E651">.</font><font color="#ff0000">63T 677G </font><font color="#bb00ff">2</font><font color="#F3E651">.</font><font color="#ff0000">97T - - </font><font color="#bb00ff">3</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">18</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">1</font><font color="#F3E651">.</font><font color="#ff0000">00x ONLINE -</font>
<font color="#ff0000">zroot 472G </font><font color="#bb00ff">68</font><font color="#F3E651">.</font><font color="#ff0000">4G 404G - - </font><font color="#bb00ff">13</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">14</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">1</font><font color="#F3E651">.</font><font color="#ff0000">00x ONLINE -</font>
<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas camcontrol devlist</font>
<font color="#F3E651"><</font><font color="#ff0000">512GB SSD D910R170</font><font color="#F3E651">></font><font color="#ff0000"> at scbus0 target </font><font color="#bb00ff">0</font><font color="#ff0000"> lun </font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#F3E651">(</font><font color="#ff0000">pass0</font><font color="#F3E651">,</font><font color="#ff0000">ada0</font><font color="#F3E651">)</font>
<font color="#F3E651"><</font><font color="#ff0000">SD Ultra 3D 4TB 530500WD</font><font color="#F3E651">></font><font color="#ff0000"> at scbus1 target </font><font color="#bb00ff">0</font><font color="#ff0000"> lun </font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#F3E651">(</font><font color="#ff0000">pass1</font><font color="#F3E651">,</font><font color="#ff0000">ada1</font><font color="#F3E651">)</font>
<font color="#F3E651"><</font><font color="#ff0000">Generic Flash Disk </font><font color="#bb00ff">8.07</font><font color="#F3E651">></font><font color="#ff0000"> at scbus2 target </font><font color="#bb00ff">0</font><font color="#ff0000"> lun </font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#F3E651">(</font><font color="#ff0000">da0</font><font color="#F3E651">,</font><font color="#ff0000">pass2</font><font color="#F3E651">)</font>
</pre>
<br />
<span>We're still using different SSD models on f1 (WD Blue SA510 4TB) to avoid simultaneous failures:</span><br />
<br />
<!-- Generator: GNU source-highlight 3.1.9
by Lorenzo Bettini
http://www.lorenzobettini.it
http://www.gnu.org/software/src-highlite -->
<pre><font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas camcontrol devlist</font>
<font color="#F3E651"><</font><font color="#ff0000">512GB SSD D910R170</font><font color="#F3E651">></font><font color="#ff0000"> at scbus0 target </font><font color="#bb00ff">0</font><font color="#ff0000"> lun </font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#F3E651">(</font><font color="#ff0000">pass0</font><font color="#F3E651">,</font><font color="#ff0000">ada0</font><font color="#F3E651">)</font>
<font color="#F3E651"><</font><font color="#ff0000">WD Blue SA510 </font><font color="#bb00ff">2.5</font><font color="#ff0000"> 4TB 530500WD</font><font color="#F3E651">></font><font color="#ff0000"> at scbus1 target </font><font color="#bb00ff">0</font><font color="#ff0000"> lun </font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#F3E651">(</font><font color="#ff0000">pass1</font><font color="#F3E651">,</font><font color="#ff0000">ada1</font><font color="#F3E651">)</font>
<font color="#F3E651"><</font><font color="#ff0000">Generic Flash Disk </font><font color="#bb00ff">8.07</font><font color="#F3E651">></font><font color="#ff0000"> at scbus2 target </font><font color="#bb00ff">0</font><font color="#ff0000"> lun </font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#F3E651">(</font><font color="#ff0000">da0</font><font color="#F3E651">,</font><font color="#ff0000">pass2</font><font color="#F3E651">)</font>
</pre>
<br />
<h2 style='display: inline' id='conclusion'>Conclusion</h2><br />
<br />
<span>We've built a robust, encrypted storage system for our FreeBSD-based Kubernetes cluster that provides:</span><br />
<br />
<ul>
<li>High Availability: CARP ensures the storage VIP moves automatically during failures</li>
<li>Data Protection: ZFS encryption protects data at rest, stunnel protects data in transit</li>
<li>Continuous Replication: 1-minute RPO for the data, automated via <span class='inlinecode'>zrepl</span></li>
<li>Secure Access: Client certificate authentication prevents unauthorised access</li>
</ul><br />
<span>Some key lessons learned are:</span><br />
<br />
<ul>
<li>Stunnel vs Native NFS/TLS: While native encryption would be ideal, stunnel provides better cross-platform compatibility</li>
<li>Manual vs Automatic Failover: For storage systems, controlled failover often prevents more problems than it causes</li>
<li>Client Compatibility: Different NFS implementations behave differently - test thoroughly</li>
</ul><br />
<h2 style='display: inline' id='future-storage-explorations'>Future Storage Explorations</h2><br />
<br />
<span>While <span class='inlinecode'>zrepl</span> provides excellent snapshot-based replication for disaster recovery, there are other storage technologies worth exploring for the f3s project:</span><br />
<br />
<h3 style='display: inline' id='minio-for-s3-compatible-object-storage'>MinIO for S3-Compatible Object Storage</h3><br />
<br />
<span>MinIO is a high-performance, S3-compatible object storage system that could complement our ZFS-based storage. Some potential use cases:</span><br />
<br />
<ul>
<li>S3 API compatibility: Many modern applications expect S3-style object storage APIs. MinIO could provide this interface while using our ZFS storage as the backend.</li>
<li>Multi-site replication: MinIO supports active-active replication across multiple sites, which could work well with our f0/f1/f2 node setup.</li>
<li>Kubernetes native: MinIO has excellent Kubernetes integration with operators and CSI drivers, making it ideal for the f3s k3s environment.</li>
</ul><br />
<h3 style='display: inline' id='moosefs-for-distributed-high-availability'>MooseFS for Distributed High Availability</h3><br />
<br />
<span>MooseFS is a fault-tolerant, distributed file system that could provide proper high-availability storage:</span><br />
<br />
<ul>
<li>True HA: Unlike our current setup, which requires manual failover, MooseFS provides automatic failover with no single point of failure.</li>
<li>POSIX compliance: Applications can use MooseFS like any regular filesystem, no code changes needed.</li>
<li>Flexible redundancy: Configure different replication levels per directory or file, optimising storage efficiency.</li>
<li>FreeBSD support: MooseFS has native FreeBSD support, making it a natural fit for the f3s project.</li>
</ul><br />
<span>Both technologies could run on top of our encrypted ZFS volumes, combining ZFS's data integrity and encryption features with distributed storage capabilities. This would be particularly interesting for workloads that need either S3-compatible APIs (MinIO) or transparent distributed POSIX storage (MooseFS). What about Ceph and GlusterFS? Unfortunately, there doesn't seem to be great native FreeBSD support for them. However, other alternatives also appear suitable for my use case.</span><br />
<br />
<span>Read the next post of this series:</span><br />
<br />
<a class='textlink' href='./2025-10-02-f3s-kubernetes-with-freebsd-part-7.html'>f3s: Kubernetes with FreeBSD - Part 7: k3s and first pod deployments</a><br />
<br />
<span>Other *BSD-related posts:</span><br />
<br />
<a class='textlink' href='./2025-12-07-f3s-kubernetes-with-freebsd-part-8.html'>2025-12-07 f3s: Kubernetes with FreeBSD - Part 8: Observability</a><br />
<a class='textlink' href='./2025-10-02-f3s-kubernetes-with-freebsd-part-7.html'>2025-10-02 f3s: Kubernetes with FreeBSD - Part 7: k3s and first pod deployments</a><br />
<a class='textlink' href='./2025-07-14-f3s-kubernetes-with-freebsd-part-6.html'>2025-07-14 f3s: Kubernetes with FreeBSD - Part 6: Storage (You are currently reading this)</a><br />
<a class='textlink' href='./2025-05-11-f3s-kubernetes-with-freebsd-part-5.html'>2025-05-11 f3s: Kubernetes with FreeBSD - Part 5: WireGuard mesh network</a><br />
<a class='textlink' href='./2025-04-05-f3s-kubernetes-with-freebsd-part-4.html'>2025-04-05 f3s: Kubernetes with FreeBSD - Part 4: Rocky Linux Bhyve VMs</a><br />
<a class='textlink' href='./2025-02-01-f3s-kubernetes-with-freebsd-part-3.html'>2025-02-01 f3s: Kubernetes with FreeBSD - Part 3: Protecting from power cuts</a><br />
<a class='textlink' href='./2024-12-03-f3s-kubernetes-with-freebsd-part-2.html'>2024-12-03 f3s: Kubernetes with FreeBSD - Part 2: Hardware and base installation</a><br />
<a class='textlink' href='./2024-11-17-f3s-kubernetes-with-freebsd-part-1.html'>2024-11-17 f3s: Kubernetes with FreeBSD - Part 1: Setting the stage</a><br />
<a class='textlink' href='./2024-04-01-KISS-high-availability-with-OpenBSD.html'>2024-04-01 KISS high-availability with OpenBSD</a><br />
<a class='textlink' href='./2024-01-13-one-reason-why-i-love-openbsd.html'>2024-01-13 One reason why I love OpenBSD</a><br />
<a class='textlink' href='./2022-10-30-installing-dtail-on-openbsd.html'>2022-10-30 Installing DTail on OpenBSD</a><br />
<a class='textlink' href='./2022-07-30-lets-encrypt-with-openbsd-and-rex.html'>2022-07-30 Let's Encrypt with OpenBSD and Rex</a><br />
<a class='textlink' href='./2016-04-09-jails-and-zfs-on-freebsd-with-puppet.html'>2016-04-09 Jails and ZFS with Puppet on FreeBSD</a><br />
<br />
<span>E-Mail your comments to <span class='inlinecode'>paul@nospam.buetow.org</span></span><br />
<br />
<a class='textlink' href='../'>Back to the main site</a><br />
<p class="footer">
Generated with <a href="https://codeberg.org/snonux/gemtexter">Gemtexter 3.0.1-develop</a> |
served by <a href="https://www.OpenBSD.org">OpenBSD</a>/<a href="https://man.openbsd.org/relayd.8">relayd(8)</a>+<a href="https://man.openbsd.org/httpd.8">httpd(8)</a> |
<a href="https://foo.zone/site-mirrors.html">Site Mirrors</a>
<br />
Webring: <a href="https://shring.sh/foo.zone/previous">previous</a> | <a href="https://shring.sh">shring</a> | <a href="https://shring.sh/foo.zone/next">next</a>
</p>
</body>
</html>
|
