diff options
Diffstat (limited to 'gemfeed/2025-07-14-f3s-kubernetes-with-freebsd-part-6.html')
| -rw-r--r-- | gemfeed/2025-07-14-f3s-kubernetes-with-freebsd-part-6.html | 1840 |
1 files changed, 923 insertions, 917 deletions
diff --git a/gemfeed/2025-07-14-f3s-kubernetes-with-freebsd-part-6.html b/gemfeed/2025-07-14-f3s-kubernetes-with-freebsd-part-6.html index a66bad32..d3d00045 100644 --- a/gemfeed/2025-07-14-f3s-kubernetes-with-freebsd-part-6.html +++ b/gemfeed/2025-07-14-f3s-kubernetes-with-freebsd-part-6.html @@ -2,12 +2,17 @@ <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"> <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> +<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <title>f3s: Kubernetes with FreeBSD - Part 6: Storage</title> <link rel="shortcut icon" type="image/gif" href="/favicon.ico" /> <link rel="stylesheet" href="../style.css" /> <link rel="stylesheet" href="style-override.css" /> </head> -<body> +<body class="rfx-boot"> +<div class="rfx-overlay-grid"></div> +<div class="rfx-overlay-scanlines"></div> +<div id="rfx-stars"></div> +<div class="rfx-vignette"></div> <p class="header"> <a href="https://foo.zone">Home</a> | <a href="https://codeberg.org/snonux/foo.zone/src/branch/content-md/gemfeed/2025-07-14-f3s-kubernetes-with-freebsd-part-6.md">Markdown</a> | <a href="gemini://foo.zone/gemfeed/2025-07-14-f3s-kubernetes-with-freebsd-part-6.gmi">Gemini</a> </p> @@ -119,16 +124,16 @@ by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:~ % doas zpool create -m /data zdata /dev/ada<font color="#000000">1</font> -paul@f0:~ % zpool list -NAME SIZE ALLOC FREE CKPOINT EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT -zdata 928G <font color="#000000">12</font>.1M 928G - - <font color="#000000">0</font>% <font color="#000000">0</font>% <font color="#000000">1</font>.00x ONLINE - -zroot 472G <font color="#000000">29</font>.0G 443G - - <font color="#000000">0</font>% <font color="#000000">6</font>% <font color="#000000">1</font>.00x ONLINE - +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zpool create -m /data zdata /dev/ada</font><font color="#bb00ff">1</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> zpool list</font> +<font color="#ff0000">NAME SIZE ALLOC FREE CKPOINT EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT</font> +<font color="#ff0000">zdata 928G </font><font color="#bb00ff">12</font><font color="#F3E651">.</font><font color="#ff0000">1M 928G - - </font><font color="#bb00ff">0</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">0</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">1</font><font color="#F3E651">.</font><font color="#ff0000">00x ONLINE -</font> +<font color="#ff0000">zroot 472G </font><font color="#bb00ff">29</font><font color="#F3E651">.</font><font color="#ff0000">0G 443G - - </font><font color="#bb00ff">0</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">6</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">1</font><font color="#F3E651">.</font><font color="#ff0000">00x ONLINE -</font> -paul@f0:/ % doas camcontrol devlist -<512GB SSD D910R170> at scbus0 target <font color="#000000">0</font> lun <font color="#000000">0</font> (pass0,ada0) -<Samsung SSD <font color="#000000">870</font> EVO 1TB SVT03B6Q> at scbus1 target <font color="#000000">0</font> lun <font color="#000000">0</font> (pass1,ada1) -paul@f0:/ % +<font color="#ff0000">paul@f0</font><font color="#F3E651">:/</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas camcontrol devlist</font> +<font color="#F3E651"><</font><font color="#ff0000">512GB SSD D910R170</font><font color="#F3E651">></font><font color="#ff0000"> at scbus0 target </font><font color="#bb00ff">0</font><font color="#ff0000"> lun </font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#F3E651">(</font><font color="#ff0000">pass0</font><font color="#F3E651">,</font><font color="#ff0000">ada0</font><font color="#F3E651">)</font> +<font color="#F3E651"><</font><font color="#ff0000">Samsung SSD </font><font color="#bb00ff">870</font><font color="#ff0000"> EVO 1TB SVT03B6Q</font><font color="#F3E651">></font><font color="#ff0000"> at scbus1 target </font><font color="#bb00ff">0</font><font color="#ff0000"> lun </font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#F3E651">(</font><font color="#ff0000">pass1</font><font color="#F3E651">,</font><font color="#ff0000">ada1</font><font color="#F3E651">)</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:/</font><font color="#ff0000"> </font><font color="#F3E651">%</font> </pre> <br /> <span>To verify that we have a different SSD on the second node (the third node has the same drive as the first):</span><br /> @@ -137,9 +142,9 @@ paul@f0:/ % by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f1:/ % doas camcontrol devlist -<512GB SSD D910R170> at scbus0 target <font color="#000000">0</font> lun <font color="#000000">0</font> (pass0,ada0) -<CT1000BX500SSD1 M6CR072> at scbus1 target <font color="#000000">0</font> lun <font color="#000000">0</font> (pass1,ada1) +<pre><font color="#ff0000">paul@f1</font><font color="#F3E651">:/</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas camcontrol devlist</font> +<font color="#F3E651"><</font><font color="#ff0000">512GB SSD D910R170</font><font color="#F3E651">></font><font color="#ff0000"> at scbus0 target </font><font color="#bb00ff">0</font><font color="#ff0000"> lun </font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#F3E651">(</font><font color="#ff0000">pass0</font><font color="#F3E651">,</font><font color="#ff0000">ada0</font><font color="#F3E651">)</font> +<font color="#F3E651"><</font><font color="#ff0000">CT1000BX500SSD1 M6CR072</font><font color="#F3E651">></font><font color="#ff0000"> at scbus1 target </font><font color="#bb00ff">0</font><font color="#ff0000"> lun </font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#F3E651">(</font><font color="#ff0000">pass1</font><font color="#F3E651">,</font><font color="#ff0000">ada1</font><font color="#F3E651">)</font> </pre> <br /> <h2 style='display: inline' id='zfs-encryption-keys'>ZFS encryption keys</h2><br /> @@ -177,21 +182,21 @@ paul@f0:/ % by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:/ % doas newfs /dev/da<font color="#000000">0</font> -/dev/da<font color="#000000">0</font>: <font color="#000000">15000</font>.0MB (<font color="#000000">30720000</font> sectors) block size <font color="#000000">32768</font>, fragment size <font color="#000000">4096</font> - using <font color="#000000">24</font> cylinder groups of <font color="#000000">625</font>.22MB, <font color="#000000">20007</font> blks, <font color="#000000">80128</font> inodes. - with soft updates -super-block backups (<b><u><font color="#000000">for</font></u></b> fsck_ffs -b <i><font color="silver">#) at:</font></i> - <font color="#000000">192</font>, <font color="#000000">1280640</font>, <font color="#000000">2561088</font>, <font color="#000000">3841536</font>, <font color="#000000">5121984</font>, <font color="#000000">6402432</font>, <font color="#000000">7682880</font>, <font color="#000000">8963328</font>, <font color="#000000">10243776</font>, -<font color="#000000">11524224</font>, <font color="#000000">12804672</font>, <font color="#000000">14085120</font>, <font color="#000000">15365568</font>, <font color="#000000">16646016</font>, <font color="#000000">17926464</font>, <font color="#000000">19206912</font>,k <font color="#000000">20487360</font>, -... +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:/</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas newfs /dev/da</font><font color="#bb00ff">0</font> +<font color="#ff0000">/dev/da</font><font color="#bb00ff">0</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">15000</font><font color="#F3E651">.</font><font color="#ff0000">0MB </font><font color="#F3E651">(</font><font color="#bb00ff">30720000</font><font color="#ff0000"> sectors</font><font color="#F3E651">)</font><font color="#ff0000"> block size </font><font color="#bb00ff">32768</font><font color="#F3E651">,</font><font color="#ff0000"> fragment size </font><font color="#bb00ff">4096</font> +<font color="#ff0000"> using </font><font color="#bb00ff">24</font><font color="#ff0000"> cylinder groups of </font><font color="#bb00ff">625</font><font color="#F3E651">.</font><font color="#ff0000">22MB</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">20007</font><font color="#ff0000"> blks</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">80128</font><font color="#ff0000"> inodes</font><font color="#F3E651">.</font> +<font color="#ff0000"> with soft updates</font> +<font color="#ff0000">super-block backups </font><font color="#F3E651">(</font><b><font color="#ffffff">for</font></b><font color="#ff0000"> fsck_ffs -b </font><i><font color="#ababab">#) at:</font></i> +<font color="#ff0000"> </font><font color="#bb00ff">192</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">1280640</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">2561088</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">3841536</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">5121984</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">6402432</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">7682880</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">8963328</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">10243776</font><font color="#F3E651">,</font> +<font color="#bb00ff">11524224</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">12804672</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">14085120</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">15365568</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">16646016</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">17926464</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">19206912</font><font color="#F3E651">,</font><font color="#ff0000">k </font><font color="#bb00ff">20487360</font><font color="#F3E651">,</font> +<font color="#F3E651">...</font> -paul@f0:/ % echo <font color="#808080">'/dev/da0 /keys ufs rw 0 2'</font> | doas tee -a /etc/fstab -/dev/da<font color="#000000">0</font> /keys ufs rw <font color="#000000">0</font> <font color="#000000">2</font> -paul@f0:/ % doas mkdir /keys -paul@f0:/ % doas mount /keys -paul@f0:/ % df | grep keys -/dev/da<font color="#000000">0</font> <font color="#000000">14877596</font> <font color="#000000">8</font> <font color="#000000">13687384</font> <font color="#000000">0</font>% /keys +<font color="#ff0000">paul@f0</font><font color="#F3E651">:/</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> echo </font><font color="#bb00ff">'/dev/da0 /keys ufs rw 0 2'</font><font color="#ff0000"> </font><font color="#F3E651">|</font><font color="#ff0000"> doas tee -a /etc/fstab</font> +<font color="#ff0000">/dev/da</font><font color="#bb00ff">0</font><font color="#ff0000"> /keys ufs rw </font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#bb00ff">2</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:/</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas mkdir /keys</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:/</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas mount /keys</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:/</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> df </font><font color="#F3E651">|</font><font color="#ff0000"> grep keys</font> +<font color="#ff0000">/dev/da</font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#bb00ff">14877596</font><font color="#ff0000"> </font><font color="#bb00ff">8</font><font color="#ff0000"> </font><font color="#bb00ff">13687384</font><font color="#ff0000"> </font><font color="#bb00ff">0</font><font color="#F3E651">%</font><font color="#ff0000"> /keys</font> </pre> <br /> <a href='./f3s-kubernetes-with-freebsd-part-6/usbkeys2.jpg'><img alt='USB keys stuck in' title='USB keys stuck in' src='./f3s-kubernetes-with-freebsd-part-6/usbkeys2.jpg' /></a><br /> @@ -230,18 +235,18 @@ total 20 by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:/keys % doas zfs create -o encryption=on -o keyformat=raw -o \ - keylocation=file:///keys/`hostname`:zdata.key zdata/enc -paul@f0:/ % zfs list | grep zdata -zdata 836K 899G 96K /data -zdata/enc 200K 899G 200K /data/enc +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs create -o </font><font color="#ff0000">encryption</font><font color="#F3E651">=</font><font color="#ff0000">on -o </font><font color="#ff0000">keyformat</font><font color="#F3E651">=</font><font color="#ff0000">raw -o </font><font color="#F3E651">\</font> +<font color="#ff0000"> </font><font color="#ff0000">keylocation</font><font color="#F3E651">=</font><font color="#ff0000">file</font><font color="#F3E651">:</font><font color="#ff0000">///keys</font><font color="#F3E651">/</font><font color="#ff0000">`hostname`</font><font color="#F3E651">:</font><font color="#ff0000">zdata</font><font color="#F3E651">.</font><font color="#ff0000">key zdata/enc</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:/</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> zfs list </font><font color="#F3E651">|</font><font color="#ff0000"> grep zdata</font> +<font color="#ff0000">zdata 836K 899G 96K /data</font> +<font color="#ff0000">zdata/enc 200K 899G 200K /data/enc</font> -paul@f0:/keys % zfs get all zdata/enc | grep -E -i <font color="#808080">'(encryption|key)'</font> -zdata/enc encryption aes-<font color="#000000">256</font>-gcm - -zdata/enc keylocation file:///keys/f<font color="#000000">0</font>.lan.buetow.org:zdata.key <b><u><font color="#000000">local</font></u></b> -zdata/enc keyformat raw - -zdata/enc encryptionroot zdata/enc - -zdata/enc keystatus available - +<font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> zfs get all zdata/enc </font><font color="#F3E651">|</font><font color="#ff0000"> grep -E -i </font><font color="#bb00ff">'(encryption|key)'</font> +<font color="#ff0000">zdata/enc encryption aes-</font><font color="#bb00ff">256</font><font color="#ff0000">-gcm -</font> +<font color="#ff0000">zdata/enc keylocation file</font><font color="#F3E651">:</font><font color="#ff0000">///keys/f</font><font color="#bb00ff">0</font><font color="#F3E651">.</font><font color="#ff0000">lan</font><font color="#F3E651">.</font><font color="#ff0000">buetow</font><font color="#F3E651">.</font><font color="#ff0000">org</font><font color="#F3E651">:</font><font color="#ff0000">zdata</font><font color="#F3E651">.</font><font color="#ff0000">key </font><b><font color="#ffffff">local</font></b> +<font color="#ff0000">zdata/enc keyformat raw -</font> +<font color="#ff0000">zdata/enc encryptionroot zdata/enc -</font> +<font color="#ff0000">zdata/enc keystatus available -</font> </pre> <br /> <span>All future data sets within <span class='inlinecode'>zdata/enc</span> will inherit the same encryption key.</span><br /> @@ -254,12 +259,12 @@ zdata/enc keystatus available - by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:/keys % doas vm stop rocky -Sending ACPI shutdown to rocky +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> doas vm stop rocky</font> +<font color="#ff0000">Sending ACPI shutdown to rocky</font> -paul@f0:/keys % doas vm list -NAME DATASTORE LOADER CPU MEMORY VNC AUTO STATE -rocky default uefi <font color="#000000">4</font> 14G - Yes [<font color="#000000">1</font>] Stopped +<font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> doas vm list</font> +<font color="#ff0000">NAME DATASTORE LOADER CPU MEMORY VNC AUTO STATE</font> +<font color="#ff0000">rocky default uefi </font><font color="#bb00ff">4</font><font color="#ff0000"> 14G - Yes </font><font color="#F3E651">[</font><font color="#bb00ff">1</font><font color="#F3E651">]</font><font color="#ff0000"> Stopped</font> </pre> <br /> <span>After this, we rename the unencrypted data set to <span class='inlinecode'>_old</span>, create a new encrypted data set, and also snapshot it as <span class='inlinecode'>@hamburger</span>.</span><br /> @@ -268,14 +273,14 @@ rocky default uefi <font color="#000000">4</font> 14G - Ye by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:/keys % doas zfs rename zroot/bhyve zroot/bhyve_old -paul@f0:/keys % doas zfs <b><u><font color="#000000">set</font></u></b> mountpoint=/mnt zroot/bhyve_old -paul@f0:/keys % doas zfs snapshot zroot/bhyve_old/rocky@hamburger +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs rename zroot/bhyve zroot/bhyve_old</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs </font><b><font color="#ffffff">set</font></b><font color="#ff0000"> </font><font color="#ff0000">mountpoint</font><font color="#F3E651">=</font><font color="#ff0000">/mnt zroot/bhyve_old</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs snapshot zroot/bhyve_old/rocky@hamburger</font> -paul@f0:/keys % doas zfs create -o encryption=on -o keyformat=raw -o \ - keylocation=file:///keys/`hostname`:bhyve.key zroot/bhyve -paul@f0:/keys % doas zfs <b><u><font color="#000000">set</font></u></b> mountpoint=/zroot/bhyve zroot/bhyve -paul@f0:/keys % doas zfs <b><u><font color="#000000">set</font></u></b> mountpoint=/zroot/bhyve/rocky zroot/bhyve/rocky +<font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs create -o </font><font color="#ff0000">encryption</font><font color="#F3E651">=</font><font color="#ff0000">on -o </font><font color="#ff0000">keyformat</font><font color="#F3E651">=</font><font color="#ff0000">raw -o </font><font color="#F3E651">\</font> +<font color="#ff0000"> </font><font color="#ff0000">keylocation</font><font color="#F3E651">=</font><font color="#ff0000">file</font><font color="#F3E651">:</font><font color="#ff0000">///keys</font><font color="#F3E651">/</font><font color="#ff0000">`hostname`</font><font color="#F3E651">:</font><font color="#ff0000">bhyve</font><font color="#F3E651">.</font><font color="#ff0000">key zroot/bhyve</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs </font><b><font color="#ffffff">set</font></b><font color="#ff0000"> </font><font color="#ff0000">mountpoint</font><font color="#F3E651">=</font><font color="#ff0000">/zroot/bhyve zroot/bhyve</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs </font><b><font color="#ffffff">set</font></b><font color="#ff0000"> </font><font color="#ff0000">mountpoint</font><font color="#F3E651">=</font><font color="#ff0000">/zroot/bhyve/rocky zroot/bhyve/rocky</font> </pre> <br /> <span>Once done, we import the snapshot into the encrypted dataset and also copy some other metadata files from <span class='inlinecode'>vm-bhyve</span> back over.</span><br /> @@ -295,17 +300,17 @@ paul@f0:/keys % doas cp -Rp /mnt/.iso /zroot/bhyve/ by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:/keys % doas sysrc zfskeys_enable=YES -zfskeys_enable: -> YES -paul@f0:/keys % doas vm init -paul@f0:/keys % doas reboot -. -. -. -paul@f0:~ % doas vm list -paul@f0:~ % doas vm list -NAME DATASTORE LOADER CPU MEMORY VNC AUTO STATE -rocky default uefi <font color="#000000">4</font> 14G <font color="#000000">0.0</font>.<font color="#000000">0.0</font>:<font color="#000000">5900</font> Yes [<font color="#000000">1</font>] Running (<font color="#000000">2265</font>) +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">zfskeys_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font> +<font color="#ff0000">zfskeys_enable</font><font color="#F3E651">:</font><font color="#ff0000"> -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> doas vm init</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:</font><font color="#ff0000">/keys </font><font color="#F3E651">%</font><font color="#ff0000"> doas reboot</font> +<font color="#F3E651">.</font> +<font color="#F3E651">.</font> +<font color="#F3E651">.</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas vm list</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas vm list</font> +<font color="#ff0000">NAME DATASTORE LOADER CPU MEMORY VNC AUTO STATE</font> +<font color="#ff0000">rocky default uefi </font><font color="#bb00ff">4</font><font color="#ff0000"> 14G </font><font color="#bb00ff">0.0</font><font color="#F3E651">.</font><font color="#bb00ff">0.0</font><font color="#F3E651">:</font><font color="#bb00ff">5900</font><font color="#ff0000"> Yes </font><font color="#F3E651">[</font><font color="#bb00ff">1</font><font color="#F3E651">]</font><font color="#ff0000"> Running </font><font color="#F3E651">(</font><font color="#bb00ff">2265</font><font color="#F3E651">)</font> </pre> <br /> <span>As you can see, the VM is running. This means the encrypted <span class='inlinecode'>zroot/bhyve</span> was mounted successfully after the reboot! Now we can destroy the old, unencrypted, and now unused bhyve dataset:</span><br /> @@ -314,7 +319,7 @@ rocky default uefi <font color="#000000">4</font> 14G <font c by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:~ % doas zfs destroy -R zroot/bhyve_old +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs destroy -R zroot/bhyve_old</font> </pre> <br /> <span>To verify once again that <span class='inlinecode'>zroot/bhyve</span> and <span class='inlinecode'>zroot/bhyve/rocky</span> are now both encrypted, we run:</span><br /> @@ -323,19 +328,19 @@ http://www.gnu.org/software/src-highlite --> by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:~ % zfs get all zroot/bhyve | grep -E <font color="#808080">'(encryption|key)'</font> -zroot/bhyve encryption aes-<font color="#000000">256</font>-gcm - -zroot/bhyve keylocation file:///keys/f<font color="#000000">0</font>.lan.buetow.org:bhyve.key <b><u><font color="#000000">local</font></u></b> -zroot/bhyve keyformat raw - -zroot/bhyve encryptionroot zroot/bhyve - -zroot/bhyve keystatus available - +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> zfs get all zroot/bhyve </font><font color="#F3E651">|</font><font color="#ff0000"> grep -E </font><font color="#bb00ff">'(encryption|key)'</font> +<font color="#ff0000">zroot/bhyve encryption aes-</font><font color="#bb00ff">256</font><font color="#ff0000">-gcm -</font> +<font color="#ff0000">zroot/bhyve keylocation file</font><font color="#F3E651">:</font><font color="#ff0000">///keys/f</font><font color="#bb00ff">0</font><font color="#F3E651">.</font><font color="#ff0000">lan</font><font color="#F3E651">.</font><font color="#ff0000">buetow</font><font color="#F3E651">.</font><font color="#ff0000">org</font><font color="#F3E651">:</font><font color="#ff0000">bhyve</font><font color="#F3E651">.</font><font color="#ff0000">key </font><b><font color="#ffffff">local</font></b> +<font color="#ff0000">zroot/bhyve keyformat raw -</font> +<font color="#ff0000">zroot/bhyve encryptionroot zroot/bhyve -</font> +<font color="#ff0000">zroot/bhyve keystatus available -</font> -paul@f0:~ % zfs get all zroot/bhyve/rocky | grep -E <font color="#808080">'(encryption|key)'</font> -zroot/bhyve/rocky encryption aes-<font color="#000000">256</font>-gcm - -zroot/bhyve/rocky keylocation none default -zroot/bhyve/rocky keyformat raw - -zroot/bhyve/rocky encryptionroot zroot/bhyve - -zroot/bhyve/rocky keystatus available - +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> zfs get all zroot/bhyve/rocky </font><font color="#F3E651">|</font><font color="#ff0000"> grep -E </font><font color="#bb00ff">'(encryption|key)'</font> +<font color="#ff0000">zroot/bhyve/rocky encryption aes-</font><font color="#bb00ff">256</font><font color="#ff0000">-gcm -</font> +<font color="#ff0000">zroot/bhyve/rocky keylocation none default</font> +<font color="#ff0000">zroot/bhyve/rocky keyformat raw -</font> +<font color="#ff0000">zroot/bhyve/rocky encryptionroot zroot/bhyve -</font> +<font color="#ff0000">zroot/bhyve/rocky keystatus available -</font> </pre> <br /> <h2 style='display: inline' id='zfs-replication-with-zrepl'>ZFS Replication with <span class='inlinecode'>zrepl</span></h2><br /> @@ -370,7 +375,7 @@ zroot/bhyve/rocky keystatus available - by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:~ % doas pkg install -y zrepl +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas pkg install -y zrepl</font> </pre> <br /> <span>Then, we verify the pools and datasets on both hosts:</span><br /> @@ -379,25 +384,25 @@ http://www.gnu.org/software/src-highlite --> by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># On f0</font></i> -paul@f0:~ % doas zpool list -NAME SIZE ALLOC FREE CKPOINT EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT -zdata 928G <font color="#000000">1</font>.03M 928G - - <font color="#000000">0</font>% <font color="#000000">0</font>% <font color="#000000">1</font>.00x ONLINE - -zroot 472G <font color="#000000">26</font>.7G 445G - - <font color="#000000">0</font>% <font color="#000000">5</font>% <font color="#000000">1</font>.00x ONLINE - +<pre><i><font color="#ababab"># On f0</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zpool list</font> +<font color="#ff0000">NAME SIZE ALLOC FREE CKPOINT EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT</font> +<font color="#ff0000">zdata 928G </font><font color="#bb00ff">1</font><font color="#F3E651">.</font><font color="#ff0000">03M 928G - - </font><font color="#bb00ff">0</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">0</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">1</font><font color="#F3E651">.</font><font color="#ff0000">00x ONLINE -</font> +<font color="#ff0000">zroot 472G </font><font color="#bb00ff">26</font><font color="#F3E651">.</font><font color="#ff0000">7G 445G - - </font><font color="#bb00ff">0</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">5</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">1</font><font color="#F3E651">.</font><font color="#ff0000">00x ONLINE -</font> -paul@f0:~ % doas zfs list -r zdata/enc -NAME USED AVAIL REFER MOUNTPOINT -zdata/enc 200K 899G 200K /data/enc +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs list -r zdata/enc</font> +<font color="#ff0000">NAME USED AVAIL REFER MOUNTPOINT</font> +<font color="#ff0000">zdata/enc 200K 899G 200K /data/enc</font> -<i><font color="silver"># On f1</font></i> -paul@f1:~ % doas zpool list -NAME SIZE ALLOC FREE CKPOINT EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT -zdata 928G 956K 928G - - <font color="#000000">0</font>% <font color="#000000">0</font>% <font color="#000000">1</font>.00x ONLINE - -zroot 472G <font color="#000000">11</font>.7G 460G - - <font color="#000000">0</font>% <font color="#000000">2</font>% <font color="#000000">1</font>.00x ONLINE - +<i><font color="#ababab"># On f1</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zpool list</font> +<font color="#ff0000">NAME SIZE ALLOC FREE CKPOINT EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT</font> +<font color="#ff0000">zdata 928G 956K 928G - - </font><font color="#bb00ff">0</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">0</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">1</font><font color="#F3E651">.</font><font color="#ff0000">00x ONLINE -</font> +<font color="#ff0000">zroot 472G </font><font color="#bb00ff">11</font><font color="#F3E651">.</font><font color="#ff0000">7G 460G - - </font><font color="#bb00ff">0</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">2</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">1</font><font color="#F3E651">.</font><font color="#ff0000">00x ONLINE -</font> -paul@f1:~ % doas zfs list -r zdata/enc -NAME USED AVAIL REFER MOUNTPOINT -zdata/enc 200K 899G 200K /data/enc +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs list -r zdata/enc</font> +<font color="#ff0000">NAME USED AVAIL REFER MOUNTPOINT</font> +<font color="#ff0000">zdata/enc 200K 899G 200K /data/enc</font> </pre> <br /> <span>Since we have a WireGuard tunnel between <span class='inlinecode'>f0</span> and f1, we'll use TCP transport over the secure tunnel instead of SSH. First, check the WireGuard IP addresses:</span><br /> @@ -406,12 +411,12 @@ zdata/enc 200K 899G 200K /data/enc by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># Check WireGuard interface IPs</font></i> -paul@f0:~ % ifconfig wg0 | grep inet - inet <font color="#000000">192.168</font>.<font color="#000000">2.130</font> netmask <font color="#000000">0xffffff00</font> +<pre><i><font color="#ababab"># Check WireGuard interface IPs</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> ifconfig wg0 </font><font color="#F3E651">|</font><font color="#ff0000"> grep inet</font> +<font color="#ff0000"> inet </font><font color="#bb00ff">192.168</font><font color="#F3E651">.</font><font color="#bb00ff">2.130</font><font color="#ff0000"> netmask </font><font color="#bb00ff">0xffffff00</font> -paul@f1:~ % ifconfig wg0 | grep inet - inet <font color="#000000">192.168</font>.<font color="#000000">2.131</font> netmask <font color="#000000">0xffffff00</font> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> ifconfig wg0 </font><font color="#F3E651">|</font><font color="#ff0000"> grep inet</font> +<font color="#ff0000"> inet </font><font color="#bb00ff">192.168</font><font color="#F3E651">.</font><font color="#bb00ff">2.131</font><font color="#ff0000"> netmask </font><font color="#bb00ff">0xffffff00</font> </pre> <br /> <span>Let's create a dedicated dataset for NFS data that will be replicated:</span><br /> @@ -420,8 +425,8 @@ paul@f1:~ % ifconfig wg0 | grep inet by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># Create the nfsdata dataset that will hold all data exposed via NFS</font></i> -paul@f0:~ % doas zfs create zdata/enc/nfsdata +<pre><i><font color="#ababab"># Create the nfsdata dataset that will hold all data exposed via NFS</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs create zdata/enc/nfsdata</font> </pre> <br /> <span>Afterwards, we create the <span class='inlinecode'>zrepl</span> configuration on <span class='inlinecode'>f0</span>:</span><br /> @@ -430,68 +435,68 @@ paul@f0:~ % doas zfs create zdata/enc/nfsdata by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:~ % doas tee /usr/local/etc/zrepl/zrepl.yml <<<font color="#808080">'EOF'</font> -global: - logging: - - <b><u><font color="#000000">type</font></u></b>: stdout - level: info - format: human - -<b><u><font color="#000000">jobs</font></u></b>: - - name: f0_to_f1_nfsdata - <b><u><font color="#000000">type</font></u></b>: push - connect: - <b><u><font color="#000000">type</font></u></b>: tcp - address: <font color="#808080">"192.168.2.131:8888"</font> - filesystems: - <font color="#808080">"zdata/enc/nfsdata"</font>: <b><u><font color="#000000">true</font></u></b> - send: - encrypted: <b><u><font color="#000000">true</font></u></b> - snapshotting: - <b><u><font color="#000000">type</font></u></b>: periodic - prefix: zrepl_ - interval: 1m - pruning: - keep_sender: - - <b><u><font color="#000000">type</font></u></b>: last_n - count: <font color="#000000">10</font> - - <b><u><font color="#000000">type</font></u></b>: grid - grid: 4x7d | 6x30d - regex: <font color="#808080">"^zrepl_.*"</font> - keep_receiver: - - <b><u><font color="#000000">type</font></u></b>: last_n - count: <font color="#000000">10</font> - - <b><u><font color="#000000">type</font></u></b>: grid - grid: 4x7d | 6x30d - regex: <font color="#808080">"^zrepl_.*"</font> - - - name: f0_to_f1_freebsd - <b><u><font color="#000000">type</font></u></b>: push - connect: - <b><u><font color="#000000">type</font></u></b>: tcp - address: <font color="#808080">"192.168.2.131:8888"</font> - filesystems: - <font color="#808080">"zroot/bhyve/freebsd"</font>: <b><u><font color="#000000">true</font></u></b> - send: - encrypted: <b><u><font color="#000000">true</font></u></b> - snapshotting: - <b><u><font color="#000000">type</font></u></b>: periodic - prefix: zrepl_ - interval: 10m - pruning: - keep_sender: - - <b><u><font color="#000000">type</font></u></b>: last_n - count: <font color="#000000">10</font> - - <b><u><font color="#000000">type</font></u></b>: grid - grid: 4x7d - regex: <font color="#808080">"^zrepl_.*"</font> - keep_receiver: - - <b><u><font color="#000000">type</font></u></b>: last_n - count: <font color="#000000">10</font> - - <b><u><font color="#000000">type</font></u></b>: grid - grid: 4x7d - regex: <font color="#808080">"^zrepl_.*"</font> -EOF +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas tee /usr/local/etc/zrepl/zrepl</font><font color="#F3E651">.</font><font color="#ff0000">yml </font><font color="#F3E651"><<</font><font color="#bb00ff">'EOF'</font> +<font color="#ff0000">global</font><font color="#F3E651">:</font> +<font color="#ff0000"> logging</font><font color="#F3E651">:</font> +<font color="#ff0000"> - </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> stdout</font> +<font color="#ff0000"> level</font><font color="#F3E651">:</font><font color="#ff0000"> info</font> +<font color="#ff0000"> format</font><font color="#F3E651">:</font><font color="#ff0000"> human</font> + +<b><font color="#ffffff">jobs</font></b><font color="#F3E651">:</font> +<font color="#ff0000"> - name</font><font color="#F3E651">:</font><font color="#ff0000"> f0_to_f1_nfsdata</font> +<font color="#ff0000"> </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> push</font> +<font color="#ff0000"> connect</font><font color="#F3E651">:</font> +<font color="#ff0000"> </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> tcp</font> +<font color="#ff0000"> address</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">"192.168.2.131:8888"</font> +<font color="#ff0000"> filesystems</font><font color="#F3E651">:</font> +<font color="#ff0000"> </font><font color="#bb00ff">"zdata/enc/nfsdata"</font><font color="#F3E651">:</font><font color="#ff0000"> </font><b><font color="#ffffff">true</font></b> +<font color="#ff0000"> send</font><font color="#F3E651">:</font> +<font color="#ff0000"> encrypted</font><font color="#F3E651">:</font><font color="#ff0000"> </font><b><font color="#ffffff">true</font></b> +<font color="#ff0000"> snapshotting</font><font color="#F3E651">:</font> +<font color="#ff0000"> </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> periodic</font> +<font color="#ff0000"> prefix</font><font color="#F3E651">:</font><font color="#ff0000"> zrepl_</font> +<font color="#ff0000"> interval</font><font color="#F3E651">:</font><font color="#ff0000"> 1m</font> +<font color="#ff0000"> pruning</font><font color="#F3E651">:</font> +<font color="#ff0000"> keep_sender</font><font color="#F3E651">:</font> +<font color="#ff0000"> - </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> last_n</font> +<font color="#ff0000"> count</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">10</font> +<font color="#ff0000"> - </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> grid</font> +<font color="#ff0000"> grid</font><font color="#F3E651">:</font><font color="#ff0000"> 4x7d </font><font color="#F3E651">|</font><font color="#ff0000"> 6x30d</font> +<font color="#ff0000"> regex</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">"^zrepl_.*"</font> +<font color="#ff0000"> keep_receiver</font><font color="#F3E651">:</font> +<font color="#ff0000"> - </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> last_n</font> +<font color="#ff0000"> count</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">10</font> +<font color="#ff0000"> - </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> grid</font> +<font color="#ff0000"> grid</font><font color="#F3E651">:</font><font color="#ff0000"> 4x7d </font><font color="#F3E651">|</font><font color="#ff0000"> 6x30d</font> +<font color="#ff0000"> regex</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">"^zrepl_.*"</font> + +<font color="#ff0000"> - name</font><font color="#F3E651">:</font><font color="#ff0000"> f0_to_f1_freebsd</font> +<font color="#ff0000"> </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> push</font> +<font color="#ff0000"> connect</font><font color="#F3E651">:</font> +<font color="#ff0000"> </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> tcp</font> +<font color="#ff0000"> address</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">"192.168.2.131:8888"</font> +<font color="#ff0000"> filesystems</font><font color="#F3E651">:</font> +<font color="#ff0000"> </font><font color="#bb00ff">"zroot/bhyve/freebsd"</font><font color="#F3E651">:</font><font color="#ff0000"> </font><b><font color="#ffffff">true</font></b> +<font color="#ff0000"> send</font><font color="#F3E651">:</font> +<font color="#ff0000"> encrypted</font><font color="#F3E651">:</font><font color="#ff0000"> </font><b><font color="#ffffff">true</font></b> +<font color="#ff0000"> snapshotting</font><font color="#F3E651">:</font> +<font color="#ff0000"> </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> periodic</font> +<font color="#ff0000"> prefix</font><font color="#F3E651">:</font><font color="#ff0000"> zrepl_</font> +<font color="#ff0000"> interval</font><font color="#F3E651">:</font><font color="#ff0000"> 10m</font> +<font color="#ff0000"> pruning</font><font color="#F3E651">:</font> +<font color="#ff0000"> keep_sender</font><font color="#F3E651">:</font> +<font color="#ff0000"> - </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> last_n</font> +<font color="#ff0000"> count</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">10</font> +<font color="#ff0000"> - </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> grid</font> +<font color="#ff0000"> grid</font><font color="#F3E651">:</font><font color="#ff0000"> 4x7d</font> +<font color="#ff0000"> regex</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">"^zrepl_.*"</font> +<font color="#ff0000"> keep_receiver</font><font color="#F3E651">:</font> +<font color="#ff0000"> - </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> last_n</font> +<font color="#ff0000"> count</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">10</font> +<font color="#ff0000"> - </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> grid</font> +<font color="#ff0000"> grid</font><font color="#F3E651">:</font><font color="#ff0000"> 4x7d</font> +<font color="#ff0000"> regex</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">"^zrepl_.*"</font> +<font color="#ff0000">EOF</font> </pre> <br /> <span> We're using two separate replication jobs with different intervals:</span><br /> @@ -516,29 +521,29 @@ EOF by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># First, create a dedicated sink dataset</font></i> -paul@f1:~ % doas zfs create zdata/sink +<pre><i><font color="#ababab"># First, create a dedicated sink dataset</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs create zdata/sink</font> -paul@f1:~ % doas tee /usr/local/etc/zrepl/zrepl.yml <<<font color="#808080">'EOF'</font> -global: - logging: - - <b><u><font color="#000000">type</font></u></b>: stdout - level: info - format: human +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas tee /usr/local/etc/zrepl/zrepl</font><font color="#F3E651">.</font><font color="#ff0000">yml </font><font color="#F3E651"><<</font><font color="#bb00ff">'EOF'</font> +<font color="#ff0000">global</font><font color="#F3E651">:</font> +<font color="#ff0000"> logging</font><font color="#F3E651">:</font> +<font color="#ff0000"> - </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> stdout</font> +<font color="#ff0000"> level</font><font color="#F3E651">:</font><font color="#ff0000"> info</font> +<font color="#ff0000"> format</font><font color="#F3E651">:</font><font color="#ff0000"> human</font> -<b><u><font color="#000000">jobs</font></u></b>: - - name: sink - <b><u><font color="#000000">type</font></u></b>: sink - serve: - <b><u><font color="#000000">type</font></u></b>: tcp - listen: <font color="#808080">"192.168.2.131:8888"</font> - clients: - <font color="#808080">"192.168.2.130"</font>: <font color="#808080">"f0"</font> - recv: - placeholder: - encryption: inherit - root_fs: <font color="#808080">"zdata/sink"</font> -EOF +<b><font color="#ffffff">jobs</font></b><font color="#F3E651">:</font> +<font color="#ff0000"> - name</font><font color="#F3E651">:</font><font color="#ff0000"> sink</font> +<font color="#ff0000"> </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> sink</font> +<font color="#ff0000"> serve</font><font color="#F3E651">:</font> +<font color="#ff0000"> </font><b><font color="#ffffff">type</font></b><font color="#F3E651">:</font><font color="#ff0000"> tcp</font> +<font color="#ff0000"> listen</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">"192.168.2.131:8888"</font> +<font color="#ff0000"> clients</font><font color="#F3E651">:</font> +<font color="#ff0000"> </font><font color="#bb00ff">"192.168.2.130"</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">"f0"</font> +<font color="#ff0000"> recv</font><font color="#F3E651">:</font> +<font color="#ff0000"> placeholder</font><font color="#F3E651">:</font> +<font color="#ff0000"> encryption</font><font color="#F3E651">:</font><font color="#ff0000"> inherit</font> +<font color="#ff0000"> root_fs</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">"zdata/sink"</font> +<font color="#ff0000">EOF</font> </pre> <br /> <h3 style='display: inline' id='enabling-and-starting-zrepl-services'>Enabling and starting <span class='inlinecode'>zrepl</span> services</h3><br /> @@ -549,17 +554,17 @@ EOF by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># On f0</font></i> -paul@f0:~ % doas sysrc zrepl_enable=YES -zrepl_enable: -> YES -paul@f0:~ % doas service `zrepl` start -Starting zrepl. +<pre><i><font color="#ababab"># On f0</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">zrepl_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font> +<font color="#ff0000">zrepl_enable</font><font color="#F3E651">:</font><font color="#ff0000"> -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service `zrepl` start</font> +<font color="#ff0000">Starting zrepl</font><font color="#F3E651">.</font> -<i><font color="silver"># On f1</font></i> -paul@f1:~ % doas sysrc zrepl_enable=YES -zrepl_enable: -> YES -paul@f1:~ % doas service `zrepl` start -Starting zrepl. +<i><font color="#ababab"># On f1</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">zrepl_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font> +<font color="#ff0000">zrepl_enable</font><font color="#F3E651">:</font><font color="#ff0000"> -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service `zrepl` start</font> +<font color="#ff0000">Starting zrepl</font><font color="#F3E651">.</font> </pre> <br /> <span>To check the replication status, we run:</span><br /> @@ -568,35 +573,35 @@ Starting zrepl. by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># On f0, check `zrepl` status (use raw mode for non-tty)</font></i> -paul@f0:~ % doas pkg install jq -paul@f0:~ % doas zrepl status --mode raw | grep -A<font color="#000000">2</font> <font color="#808080">"Replication"</font> | jq . -<font color="#808080">"Replication"</font>:{<font color="#808080">"StartAt"</font>:<font color="#808080">"2025-07-01T22:31:48.712143123+03:00"</font>... +<pre><i><font color="#ababab"># On f0, check `zrepl` status (use raw mode for non-tty)</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas pkg install jq</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zrepl status --mode raw </font><font color="#F3E651">|</font><font color="#ff0000"> grep -A</font><font color="#bb00ff">2</font><font color="#ff0000"> </font><font color="#bb00ff">"Replication"</font><font color="#ff0000"> </font><font color="#F3E651">|</font><font color="#ff0000"> jq </font><font color="#F3E651">.</font> +<font color="#bb00ff">"Replication"</font><font color="#F3E651">:</font><font color="#ff0000">{</font><font color="#bb00ff">"StartAt"</font><font color="#F3E651">:</font><font color="#bb00ff">"2025-07-01T22:31:48.712143123+03:00"</font><font color="#F3E651">...</font> -<i><font color="silver"># Check if services are running</font></i> -paul@f0:~ % doas service zrepl status -zrepl is running as pid <font color="#000000">2649</font>. +<i><font color="#ababab"># Check if services are running</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service zrepl status</font> +<font color="#ff0000">zrepl is running as pid </font><font color="#bb00ff">2649</font><font color="#F3E651">.</font> -paul@f1:~ % doas service zrepl status -zrepl is running as pid <font color="#000000">2574</font>. +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service zrepl status</font> +<font color="#ff0000">zrepl is running as pid </font><font color="#bb00ff">2574</font><font color="#F3E651">.</font> -<i><font color="silver"># Check for `zrepl` snapshots on source</font></i> -paul@f0:~ % doas zfs list -t snapshot -r zdata/enc | grep zrepl -zdata/enc@zrepl_20250701_193148_000 0B - 176K - +<i><font color="#ababab"># Check for `zrepl` snapshots on source</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs list -t snapshot -r zdata/enc </font><font color="#F3E651">|</font><font color="#ff0000"> grep zrepl</font> +<font color="#ff0000">zdata/enc@zrepl_20250701_193148_000 0B - 176K -</font> -<i><font color="silver"># On f1, verify the replicated datasets </font></i> -paul@f1:~ % doas zfs list -r zdata | grep f0 -zdata/f<font color="#000000">0</font> 576K 899G 200K none -zdata/f<font color="#000000">0</font>/zdata 376K 899G 200K none -zdata/f<font color="#000000">0</font>/zdata/enc 176K 899G 176K none +<i><font color="#ababab"># On f1, verify the replicated datasets </font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs list -r zdata </font><font color="#F3E651">|</font><font color="#ff0000"> grep f0</font> +<font color="#ff0000">zdata/f</font><font color="#bb00ff">0</font><font color="#ff0000"> 576K 899G 200K none</font> +<font color="#ff0000">zdata/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata 376K 899G 200K none</font> +<font color="#ff0000">zdata/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc 176K 899G 176K none</font> -<i><font color="silver"># Check replicated snapshots on f1</font></i> -paul@f1:~ % doas zfs list -t snapshot -r zdata | grep zrepl -zdata/f<font color="#000000">0</font>/zdata/enc@zrepl_20250701_193148_000 0B - 176K - -zdata/f<font color="#000000">0</font>/zdata/enc@zrepl_20250701_194148_000 0B - 176K - -. -. -. +<i><font color="#ababab"># Check replicated snapshots on f1</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs list -t snapshot -r zdata </font><font color="#F3E651">|</font><font color="#ff0000"> grep zrepl</font> +<font color="#ff0000">zdata/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc@zrepl_20250701_193148_000 0B - 176K -</font> +<font color="#ff0000">zdata/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc@zrepl_20250701_194148_000 0B - 176K -</font> +<font color="#F3E651">.</font> +<font color="#F3E651">.</font> +<font color="#F3E651">.</font> </pre> <br /> <h3 style='display: inline' id='monitoring-replication'>Monitoring replication</h3><br /> @@ -607,7 +612,7 @@ zdata/f<font color="#000000">0</font>/zdata/enc@zrepl_20250701_194148_000 0B by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:~ % doas zrepl status +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zrepl status</font> </pre> <br /> <a href='./f3s-kubernetes-with-freebsd-part-6/zrepl.png'><img alt='zrepl status' title='zrepl status' src='./f3s-kubernetes-with-freebsd-part-6/zrepl.png' /></a><br /> @@ -630,29 +635,29 @@ http://www.gnu.org/software/src-highlite --> by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:~ % uptime -<font color="#000000">11</font>:17PM up <font color="#000000">1</font> min, <font color="#000000">0</font> users, load averages: <font color="#000000">0.16</font>, <font color="#000000">0.06</font>, <font color="#000000">0.02</font> +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> uptime</font> +<font color="#bb00ff">11</font><font color="#F3E651">:</font><font color="#ff0000">17PM up </font><font color="#bb00ff">1</font><font color="#ff0000"> min</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">0</font><font color="#ff0000"> users</font><font color="#F3E651">,</font><font color="#ff0000"> load averages</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">0.16</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">0.06</font><font color="#F3E651">,</font><font color="#ff0000"> </font><font color="#bb00ff">0.02</font> -paul@f0:~ % doas service `zrepl` status -zrepl is running as pid <font color="#000000">2366</font>. +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service `zrepl` status</font> +<font color="#ff0000">zrepl is running as pid </font><font color="#bb00ff">2366</font><font color="#F3E651">.</font> -paul@f1:~ % doas service `zrepl` status -zrepl is running as pid <font color="#000000">2309</font>. +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service `zrepl` status</font> +<font color="#ff0000">zrepl is running as pid </font><font color="#bb00ff">2309</font><font color="#F3E651">.</font> -<i><font color="silver"># Check that new snapshots are being created and replicated</font></i> -paul@f0:~ % doas zfs list -t snapshot | grep `zrepl` | tail -<font color="#000000">2</font> -zdata/enc/nfsdata@zrepl_20250701_202530_000 0B - 200K - -zroot/bhyve/freebsd@zrepl_20250701_202530_000 0B - <font color="#000000">2</font>.97G - -. -. -. +<i><font color="#ababab"># Check that new snapshots are being created and replicated</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs list -t snapshot </font><font color="#F3E651">|</font><font color="#ff0000"> grep `zrepl` </font><font color="#F3E651">|</font><font color="#ff0000"> tail -</font><font color="#bb00ff">2</font> +<font color="#ff0000">zdata/enc/nfsdata@zrepl_20250701_202530_000 0B - 200K -</font> +<font color="#ff0000">zroot/bhyve/freebsd@zrepl_20250701_202530_000 0B - </font><font color="#bb00ff">2</font><font color="#F3E651">.</font><font color="#ff0000">97G -</font> +<font color="#F3E651">.</font> +<font color="#F3E651">.</font> +<font color="#F3E651">.</font> -paul@f1:~ % doas zfs list -t snapshot -r zdata/sink | grep <font color="#000000">202530</font> -zdata/sink/f<font color="#000000">0</font>/zdata/enc/nfsdata@zrepl_20250701_202530_000 0B - 176K - -zdata/sink/f<font color="#000000">0</font>/zroot/bhyve/freebsd@zrepl_20250701_202530_000 0B - <font color="#000000">2</font>.97G - -. -. -. +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs list -t snapshot -r zdata/sink </font><font color="#F3E651">|</font><font color="#ff0000"> grep </font><font color="#bb00ff">202530</font> +<font color="#ff0000">zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata@zrepl_20250701_202530_000 0B - 176K -</font> +<font color="#ff0000">zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zroot/bhyve/freebsd@zrepl_20250701_202530_000 0B - </font><font color="#bb00ff">2</font><font color="#F3E651">.</font><font color="#ff0000">97G -</font> +<font color="#F3E651">.</font> +<font color="#F3E651">.</font> +<font color="#F3E651">.</font> </pre> <br /> <span>The timestamps confirm that replication resumed automatically after the reboot, ensuring continuous data protection. We can also write a test file to the NFS data directory on <span class='inlinecode'>f0</span> and verify whether it appears on <span class='inlinecode'>f1</span> after a minute.</span><br /> @@ -677,14 +682,14 @@ zdata/sink/f<font color="#000000">0</font>/zroot/bhyve/freebsd@zrepl_20250701_20 by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># On f0 - set mountpoint for the primary nfsdata</font></i> -paul@f0:~ % doas zfs <b><u><font color="#000000">set</font></u></b> mountpoint=/data/nfs zdata/enc/nfsdata -paul@f0:~ % doas mkdir -p /data/nfs +<pre><i><font color="#ababab"># On f0 - set mountpoint for the primary nfsdata</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs </font><b><font color="#ffffff">set</font></b><font color="#ff0000"> </font><font color="#ff0000">mountpoint</font><font color="#F3E651">=</font><font color="#ff0000">/data/nfs zdata/enc/nfsdata</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas mkdir -p /data/nfs</font> -<i><font color="silver"># Verify it's mounted</font></i> -paul@f0:~ % df -h /data/nfs -Filesystem Size Used Avail Capacity Mounted on -zdata/enc/nfsdata 899G 204K 899G <font color="#000000">0</font>% /data/nfs +<i><font color="#ababab"># Verify it's mounted</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> df -h /data/nfs</font> +<font color="#ff0000">Filesystem Size Used Avail Capacity Mounted on</font> +<font color="#ff0000">zdata/enc/nfsdata 899G 204K 899G </font><font color="#bb00ff">0</font><font color="#F3E651">%</font><font color="#ff0000"> /data/nfs</font> </pre> <br /> <span>On <span class='inlinecode'>f1</span>, we need to handle the encryption key and mount the standby copy:</span><br /> @@ -693,27 +698,27 @@ zdata/enc/nfsdata 899G 204K 899G <font color="#000000">0</font>% by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># On f1 - first check encryption status</font></i> -paul@f1:~ % doas zfs get keystatus zdata/sink/f<font color="#000000">0</font>/zdata/enc/nfsdata -NAME PROPERTY VALUE SOURCE -zdata/sink/f<font color="#000000">0</font>/zdata/enc/nfsdata keystatus unavailable - +<pre><i><font color="#ababab"># On f1 - first check encryption status</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs get keystatus zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font> +<font color="#ff0000">NAME PROPERTY VALUE SOURCE</font> +<font color="#ff0000">zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata keystatus unavailable -</font> -<i><font color="silver"># Load the encryption key (using f0's key stored on the USB)</font></i> -paul@f1:~ % doas zfs load-key -L file:///keys/f<font color="#000000">0</font>.lan.buetow.org:zdata.key \ - zdata/sink/f<font color="#000000">0</font>/zdata/enc/nfsdata +<i><font color="#ababab"># Load the encryption key (using f0's key stored on the USB)</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs load-key -L file</font><font color="#F3E651">:</font><font color="#ff0000">///keys/f</font><font color="#bb00ff">0</font><font color="#F3E651">.</font><font color="#ff0000">lan</font><font color="#F3E651">.</font><font color="#ff0000">buetow</font><font color="#F3E651">.</font><font color="#ff0000">org</font><font color="#F3E651">:</font><font color="#ff0000">zdata</font><font color="#F3E651">.</font><font color="#ff0000">key </font><font color="#F3E651">\</font> +<font color="#ff0000"> zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font> -<i><font color="silver"># Set mountpoint and mount (same path as f0 for easier failover)</font></i> -paul@f1:~ % doas mkdir -p /data/nfs -paul@f1:~ % doas zfs <b><u><font color="#000000">set</font></u></b> mountpoint=/data/nfs zdata/sink/f<font color="#000000">0</font>/zdata/enc/nfsdata -paul@f1:~ % doas zfs mount zdata/sink/f<font color="#000000">0</font>/zdata/enc/nfsdata +<i><font color="#ababab"># Set mountpoint and mount (same path as f0 for easier failover)</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas mkdir -p /data/nfs</font> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs </font><b><font color="#ffffff">set</font></b><font color="#ff0000"> </font><font color="#ff0000">mountpoint</font><font color="#F3E651">=</font><font color="#ff0000">/data/nfs zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs mount zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font> -<i><font color="silver"># Make it read-only to prevent accidental writes that would break replication</font></i> -paul@f1:~ % doas zfs <b><u><font color="#000000">set</font></u></b> <b><u><font color="#000000">readonly</font></u></b>=on zdata/sink/f<font color="#000000">0</font>/zdata/enc/nfsdata +<i><font color="#ababab"># Make it read-only to prevent accidental writes that would break replication</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs </font><b><font color="#ffffff">set</font></b><font color="#ff0000"> </font><b><font color="#ffffff">readonly</font></b><font color="#F3E651">=</font><font color="#ff0000">on zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font> -<i><font color="silver"># Verify</font></i> -paul@f1:~ % df -h /data/nfs -Filesystem Size Used Avail Capacity Mounted on -zdata/sink/f<font color="#000000">0</font>/zdata/enc/nfsdata 896G 204K 896G <font color="#000000">0</font>% /data/nfs +<i><font color="#ababab"># Verify</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> df -h /data/nfs</font> +<font color="#ff0000">Filesystem Size Used Avail Capacity Mounted on</font> +<font color="#ff0000">zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata 896G 204K 896G </font><font color="#bb00ff">0</font><font color="#F3E651">%</font><font color="#ff0000"> /data/nfs</font> </pre> <br /> <span>Note: The dataset is mounted at the same path (<span class='inlinecode'>/data/nfs</span>) on both hosts to simplify failover procedures. The dataset on <span class='inlinecode'>f1</span> is set to <span class='inlinecode'>readonly=on</span> to prevent accidental modifications, which, as mentioned earlier, would break replication. If we did, replication from <span class='inlinecode'>f0</span> to <span class='inlinecode'>f1</span> would fail like this:</span><br /> @@ -726,11 +731,11 @@ zdata/sink/f<font color="#000000">0</font>/zdata/enc/nfsdata 896G 204K by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># Option 1: Rollback to the last common snapshot (loses local changes)</font></i> -paul@f1:~ % doas zfs rollback zdata/sink/f<font color="#000000">0</font>/zdata/enc/nfsdata@zrepl_20250701_204054_000 +<pre><i><font color="#ababab"># Option 1: Rollback to the last common snapshot (loses local changes)</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs rollback zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata@zrepl_20250701_204054_000</font> -<i><font color="silver"># Option 2: Make it read-only to prevent accidents again</font></i> -paul@f1:~ % doas zfs <b><u><font color="#000000">set</font></u></b> <b><u><font color="#000000">readonly</font></u></b>=on zdata/sink/f<font color="#000000">0</font>/zdata/enc/nfsdata +<i><font color="#ababab"># Option 2: Make it read-only to prevent accidents again</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs </font><b><font color="#ffffff">set</font></b><font color="#ff0000"> </font><b><font color="#ffffff">readonly</font></b><font color="#F3E651">=</font><font color="#ff0000">on zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font> </pre> <br /> <span>And replication should work again!</span><br /> @@ -743,8 +748,8 @@ paul@f1:~ % doas zfs <b><u><font color="#000000">set</font></u></b> <b><u><font by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:~ % doas zfs list -o name,mountpoint,mounted | grep nfsdata -zdata/enc/nfsdata /data/nfs yes +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs list -o name</font><font color="#F3E651">,</font><font color="#ff0000">mountpoint</font><font color="#F3E651">,</font><font color="#ff0000">mounted </font><font color="#F3E651">|</font><font color="#ff0000"> grep nfsdata</font> +<font color="#ff0000">zdata/enc/nfsdata /data/nfs yes</font> </pre> <br /> <span>If it shows <span class='inlinecode'>no</span>, the dataset isn't mounted! This means files are being written to the root filesystem, not ZFS. Next, we should check whether the encryption key is loaded:</span><br /> @@ -753,12 +758,12 @@ zdata/enc/nfsdata /data/nfs yes by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:~ % doas zfs get keystatus zdata/enc/nfsdata -NAME PROPERTY VALUE SOURCE -zdata/enc/nfsdata keystatus available - -<i><font color="silver"># If "unavailable", load the key:</font></i> -paul@f0:~ % doas zfs load-key -L file:///keys/f<font color="#000000">0</font>.lan.buetow.org:zdata.key zdata/enc/nfsdata -paul@f0:~ % doas zfs mount zdata/enc/nfsdata +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs get keystatus zdata/enc/nfsdata</font> +<font color="#ff0000">NAME PROPERTY VALUE SOURCE</font> +<font color="#ff0000">zdata/enc/nfsdata keystatus available -</font> +<i><font color="#ababab"># If "unavailable", load the key:</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs load-key -L file</font><font color="#F3E651">:</font><font color="#ff0000">///keys/f</font><font color="#bb00ff">0</font><font color="#F3E651">.</font><font color="#ff0000">lan</font><font color="#F3E651">.</font><font color="#ff0000">buetow</font><font color="#F3E651">.</font><font color="#ff0000">org</font><font color="#F3E651">:</font><font color="#ff0000">zdata</font><font color="#F3E651">.</font><font color="#ff0000">key zdata/enc/nfsdata</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs mount zdata/enc/nfsdata</font> </pre> <br /> <span>You can also verify that files are in the snapshot (not just the directory):</span><br /> @@ -767,7 +772,7 @@ paul@f0:~ % doas zfs mount zdata/enc/nfsdata by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:~ % ls -la /data/nfs/.zfs/snapshot/zrepl_*/ +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> ls -la /data/nfs</font><font color="#F3E651">/.</font><font color="#ff0000">zfs/snapshot/zrepl_</font><font color="#F3E651">*/</font> </pre> <br /> <span>This issue commonly occurs after a reboot if the encryption keys aren't configured to load automatically.</span><br /> @@ -780,26 +785,26 @@ http://www.gnu.org/software/src-highlite --> by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># On f0 - configure all encrypted datasets</font></i> -paul@f0:~ % doas sysrc zfskeys_enable=YES -zfskeys_enable: YES -> YES -paul@f0:~ % doas sysrc zfskeys_datasets=<font color="#808080">"zdata/enc zdata/enc/nfsdata zroot/bhyve"</font> -zfskeys_datasets: -> zdata/enc zdata/enc/nfsdata zroot/bhyve +<pre><i><font color="#ababab"># On f0 - configure all encrypted datasets</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">zfskeys_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font> +<font color="#ff0000">zfskeys_enable</font><font color="#F3E651">:</font><font color="#ff0000"> YES -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">zfskeys_datasets</font><font color="#F3E651">=</font><font color="#bb00ff">"zdata/enc zdata/enc/nfsdata zroot/bhyve"</font> +<font color="#ff0000">zfskeys_datasets</font><font color="#F3E651">:</font><font color="#ff0000"> -</font><font color="#F3E651">></font><font color="#ff0000"> zdata/enc zdata/enc/nfsdata zroot/bhyve</font> -<i><font color="silver"># Set correct key locations for all datasets</font></i> -paul@f0:~ % doas zfs <b><u><font color="#000000">set</font></u></b> \ - keylocation=file:///keys/f<font color="#000000">0</font>.lan.buetow.org:zdata.key zdata/enc/nfsdata +<i><font color="#ababab"># Set correct key locations for all datasets</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs </font><b><font color="#ffffff">set</font></b><font color="#ff0000"> </font><font color="#F3E651">\</font> +<font color="#ff0000"> </font><font color="#ff0000">keylocation</font><font color="#F3E651">=</font><font color="#ff0000">file</font><font color="#F3E651">:</font><font color="#ff0000">///keys/f</font><font color="#bb00ff">0</font><font color="#F3E651">.</font><font color="#ff0000">lan</font><font color="#F3E651">.</font><font color="#ff0000">buetow</font><font color="#F3E651">.</font><font color="#ff0000">org</font><font color="#F3E651">:</font><font color="#ff0000">zdata</font><font color="#F3E651">.</font><font color="#ff0000">key zdata/enc/nfsdata</font> -<i><font color="silver"># On f1 - include the replicated dataset</font></i> -paul@f1:~ % doas sysrc zfskeys_enable=YES -zfskeys_enable: YES -> YES -paul@f1:~ % doas sysrc \ - zfskeys_datasets=<font color="#808080">"zdata/enc zroot/bhyve zdata/sink/f0/zdata/enc/nfsdata"</font> -zfskeys_datasets: -> zdata/enc zroot/bhyve zdata/sink/f<font color="#000000">0</font>/zdata/enc/nfsdata +<i><font color="#ababab"># On f1 - include the replicated dataset</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">zfskeys_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font> +<font color="#ff0000">zfskeys_enable</font><font color="#F3E651">:</font><font color="#ff0000"> YES -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#F3E651">\</font> +<font color="#ff0000"> </font><font color="#ff0000">zfskeys_datasets</font><font color="#F3E651">=</font><font color="#bb00ff">"zdata/enc zroot/bhyve zdata/sink/f0/zdata/enc/nfsdata"</font> +<font color="#ff0000">zfskeys_datasets</font><font color="#F3E651">:</font><font color="#ff0000"> -</font><font color="#F3E651">></font><font color="#ff0000"> zdata/enc zroot/bhyve zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font> -<i><font color="silver"># Set key location for replicated dataset</font></i> -paul@f1:~ % doas zfs <b><u><font color="#000000">set</font></u></b> \ - keylocation=file:///keys/f<font color="#000000">0</font>.lan.buetow.org:zdata.key zdata/sink/f<font color="#000000">0</font>/zdata/enc/nfsdata +<i><font color="#ababab"># Set key location for replicated dataset</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs </font><b><font color="#ffffff">set</font></b><font color="#ff0000"> </font><font color="#F3E651">\</font> +<font color="#ff0000"> </font><font color="#ff0000">keylocation</font><font color="#F3E651">=</font><font color="#ff0000">file</font><font color="#F3E651">:</font><font color="#ff0000">///keys/f</font><font color="#bb00ff">0</font><font color="#F3E651">.</font><font color="#ff0000">lan</font><font color="#F3E651">.</font><font color="#ff0000">buetow</font><font color="#F3E651">.</font><font color="#ff0000">org</font><font color="#F3E651">:</font><font color="#ff0000">zdata</font><font color="#F3E651">.</font><font color="#ff0000">key zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font> </pre> <br /> <span>Important notes:</span><br /> @@ -822,13 +827,13 @@ paul@f1:~ % doas zfs <b><u><font color="#000000">set</font></u></b> \ by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># Check service status on both f0 and f1</font></i> -paul@f0:~ % doas service zrepl status -paul@f1:~ % doas service zrepl status +<pre><i><font color="#ababab"># Check service status on both f0 and f1</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service zrepl status</font> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service zrepl status</font> -<i><font color="silver"># If not running, start the service</font></i> -paul@f0:~ % doas service zrepl start -paul@f1:~ % doas service zrepl start +<i><font color="#ababab"># If not running, start the service</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service zrepl start</font> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service zrepl start</font> </pre> <br /> <h3 style='display: inline' id='check-zrepl-status-for-errors'>Check zrepl Status for Errors</h3><br /> @@ -839,11 +844,11 @@ paul@f1:~ % doas service zrepl start by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># Check detailed status (use --mode raw for non-tty environments)</font></i> -paul@f0:~ % doas zrepl status --mode raw +<pre><i><font color="#ababab"># Check detailed status (use --mode raw for non-tty environments)</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zrepl status --mode raw</font> -<i><font color="silver"># Look for error messages in the replication section</font></i> -<i><font color="silver"># Common errors include "no common snapshot" or connection failures</font></i> +<i><font color="#ababab"># Look for error messages in the replication section</font></i> +<i><font color="#ababab"># Common errors include "no common snapshot" or connection failures</font></i> </pre> <br /> <h3 style='display: inline' id='fixing-no-common-snapshot-errors'>Fixing "No Common Snapshot" Errors</h3><br /> @@ -866,27 +871,27 @@ no common snapshot or suitable bookmark between sender and receiver by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># First, identify the destination dataset on f1</font></i> -paul@f1:~ % doas zfs list | grep sink +<pre><i><font color="#ababab"># First, identify the destination dataset on f1</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs list </font><font color="#F3E651">|</font><font color="#ff0000"> grep sink</font> -<i><font color="silver"># Check existing snapshots on the problematic dataset</font></i> -paul@f1:~ % doas zfs list -t snapshot | grep nfsdata +<i><font color="#ababab"># Check existing snapshots on the problematic dataset</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs list -t snapshot </font><font color="#F3E651">|</font><font color="#ff0000"> grep nfsdata</font> -<i><font color="silver"># If you see snapshots with different naming (e.g., @daily-*, @weekly-*)</font></i> -<i><font color="silver"># these conflict with zrepl's @zrepl_* snapshots</font></i> +<i><font color="#ababab"># If you see snapshots with different naming (e.g., @daily-*, @weekly-*)</font></i> +<i><font color="#ababab"># these conflict with zrepl's @zrepl_* snapshots</font></i> -<i><font color="silver"># Destroy the entire destination dataset to allow clean replication</font></i> -paul@f1:~ % doas zfs destroy -r zdata/sink/f<font color="#000000">0</font>/zdata/enc/nfsdata +<i><font color="#ababab"># Destroy the entire destination dataset to allow clean replication</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs destroy -r zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font> -<i><font color="silver"># For VM replication, do the same for the freebsd dataset</font></i> -paul@f1:~ % doas zfs destroy -r zdata/sink/f<font color="#000000">0</font>/zroot/bhyve/freebsd +<i><font color="#ababab"># For VM replication, do the same for the freebsd dataset</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs destroy -r zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zroot/bhyve/freebsd</font> -<i><font color="silver"># Wake up zrepl to start fresh replication</font></i> -paul@f0:~ % doas zrepl signal wakeup f0_to_f1_nfsdata -paul@f0:~ % doas zrepl signal wakeup f0_to_f1_freebsd +<i><font color="#ababab"># Wake up zrepl to start fresh replication</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zrepl signal wakeup f0_to_f1_nfsdata</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zrepl signal wakeup f0_to_f1_freebsd</font> -<i><font color="silver"># Check replication status</font></i> -paul@f0:~ % doas zrepl status --mode raw +<i><font color="#ababab"># Check replication status</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zrepl status --mode raw</font> </pre> <br /> <span>**Verification that replication is working:**</span><br /> @@ -895,14 +900,14 @@ paul@f0:~ % doas zrepl status --mode raw by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># Look for "stepping" state and active zfs send processes</font></i> -paul@f0:~ % doas zrepl status --mode raw | grep -A<font color="#000000">5</font> <font color="#808080">"State.*stepping"</font> +<pre><i><font color="#ababab"># Look for "stepping" state and active zfs send processes</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zrepl status --mode raw </font><font color="#F3E651">|</font><font color="#ff0000"> grep -A</font><font color="#bb00ff">5</font><font color="#ff0000"> </font><font color="#bb00ff">"State.*stepping"</font> -<i><font color="silver"># Check for active ZFS commands</font></i> -paul@f0:~ % doas zrepl status --mode raw | grep -A<font color="#000000">10</font> <font color="#808080">"ZFSCmds.*Active"</font> +<i><font color="#ababab"># Check for active ZFS commands</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zrepl status --mode raw </font><font color="#F3E651">|</font><font color="#ff0000"> grep -A</font><font color="#bb00ff">10</font><font color="#ff0000"> </font><font color="#bb00ff">"ZFSCmds.*Active"</font> -<i><font color="silver"># Monitor progress - bytes replicated should be increasing</font></i> -paul@f0:~ % doas zrepl status --mode raw | grep BytesReplicated +<i><font color="#ababab"># Monitor progress - bytes replicated should be increasing</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zrepl status --mode raw </font><font color="#F3E651">|</font><font color="#ff0000"> grep BytesReplicated</font> </pre> <br /> <h3 style='display: inline' id='network-connectivity-issues'>Network Connectivity Issues</h3><br /> @@ -913,14 +918,14 @@ paul@f0:~ % doas zrepl status --mode raw | grep BytesReplicated by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># Test connectivity between nodes</font></i> -paul@f0:~ % nc -zv <font color="#000000">192.168</font>.<font color="#000000">2.131</font> <font color="#000000">8888</font> +<pre><i><font color="#ababab"># Test connectivity between nodes</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> nc -zv </font><font color="#bb00ff">192.168</font><font color="#F3E651">.</font><font color="#bb00ff">2.131</font><font color="#ff0000"> </font><font color="#bb00ff">8888</font> -<i><font color="silver"># Check if zrepl is listening on f1</font></i> -paul@f1:~ % doas netstat -an | grep <font color="#000000">8888</font> +<i><font color="#ababab"># Check if zrepl is listening on f1</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas netstat -an </font><font color="#F3E651">|</font><font color="#ff0000"> grep </font><font color="#bb00ff">8888</font> -<i><font color="silver"># Verify WireGuard tunnel is working</font></i> -paul@f0:~ % ping <font color="#000000">192.168</font>.<font color="#000000">2.131</font> +<i><font color="#ababab"># Verify WireGuard tunnel is working</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> ping </font><font color="#bb00ff">192.168</font><font color="#F3E651">.</font><font color="#bb00ff">2.131</font> </pre> <br /> <h3 style='display: inline' id='encryption-key-issues'>Encryption Key Issues</h3><br /> @@ -931,13 +936,13 @@ paul@f0:~ % ping <font color="#000000">192.168</font>.<font color="#000000">2.13 by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># Verify encryption keys are available on both nodes</font></i> -paul@f0:~ % doas zfs get keystatus zdata/enc/nfsdata -paul@f1:~ % doas zfs get keystatus zdata/sink/f<font color="#000000">0</font>/zdata/enc/nfsdata +<pre><i><font color="#ababab"># Verify encryption keys are available on both nodes</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs get keystatus zdata/enc/nfsdata</font> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs get keystatus zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font> -<i><font color="silver"># Load keys if unavailable</font></i> -paul@f1:~ % doas zfs load-key -L file:///keys/f<font color="#000000">0</font>.lan.buetow.org:zdata.key \ - zdata/sink/f<font color="#000000">0</font>/zdata/enc/nfsdata +<i><font color="#ababab"># Load keys if unavailable</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs load-key -L file</font><font color="#F3E651">:</font><font color="#ff0000">///keys/f</font><font color="#bb00ff">0</font><font color="#F3E651">.</font><font color="#ff0000">lan</font><font color="#F3E651">.</font><font color="#ff0000">buetow</font><font color="#F3E651">.</font><font color="#ff0000">org</font><font color="#F3E651">:</font><font color="#ff0000">zdata</font><font color="#F3E651">.</font><font color="#ff0000">key </font><font color="#F3E651">\</font> +<font color="#ff0000"> zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font> </pre> <br /> <h3 style='display: inline' id='monitoring-ongoing-replication'>Monitoring Ongoing Replication</h3><br /> @@ -948,18 +953,18 @@ paul@f1:~ % doas zfs load-key -L file:///keys/f<font color="#000000">0</font>.la by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># Monitor replication progress (run repeatedly to check status)</font></i> -paul@f0:~ % doas zrepl status --mode raw | grep -A<font color="#000000">10</font> BytesReplicated +<pre><i><font color="#ababab"># Monitor replication progress (run repeatedly to check status)</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zrepl status --mode raw </font><font color="#F3E651">|</font><font color="#ff0000"> grep -A</font><font color="#bb00ff">10</font><font color="#ff0000"> BytesReplicated</font> -<i><font color="silver"># Or install watch from ports and use it</font></i> -paul@f0:~ % doas pkg install watch -paul@f0:~ % watch -n <font color="#000000">5</font> <font color="#808080">'doas zrepl status --mode raw | grep -A10 BytesReplicated'</font> +<i><font color="#ababab"># Or install watch from ports and use it</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas pkg install watch</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> watch -n </font><font color="#bb00ff">5</font><font color="#ff0000"> </font><font color="#bb00ff">'doas zrepl status --mode raw | grep -A10 BytesReplicated'</font> -<i><font color="silver"># Check for new snapshots being created</font></i> -paul@f0:~ % doas zfs list -t snapshot | grep zrepl | tail -<font color="#000000">5</font> +<i><font color="#ababab"># Check for new snapshots being created</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs list -t snapshot </font><font color="#F3E651">|</font><font color="#ff0000"> grep zrepl </font><font color="#F3E651">|</font><font color="#ff0000"> tail -</font><font color="#bb00ff">5</font> -<i><font color="silver"># Verify snapshots appear on receiver</font></i> -paul@f1:~ % doas zfs list -t snapshot -r zdata/sink | grep zrepl | tail -<font color="#000000">5</font> +<i><font color="#ababab"># Verify snapshots appear on receiver</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs list -t snapshot -r zdata/sink </font><font color="#F3E651">|</font><font color="#ff0000"> grep zrepl </font><font color="#F3E651">|</font><font color="#ff0000"> tail -</font><font color="#bb00ff">5</font> </pre> <br /> <span>This troubleshooting process resolves the most common <span class='inlinecode'>zrepl</span> issues and ensures continuous data replication between your storage nodes.</span><br /> @@ -992,11 +997,11 @@ paul@f1:~ % doas zfs list -t snapshot -r zdata/sink | grep zrepl | tail -<font c by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># On f0 - The virtual IP 192.168.1.138 will float between f0 and f1</font></i> -ifconfig_re0_alias0=<font color="#808080">"inet vhid 1 pass testpass alias 192.168.1.138/32"</font> +<pre><i><font color="#ababab"># On f0 - The virtual IP 192.168.1.138 will float between f0 and f1</font></i> +<font color="#ff0000">ifconfig_re0_alias0</font><font color="#F3E651">=</font><font color="#bb00ff">"inet vhid 1 pass testpass alias 192.168.1.138/32"</font> -<i><font color="silver"># On f1 - Higher advskew means lower priority, so f0 wins elections</font></i> -ifconfig_re0_alias0=<font color="#808080">"inet vhid 1 advskew 100 pass testpass alias 192.168.1.138/32"</font> +<i><font color="#ababab"># On f1 - Higher advskew means lower priority, so f0 wins elections</font></i> +<font color="#ff0000">ifconfig_re0_alias0</font><font color="#F3E651">=</font><font color="#bb00ff">"inet vhid 1 advskew 100 pass testpass alias 192.168.1.138/32"</font> </pre> <br /> <span>Whereas:</span><br /> @@ -1026,16 +1031,16 @@ fd42:beef:cafe:2::138 f3s-storage-ha f3s-storage-ha.wg0 f3s-storage-ha.wg0.wan.b by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:~ % cat <<END | doas tee -a /etc/devd.conf -notify <font color="#000000">0</font> { - match <font color="#808080">"system"</font> <font color="#808080">"CARP"</font>; - match <font color="#808080">"subsystem"</font> <font color="#808080">"[0-9]+@[0-9a-z.]+"</font>; - match <font color="#808080">"type"</font> <font color="#808080">"(MASTER|BACKUP)"</font>; - action <font color="#808080">"/usr/local/bin/carpcontrol.sh $subsystem $type"</font>; -}; -END +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> cat </font><font color="#F3E651"><<</font><font color="#ff0000">END </font><font color="#F3E651">|</font><font color="#ff0000"> doas tee -a /etc/devd</font><font color="#F3E651">.</font><font color="#ff0000">conf</font> +<font color="#ff0000">notify </font><font color="#bb00ff">0</font><font color="#ff0000"> {</font> +<font color="#ff0000"> match </font><font color="#bb00ff">"system"</font><font color="#ff0000"> </font><font color="#bb00ff">"CARP"</font><font color="#F3E651">;</font> +<font color="#ff0000"> match </font><font color="#bb00ff">"subsystem"</font><font color="#ff0000"> </font><font color="#bb00ff">"[0-9]+@[0-9a-z.]+"</font><font color="#F3E651">;</font> +<font color="#ff0000"> match </font><font color="#bb00ff">"type"</font><font color="#ff0000"> </font><font color="#bb00ff">"(MASTER|BACKUP)"</font><font color="#F3E651">;</font> +<font color="#ff0000"> action </font><font color="#bb00ff">"/usr/local/bin/carpcontrol.sh $subsystem $type"</font><font color="#F3E651">;</font> +<font color="#ff0000">}</font><font color="#F3E651">;</font> +<font color="#ff0000">END</font> -paul@f0:~ % doas service devd restart +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service devd restart</font> </pre> <br /> <span>Next, we create the CARP control script that will restart stunnel when the CARP state changes:</span><br /> @@ -1046,29 +1051,29 @@ paul@f0:~ % doas service devd restart by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:~ % doas tee /usr/local/bin/carpcontrol.sh <<<font color="#808080">'EOF'</font> -<i><font color="silver">#!/bin/sh</font></i> -<i><font color="silver"># CARP state change control script</font></i> +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas tee /usr/local/bin/carpcontrol</font><font color="#F3E651">.</font><font color="#ff0000">sh </font><font color="#F3E651"><<</font><font color="#bb00ff">'EOF'</font> +<i><font color="#ababab">#!/bin/sh</font></i> +<i><font color="#ababab"># CARP state change control script</font></i> -<b><u><font color="#000000">case</font></u></b> <font color="#808080">"$2"</font> <b><u><font color="#000000">in</font></u></b> - MASTER) - logger <font color="#808080">"CARP state changed to MASTER, starting services"</font> - ;; - BACKUP) - logger <font color="#808080">"CARP state changed to BACKUP, stopping services"</font> - ;; - *) - logger <font color="#808080">"CARP state changed to $2 (unhandled)"</font> - ;; -<b><u><font color="#000000">esac</font></u></b> -EOF +<b><font color="#ffffff">case</font></b><font color="#ff0000"> </font><font color="#bb00ff">"$2"</font><font color="#ff0000"> </font><b><font color="#ffffff">in</font></b> +<font color="#ff0000"> MASTER</font><font color="#F3E651">)</font> +<font color="#ff0000"> logger </font><font color="#bb00ff">"CARP state changed to MASTER, starting services"</font> +<font color="#ff0000"> </font><font color="#F3E651">;;</font> +<font color="#ff0000"> BACKUP</font><font color="#F3E651">)</font> +<font color="#ff0000"> logger </font><font color="#bb00ff">"CARP state changed to BACKUP, stopping services"</font> +<font color="#ff0000"> </font><font color="#F3E651">;;</font> +<font color="#ff0000"> </font><font color="#F3E651">*)</font> +<font color="#ff0000"> logger </font><font color="#bb00ff">"CARP state changed to $2 (unhandled)"</font> +<font color="#ff0000"> </font><font color="#F3E651">;;</font> +<b><font color="#ffffff">esac</font></b> +<font color="#ff0000">EOF</font> -paul@f0:~ % doas chmod +x /usr/local/bin/carpcontrol.sh +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas chmod </font><font color="#F3E651">+</font><font color="#ff0000">x /usr/local/bin/carpcontrol</font><font color="#F3E651">.</font><font color="#ff0000">sh</font> -<i><font color="silver"># Copy the same script to f1</font></i> -paul@f0:~ % scp /usr/local/bin/carpcontrol.sh f1:/tmp/ -paul@f1:~ % doas mv /tmp/carpcontrol.sh /usr/local/bin/ -paul@f1:~ % doas chmod +x /usr/local/bin/carpcontrol.sh +<i><font color="#ababab"># Copy the same script to f1</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> scp /usr/local/bin/carpcontrol</font><font color="#F3E651">.</font><font color="#ff0000">sh f1</font><font color="#F3E651">:</font><font color="#ff0000">/tmp</font><font color="#F3E651">/</font> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas mv /tmp/carpcontrol</font><font color="#F3E651">.</font><font color="#ff0000">sh /usr/local/bin</font><font color="#F3E651">/</font> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas chmod </font><font color="#F3E651">+</font><font color="#ff0000">x /usr/local/bin/carpcontrol</font><font color="#F3E651">.</font><font color="#ff0000">sh</font> </pre> <br /> <span>Note that <span class='inlinecode'>carpcontrol.sh</span> doesn't do anything useful yet. We will provide more details (including starting and stopping services upon failover) later in this blog post.</span><br /> @@ -1079,10 +1084,10 @@ paul@f1:~ % doas chmod +x /usr/local/bin/carpcontrol.sh by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:~ % echo <font color="#808080">'carp_load="YES"'</font> | doas tee -a /boot/loader.conf -carp_load=<font color="#808080">"YES"</font> -paul@f1:~ % echo <font color="#808080">'carp_load="YES"'</font> | doas tee -a /boot/loader.conf -carp_load=<font color="#808080">"YES"</font> +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> echo </font><font color="#bb00ff">'carp_load="YES"'</font><font color="#ff0000"> </font><font color="#F3E651">|</font><font color="#ff0000"> doas tee -a /boot/loader</font><font color="#F3E651">.</font><font color="#ff0000">conf</font> +<font color="#ff0000">carp_load</font><font color="#F3E651">=</font><font color="#bb00ff">"YES"</font> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> echo </font><font color="#bb00ff">'carp_load="YES"'</font><font color="#ff0000"> </font><font color="#F3E651">|</font><font color="#ff0000"> doas tee -a /boot/loader</font><font color="#F3E651">.</font><font color="#ff0000">conf </font> +<font color="#ff0000">carp_load</font><font color="#F3E651">=</font><font color="#bb00ff">"YES"</font> </pre> <br /> <span>Then reboot both hosts or run <span class='inlinecode'>doas kldload carp</span> to load the module immediately. </span><br /> @@ -1099,18 +1104,18 @@ carp_load=<font color="#808080">"YES"</font> by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:~ % doas sysrc nfs_server_enable=YES -nfs_server_enable: YES -> YES -paul@f0:~ % doas sysrc nfsv4_server_enable=YES -nfsv4_server_enable: YES -> YES -paul@f0:~ % doas sysrc nfsuserd_enable=YES -nfsuserd_enable: YES -> YES -paul@f0:~ % doas sysrc nfsuserd_flags=<font color="#808080">"-domain lan.buetow.org"</font> -nfsuserd_flags: <font color="#808080">""</font> -> <font color="#808080">"-domain lan.buetow.org"</font> -paul@f0:~ % doas sysrc mountd_enable=YES -mountd_enable: NO -> YES -paul@f0:~ % doas sysrc rpcbind_enable=YES -rpcbind_enable: NO -> YES +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">nfs_server_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font> +<font color="#ff0000">nfs_server_enable</font><font color="#F3E651">:</font><font color="#ff0000"> YES -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">nfsv4_server_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font> +<font color="#ff0000">nfsv4_server_enable</font><font color="#F3E651">:</font><font color="#ff0000"> YES -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">nfsuserd_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font> +<font color="#ff0000">nfsuserd_enable</font><font color="#F3E651">:</font><font color="#ff0000"> YES -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">nfsuserd_flags</font><font color="#F3E651">=</font><font color="#bb00ff">"-domain lan.buetow.org"</font> +<font color="#ff0000">nfsuserd_flags</font><font color="#F3E651">:</font><font color="#ff0000"> </font><font color="#bb00ff">""</font><font color="#ff0000"> -</font><font color="#F3E651">></font><font color="#ff0000"> </font><font color="#bb00ff">"-domain lan.buetow.org"</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">mountd_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font> +<font color="#ff0000">mountd_enable</font><font color="#F3E651">:</font><font color="#ff0000"> NO -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">rpcbind_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font> +<font color="#ff0000">rpcbind_enable</font><font color="#F3E651">:</font><font color="#ff0000"> NO -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font> </pre> <br /> <span class='quote'>Update: 08.08.2025: I've added the domain to <span class='inlinecode'>nfsuserd_flags</span></span><br /> @@ -1121,14 +1126,14 @@ rpcbind_enable: NO -> YES by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># First, ensure the dataset is mounted</font></i> -paul@f0:~ % doas zfs get mounted zdata/enc/nfsdata -NAME PROPERTY VALUE SOURCE -zdata/enc/nfsdata mounted yes - +<pre><i><font color="#ababab"># First, ensure the dataset is mounted</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zfs get mounted zdata/enc/nfsdata</font> +<font color="#ff0000">NAME PROPERTY VALUE SOURCE</font> +<font color="#ff0000">zdata/enc/nfsdata mounted yes -</font> -<i><font color="silver"># Create the k3svolumes directory</font></i> -paul@f0:~ % doas mkdir -p /data/nfs/k3svolumes -paul@f0:~ % doas chmod <font color="#000000">755</font> /data/nfs/k3svolumes +<i><font color="#ababab"># Create the k3svolumes directory</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas mkdir -p /data/nfs/k3svolumes</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas chmod </font><font color="#bb00ff">755</font><font color="#ff0000"> /data/nfs/k3svolumes</font> </pre> <br /> <span>We also create the <span class='inlinecode'>/etc/exports</span> file. Since we're using stunnel for encryption, ALL clients must connect through stunnel, which appears as localhost (<span class='inlinecode'>127.0.0.1</span>) to the NFS server:</span><br /> @@ -1137,10 +1142,10 @@ paul@f0:~ % doas chmod <font color="#000000">755</font> /data/nfs/k3svolumes by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:~ % doas tee /etc/exports <<<font color="#808080">'EOF'</font> -V4: /data/nfs -sec=sys -/data/nfs -alldirs -maproot=root -network <font color="#000000">127.0</font>.<font color="#000000">0.1</font> -mask <font color="#000000">255.255</font>.<font color="#000000">255.255</font> -EOF +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas tee /etc/exports </font><font color="#F3E651"><<</font><font color="#bb00ff">'EOF'</font> +<font color="#ff0000">V4</font><font color="#F3E651">:</font><font color="#ff0000"> /data/nfs -sec</font><font color="#F3E651">=</font><font color="#ff0000">sys</font> +<font color="#ff0000">/data/nfs -alldirs -maproot</font><font color="#F3E651">=</font><font color="#ff0000">root -network </font><font color="#bb00ff">127.0</font><font color="#F3E651">.</font><font color="#bb00ff">0.1</font><font color="#ff0000"> -mask </font><font color="#bb00ff">255.255</font><font color="#F3E651">.</font><font color="#bb00ff">255.255</font> +<font color="#ff0000">EOF</font> </pre> <br /> <span>The exports configuration:</span><br /> @@ -1156,14 +1161,14 @@ EOF by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:~ % doas service rpcbind start -Starting rpcbind. -paul@f0:~ % doas service mountd start -Starting mountd. -paul@f0:~ % doas service nfsd start -Starting nfsd. -paul@f0:~ % doas service nfsuserd start -Starting nfsuserd. +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service rpcbind start</font> +<font color="#ff0000">Starting rpcbind</font><font color="#F3E651">.</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service mountd start</font> +<font color="#ff0000">Starting mountd</font><font color="#F3E651">.</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service nfsd start</font> +<font color="#ff0000">Starting nfsd</font><font color="#F3E651">.</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service nfsuserd start</font> +<font color="#ff0000">Starting nfsuserd</font><font color="#F3E651">.</font> </pre> <br /> <h3 style='display: inline' id='configuring-stunnel-for-nfs-encryption-with-carp-failover'>Configuring Stunnel for NFS Encryption with CARP Failover</h3><br /> @@ -1202,32 +1207,32 @@ Starting nfsuserd. by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># On f0 - Create CA</font></i> -paul@f0:~ % doas mkdir -p /usr/local/etc/stunnel/ca -paul@f0:~ % cd /usr/local/etc/stunnel/ca -paul@f0:~ % doas openssl genrsa -out ca-key.pem <font color="#000000">4096</font> -paul@f0:~ % doas openssl req -new -x<font color="#000000">509</font> -days <font color="#000000">3650</font> -key ca-key.pem -out ca-cert.pem \ - -subj <font color="#808080">'/C=US/ST=State/L=City/O=F3S Storage/CN=F3S Stunnel CA'</font> +<pre><i><font color="#ababab"># On f0 - Create CA</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas mkdir -p /usr/local/etc/stunnel/ca</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> cd /usr/local/etc/stunnel/ca</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas openssl genrsa -out ca-key</font><font color="#F3E651">.</font><font color="#ff0000">pem </font><font color="#bb00ff">4096</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas openssl req -new -x</font><font color="#bb00ff">509</font><font color="#ff0000"> -days </font><font color="#bb00ff">3650</font><font color="#ff0000"> -key ca-key</font><font color="#F3E651">.</font><font color="#ff0000">pem -out ca-cert</font><font color="#F3E651">.</font><font color="#ff0000">pem </font><font color="#F3E651">\</font> +<font color="#ff0000"> -subj </font><font color="#bb00ff">'/C=US/ST=State/L=City/O=F3S Storage/CN=F3S Stunnel CA'</font> -<i><font color="silver"># Create server certificate</font></i> -paul@f0:~ % cd /usr/local/etc/stunnel -paul@f0:~ % doas openssl genrsa -out server-key.pem <font color="#000000">4096</font> -paul@f0:~ % doas openssl req -new -key server-key.pem -out server.csr \ - -subj <font color="#808080">'/C=US/ST=State/L=City/O=F3S Storage/CN=f3s-storage-ha.lan'</font> -paul@f0:~ % doas openssl x509 -req -days <font color="#000000">3650</font> -in server.csr -CA ca/ca-cert.pem \ - -CAkey ca/ca-key.pem -CAcreateserial -out server-cert.pem +<i><font color="#ababab"># Create server certificate</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> cd /usr/local/etc/stunnel</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas openssl genrsa -out server-key</font><font color="#F3E651">.</font><font color="#ff0000">pem </font><font color="#bb00ff">4096</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas openssl req -new -key server-key</font><font color="#F3E651">.</font><font color="#ff0000">pem -out server</font><font color="#F3E651">.</font><font color="#ff0000">csr </font><font color="#F3E651">\</font> +<font color="#ff0000"> -subj </font><font color="#bb00ff">'/C=US/ST=State/L=City/O=F3S Storage/CN=f3s-storage-ha.lan'</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas openssl x509 -req -days </font><font color="#bb00ff">3650</font><font color="#ff0000"> -in server</font><font color="#F3E651">.</font><font color="#ff0000">csr -CA ca/ca-cert</font><font color="#F3E651">.</font><font color="#ff0000">pem </font><font color="#F3E651">\</font> +<font color="#ff0000"> -CAkey ca/ca-key</font><font color="#F3E651">.</font><font color="#ff0000">pem -CAcreateserial -out server-cert</font><font color="#F3E651">.</font><font color="#ff0000">pem</font> -<i><font color="silver"># Create client certificates for authorised clients</font></i> -paul@f0:~ % cd /usr/local/etc/stunnel/ca -paul@f0:~ % doas sh -c <font color="#808080">'for client in r0 r1 r2 earth; do </font> -<font color="#808080"> openssl genrsa -out ${client}-key.pem 4096</font> -<font color="#808080"> openssl req -new -key ${client}-key.pem -out ${client}.csr \</font> -<font color="#808080"> -subj "/C=US/ST=State/L=City/O=F3S Storage/CN=${client}.lan.buetow.org"</font> -<font color="#808080"> openssl x509 -req -days 3650 -in ${client}.csr -CA ca-cert.pem \</font> -<font color="#808080"> -CAkey ca-key.pem -CAcreateserial -out ${client}-cert.pem</font> -<font color="#808080"> # Combine cert and key into a single file for stunnel client</font> -<font color="#808080"> cat ${client}-cert.pem ${client}-key.pem > ${client}-stunnel.pem</font> -<font color="#808080">done'</font> +<i><font color="#ababab"># Create client certificates for authorised clients</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> cd /usr/local/etc/stunnel/ca</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sh -c </font><font color="#bb00ff">'for client in r0 r1 r2 earth; do </font> +<font color="#bb00ff"> openssl genrsa -out ${client}-key.pem 4096</font> +<font color="#bb00ff"> openssl req -new -key ${client}-key.pem -out ${client}.csr \</font> +<font color="#bb00ff"> -subj "/C=US/ST=State/L=City/O=F3S Storage/CN=${client}.lan.buetow.org"</font> +<font color="#bb00ff"> openssl x509 -req -days 3650 -in ${client}.csr -CA ca-cert.pem \</font> +<font color="#bb00ff"> -CAkey ca-key.pem -CAcreateserial -out ${client}-cert.pem</font> +<font color="#bb00ff"> # Combine cert and key into a single file for stunnel client</font> +<font color="#bb00ff"> cat ${client}-cert.pem ${client}-key.pem > ${client}-stunnel.pem</font> +<font color="#bb00ff">done'</font> </pre> <br /> <h3 style='display: inline' id='install-and-configure-stunnel-on-f0'>Install and Configure Stunnel on <span class='inlinecode'>f0</span></h3><br /> @@ -1236,35 +1241,35 @@ paul@f0:~ % doas sh -c <font color="#808080">'for client in r0 r1 r2 earth; do < by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># Install stunnel</font></i> -paul@f0:~ % doas pkg install -y stunnel +<pre><i><font color="#ababab"># Install stunnel</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas pkg install -y stunnel</font> -<i><font color="silver"># Configure stunnel server with client certificate authentication</font></i> -paul@f0:~ % doas tee /usr/local/etc/stunnel/stunnel.conf <<<font color="#808080">'EOF'</font> -cert = /usr/local/etc/stunnel/server-cert.pem -key = /usr/local/etc/stunnel/server-key.pem +<i><font color="#ababab"># Configure stunnel server with client certificate authentication</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas tee /usr/local/etc/stunnel/stunnel</font><font color="#F3E651">.</font><font color="#ff0000">conf </font><font color="#F3E651"><<</font><font color="#bb00ff">'EOF'</font> +<font color="#ff0000">cert </font><font color="#F3E651">=</font><font color="#ff0000"> /usr/local/etc/stunnel/server-cert</font><font color="#F3E651">.</font><font color="#ff0000">pem</font> +<font color="#ff0000">key </font><font color="#F3E651">=</font><font color="#ff0000"> /usr/local/etc/stunnel/server-key</font><font color="#F3E651">.</font><font color="#ff0000">pem</font> -setuid = stunnel -setgid = stunnel +<font color="#ff0000">setuid </font><font color="#F3E651">=</font><font color="#ff0000"> stunnel</font> +<font color="#ff0000">setgid </font><font color="#F3E651">=</font><font color="#ff0000"> stunnel</font> -[nfs-tls] -accept = <font color="#000000">192.168</font>.<font color="#000000">1.138</font>:<font color="#000000">2323</font> -connect = <font color="#000000">127.0</font>.<font color="#000000">0.1</font>:<font color="#000000">2049</font> -CAfile = /usr/local/etc/stunnel/ca/ca-cert.pem -verify = <font color="#000000">2</font> -requireCert = yes -EOF +<font color="#F3E651">[</font><font color="#ff0000">nfs-tls</font><font color="#F3E651">]</font> +<font color="#ff0000">accept </font><font color="#F3E651">=</font><font color="#ff0000"> </font><font color="#bb00ff">192.168</font><font color="#F3E651">.</font><font color="#bb00ff">1.138</font><font color="#F3E651">:</font><font color="#bb00ff">2323</font> +<font color="#ff0000">connect </font><font color="#F3E651">=</font><font color="#ff0000"> </font><font color="#bb00ff">127.0</font><font color="#F3E651">.</font><font color="#bb00ff">0.1</font><font color="#F3E651">:</font><font color="#bb00ff">2049</font> +<font color="#ff0000">CAfile </font><font color="#F3E651">=</font><font color="#ff0000"> /usr/local/etc/stunnel/ca/ca-cert</font><font color="#F3E651">.</font><font color="#ff0000">pem</font> +<font color="#ff0000">verify </font><font color="#F3E651">=</font><font color="#ff0000"> </font><font color="#bb00ff">2</font> +<font color="#ff0000">requireCert </font><font color="#F3E651">=</font><font color="#ff0000"> yes</font> +<font color="#ff0000">EOF</font> -<i><font color="silver"># Enable and start stunnel</font></i> -paul@f0:~ % doas sysrc stunnel_enable=YES -stunnel_enable: -> YES -paul@f0:~ % doas service stunnel start -Starting stunnel. +<i><font color="#ababab"># Enable and start stunnel</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">stunnel_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font> +<font color="#ff0000">stunnel_enable</font><font color="#F3E651">:</font><font color="#ff0000"> -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service stunnel start</font> +<font color="#ff0000">Starting stunnel</font><font color="#F3E651">.</font> -<i><font color="silver"># Restart stunnel to apply the CARP VIP binding</font></i> -paul@f0:~ % doas service stunnel restart -Stopping stunnel. -Starting stunnel. +<i><font color="#ababab"># Restart stunnel to apply the CARP VIP binding</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service stunnel restart</font> +<font color="#ff0000">Stopping stunnel</font><font color="#F3E651">.</font> +<font color="#ff0000">Starting stunnel</font><font color="#F3E651">.</font> </pre> <br /> <span>The configuration includes:</span><br /> @@ -1282,30 +1287,30 @@ Starting stunnel. by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f1:~ % doas sysrc nfs_server_enable=YES -nfs_server_enable: NO -> YES -paul@f1:~ % doas sysrc nfsv4_server_enable=YES -nfsv4_server_enable: NO -> YES -paul@f1:~ % doas sysrc nfsuserd_enable=YES -nfsuserd_enable: NO -> YES -paul@f1:~ % doas sysrc mountd_enable=YES -mountd_enable: NO -> YES -paul@f1:~ % doas sysrc rpcbind_enable=YES -rpcbind_enable: NO -> YES +<pre><font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">nfs_server_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font> +<font color="#ff0000">nfs_server_enable</font><font color="#F3E651">:</font><font color="#ff0000"> NO -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">nfsv4_server_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font> +<font color="#ff0000">nfsv4_server_enable</font><font color="#F3E651">:</font><font color="#ff0000"> NO -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">nfsuserd_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font> +<font color="#ff0000">nfsuserd_enable</font><font color="#F3E651">:</font><font color="#ff0000"> NO -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">mountd_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font> +<font color="#ff0000">mountd_enable</font><font color="#F3E651">:</font><font color="#ff0000"> NO -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">rpcbind_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font> +<font color="#ff0000">rpcbind_enable</font><font color="#F3E651">:</font><font color="#ff0000"> NO -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font> -paul@f1:~ % doas tee /etc/exports <<<font color="#808080">'EOF'</font> -V4: /data/nfs -sec=sys -/data/nfs -alldirs -maproot=root -network <font color="#000000">127.0</font>.<font color="#000000">0.1</font> -mask <font color="#000000">255.255</font>.<font color="#000000">255.255</font> -EOF +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas tee /etc/exports </font><font color="#F3E651"><<</font><font color="#bb00ff">'EOF'</font> +<font color="#ff0000">V4</font><font color="#F3E651">:</font><font color="#ff0000"> /data/nfs -sec</font><font color="#F3E651">=</font><font color="#ff0000">sys</font> +<font color="#ff0000">/data/nfs -alldirs -maproot</font><font color="#F3E651">=</font><font color="#ff0000">root -network </font><font color="#bb00ff">127.0</font><font color="#F3E651">.</font><font color="#bb00ff">0.1</font><font color="#ff0000"> -mask </font><font color="#bb00ff">255.255</font><font color="#F3E651">.</font><font color="#bb00ff">255.255</font> +<font color="#ff0000">EOF</font> -paul@f1:~ % doas service rpcbind start -Starting rpcbind. -paul@f1:~ % doas service mountd start -Starting mountd. -paul@f1:~ % doas service nfsd start -Starting nfsd. -paul@f1:~ % doas service nfsuserd start -Starting nfsuserd. +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service rpcbind start</font> +<font color="#ff0000">Starting rpcbind</font><font color="#F3E651">.</font> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service mountd start</font> +<font color="#ff0000">Starting mountd</font><font color="#F3E651">.</font> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service nfsd start</font> +<font color="#ff0000">Starting nfsd</font><font color="#F3E651">.</font> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service nfsuserd start</font> +<font color="#ff0000">Starting nfsuserd</font><font color="#F3E651">.</font> </pre> <br /> <span>And to configure stunnel on <span class='inlinecode'>f1</span>, we run:</span><br /> @@ -1314,42 +1319,42 @@ Starting nfsuserd. by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># Install stunnel</font></i> -paul@f1:~ % doas pkg install -y stunnel +<pre><i><font color="#ababab"># Install stunnel</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas pkg install -y stunnel</font> -<i><font color="silver"># Copy certificates from f0</font></i> -paul@f0:~ % doas tar -cf /tmp/stunnel-certs.tar \ - -C /usr/local/etc/stunnel server-cert.pem server-key.pem ca -paul@f0:~ % scp /tmp/stunnel-certs.tar f1:/tmp/ +<i><font color="#ababab"># Copy certificates from f0</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas tar -cf /tmp/stunnel-certs</font><font color="#F3E651">.</font><font color="#ff0000">tar </font><font color="#F3E651">\</font> +<font color="#ff0000"> -C /usr/local/etc/stunnel server-cert</font><font color="#F3E651">.</font><font color="#ff0000">pem server-key</font><font color="#F3E651">.</font><font color="#ff0000">pem ca</font> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> scp /tmp/stunnel-certs</font><font color="#F3E651">.</font><font color="#ff0000">tar f1</font><font color="#F3E651">:</font><font color="#ff0000">/tmp</font><font color="#F3E651">/</font> -paul@f1:~ % cd /usr/local/etc/stunnel && doas tar -xf /tmp/stunnel-certs.tar +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> cd /usr/local/etc/stunnel </font><font color="#F3E651">&&</font><font color="#ff0000"> doas tar -xf /tmp/stunnel-certs</font><font color="#F3E651">.</font><font color="#ff0000">tar</font> -<i><font color="silver"># Configure stunnel server on f1 with client certificate authentication</font></i> -paul@f1:~ % doas tee /usr/local/etc/stunnel/stunnel.conf <<<font color="#808080">'EOF'</font> -cert = /usr/local/etc/stunnel/server-cert.pem -key = /usr/local/etc/stunnel/server-key.pem +<i><font color="#ababab"># Configure stunnel server on f1 with client certificate authentication</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas tee /usr/local/etc/stunnel/stunnel</font><font color="#F3E651">.</font><font color="#ff0000">conf </font><font color="#F3E651"><<</font><font color="#bb00ff">'EOF'</font> +<font color="#ff0000">cert </font><font color="#F3E651">=</font><font color="#ff0000"> /usr/local/etc/stunnel/server-cert</font><font color="#F3E651">.</font><font color="#ff0000">pem</font> +<font color="#ff0000">key </font><font color="#F3E651">=</font><font color="#ff0000"> /usr/local/etc/stunnel/server-key</font><font color="#F3E651">.</font><font color="#ff0000">pem</font> -setuid = stunnel -setgid = stunnel +<font color="#ff0000">setuid </font><font color="#F3E651">=</font><font color="#ff0000"> stunnel</font> +<font color="#ff0000">setgid </font><font color="#F3E651">=</font><font color="#ff0000"> stunnel</font> -[nfs-tls] -accept = <font color="#000000">192.168</font>.<font color="#000000">1.138</font>:<font color="#000000">2323</font> -connect = <font color="#000000">127.0</font>.<font color="#000000">0.1</font>:<font color="#000000">2049</font> -CAfile = /usr/local/etc/stunnel/ca/ca-cert.pem -verify = <font color="#000000">2</font> -requireCert = yes -EOF +<font color="#F3E651">[</font><font color="#ff0000">nfs-tls</font><font color="#F3E651">]</font> +<font color="#ff0000">accept </font><font color="#F3E651">=</font><font color="#ff0000"> </font><font color="#bb00ff">192.168</font><font color="#F3E651">.</font><font color="#bb00ff">1.138</font><font color="#F3E651">:</font><font color="#bb00ff">2323</font> +<font color="#ff0000">connect </font><font color="#F3E651">=</font><font color="#ff0000"> </font><font color="#bb00ff">127.0</font><font color="#F3E651">.</font><font color="#bb00ff">0.1</font><font color="#F3E651">:</font><font color="#bb00ff">2049</font> +<font color="#ff0000">CAfile </font><font color="#F3E651">=</font><font color="#ff0000"> /usr/local/etc/stunnel/ca/ca-cert</font><font color="#F3E651">.</font><font color="#ff0000">pem</font> +<font color="#ff0000">verify </font><font color="#F3E651">=</font><font color="#ff0000"> </font><font color="#bb00ff">2</font> +<font color="#ff0000">requireCert </font><font color="#F3E651">=</font><font color="#ff0000"> yes</font> +<font color="#ff0000">EOF</font> -<i><font color="silver"># Enable and start stunnel</font></i> -paul@f1:~ % doas sysrc stunnel_enable=YES -stunnel_enable: -> YES -paul@f1:~ % doas service stunnel start -Starting stunnel. +<i><font color="#ababab"># Enable and start stunnel</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sysrc </font><font color="#ff0000">stunnel_enable</font><font color="#F3E651">=</font><font color="#ff0000">YES</font> +<font color="#ff0000">stunnel_enable</font><font color="#F3E651">:</font><font color="#ff0000"> -</font><font color="#F3E651">></font><font color="#ff0000"> YES</font> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service stunnel start</font> +<font color="#ff0000">Starting stunnel</font><font color="#F3E651">.</font> -<i><font color="silver"># Restart stunnel to apply the CARP VIP binding</font></i> -paul@f1:~ % doas service stunnel restart -Stopping stunnel. -Starting stunnel. +<i><font color="#ababab"># Restart stunnel to apply the CARP VIP binding</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas service stunnel restart</font> +<font color="#ff0000">Stopping stunnel</font><font color="#F3E651">.</font> +<font color="#ff0000">Starting stunnel</font><font color="#F3E651">.</font> </pre> <br /> <h3 style='display: inline' id='carp-control-script-for-clean-failover'>CARP Control Script for Clean Failover</h3><br /> @@ -1376,54 +1381,54 @@ Starting stunnel. by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># Create CARP control script on both f0 and f1</font></i> -paul@f0:~ % doas tee /usr/local/bin/carpcontrol.sh <<<font color="#808080">'EOF'</font> -<i><font color="silver">#!/bin/sh</font></i> -<i><font color="silver"># CARP state change control script</font></i> - -HOSTNAME=`hostname` - -<b><u><font color="#000000">if</font></u></b> [ ! -f /data/nfs/nfs.DO_NOT_REMOVE ]; <b><u><font color="#000000">then</font></u></b> - logger <font color="#808080">'/data/nfs not mounted, mounting it now!'</font> - <b><u><font color="#000000">if</font></u></b> [ <font color="#808080">"$HOSTNAME"</font> = <font color="#808080">'f0.lan.buetow.org'</font> ]; <b><u><font color="#000000">then</font></u></b> - zfs load-key -L file:///keys/f<font color="#000000">0</font>.lan.buetow.org:zdata.key zdata/enc/nfsdata - zfs <b><u><font color="#000000">set</font></u></b> mountpoint=/data/nfs zdata/enc/nfsdata - <b><u><font color="#000000">else</font></u></b> - zfs load-key -L file:///keys/f<font color="#000000">0</font>.lan.buetow.org:zdata.key zdata/sink/f<font color="#000000">0</font>/zdata/enc/nfsdata - zfs <b><u><font color="#000000">set</font></u></b> mountpoint=/data/nfs zdata/sink/f<font color="#000000">0</font>/zdata/enc/nfsdata - zfs mount zdata/sink/f<font color="#000000">0</font>/zdata/enc/nfsdata - zfs <b><u><font color="#000000">set</font></u></b> <b><u><font color="#000000">readonly</font></u></b>=on zdata/sink/f<font color="#000000">0</font>/zdata/enc/nfsdata - <b><u><font color="#000000">fi</font></u></b> - service nfsd stop <font color="#000000">2</font>>&<font color="#000000">1</font> - service mountd stop <font color="#000000">2</font>>&<font color="#000000">1</font> -<b><u><font color="#000000">fi</font></u></b> - - -<b><u><font color="#000000">case</font></u></b> <font color="#808080">"$2"</font> <b><u><font color="#000000">in</font></u></b> - MASTER) - logger <font color="#808080">"CARP state changed to MASTER, starting services"</font> - service rpcbind start >/dev/null <font color="#000000">2</font>>&<font color="#000000">1</font> - service mountd start >/dev/null <font color="#000000">2</font>>&<font color="#000000">1</font> - service nfsd start >/dev/null <font color="#000000">2</font>>&<font color="#000000">1</font> - service nfsuserd start >/dev/null <font color="#000000">2</font>>&<font color="#000000">1</font> - service stunnel restart >/dev/null <font color="#000000">2</font>>&<font color="#000000">1</font> - logger <font color="#808080">"CARP MASTER: NFS and stunnel services started"</font> - ;; - BACKUP) - logger <font color="#808080">"CARP state changed to BACKUP, stopping services"</font> - service stunnel stop >/dev/null <font color="#000000">2</font>>&<font color="#000000">1</font> - service nfsd stop >/dev/null <font color="#000000">2</font>>&<font color="#000000">1</font> - service mountd stop >/dev/null <font color="#000000">2</font>>&<font color="#000000">1</font> - service nfsuserd stop >/dev/null <font color="#000000">2</font>>&<font color="#000000">1</font> - logger <font color="#808080">"CARP BACKUP: NFS and stunnel services stopped"</font> - ;; - *) - logger <font color="#808080">"CARP state changed to $2 (unhandled)"</font> - ;; -<b><u><font color="#000000">esac</font></u></b> -EOF - -paul@f0:~ % doas chmod +x /usr/local/bin/carpcontrol.sh +<pre><i><font color="#ababab"># Create CARP control script on both f0 and f1</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas tee /usr/local/bin/carpcontrol</font><font color="#F3E651">.</font><font color="#ff0000">sh </font><font color="#F3E651"><<</font><font color="#bb00ff">'EOF'</font> +<i><font color="#ababab">#!/bin/sh</font></i> +<i><font color="#ababab"># CARP state change control script</font></i> + +<font color="#ff0000">HOSTNAME</font><font color="#F3E651">=</font><font color="#ff0000">`hostname`</font> + +<b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">[</font><font color="#ff0000"> </font><font color="#F3E651">!</font><font color="#ff0000"> -f /data/nfs/nfs</font><font color="#F3E651">.</font><font color="#ff0000">DO_NOT_REMOVE </font><font color="#F3E651">];</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b> +<font color="#ff0000"> logger </font><font color="#bb00ff">'/data/nfs not mounted, mounting it now!'</font> +<font color="#ff0000"> </font><b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">[</font><font color="#ff0000"> </font><font color="#bb00ff">"$HOSTNAME"</font><font color="#ff0000"> </font><font color="#F3E651">=</font><font color="#ff0000"> </font><font color="#bb00ff">'f0.lan.buetow.org'</font><font color="#ff0000"> </font><font color="#F3E651">];</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b> +<font color="#ff0000"> zfs load-key -L file</font><font color="#F3E651">:</font><font color="#ff0000">///keys/f</font><font color="#bb00ff">0</font><font color="#F3E651">.</font><font color="#ff0000">lan</font><font color="#F3E651">.</font><font color="#ff0000">buetow</font><font color="#F3E651">.</font><font color="#ff0000">org</font><font color="#F3E651">:</font><font color="#ff0000">zdata</font><font color="#F3E651">.</font><font color="#ff0000">key zdata/enc/nfsdata</font> +<font color="#ff0000"> zfs </font><b><font color="#ffffff">set</font></b><font color="#ff0000"> </font><font color="#ff0000">mountpoint</font><font color="#F3E651">=</font><font color="#ff0000">/data/nfs zdata/enc/nfsdata</font> +<font color="#ff0000"> </font><b><font color="#ffffff">else</font></b> +<font color="#ff0000"> zfs load-key -L file</font><font color="#F3E651">:</font><font color="#ff0000">///keys/f</font><font color="#bb00ff">0</font><font color="#F3E651">.</font><font color="#ff0000">lan</font><font color="#F3E651">.</font><font color="#ff0000">buetow</font><font color="#F3E651">.</font><font color="#ff0000">org</font><font color="#F3E651">:</font><font color="#ff0000">zdata</font><font color="#F3E651">.</font><font color="#ff0000">key zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font> +<font color="#ff0000"> zfs </font><b><font color="#ffffff">set</font></b><font color="#ff0000"> </font><font color="#ff0000">mountpoint</font><font color="#F3E651">=</font><font color="#ff0000">/data/nfs zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font> +<font color="#ff0000"> zfs mount zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font> +<font color="#ff0000"> zfs </font><b><font color="#ffffff">set</font></b><font color="#ff0000"> </font><b><font color="#ffffff">readonly</font></b><font color="#F3E651">=</font><font color="#ff0000">on zdata/sink/f</font><font color="#bb00ff">0</font><font color="#ff0000">/zdata/enc/nfsdata</font> +<font color="#ff0000"> </font><b><font color="#ffffff">fi</font></b> +<font color="#ff0000"> service nfsd stop </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font> +<font color="#ff0000"> service mountd stop </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font> +<b><font color="#ffffff">fi</font></b> + + +<b><font color="#ffffff">case</font></b><font color="#ff0000"> </font><font color="#bb00ff">"$2"</font><font color="#ff0000"> </font><b><font color="#ffffff">in</font></b> +<font color="#ff0000"> MASTER</font><font color="#F3E651">)</font> +<font color="#ff0000"> logger </font><font color="#bb00ff">"CARP state changed to MASTER, starting services"</font> +<font color="#ff0000"> service rpcbind start </font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font> +<font color="#ff0000"> service mountd start </font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font> +<font color="#ff0000"> service nfsd start </font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font> +<font color="#ff0000"> service nfsuserd start </font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font> +<font color="#ff0000"> service stunnel restart </font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font> +<font color="#ff0000"> logger </font><font color="#bb00ff">"CARP MASTER: NFS and stunnel services started"</font> +<font color="#ff0000"> </font><font color="#F3E651">;;</font> +<font color="#ff0000"> BACKUP</font><font color="#F3E651">)</font> +<font color="#ff0000"> logger </font><font color="#bb00ff">"CARP state changed to BACKUP, stopping services"</font> +<font color="#ff0000"> service stunnel stop </font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font> +<font color="#ff0000"> service nfsd stop </font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font> +<font color="#ff0000"> service mountd stop </font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font> +<font color="#ff0000"> service nfsuserd stop </font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font> +<font color="#ff0000"> logger </font><font color="#bb00ff">"CARP BACKUP: NFS and stunnel services stopped"</font> +<font color="#ff0000"> </font><font color="#F3E651">;;</font> +<font color="#ff0000"> </font><font color="#F3E651">*)</font> +<font color="#ff0000"> logger </font><font color="#bb00ff">"CARP state changed to $2 (unhandled)"</font> +<font color="#ff0000"> </font><font color="#F3E651">;;</font> +<b><font color="#ffffff">esac</font></b> +<font color="#ff0000">EOF</font> + +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas chmod </font><font color="#F3E651">+</font><font color="#ff0000">x /usr/local/bin/carpcontrol</font><font color="#F3E651">.</font><font color="#ff0000">sh</font> </pre> <br /> <h3 style='display: inline' id='carp-management-script'>CARP Management Script</h3><br /> @@ -1434,113 +1439,113 @@ paul@f0:~ % doas chmod +x /usr/local/bin/carpcontrol.sh by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># Create the CARP management script</font></i> -paul@f0:~ % doas tee /usr/local/bin/carp <<<font color="#808080">'EOF'</font> -<i><font color="silver">#!/bin/sh</font></i> -<i><font color="silver"># CARP state management script</font></i> -<i><font color="silver"># Usage: carp [master|backup|auto-failback enable|auto-failback disable]</font></i> -<i><font color="silver"># Without arguments: shows current state</font></i> - -<i><font color="silver"># Find the interface with CARP configured</font></i> -CARP_IF=$(ifconfig -l | xargs -n<font color="#000000">1</font> | <b><u><font color="#000000">while</font></u></b> <b><u><font color="#000000">read</font></u></b> <b><u><font color="#000000">if</font></u></b>; <b><u><font color="#000000">do</font></u></b> - ifconfig <font color="#808080">"$if"</font> <font color="#000000">2</font>>/dev/null | grep -q <font color="#808080">"carp:"</font> && echo <font color="#808080">"$if"</font> && <b><u><font color="#000000">break</font></u></b> -<b><u><font color="#000000">done</font></u></b>) - -<b><u><font color="#000000">if</font></u></b> [ -z <font color="#808080">"$CARP_IF"</font> ]; <b><u><font color="#000000">then</font></u></b> - echo <font color="#808080">"Error: No CARP interface found"</font> - <b><u><font color="#000000">exit</font></u></b> <font color="#000000">1</font> -<b><u><font color="#000000">fi</font></u></b> - -<i><font color="silver"># Get CARP VHID</font></i> -VHID=$(ifconfig <font color="#808080">"$CARP_IF"</font> | grep <font color="#808080">"carp:"</font> | sed -n <font color="#808080">'s/.*vhid </font>\(<font color="#808080">[0-9]*</font>\)<font color="#808080">.*/</font>\1<font color="#808080">/p'</font>) - -<b><u><font color="#000000">if</font></u></b> [ -z <font color="#808080">"$VHID"</font> ]; <b><u><font color="#000000">then</font></u></b> - echo <font color="#808080">"Error: Could not determine CARP VHID"</font> - <b><u><font color="#000000">exit</font></u></b> <font color="#000000">1</font> -<b><u><font color="#000000">fi</font></u></b> - -<i><font color="silver"># Function to get the current state</font></i> -get_state() { - ifconfig <font color="#808080">"$CARP_IF"</font> | grep <font color="#808080">"carp:"</font> | awk <font color="#808080">'{print $2}'</font> -} - -<i><font color="silver"># Check for auto-failback block file</font></i> -BLOCK_FILE=<font color="#808080">"/data/nfs/nfs.NO_AUTO_FAILBACK"</font> -check_auto_failback() { - <b><u><font color="#000000">if</font></u></b> [ -f <font color="#808080">"$BLOCK_FILE"</font> ]; <b><u><font color="#000000">then</font></u></b> - echo <font color="#808080">"WARNING: Auto-failback is DISABLED (file exists: $BLOCK_FILE)"</font> - <b><u><font color="#000000">fi</font></u></b> -} - -<i><font color="silver"># Main logic</font></i> -<b><u><font color="#000000">case</font></u></b> <font color="#808080">"$1"</font> <b><u><font color="#000000">in</font></u></b> - <font color="#808080">""</font>) - <i><font color="silver"># No argument - show current state</font></i> - STATE=$(get_state) - echo <font color="#808080">"CARP state on $CARP_IF (vhid $VHID): $STATE"</font> - check_auto_failback - ;; - master) - <i><font color="silver"># Force to MASTER state</font></i> - echo <font color="#808080">"Setting CARP to MASTER state..."</font> - ifconfig <font color="#808080">"$CARP_IF"</font> vhid <font color="#808080">"$VHID"</font> state master - sleep <font color="#000000">1</font> - STATE=$(get_state) - echo <font color="#808080">"CARP state on $CARP_IF (vhid $VHID): $STATE"</font> - check_auto_failback - ;; - backup) - <i><font color="silver"># Force to BACKUP state</font></i> - echo <font color="#808080">"Setting CARP to BACKUP state..."</font> - ifconfig <font color="#808080">"$CARP_IF"</font> vhid <font color="#808080">"$VHID"</font> state backup - sleep <font color="#000000">1</font> - STATE=$(get_state) - echo <font color="#808080">"CARP state on $CARP_IF (vhid $VHID): $STATE"</font> - check_auto_failback - ;; - auto-failback) - <b><u><font color="#000000">case</font></u></b> <font color="#808080">"$2"</font> <b><u><font color="#000000">in</font></u></b> - <b><u><font color="#000000">enable</font></u></b>) - <b><u><font color="#000000">if</font></u></b> [ -f <font color="#808080">"$BLOCK_FILE"</font> ]; <b><u><font color="#000000">then</font></u></b> - rm <font color="#808080">"$BLOCK_FILE"</font> - echo <font color="#808080">"Auto-failback ENABLED (removed $BLOCK_FILE)"</font> - <b><u><font color="#000000">else</font></u></b> - echo <font color="#808080">"Auto-failback was already enabled"</font> - <b><u><font color="#000000">fi</font></u></b> - ;; - disable) - <b><u><font color="#000000">if</font></u></b> [ ! -f <font color="#808080">"$BLOCK_FILE"</font> ]; <b><u><font color="#000000">then</font></u></b> - touch <font color="#808080">"$BLOCK_FILE"</font> - echo <font color="#808080">"Auto-failback DISABLED (created $BLOCK_FILE)"</font> - <b><u><font color="#000000">else</font></u></b> - echo <font color="#808080">"Auto-failback was already disabled"</font> - <b><u><font color="#000000">fi</font></u></b> - ;; - *) - echo <font color="#808080">"Usage: $0 auto-failback [enable|disable]"</font> - echo <font color="#808080">" enable: Remove block file to allow automatic failback"</font> - echo <font color="#808080">" disable: Create block file to prevent automatic failback"</font> - <b><u><font color="#000000">exit</font></u></b> <font color="#000000">1</font> - ;; - <b><u><font color="#000000">esac</font></u></b> - ;; - *) - echo <font color="#808080">"Usage: $0 [master|backup|auto-failback enable|auto-failback disable]"</font> - echo <font color="#808080">" Without arguments: show current CARP state"</font> - echo <font color="#808080">" master: force this node to become CARP MASTER"</font> - echo <font color="#808080">" backup: force this node to become CARP BACKUP"</font> - echo <font color="#808080">" auto-failback enable: allow automatic failback to f0"</font> - echo <font color="#808080">" auto-failback disable: prevent automatic failback to f0"</font> - <b><u><font color="#000000">exit</font></u></b> <font color="#000000">1</font> - ;; -<b><u><font color="#000000">esac</font></u></b> -EOF - -paul@f0:~ % doas chmod +x /usr/local/bin/carp - -<i><font color="silver"># Copy to f1 as well</font></i> -paul@f0:~ % scp /usr/local/bin/carp f1:/tmp/ -paul@f1:~ % doas cp /tmp/carp /usr/local/bin/carp && doas chmod +x /usr/local/bin/carp +<pre><i><font color="#ababab"># Create the CARP management script</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas tee /usr/local/bin/carp </font><font color="#F3E651"><<</font><font color="#bb00ff">'EOF'</font> +<i><font color="#ababab">#!/bin/sh</font></i> +<i><font color="#ababab"># CARP state management script</font></i> +<i><font color="#ababab"># Usage: carp [master|backup|auto-failback enable|auto-failback disable]</font></i> +<i><font color="#ababab"># Without arguments: shows current state</font></i> + +<i><font color="#ababab"># Find the interface with CARP configured</font></i> +<font color="#ff0000">CARP_IF</font><font color="#F3E651">=</font><font color="#ff0000">$(</font><font color="#ff0000">ifconfig -l </font><font color="#F3E651">|</font><font color="#ff0000"> xargs -n</font><font color="#bb00ff">1</font><font color="#ff0000"> </font><font color="#F3E651">|</font><font color="#ff0000"> </font><b><font color="#ffffff">while</font></b><font color="#ff0000"> </font><b><font color="#ffffff">read</font></b><font color="#ff0000"> </font><b><font color="#ffffff">if</font></b><font color="#F3E651">;</font><font color="#ff0000"> </font><b><font color="#ffffff">do</font></b> +<font color="#ff0000"> ifconfig </font><font color="#bb00ff">"$if"</font><font color="#ff0000"> </font><font color="#bb00ff">2</font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#F3E651">|</font><font color="#ff0000"> grep -q </font><font color="#bb00ff">"carp:"</font><font color="#ff0000"> </font><font color="#F3E651">&&</font><font color="#ff0000"> echo </font><font color="#bb00ff">"$if"</font><font color="#ff0000"> </font><font color="#F3E651">&&</font><font color="#ff0000"> </font><b><font color="#ffffff">break</font></b> +<b><font color="#ffffff">done</font></b><font color="#F3E651">)</font> + +<b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">[</font><font color="#ff0000"> -z </font><font color="#bb00ff">"$CARP_IF"</font><font color="#ff0000"> </font><font color="#F3E651">];</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b> +<font color="#ff0000"> echo </font><font color="#bb00ff">"Error: No CARP interface found"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">exit</font></b><font color="#ff0000"> </font><font color="#bb00ff">1</font> +<b><font color="#ffffff">fi</font></b> + +<i><font color="#ababab"># Get CARP VHID</font></i> +<font color="#ff0000">VHID</font><font color="#F3E651">=</font><font color="#ff0000">$(</font><font color="#ff0000">ifconfig </font><font color="#bb00ff">"$CARP_IF"</font><font color="#ff0000"> </font><font color="#F3E651">|</font><font color="#ff0000"> grep </font><font color="#bb00ff">"carp:"</font><font color="#ff0000"> </font><font color="#F3E651">|</font><font color="#ff0000"> sed -n </font><font color="#bb00ff">'s/.*vhid </font><font color="#ffffff">\(</font><font color="#bb00ff">[0-9]*</font><font color="#ffffff">\)</font><font color="#bb00ff">.*/</font><font color="#ffffff">\1</font><font color="#bb00ff">/p'</font><font color="#F3E651">)</font> + +<b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">[</font><font color="#ff0000"> -z </font><font color="#bb00ff">"$VHID"</font><font color="#ff0000"> </font><font color="#F3E651">];</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b> +<font color="#ff0000"> echo </font><font color="#bb00ff">"Error: Could not determine CARP VHID"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">exit</font></b><font color="#ff0000"> </font><font color="#bb00ff">1</font> +<b><font color="#ffffff">fi</font></b> + +<i><font color="#ababab"># Function to get the current state</font></i> +<font color="#7bc710">get_state()</font><font color="#ff0000"> {</font> +<font color="#ff0000"> ifconfig </font><font color="#bb00ff">"$CARP_IF"</font><font color="#ff0000"> </font><font color="#F3E651">|</font><font color="#ff0000"> grep </font><font color="#bb00ff">"carp:"</font><font color="#ff0000"> </font><font color="#F3E651">|</font><font color="#ff0000"> awk </font><font color="#bb00ff">'{print $2}'</font> +<font color="#ff0000">}</font> + +<i><font color="#ababab"># Check for auto-failback block file</font></i> +<font color="#ff0000">BLOCK_FILE</font><font color="#F3E651">=</font><font color="#bb00ff">"/data/nfs/nfs.NO_AUTO_FAILBACK"</font> +<font color="#7bc710">check_auto_failback()</font><font color="#ff0000"> {</font> +<font color="#ff0000"> </font><b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">[</font><font color="#ff0000"> -f </font><font color="#bb00ff">"$BLOCK_FILE"</font><font color="#ff0000"> </font><font color="#F3E651">];</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b> +<font color="#ff0000"> echo </font><font color="#bb00ff">"WARNING: Auto-failback is DISABLED (file exists: $BLOCK_FILE)"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">fi</font></b> +<font color="#ff0000">}</font> + +<i><font color="#ababab"># Main logic</font></i> +<b><font color="#ffffff">case</font></b><font color="#ff0000"> </font><font color="#bb00ff">"$1"</font><font color="#ff0000"> </font><b><font color="#ffffff">in</font></b> +<font color="#ff0000"> </font><font color="#bb00ff">""</font><font color="#F3E651">)</font> +<font color="#ff0000"> </font><i><font color="#ababab"># No argument - show current state</font></i> +<font color="#ff0000"> </font><font color="#ff0000">STATE</font><font color="#F3E651">=</font><font color="#ff0000">$(get_state)</font> +<font color="#ff0000"> echo </font><font color="#bb00ff">"CARP state on $CARP_IF (vhid $VHID): $STATE"</font> +<font color="#ff0000"> check_auto_failback</font> +<font color="#ff0000"> </font><font color="#F3E651">;;</font> +<font color="#ff0000"> master</font><font color="#F3E651">)</font> +<font color="#ff0000"> </font><i><font color="#ababab"># Force to MASTER state</font></i> +<font color="#ff0000"> echo </font><font color="#bb00ff">"Setting CARP to MASTER state..."</font> +<font color="#ff0000"> ifconfig </font><font color="#bb00ff">"$CARP_IF"</font><font color="#ff0000"> vhid </font><font color="#bb00ff">"$VHID"</font><font color="#ff0000"> state master</font> +<font color="#ff0000"> sleep </font><font color="#bb00ff">1</font> +<font color="#ff0000"> </font><font color="#ff0000">STATE</font><font color="#F3E651">=</font><font color="#ff0000">$(get_state)</font> +<font color="#ff0000"> echo </font><font color="#bb00ff">"CARP state on $CARP_IF (vhid $VHID): $STATE"</font> +<font color="#ff0000"> check_auto_failback</font> +<font color="#ff0000"> </font><font color="#F3E651">;;</font> +<font color="#ff0000"> backup</font><font color="#F3E651">)</font> +<font color="#ff0000"> </font><i><font color="#ababab"># Force to BACKUP state</font></i> +<font color="#ff0000"> echo </font><font color="#bb00ff">"Setting CARP to BACKUP state..."</font> +<font color="#ff0000"> ifconfig </font><font color="#bb00ff">"$CARP_IF"</font><font color="#ff0000"> vhid </font><font color="#bb00ff">"$VHID"</font><font color="#ff0000"> state backup</font> +<font color="#ff0000"> sleep </font><font color="#bb00ff">1</font> +<font color="#ff0000"> </font><font color="#ff0000">STATE</font><font color="#F3E651">=</font><font color="#ff0000">$(get_state)</font> +<font color="#ff0000"> echo </font><font color="#bb00ff">"CARP state on $CARP_IF (vhid $VHID): $STATE"</font> +<font color="#ff0000"> check_auto_failback</font> +<font color="#ff0000"> </font><font color="#F3E651">;;</font> +<font color="#ff0000"> auto-failback</font><font color="#F3E651">)</font> +<font color="#ff0000"> </font><b><font color="#ffffff">case</font></b><font color="#ff0000"> </font><font color="#bb00ff">"$2"</font><font color="#ff0000"> </font><b><font color="#ffffff">in</font></b> +<font color="#ff0000"> </font><b><font color="#ffffff">enable</font></b><font color="#F3E651">)</font> +<font color="#ff0000"> </font><b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">[</font><font color="#ff0000"> -f </font><font color="#bb00ff">"$BLOCK_FILE"</font><font color="#ff0000"> </font><font color="#F3E651">];</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b> +<font color="#ff0000"> rm </font><font color="#bb00ff">"$BLOCK_FILE"</font> +<font color="#ff0000"> echo </font><font color="#bb00ff">"Auto-failback ENABLED (removed $BLOCK_FILE)"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">else</font></b> +<font color="#ff0000"> echo </font><font color="#bb00ff">"Auto-failback was already enabled"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">fi</font></b> +<font color="#ff0000"> </font><font color="#F3E651">;;</font> +<font color="#ff0000"> disable</font><font color="#F3E651">)</font> +<font color="#ff0000"> </font><b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">[</font><font color="#ff0000"> </font><font color="#F3E651">!</font><font color="#ff0000"> -f </font><font color="#bb00ff">"$BLOCK_FILE"</font><font color="#ff0000"> </font><font color="#F3E651">];</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b> +<font color="#ff0000"> touch </font><font color="#bb00ff">"$BLOCK_FILE"</font> +<font color="#ff0000"> echo </font><font color="#bb00ff">"Auto-failback DISABLED (created $BLOCK_FILE)"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">else</font></b> +<font color="#ff0000"> echo </font><font color="#bb00ff">"Auto-failback was already disabled"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">fi</font></b> +<font color="#ff0000"> </font><font color="#F3E651">;;</font> +<font color="#ff0000"> </font><font color="#F3E651">*)</font> +<font color="#ff0000"> echo </font><font color="#bb00ff">"Usage: $0 auto-failback [enable|disable]"</font> +<font color="#ff0000"> echo </font><font color="#bb00ff">" enable: Remove block file to allow automatic failback"</font> +<font color="#ff0000"> echo </font><font color="#bb00ff">" disable: Create block file to prevent automatic failback"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">exit</font></b><font color="#ff0000"> </font><font color="#bb00ff">1</font> +<font color="#ff0000"> </font><font color="#F3E651">;;</font> +<font color="#ff0000"> </font><b><font color="#ffffff">esac</font></b> +<font color="#ff0000"> </font><font color="#F3E651">;;</font> +<font color="#ff0000"> </font><font color="#F3E651">*)</font> +<font color="#ff0000"> echo </font><font color="#bb00ff">"Usage: $0 [master|backup|auto-failback enable|auto-failback disable]"</font> +<font color="#ff0000"> echo </font><font color="#bb00ff">" Without arguments: show current CARP state"</font> +<font color="#ff0000"> echo </font><font color="#bb00ff">" master: force this node to become CARP MASTER"</font> +<font color="#ff0000"> echo </font><font color="#bb00ff">" backup: force this node to become CARP BACKUP"</font> +<font color="#ff0000"> echo </font><font color="#bb00ff">" auto-failback enable: allow automatic failback to f0"</font> +<font color="#ff0000"> echo </font><font color="#bb00ff">" auto-failback disable: prevent automatic failback to f0"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">exit</font></b><font color="#ff0000"> </font><font color="#bb00ff">1</font> +<font color="#ff0000"> </font><font color="#F3E651">;;</font> +<b><font color="#ffffff">esac</font></b> +<font color="#ff0000">EOF</font> + +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas chmod </font><font color="#F3E651">+</font><font color="#ff0000">x /usr/local/bin/carp</font> + +<i><font color="#ababab"># Copy to f1 as well</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> scp /usr/local/bin/carp f1</font><font color="#F3E651">:</font><font color="#ff0000">/tmp</font><font color="#F3E651">/</font> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas cp /tmp/carp /usr/local/bin/carp </font><font color="#F3E651">&&</font><font color="#ff0000"> doas chmod </font><font color="#F3E651">+</font><font color="#ff0000">x /usr/local/bin/carp</font> </pre> <br /> <span>Now you can easily manage CARP states and auto-failback:</span><br /> @@ -1549,27 +1554,27 @@ paul@f1:~ % doas cp /tmp/carp /usr/local/bin/carp && doas chmod +x /usr/ by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># Check current CARP state</font></i> -paul@f0:~ % doas carp -CARP state on re0 (vhid <font color="#000000">1</font>): MASTER +<pre><i><font color="#ababab"># Check current CARP state</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas carp</font> +<font color="#ff0000">CARP state on re0 </font><font color="#F3E651">(</font><font color="#ff0000">vhid </font><font color="#bb00ff">1</font><font color="#F3E651">):</font><font color="#ff0000"> MASTER</font> -<i><font color="silver"># If auto-failback is disabled, you'll see a warning</font></i> -paul@f0:~ % doas carp -CARP state on re0 (vhid <font color="#000000">1</font>): MASTER -WARNING: Auto-failback is DISABLED (file exists: /data/nfs/nfs.NO_AUTO_FAILBACK) +<i><font color="#ababab"># If auto-failback is disabled, you'll see a warning</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas carp</font> +<font color="#ff0000">CARP state on re0 </font><font color="#F3E651">(</font><font color="#ff0000">vhid </font><font color="#bb00ff">1</font><font color="#F3E651">):</font><font color="#ff0000"> MASTER</font> +<font color="#ff0000">WARNING</font><font color="#F3E651">:</font><font color="#ff0000"> Auto-failback is DISABLED </font><font color="#F3E651">(</font><font color="#ff0000">file exists</font><font color="#F3E651">:</font><font color="#ff0000"> /data/nfs/nfs</font><font color="#F3E651">.</font><font color="#ff0000">NO_AUTO_FAILBACK</font><font color="#F3E651">)</font> -<i><font color="silver"># Force f0 to become BACKUP (triggers failover to f1)</font></i> -paul@f0:~ % doas carp backup -Setting CARP to BACKUP state... -CARP state on re0 (vhid <font color="#000000">1</font>): BACKUP +<i><font color="#ababab"># Force f0 to become BACKUP (triggers failover to f1)</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas carp backup</font> +<font color="#ff0000">Setting CARP to BACKUP state</font><font color="#F3E651">...</font> +<font color="#ff0000">CARP state on re0 </font><font color="#F3E651">(</font><font color="#ff0000">vhid </font><font color="#bb00ff">1</font><font color="#F3E651">):</font><font color="#ff0000"> BACKUP</font> -<i><font color="silver"># Disable auto-failback (useful for maintenance)</font></i> -paul@f0:~ % doas carp auto-failback disable -Auto-failback DISABLED (created /data/nfs/nfs.NO_AUTO_FAILBACK) +<i><font color="#ababab"># Disable auto-failback (useful for maintenance)</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas carp auto-failback disable</font> +<font color="#ff0000">Auto-failback DISABLED </font><font color="#F3E651">(</font><font color="#ff0000">created /data/nfs/nfs</font><font color="#F3E651">.</font><font color="#ff0000">NO_AUTO_FAILBACK</font><font color="#F3E651">)</font> -<i><font color="silver"># Enable auto-failback</font></i> -paul@f0:~ % doas carp auto-failback <b><u><font color="#000000">enable</font></u></b> -Auto-failback ENABLED (removed /data/nfs/nfs.NO_AUTO_FAILBACK) +<i><font color="#ababab"># Enable auto-failback</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas carp auto-failback </font><b><font color="#ffffff">enable</font></b> +<font color="#ff0000">Auto-failback ENABLED </font><font color="#F3E651">(</font><font color="#ff0000">removed /data/nfs/nfs</font><font color="#F3E651">.</font><font color="#ff0000">NO_AUTO_FAILBACK</font><font color="#F3E651">)</font> </pre> <br /> <h3 style='display: inline' id='automatic-failback-after-reboot'>Automatic Failback After Reboot</h3><br /> @@ -1582,60 +1587,60 @@ Auto-failback ENABLED (removed /data/nfs/nfs.NO_AUTO_FAILBACK) by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:~ % doas tee /usr/local/bin/carp-auto-failback.sh <<<font color="#808080">'EOF'</font> -<i><font color="silver">#!/bin/sh</font></i> -<i><font color="silver"># CARP automatic failback script for f0</font></i> -<i><font color="silver"># Ensures f0 reclaims MASTER role after reboot when storage is ready</font></i> +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas tee /usr/local/bin/carp-auto-failback</font><font color="#F3E651">.</font><font color="#ff0000">sh </font><font color="#F3E651"><<</font><font color="#bb00ff">'EOF'</font> +<i><font color="#ababab">#!/bin/sh</font></i> +<i><font color="#ababab"># CARP automatic failback script for f0</font></i> +<i><font color="#ababab"># Ensures f0 reclaims MASTER role after reboot when storage is ready</font></i> -LOGFILE=<font color="#808080">"/var/log/carp-auto-failback.log"</font> -MARKER_FILE=<font color="#808080">"/data/nfs/nfs.DO_NOT_REMOVE"</font> -BLOCK_FILE=<font color="#808080">"/data/nfs/nfs.NO_AUTO_FAILBACK"</font> +<font color="#ff0000">LOGFILE</font><font color="#F3E651">=</font><font color="#bb00ff">"/var/log/carp-auto-failback.log"</font> +<font color="#ff0000">MARKER_FILE</font><font color="#F3E651">=</font><font color="#bb00ff">"/data/nfs/nfs.DO_NOT_REMOVE"</font> +<font color="#ff0000">BLOCK_FILE</font><font color="#F3E651">=</font><font color="#bb00ff">"/data/nfs/nfs.NO_AUTO_FAILBACK"</font> -log_message() { - echo <font color="#808080">"$(date '+%Y-%m-%d %H:%M:%S') - $1"</font> >> <font color="#808080">"$LOGFILE"</font> -} +<font color="#7bc710">log_message()</font><font color="#ff0000"> {</font> +<font color="#ff0000"> echo </font><font color="#bb00ff">"$(date '+%Y-%m-%d %H:%M:%S') - $1"</font><font color="#ff0000"> </font><font color="#F3E651">>></font><font color="#ff0000"> </font><font color="#bb00ff">"$LOGFILE"</font> +<font color="#ff0000">}</font> -<i><font color="silver"># Check if we're already MASTER</font></i> -CURRENT_STATE=$(/usr/local/bin/carp | awk <font color="#808080">'{print $NF}'</font>) -<b><u><font color="#000000">if</font></u></b> [ <font color="#808080">"$CURRENT_STATE"</font> = <font color="#808080">"MASTER"</font> ]; <b><u><font color="#000000">then</font></u></b> - <b><u><font color="#000000">exit</font></u></b> <font color="#000000">0</font> -<b><u><font color="#000000">fi</font></u></b> +<i><font color="#ababab"># Check if we're already MASTER</font></i> +<font color="#ff0000">CURRENT_STATE</font><font color="#F3E651">=</font><font color="#ff0000">$(</font><font color="#ff0000">/usr/local/bin/carp </font><font color="#F3E651">|</font><font color="#ff0000"> awk </font><font color="#bb00ff">'{print $NF}'</font><font color="#F3E651">)</font> +<b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">[</font><font color="#ff0000"> </font><font color="#bb00ff">"$CURRENT_STATE"</font><font color="#ff0000"> </font><font color="#F3E651">=</font><font color="#ff0000"> </font><font color="#bb00ff">"MASTER"</font><font color="#ff0000"> </font><font color="#F3E651">];</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b> +<font color="#ff0000"> </font><b><font color="#ffffff">exit</font></b><font color="#ff0000"> </font><font color="#bb00ff">0</font> +<b><font color="#ffffff">fi</font></b> -<i><font color="silver"># Check if /data/nfs is mounted</font></i> -<b><u><font color="#000000">if</font></u></b> ! mount | grep -q <font color="#808080">"on /data/nfs "</font>; <b><u><font color="#000000">then</font></u></b> - log_message <font color="#808080">"SKIP: /data/nfs not mounted"</font> - <b><u><font color="#000000">exit</font></u></b> <font color="#000000">0</font> -<b><u><font color="#000000">fi</font></u></b> +<i><font color="#ababab"># Check if /data/nfs is mounted</font></i> +<b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">!</font><font color="#ff0000"> mount </font><font color="#F3E651">|</font><font color="#ff0000"> grep -q </font><font color="#bb00ff">"on /data/nfs "</font><font color="#F3E651">;</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b> +<font color="#ff0000"> log_message </font><font color="#bb00ff">"SKIP: /data/nfs not mounted"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">exit</font></b><font color="#ff0000"> </font><font color="#bb00ff">0</font> +<b><font color="#ffffff">fi</font></b> -<i><font color="silver"># Check if the marker file exists</font></i> -<i><font color="silver"># (identifies that the ZFS data set is properly mounted)</font></i> -<b><u><font color="#000000">if</font></u></b> [ ! -f <font color="#808080">"$MARKER_FILE"</font> ]; <b><u><font color="#000000">then</font></u></b> - log_message <font color="#808080">"SKIP: Marker file $MARKER_FILE not found"</font> - <b><u><font color="#000000">exit</font></u></b> <font color="#000000">0</font> -<b><u><font color="#000000">fi</font></u></b> +<i><font color="#ababab"># Check if the marker file exists</font></i> +<i><font color="#ababab"># (identifies that the ZFS data set is properly mounted)</font></i> +<b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">[</font><font color="#ff0000"> </font><font color="#F3E651">!</font><font color="#ff0000"> -f </font><font color="#bb00ff">"$MARKER_FILE"</font><font color="#ff0000"> </font><font color="#F3E651">];</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b> +<font color="#ff0000"> log_message </font><font color="#bb00ff">"SKIP: Marker file $MARKER_FILE not found"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">exit</font></b><font color="#ff0000"> </font><font color="#bb00ff">0</font> +<b><font color="#ffffff">fi</font></b> -<i><font color="silver"># Check if failback is blocked (for maintenance)</font></i> -<b><u><font color="#000000">if</font></u></b> [ -f <font color="#808080">"$BLOCK_FILE"</font> ]; <b><u><font color="#000000">then</font></u></b> - log_message <font color="#808080">"SKIP: Failback blocked by $BLOCK_FILE"</font> - <b><u><font color="#000000">exit</font></u></b> <font color="#000000">0</font> -<b><u><font color="#000000">fi</font></u></b> +<i><font color="#ababab"># Check if failback is blocked (for maintenance)</font></i> +<b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">[</font><font color="#ff0000"> -f </font><font color="#bb00ff">"$BLOCK_FILE"</font><font color="#ff0000"> </font><font color="#F3E651">];</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b> +<font color="#ff0000"> log_message </font><font color="#bb00ff">"SKIP: Failback blocked by $BLOCK_FILE"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">exit</font></b><font color="#ff0000"> </font><font color="#bb00ff">0</font> +<b><font color="#ffffff">fi</font></b> -<i><font color="silver"># All conditions met - promote to MASTER</font></i> -log_message <font color="#808080">"CONDITIONS MET: Promoting to MASTER (was $CURRENT_STATE)"</font> -/usr/local/bin/carp master +<i><font color="#ababab"># All conditions met - promote to MASTER</font></i> +<font color="#ff0000">log_message </font><font color="#bb00ff">"CONDITIONS MET: Promoting to MASTER (was $CURRENT_STATE)"</font> +<font color="#ff0000">/usr/local/bin/carp master</font> -<i><font color="silver"># Log result</font></i> -sleep <font color="#000000">2</font> -NEW_STATE=$(/usr/local/bin/carp | awk <font color="#808080">'{print $NF}'</font>) -log_message <font color="#808080">"Failback complete: State is now $NEW_STATE"</font> +<i><font color="#ababab"># Log result</font></i> +<font color="#ff0000">sleep </font><font color="#bb00ff">2</font> +<font color="#ff0000">NEW_STATE</font><font color="#F3E651">=</font><font color="#ff0000">$(</font><font color="#ff0000">/usr/local/bin/carp </font><font color="#F3E651">|</font><font color="#ff0000"> awk </font><font color="#bb00ff">'{print $NF}'</font><font color="#F3E651">)</font> +<font color="#ff0000">log_message </font><font color="#bb00ff">"Failback complete: State is now $NEW_STATE"</font> -<i><font color="silver"># If successful, log to the system log too</font></i> -<b><u><font color="#000000">if</font></u></b> [ <font color="#808080">"$NEW_STATE"</font> = <font color="#808080">"MASTER"</font> ]; <b><u><font color="#000000">then</font></u></b> - logger <font color="#808080">"CARP: f0 automatically reclaimed MASTER role"</font> -<b><u><font color="#000000">fi</font></u></b> -EOF +<i><font color="#ababab"># If successful, log to the system log too</font></i> +<b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">[</font><font color="#ff0000"> </font><font color="#bb00ff">"$NEW_STATE"</font><font color="#ff0000"> </font><font color="#F3E651">=</font><font color="#ff0000"> </font><font color="#bb00ff">"MASTER"</font><font color="#ff0000"> </font><font color="#F3E651">];</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b> +<font color="#ff0000"> logger </font><font color="#bb00ff">"CARP: f0 automatically reclaimed MASTER role"</font> +<b><font color="#ffffff">fi</font></b> +<font color="#ff0000">EOF</font> -paul@f0:~ % doas chmod +x /usr/local/bin/carp-auto-failback.sh +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas chmod </font><font color="#F3E651">+</font><font color="#ff0000">x /usr/local/bin/carp-auto-failback</font><font color="#F3E651">.</font><font color="#ff0000">sh</font> </pre> <br /> <span>The marker file identifies that the ZFS data set is mounted correctly. We create it with:</span><br /> @@ -1644,7 +1649,7 @@ paul@f0:~ % doas chmod +x /usr/local/bin/carp-auto-failback.sh by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:~ % doas touch /data/nfs/nfs.DO_NOT_REMOVE +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas touch /data/nfs/nfs</font><font color="#F3E651">.</font><font color="#ff0000">DO_NOT_REMOVE</font> </pre> <br /> <span>We add a cron job to check every minute:</span><br /> @@ -1653,7 +1658,7 @@ http://www.gnu.org/software/src-highlite --> by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:~ % echo <font color="#808080">"* * * * * /usr/local/bin/carp-auto-failback.sh"</font> | doas crontab - +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> echo </font><font color="#bb00ff">"* * * * * /usr/local/bin/carp-auto-failback.sh"</font><font color="#ff0000"> </font><font color="#F3E651">|</font><font color="#ff0000"> doas crontab -</font> </pre> <br /> <span>The enhanced CARP script provides integrated control over auto-failback. To temporarily turn off automatic failback (e.g., for <span class='inlinecode'>f0</span> maintenance), we run:</span><br /> @@ -1662,8 +1667,8 @@ http://www.gnu.org/software/src-highlite --> by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:~ % doas carp auto-failback disable -Auto-failback DISABLED (created /data/nfs/nfs.NO_AUTO_FAILBACK) +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas carp auto-failback disable</font> +<font color="#ff0000">Auto-failback DISABLED </font><font color="#F3E651">(</font><font color="#ff0000">created /data/nfs/nfs</font><font color="#F3E651">.</font><font color="#ff0000">NO_AUTO_FAILBACK</font><font color="#F3E651">)</font> </pre> <br /> <span>And to re-enable it:</span><br /> @@ -1672,8 +1677,8 @@ Auto-failback DISABLED (created /data/nfs/nfs.NO_AUTO_FAILBACK) by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:~ % doas carp auto-failback <b><u><font color="#000000">enable</font></u></b> -Auto-failback ENABLED (removed /data/nfs/nfs.NO_AUTO_FAILBACK) +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas carp auto-failback </font><b><font color="#ffffff">enable</font></b> +<font color="#ff0000">Auto-failback ENABLED </font><font color="#F3E651">(</font><font color="#ff0000">removed /data/nfs/nfs</font><font color="#F3E651">.</font><font color="#ff0000">NO_AUTO_FAILBACK</font><font color="#F3E651">)</font> </pre> <br /> <span>To check whether auto-failback is enabled, we run:</span><br /> @@ -1682,9 +1687,9 @@ Auto-failback ENABLED (removed /data/nfs/nfs.NO_AUTO_FAILBACK) by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:~ % doas carp -CARP state on re0 (vhid <font color="#000000">1</font>): MASTER -<i><font color="silver"># If disabled, you'll see: WARNING: Auto-failback is DISABLED</font></i> +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas carp</font> +<font color="#ff0000">CARP state on re0 </font><font color="#F3E651">(</font><font color="#ff0000">vhid </font><font color="#bb00ff">1</font><font color="#F3E651">):</font><font color="#ff0000"> MASTER</font> +<i><font color="#ababab"># If disabled, you'll see: WARNING: Auto-failback is DISABLED</font></i> </pre> <br /> <span>The failback attempts are logged to <span class='inlinecode'>/var/log/carp-auto-failback.log</span>!</span><br /> @@ -1711,29 +1716,29 @@ CARP state on re0 (vhid <font color="#000000">1</font>): MASTER by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># Install stunnel on client (example for `r0`)</font></i> -[root@r0 ~]<i><font color="silver"># dnf install -y stunnel nfs-utils</font></i> +<pre><i><font color="#ababab"># Install stunnel on client (example for `r0`)</font></i> +<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># dnf install -y stunnel nfs-utils</font></i> -<i><font color="silver"># Copy client certificate and CA certificate from f0</font></i> -[root@r0 ~]<i><font color="silver"># scp f0:/usr/local/etc/stunnel/ca/r0-stunnel.pem /etc/stunnel/</font></i> -[root@r0 ~]<i><font color="silver"># scp f0:/usr/local/etc/stunnel/ca/ca-cert.pem /etc/stunnel/</font></i> +<i><font color="#ababab"># Copy client certificate and CA certificate from f0</font></i> +<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># scp f0:/usr/local/etc/stunnel/ca/r0-stunnel.pem /etc/stunnel/</font></i> +<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># scp f0:/usr/local/etc/stunnel/ca/ca-cert.pem /etc/stunnel/</font></i> -<i><font color="silver"># Configure stunnel client with certificate authentication</font></i> -[root@r0 ~]<i><font color="silver"># tee /etc/stunnel/stunnel.conf <<'EOF'</font></i> -cert = /etc/stunnel/r<font color="#000000">0</font>-stunnel.pem -CAfile = /etc/stunnel/ca-cert.pem -client = yes -verify = <font color="#000000">2</font> +<i><font color="#ababab"># Configure stunnel client with certificate authentication</font></i> +<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># tee /etc/stunnel/stunnel.conf <<'EOF'</font></i> +<font color="#ff0000">cert </font><font color="#F3E651">=</font><font color="#ff0000"> /etc/stunnel/r</font><font color="#bb00ff">0</font><font color="#ff0000">-stunnel</font><font color="#F3E651">.</font><font color="#ff0000">pem</font> +<font color="#ff0000">CAfile </font><font color="#F3E651">=</font><font color="#ff0000"> /etc/stunnel/ca-cert</font><font color="#F3E651">.</font><font color="#ff0000">pem</font> +<font color="#ff0000">client </font><font color="#F3E651">=</font><font color="#ff0000"> yes</font> +<font color="#ff0000">verify </font><font color="#F3E651">=</font><font color="#ff0000"> </font><font color="#bb00ff">2</font> -[nfs-ha] -accept = <font color="#000000">127.0</font>.<font color="#000000">0.1</font>:<font color="#000000">2323</font> -connect = <font color="#000000">192.168</font>.<font color="#000000">1.138</font>:<font color="#000000">2323</font> -EOF +<font color="#F3E651">[</font><font color="#ff0000">nfs-ha</font><font color="#F3E651">]</font> +<font color="#ff0000">accept </font><font color="#F3E651">=</font><font color="#ff0000"> </font><font color="#bb00ff">127.0</font><font color="#F3E651">.</font><font color="#bb00ff">0.1</font><font color="#F3E651">:</font><font color="#bb00ff">2323</font> +<font color="#ff0000">connect </font><font color="#F3E651">=</font><font color="#ff0000"> </font><font color="#bb00ff">192.168</font><font color="#F3E651">.</font><font color="#bb00ff">1.138</font><font color="#F3E651">:</font><font color="#bb00ff">2323</font> +<font color="#ff0000">EOF</font> -<i><font color="silver"># Enable and start stunnel</font></i> -[root@r0 ~]<i><font color="silver"># systemctl enable --now stunnel</font></i> +<i><font color="#ababab"># Enable and start stunnel</font></i> +<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># systemctl enable --now stunnel</font></i> -<i><font color="silver"># Repeat for r1 and r2 with their respective certificates</font></i> +<i><font color="#ababab"># Repeat for r1 and r2 with their respective certificates</font></i> </pre> <br /> <span>Note: Each client must use its certificate file (<span class='inlinecode'>r0-stunnel.pem</span>, <span class='inlinecode'>r1-stunnel.pem</span>, <span class='inlinecode'>r2-stunnel.pem</span>, or <span class='inlinecode'>earth-stunnel.pem</span> - the latter is for my Laptop, which can also mount the NFS shares).</span><br /> @@ -1759,8 +1764,8 @@ Domain = lan.buetow.org by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>[root@r0 ~]<i><font color="silver"># echo 'fs.inotify.max_user_instances = 512' > /etc/sysctl.d/99-inotify.conf</font></i> -[root@r0 ~]<i><font color="silver"># sysctl -w fs.inotify.max_user_instances=512</font></i> +<pre><font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># echo 'fs.inotify.max_user_instances = 512' > /etc/sysctl.d/99-inotify.conf</font></i> +<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># sysctl -w fs.inotify.max_user_instances=512</font></i> </pre> <br /> <span>And afterwards, we need to run the following on all 3 Rocky hosts:</span><br /> @@ -1769,8 +1774,8 @@ http://www.gnu.org/software/src-highlite --> by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>[root@r0 ~]<i><font color="silver"># systemctl start nfs-idmapd</font></i> -[root@r0 ~]<i><font color="silver"># systemctl enable --now nfs-client.target</font></i> +<pre><font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># systemctl start nfs-idmapd</font></i> +<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># systemctl enable --now nfs-client.target</font></i> </pre> <br /> <span>and then, safest, reboot those.</span><br /> @@ -1783,21 +1788,21 @@ http://www.gnu.org/software/src-highlite --> by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># Create a mount point</font></i> -[root@r0 ~]<i><font color="silver"># mkdir -p /data/nfs/k3svolumes</font></i> +<pre><i><font color="#ababab"># Create a mount point</font></i> +<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># mkdir -p /data/nfs/k3svolumes</font></i> -<i><font color="silver"># Mount through stunnel (using localhost and NFSv4)</font></i> -[root@r0 ~]<i><font color="silver"># mount -t nfs4 -o port=2323 127.0.0.1:/k3svolumes /data/nfs/k3svolumes</font></i> +<i><font color="#ababab"># Mount through stunnel (using localhost and NFSv4)</font></i> +<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># mount -t nfs4 -o port=2323 127.0.0.1:/k3svolumes /data/nfs/k3svolumes</font></i> -<i><font color="silver"># Verify mount</font></i> -[root@r0 ~]<i><font color="silver"># mount | grep k3svolumes</font></i> -<font color="#000000">127.0</font>.<font color="#000000">0.1</font>:/k3svolumes on /data/nfs/k3svolumes - <b><u><font color="#000000">type</font></u></b> nfs4 (rw,relatime,vers=<font color="#000000">4.2</font>,rsize=<font color="#000000">131072</font>,wsize=<font color="#000000">131072</font>, - namlen=<font color="#000000">255</font>,hard,proto=tcp,port=<font color="#000000">2323</font>,timeo=<font color="#000000">600</font>,retrans=<font color="#000000">2</font>,sec=sys, - clientaddr=<font color="#000000">127.0</font>.<font color="#000000">0.1</font>,local_lock=none,addr=<font color="#000000">127.0</font>.<font color="#000000">0.1</font>) +<i><font color="#ababab"># Verify mount</font></i> +<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># mount | grep k3svolumes</font></i> +<font color="#bb00ff">127.0</font><font color="#F3E651">.</font><font color="#bb00ff">0.1</font><font color="#F3E651">:</font><font color="#ff0000">/k3svolumes on /data/nfs/k3svolumes </font> +<font color="#ff0000"> </font><b><font color="#ffffff">type</font></b><font color="#ff0000"> nfs4 </font><font color="#F3E651">(</font><font color="#ff0000">rw</font><font color="#F3E651">,</font><font color="#ff0000">relatime</font><font color="#F3E651">,</font><font color="#ff0000">vers</font><font color="#F3E651">=</font><font color="#bb00ff">4.2</font><font color="#F3E651">,</font><font color="#ff0000">rsize</font><font color="#F3E651">=</font><font color="#bb00ff">131072</font><font color="#F3E651">,</font><font color="#ff0000">wsize</font><font color="#F3E651">=</font><font color="#bb00ff">131072</font><font color="#F3E651">,</font> +<font color="#ff0000"> </font><font color="#ff0000">namlen</font><font color="#F3E651">=</font><font color="#bb00ff">255</font><font color="#F3E651">,</font><font color="#ff0000">hard</font><font color="#F3E651">,</font><font color="#ff0000">proto</font><font color="#F3E651">=</font><font color="#ff0000">tcp</font><font color="#F3E651">,</font><font color="#ff0000">port</font><font color="#F3E651">=</font><font color="#bb00ff">2323</font><font color="#F3E651">,</font><font color="#ff0000">timeo</font><font color="#F3E651">=</font><font color="#bb00ff">600</font><font color="#F3E651">,</font><font color="#ff0000">retrans</font><font color="#F3E651">=</font><font color="#bb00ff">2</font><font color="#F3E651">,</font><font color="#ff0000">sec</font><font color="#F3E651">=</font><font color="#ff0000">sys</font><font color="#F3E651">,</font> +<font color="#ff0000"> </font><font color="#ff0000">clientaddr</font><font color="#F3E651">=</font><font color="#bb00ff">127.0</font><font color="#F3E651">.</font><font color="#bb00ff">0.1</font><font color="#F3E651">,</font><font color="#ff0000">local_lock</font><font color="#F3E651">=</font><font color="#ff0000">none</font><font color="#F3E651">,</font><font color="#ff0000">addr</font><font color="#F3E651">=</font><font color="#bb00ff">127.0</font><font color="#F3E651">.</font><font color="#bb00ff">0.1</font><font color="#F3E651">)</font> -<i><font color="silver"># For persistent mount, add to /etc/fstab:</font></i> -<font color="#000000">127.0</font>.<font color="#000000">0.1</font>:/k3svolumes /data/nfs/k3svolumes nfs4 port=<font color="#000000">2323</font>,_netdev,soft,timeo=<font color="#000000">10</font>,retrans=<font color="#000000">2</font>,intr <font color="#000000">0</font> <font color="#000000">0</font> +<i><font color="#ababab"># For persistent mount, add to /etc/fstab:</font></i> +<font color="#bb00ff">127.0</font><font color="#F3E651">.</font><font color="#bb00ff">0.1</font><font color="#F3E651">:</font><font color="#ff0000">/k3svolumes /data/nfs/k3svolumes nfs4 </font><font color="#ff0000">port</font><font color="#F3E651">=</font><font color="#bb00ff">2323</font><font color="#F3E651">,</font><font color="#ff0000">_netdev</font><font color="#F3E651">,</font><font color="#ff0000">soft</font><font color="#F3E651">,</font><font color="#ff0000">timeo</font><font color="#F3E651">=</font><font color="#bb00ff">10</font><font color="#F3E651">,</font><font color="#ff0000">retrans</font><font color="#F3E651">=</font><font color="#bb00ff">2</font><font color="#F3E651">,</font><font color="#ff0000">intr </font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#bb00ff">0</font> </pre> <br /> <span>Note: The mount uses localhost (<span class='inlinecode'>127.0.0.1</span>) because stunnel is listening locally and forwarding the encrypted traffic to the remote server.</span><br /> @@ -1810,20 +1815,20 @@ http://www.gnu.org/software/src-highlite --> by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># On f0 (current MASTER) - trigger failover</font></i> -paul@f0:~ % doas ifconfig re0 vhid <font color="#000000">1</font> state backup +<pre><i><font color="#ababab"># On f0 (current MASTER) - trigger failover</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas ifconfig re0 vhid </font><font color="#bb00ff">1</font><font color="#ff0000"> state backup</font> -<i><font color="silver"># On f1 - verify it becomes MASTER</font></i> -paul@f1:~ % ifconfig re0 | grep carp - inet <font color="#000000">192.168</font>.<font color="#000000">1.138</font> netmask <font color="#000000">0xffffffff</font> broadcast <font color="#000000">192.168</font>.<font color="#000000">1.138</font> vhid <font color="#000000">1</font> +<i><font color="#ababab"># On f1 - verify it becomes MASTER</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> ifconfig re0 </font><font color="#F3E651">|</font><font color="#ff0000"> grep carp</font> +<font color="#ff0000"> inet </font><font color="#bb00ff">192.168</font><font color="#F3E651">.</font><font color="#bb00ff">1.138</font><font color="#ff0000"> netmask </font><font color="#bb00ff">0xffffffff</font><font color="#ff0000"> broadcast </font><font color="#bb00ff">192.168</font><font color="#F3E651">.</font><font color="#bb00ff">1.138</font><font color="#ff0000"> vhid </font><font color="#bb00ff">1</font> -<i><font color="silver"># Check stunnel is now listening on f1</font></i> -paul@f1:~ % doas sockstat -l | grep <font color="#000000">2323</font> -stunnel stunnel <font color="#000000">4567</font> <font color="#000000">3</font> tcp4 <font color="#000000">192.168</font>.<font color="#000000">1.138</font>:<font color="#000000">2323</font> *:* +<i><font color="#ababab"># Check stunnel is now listening on f1</font></i> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas sockstat -l </font><font color="#F3E651">|</font><font color="#ff0000"> grep </font><font color="#bb00ff">2323</font> +<font color="#ff0000">stunnel stunnel </font><font color="#bb00ff">4567</font><font color="#ff0000"> </font><font color="#bb00ff">3</font><font color="#ff0000"> tcp4 </font><font color="#bb00ff">192.168</font><font color="#F3E651">.</font><font color="#bb00ff">1.138</font><font color="#F3E651">:</font><font color="#bb00ff">2323</font><font color="#ff0000"> </font><font color="#F3E651">*:*</font> -<i><font color="silver"># On client - verify NFS mount still works</font></i> -[root@r0 ~]<i><font color="silver"># ls /data/nfs/k3svolumes/</font></i> -[root@r0 ~]<i><font color="silver"># echo "Test after failover" > /data/nfs/k3svolumes/failover-test.txt</font></i> +<i><font color="#ababab"># On client - verify NFS mount still works</font></i> +<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># ls /data/nfs/k3svolumes/</font></i> +<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># echo "Test after failover" > /data/nfs/k3svolumes/failover-test.txt</font></i> </pre> <br /> <span>After a CARP failover, NFS clients may experience "Stale file handle" errors because they cached file handles from the previous server. To resolve this manually, we can run:</span><br /> @@ -1832,9 +1837,9 @@ stunnel stunnel <font color="#000000">4567</font> <font color="#000000">3</ by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># Force unmount and remount</font></i> -[root@r0 ~]<i><font color="silver"># umount -f /data/nfs/k3svolumes</font></i> -[root@r0 ~]<i><font color="silver"># mount /data/nfs/k3svolumes</font></i> +<pre><i><font color="#ababab"># Force unmount and remount</font></i> +<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># umount -f /data/nfs/k3svolumes</font></i> +<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># mount /data/nfs/k3svolumes</font></i> </pre> <br /> <span>For the automatic recovery, we create a script:</span><br /> @@ -1843,72 +1848,72 @@ http://www.gnu.org/software/src-highlite --> by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>[root@r0 ~]<i><font color="silver"># cat > /usr/local/bin/check-nfs-mount.sh << 'EOF'</font></i> -<i><font color="silver">#!/bin/bash</font></i> -<i><font color="silver"># Fast NFS mount health monitor - runs every 10 seconds via systemd timer</font></i> - -MOUNT_POINT=<font color="#808080">"/data/nfs/k3svolumes"</font> -LOCK_FILE=<font color="#808080">"/var/run/nfs-mount-check.lock"</font> - -<i><font color="silver"># Use a lock file to prevent concurrent runs</font></i> -<b><u><font color="#000000">if</font></u></b> [ -f <font color="#808080">"$LOCK_FILE"</font> ]; <b><u><font color="#000000">then</font></u></b> - <b><u><font color="#000000">exit</font></u></b> <font color="#000000">0</font> -<b><u><font color="#000000">fi</font></u></b> -touch <font color="#808080">"$LOCK_FILE"</font> -<b><u><font color="#000000">trap</font></u></b> <font color="#808080">"rm -f $LOCK_FILE"</font> EXIT - -fix_mount () { - echo <font color="#808080">"Attempting to remount NFS mount $MOUNT_POINT"</font> - <b><u><font color="#000000">if</font></u></b> mount -o remount -f <font color="#808080">"$MOUNT_POINT"</font> <font color="#000000">2</font>>/dev/null; <b><u><font color="#000000">then</font></u></b> - echo <font color="#808080">"Remount command issued for $MOUNT_POINT"</font> - <b><u><font color="#000000">else</font></u></b> - echo <font color="#808080">"Failed to remount NFS mount $MOUNT_POINT"</font> - <b><u><font color="#000000">fi</font></u></b> - - echo <font color="#808080">"Checking if $MOUNT_POINT is a mountpoint"</font> - <b><u><font color="#000000">if</font></u></b> mountpoint <font color="#808080">"$MOUNT_POINT"</font> >/dev/null <font color="#000000">2</font>>&<font color="#000000">1</font>; <b><u><font color="#000000">then</font></u></b> - echo <font color="#808080">"$MOUNT_POINT is a valid mountpoint"</font> - <b><u><font color="#000000">else</font></u></b> - echo <font color="#808080">"$MOUNT_POINT is not a valid mountpoint, attempting mount"</font> - <b><u><font color="#000000">if</font></u></b> mount <font color="#808080">"$MOUNT_POINT"</font>; <b><u><font color="#000000">then</font></u></b> - echo <font color="#808080">"Successfully mounted $MOUNT_POINT"</font> - <b><u><font color="#000000">return</font></u></b> - <b><u><font color="#000000">else</font></u></b> - echo <font color="#808080">"Failed to mount $MOUNT_POINT"</font> - <b><u><font color="#000000">fi</font></u></b> - <b><u><font color="#000000">fi</font></u></b> - - echo <font color="#808080">"Attempting to unmount $MOUNT_POINT"</font> - <b><u><font color="#000000">if</font></u></b> umount -f <font color="#808080">"$MOUNT_POINT"</font> <font color="#000000">2</font>>/dev/null; <b><u><font color="#000000">then</font></u></b> - echo <font color="#808080">"Successfully unmounted $MOUNT_POINT"</font> - <b><u><font color="#000000">else</font></u></b> - echo <font color="#808080">"Failed to unmount $MOUNT_POINT (it might not be mounted)"</font> - <b><u><font color="#000000">fi</font></u></b> - - echo <font color="#808080">"Attempting to mount $MOUNT_POINT"</font> - <b><u><font color="#000000">if</font></u></b> mount <font color="#808080">"$MOUNT_POINT"</font>; <b><u><font color="#000000">then</font></u></b> - echo <font color="#808080">"NFS mount $MOUNT_POINT mounted successfully"</font> - <b><u><font color="#000000">return</font></u></b> - <b><u><font color="#000000">else</font></u></b> - echo <font color="#808080">"Failed to mount NFS mount $MOUNT_POINT"</font> - <b><u><font color="#000000">fi</font></u></b> - - echo <font color="#808080">"Failed to fix NFS mount $MOUNT_POINT"</font> - <b><u><font color="#000000">exit</font></u></b> <font color="#000000">1</font> -} - -<b><u><font color="#000000">if</font></u></b> ! mountpoint <font color="#808080">"$MOUNT_POINT"</font> >/dev/null <font color="#000000">2</font>>&<font color="#000000">1</font>; <b><u><font color="#000000">then</font></u></b> - echo <font color="#808080">"NFS mount $MOUNT_POINT not found"</font> - fix_mount -<b><u><font color="#000000">fi</font></u></b> - -<b><u><font color="#000000">if</font></u></b> ! timeout 2s stat <font color="#808080">"$MOUNT_POINT"</font> >/dev/null <font color="#000000">2</font>>&<font color="#000000">1</font>; <b><u><font color="#000000">then</font></u></b> - echo <font color="#808080">"NFS mount $MOUNT_POINT appears to be unresponsive"</font> - fix_mount -<b><u><font color="#000000">fi</font></u></b> -EOF - -[root@r0 ~]<i><font color="silver"># chmod +x /usr/local/bin/check-nfs-mount.sh</font></i> +<pre><font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># cat > /usr/local/bin/check-nfs-mount.sh << 'EOF'</font></i> +<i><font color="#ababab">#!/bin/bash</font></i> +<i><font color="#ababab"># Fast NFS mount health monitor - runs every 10 seconds via systemd timer</font></i> + +<font color="#ff0000">MOUNT_POINT</font><font color="#F3E651">=</font><font color="#bb00ff">"/data/nfs/k3svolumes"</font> +<font color="#ff0000">LOCK_FILE</font><font color="#F3E651">=</font><font color="#bb00ff">"/var/run/nfs-mount-check.lock"</font> + +<i><font color="#ababab"># Use a lock file to prevent concurrent runs</font></i> +<b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">[</font><font color="#ff0000"> -f </font><font color="#bb00ff">"$LOCK_FILE"</font><font color="#ff0000"> </font><font color="#F3E651">];</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b> +<font color="#ff0000"> </font><b><font color="#ffffff">exit</font></b><font color="#ff0000"> </font><font color="#bb00ff">0</font> +<b><font color="#ffffff">fi</font></b> +<font color="#ff0000">touch </font><font color="#bb00ff">"$LOCK_FILE"</font> +<b><font color="#ffffff">trap</font></b><font color="#ff0000"> </font><font color="#bb00ff">"rm -f $LOCK_FILE"</font><font color="#ff0000"> EXIT</font> + +<font color="#7bc710">fix_mount ()</font><font color="#ff0000"> {</font> +<font color="#ff0000"> echo </font><font color="#bb00ff">"Attempting to remount NFS mount $MOUNT_POINT"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">if</font></b><font color="#ff0000"> mount -o remount -f </font><font color="#bb00ff">"$MOUNT_POINT"</font><font color="#ff0000"> </font><font color="#bb00ff">2</font><font color="#F3E651">></font><font color="#ff0000">/dev/null</font><font color="#F3E651">;</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b> +<font color="#ff0000"> echo </font><font color="#bb00ff">"Remount command issued for $MOUNT_POINT"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">else</font></b> +<font color="#ff0000"> echo </font><font color="#bb00ff">"Failed to remount NFS mount $MOUNT_POINT"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">fi</font></b> + +<font color="#ff0000"> echo </font><font color="#bb00ff">"Checking if $MOUNT_POINT is a mountpoint"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">if</font></b><font color="#ff0000"> mountpoint </font><font color="#bb00ff">"$MOUNT_POINT"</font><font color="#ff0000"> </font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font><font color="#F3E651">;</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b> +<font color="#ff0000"> echo </font><font color="#bb00ff">"$MOUNT_POINT is a valid mountpoint"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">else</font></b> +<font color="#ff0000"> echo </font><font color="#bb00ff">"$MOUNT_POINT is not a valid mountpoint, attempting mount"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">if</font></b><font color="#ff0000"> mount </font><font color="#bb00ff">"$MOUNT_POINT"</font><font color="#F3E651">;</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b> +<font color="#ff0000"> echo </font><font color="#bb00ff">"Successfully mounted $MOUNT_POINT"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">return</font></b> +<font color="#ff0000"> </font><b><font color="#ffffff">else</font></b> +<font color="#ff0000"> echo </font><font color="#bb00ff">"Failed to mount $MOUNT_POINT"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">fi</font></b> +<font color="#ff0000"> </font><b><font color="#ffffff">fi</font></b> + +<font color="#ff0000"> echo </font><font color="#bb00ff">"Attempting to unmount $MOUNT_POINT"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">if</font></b><font color="#ff0000"> umount -f </font><font color="#bb00ff">"$MOUNT_POINT"</font><font color="#ff0000"> </font><font color="#bb00ff">2</font><font color="#F3E651">></font><font color="#ff0000">/dev/null</font><font color="#F3E651">;</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b> +<font color="#ff0000"> echo </font><font color="#bb00ff">"Successfully unmounted $MOUNT_POINT"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">else</font></b> +<font color="#ff0000"> echo </font><font color="#bb00ff">"Failed to unmount $MOUNT_POINT (it might not be mounted)"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">fi</font></b> + +<font color="#ff0000"> echo </font><font color="#bb00ff">"Attempting to mount $MOUNT_POINT"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">if</font></b><font color="#ff0000"> mount </font><font color="#bb00ff">"$MOUNT_POINT"</font><font color="#F3E651">;</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b> +<font color="#ff0000"> echo </font><font color="#bb00ff">"NFS mount $MOUNT_POINT mounted successfully"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">return</font></b> +<font color="#ff0000"> </font><b><font color="#ffffff">else</font></b> +<font color="#ff0000"> echo </font><font color="#bb00ff">"Failed to mount NFS mount $MOUNT_POINT"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">fi</font></b> + +<font color="#ff0000"> echo </font><font color="#bb00ff">"Failed to fix NFS mount $MOUNT_POINT"</font> +<font color="#ff0000"> </font><b><font color="#ffffff">exit</font></b><font color="#ff0000"> </font><font color="#bb00ff">1</font> +<font color="#ff0000">}</font> + +<b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">!</font><font color="#ff0000"> mountpoint </font><font color="#bb00ff">"$MOUNT_POINT"</font><font color="#ff0000"> </font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font><font color="#F3E651">;</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b> +<font color="#ff0000"> echo </font><font color="#bb00ff">"NFS mount $MOUNT_POINT not found"</font> +<font color="#ff0000"> fix_mount</font> +<b><font color="#ffffff">fi</font></b> + +<b><font color="#ffffff">if</font></b><font color="#ff0000"> </font><font color="#F3E651">!</font><font color="#ff0000"> timeout 2s stat </font><font color="#bb00ff">"$MOUNT_POINT"</font><font color="#ff0000"> </font><font color="#F3E651">></font><font color="#ff0000">/dev/null </font><font color="#bb00ff">2</font><font color="#F3E651">>&</font><font color="#bb00ff">1</font><font color="#F3E651">;</font><font color="#ff0000"> </font><b><font color="#ffffff">then</font></b> +<font color="#ff0000"> echo </font><font color="#bb00ff">"NFS mount $MOUNT_POINT appears to be unresponsive"</font> +<font color="#ff0000"> fix_mount</font> +<b><font color="#ffffff">fi</font></b> +<font color="#ff0000">EOF</font> + +<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># chmod +x /usr/local/bin/check-nfs-mount.sh</font></i> </pre> <br /> <span>And we create the systemd service as follows:</span><br /> @@ -1917,17 +1922,17 @@ EOF by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>[root@r0 ~]<i><font color="silver"># cat > /etc/systemd/system/nfs-mount-monitor.service << 'EOF'</font></i> -[Unit] -Description=NFS Mount Health Monitor -After=network-online.target +<pre><font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># cat > /etc/systemd/system/nfs-mount-monitor.service << 'EOF'</font></i> +<font color="#F3E651">[</font><font color="#ff0000">Unit</font><font color="#F3E651">]</font> +<font color="#ff0000">Description</font><font color="#F3E651">=</font><font color="#ff0000">NFS Mount Health Monitor</font> +<font color="#ff0000">After</font><font color="#F3E651">=</font><font color="#ff0000">network-online</font><font color="#F3E651">.</font><font color="#ff0000">target</font> -[Service] -Type=oneshot -ExecStart=/usr/local/bin/check-nfs-mount.sh -StandardOutput=journal -StandardError=journal -EOF +<font color="#F3E651">[</font><font color="#ff0000">Service</font><font color="#F3E651">]</font> +<font color="#ff0000">Type</font><font color="#F3E651">=</font><font color="#ff0000">oneshot</font> +<font color="#ff0000">ExecStart</font><font color="#F3E651">=</font><font color="#ff0000">/usr/local/bin/check-nfs-mount</font><font color="#F3E651">.</font><font color="#ff0000">sh</font> +<font color="#ff0000">StandardOutput</font><font color="#F3E651">=</font><font color="#ff0000">journal</font> +<font color="#ff0000">StandardError</font><font color="#F3E651">=</font><font color="#ff0000">journal</font> +<font color="#ff0000">EOF</font> </pre> <br /> <span>And we also create the systemd timer (runs every 10 seconds):</span><br /> @@ -1936,19 +1941,19 @@ EOF by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>[root@r0 ~]<i><font color="silver"># cat > /etc/systemd/system/nfs-mount-monitor.timer << 'EOF'</font></i> -[Unit] -Description=Run NFS Mount Health Monitor every <font color="#000000">10</font> seconds -Requires=nfs-mount-monitor.service +<pre><font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># cat > /etc/systemd/system/nfs-mount-monitor.timer << 'EOF'</font></i> +<font color="#F3E651">[</font><font color="#ff0000">Unit</font><font color="#F3E651">]</font> +<font color="#ff0000">Description</font><font color="#F3E651">=</font><font color="#ff0000">Run NFS Mount Health Monitor every </font><font color="#bb00ff">10</font><font color="#ff0000"> seconds</font> +<font color="#ff0000">Requires</font><font color="#F3E651">=</font><font color="#ff0000">nfs-mount-monitor</font><font color="#F3E651">.</font><font color="#ff0000">service</font> -[Timer] -OnBootSec=30s -OnUnitActiveSec=10s -AccuracySec=1s +<font color="#F3E651">[</font><font color="#ff0000">Timer</font><font color="#F3E651">]</font> +<font color="#ff0000">OnBootSec</font><font color="#F3E651">=</font><font color="#ff0000">30s</font> +<font color="#ff0000">OnUnitActiveSec</font><font color="#F3E651">=</font><font color="#ff0000">10s</font> +<font color="#ff0000">AccuracySec</font><font color="#F3E651">=</font><font color="#ff0000">1s</font> -[Install] -WantedBy=timers.target -EOF +<font color="#F3E651">[</font><font color="#ff0000">Install</font><font color="#F3E651">]</font> +<font color="#ff0000">WantedBy</font><font color="#F3E651">=</font><font color="#ff0000">timers</font><font color="#F3E651">.</font><font color="#ff0000">target</font> +<font color="#ff0000">EOF</font> </pre> <br /> <span>To enable and start the timer, we run:</span><br /> @@ -1957,19 +1962,19 @@ EOF by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>[root@r0 ~]<i><font color="silver"># systemctl daemon-reload</font></i> -[root@r0 ~]<i><font color="silver"># systemctl enable nfs-mount-monitor.timer</font></i> -[root@r0 ~]<i><font color="silver"># systemctl start nfs-mount-monitor.timer</font></i> +<pre><font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># systemctl daemon-reload</font></i> +<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># systemctl enable nfs-mount-monitor.timer</font></i> +<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># systemctl start nfs-mount-monitor.timer</font></i> -<i><font color="silver"># Check status</font></i> -[root@r0 ~]<i><font color="silver"># systemctl status nfs-mount-monitor.timer</font></i> -● nfs-mount-monitor.timer - Run NFS Mount Health Monitor every <font color="#000000">10</font> seconds - Loaded: loaded (/etc/systemd/system/nfs-mount-monitor.timer; enabled) - Active: active (waiting) since Sat <font color="#000000">2025</font>-<font color="#000000">07</font>-<font color="#000000">06</font> <font color="#000000">10</font>:<font color="#000000">00</font>:<font color="#000000">00</font> EEST - Trigger: Sat <font color="#000000">2025</font>-<font color="#000000">07</font>-<font color="#000000">06</font> <font color="#000000">10</font>:<font color="#000000">00</font>:<font color="#000000">10</font> EEST; 8s left +<i><font color="#ababab"># Check status</font></i> +<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># systemctl status nfs-mount-monitor.timer</font></i> +<font color="#ff0000">● nfs-mount-monitor</font><font color="#F3E651">.</font><font color="#ff0000">timer - Run NFS Mount Health Monitor every </font><font color="#bb00ff">10</font><font color="#ff0000"> seconds</font> +<font color="#ff0000"> Loaded</font><font color="#F3E651">:</font><font color="#ff0000"> loaded </font><font color="#F3E651">(</font><font color="#ff0000">/etc/systemd/system/nfs-mount-monitor</font><font color="#F3E651">.</font><font color="#ff0000">timer</font><font color="#F3E651">;</font><font color="#ff0000"> enabled</font><font color="#F3E651">)</font> +<font color="#ff0000"> Active</font><font color="#F3E651">:</font><font color="#ff0000"> active </font><font color="#F3E651">(</font><font color="#ff0000">waiting</font><font color="#F3E651">)</font><font color="#ff0000"> since Sat </font><font color="#bb00ff">2025</font><font color="#ff0000">-</font><font color="#bb00ff">07</font><font color="#ff0000">-</font><font color="#bb00ff">06</font><font color="#ff0000"> </font><font color="#bb00ff">10</font><font color="#F3E651">:</font><font color="#bb00ff">00</font><font color="#F3E651">:</font><font color="#bb00ff">00</font><font color="#ff0000"> EEST</font> +<font color="#ff0000"> Trigger</font><font color="#F3E651">:</font><font color="#ff0000"> Sat </font><font color="#bb00ff">2025</font><font color="#ff0000">-</font><font color="#bb00ff">07</font><font color="#ff0000">-</font><font color="#bb00ff">06</font><font color="#ff0000"> </font><font color="#bb00ff">10</font><font color="#F3E651">:</font><font color="#bb00ff">00</font><font color="#F3E651">:</font><font color="#bb00ff">10</font><font color="#ff0000"> EEST</font><font color="#F3E651">;</font><font color="#ff0000"> 8s left</font> -<i><font color="silver"># Monitor logs</font></i> -[root@r0 ~]<i><font color="silver"># journalctl -u nfs-mount-monitor -f</font></i> +<i><font color="#ababab"># Monitor logs</font></i> +<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># journalctl -u nfs-mount-monitor -f</font></i> </pre> <br /> <span>Note: Stale file handles are inherent to NFS failover because file handles are server-specific. The best approach depends on your application's tolerance for brief disruptions. Of course, all the changes made to <span class='inlinecode'>r0</span> above must also be applied to <span class='inlinecode'>r1</span> and <span class='inlinecode'>r2</span>.</span><br /> @@ -1982,30 +1987,30 @@ http://www.gnu.org/software/src-highlite --> by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre><i><font color="silver"># 1. Check the initial state</font></i> -paul@f0:~ % ifconfig re0 | grep carp - carp: MASTER vhid <font color="#000000">1</font> advbase <font color="#000000">1</font> advskew <font color="#000000">0</font> -paul@f1:~ % ifconfig re0 | grep carp - carp: BACKUP vhid <font color="#000000">1</font> advbase <font color="#000000">1</font> advskew <font color="#000000">100</font> +<pre><i><font color="#ababab"># 1. Check the initial state</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> ifconfig re0 </font><font color="#F3E651">|</font><font color="#ff0000"> grep carp</font> +<font color="#ff0000"> carp</font><font color="#F3E651">:</font><font color="#ff0000"> MASTER vhid </font><font color="#bb00ff">1</font><font color="#ff0000"> advbase </font><font color="#bb00ff">1</font><font color="#ff0000"> advskew </font><font color="#bb00ff">0</font> +<font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> ifconfig re0 </font><font color="#F3E651">|</font><font color="#ff0000"> grep carp</font> +<font color="#ff0000"> carp</font><font color="#F3E651">:</font><font color="#ff0000"> BACKUP vhid </font><font color="#bb00ff">1</font><font color="#ff0000"> advbase </font><font color="#bb00ff">1</font><font color="#ff0000"> advskew </font><font color="#bb00ff">100</font> -<i><font color="silver"># 2. Create a test file from a client</font></i> -[root@r0 ~]<i><font color="silver"># echo "test before failover" > /data/nfs/k3svolumes/test-before.txt</font></i> +<i><font color="#ababab"># 2. Create a test file from a client</font></i> +<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># echo "test before failover" > /data/nfs/k3svolumes/test-before.txt</font></i> -<i><font color="silver"># 3. Trigger failover (f0 → f1)</font></i> -paul@f0:~ % doas ifconfig re0 vhid <font color="#000000">1</font> state backup +<i><font color="#ababab"># 3. Trigger failover (f0 → f1)</font></i> +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas ifconfig re0 vhid </font><font color="#bb00ff">1</font><font color="#ff0000"> state backup</font> -<i><font color="silver"># 4. Monitor client behaviour</font></i> -[root@r0 ~]<i><font color="silver"># ls /data/nfs/k3svolumes/</font></i> -ls: cannot access <font color="#808080">'/data/nfs/k3svolumes/'</font>: Stale file handle +<i><font color="#ababab"># 4. Monitor client behaviour</font></i> +<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># ls /data/nfs/k3svolumes/</font></i> +<font color="#ff0000">ls</font><font color="#F3E651">:</font><font color="#ff0000"> cannot access </font><font color="#bb00ff">'/data/nfs/k3svolumes/'</font><font color="#F3E651">:</font><font color="#ff0000"> Stale file handle</font> -<i><font color="silver"># 5. Check automatic recovery (within 10 seconds)</font></i> -[root@r0 ~]<i><font color="silver"># journalctl -u nfs-mount-monitor -f</font></i> -Jul <font color="#000000">06</font> <font color="#000000">10</font>:<font color="#000000">15</font>:<font color="#000000">32</font> r0 nfs-monitor[<font color="#000000">1234</font>]: NFS mount unhealthy detected at \ - Sun Jul <font color="#000000">6</font> <font color="#000000">10</font>:<font color="#000000">15</font>:<font color="#000000">32</font> EEST <font color="#000000">2025</font> -Jul <font color="#000000">06</font> <font color="#000000">10</font>:<font color="#000000">15</font>:<font color="#000000">32</font> r0 nfs-monitor[<font color="#000000">1234</font>]: Attempting to fix stale NFS mount at \ - Sun Jul <font color="#000000">6</font> <font color="#000000">10</font>:<font color="#000000">15</font>:<font color="#000000">32</font> EEST <font color="#000000">2025</font> -Jul <font color="#000000">06</font> <font color="#000000">10</font>:<font color="#000000">15</font>:<font color="#000000">33</font> r0 nfs-monitor[<font color="#000000">1234</font>]: NFS mount fixed at \ - Sun Jul <font color="#000000">6</font> <font color="#000000">10</font>:<font color="#000000">15</font>:<font color="#000000">33</font> EEST <font color="#000000">2025</font> +<i><font color="#ababab"># 5. Check automatic recovery (within 10 seconds)</font></i> +<font color="#F3E651">[</font><font color="#ff0000">root@r0 </font><font color="#F3E651">~]</font><i><font color="#ababab"># journalctl -u nfs-mount-monitor -f</font></i> +<font color="#ff0000">Jul </font><font color="#bb00ff">06</font><font color="#ff0000"> </font><font color="#bb00ff">10</font><font color="#F3E651">:</font><font color="#bb00ff">15</font><font color="#F3E651">:</font><font color="#bb00ff">32</font><font color="#ff0000"> r0 nfs-monitor</font><font color="#F3E651">[</font><font color="#bb00ff">1234</font><font color="#F3E651">]:</font><font color="#ff0000"> NFS mount unhealthy detected at </font><font color="#F3E651">\</font> +<font color="#ff0000"> Sun Jul </font><font color="#bb00ff">6</font><font color="#ff0000"> </font><font color="#bb00ff">10</font><font color="#F3E651">:</font><font color="#bb00ff">15</font><font color="#F3E651">:</font><font color="#bb00ff">32</font><font color="#ff0000"> EEST </font><font color="#bb00ff">2025</font> +<font color="#ff0000">Jul </font><font color="#bb00ff">06</font><font color="#ff0000"> </font><font color="#bb00ff">10</font><font color="#F3E651">:</font><font color="#bb00ff">15</font><font color="#F3E651">:</font><font color="#bb00ff">32</font><font color="#ff0000"> r0 nfs-monitor</font><font color="#F3E651">[</font><font color="#bb00ff">1234</font><font color="#F3E651">]:</font><font color="#ff0000"> Attempting to fix stale NFS mount at </font><font color="#F3E651">\</font> +<font color="#ff0000"> Sun Jul </font><font color="#bb00ff">6</font><font color="#ff0000"> </font><font color="#bb00ff">10</font><font color="#F3E651">:</font><font color="#bb00ff">15</font><font color="#F3E651">:</font><font color="#bb00ff">32</font><font color="#ff0000"> EEST </font><font color="#bb00ff">2025</font> +<font color="#ff0000">Jul </font><font color="#bb00ff">06</font><font color="#ff0000"> </font><font color="#bb00ff">10</font><font color="#F3E651">:</font><font color="#bb00ff">15</font><font color="#F3E651">:</font><font color="#bb00ff">33</font><font color="#ff0000"> r0 nfs-monitor</font><font color="#F3E651">[</font><font color="#bb00ff">1234</font><font color="#F3E651">]:</font><font color="#ff0000"> NFS mount fixed at </font><font color="#F3E651">\</font> +<font color="#ff0000"> Sun Jul </font><font color="#bb00ff">6</font><font color="#ff0000"> </font><font color="#bb00ff">10</font><font color="#F3E651">:</font><font color="#bb00ff">15</font><font color="#F3E651">:</font><font color="#bb00ff">33</font><font color="#ff0000"> EEST </font><font color="#bb00ff">2025</font> </pre> <br /> <span>Failover Timeline:</span><br /> @@ -2063,7 +2068,7 @@ Jul <font color="#000000">06</font> <font color="#000000">10</font>:<font color= by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:~ % doas zpool online -e /dev/ada<font color="#000000">1</font> +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zpool online -e /dev/ada</font><font color="#bb00ff">1</font> </pre> <br /> <ul> @@ -2076,15 +2081,15 @@ http://www.gnu.org/software/src-highlite --> by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f0:~ % doas zpool list -NAME SIZE ALLOC FREE CKPOINT EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT -zdata <font color="#000000">3</font>.63T 677G <font color="#000000">2</font>.97T - - <font color="#000000">3</font>% <font color="#000000">18</font>% <font color="#000000">1</font>.00x ONLINE - -zroot 472G <font color="#000000">68</font>.4G 404G - - <font color="#000000">13</font>% <font color="#000000">14</font>% <font color="#000000">1</font>.00x ONLINE - +<pre><font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas zpool list</font> +<font color="#ff0000">NAME SIZE ALLOC FREE CKPOINT EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT</font> +<font color="#ff0000">zdata </font><font color="#bb00ff">3</font><font color="#F3E651">.</font><font color="#ff0000">63T 677G </font><font color="#bb00ff">2</font><font color="#F3E651">.</font><font color="#ff0000">97T - - </font><font color="#bb00ff">3</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">18</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">1</font><font color="#F3E651">.</font><font color="#ff0000">00x ONLINE -</font> +<font color="#ff0000">zroot 472G </font><font color="#bb00ff">68</font><font color="#F3E651">.</font><font color="#ff0000">4G 404G - - </font><font color="#bb00ff">13</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">14</font><font color="#F3E651">%</font><font color="#ff0000"> </font><font color="#bb00ff">1</font><font color="#F3E651">.</font><font color="#ff0000">00x ONLINE -</font> -paul@f0:~ % doas camcontrol devlist -<512GB SSD D910R170> at scbus0 target <font color="#000000">0</font> lun <font color="#000000">0</font> (pass0,ada0) -<SD Ultra 3D 4TB 530500WD> at scbus1 target <font color="#000000">0</font> lun <font color="#000000">0</font> (pass1,ada1) -<Generic Flash Disk <font color="#000000">8.07</font>> at scbus2 target <font color="#000000">0</font> lun <font color="#000000">0</font> (da0,pass2) +<font color="#ff0000">paul@f0</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas camcontrol devlist</font> +<font color="#F3E651"><</font><font color="#ff0000">512GB SSD D910R170</font><font color="#F3E651">></font><font color="#ff0000"> at scbus0 target </font><font color="#bb00ff">0</font><font color="#ff0000"> lun </font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#F3E651">(</font><font color="#ff0000">pass0</font><font color="#F3E651">,</font><font color="#ff0000">ada0</font><font color="#F3E651">)</font> +<font color="#F3E651"><</font><font color="#ff0000">SD Ultra 3D 4TB 530500WD</font><font color="#F3E651">></font><font color="#ff0000"> at scbus1 target </font><font color="#bb00ff">0</font><font color="#ff0000"> lun </font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#F3E651">(</font><font color="#ff0000">pass1</font><font color="#F3E651">,</font><font color="#ff0000">ada1</font><font color="#F3E651">)</font> +<font color="#F3E651"><</font><font color="#ff0000">Generic Flash Disk </font><font color="#bb00ff">8.07</font><font color="#F3E651">></font><font color="#ff0000"> at scbus2 target </font><font color="#bb00ff">0</font><font color="#ff0000"> lun </font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#F3E651">(</font><font color="#ff0000">da0</font><font color="#F3E651">,</font><font color="#ff0000">pass2</font><font color="#F3E651">)</font> </pre> <br /> <span>We're still using different SSD models on f1 (WD Blue SA510 4TB) to avoid simultaneous failures:</span><br /> @@ -2093,10 +2098,10 @@ paul@f0:~ % doas camcontrol devlist by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -<pre>paul@f1:~ % doas camcontrol devlist -<512GB SSD D910R170> at scbus0 target <font color="#000000">0</font> lun <font color="#000000">0</font> (pass0,ada0) -<WD Blue SA510 <font color="#000000">2.5</font> 4TB 530500WD> at scbus1 target <font color="#000000">0</font> lun <font color="#000000">0</font> (pass1,ada1) -<Generic Flash Disk <font color="#000000">8.07</font>> at scbus2 target <font color="#000000">0</font> lun <font color="#000000">0</font> (da0,pass2) +<pre><font color="#ff0000">paul@f1</font><font color="#F3E651">:~</font><font color="#ff0000"> </font><font color="#F3E651">%</font><font color="#ff0000"> doas camcontrol devlist</font> +<font color="#F3E651"><</font><font color="#ff0000">512GB SSD D910R170</font><font color="#F3E651">></font><font color="#ff0000"> at scbus0 target </font><font color="#bb00ff">0</font><font color="#ff0000"> lun </font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#F3E651">(</font><font color="#ff0000">pass0</font><font color="#F3E651">,</font><font color="#ff0000">ada0</font><font color="#F3E651">)</font> +<font color="#F3E651"><</font><font color="#ff0000">WD Blue SA510 </font><font color="#bb00ff">2.5</font><font color="#ff0000"> 4TB 530500WD</font><font color="#F3E651">></font><font color="#ff0000"> at scbus1 target </font><font color="#bb00ff">0</font><font color="#ff0000"> lun </font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#F3E651">(</font><font color="#ff0000">pass1</font><font color="#F3E651">,</font><font color="#ff0000">ada1</font><font color="#F3E651">)</font> +<font color="#F3E651"><</font><font color="#ff0000">Generic Flash Disk </font><font color="#bb00ff">8.07</font><font color="#F3E651">></font><font color="#ff0000"> at scbus2 target </font><font color="#bb00ff">0</font><font color="#ff0000"> lun </font><font color="#bb00ff">0</font><font color="#ff0000"> </font><font color="#F3E651">(</font><font color="#ff0000">da0</font><font color="#F3E651">,</font><font color="#ff0000">pass2</font><font color="#F3E651">)</font> </pre> <br /> <h2 style='display: inline' id='conclusion'>Conclusion</h2><br /> @@ -2165,11 +2170,12 @@ http://www.gnu.org/software/src-highlite --> <br /> <a class='textlink' href='../'>Back to the main site</a><br /> <p class="footer"> - Generated with <a href="https://codeberg.org/snonux/gemtexter">Gemtexter 3.0.1-develop</a> | - served by <a href="https://www.OpenBSD.org">OpenBSD</a>/<a href="https://man.openbsd.org/relayd.8">relayd(8)</a>+<a href="https://man.openbsd.org/httpd.8">httpd(8)</a> | - <a href="https://foo.zone/site-mirrors.html">Site Mirrors</a> - <br /> - Webring: <a href="https://shring.sh/foo.zone/previous">previous</a> | <a href="https://shring.sh">shring</a> | <a href="https://shring.sh/foo.zone/next">next</a> + Generated with <a href="https://codeberg.org/snonux/gemtexter">Gemtexter 3.0.1-develop</a> | + served by <a href="https://www.OpenBSD.org">OpenBSD</a>/<a href="https://man.openbsd.org/relayd.8">relayd(8)</a>+<a href="https://man.openbsd.org/httpd.8">httpd(8)</a> | + <a href="https://foo.zone/site-mirrors.html">Site Mirrors</a> + <br /> + Webring: <a href="https://shring.sh/foo.zone/previous">previous</a> | <a href="https://shring.sh">shring</a> | <a href="https://shring.sh/foo.zone/next">next</a> </p> +<script type="text/javascript" src="../retrofuturistic.js"></script> </body> </html> |