Update content for md

1 files changed, 69 insertions, 164 deletions

diff --git a/gemfeed/DRAFT-f3s-kubernetes-with-freebsd-part-6.md b/gemfeed/DRAFT-f3s-kubernetes-with-freebsd-part-6.md
index 32783a47..0aa6d893 100644
--- a/gemfeed/DRAFT-f3s-kubernetes-with-freebsd-part-6.md
+++ b/gemfeed/DRAFT-f3s-kubernetes-with-freebsd-part-6.md
@@ -16,10 +16,12 @@ This is the sixth blog post about the f3s series for self-hosting demands in a h
 
 * [⇢ f3s: Kubernetes with FreeBSD - Part 6: Storage](#f3s-kubernetes-with-freebsd---part-6-storage)
 * [⇢ ⇢ Introduction](#introduction)
-* [⇢ ⇢ UFS Setup](#ufs-setup)
-* [⇢ ⇢ ZFS Setup](#zfs-setup)
-* [⇢ ⇢ ⇢ Encryption](#encryption)
-* [⇢ ⇢ HAST](#hast)
+* [⇢ ⇢ ZFS encryption keys](#zfs-encryption-keys)
+* [⇢ ⇢ ⇢ UFS on USB keys](#ufs-on-usb-keys)
+* [⇢ ⇢ ⇢ Generating encryption keys](#generating-encryption-keys)
+* [⇢ ⇢ ⇢ Configuring `zdata` ZFS pool and encryption](#configuring-zdata-zfs-pool-and-encryption)
+* [⇢ ⇢ ⇢ Migrating Bhyve VMs to encrypted `bhyve` ZFS volume](#migrating-bhyve-vms-to-encrypted-bhyve-zfs-volume)
+* [⇢ ⇢ CARP](#carp)
 
 ## Introduction
 
@@ -27,6 +29,10 @@ In this blog post, we are going to extend the Beelinks with some additional stor
 
 Some photos here, describe why there are 2 different models of SSD drives (replication etc)
 
+## ZFS encryption keys
+
+### UFS on USB keys
+
 ```
 paul@f0:/ % doas camcontrol devlist
 <512GB SSD D910R170>               at scbus0 target 0 lun 0 (pass0,ada0)
@@ -43,8 +49,6 @@ paul@f1:/ % doas camcontrol devlist
 paul@f1:/ %
 ```
 
-## UFS Setup
-
 ```sh
 paul@f0:/ % doas newfs /dev/da0
 /dev/da0: 15000.0MB (30720000 sectors) block size 32768, fragment size 4096
@@ -63,20 +67,47 @@ paul@f0:/ % df | grep keys
 /dev/da0             14877596       8  13687384     0%    /keys
 ```
 
-## ZFS Setup
+### Generating encryption keys
 
-```sh
-paul@f0:/dev % doas zpool create -m /data zdata /dev/ada1
-paul@f0:/dev % zpool list
-NAME    SIZE  ALLOC   FREE  CKPOINT  EXPANDSZ   FRAG    CAP  DEDUP    HEALTH  ALTROOT
-zdata   928G   432K   928G        -         -     0%     0%  1.00x    ONLINE  -
-zroot   472G  19.8G   452G        -         -     0%     4%  1.00x    ONLINE  -
+paul@f0:/keys % doas openssl rand -out /keys/f0.lan.buetow.org:bhyve.key 32
+paul@f0:/keys % doas openssl rand -out /keys/f1.lan.buetow.org:bhyve.key 32
+paul@f0:/keys % doas openssl rand -out /keys/f2.lan.buetow.org:bhyve.key 32
+paul@f0:/keys % doas openssl rand -out /keys/f0.lan.buetow.org:zdata.key 32
+paul@f0:/keys % doas openssl rand -out /keys/f1.lan.buetow.org:zdata.key 32
+paul@f0:/keys % doas openssl rand -out /keys/f2.lan.buetow.org:zdata.key 32
+paul@f0:/keys % doas chown root *
+paul@f0:/keys % doas chmod 400 *
 
-```
+paul@f0:/keys % ls -l
+total 20
+-r--------  1 root wheel 32 May 25 13:07 f0.lan.buetow.org:bhyve.key
+-r--------  1 root wheel 32 May 25 13:07 f1.lan.buetow.org:bhyve.key
+-r--------  1 root wheel 32 May 25 13:07 f2.lan.buetow.org:bhyve.key
+-r--------  1 root wheel 32 May 25 13:07 f0.lan.buetow.org:zdata.key
+-r--------  1 root wheel 32 May 25 13:07 f1.lan.buetow.org:zdata.key
+-r--------  1 root wheel 32 May 25 13:07 f2.lan.buetow.org:zdata.key
+
+Copy those to all 3 nodes to /keys
+
+### Configuring `zdata` ZFS pool and encryption
+
+```sh
+paul@f0:/keys % doas zpool create -m /data zdata /dev/ada1
+paul@f0:/keys % doas zfs create -o encryption=on -o keyformat=raw -o keylocation=file:///keys/`hostname`:zdata.key zdata/enc
+paul@f0:/ % zfs list | grep zdata
+zdata                                          836K   899G    96K  /data
+zdata/enc                                      200K   899G   200K  /data/enc
+paul@f0:/keys % zfs get all zdata/enc | grep -E -i '(encryption|key)'
+zdata/enc  encryption            aes-256-gcm                               -
+zdata/enc  keylocation           file:///keys/f0.lan.buetow.org:zdata.key  local
+zdata/enc  keyformat             raw                                       -
+zdata/enc  encryptionroot        zdata/enc                                 -
+zdata/enc  keystatus             available                                 -
+````
 
-### Encryption
+### Migrating Bhyve VMs to encrypted `bhyve` ZFS volume
 
-USB key for key location
+Run on all 3 nodes
 
 ```sh
 paul@f0:/keys % doas vm stop rocky
@@ -92,15 +123,7 @@ paul@f0:/keys % doas zfs set mountpoint=/mnt zroot/bhyve_old
 paul@f0:/keys % doas zfs snapshot zroot/bhyve_old/rocky@hamburger
 
 
-paul@f0:/keys % doas openssl rand -out /keys/`hostname`:bhyve.key 32
-paul@f0:/keys % doas openssl rand -out /keys/`hostname`:zdata.key 32
-paul@f0:/keys % ls -ltr
-total 8
--rw-r--r--  1 root wheel 16 May 25 11:54 f0.lan.buetow.org:bhyve.key
--rw-r--r--  1 root wheel 16 May 25 11:54 f0.lan.buetow.org:zdata.key
-
 paul@f0:/keys % doas zfs create -o encryption=on -o keyformat=raw -o keylocation=file:///keys/`hostname`:bhyve.key zroot/bhyve
-paul@f0:/keys % doas zfs create -o encryption=on -o keyformat=raw -o keylocation=file:///keys/`hostname`:zdata.key zdata/enc
 paul@f0:/keys % doas zfs set mountpoint=/zroot/bhyve zroot/bhyve
 paul@f0:/keys % doas zfs set mountpoint=/zroot/bhyve/rocky zroot/bhyve/rocky
 
@@ -112,24 +135,6 @@ paul@f0:/keys % doas cp -Rp /mnt/.iso /zroot/bhyve/
 
 paul@f0:/keys % doas sysrc zfskeys_enable=YES
 zfskeys_enable:  -> YES
-```
-
-Copied over all the keys from the partner node to each node, so they backup each other:
-
-```sh
-paul@f0:/keys % doas chown root *
-paul@f0:/keys % doas chmod 400 *
-paul@f0:/keys % ls -ltr
-total 24
--r--------  1 root paul 16 May 25 11:56 f0.lan.buetow.org:zdata.key
--r--------  1 root paul 16 May 25 11:56 f0.lan.buetow.org:bhyve.key
--r--------  1 root paul 16 May 25 11:56 f1.lan.buetow.org:zdata.key
--r--------  1 root paul 16 May 25 11:56 f1.lan.buetow.org:bhyve.key
--r--------  1 root paul 16 May 25 11:57 f2.lan.buetow.org:zdata.key
--r--------  1 root paul 16 May 25 11:57 f2.lan.buetow.org:bhyve.key
-```
-
-```sh
 paul@f0:/keys % doas vm init
 paul@f0:/keys % doas reboot
 .
@@ -144,12 +149,6 @@ rocky    default    uefi       4    14G     0.0.0.0:5900  Yes [1]  Running (2265
 ```sh
 paul@f0:~ % doas zfs destroy -R zroot/bhyve_old
 
-paul@f0:~ % zfs get all zdata/enc | grep -E '(encryption|key)'
-zdata/enc  encryption            aes-256-gcm                               -
-zdata/enc  keylocation           file:///keys/f0.lan.buetow.org:zdata.key  local
-zdata/enc  keyformat             raw                                       -
-zdata/enc  encryptionroot        zdata/enc                                 -
-zdata/enc  keystatus             available                                 -
 paul@f0:~ % zfs get all zroot/bhyve | grep -E '(encryption|key)'
 zroot/bhyve  encryption            aes-256-gcm                               -
 zroot/bhyve  keylocation           file:///keys/f0.lan.buetow.org:bhyve.key  local
@@ -164,129 +163,30 @@ zroot/bhyve/rocky  encryptionroot        zroot/bhyve            -
 zroot/bhyve/rocky  keystatus             available              -
 ```
 
-```
-	paul@f0:~ % zpool status
-  pool: zdata
- state: ONLINE
-config:
-
-        NAME        STATE     READ WRITE CKSUM
-        zdata       ONLINE       0     0     0
-          ada1      ONLINE       0     0     0
-
-errors: No known data errors
-
-  pool: zroot
- state: ONLINE
-config:
-
-        NAME        STATE     READ WRITE CKSUM
-        zroot       ONLINE       0     0     0
-          ada0p4    ONLINE       0     0     0
-
-errors: No known data errors
-```
-## HAST
-
-```
-doas zpool export zdata
-
-paul@f0:/etc/rc.d % cat /etc/hast.conf
-resource storage {
-        on f0 {
-                local /dev/ada1
-                remote 192.168.1.130
-        }
-        on f1 {
-                local /dev/ada1
-                remote 192.168.1.131
-        }
-}
-
-paul@f0:/etc/rc.d % doas hastctl create storage
-paul@f0:/etc/rc.d % doas hastctl role primary storage
-paul@f0:/etc/rc.d % doas service hastd onestart
-Starting hastd.
-
-paul@f1:/etc/rc.d % doas hastctl create storage
-paul@f1:/etc/rc.d % doas hastctl role secondary storage
-paul@f1:/etc/rc.d % doas service hastd onestart
-Starting hastd.
-
-
-paul@f0:/var/log % doas hastctl status
-Name    Status   Role           Components
-storage complete primary        /dev/ada1       192.168.1.131
-
-paul@f1:/var/log % doas hastctl status
-Name    Status   Role           Components
-storage complete secondary      /dev/ada1       192.168.1.130
+## CARP
 
+adding to /etc/rc.conf on f0 and f1:
+ifconfig_re0_alias0="inet vhid 1 pass testpass alias 192.168.1.138/32"
 
+adding to /etc/hosts:
 
-paul@f0:/dev/hast % ls -l /dev/hast/storage
-crw-r-----  1 root operator 0x83 Jun  6 00:08 /dev/hast/storage
+192.168.1.138 f3s-storage-ha f3s-storage-ha.lan f3s-storage-ha.lan.buetow.org
 
-paul@f0:/dev/hast % doas zpool create -m /zhast zhast /dev/hast/storage
-paul@f0:/dev/hast % doas zpool status zhast
-  pool: zhast
- state: ONLINE
-config:
+Adding on f0 and f1:
 
-        NAME            STATE     READ WRITE CKSUM
-        zhast           ONLINE       0     0     0
-          hast/storage  ONLINE       0     0     0
+paul@f0:~ % cat <<END | doas tee -a /etc/devd.conf
+notify 0 {
+        match "system"          "CARP";
+        match "subsystem"       "[0-9]+@[0-9a-z.]+";
+        match "type"            "(MASTER|BACKUP)";
+        action "/usr/local/bin/carpcontrol.sh $subsystem $type";
+};
+END
 
-errors: No known data errors
-paul@f0:/dev/hast % doas zpool list
-NAME    SIZE  ALLOC   FREE  CKPOINT  EXPANDSZ   FRAG    CAP  DEDUP    HEALTH  ALTROOT
-zhast   928G   420K   928G        -         -     0%     0%  1.00x    ONLINE  -
-zroot   472G  21.0G   451G        -         -     0%     4%  1.00x    ONLINE  -```
+next, copied that script /usr/local/bin/carpcontrol.sh and adjusted the disk to storage
 
-
-paul@f0:/dev/hast % doas openssl rand -out /keys/zhast.key 32
-paul@f0:/dev/hast % doas zfs create -o encryption=on -o keyformat=raw -o keylocation=file:///keys/zhast.key zhast/enc
-paul@f0:/data/enc % zfs list | grep hast
-zhast                                          764K   899G    96K  /zhast
-zhast/enc                                      200K   899G   200K  /zhast/enc
-
-... copying the key to f1
-
-
-paul@f1:/var/log % doas hastctl list
-storage:
-  role: secondary
-  provname: storage
-  localpath: /dev/ada1
-  extentsize: 2097152 (2.0MB)
-  keepdirty: 0
-  remoteaddr: 192.168.1.130
-  replication: memsync
-  status: complete
-  workerpid: 2546
-  dirty: 0 (0B)
-  statistics:
-    reads: 0
-    writes: 26
-    deletes: 0
-    flushes: 0
-    activemap updates: 0
-    local errors: read: 0, write: 0, delete: 0, flush: 0
-    queues: local: 0, send: 0, recv: 0, done: 0, idle: 255
-
-
-
-
-
-paul@f1:/var/log % zfs get all zhast/enc | grep -E '(encryption|key)'
-zhast/enc  encryption            aes-256-gcm             -
-zhast/enc  keylocation           file:///keys/zhast.key  local
-zhast/enc  keyformat             raw                     -
-zhast/enc  encryptionroot        zhast/enc               -
-zhast/enc  keystatus             unavailable             -
-
-root@f0:/zhast/enc # sysrc hastd_enable=YES
-hastd_enable: NO -> YES
+/boot/loader.conf add carp_load="YES"
+reboot or run doas kldload carp0 
 
 
 ZFS auto scrubbing....~?
@@ -311,3 +211,8 @@ E-Mail your comments to `paul@nospam.buetow.org`
 [Back to the main site](../)  
 
 https://forums.freebsd.org/threads/hast-and-zfs-with-carp-failover.29639/
+
+
+E-Mail your comments to `paul@nospam.buetow.org`
+
+[Back to the main site](../)