zpool list
zpool list <poolname>
zpool status
zpool status -x
zpool status -v <poolname>
zpool iostat
zpool iostat -v
zpool history

Rule of thumb:
2 GB + 1GB Cache per 1 TB Storage

Set Cache Min/Max

cat << 'EOF' > /etc/modprobe.d/zfs.conf
options zfs zfs_arc_min=3221225472
options zfs zfs_arc_max=4294967296
EOF

update-initramfs -u && reboot

4x NVMe as VM datastore (Hypervisor installed on nvme0n1 and nvme1n1)

zpool create -o ashift=12 -O compression=lz4 -O atime=off -O xattr=sa -O local-datastore-nvme01 mirror /dev/nvme2n1 /dev/nvme3n1 mirror /dev/nvme4n1 /dev/nvme5n1

Parameters

ashift=12              # Aligns to 4K sectors (SSD optimal)
autotrim=on	       # Enables TRIM for SSDs (optional - alternatively create cronjob running during off-peak hours)
compression=lz4	       # Transparent compression, fast
atime=off	       # Disables access time writes
xattr=sa	       # Stores xattrs in inode — good for VMs

by-id

zpool create -o ashift=12 -O compression=lz4 -O atime=off -O xattr=sa local-datastore-nvme01 \ 
mirror /dev/disk/by-id/nvme-SAMSUNG_MZQL23T8HCLS-00A07_S3R14LNUMB3R1 /dev/disk/by-id/nvme-SAMSUNG_MZQL23T8HCLS-00A07_S3R14LNUMB3R2 \
mirror /dev/disk/by-id/nvme-SAMSUNG_MZQL23T8HCLS-00A07_S3R14LNUMB3R3 /dev/disk/by-id/nvme-SAMSUNG_MZQL23T8HCLS-00A07_S3R14LNUMB3R4

zpool destroy <poolname>
wipefs -a /dev/nvme2n1 /dev/nvme3n1 mirror /dev/nvme4n1 /dev/nvme2n1
wipefs -a /dev/nvme2n1 /dev/nvme3n1 mirror /dev/nvme4n1 /dev/nvme3n1
wipefs -a /dev/nvme2n1 /dev/nvme3n1 mirror /dev/nvme4n1 /dev/nvme4n1
wipefs -a /dev/nvme2n1 /dev/nvme3n1 mirror /dev/nvme4n1 /dev/nvme5n1

apt update && apt install nvme-cli

Check column “Fmt”. “+” marks the current format. “Rel_Perf” indicates the higher performance low level formats. The lower the better. 0 being the best.

root@pve01: smartctl -a /dev/nvme4n1
{...}
Supported LBA Sizes (NSID 0x1)
Id Fmt  Data  Metadt  Rel_Perf
 0 +     512       0         0
 1 -    4096       0         0
{...}

Format the device with bigger sector size. Column “Id” above.

nvme format /dev/nvme4n1 -l 1 --force

Verify the change.

root@pve01: smartctl -a /dev/nvme4n1
{...}
Supported LBA Sizes (NSID 0x1)
Id Fmt  Data  Metadt  Rel_Perf
 0 -     512       0         0
 1 +    4096       0         0
{...}

https://openzfs.github.io/openzfs-docs/Performance%20and%20Tuning/Hardware.html#nvme-low-level-formatting

lsblk -d -o NAME,PHY-SeC,LOG-SEC
nvme id-ns -H /dev/nvme4n1 | grep 'LBA'

Debian fails to start import service of an orphaned pool.

zpool set cachefile=/etc/zfs/zpool.cache poolname    # set cache file for running pools
update-initramfs -k all -u
reboot

check labels on disks

zdb -l /dev/nvme1n1p1