zpool list zpool list <poolname> zpool status zpool status -x zpool status -v <poolname> zpool iostat zpool iostat -v zpool history
Rule of thumb:
2 GB + 1GB Cache per 1 TB Storage
Set Cache Min/Max
cat << 'EOF' > /etc/modprobe.d/zfs.conf options zfs zfs_arc_min=3221225472 options zfs zfs_arc_max=4294967296 EOF
update-initramfs -u && reboot
4x NVMe as VM datastore (Hypervisor installed on nvme0n1 and nvme1n1)
zpool create -o ashift=12 -O compression=lz4 -O atime=off -O xattr=sa -O local-datastore-nvme01 mirror /dev/nvme2n1 /dev/nvme3n1 mirror /dev/nvme4n1 /dev/nvme5n1
Parameters
ashift=12 # Aligns to 4K sectors (SSD optimal) autotrim=on # Enables TRIM for SSDs (optional - alternatively create cronjob running during off-peak hours) compression=lz4 # Transparent compression, fast atime=off # Disables access time writes xattr=sa # Stores xattrs in inode — good for VMs
by-id
zpool create -o ashift=12 -O compression=lz4 -O atime=off -O xattr=sa local-datastore-nvme01 \ mirror /dev/disk/by-id/nvme-SAMSUNG_MZQL23T8HCLS-00A07_S3R14LNUMB3R1 /dev/disk/by-id/nvme-SAMSUNG_MZQL23T8HCLS-00A07_S3R14LNUMB3R2 \ mirror /dev/disk/by-id/nvme-SAMSUNG_MZQL23T8HCLS-00A07_S3R14LNUMB3R3 /dev/disk/by-id/nvme-SAMSUNG_MZQL23T8HCLS-00A07_S3R14LNUMB3R4
zpool destroy <poolname> wipefs -a /dev/nvme2n1 /dev/nvme3n1 mirror /dev/nvme4n1 /dev/nvme2n1 wipefs -a /dev/nvme2n1 /dev/nvme3n1 mirror /dev/nvme4n1 /dev/nvme3n1 wipefs -a /dev/nvme2n1 /dev/nvme3n1 mirror /dev/nvme4n1 /dev/nvme4n1 wipefs -a /dev/nvme2n1 /dev/nvme3n1 mirror /dev/nvme4n1 /dev/nvme5n1
apt update && apt install nvme-cli
Check column “Fmt”. “+” marks the current format. “Rel_Perf” indicates the higher performance low level formats. The lower the better. 0 being the best.
root@pve01: smartctl -a /dev/nvme4n1
{...}
Supported LBA Sizes (NSID 0x1)
Id Fmt Data Metadt Rel_Perf
0 + 512 0 0
1 - 4096 0 0
{...}
Format the device with bigger sector size. Column “Id” above.
nvme format /dev/nvme4n1 -l 1 --force
Verify the change.
root@pve01: smartctl -a /dev/nvme4n1
{...}
Supported LBA Sizes (NSID 0x1)
Id Fmt Data Metadt Rel_Perf
0 - 512 0 0
1 + 4096 0 0
{...}
lsblk -d -o NAME,PHY-SeC,LOG-SEC nvme id-ns -H /dev/nvme4n1 | grep 'LBA'
Debian fails to start import service of an orphaned pool.
zpool set cachefile=/etc/zfs/zpool.cache poolname # set cache file for running pools update-initramfs -k all -u reboot
check labels on disks
zdb -l /dev/nvme1n1p1