Differences

This shows you the differences between two versions of the page.

--- linux:filesystem:zfs [2024/06/14 17:03] – created gsys
+++ linux:filesystem:zfs [2025/10/08 03:44] (current) – [Format NVMe with bigger sector size] gsys
@@ Line 10: / Line 10: @@
 zpool iostat -v
 zpool history
+</code>
+===== Adaptive Replacement Cache (ARC) =====
+Rule of thumb: \\
+''2 GB + 1GB Cache per 1 TB Storage'' \\
+Set Cache Min/Max
+<code>
+cat << 'EOF' > /etc/modprobe.d/zfs.conf
+options zfs zfs_arc_min=3221225472
+options zfs zfs_arc_max=4294967296
+EOF
+</code>
+<code>
+update-initramfs -u && reboot
+</code>
+===== Setups =====
+x NVMe as VM datastore (Hypervisor installed on nvme0n1 and nvme1n1)
+<code>
+zpool create -o ashift=12 -O compression=lz4 -O atime=off -O xattr=sa -O local-datastore-nvme01 mirror /dev/nvme2n1 /dev/nvme3n1 mirror /dev/nvme4n1 /dev/nvme5n1
+</code>
+Parameters
+<code>
+ashift=12              # Aligns to 4K sectors (SSD optimal)
+autotrim=on	       # Enables TRIM for SSDs (optional - alternatively create cronjob running during off-peak hours)
+compression=lz4	       # Transparent compression, fast
+atime=off	       # Disables access time writes
+xattr=sa	       # Stores xattrs in inode — good for VMs
+</code>
+by-id
+<code>
+zpool create -o ashift=12 -O compression=lz4 -O atime=off -O xattr=sa local-datastore-nvme01 \
+mirror /dev/disk/by-id/nvme-SAMSUNG_MZQL23T8HCLS-00A07_S3R14LNUMB3R1 /dev/disk/by-id/nvme-SAMSUNG_MZQL23T8HCLS-00A07_S3R14LNUMB3R2 \
+mirror /dev/disk/by-id/nvme-SAMSUNG_MZQL23T8HCLS-00A07_S3R14LNUMB3R3 /dev/disk/by-id/nvme-SAMSUNG_MZQL23T8HCLS-00A07_S3R14LNUMB3R4
+</code>
+===== Delete ZFS Pool =====
+<code>
+zpool destroy <poolname>
+wipefs -a /dev/nvme2n1 /dev/nvme3n1 mirror /dev/nvme4n1 /dev/nvme2n1
+wipefs -a /dev/nvme2n1 /dev/nvme3n1 mirror /dev/nvme4n1 /dev/nvme3n1
+wipefs -a /dev/nvme2n1 /dev/nvme3n1 mirror /dev/nvme4n1 /dev/nvme4n1
+wipefs -a /dev/nvme2n1 /dev/nvme3n1 mirror /dev/nvme4n1 /dev/nvme5n1
+</code>
+===== NVMe =====
+==== Format NVMe with bigger sector size ====
+<code>
+apt update && apt install nvme-cli
+</code>
+Check column "Fmt". "+" marks the current format. "Rel_Perf" indicates the higher performance low level formats. The lower the better. 0 being the best.
+<code>
+root@pve01: smartctl -a /dev/nvme4n1
+{...}
+Supported LBA Sizes (NSID 0x1)
+Id Fmt  Data  Metadt  Rel_Perf
++     512       0         0
+-    4096       0         0
+{...}
+</code>
+Format the device with bigger sector size. Column "Id" above.
+<code>
+nvme format /dev/nvme4n1 -l 1 --force
+</code>
+Verify the change.
+<code>
+root@pve01: smartctl -a /dev/nvme4n1
+{...}
+Supported LBA Sizes (NSID 0x1)
+Id Fmt  Data  Metadt  Rel_Perf
+-     512       0         0
++    4096       0         0
+{...}
+</code>
+https://openzfs.github.io/openzfs-docs/Performance%20and%20Tuning/Hardware.html#nvme-low-level-formatting
+<code>
+lsblk -d -o NAME,PHY-SeC,LOG-SEC
+nvme id-ns -H /dev/nvme4n1 | grep 'LBA'
+</code>
+===== Misc =====
+==== Remove orphaned pool ====
+Debian fails to start import service of an orphaned pool.
+<code>
+zpool set cachefile=/etc/zfs/zpool.cache poolname    # set cache file for running pools
+update-initramfs -k all -u
+reboot
+</code>
+check labels on disks
+<code>
+zdb -l /dev/nvme1n1p1
 </code>