在ZFS上安装Arch Linux (zstd + syslinux)

原创文章，转载请注明出处
https://qiedd.com/

参考Wiki：Install Arch Linux on ZFS

使用的机器为腾讯云香港的 2C 2G
由于 Grub 暂时不支持 zstd 作为根分区的压缩方式，因此使用了 syslinux
如需 lz4 + Grub，请看这里

启动到ArchISO

将系统重装为 Debian 10，然后安装 grub-imageboot

# 安装 grub-imageboot
apt update -y && apt install grub-imageboot -y

# 新建文件夹用于存放ArchISO
mkdir /boot/images

# 进入文件夹
cd /boot/images

# 下载镜像
wget http://mirrors.tencentyun.com/archlinux/iso/2021.12.01/archlinux-2021.12.01-x86_64.iso

# 可选镜像
wget https://mirrors.ustc.edu.cn/archlinux/iso/2021.12.01/archlinux-2021.12.01-x86_64.iso
wget http://mirror.rackspace.com/archlinux/iso/2021.12.01/archlinux-2021.12.01-x86_64.iso

配置grub

# 更新grub
update-grub2

# 按照上面的提示，选择默认启动项，我这里是ArchISO是第三项
vim /etc/default/grub

# 从0开始，所以这里是2
GRUB_DEFAULT=2

# 更新grub
udpate-grub2

# 重启等待即可，登录到VNC
reboot

安装Arch

安装前配置

https://github.com/eoli3n/archiso-zfs

# 安装迁配置，开启ntp
timedatectl set-ntp true

# 设置密码，用于ssh连接
passwd

# 启动sshd
systemctl start sshd

# 安装ZFS模块
curl -s https://eoli3n.github.io/archzfs/init | bash -x

# 查看硬盘
fdisk -l

# 对硬盘进行分区
fdisk /dev/vda

硬盘分区实例

2G作为启动分区，剩余为根分区

root@VM-20-14-debian ~ # fdisk /dev/vda

Welcome to fdisk (util-linux 2.37.2).
Changes will remain in memory only, until you decide to write them.
Be careful before using the write command.


Command (m for help): o
Created a new DOS disklabel with disk identifier 0x5b2e861f.

Command (m for help): n
Partition type
   p   primary (0 primary, 0 extended, 4 free)
   e   extended (container for logical partitions)
Select (default p): 

Using default response p.
Partition number (1-4, default 1): 
First sector (2048-167772159, default 2048): 
Last sector, +/-sectors or +/-size{K,M,G,T,P} (2048-167772159, default 167772159): +2G

Created a new partition 1 of type 'Linux' and of size 2 GiB.
Partition #1 contains a ext4 signature.

Do you want to remove the signature? [Y]es/[N]o: y

The signature will be removed by a write command.

Command (m for help): n
Partition type
   p   primary (1 primary, 0 extended, 3 free)
   e   extended (container for logical partitions)
Select (default p): 

Using default response p.
Partition number (2-4, default 2): 
First sector (4196352-167772159, default 4196352): 
Last sector, +/-sectors or +/-size{K,M,G,T,P} (4196352-167772159, default 167772159): 

Created a new partition 2 of type 'Linux' and of size 78 GiB.

Command (m for help): w
The partition table has been altered.
Calling ioctl() to re-read partition table.
Syncing disks.

Disk /dev/vda: 80 GiB, 85899345920 bytes, 167772160 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disklabel type: dos
Disk identifier: 0x5b2e861f

Device     Boot   Start       End   Sectors Size Id Type
/dev/vda1          2048   4196351   4194304   2G 83 Linux
/dev/vda2       4196352 167772159 163575808  78G 83 Linux

ZFS配置

# 创建zpool
zpool create -f -o ashift=12           \
             -O acltype=posixacl       \
             -O relatime=on            \
             -O xattr=sa               \
             -O dnodesize=legacy       \
             -O normalization=formD    \
             -O mountpoint=none        \
             -O canmount=off           \
             -O devices=off            \
             -O compression=zstd       \
             -R /mnt                   \
             zroot /dev/vda2                  
             
# 创建datasets
zfs create -o mountpoint=none zroot/data
zfs create -o mountpoint=none zroot/ROOT
zfs create -o mountpoint=/ -o canmount=noauto zroot/ROOT/default
zfs create -o mountpoint=/home zroot/data/home
zfs create -o mountpoint=/www zroot/data/www

# 测试zpool是否能够导入导出
zpool export zroot
zpool import zroot -R /mnt

# 挂载zpool
zfs mount zroot/ROOT/default
zfs mount -a

# 设置启动分区
zpool set bootfs=zroot/ROOT/default zroot

# 设置zpool缓存
cp /etc/zfs/zpool.cache /mnt/etc/zfs/zpool.cache

# 如果提示：cp: cannot create regular file '/mnt/etc/zfs/zpool.cache': No such file or directory
zpool set cachefile=/etc/zfs/zpool.cache zroot

# 查看是否有挂载上
df -h
zroot/ROOT/default   37G  128K   37G   1% /mnt

# 如果配置出错，需要重新配置
# umount -R /mnt
# zpool destroy zroot

将Arch装进硬盘中

# 格式化boot分区
mkfs.ext4 /dev/vda1

# 创建/boot
mkdir /mnt/boot

# 挂载/boot
mount /dev/vda1 /mnt/boot

# 设置镜像（腾讯云内网专用）
vim /etc/pacman.d/mirrorlist
Server = http://mirrors.tencentyun.com/archlinux/$repo/os/$arch

# 其他镜像（可选）
Server = https://mirrors.ustc.edu.cn/archlinux/$repo/os/$arch
Server = https://mirrors.tuna.tsinghua.edu.cn/archlinux/$repo/os/$arch
Server = https://mirrors.cloud.tencent.com/archlinux/$repo/os/$arch

# 安装系统,由于内存不足无法使用zfs-dkms，因此先使用zfs-linux，进入系统后再替换
# AMD CPU
pacstrap /mnt base linux linux-firmware linux-headers base-devel vim os-prober amd-ucode openssh wget networkmanager zfs-linux syslinux

# INTEL CPU
pacstrap /mnt base linux linux-firmware linux-headers base-devel vim os-prober intel-ucode openssh wget networkmanager zfs-linux syslinux

# 设置挂载
genfstab -U /mnt >> /mnt/etc/fstab
cat /mnt/etc/fstab

# chroot进Arch
arch-chroot /mnt

# 设置时区
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
hwclock --systohc
date

# 安装语言支持，放在最前面即可
vim /etc/locale.gen
en_US.UTF-8 UTF-8
zh_CN.UTF-8 UTF-8
zh_TW.UTF-8 UTF-8

# 应用配置
locale-gen

# 默认语言
vim /etc/locale.conf
LANG=en_US.UTF-8

# 配置内核钩子，加入zfs支持
vim /etc/mkinitcpio.conf
HOOKS=(base udev autodetect modconf block zfs filesystems keyboard fsck)

# 加载zfs
mkinitcpio -P

# 安装syslinux
syslinux-install_update -i -a -m

# 配置文件
vim /boot/syslinux/syslinux.cfg

# 修改两个APPEDND，两个都要修改
APPEND root=ZFS=zroot/ROOT/default rw

# 设置zpool缓存
zpool set cachefile=/etc/zfs/zpool.cache zroot

# 让zfs服务在开机时自启动
systemctl enable zfs.target
systemctl enable zfs-import-cache
systemctl enable zfs-mount
systemctl enable zfs-import.target
systemctl enable sshd
systemctl enable NetworkManager

# 设置root密码
passwd

# 允许SSH root登录
vim /etc/ssh/sshd_config
*** 示例 ***
PermitRootLogin yes

# 退出chroot
exit

# 取消挂载
umount -R /mnt
zfs umount -a
zpool export zroot

# 同步并重启
sync
reboot

重启后配置

# 添加archlinuxcn源和archzfs源
vim /etc/pacman.conf 

[archlinuxcn]
Server = http://mirrors.tencentyun.com/archlinuxcn/$arch

[archzfs]
Server = http://archzfs.com/$repo/x86_64

# 可选镜像源
# archlinuxcn
# Server = https://mirrors.cloud.tencent.com/archlinuxcn/$arch
# Server = https://mirrors.ustc.edu.cn/archlinuxcn/$arch
# Server = https://repo.archlinuxcn.org/$arch

# archzfs
# Origin Server - France
# Server = http://archzfs.com/$repo/x86_64
# Mirror - Germany
# Server = http://mirror.sum7.eu/archlinux/archzfs/$repo/x86_64
# Mirror - Germany
# Server = https://mirror.biocrafting.net/archlinux/archzfs/$repo/x86_64
# Mirror - India
# Server = https://mirror.in.themindsmaze.com/archzfs/$repo/x86_64
# Mirror - US
# Server = https://zxcvfdsa.com/archzfs/$repo/$arch

# 安装key
pacman-key -r DDF7DB817396A49B2A2723F7403BD972F75D9D76
pacman-key --lsign-key DDF7DB817396A49B2A2723F7403BD972F75D9D76
pacman -Syu archlinuxcn-keyring

# 替换成zfs-dkms
pacman -Rs zfs-linux zfs-utils
pacman -Sy zfs-dkms

# (可选)缩小ARC最大内存占用，请按实际情况设置
vim /boot/syslinux/syslinux.cfg

# 修改APPEND(限制到256M)
APPEND root=ZFS=zroot/ROOT/default rw zfs.zfs_arc_max=268435456

# 更新syslinux配置
syslinux-install_update -u

# 开启bbr
vim /etc/sysctl.conf 

net.core.default_qdisc = cake
net.ipv4.tcp_congestion_control = bbr
net.ipv4.tcp_fastopen = 3

# 设置bbr开机启动
vim /etc/sysctl.d/bbr.conf 

net.core.default_qdisc = cake
net.ipv4.tcp_congestion_control = bbr
net.ipv4.tcp_fastopen = 3

# 取消NetworkManager接管DNS
vim /etc/NetworkManager/conf.d/dns.conf

[main]
dns=none

# 设置dns
vim /etc/resolv.conf

nameserver 1.1.1.1
nameserver 1.0.0.1

# 重启
reboot

# 查看ARC内存占用
cat /proc/spl/kstat/zfs/arcstats|grep c_max

# 清空包缓存
pacman -Scc

# 查看压缩率
zfs get compressratio

NAME                PROPERTY       VALUE  SOURCE
zroot               compressratio  2.08x  -
zroot/ROOT          compressratio  2.08x  -
zroot/ROOT/default  compressratio  2.08x  -
zroot/data          compressratio  1.12x  -
zroot/data/home     compressratio  1.12x  -
zroot/data/www      compressratio  1.00x  -

性能测试

dd 测试

# 4k
time dd if=/dev/random of=test bs=4k count=1048576 status=progress

4294967296 bytes (4.3 GB, 4.0 GiB) copied, 33.4941 s, 128 MB/s

________________________________________________________
Executed in   33.50 secs    fish           external
   usr time    0.73 secs  278.00 micros    0.73 secs
   sys time   31.24 secs  234.00 micros   31.24 secs

# 64k 
time dd if=/dev/random of=test bs=64k count=65536 status=progress

4294967296 bytes (4.3 GB, 4.0 GiB) copied, 26.5634 s, 162 MB/s

________________________________________________________
Executed in   26.57 secs    fish           external
   usr time    0.10 secs  528.00 micros    0.10 secs
   sys time   25.46 secs    0.00 micros   25.46 secs

# 256k
time dd if=/dev/random of=test bs=256k count=16384 status=progress

4294967296 bytes (4.3 GB, 4.0 GiB) copied, 26.2403 s, 164 MB/s

________________________________________________________
Executed in   26.24 secs    fish           external
   usr time    0.04 secs  318.00 micros    0.04 secs
   sys time   24.51 secs  222.00 micros   24.51 secs

# 1M
time dd if=/dev/random of=test bs=1M count=4096 status=progress

4294967296 bytes (4.3 GB, 4.0 GiB) copied, 26.9812 s, 159 MB/s

________________________________________________________
Executed in   26.98 secs    fish           external
   usr time    0.02 secs  323.00 micros    0.02 secs
   sys time   24.10 secs  239.00 micros   24.10 secs

FIO 64K成绩

# 写入
fio --name=seqwrite --rw=write --direct=1 --ioengine=psync --bs=64k --iodepth=64 --numjobs=10 --size=1G --group_reporting

seqwrite: (g=0): rw=write, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=psync, iodepth=64
...
fio-3.28
Starting 10 processes
seqwrite: Laying out IO file (1 file / 1024MiB)
seqwrite: Laying out IO file (1 file / 1024MiB)
seqwrite: Laying out IO file (1 file / 1024MiB)
seqwrite: Laying out IO file (1 file / 1024MiB)
seqwrite: Laying out IO file (1 file / 1024MiB)
seqwrite: Laying out IO file (1 file / 1024MiB)
seqwrite: Laying out IO file (1 file / 1024MiB)
seqwrite: Laying out IO file (1 file / 1024MiB)
seqwrite: Laying out IO file (1 file / 1024MiB)
seqwrite: Laying out IO file (1 file / 1024MiB)
Jobs: 2 (f=2): [_(1),W(2),_(7)][95.6%][w=49.6MiB/s][w=794 IOPS][eta 00m:05s]           
seqwrite: (groupid=0, jobs=10): err= 0: pid=2064: Fri Jan 21 00:16:42 2022
  write: IOPS=1530, BW=95.6MiB/s (100MB/s)(10.0GiB/107081msec); 0 zone resets
    clat (usec): min=8, max=1182.3k, avg=6266.89, stdev=69800.72
     lat (usec): min=9, max=1182.3k, avg=6271.26, stdev=69800.84
    clat percentiles (usec):
     |  1.00th=[     12],  5.00th=[     15], 10.00th=[     17],
     | 20.00th=[     18], 30.00th=[     19], 40.00th=[     32],
     | 50.00th=[     44], 60.00th=[     59], 70.00th=[     80],
     | 80.00th=[     91], 90.00th=[    429], 95.00th=[    881],
     | 99.00th=[  20579], 99.50th=[ 574620], 99.90th=[1061159],
     | 99.95th=[1149240], 99.99th=[1166017]
   bw (  KiB/s): min= 3584, max=794188, per=100.00%, avg=152999.29, stdev=12897.55, samples=1358
   iops        : min=   56, max=12408, avg=2390.26, stdev=201.52, samples=1358
  lat (usec)   : 10=0.94%, 20=33.73%, 50=18.15%, 100=27.47%, 250=5.09%
  lat (usec)   : 500=5.79%, 750=2.95%, 1000=1.42%
  lat (msec)   : 2=1.33%, 4=0.77%, 10=0.82%, 20=0.53%, 50=0.18%
  lat (msec)   : 100=0.01%, 500=0.28%, 750=0.13%, 1000=0.25%, 2000=0.17%
  cpu          : usr=0.07%, sys=0.66%, ctx=48740, majf=0, minf=125
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=0,163840,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=64

Run status group 0 (all jobs):
  WRITE: bw=95.6MiB/s (100MB/s), 95.6MiB/s-95.6MiB/s (100MB/s-100MB/s), io=10.0GiB (10.7GB), run=107081-107081msec

# 读取
fio --name=seqread --rw=read --direct=1 --ioengine=psync --bs=64k --iodepth=64 --numjobs=10 --size=1G --group_reporting

seqread: (g=0): rw=read, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=psync, iodepth=64
...
fio-3.28
Starting 10 processes
seqread: Laying out IO file (1 file / 1024MiB)
seqread: Laying out IO file (1 file / 1024MiB)
seqread: Laying out IO file (1 file / 1024MiB)
seqread: Laying out IO file (1 file / 1024MiB)
seqread: Laying out IO file (1 file / 1024MiB)
seqread: Laying out IO file (1 file / 1024MiB)
seqread: Laying out IO file (1 file / 1024MiB)
seqread: Laying out IO file (1 file / 1024MiB)
seqread: Laying out IO file (1 file / 1024MiB)
seqread: Laying out IO file (1 file / 1024MiB)
Jobs: 2 (f=2): [_(6),R(1),_(1),R(1),_(1)][97.5%][r=265MiB/s][r=4243 IOPS][eta 00m:01s]
seqread: (groupid=0, jobs=10): err= 0: pid=2914: Fri Jan 21 00:18:53 2022
  read: IOPS=4191, BW=262MiB/s (275MB/s)(10.0GiB/39089msec)
    clat (usec): min=7, max=87430, avg=2321.68, stdev=9929.98
     lat (usec): min=7, max=87431, avg=2322.21, stdev=9929.98
    clat percentiles (usec):
     |  1.00th=[    8],  5.00th=[    8], 10.00th=[    8], 20.00th=[    8],
     | 30.00th=[    9], 40.00th=[   12], 50.00th=[   28], 60.00th=[   69],
     | 70.00th=[   72], 80.00th=[   74], 90.00th=[   88], 95.00th=[ 7111],
     | 99.00th=[49546], 99.50th=[50070], 99.90th=[51643], 99.95th=[53216],
     | 99.99th=[79168]
   bw (  KiB/s): min=123096, max=570212, per=100.00%, avg=269605.52, stdev=5995.38, samples=753
   iops        : min= 1923, max= 8906, avg=4212.24, stdev=93.66, samples=753
  lat (usec)   : 10=35.83%, 20=13.40%, 50=3.20%, 100=39.33%, 250=1.40%
  lat (usec)   : 500=0.09%, 750=0.04%, 1000=0.03%
  lat (msec)   : 2=0.12%, 4=1.26%, 10=0.36%, 20=0.20%, 50=4.28%
  lat (msec)   : 100=0.45%
  cpu          : usr=0.10%, sys=1.85%, ctx=12146, majf=0, minf=283
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=163840,0,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=64

Run status group 0 (all jobs):
   READ: bw=262MiB/s (275MB/s), 262MiB/s-262MiB/s (275MB/s-275MB/s), io=10.0GiB (10.7GB), run=39089-39089msec

-------------------- A Bench.sh Script By Teddysun -------------------
 Intro              : https://teddysun.com/444.html
 Version            : v2022-01-01
 Usage              : wget -qO- bench.sh | bash
----------------------------------------------------------------------
 CPU Model          : Intel(R) Xeon(R) CPU E5-26xx v4
 CPU Cores          : 2
 CPU Frequency      : 2394.446 MHz
 CPU Cache          : 4096 KB
 Total Disk         : 85.0 GB (1.7 GB Used)
 Total Mem          : 1976 MB (167 MB Used)
 Total Swap         : 0 MB (0 MB Used)
 System uptime      : 0 days, 0 hour 6 min
 Load average       : 0.34, 1.62, 0.99
 OS                 : Arch Linux 
 Arch               : x86_64 (64 Bit)
 Kernel             : 5.16.1-arch1-1
 TCP CC             : bbr
 Virtualization     : KVM
 Organization       : AS132203 Tencent Building, Kejizhongyi Avenue
 Location           : Sham Shui Po / HK
 Region             : Sham Shui Po
----------------------------------------------------------------------
 I/O Speed(1st run) : 2.0 GB/s
 I/O Speed(2nd run) : 2.2 GB/s
 I/O Speed(3rd run) : 2.2 GB/s
 I/O Speed(average) : 2184.5 MB/s
----------------------------------------------------------------------
SSL_INIT
 Node Name        Upload Speed      Download Speed      Latency     
 Speedtest.net    30.92 Mbps        692.68 Mbps         1.21 ms     
 Los Angeles, US  31.12 Mbps        1647.06 Mbps        153.10 ms   
 Montreal, CA     29.78 Mbps        315.24 Mbps         237.16 ms   
 Amsterdam, NL    30.83 Mbps        579.81 Mbps         303.70 ms   
 Shanghai, CN     32.78 Mbps        1587.13 Mbps        34.95 ms    
 Nanjing, CN      29.81 Mbps        1717.84 Mbps        56.82 ms    
 Guangzhou, CN    2.90 Mbps         448.50 Mbps         195.47 ms   
 Hongkong, CN     31.13 Mbps        1584.85 Mbps        1.39 ms     
 Tokyo, JP        31.04 Mbps        744.83 Mbps         57.67 ms    
----------------------------------------------------------------------
 Finished in        : 5 min 19 sec
 Timestamp          : 2022-01-21 00:14:37
----------------------------------------------------------------------