1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
|
# vi: set ft=ruby :
# frozen_string_literal: true
require 'yaml'
if File.file?('vm-config.yaml')
vm_config = YAML.load_file('vm-config.yaml')
DPDK_VERSION = vm_config['dpdk_version']
E1000_NIC_COUNT = vm_config['nics']['e1000']
VIRTIO_NIC_COUNT = vm_config['nics']['virtio']
DEMO_MOUNT_POINT = vm_config['mount_point']
HUGETLB_2MG_COUNT = vm_config['huge2mb']
PXB_COUNT = vm_config['pxb']
NUMA_COUNT = vm_config['numa']
TOTAL_MEMORY = vm_config['memory']
VCORE_COUNT = vm_config['vcore']
else
# defaults
DPDK_VERSION = '20.11.3'
E1000_NIC_COUNT = 4
VIRTIO_NIC_COUNT = 4
DEMO_MOUNT_POINT = '/home/vagrant/pfsense'
HUGETLB_2MG_COUNT = 512
PXB_COUNT = 2
NUMA_COUNT = 2
TOTAL_MEMORY = 12_288
VCORE_COUNT = 8
end
ENV['VAGRANT_DEFAULT_PROVIDER'] = 'libvirt'
Vagrant.require_version '>= 2.2.6'
Vagrant.configure('2') do |config|
config.vm.box = 'generic/debian10'
config.vm.box_version = '3.4.2'
config.vm.box_check_update = false
config.vm.synced_folder './source', DEMO_MOUNT_POINT, type: 'nfs'
config.vm.hostname = 'pfsense'
config.vm.boot_timeout = 30
config.vm.graceful_halt_timeout = 30
# CFLAGS="-I/opt/vagrant/embedded/include/ruby-3.0.0/ruby" vagrant plugin install vagrant-reload
# CFLAGS="-I/opt/vagrant/embedded/include/ruby-3.0.0/ruby" vagrant plugin install vagrant-libvirt
config.vagrant.plugins = ['vagrant-reload', { 'vagrant-libvirt' => { 'version' => '^0.6.2' } }]
config.vm.provider 'libvirt' do |libvirt|
libvirt.default_prefix = 'deb10-dpdk-'
libvirt.driver = 'kvm'
libvirt.nested = true
libvirt.machine_type = 'pc-q35-3.1'
libvirt.nic_adapter_count = E1000_NIC_COUNT + VIRTIO_NIC_COUNT + 1
libvirt.qemuargs value: '-device'
libvirt.qemuargs value: 'intel-iommu,caching-mode=on,intremap=off,pt=true'
(0..PXB_COUNT - 1).each do |i|
libvirt.qemuargs value: '-device'
libvirt.qemuargs value: "pxb-pcie,id=pcie.#{i + 1},bus=pcie.0,bus_nr=#{180 + i * 20},addr=#{i + 10},numa_node=#{i % NUMA_COUNT}"
end
(0..VIRTIO_NIC_COUNT + E1000_NIC_COUNT - 1).each do |i|
libvirt.qemuargs value: '-device'
libvirt.qemuargs value: "ioh3420,port=0x3#{i},id=rppci.#{i},bus=pcie.#{(i / 4) + 1},chassis=#{i + 7},slot=#{i % 4},addr=0x#{i % 4}"
end
(0..VIRTIO_NIC_COUNT - 1).each do |i|
libvirt.qemuargs value: '-netdev'
libvirt.qemuargs value: "user,id=net#{i + 4}"
libvirt.qemuargs value: '-device'
libvirt.qemuargs value: "virtio-net-pci,netdev=net#{i + 4},mac=de:ad:be:ef:00:0#{i + 4},bus=rppci.#{i},mq=on,vectors=32,status=off"
end
(0..E1000_NIC_COUNT - 1).each do |i|
libvirt.qemuargs value: '-netdev'
libvirt.qemuargs value: "user,id=net#{i}"
libvirt.qemuargs value: '-device'
libvirt.qemuargs value: "e1000,netdev=net#{i},mac=de:ad:be:ef:00:0#{i},bus=rppci.#{i + 4}"
end
libvirt.qemuargs value: '-object'
libvirt.qemuargs value: 'rng-random,id=rng0,filename=/dev/urandom'
libvirt.qemuargs value: '-device'
libvirt.qemuargs value: 'virtio-rng-pci,rng=rng0,bus=pcie.0'
libvirt.qemuargs value: '-nographic'
libvirt.qemuargs value: '-nodefaults'
libvirt.qemuargs value: '-no-user-config'
libvirt.qemuargs value: '-enable-kvm'
# libvirt.qemuargs :value => "-chardev"
# libvirt.qemuargs :value => "socket,path=/tmp/ivshmem_socket,id=ivshmem_socket"
# libvirt.qemuargs :value => "-device"
# libvirt.qemuargs :value => "ivshmem,chardev=ivshmem_socket,size=1m"
libvirt.sound_type = nil
libvirt.cpus = VCORE_COUNT
libvirt.cputopology sockets: NUMA_COUNT.to_s, cores: (VCORE_COUNT / NUMA_COUNT).to_s, threads: '1'
libvirt.cpu_fallback = 'forbid'
libvirt.numa_nodes = [
{ cpus: '0-3', memory: (TOTAL_MEMORY / 2).to_s },
{ cpus: '4-7', memory: (TOTAL_MEMORY / 2).to_s }
]
libvirt.random model: 'random'
libvirt.autostart = true
libvirt.features = %w[acpi apic]
libvirt.channel type: 'unix', target_name: 'org.qemu.guest_agent.0', target_type: 'virtio'
libvirt.disk_driver cache: 'writeback', io: nil, copy_on_read: 'on', discard: nil, detect_zeroes: nil
libvirt.clock_timer name: 'hpet', present: 'yes'
end
config.vm.provision 'enable-iommu', type: 'shell', name: 'enable-iommu', privileged: true, reboot: false,
inline: <<-SHELL
set -ex
cp #{DEMO_MOUNT_POINT}/conf/grub /etc/default/grub
echo "options vfio_iommu_type1 allow_unsafe_interrupts=1" > /etc/modprobe.d/unsafe-interrupts.conf
update-grub
SHELL
# https://github.com/vagrant-libvirt/vagrant-libvirt/issues/1366
# https://github.com/hashicorp/vagrant/issues/11632
config.vm.provision :reload
config.vm.provision 'hugepages', type: 'shell', name: 'hugepages', privileged: true, reboot: false, reset: true,
run: 'always', inline: <<-SHELL
set -ex
groupadd dpdk || true
usermod -aG dpdk vagrant
echo "@dpdk - memlock unlimited" >> /etc/security/limits.conf
/usr/bin/bash -c "echo #{HUGETLB_2MG_COUNT} > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages"
/usr/bin/bash -c "echo #{HUGETLB_2MG_COUNT} > /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages"
mkdir -p /dev/hugepages-2MB &&\
mountpoint -q /dev/hugepages-2MB || mount -o pagesize=2097152 -t hugetlbfs nodev /dev/hugepages-2MB
mkdir -p /dev/hugepages-1GB &&\
mountpoint -q /dev/hugepages-1GB || mount -o pagesize=1073741824 -t hugetlbfs nodev /dev/hugepages-1GB
echo "nodev /mnt/huge hugetlbfs defaults 0 0" >> /etc/fstab
# recommended to increase limit of open files when using a lot of huge pages
ulimit -Sn 2048
SHELL
config.vm.provision 'dpdk-install', name: 'dpdk-install', type: 'shell', privileged: false, reboot: false,
inline: <<-SHELL
set -ex
sudo apt update && sudo apt upgrade -y
sudo apt install -y build-essential
sudo apt install -y libnuma-dev python3-pyelftools libarchive-dev libelf-dev python3 python3-pip meson ninja-build libssl-dev zlib1g-dev
sudo apt install -y tmux vim wget universal-ctags cscope git devscripts w3m cmake gdb sshfs libhugetlbfs-bin ethtool
cd /home/vagrant &&\
wget -q https://fast.dpdk.org/rel/dpdk-#{DPDK_VERSION}.tar.xz &&\
tar -xvf dpdk-#{DPDK_VERSION}.tar.xz &&\
cd dpdk-stable-#{DPDK_VERSION} &&\
meson build &&\
cd build &&\
meson configure -Dexamples=all -Duse_hpet=true -Dbuildtype=debug -Denable_trace_fp=true &&\
ninja &&\
ninja -t compdb &&\
sudo ninja install &&\
sudo ldconfig
SHELL
config.vm.provision 'dpdk-pmd', type: 'shell', name: 'dpdk-pmd', privileged: true, reboot: false, run: 'always',
inline: <<-SHELL
set -ex
modprobe vfio-pci
NICS=$(lspci -nn -D | grep -i "gigabit\ ethernet" | gawk '{sum=sum" "$1}END{print sum}')
IFS=" ";read -ra NICS_ARRAY <<< "$NICS"
for NIC in "${NICS_ARRAY[@]}";do
/home/vagrant/dpdk-stable-#{DPDK_VERSION}/usertools/dpdk-devbind.py --bind vfio-pci $NIC || true
done
# we don't wanna blow out the one NIC that's giving us internet/ssh
EXCLUSION_LIST=$(/sbin/ethtool -i eth0 | grep bus-info | gawk '{print $2}')
NICS=$(lspci -nn -D | grep -v $EXCLUSION_LIST | grep -i "virtio\ network\ device" | gawk '{sum=sum" "$1}END{print sum}')
IFS=" ";read -ra NICS_ARRAY <<< "$NICS"
for NIC in "${NICS_ARRAY[@]}";do
/home/vagrant/dpdk-stable-#{DPDK_VERSION}/usertools/dpdk-devbind.py --bind vfio-pci $NIC || true
done
# kill ASLR. we need to do this for multiprocess
echo 0 > /proc/sys/kernel/randomize_va_space
SHELL
config.vm.provision 'dpdk-sudoless', name: 'dpdk-sudoless', type: 'shell', privileged: true, reboot: false,
run: 'always', inline: <<-SHELL
set -ex
chown root:dpdk /dev/hpet
chown -R root:dpdk /dev/vfio
chmod 660 /dev/hpet
chmod 770 /dev/vfio
chmod 660 /dev/vfio/*
chmod 770 /dev/hugepages-1GB
chown root:dpdk /dev/hugepages-1GB
chmod 770 /dev/hugepages-2MB
chown root:dpdk /dev/hugepages-2MB
SHELL
config.vm.provision 'dpdk-test', type: 'shell', name: 'dpdk-test', privileged: false, reboot: false, run: 'never',
inline: <<-SHELL
set -ex
cd /home/vagrant/dpdk-stable-#{DPDK_VERSION}/build &&\
sudo meson test --suite fast-tests --timeout 100
cd /home/vagrant/dpdk-stable-#{DPDK_VERSION}/build &&\
meson test --suite driver-tests
/home/vagrant/dpdk-stable-#{DPDK_VERSION}/build/app/test/dpdk-test -n2 -l4 --vfio-int=legacy --huge-dir=/dev/hugepages-2MB --allow 0000:00:03.0 --allow 0000:00:04.0
make -C /home/vagrant/dpdk-stable-#{DPDK_VERSION}/examples/helloworld &&\
/home/vagra/tdpdk-stable-#{DPDK_VERSION}/examples/helloworld/build/helloworld
/home/vagra/tdpdk-stable-#{DPDK_VERSION}/build/app/dpdk-testpmd -l 0-3 -n 4 --vfio-int=legacy -- -i
make -C /home/vagrant/dpdk-stable-#{DPDK_VERSION}/examples/skeleton &&\
/home/vagra/tdpdk-stable-#{DPDK_VERSION}/examples/skeleton/build/basicfwd -l1 -n4 --vfio-int=legacy
SHELL
if ARGV[0] == '--farzad'
config.vm.provision 'farzad', type: 'shell', name: 'farzad', privileged: false, reboot: false, reset: true,
inline: <<-SHELL
set -ex
sudo apt install -y apt-file hwloc numactl strace ltrace babeltrace htop
sudo apt-file update
echo "set -o vi" >> ~/.bashrc
SHELL
end
end
|
