aboutsummaryrefslogtreecommitdiffstats
path: root/vagrant/pfsense/Vagrantfile
blob: 26f58d1a0999b7c36e89ee50a83cefc002597bd2 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
# vi: set ft=ruby :
# frozen_string_literal: true

require 'yaml'

if File.file?('vm-config.yaml')
  vm_config = YAML.load_file('vm-config.yaml')
  DPDK_VERSION = vm_config['dpdk_version']
  E1000_NIC_COUNT = vm_config['nics']['e1000']
  VIRTIO_NIC_COUNT = vm_config['nics']['virtio']
  DEMO_MOUNT_POINT = vm_config['mount_point']
  HUGETLB_2MG_COUNT = vm_config['huge2mb']
  PXB_COUNT = vm_config['pxb']
  NUMA_COUNT = vm_config['numa']
  TOTAL_MEMORY = vm_config['memory']
  VCORE_COUNT = vm_config['vcore']
else
  # defaults
  DPDK_VERSION = '20.11.3'
  E1000_NIC_COUNT = 4
  VIRTIO_NIC_COUNT = 4
  DEMO_MOUNT_POINT = '/home/vagrant/pfsense'
  HUGETLB_2MG_COUNT = 512
  PXB_COUNT = 2
  NUMA_COUNT = 2
  TOTAL_MEMORY = 12_288
  VCORE_COUNT = 8
end

ENV['VAGRANT_DEFAULT_PROVIDER'] = 'libvirt'
Vagrant.require_version '>= 2.2.6'
Vagrant.configure('2') do |config|
  config.vm.box = 'generic/debian10'
  config.vm.box_version = '3.4.2'
  config.vm.box_check_update = false
  config.vm.synced_folder './source', DEMO_MOUNT_POINT, type: 'nfs'
  config.vm.hostname = 'pfsense'

  config.vm.boot_timeout = 30
  config.vm.graceful_halt_timeout = 30

  # CFLAGS="-I/opt/vagrant/embedded/include/ruby-3.0.0/ruby" vagrant plugin install vagrant-reload
  # CFLAGS="-I/opt/vagrant/embedded/include/ruby-3.0.0/ruby" vagrant plugin install vagrant-libvirt
  config.vagrant.plugins = ['vagrant-reload', { 'vagrant-libvirt' => { 'version' => '^0.6.2' } }]

  config.vm.provider 'libvirt' do |libvirt|
    libvirt.default_prefix = 'deb10-dpdk-'
    libvirt.driver = 'kvm'
    libvirt.nested = true
    libvirt.machine_type = 'pc-q35-3.1'
    libvirt.nic_adapter_count = E1000_NIC_COUNT + VIRTIO_NIC_COUNT + 1
    libvirt.qemuargs value: '-device'
    libvirt.qemuargs value: 'intel-iommu,caching-mode=on,intremap=off,pt=true'
    (0..PXB_COUNT - 1).each do |i|
      libvirt.qemuargs value: '-device'
      libvirt.qemuargs value: "pxb-pcie,id=pcie.#{i + 1},bus=pcie.0,bus_nr=#{180 + i * 20},addr=#{i + 10},numa_node=#{i % NUMA_COUNT}"
    end
    (0..VIRTIO_NIC_COUNT + E1000_NIC_COUNT - 1).each do |i|
      libvirt.qemuargs value: '-device'
      libvirt.qemuargs value: "ioh3420,port=0x3#{i},id=rppci.#{i},bus=pcie.#{(i / 4) + 1},chassis=#{i + 7},slot=#{i % 4},addr=0x#{i % 4}"
    end
    (0..VIRTIO_NIC_COUNT - 1).each do |i|
      libvirt.qemuargs value: '-netdev'
      libvirt.qemuargs value: "user,id=net#{i + 4}"
      libvirt.qemuargs value: '-device'
      libvirt.qemuargs value: "virtio-net-pci,netdev=net#{i + 4},mac=de:ad:be:ef:00:0#{i + 4},bus=rppci.#{i},mq=on,vectors=32,status=off"
    end
    (0..E1000_NIC_COUNT - 1).each do |i|
      libvirt.qemuargs value: '-netdev'
      libvirt.qemuargs value: "user,id=net#{i}"
      libvirt.qemuargs value: '-device'
      libvirt.qemuargs value: "e1000,netdev=net#{i},mac=de:ad:be:ef:00:0#{i},bus=rppci.#{i + 4}"
    end
    libvirt.qemuargs value: '-object'
    libvirt.qemuargs value: 'rng-random,id=rng0,filename=/dev/urandom'
    libvirt.qemuargs value: '-device'
    libvirt.qemuargs value: 'virtio-rng-pci,rng=rng0,bus=pcie.0'
    libvirt.qemuargs value: '-nographic'
    libvirt.qemuargs value: '-nodefaults'
    libvirt.qemuargs value: '-no-user-config'
    libvirt.qemuargs value: '-enable-kvm'
    # libvirt.qemuargs :value => "-chardev"
    # libvirt.qemuargs :value => "socket,path=/tmp/ivshmem_socket,id=ivshmem_socket"
    # libvirt.qemuargs :value => "-device"
    # libvirt.qemuargs :value => "ivshmem,chardev=ivshmem_socket,size=1m"
    libvirt.sound_type = nil
    libvirt.cpus = VCORE_COUNT
    libvirt.cputopology sockets: NUMA_COUNT.to_s, cores: (VCORE_COUNT / NUMA_COUNT).to_s, threads: '1'
    libvirt.cpu_fallback = 'forbid'
    libvirt.numa_nodes = [
      { cpus: '0-3', memory: (TOTAL_MEMORY / 2).to_s },
      { cpus: '4-7', memory: (TOTAL_MEMORY / 2).to_s }
    ]
    libvirt.random model: 'random'
    libvirt.autostart = true
    libvirt.features = %w[acpi apic]
    libvirt.channel type: 'unix', target_name: 'org.qemu.guest_agent.0', target_type: 'virtio'
    libvirt.disk_driver cache: 'writeback', io: nil, copy_on_read: 'on', discard: nil, detect_zeroes: nil
    libvirt.clock_timer name: 'hpet', present: 'yes'
  end

  config.vm.provision 'enable-iommu', type: 'shell', name: 'enable-iommu', privileged: true, reboot: false,
                                      inline: <<-SHELL
    set -ex
    cp #{DEMO_MOUNT_POINT}/conf/grub /etc/default/grub
    echo "options vfio_iommu_type1 allow_unsafe_interrupts=1" > /etc/modprobe.d/unsafe-interrupts.conf
    update-grub
                                      SHELL

  # https://github.com/vagrant-libvirt/vagrant-libvirt/issues/1366
  # https://github.com/hashicorp/vagrant/issues/11632
  config.vm.provision :reload

  config.vm.provision 'hugepages', type: 'shell', name: 'hugepages', privileged: true, reboot: false, reset: true,
                                   run: 'always', inline: <<-SHELL
    set -ex
    groupadd dpdk || true
    usermod -aG dpdk vagrant
    echo "@dpdk - memlock unlimited" >> /etc/security/limits.conf
    /usr/bin/bash -c "echo #{HUGETLB_2MG_COUNT} > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages"
    /usr/bin/bash -c "echo #{HUGETLB_2MG_COUNT} > /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages"
    mkdir -p /dev/hugepages-2MB &&\
      mountpoint -q /dev/hugepages-2MB || mount -o pagesize=2097152 -t hugetlbfs nodev /dev/hugepages-2MB
    mkdir -p /dev/hugepages-1GB &&\
      mountpoint -q /dev/hugepages-1GB || mount -o pagesize=1073741824 -t hugetlbfs nodev /dev/hugepages-1GB
    echo "nodev /mnt/huge hugetlbfs defaults 0 0" >> /etc/fstab
    # recommended to increase limit of open files when using a lot of huge pages
    ulimit -Sn 2048
                                   SHELL

  config.vm.provision 'dpdk-install', name: 'dpdk-install', type: 'shell', privileged: false, reboot: false,
                                      inline: <<-SHELL
    set -ex
    sudo apt update && sudo apt upgrade -y
    sudo apt install -y build-essential
    sudo apt install -y libnuma-dev python3-pyelftools libarchive-dev libelf-dev python3 python3-pip meson ninja-build libssl-dev zlib1g-dev
    sudo apt install -y tmux vim wget universal-ctags cscope git devscripts w3m cmake gdb sshfs libhugetlbfs-bin ethtool
    cd /home/vagrant &&\
      wget -q https://fast.dpdk.org/rel/dpdk-#{DPDK_VERSION}.tar.xz &&\
      tar -xvf dpdk-#{DPDK_VERSION}.tar.xz &&\
      cd dpdk-stable-#{DPDK_VERSION} &&\
      meson build &&\
      cd build &&\
      meson configure -Dexamples=all -Duse_hpet=true -Dbuildtype=debug -Denable_trace_fp=true &&\
      ninja &&\
      ninja -t compdb &&\
      sudo ninja install &&\
      sudo ldconfig
                                      SHELL

  config.vm.provision 'dpdk-pmd', type: 'shell', name: 'dpdk-pmd', privileged: true, reboot: false, run: 'always',
                                  inline: <<-SHELL
    set -ex
    modprobe vfio-pci
    NICS=$(lspci -nn -D | grep -i "gigabit\ ethernet" | gawk '{sum=sum" "$1}END{print sum}')
    IFS=" ";read -ra NICS_ARRAY <<< "$NICS"
    for NIC in "${NICS_ARRAY[@]}";do
      /home/vagrant/dpdk-stable-#{DPDK_VERSION}/usertools/dpdk-devbind.py --bind vfio-pci $NIC || true
    done
    # we don't wanna blow out the one NIC that's giving us internet/ssh
    EXCLUSION_LIST=$(/sbin/ethtool -i eth0 | grep bus-info | gawk '{print $2}')
    NICS=$(lspci -nn -D | grep -v $EXCLUSION_LIST | grep -i "virtio\ network\ device" | gawk '{sum=sum" "$1}END{print sum}')
    IFS=" ";read -ra NICS_ARRAY <<< "$NICS"
    for NIC in "${NICS_ARRAY[@]}";do
      /home/vagrant/dpdk-stable-#{DPDK_VERSION}/usertools/dpdk-devbind.py --bind vfio-pci $NIC || true
    done
    # kill ASLR. we need to do this for multiprocess
    echo 0 > /proc/sys/kernel/randomize_va_space
                                  SHELL

  config.vm.provision 'dpdk-sudoless', name: 'dpdk-sudoless', type: 'shell', privileged: true, reboot: false,
                                       run: 'always', inline: <<-SHELL
    set -ex
    chown root:dpdk /dev/hpet
    chown -R root:dpdk /dev/vfio
    chmod 660 /dev/hpet
    chmod 770 /dev/vfio
    chmod 660 /dev/vfio/*
    chmod 770 /dev/hugepages-1GB
    chown root:dpdk /dev/hugepages-1GB
    chmod 770 /dev/hugepages-2MB
    chown root:dpdk /dev/hugepages-2MB
                                       SHELL

  config.vm.provision 'dpdk-test', type: 'shell', name: 'dpdk-test', privileged: false, reboot: false, run: 'never',
                                   inline: <<-SHELL
    set -ex
    cd /home/vagrant/dpdk-stable-#{DPDK_VERSION}/build &&\
      sudo meson test --suite fast-tests --timeout 100
    cd /home/vagrant/dpdk-stable-#{DPDK_VERSION}/build &&\
      meson test --suite driver-tests
    /home/vagrant/dpdk-stable-#{DPDK_VERSION}/build/app/test/dpdk-test -n2 -l4 --vfio-int=legacy --huge-dir=/dev/hugepages-2MB --allow 0000:00:03.0 --allow 0000:00:04.0
    make -C /home/vagrant/dpdk-stable-#{DPDK_VERSION}/examples/helloworld &&\
      /home/vagra/tdpdk-stable-#{DPDK_VERSION}/examples/helloworld/build/helloworld
    /home/vagra/tdpdk-stable-#{DPDK_VERSION}/build/app/dpdk-testpmd -l 0-3 -n 4 --vfio-int=legacy -- -i
    make -C /home/vagrant/dpdk-stable-#{DPDK_VERSION}/examples/skeleton &&\
      /home/vagra/tdpdk-stable-#{DPDK_VERSION}/examples/skeleton/build/basicfwd -l1 -n4 --vfio-int=legacy
                                   SHELL

  if ARGV[0] == '--farzad'
    config.vm.provision 'farzad', type: 'shell', name: 'farzad', privileged: false, reboot: false, reset: true,
                                  inline: <<-SHELL
      set -ex
      sudo apt install -y apt-file hwloc numactl strace ltrace babeltrace htop
      sudo apt-file update
      echo "set -o vi" >> ~/.bashrc
                                  SHELL
  end
end