VyOS 10gbe throughput optimization

Disable offloading, enable ntuple

ethtool -K eth1 rx on tx on sg on tso off gso off gro off lro off ntuple on rxhash on
ethtool -K eth3 rx on tx on sg on tso off gso off gro off lro off ntuple on rxhash on

Configure MAX RPS flow counts for queues

echo 32768 > /proc/sys/net/core/rps_sock_flow_entries
for card in eth1 eth3; do
  for queue in $(seq 0 5); do
    echo 5461 > /sys/class/net/$card/queues/rx-$queue/rps_flow_cnt
  done
done

TCP Tweaks:

set parameter net.core.netdev_budget value '100000'
set parameter net.core.netdev_max_backlog value '300000'
set parameter net.core.somaxconn value '8192'
set parameter net.ipv4.icmp_msgs_burst value '100000'
set parameter net.ipv4.icmp_msgs_per_sec value '90000'
set parameter net.ipv4.icmp_ratelimit value '0'
set parameter net.ipv4.icmp_ratemask value '88089'
set parameter net.ipv4.ipfrag_high_thresh value '544288'
set parameter net.ipv4.ipfrag_low_thresh value '393216'
set parameter net.ipv4.tcp_keepalive_intvl value '10'
set parameter net.ipv4.tcp_keepalive_probes value '3'
set parameter net.ipv4.tcp_keepalive_time value '600'
set parameter net.ipv4.tcp_max_syn_backlog value '8192'
set parameter net.ipv4.tcp_mem value '10000000 10000000 10000000'
set parameter net.ipv4.tcp_moderate_rcvbuf value '1'
set parameter net.ipv4.tcp_orphan_retries value '1'
set parameter net.ipv4.tcp_retries2 value '5'
set parameter net.ipv4.tcp_rmem value '10000000 10000000 10000000'
set parameter net.ipv4.tcp_synack_retries value '3'
set parameter net.ipv4.tcp_syn_retries value '3'
set parameter net.ipv4.tcp_tso_win_divisor value '8'
set parameter net.ipv4.tcp_wmem value '10000000 10000000 10000000'
set parameter net.netfilter.nf_conntrack_buckets value '8388608'
set parameter net.netfilter.nf_conntrack_tcp_timeout_established value '600'

Setting the MaxReadReq to 2048

#To get the current value use this command:
setpci -s 04:00.1 a8.w

# This should return 283f. If not, then something is not working as expected.
# Change it:
setpci -s 04:00.1 a8.w=483f

Disable all offloading on the interfaces.