Vpn-policy-routing performance issue

Dear all,

so far I have used openwrt with vpn-policy-routing for >1,5 years. Due to several changes and especially a new openwrt version (OpenWrt 21.02.2 r16), I have updated openwrt from scratch on my router.

Sadly, the performance I have now is really bad and not sufficient for streaming and/or video calls (with vpn-policy-routing I get only 1,34 - 3,03 mbit/s down). Here are some numbers:

100 mbit/s down / VPN-policy-routing / openwrt no VPN
VPN connection / 1,34 mbit/s down / 24 mbit/s down (vpn on pc)
direct wan access / 30 mbit/s down / 66 mbit/s down

  • I have used speedtest on Debian for these numbers
  • VPN-policiy-routing is installed on a linksys wrt1900 acs
  • openwrt without VPN is installed on a tp-link archer C6 V2
  • by the way, I could measure also 88 mbit/s down with a different router

As recommended on the docs.openwrt.melmac.net page, I provide the required documents enclosed. Maybe someone can point out to me what I have missed during the configuration...

1- content of /etc/config/dhcp
2- content of /etc/config/firewall
3- content of /etc/config/network
4- content of /etc/config/vpn-policy-routing
5- the output of /etc/init.d/vpn-policy-routing support
6- the output of /etc/init.d/vpn-policy-routing reload with verbosity setting set to 2

BR Martin

1- dhcp

config dnsmasq
	option domainneeded '1'
	option localise_queries '1'
	option rebind_protection '1'
	option rebind_localhost '1'
	option local '/lan/'
	option domain 'lan'
	option expandhosts '1'
	option authoritative '1'
	option readethers '1'
	option leasefile '/tmp/dhcp.leases'
	option resolvfile '/tmp/resolv.conf.d/resolv.conf.auto'
	option localservice '1'
	option ednspacket_max '1232'

config dhcp 'lan'
	option interface 'lan'
	option leasetime '12h'
	option dhcpv4 'server'
	option dhcpv6 'server'
	option ra 'server'
	list ra_flags 'managed-config'
	list ra_flags 'other-config'
	option start '210'
	option limit '11'

config dhcp 'wan'
	option interface 'wan'
	option ignore '1'

config odhcpd 'odhcpd'
	option maindhcp '0'
	option leasefile '/tmp/hosts/odhcpd'
	option leasetrigger '/usr/sbin/odhcpd-update'
	option loglevel '4'

config host
	option name 'mmnat2'
	option dns '1'
	option mac '20:89:84:38:78:A5'
	option ip '192.168.33.122'

2- firewall

config defaults
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'REJECT'
	option synflood_protect '1'

config zone
	option name 'lan'
	list network 'lan'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'ACCEPT'

config zone
	option name 'wan'
	list network 'wan'
	list network 'wan6'
	option input 'REJECT'
	option output 'ACCEPT'
	option forward 'REJECT'
	option masq '1'
	option mtu_fix '1'

config forwarding
	option src 'lan'
	option dest 'wan'

config rule
	option name 'Allow-DHCP-Renew'
	option src 'wan'
	option proto 'udp'
	option dest_port '68'
	option target 'ACCEPT'
	option family 'ipv4'

config rule
	option name 'Allow-Ping'
	option src 'wan'
	option proto 'icmp'
	option icmp_type 'echo-request'
	option family 'ipv4'
	option target 'ACCEPT'

config rule
	option name 'Allow-IGMP'
	option src 'wan'
	option proto 'igmp'
	option family 'ipv4'
	option target 'ACCEPT'

config rule
	option name 'Allow-DHCPv6'
	option src 'wan'
	option proto 'udp'
	option src_ip 'fc00::/6'
	option dest_ip 'fc00::/6'
	option dest_port '546'
	option family 'ipv6'
	option target 'ACCEPT'

config rule
	option name 'Allow-MLD'
	option src 'wan'
	option proto 'icmp'
	option src_ip 'fe80::/10'
	list icmp_type '130/0'
	list icmp_type '131/0'
	list icmp_type '132/0'
	list icmp_type '143/0'
	option family 'ipv6'
	option target 'ACCEPT'

config rule
	option name 'Allow-ICMPv6-Input'
	option src 'wan'
	option proto 'icmp'
	list icmp_type 'echo-request'
	list icmp_type 'echo-reply'
	list icmp_type 'destination-unreachable'
	list icmp_type 'packet-too-big'
	list icmp_type 'time-exceeded'
	list icmp_type 'bad-header'
	list icmp_type 'unknown-header-type'
	list icmp_type 'router-solicitation'
	list icmp_type 'neighbour-solicitation'
	list icmp_type 'router-advertisement'
	list icmp_type 'neighbour-advertisement'
	option limit '1000/sec'
	option family 'ipv6'
	option target 'ACCEPT'

config rule
	option name 'Allow-ICMPv6-Forward'
	option src 'wan'
	option dest '*'
	option proto 'icmp'
	list icmp_type 'echo-request'
	list icmp_type 'echo-reply'
	list icmp_type 'destination-unreachable'
	list icmp_type 'packet-too-big'
	list icmp_type 'time-exceeded'
	list icmp_type 'bad-header'
	list icmp_type 'unknown-header-type'
	option limit '1000/sec'
	option family 'ipv6'
	option target 'ACCEPT'

config rule
	option name 'Allow-IPSec-ESP'
	option src 'wan'
	option dest 'lan'
	option proto 'esp'
	option target 'ACCEPT'

config rule
	option name 'Allow-ISAKMP'
	option src 'wan'
	option dest 'lan'
	option dest_port '500'
	option proto 'udp'
	option target 'ACCEPT'

config rule
	option name 'Support-UDP-Traceroute'
	option src 'wan'
	option dest_port '33434:33689'
	option proto 'udp'
	option family 'ipv4'
	option target 'REJECT'
	option enabled 'false'

config include
	option path '/etc/firewall.user'

config zone
	option input 'REJECT'
	option forward 'REJECT'
	option output 'ACCEPT'
	option name 'vpn1'
	option masq '1'
	option network 'vpn1'
	option mtu_fix '1'

config zone
	option input 'REJECT'
	option forward 'REJECT'
	option output 'ACCEPT'
	option name 'vpn2'
	option masq '1'
	option network 'vpn2'
	option mtu_fix '1'

config zone
	option forward 'REJECT'
	option output 'ACCEPT'
	option name 'vpn3'
	option masq '1'
	option network 'vpn3'
	option mtu_fix '1'
	option input 'ACCEPT'

config forwarding
	option dest 'vpn1'
	option src 'lan'

config forwarding
	option dest 'vpn2'
	option src 'lan'

config forwarding
	option dest 'vpn3'
	option src 'lan'

3- network

config interface 'loopback'
	option device 'lo'
	option proto 'static'
	option ipaddr '127.0.0.1'
	option netmask '255.0.0.0'

config globals 'globals'

config device
	option name 'br-lan'
	option type 'bridge'
	list ports 'lan1'
	list ports 'lan2'
	list ports 'lan3'
	list ports 'lan4'

config interface 'lan'
	option device 'br-lan'
	option proto 'static'
	option netmask '255.255.255.0'
	option ip6assign '60'
	list dns '103.86.96.100'
	list dns '103.86.99.100'
	option ipaddr '192.168.33.1'

config device
	option name 'wan'
	option macaddr '32:23:03:9f:45:1b'

config interface 'wan'
	option device 'wan'
	option proto 'dhcp'

config interface 'wan6'
	option device 'wan'
	option proto 'dhcpv6'

config interface 'vpn1'
	option proto 'none'
	list dns '103.86.96.100'
	list dns '103.86.99.100'
	option device 'tun-vpn1'

config interface 'vpn2'
	option proto 'none'
	list dns '103.86.96.100'
	list dns '103.86.99.100'
	option device 'tun-vpn2'

config interface 'vpn3'
	option proto 'none'
	list dns '103.86.96.100'
	list dns '103.86.99.100'
	option device 'tun-vpn3'

4- vpn-policy-routing

config policy
	option interface 'vpn1'
	option name 'test1'
	option src_addr '192.168.33.251'

config policy
	option src_addr '192.168.33.81'
	option interface 'vpn1'
	option name 'voip1'

config policy
	option src_addr '192.168.33.82'
	option interface 'vpn1'
	option name 'voip2'

config policy
	option src_addr '192.168.33.83'
	option interface 'vpn1'
	option name 'voip3'

config policy
	option interface 'vpn2'
	option name 'test2'
	option src_addr '192.168.33.252'
	option enabled '0'

config policy
	option name 'test3'
	option src_addr '192.168.33.253'
	option interface 'vpn3'
	option enabled '0'

config policy
	option interface 'wan'
	option name 'test0'
	option src_addr '192.168.33.250'

config policy
	option interface 'wan'
	option name 'ps4wzi'
	option src_addr '192.168.33.160'

config policy
	option interface 'wan'
	option src_addr '192.168.33.120'
	option name 'nat'

config policy
	option name 'default'
	option src_addr '192.168.33.0/24'
	option interface 'vpn2'

config vpn-policy-routing 'config'
	option strict_enforcement '1'
	option src_ipset '0'
	option dest_ipset '0'
	option resolver_ipset 'dnsmasq.ipset'
	option ipv6_enabled '0'
	option boot_timeout '30'
	option iptables_rule_option 'append'
	option procd_reload_delay '1'
	option webui_sorting '1'
	option enabled '1'
	list webui_supported_protocol 'tcp'
	list webui_supported_protocol 'udp'
	list webui_supported_protocol 'tcp udp'
	list webui_supported_protocol 'icmp'
	list webui_supported_protocol 'all'
	option verbosity '2'
	list supported_interface 'vpn1 vpn2 vpn3'
	list ignored_interface 'vpnserver wgserver'
	list ignored_interface 'wgserver'
	option webui_enable_column '1'
	option webui_protocol_column '1'
	option webui_chain_column '1'
	option webui_show_ignore_target '1'

config include
	option path '/etc/vpn-policy-routing.netflix.user'
	option enabled '0'

config include
	option path '/etc/vpn-policy-routing.aws.user'
	option enabled '0'

config include
	option path '/etc/vpn-policy-routing.netflix.user'
	option enabled '0'

config include
	option path '/etc/vpn-policy-routing.aws.user'
	option enabled '0'

5- output support

vpn-policy-routing 0.3.4-8 running on OpenWrt 21.02.2.
                                                            
Dnsmasq version 2.85  Copyright (c) 2000-2021 Simon Kelley
Compile time options: IPv6 GNU-getopt no-DBus UBus no-i18n no-IDN DHCP no-DHCPv6 no-Lua TFTP no-conntrack no-ipset no-auth no-cryptohash no-DNSSEC no-ID loop-detect inotify dumpfile
                                                            
Routes/IP Rules
default         192.168.7.3     0.0.0.0         UG    0      0        0 wan

IPv4 Table 201: default via 192.168.7.3 dev wan 
192.168.33.0/24 dev br-lan proto kernel scope link src 192.168.33.1 
IPv4 Table 201 Rules:
32749:	from all fwmark 0x10000/0xff0000 lookup wan

IPv4 Table 202: default via 10.7.2.2 dev tun-vpn1 
192.168.33.0/24 dev br-lan proto kernel scope link src 192.168.33.1 
IPv4 Table 202 Rules:
32748:	from all fwmark 0x20000/0xff0000 lookup vpn1

IPv4 Table 203: default via 10.7.1.4 dev tun-vpn2 
192.168.33.0/24 dev br-lan proto kernel scope link src 192.168.33.1 
IPv4 Table 203 Rules:
32747:	from all fwmark 0x30000/0xff0000 lookup vpn2

IPv4 Table 204: unreachable default 
192.168.33.0/24 dev br-lan proto kernel scope link src 192.168.33.1 
IPv4 Table 204 Rules:
32746:	from all fwmark 0x40000/0xff0000 lookup vpn3
                                                            
Mangle IP Table: PREROUTING
-N VPR_PREROUTING
-A VPR_PREROUTING -s 192.168.33.251/32 -m comment --comment test1 -c 0 0 -g VPR_MARK0x020000
-A VPR_PREROUTING -s 192.168.33.81/32 -m comment --comment voip1 -c 0 0 -g VPR_MARK0x020000
-A VPR_PREROUTING -s 192.168.33.82/32 -m comment --comment voip2 -c 0 0 -g VPR_MARK0x020000
-A VPR_PREROUTING -s 192.168.33.83/32 -m comment --comment voip3 -c 0 0 -g VPR_MARK0x020000
-A VPR_PREROUTING -s 192.168.33.161/32 -m comment --comment tvhome -c 0 0 -g VPR_MARK0x020000
-A VPR_PREROUTING -s 192.168.33.162/32 -m comment --comment tvbana -c 0 0 -g VPR_MARK0x020000
-A VPR_PREROUTING -s 192.168.33.163/32 -m comment --comment tvtuca -c 0 0 -g VPR_MARK0x020000
-A VPR_PREROUTING -s 192.168.33.250/32 -m comment --comment test0 -c 80928 45253029 -g VPR_MARK0x010000
-A VPR_PREROUTING -s 192.168.33.160/32 -m comment --comment ps4wzi -c 0 0 -g VPR_MARK0x010000
-A VPR_PREROUTING -s 192.168.33.120/32 -m comment --comment nat -c 126 22806 -g VPR_MARK0x010000
-A VPR_PREROUTING -s 192.168.33.22/32 -m comment --comment mmcam1 -c 65 9518 -g VPR_MARK0x010000
-A VPR_PREROUTING -s 192.168.33.0/24 -m comment --comment default -c 11394 4941339 -g VPR_MARK0x030000
                                                            
Mangle IP Table MARK Chain: VPR_MARK0x010000
-N VPR_MARK0x010000
-A VPR_MARK0x010000 -c 81119 45285353 -j MARK --set-xmark 0x10000/0xff0000
-A VPR_MARK0x010000 -c 81119 45285353 -j RETURN
                                                            
Mangle IP Table MARK Chain: VPR_MARK0x020000
-N VPR_MARK0x020000
-A VPR_MARK0x020000 -c 0 0 -j MARK --set-xmark 0x20000/0xff0000
-A VPR_MARK0x020000 -c 0 0 -j RETURN
                                                            
Mangle IP Table MARK Chain: VPR_MARK0x030000
-N VPR_MARK0x030000
-A VPR_MARK0x030000 -c 11403 4941807 -j MARK --set-xmark 0x30000/0xff0000
-A VPR_MARK0x030000 -c 11403 4941807 -j RETURN
                                                            
Mangle IP Table MARK Chain: VPR_MARK0x040000
-N VPR_MARK0x040000
-A VPR_MARK0x040000 -c 0 0 -j MARK --set-xmark 0x40000/0xff0000
-A VPR_MARK0x040000 -c 0 0 -j RETURN
                                                            
Current ipsets

6- output reload

Creating table 'wan/192.168.7.3' [✓]
Creating table 'vpn1/tun-vpn1/10.7.2.2' [✓]
Creating table 'vpn2/tun-vpn2/10.7.1.4' [✓]
Creating table 'vpn3/tun-vpn3/0.0.0.0' [✓]
Routing 'test1' via vpn1 [✓]
Routing 'voip1' via vpn1 [✓]
Routing 'voip2' via vpn1 [✓]
Routing 'voip3' via vpn1 [✓]
Routing 'tvhome' via vpn1 [✓]
Routing 'tvbana' via vpn1 [✓]
Routing 'tvtuca' via vpn1 [✓]
Routing 'test0' via wan [✓]
Routing 'ps4wzi' via wan [✓]
Routing 'nat' via wan [✓]
Routing 'mmcam1' via wan [✓]
Routing 'default' via vpn2 [✓]
vpn-policy-routing 0.3.4-8 monitoring interfaces: wan vpn1 vpn2 vpn3 [✓]
vpn-policy-routing 0.3.4-8 started with gateways:
wan/192.168.7.3 [✓]
vpn1/tun-vpn1/10.7.2.2
vpn2/tun-vpn2/10.7.1.4
vpn3/tun-vpn3/0.0.0.0

Update

I have installed openwrt on a x86: i7 / 8cores / 16gb ram / 240gb ssd / 2 eth 1000 interfaces

Setup with vpn-policiy routing as with my router.
Surprise: vpn is even slower on the x86

Any idea why?

maybe - is openwrt loadbalancing vpn operations on different cores?