Multiple Wans Port forwarding

Tueem · May 3, 2024, 2:42pm

Network topography:

Hey,
I've got two wans. One VPN with public static IP and the other just my normal ISP.
My server is routed through wan and not VPN. I want to access my server through the public IP of the VPN and have set up port forwarding from the vpn to my server.
I've checked using tcpdump that the packets arrive at my server and also get replied to but the problem is that aforementioned replies get routed through its default gateway (wan) and not to the vpn where the connection is coming from which results in a non working website.

I've also tried using firewall marks and routing the reply packets using that but that has failed (probably because of misconfiguration)

I would prefer a Luci config so I can easily see changes I made on the routing behaviour.

Thanks for any help!

egc · May 3, 2024, 2:58pm

You must use Policy Based Routing (PB) to route traffic from the server via the VPN.

This is what I use for routing a specific client via my VPN, I use Wireguard but hopefully you get the idea:
/etc/config/network

config route
	option interface 'wg_mullv_se'
	option table '102'
	option target '0.0.0.0/0'

config rule
	option src '192.168.30.3/32'
	option lookup '102'

Basically it creates a routing table 102 with default route via the VPN interface and a routing rule which lets my client 192.168.30.3 use this routing table

Tueem · May 3, 2024, 3:13pm

Yeah I'm actually using pbr for some other vms but I want that one to have its usual traffic routed through wan. Only the replies to Port forwards should be routed through the VPN

krazeh · May 3, 2024, 3:20pm

That makes things trickier..

Please connect to your OpenWrt device using ssh and copy the output of the following commands and post it here using the "Preformatted text </> " button:

Remember to redact passwords, MAC addresses and any public IP addresses you may have:

ubus call system board
cat /etc/config/network
cat /etc/config/wireless
cat /etc/config/dhcp
cat /etc/config/firewall

Tueem · May 3, 2024, 3:32pm

ubus call system board

{
        "kernel": "5.15.150",
        "hostname": "OpenWrt",
        "system": "ARMv8 Processor rev 3",
        "model": "Raspberry Pi 4 Model B Rev 1.4",
        "board_name": "raspberrypi,4-model-b",
        "rootfs_type": "squashfs",
        "release": {
                "distribution": "OpenWrt",
                "version": "23.05.3",
                "revision": "r23809-234f1a2efa",
                "target": "bcm27xx/bcm2711",
                "description": "OpenWrt 23.05.3 r23809-234f1a2efa"
        }
}

cat /etc/config/network

config interface 'loopback'
        option device 'lo'
        option proto 'static'
        option ipaddr '127.0.0.1'
        option netmask '255.0.0.0'

config globals 'globals'
        option ula_prefix 'fdc3:26f5:2cf6::/48'

config device
        option name 'br-lan'
        option type 'bridge'
        list ports 'eth1'
        option mtu '1500'
        option macaddr '00:E0:4C:42:59:3F'

config interface 'lan'
        option proto 'static'
        option ipaddr '10.0.0.1'
        option netmask '255.255.255.0'
        option device 'br-lan.1'

config device
        option type 'bridge'
        option name 'br-wan'
        list ports 'eth0'
        option mtu '1500'
        option macaddr 'E4:5F:01:7F:2C:7C'

config interface 'wan'
        option proto 'static'
        option device 'br-wan'
        option ipaddr '192.168.178.2'
        option netmask '255.255.255.0'
        list dns '192.168.178.1'
        option gateway '192.168.178.1'

config interface 'vmLan'
        option proto 'static'
        option ipaddr '10.0.1.1'
        option netmask '255.255.255.0'
        option device 'br-lan.10'

config bridge-vlan
        option device 'br-lan'
        option vlan '1'
        list ports 'eth1:u*'

config bridge-vlan
        option device 'br-lan'
        option vlan '10'
        list ports 'eth1:t'

config interface 'VPN'
        option proto 'wireguard'
        option private_key 'REDACTED'
        list addresses 'REDACTED'
        list addresses 'REDACTED'

config wireguard_VPN
        option description 'wireguard.conf'
        option public_key 'REDACTED'
        list allowed_ips '0.0.0.0/0'
        list allowed_ips '::/0'
        option persistent_keepalive '25'
        option endpoint_host 'REDACTED'
        option endpoint_port '8888'

config device
        option name 'br-lan.1'
        option type '8021q'
        option ifname 'br-lan'
        option vid '1'
        option mtu '1500'

config device
        option name 'br-lan.10'
        option type '8021q'
        option ifname 'br-lan'
        option vid '10'
        option mtu '1500'

config device
        option name 'VPN'

cat /etc/config/wireless
disabled
cat /etc/config/dhcp

config dnsmasq
        option domainneeded '1'
        option localise_queries '1'
        option rebind_protection '1'
        option rebind_localhost '1'
        option local '/lan/'
        option domain 'lan'
        option expandhosts '1'
        option cachesize '1000'
        option authoritative '1'
        option readethers '1'
        option leasefile '/tmp/dhcp.leases'
        option resolvfile '/tmp/resolv.conf.d/resolv.conf.auto'
        option localservice '1'
        option ednspacket_max '1232'
        option confdir '/tmp/dnsmasq.d'
        option sequential_ip '1'

config odhcpd 'odhcpd'
        option maindhcp '0'
        option leasefile '/tmp/hosts/odhcpd'
        option leasetrigger '/usr/sbin/odhcpd-update'
        option loglevel '4'

config dhcp 'lan'
        option interface 'lan'
        option start '100'
        option limit '255'
        option leasetime '12h'

config domain
        option name 'pve.local'
        option ip '10.0.0.10'

config domain
        option name 'router.local'
        option ip '10.0.0.1'

config dhcp 'vmLan'
        option interface 'vmLan'
        option start '2'
        option limit '255'
        option leasetime '12h'

cat /etc/config/firewall

config defaults
        option input 'REJECT'
        option output 'ACCEPT'
        option forward 'REJECT'
        option synflood_protect '1'

config zone
        option name 'lan'
        option input 'ACCEPT'
        option output 'ACCEPT'
        option forward 'ACCEPT'
        option log '1'
        list network 'lan'

config zone
        option name 'wan'
        option input 'REJECT'
        option output 'ACCEPT'
        option forward 'REJECT'
        option masq '1'
        option mtu_fix '1'
        list network 'wan'

config forwarding
        option src 'lan'
        option dest 'wan'

config rule
        option name 'Allow-DHCP-Renew'
        option src 'wan'
        option proto 'udp'
        option dest_port '68'
        option target 'ACCEPT'
        option family 'ipv4'

config rule
        option name 'Allow-Ping'
        option src 'wan'
        option proto 'icmp'
        option icmp_type 'echo-request'
        option family 'ipv4'
        option target 'ACCEPT'

config rule
        option name 'Allow-IGMP'
        option src 'wan'
        option proto 'igmp'
        option family 'ipv4'
        option target 'ACCEPT'

config rule
        option name 'Allow-DHCPv6'
        option src 'wan'
        option proto 'udp'
        option dest_port '546'
        option family 'ipv6'
        option target 'ACCEPT'

config rule
        option name 'Allow-MLD'
        option src 'wan'
        option proto 'icmp'
        option src_ip 'fe80::/10'
        list icmp_type '130/0'
        list icmp_type '131/0'
        list icmp_type '132/0'
        list icmp_type '143/0'
        option family 'ipv6'
        option target 'ACCEPT'

config rule
        option name 'Allow-ICMPv6-Input'
        option src 'wan'
        option proto 'icmp'
        list icmp_type 'echo-request'
        list icmp_type 'echo-reply'
        list icmp_type 'destination-unreachable'
        list icmp_type 'packet-too-big'
        list icmp_type 'time-exceeded'
        list icmp_type 'bad-header'
        list icmp_type 'unknown-header-type'
        list icmp_type 'router-solicitation'
        list icmp_type 'neighbour-solicitation'
        list icmp_type 'router-advertisement'
        list icmp_type 'neighbour-advertisement'
        option limit '1000/sec'
        option family 'ipv6'
        option target 'ACCEPT'

config rule
        option name 'Allow-ICMPv6-Forward'
        option src 'wan'
        option dest '*'
        option proto 'icmp'
        list icmp_type 'echo-request'
        list icmp_type 'echo-reply'
        list icmp_type 'destination-unreachable'
        list icmp_type 'packet-too-big'
        list icmp_type 'time-exceeded'
        list icmp_type 'bad-header'
        list icmp_type 'unknown-header-type'
        option limit '1000/sec'
        option family 'ipv6'
        option target 'ACCEPT'

config rule
        option name 'Allow-IPSec-ESP'
        option src 'wan'
        option dest 'lan'
        option proto 'esp'
        option target 'ACCEPT'

config rule
        option name 'Allow-ISAKMP'
        option src 'wan'
        option dest 'lan'
        option dest_port '500'
        option proto 'udp'
        option target 'ACCEPT'

config forwarding
        option src 'wan'
        option dest 'lan'

config rule
        option src 'wan'
        option dest_port '80 443 22'
        option target 'ACCEPT'
        option name 'Allow-Admin'

config include 'pbr'
        option fw4_compatible '1'
        option type 'script'
        option path '/usr/share/pbr/pbr.firewall.include'

config zone
        option name 'vmLan'
        option input 'ACCEPT'
        option output 'ACCEPT'
        option forward 'ACCEPT'
        list network 'vmLan'

config forwarding
        option src 'wan'
        option dest 'vmLan'

config zone
        option name 'vpn'
        option input 'REJECT'
        option output 'ACCEPT'
        option forward 'REJECT'
        option masq '1'
        option mtu_fix '1'
        list network 'VPN'

config forwarding
        option src 'vmLan'
        option dest 'vpn'

config forwarding
        option src 'lan'
        option dest 'vmLan'

config redirect
        option dest 'lan'
        option target 'DNAT'
        option name 'testproxmox'
        option src 'vpn'
        option src_dport '8006'
        option dest_ip '10.0.0.10'
        option dest_port '8006'
        option reflection_src 'external'

config redirect
        option dest 'vmLan'
        option target 'DNAT'
        option name 'SE VM101'
        option src 'vpn'
        option src_dport '27016'
        option dest_ip '10.0.1.183'
        option dest_port '27016'
        list reflection_zone 'lan'

config rule
        option name 'Allow Admin test'
        option src 'vpn'
        option dest 'lan'
        option target 'ACCEPT'
        option enabled '0'

config forwarding
        option src 'vpn'
        option dest 'lan'

config forwarding
        option src 'lan'
        option dest 'vpn'

config rule
        option name 'VPN apply mark'
        option src 'vpn'
        option dest '*'
        option target 'MARK'
        option set_mark '1'

config rule

config redirect
        option dest 'vmLan'
        option target 'DNAT'
        option name 'CloudNet Minecraft'
        option src 'vpn'
        option src_dport '25565-25665'
        option dest_ip '10.0.1.3'
        option dest_port '25565-25665'
        list reflection_zone 'lan'

egc · May 3, 2024, 3:33pm

That is indeed trickier.

However I use this for an OpenVPN server which uses a fixed port to reply so I only route that sourceport via the VPN and not the whole server

Unfortunately there is no rule yet for sport/dport but it will come
In the meantime add to Startup/Local startup:

ip rule del sport 1194 table 102
ip rule add sport 1194 table 102

This adds the rule to route sourceport 1194 to table 102 thus routing via the VPN

Maybe this helps?

Tueem · May 3, 2024, 3:45pm

I could do that but it would be kind of repetetive having to do this for every port I forward. I was looking for a more elegant solution, which automates that.

Tueem · May 3, 2024, 4:08pm

The same thing happens with ICMP packets. The are coming from the VPN connection but the replies are sent to my wan.