Stuck in (virtual) Networking hell! Connecting to VMs on a different subnet

Hey guys I already spent almost a full day tinkering around with this issue but my limited networking knowledge is prohibiting any further progress it seem.

My main network is run by a openWRT router at 192.168.1.1 which also has my server in the same subnet. The server is running a openVPN container that is used to route some filesharing containers through a commercial VPN. The openVPN container uses 2 virtual network devices (bridges). One connects to the main router at 192.168.1.1 and the other (192.168.50.1) is used to connect to the filesharing containers and route their traffic through the VPN.
The current state is that I can access the openVPN container from the main network but I have no access to the other VMs. The other VMs get routed through the VPN and can access each other though.
The OpenVPN VM is running DHCP for the second subnet.

I already tried countless things but I am so stuck and everything I did until now was based on several tutorials I found on the net.

This is all my config and info:
OpenWRT main router:
Network:

config interface 'loopback'
	option device 'lo'
	option proto 'static'
	option ipaddr '127.0.0.1'
	option netmask '255.0.0.0'
 
config globals 'globals'
	option ula_prefix 'fdb0:92f6:e692::/48'
	option packet_steering '1'
 
config device
	option name 'br-lan'
	option type 'bridge'
	list ports 'lan1'
	list ports 'lan2'
	list ports 'lan3'
 
config interface 'lan'
	option device 'br-lan'
	option proto 'static'
	option ipaddr '192.168.1.1'
	option netmask '255.255.255.0'
	option ip6assign '60'
 
config interface 'wan'
	option device 'wan'
	option proto 'dhcp'
 
config interface 'wan6'
	option device 'wan'
	option proto 'dhcpv6'
 
config route
	option interface 'lan'
	option target '10.20.0.0/24'
	option gateway '192.168.1.157'
 
 
config device
	option type 'bridge'
	option name 'Nibelungen'
	option bridge_empty '1'
	list ports 'eth0'
	list ports 'lan3'
	list ports 'Nibelungen.1'
	list ports 'Nibelungen.2'
	list ports 'Nibelungen.3'
 
config bridge-vlan
	option device 'Nibelungen'
	option vlan '1'
	list ports 'Nibelungen.1:t'
 
config bridge-vlan
	option device 'Nibelungen'
	option vlan '2'
	list ports 'Nibelungen.2:t'
 
config bridge-vlan
	option device 'Nibelungen'
	option vlan '3'
	list ports 'Nibelungen.3:t'
 
config route
	option interface 'lan'
	option target '192.168.50.0/24'
 
config interface 'OVPN_VLAN'
	option proto 'static'
	option device 'Nibelungen'
	option ipaddr '192.168.50.0'
	option netmask '255.255.255.0'
	option gateway '192.168.50.1'
	option broadcast '192.168.50.255'
	list dns '192.168.1.1'
 
config route
	option interface 'lan'
	option target '192.168.50.129/24'
	option gateway '192.168.50.1'
 
config route
	option interface 'OVPN_VLAN'
	option target '192.168.1.0/24'

firewall:

This is not the usual config - I just set everything to accept for troubleshooting!
 
config defaults
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'ACCEPT'
	option synflood_protect '1'
 
config zone
	option name 'lan'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'ACCEPT'
	list network 'lan'
 
config zone
	option name 'wan'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'ACCEPT'
	option masq '1'
	list network 'wan'
	list network 'wan6'
 
config forwarding
	option src 'lan'
	option dest 'wan'
 
config rule
	option name 'Allow-DHCP-Renew'
	option src 'wan'
	option proto 'udp'
	option dest_port '68'
	option target 'ACCEPT'
	option family 'ipv4'
 
config rule
	option name 'Allow-Ping'
	option src 'wan'
	option proto 'icmp'
	option icmp_type 'echo-request'
	option family 'ipv4'
	option target 'ACCEPT'
 
config rule
	option name 'Allow-IGMP'
	option src 'wan'
	option proto 'igmp'
	option family 'ipv4'
	option target 'ACCEPT'
 
config rule
	option name 'Allow-DHCPv6'
	option src 'wan'
	option proto 'udp'
	option dest_port '546'
	option family 'ipv6'
	option target 'ACCEPT'
 
config rule
	option name 'Allow-MLD'
	option src 'wan'
	option proto 'icmp'
	option src_ip 'fe80::/10'
	list icmp_type '130/0'
	list icmp_type '131/0'
	list icmp_type '132/0'
	list icmp_type '143/0'
	option family 'ipv6'
	option target 'ACCEPT'
 
config rule
	option name 'Allow-ICMPv6-Input'
	option src 'wan'
	option proto 'icmp'
	list icmp_type 'echo-request'
	list icmp_type 'echo-reply'
	list icmp_type 'destination-unreachable'
	list icmp_type 'packet-too-big'
	list icmp_type 'time-exceeded'
	list icmp_type 'bad-header'
	list icmp_type 'unknown-header-type'
	list icmp_type 'router-solicitation'
	list icmp_type 'neighbour-solicitation'
	list icmp_type 'router-advertisement'
	list icmp_type 'neighbour-advertisement'
	option limit '1000/sec'
	option family 'ipv6'
	option target 'ACCEPT'
 
config rule
	option name 'Allow-ICMPv6-Forward'
	option src 'wan'
	option dest '*'
	option proto 'icmp'
	list icmp_type 'echo-request'
	list icmp_type 'echo-reply'
	list icmp_type 'destination-unreachable'
	list icmp_type 'packet-too-big'
	list icmp_type 'time-exceeded'
	list icmp_type 'bad-header'
	list icmp_type 'unknown-header-type'
	option limit '1000/sec'
	option family 'ipv6'
	option target 'ACCEPT'
 
config rule
	option name 'Allow-IPSec-ESP'
	option src 'wan'
	option dest 'lan'
	option proto 'esp'
	option target 'ACCEPT'
 
config rule
	option name 'Allow-ISAKMP'
	option src 'wan'
	option dest 'lan'
	option dest_port '500'
	option proto 'udp'
	option target 'ACCEPT'
 
 
 
config include 'pbr'
	option fw4_compatible '1'
	option type 'script'
	option path '/usr/share/pbr/firewall.include'
 
 
config zone
	option name 'OVPN_VLAN'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'ACCEPT'
	list network 'OVPN_VLAN'
 
config forwarding
	option src 'OVPN_VLAN'
	option dest 'lan'
 
config forwarding
	option src 'lan'
	option dest 'OVPN_VLAN'
 
config forwarding
	option src 'OVPN_VLAN'
	option dest 'wan'
 
config forwarding
	option src 'wan'
	option dest 'OVPN_VLAN'
 
config redirect
	option dest 'OVPN_VLAN'
	option target 'DNAT'
	option name 'tranvlan'
	option src 'lan'
	option src_dport '9091'
	option dest_port '9091'

OpenVPN VM:
VPN client:

client
dev tap0
proto udp
remote xxx
resolv-retry infinite
remote-random
nobind
tun-mtu 1500
tun-mtu-extra 32
mssfix 1450
persist-key
persist-tun
ping 15
ping-restart 0
ping-timer-rem
reneg-sec 0
 
remote-cert-tls server
 
auth-user-pass xxx
 
#comp-lzo
verb 3
pull
fast-io
cipher AES-256-CBC
 
auth SHA512
 
<ca>
-----BEGIN CERTIFICATE-----
xxx
-----END CERTIFICATE-----
</ca>
key-direction 1
<tls-auth>
#
# 2048 bit OpenVPN static key
#
-----BEGIN OpenVPN Static key V1-----
xxx
-----END OpenVPN Static key V1-----
</tls-auth>

iptables:

I have no idea what all that stuff really does, what is missing here is that I tried several routes to+from the main router
 
# Generated by iptables-save v1.8.9 on Wed Oct  9 15:35:17 2024
*filter
:INPUT DROP [19:1210]
:FORWARD ACCEPT [1:60]
:OUTPUT ACCEPT [241033:112543109]
-A INPUT -i lo -j ACCEPT
-A INPUT -p icmp -m icmp --icmp-type 8 -j ACCEPT
-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
-A INPUT -p tcp -m tcp --dport 22 -j ACCEPT
-A INPUT -p tcp -m tcp --dport 80 -j ACCEPT
-A INPUT -p tcp -m tcp --dport 443 -j ACCEPT
-A INPUT -p tcp -m tcp --dport 12320 -j ACCEPT
-A INPUT -p tcp -m tcp --dport 12321 -j ACCEPT
-A INPUT -p udp -m udp --dport 1194 -j ACCEPT
-A FORWARD -s 192.168.100.0/24 -i eth1 -o tun0 -m conntrack --ctstate NEW -j ACCEPT
-A FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
COMMIT
# Completed on Wed Oct  9 15:35:17 2024
# Generated by iptables-save v1.8.9 on Wed Oct  9 15:35:17 2024
*mangle
:PREROUTING ACCEPT [1135925:1319901755]
:INPUT ACCEPT [461544:622624500]
:FORWARD ACCEPT [671737:696660176]
:OUTPUT ACCEPT [242112:112663364]
:POSTROUTING ACCEPT [913849:809323540]
COMMIT
# Completed on Wed Oct  9 15:35:17 2024
# Generated by iptables-save v1.8.9 on Wed Oct  9 15:35:17 2024
*nat
:PREROUTING ACCEPT [709:82568]
:INPUT ACCEPT [0:0]
:OUTPUT ACCEPT [0:0]
:POSTROUTING ACCEPT [0:0]
-A POSTROUTING -o eth0 -j MASQUERADE
-A POSTROUTING -j MASQUERADE
COMMIT
# Completed on Wed Oct  9 15:35:17 2024

DHCP:

# dhcpd.conf
#
# Sample configuration file for ISC dhcpd
#
 
authoritative;
 
subnet 192.168.50.0 netmask 255.255.255.0 {
  range 192.168.50.128 192.168.50.254;
 
  option routers 192.168.50.1;
  option subnet-mask 255.255.255.0;
  option broadcast-address 192.168.50.255;
  option domain-name-servers 8.8.8.8;
 
  default-lease-time 86400;
  max-lease-time 259200;
}
 
# If this DHCP server is the official DHCP server for the local
# network, the authoritative directive should be uncommented.
#authoritative;
 
# Use this to send dhcp log messages to a different log file (you also
# have to hack syslog.conf to complete the redirection).
#log-facility local7;
 
# No service will be given on this subnet, but declaring it helps the 
# DHCP server to understand the network topology.
 
#subnet 10.152.187.0 netmask 255.255.255.0 {
#}
 
# This is a very basic subnet declaration.
 
#subnet 10.254.239.0 netmask 255.255.255.224 {
#  range 10.254.239.10 10.254.239.20;
#  option routers rtr-239-0-1.example.org, rtr-239-0-2.example.org;
#}
 
# This declaration allows BOOTP clients to get dynamic addresses,
# which we don't really recommend.
 
#subnet 10.254.239.32 netmask 255.255.255.224 {
#  range dynamic-bootp 10.254.239.40 10.254.239.60;
#  option broadcast-address 10.254.239.31;
#  option routers rtr-239-32-1.example.org;
#}
 
# A slightly different configuration for an internal subnet.
#subnet 10.5.5.0 netmask 255.255.255.224 {
#  range 10.5.5.26 10.5.5.30;
#  option domain-name-servers ns1.internal.example.org;
#  option domain-name "internal.example.org";
#  option routers 10.5.5.1;
#  option broadcast-address 10.5.5.31;
#  default-lease-time 600;
#  max-lease-time 7200;
#}
 
# Hosts which require special configuration options can be listed in
# host statements.   If no address is specified, the address will be
# allocated dynamically (if possible), but the host-specific information
# will still come from the host declaration.
 
#host passacaglia {
#  hardware ethernet 0:0:c0:5d:bd:95;
#  filename "vmunix.passacaglia";
#  server-name "toccata.example.com";
#}
 
# Fixed IP addresses can also be specified for hosts.   These addresses
# should not also be listed as being available for dynamic assignment.
# Hosts for which fixed IP addresses have been specified can boot using
# BOOTP or DHCP.   Hosts for which no fixed address is specified can only
# be booted with DHCP, unless there is an address range on the subnet
# to which a BOOTP client is connected which has the dynamic-bootp flag
# set.
#host fantasia {
#  hardware ethernet 08:00:07:26:c0:a5;
#  fixed-address fantasia.example.com;
#}
 
# You can declare a class of clients and then do address allocation
# based on that.   The example below shows a case where all clients
# in a certain class get addresses on the 10.17.224/24 subnet, and all
# other clients get addresses on the 10.0.29/24 subnet.
 
#class "foo" {
#  match if substring (option vendor-class-identifier, 0, 4) = "SUNW";
#}
 
#shared-network 224-29 {
#  subnet 10.17.224.0 netmask 255.255.255.0 {
#    option routers rtr-224.example.org;
#  }
#  subnet 10.0.29.0 netmask 255.255.255.0 {
#    option routers rtr-29.example.org;
#  }
#  pool {
#    allow members of "foo";
#    range 10.17.224.10 10.17.224.250;
#  }
#  pool {
#    deny members of "foo";
#    range 10.0.29.10 10.0.29.230;
#  }
#}
 

sysctl:

#
# /etc/sysctl.conf - Configuration file for setting system variables
# See /etc/sysctl.d/ for additional system variables.
# See sysctl.conf (5) for information.
#
net.ipv4.conf.all.forwarding = 1
#kernel.domainname = example.com
 
# Uncomment the following to stop low-level messages on console
kernel.printk = 1 4 1 7
 
###################################################################
# Functions previously found in netbase
#
 
# Uncomment the next two lines to enable Spoof protection (reverse-path filter)
# Turn on Source Address Verification in all interfaces to
# prevent some spoofing attacks
#net.ipv4.conf.default.rp_filter=1
#net.ipv4.conf.all.rp_filter=1
 
# Uncomment the next line to enable TCP/IP SYN cookies
# See http://lwn.net/Articles/277146/
# Note: This may impact IPv6 TCP sessions too
#net.ipv4.tcp_syncookies=1
 
# Uncomment the next line to enable packet forwarding for IPv4
net.ipv4.ip_forward = 1
 
# Uncomment the next line to enable packet forwarding for IPv6
#  Enabling this option disables Stateless Address Autoconfiguration
#  based on Router Advertisements for this host
#net.ipv6.conf.all.forwarding=1
 
 
###################################################################
# Additional settings - these settings can improve the network
# security of the host and prevent against some network attacks
# including spoofing attacks and man in the middle attacks through
# redirection. Some network environments, however, require that these
# settings are disabled so review and enable them as needed.
#
# Do not accept ICMP redirects (prevent MITM attacks)
#net.ipv4.conf.all.accept_redirects = 0
#net.ipv6.conf.all.accept_redirects = 0
# _or_
# Accept ICMP redirects only for gateways listed in our default
# gateway list (enabled by default)
# net.ipv4.conf.all.secure_redirects = 1
#
# Do not send ICMP redirects (we are not a router)
#net.ipv4.conf.all.send_redirects = 0
#
# Do not accept IP source route packets (we are not a router)
#net.ipv4.conf.all.accept_source_route = 0
#net.ipv6.conf.all.accept_source_route = 0
#
# Log Martian Packets
#net.ipv4.conf.all.log_martians = 1
#
 
###################################################################
# Magic system request Key
# 0=disable, 1=enable all, >1 bitmask of sysrq functions
# See https://www.kernel.org/doc/html/latest/admin-guide/sysrq.html
# for what other values do
#kernel.sysrq=438
 
# Disable TCP timestamps
net.ipv4.tcp_timestamps = 0
#

"ip addr"

1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host noprefixroute 
       valid_lft forever preferred_lft forever
2: eth0@if177: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether bc:24:11:5f:30:d2 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.1.118/24 brd 192.168.1.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::be24:11ff:fe5f:30d2/64 scope link 
       valid_lft forever preferred_lft forever
3: eth1@if178: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether bc:24:11:a7:51:f9 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.50.1/24 brd 192.168.50.255 scope global eth1
       valid_lft forever preferred_lft forever
    inet6 fe80::be24:11ff:fea7:51f9/64 scope link 
       valid_lft forever preferred_lft forever
4: tun0: <POINTOPOINT,MULTICAST,NOARP,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UNKNOWN group default qlen 500
    link/none 
    inet 10.8.8.12/24 scope global tun0
       valid_lft forever preferred_lft forever
    inet6 fe80::acfc:c999:d938:5b72/64 scope link stable-privacy 
       valid_lft forever preferred_lft forever
 

"route"

root@OpenVPN ~# route
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.8.8.1        128.0.0.0       UG    0      0        0 tun0
default         OpenWrt.lan     0.0.0.0         UG    0      0        0 eth0
10.8.8.0        0.0.0.0         255.255.255.0   U     0      0        0 tun0
128.0.0.0       10.8.8.1        128.0.0.0       UG    0      0        0 tun0
192.168.1.0     0.0.0.0         255.255.255.0   U     0      0        0 eth0
192.168.50.0    0.0.0.0         255.255.255.0   U     0      0        0 eth1
vpn-pub-address OpenWrt.lan     255.255.255.255 UGH   0      0        0 eth0

"ip addr" and "route" from one of the other VMs

root@transmission:~# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host noprefixroute 
       valid_lft forever preferred_lft forever
2: esn0@if147: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether bc:24:11:75:63:21 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.50.129/24 brd 192.168.50.255 scope global esn0
       valid_lft forever preferred_lft forever
    inet6 fe80::be24:11ff:fe75:6321/64 scope link 
       valid_lft forever preferred_lft forever
 
route:
 
root@transmission:~# route
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
default         192.168.50.1    0.0.0.0         UG    0      0        0 esn0
192.168.50.0    0.0.0.0         255.255.255.0   U     0      0        0 esn0

Even though it would not be perfect but at this point I would be okay if there was a way to let me (and other VMs) access each machines web interface.

I am so lost I will gladly buy you a coffee if you can help me set this up!!

how about putting it all in the thread ?

That would have been a lot less work but would make the thread hardly readable -.-
edit: I added a map of the network to the OP

Please get all info into thread, adding ubus call system board from openvpn machine.

OK I moved all the info from the separate pastebins into the thread.
I cant seem to be able to install the ubus command on this debian VM, is there any other way?

Your openvpn virtual machine blocking your traffic is not openwrt, ask in some relevant (commercial you mentioned?) support channel.

Routing traffic from/to a openWRT router from/to a second subnet is not about openWRT?
I dont think that the VPN is the issue and it should not make a difference if it is a private VPN or a privacy VPN in this context...

Thats YOU blocking outside openwrt, enable firewall log and fix it.

could it be that the rules from that file (iptables.up.rules) are not used at all?
This is what I am getting,

root@OpenVPN ~/ubus# iptables -t nat   
iptables v1.8.9 (legacy): no command specified
Try `iptables -h' or 'iptables --help' for more information.
root@OpenVPN ~/ubus# iptables -t nat -L
Chain PREROUTING (policy ACCEPT)
target     prot opt source               destination         

Chain INPUT (policy ACCEPT)
target     prot opt source               destination         

Chain OUTPUT (policy ACCEPT)
target     prot opt source               destination         

Chain POSTROUTING (policy ACCEPT)
target     prot opt source               destination         
MASQUERADE  all  --  anywhere             anywhere            
MASQUERADE  all  --  anywhere             anywhere            
root@OpenVPN ~/ubus# iptables -L
Chain INPUT (policy DROP)
target     prot opt source               destination         
ACCEPT     all  --  anywhere             anywhere            
ACCEPT     icmp --  anywhere             anywhere             icmp echo-request
ACCEPT     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:ssh
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:http
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:https
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:12320
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:9091
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:12321
ACCEPT     udp  --  anywhere             anywhere             udp dpt:openvpn

Chain FORWARD (policy ACCEPT)
target     prot opt source               destination         
ACCEPT     all  --  192.168.50.0/24      anywhere             ctstate NEW
ACCEPT     all  --  anywhere             anywhere             ctstate RELATED,ESTABLISHED

Chain OUTPUT (policy ACCEPT)
target     prot opt source               destination         

or am I on the wrong track again here?

It is your responsibility to fix firewall on your other system. There is a clear proof it blocks connections.

hint : log fallthru packets ? Ok? -j LOG in iptables included in OpenWRT 5y ago.

I dont think I understand your hint but I flushed the firewall completely for now and accepted all traffic but I still dont have access to the other machines and I cant ping them.

root@OpenVPNiptables -L
Chain INPUT (policy ACCEPT)
target     prot opt source               destination         

Chain FORWARD (policy ACCEPT)
target     prot opt source               destination         

Chain OUTPUT (policy ACCEPT)
target     prot opt source               destination         
root@OpenVPN ~iptable-t nat -L
Chain PREROUTING (policy ACCEPT)
target     prot opt source               destination         

Chain INPUT (policy ACCEPT)
target     prot opt source               destination         

Chain OUTPUT (policy ACCEPT)
target     prot opt source               destination         

Chain POSTROUTING (policy ACCEPT)
target     prot opt source               destination         
root@OpenVPN ~# 

Hi @dakazze ,

This is way beyond my capabilities. But I came across similar situations like yours in the past. Can't ping/reach other nodes on other subnets. What helped me is tcpdump. Hope that helps. Thanks.

-Gamma

1 Like

Thank you very much! At this point I am glad for any help I can get! I will have a look at it.
Oh and I forgot that my VMs can access the main router at 192.168.1.1 but not the other way around.

edit:
okay I just had to serial into my router because networking stopped working after adding a bridge interface.... I am starting to hate this....