Incoming SSH connections are dropping after 30 seconds to two minutes of data transfer. This is happening when external servers are trying to back up to my home NAS via SSH, but I can also repro by connecting from a remote system and transferring a large file via SFTP, or using SSH forwarding to run an OpenSpeedTest over the SSH connection.
I usually get this error message when it happens:
ssh_dispatch_run_fatal: Connection to xxxxxx port 22: message authentication code incorrect
In Borgmatic logs, it manifests it self as an error 32 (Broken pipe):
Nov 8 01:40:05 sjc02 borgmatic: INFO /home/<file path removed>: read: [Errno 32] Broken pipe
There are no logs in dmesg nor in the OpenWrt system log when this happens. It worked fine with my old router (a TP-Link ER8411).
Outgoing SSH connections seem to be fine, and other protocols seem to be fine too (eg I can run iperf3 for several minutes with no issues).
System specs:
- Intel Core i5-9500 processor.
- Intel X540-T2 with extra cooling (Noctua NF-A6x25 fan sitting directly next to network card).
- OpenWrt is running on a VM on Proxmox. I'm passing through the network card rather than using virtio.
/etc/config/network:
config interface 'loopback'
option device 'lo'
option proto 'static'
option ipaddr '127.0.0.1'
option netmask '255.0.0.0'
config globals 'globals'
option ula_prefix 'fdc7:ff6a:262b::/48'
config device
option name 'br-lan'
option type 'bridge'
list ports 'eth0'
config interface 'lan'
option device 'br-lan'
option proto 'static'
option ip6assign '60'
list ipaddr '10.1.0.1/16'
list dns '10.1.4.2'
list dns '10.1.1.8'
config interface 'wan'
option device 'eth1'
option proto 'dhcp'
config interface 'wan6'
option device 'eth1'
option proto 'dhcpv6'
config device
option type '8021q'
option ifname 'br-lan'
option vid '20'
option name 'iot-no-net'
option ipv6 '0'
config device
option type '8021q'
option ifname 'br-lan'
option vid '30'
option name 'iot-net'
config interface 'iot_no_net'
option proto 'static'
option device 'iot-no-net'
option ipaddr '10.20.0.1'
option netmask '255.255.0.0'
config interface 'iot_net'
option proto 'static'
option device 'iot-net'
option ipaddr '10.30.0.1'
option netmask '255.255.0.0'
option ip6assign '64'
list dns '10.1.4.2'
list dns '10.1.1.8'
config interface 'wg0'
option proto 'wireguard'
option private_key 'xxxxxxx'
option listen_port '46422'
list addresses '10.123.1.58'
config wireguard_wg0
option description 'la05'
option public_key 'xxxxxxx'
list allowed_ips '10.123.1.53/32'
option endpoint_host 'xxxxxxx'
option endpoint_port '38917'
option route_allowed_ips '1'
/etc/config/firewall:
config defaults
option input 'REJECT'
option output 'ACCEPT'
option forward 'REJECT'
option synflood_protect '1'
option flow_offloading '1'
option flow_offloading_hw '1'
config zone
option name 'lan'
list network 'lan'
option input 'ACCEPT'
option output 'ACCEPT'
option forward 'ACCEPT'
config zone
option name 'wan'
list network 'wan'
list network 'wan6'
option input 'REJECT'
option output 'ACCEPT'
option forward 'REJECT'
option masq '1'
option mtu_fix '1'
config forwarding
option src 'lan'
option dest 'wan'
config rule
option name 'Allow-DHCP-Renew'
option src 'wan'
option proto 'udp'
option dest_port '68'
option target 'ACCEPT'
option family 'ipv4'
config rule
option name 'Allow-Ping'
option src 'wan'
option proto 'icmp'
option icmp_type 'echo-request'
option family 'ipv4'
option target 'ACCEPT'
config rule
option name 'Allow-IGMP'
option src 'wan'
option proto 'igmp'
option family 'ipv4'
option target 'ACCEPT'
config rule
option name 'Allow-DHCPv6'
option src 'wan'
option proto 'udp'
option dest_port '546'
option family 'ipv6'
option target 'ACCEPT'
config rule
option name 'Allow-MLD'
option src 'wan'
option proto 'icmp'
option src_ip 'fe80::/10'
list icmp_type '130/0'
list icmp_type '131/0'
list icmp_type '132/0'
list icmp_type '143/0'
option family 'ipv6'
option target 'ACCEPT'
config rule
option name 'Allow-ICMPv6-Input'
option src 'wan'
option proto 'icmp'
list icmp_type 'echo-request'
list icmp_type 'echo-reply'
list icmp_type 'destination-unreachable'
list icmp_type 'packet-too-big'
list icmp_type 'time-exceeded'
list icmp_type 'bad-header'
list icmp_type 'unknown-header-type'
list icmp_type 'router-solicitation'
list icmp_type 'neighbour-solicitation'
list icmp_type 'router-advertisement'
list icmp_type 'neighbour-advertisement'
option limit '1000/sec'
option family 'ipv6'
option target 'ACCEPT'
config rule
option name 'Allow-ICMPv6-Forward'
option src 'wan'
option dest '*'
option proto 'icmp'
list icmp_type 'echo-request'
list icmp_type 'echo-reply'
list icmp_type 'destination-unreachable'
list icmp_type 'packet-too-big'
list icmp_type 'time-exceeded'
list icmp_type 'bad-header'
list icmp_type 'unknown-header-type'
option limit '1000/sec'
option family 'ipv6'
option target 'ACCEPT'
config rule
option name 'Allow-IPSec-ESP'
option src 'wan'
option dest 'lan'
option proto 'esp'
option target 'ACCEPT'
config rule
option name 'Allow-ISAKMP'
option src 'wan'
option dest 'lan'
option dest_port '500'
option proto 'udp'
option target 'ACCEPT'
config redirect
option dest 'lan'
option target 'DNAT'
option name 'Plex'
list proto 'tcp'
option src 'wan'
option src_dport '34321'
option dest_ip '10.1.1.12'
option dest_port '32400'
config redirect
option dest 'lan'
option target 'DNAT'
option name 'SWAG'
option src 'wan'
option src_dport '80'
option dest_ip '10.1.1.12'
option dest_port '8880'
config redirect
option dest 'lan'
option target 'DNAT'
option name 'SWAG_HTTPS'
option src 'wan'
option src_dport '443'
option dest_ip '10.1.1.12'
option dest_port '4443'
config redirect
option dest 'lan'
option target 'DNAT'
option name 'SyncThing'
option src 'wan'
option src_dport '22000'
option dest_ip '10.1.1.12'
option dest_port '22000'
config redirect
option dest 'lan'
option target 'DNAT'
option name 'SyncThing_Discovery'
option src 'wan'
option src_dport '21027'
option dest_ip '10.1.1.12'
option dest_port '21027'
config redirect
option dest 'lan'
option target 'DNAT'
option name 'Jumphost_SSH'
option src 'wan'
option src_dport '36153'
option dest_ip '10.1.1.1'
option dest_port '22'
list proto 'tcp'
config redirect
option dest 'lan'
option target 'DNAT'
option name 'DanHome_WireGuard'
option src 'wan'
option src_dport '46421'
option dest_ip '10.1.1.12'
option dest_port '46421'
config redirect
option dest 'lan'
option target 'DNAT'
option name 'Borgserver_SSH'
list proto 'tcp'
option src 'wan'
option src_dport '36152'
option dest_ip '10.1.1.15'
option dest_port '22'
config zone
option name 'iot_no_net'
option input 'REJECT'
option output 'ACCEPT'
option forward 'REJECT'
list network 'iot_no_net'
config zone
option name 'iot_net'
option input 'REJECT'
option output 'ACCEPT'
option forward 'REJECT'
list network 'iot_net'
config forwarding
option src 'lan'
option dest 'iot_net'
config forwarding
option src 'lan'
option dest 'iot_no_net'
config forwarding
option src 'iot_net'
option dest 'wan'
config rule
option name 'Allow-DHCP-IoT-No-Net'
option src 'iot_no_net'
option dest_port '67 68'
option target 'ACCEPT'
list proto 'udp'
config rule
option name 'Allow-DHCP-IoT-Net'
option src 'iot_net'
option dest_port '67 68 547'
option target 'ACCEPT'
list proto 'udp'
config rule
option name 'Allow-DNS-IoT-No-Net'
option src 'iot_no_net'
option dest_port '53'
option target 'ACCEPT'
option dest 'lan'
config rule
option name 'Allow-DNS-IoT-Net'
option src 'iot_net'
option dest 'lan'
option dest_port '53'
option target 'ACCEPT'
config rule
option name 'Allow-ICMPv6-IoT-Net'
list proto 'icmp'
option src 'iot_net'
option target 'ACCEPT'
option family 'ipv6'
config zone
option name 'vpn'
option input 'REJECT'
option output 'ACCEPT'
option forward 'REJECT'
list network 'wg0'
config rule
option name 'Allow-WireGuard'
list proto 'udp'
option src 'wan'
option dest_port '46422'
option target 'ACCEPT'
(the port forward for this SSH server is Jumphost_SSH)
Any ideas?
Thanks!