G'day all,
I'm using a self-compiled image using current openwrt.git.
Aside from the default config it has LuCI, openvpn, USB and a few little utils (like iperf3).
It is being used as a travel router where clients use Wifi and the ethernet port is the WAN uplink.
The thing runs for a day or so, then the wireless kicks the clients off and any attempt to re-associate is met with an "invalid password" error on the client (probably because the oom killer knocks out hostapd. I caught it in the log once while I was ssh'd in from the WAN side).
I've written some scripts to compare slab usage, and nothing sticks out.
I have another machine log in and do a free
every 30 minutes, which results in output like this :
brad@srv:~/diag$ cat fred.log | egrep "^Mem:" | less
total used free shared buff/cache available
Mem: 60100 26212 16576 96 17312 14108
Mem: 60100 26928 15888 160 17284 13432
Mem: 60100 27372 15112 160 17616 12824
Mem: 60100 27744 14600 160 17756 12388
Mem: 60100 28132 14340 160 17628 12064
Mem: 60100 28620 13852 160 17628 11576
Mem: 60100 29200 13272 160 17628 10996
Mem: 60100 29592 12880 160 17628 10604
Mem: 60100 30272 12200 160 17628 9924
Mem: 60100 30676 11796 160 17628 9520
Mem: 60100 31456 10944 160 17700 8712
Mem: 60100 31368 11032 160 17700 8800
Mem: 60100 31648 10752 160 17700 8520
Mem: 60100 31832 12596 160 15672 9168
Mem: 60100 32260 12168 160 15672 8740
Mem: 60100 32376 12052 160 15672 8624
Mem: 60100 32568 11860 160 15672 8432
Mem: 60100 33504 10924 160 15672 7496
Mem: 60100 34160 12528 160 13412 7928
Mem: 60100 35316 11372 160 13412 6772
Mem: 60100 35572 14996 160 9532 8140
Usage climbs and free falls until the OOM killer wipes out one of the hostapd tasks (as it's the biggest memory user).
I've tried with :
- Firewall disabled and a single manual MASQ entry for clients.
- USB disabled (the USB port is unused but the drivers compiled in).
- 2.4G Radio disabled with all clients on 5G.
- 5G Radio disabled with all clients on 2.4G.
Disabling the 5G radio results in this :
brad@srv:~/diag$ cat fred.log | egrep "^Mem:"
total used free shared buff/cache available
Mem: 60100 30072 12408 92 17620 12168
Mem: 60100 30076 12404 92 17620 12164
Mem: 60100 30148 12204 92 17748 12032
Mem: 60100 30228 12236 92 17636 12008
Mem: 60100 30308 12156 92 17636 11928
Mem: 60100 30264 12200 92 17636 11972
Mem: 60100 30228 12236 92 17636 12008
Mem: 60100 30224 12240 92 17636 12012
Mem: 60100 30216 12248 92 17636 12020
Mem: 60100 30228 12236 92 17636 12008
Mem: 60100 30216 12248 92 17636 12020
Mem: 60100 30216 12248 92 17636 12020
Mem: 60100 30228 12236 92 17636 12008
Mem: 60100 30264 12200 92 17636 11972
Mem: 60100 30228 12236 92 17636 12008
Mem: 60100 30240 12224 92 17636 11996
Mem: 60100 30228 12236 92 17636 12008
Mem: 60100 30232 12104 92 17764 11940
Mem: 60100 30228 12044 92 17828 11864
Mem: 60100 30284 11988 92 17828 11808
Mem: 60100 30264 12072 92 17764 11908
Mem: 60100 30252 12084 92 17764 11920
Mem: 60100 30292 12160 92 17648 11940
Mem: 60100 30344 12108 92 17648 11888
Mem: 60100 30464 11988 92 17648 11768
Mem: 60100 30540 11900 92 17660 11684
Mem: 60100 26940 15436 92 17724 13472
Mem: 60100 26964 15412 92 17724 13448
So outwardly it would look like the driver for the 5G radio is leaking memory. It's identified as " Type: MediaTek MT7610E 802.11nac".
Has anyone seen anything similar?
I've tried kmemleak, but it doesn't identify anything obvious and just brings the system to its knees. Slab tracing does the same.
root@TL-WR902AC:~# cat /etc/openwrt_*
DISTRIB_ID='OpenWrt'
DISTRIB_RELEASE='SNAPSHOT'
DISTRIB_REVISION='r14522-8cfb839907'
DISTRIB_TARGET='ramips/mt76x8'
DISTRIB_ARCH='mipsel_24kc'
DISTRIB_DESCRIPTION='OpenWrt SNAPSHOT r14522-8cfb839907'
DISTRIB_TAINTS='no-all no-ipv6 busybox'
r14522-8cfb839907
root@TL-WR902AC:~# uname -a
Linux TL-WR902AC 5.4.65 #0 Fri Sep 18 19:42:06 2020 mips GNU/Linux
Config as follows :
brad@debian64:~/build/openwrt$ scripts/diffconfig.sh
CONFIG_TARGET_ramips=y
CONFIG_TARGET_ramips_mt76x8=y
CONFIG_TARGET_ramips_mt76x8_DEVICE_tplink_tl-wr902ac-v3=y
CONFIG_BUSYBOX_CUSTOM=y
CONFIG_BUSYBOX_CONFIG_DIFF=y
# CONFIG_BUSYBOX_CONFIG_FEATURE_IPV6 is not set
# CONFIG_BUSYBOX_DEFAULT_FEATURE_IPV6 is not set
# CONFIG_IPV6 is not set
# CONFIG_KERNEL_AIO is not set
# CONFIG_KERNEL_CGROUPS is not set
# CONFIG_KERNEL_CRASHLOG is not set
# CONFIG_KERNEL_DEBUG_FS is not set
# CONFIG_KERNEL_DEBUG_INFO is not set
# CONFIG_KERNEL_DEBUG_KERNEL is not set
# CONFIG_KERNEL_ELF_CORE is not set
# CONFIG_KERNEL_FANOTIFY is not set
# CONFIG_KERNEL_FHANDLE is not set
# CONFIG_KERNEL_IPV6 is not set
# CONFIG_KERNEL_IP_MROUTE is not set
# CONFIG_KERNEL_KALLSYMS is not set
# CONFIG_KERNEL_NAMESPACES is not set
# CONFIG_KERNEL_SECCOMP is not set
CONFIG_KERNEL_SLABINFO=y
CONFIG_KERNEL_SLUB_DEBUG=y
CONFIG_KERNEL_SLUB_DEBUG_ON=y
# CONFIG_KERNEL_SWAP is not set
CONFIG_OPENSSL_ENGINE=y
CONFIG_OPENSSL_PREFER_CHACHA_OVER_GCM=y
CONFIG_OPENSSL_WITH_ASM=y
CONFIG_OPENSSL_WITH_CHACHA_POLY1305=y
CONFIG_OPENSSL_WITH_CMS=y
CONFIG_OPENSSL_WITH_DEPRECATED=y
CONFIG_OPENSSL_WITH_ERROR_MESSAGES=y
CONFIG_OPENSSL_WITH_PSK=y
CONFIG_OPENSSL_WITH_SRP=y
CONFIG_OPENSSL_WITH_TLS13=y
CONFIG_OPENVPN_openssl_ENABLE_DEF_AUTH=y
CONFIG_OPENVPN_openssl_ENABLE_FRAGMENT=y
CONFIG_OPENVPN_openssl_ENABLE_LZ4=y
CONFIG_OPENVPN_openssl_ENABLE_LZO=y
CONFIG_OPENVPN_openssl_ENABLE_MULTIHOME=y
CONFIG_OPENVPN_openssl_ENABLE_PF=y
CONFIG_OPENVPN_openssl_ENABLE_PORT_SHARE=y
CONFIG_OPENVPN_openssl_ENABLE_SERVER=y
CONFIG_OPENVPN_openssl_ENABLE_SMALL=y
# CONFIG_PACKAGE_MAC80211_DEBUGFS is not set
# CONFIG_PACKAGE_MAC80211_MESH is not set
# CONFIG_PACKAGE_ca-bundle is not set
CONFIG_PACKAGE_cgi-io=y
CONFIG_PACKAGE_iperf3=y
CONFIG_PACKAGE_kmod-crypto-crc32c=y
CONFIG_PACKAGE_kmod-crypto-hash=y
CONFIG_PACKAGE_kmod-fs-ext4=y
CONFIG_PACKAGE_kmod-ledtrig-default-on=y
CONFIG_PACKAGE_kmod-ledtrig-heartbeat=y
CONFIG_PACKAGE_kmod-ledtrig-netdev=y
CONFIG_PACKAGE_kmod-ledtrig-timer=y
CONFIG_PACKAGE_kmod-lib-crc16=y
CONFIG_PACKAGE_kmod-mt76-usb=y
CONFIG_PACKAGE_kmod-mt76x02-usb=y
CONFIG_PACKAGE_kmod-mt76x0u=y
# CONFIG_PACKAGE_kmod-nf-ipt6 is not set
CONFIG_PACKAGE_kmod-nls-cp437=y
# CONFIG_PACKAGE_kmod-ppp is not set
CONFIG_PACKAGE_kmod-scsi-core=y
CONFIG_PACKAGE_kmod-tun=y
CONFIG_PACKAGE_kmod-usb-storage=y
CONFIG_PACKAGE_libiwinfo-lua=y
CONFIG_PACKAGE_liblua=y
CONFIG_PACKAGE_liblucihttp=y
CONFIG_PACKAGE_liblucihttp-lua=y
CONFIG_PACKAGE_liblzo=y
CONFIG_PACKAGE_libopenssl=y
CONFIG_PACKAGE_librt=y
CONFIG_PACKAGE_libubus-lua=y
CONFIG_PACKAGE_libusb-1.0=y
# CONFIG_PACKAGE_libustream-wolfssl is not set
CONFIG_PACKAGE_lua=y
CONFIG_PACKAGE_luci-app-firewall=y
CONFIG_PACKAGE_luci-app-opkg=y
CONFIG_PACKAGE_luci-base=y
CONFIG_PACKAGE_luci-compat=y
CONFIG_PACKAGE_luci-lib-base=y
CONFIG_PACKAGE_luci-lib-ip=y
CONFIG_PACKAGE_luci-lib-jsonc=y
CONFIG_PACKAGE_luci-lib-nixio=y
CONFIG_PACKAGE_luci-mod-admin-full=y
CONFIG_PACKAGE_luci-mod-network=y
CONFIG_PACKAGE_luci-mod-status=y
CONFIG_PACKAGE_luci-mod-system=y
CONFIG_PACKAGE_luci-theme-bootstrap=y
CONFIG_PACKAGE_openvpn-openssl=y
# CONFIG_PACKAGE_ppp is not set
CONFIG_PACKAGE_rpcd=y
CONFIG_PACKAGE_rpcd-mod-file=y
CONFIG_PACKAGE_rpcd-mod-iwinfo=y
CONFIG_PACKAGE_rpcd-mod-luci=y
CONFIG_PACKAGE_rpcd-mod-rrdns=y
CONFIG_PACKAGE_uclibcxx=y
CONFIG_PACKAGE_uhttpd=y
CONFIG_PACKAGE_uhttpd-mod-lua=y
CONFIG_PACKAGE_uhttpd-mod-ubus=y
CONFIG_PACKAGE_usbutils=y
# CONFIG_PACKAGE_wpad-basic-wolfssl is not set
CONFIG_PACKAGE_wpad-wolfssl=y
# CONFIG_TARGET_ROOTFS_INITRAMFS is not set
CONFIG_WPA_MSG_MIN_PRIORITY=2
CONFIG_uhttpd_lua=y
CONFIG_PACKAGE_kmod-lib-crc-ccitt=y
CONFIG_PACKAGE_libip6tc=y