IPQ806x NSS Drivers

For Developers
1580

unfortunately，It's not working,booting ok,but always shows "connecting"
Screenshot from 2020-07-21 22-15-44

and here is saved.config

CONFIG_TARGET_ipq806x=y
CONFIG_TARGET_ipq806x_generic=y
CONFIG_TARGET_ipq806x_generic_DEVICE_linksys_ea7500-v1=y
CONFIG_DEVEL=y
CONFIG_TOOLCHAINOPTS=y
CONFIG_BUILD_NLS=y
CONFIG_CCACHE=y
CONFIG_GNUTLS_ALPN=y
CONFIG_GNUTLS_ANON=y
CONFIG_GNUTLS_DTLS_SRTP=y
CONFIG_GNUTLS_HEARTBEAT=y
CONFIG_GNUTLS_OCSP=y
CONFIG_GNUTLS_PSK=y
CONFIG_IMAGEOPT=y
# CONFIG_KERNEL_AIO is not set
# CONFIG_KERNEL_CGROUPS is not set
# CONFIG_KERNEL_FANOTIFY is not set
# CONFIG_KERNEL_FHANDLE is not set
# CONFIG_KERNEL_NAMESPACES is not set
# CONFIG_KERNEL_SECCOMP is not set
CONFIG_LIBCURL_COOKIES=y
CONFIG_LIBCURL_FILE=y
CONFIG_LIBCURL_FTP=y
CONFIG_LIBCURL_HTTP=y
CONFIG_LIBCURL_MBEDTLS=y
CONFIG_LIBCURL_NO_SMB="!"
CONFIG_LIBCURL_PROXY=y
CONFIG_OPENSSL_ENGINE=y
CONFIG_OPENSSL_OPTIMIZE_SPEED=y
CONFIG_OPENSSL_PREFER_CHACHA_OVER_GCM=y
CONFIG_OPENSSL_WITH_ASM=y
CONFIG_OPENSSL_WITH_CHACHA_POLY1305=y
CONFIG_OPENSSL_WITH_CMS=y
CONFIG_OPENSSL_WITH_DEPRECATED=y
CONFIG_OPENSSL_WITH_ERROR_MESSAGES=y
CONFIG_OPENSSL_WITH_NPN=y
CONFIG_OPENSSL_WITH_PSK=y
CONFIG_OPENSSL_WITH_SRP=y
CONFIG_OPENSSL_WITH_TLS13=y
CONFIG_PACKAGE_MAC80211_NSS_SUPPORT=y
CONFIG_PACKAGE_ath10k-firmware-qca9984-ct-full-htt=y
# CONFIG_PACKAGE_ath10k-firmware-qca99x0-ct is not set
CONFIG_PACKAGE_block-mount=y
CONFIG_PACKAGE_ca-bundle=y
CONFIG_PACKAGE_cgi-io=y
CONFIG_PACKAGE_curl=y
# CONFIG_PACKAGE_dnsmasq is not set
CONFIG_PACKAGE_dnsmasq-full=y
CONFIG_PACKAGE_dnsmasq_full_auth=y
CONFIG_PACKAGE_dnsmasq_full_conntrack=y
CONFIG_PACKAGE_dnsmasq_full_dhcp=y
CONFIG_PACKAGE_dnsmasq_full_dhcpv6=y
CONFIG_PACKAGE_dnsmasq_full_dnssec=y
CONFIG_PACKAGE_dnsmasq_full_ipset=y
CONFIG_PACKAGE_dnsmasq_full_noid=y
CONFIG_PACKAGE_dnsmasq_full_tftp=y
CONFIG_PACKAGE_e2fsprogs=y
CONFIG_PACKAGE_glib2=y
CONFIG_PACKAGE_hostapd-openssl=y
CONFIG_PACKAGE_htop=y
CONFIG_PACKAGE_iftop=y
CONFIG_PACKAGE_iptables-mod-conntrack-extra=y
CONFIG_PACKAGE_iptables-mod-extra=y
CONFIG_PACKAGE_kmod-asn1-decoder=y
CONFIG_PACKAGE_kmod-bonding=y
CONFIG_PACKAGE_kmod-crypto-acompress=y
CONFIG_PACKAGE_kmod-crypto-aead=y
CONFIG_PACKAGE_kmod-crypto-arc4=y
CONFIG_PACKAGE_kmod-crypto-authenc=y
CONFIG_PACKAGE_kmod-crypto-cbc=y
CONFIG_PACKAGE_kmod-crypto-ccm=y
CONFIG_PACKAGE_kmod-crypto-cmac=y
CONFIG_PACKAGE_kmod-crypto-crc32c=y
CONFIG_PACKAGE_kmod-crypto-ctr=y
CONFIG_PACKAGE_kmod-crypto-deflate=y
CONFIG_PACKAGE_kmod-crypto-des=y
CONFIG_PACKAGE_kmod-crypto-ecb=y
CONFIG_PACKAGE_kmod-crypto-echainiv=y
CONFIG_PACKAGE_kmod-crypto-gcm=y
CONFIG_PACKAGE_kmod-crypto-gf128=y
CONFIG_PACKAGE_kmod-crypto-ghash=y
CONFIG_PACKAGE_kmod-crypto-hash=y
CONFIG_PACKAGE_kmod-crypto-hmac=y
CONFIG_PACKAGE_kmod-crypto-manager=y
CONFIG_PACKAGE_kmod-crypto-md4=y
CONFIG_PACKAGE_kmod-crypto-md5=y
CONFIG_PACKAGE_kmod-crypto-null=y
CONFIG_PACKAGE_kmod-crypto-pcompress=y
CONFIG_PACKAGE_kmod-crypto-rng=y
CONFIG_PACKAGE_kmod-crypto-seqiv=y
CONFIG_PACKAGE_kmod-crypto-sha1=y
CONFIG_PACKAGE_kmod-crypto-sha256=y
CONFIG_PACKAGE_kmod-crypto-sha512=y
CONFIG_PACKAGE_kmod-fast-classifier=y
CONFIG_PACKAGE_kmod-fs-ext4=y
CONFIG_PACKAGE_kmod-fs-ntfs=y
CONFIG_PACKAGE_kmod-fs-vfat=y
CONFIG_PACKAGE_kmod-ipsec=y
CONFIG_PACKAGE_kmod-ipt-conntrack-extra=y
CONFIG_PACKAGE_kmod-ipt-extra=y
CONFIG_PACKAGE_kmod-ipt-filter=y
CONFIG_PACKAGE_kmod-ipt-ipset=y
CONFIG_PACKAGE_kmod-ipt-raw=y
CONFIG_PACKAGE_kmod-l2tp=y
CONFIG_PACKAGE_kmod-ledtrig-default-on=y
CONFIG_PACKAGE_kmod-ledtrig-heartbeat=y
CONFIG_PACKAGE_kmod-ledtrig-netdev=y
CONFIG_PACKAGE_kmod-ledtrig-timer=y
CONFIG_PACKAGE_kmod-lib-crc16=y
CONFIG_PACKAGE_kmod-lib-textsearch=y
CONFIG_PACKAGE_kmod-lib-zlib-deflate=y
CONFIG_PACKAGE_kmod-lib-zlib-inflate=y
CONFIG_PACKAGE_kmod-nf-conntrack-netlink=y
CONFIG_PACKAGE_kmod-nf-nathelper=y
CONFIG_PACKAGE_kmod-nf-nathelper-extra=y
CONFIG_PACKAGE_kmod-nfnetlink=y
CONFIG_PACKAGE_kmod-nls-cp437=y
CONFIG_PACKAGE_kmod-nls-iso8859-1=y
CONFIG_PACKAGE_kmod-nls-utf8=y
CONFIG_PACKAGE_kmod-pppol2tp=y
CONFIG_PACKAGE_kmod-qca-nss-drv=y
CONFIG_PACKAGE_kmod-qca-nss-ecm-standard=y
CONFIG_PACKAGE_kmod-qca-nss-gmac=y
CONFIG_PACKAGE_kmod-shortcut-fe=y
CONFIG_PACKAGE_kmod-udptunnel4=y
CONFIG_PACKAGE_kmod-udptunnel6=y
CONFIG_PACKAGE_kmod-usb-storage=y
CONFIG_PACKAGE_kmod-usb-storage-extras=y
CONFIG_PACKAGE_libatomic=y
CONFIG_PACKAGE_libattr=y
CONFIG_PACKAGE_libblkid=y
CONFIG_PACKAGE_libcap=y
CONFIG_PACKAGE_libcap-ng=y
CONFIG_PACKAGE_libcomerr=y
CONFIG_PACKAGE_libcurl=y
CONFIG_PACKAGE_libevent2=y
CONFIG_PACKAGE_libext2fs=y
CONFIG_PACKAGE_libffi=y
CONFIG_PACKAGE_libgmp=y
CONFIG_PACKAGE_libgnutls=y
CONFIG_PACKAGE_libiconv-full=y
CONFIG_PACKAGE_libintl-full=y
CONFIG_PACKAGE_libiwinfo-lua=y
CONFIG_PACKAGE_liblua=y
CONFIG_PACKAGE_liblucihttp=y
CONFIG_PACKAGE_liblucihttp-lua=y
CONFIG_PACKAGE_libmbedtls=y
CONFIG_PACKAGE_libminiupnpc=y
CONFIG_PACKAGE_libmnl=y
CONFIG_PACKAGE_libnatpmp=y
CONFIG_PACKAGE_libncurses=y
CONFIG_PACKAGE_libnetfilter-conntrack=y
CONFIG_PACKAGE_libnettle=y
CONFIG_PACKAGE_libnfnetlink=y
CONFIG_PACKAGE_libnl-core=y
CONFIG_PACKAGE_libnl-genl=y
CONFIG_PACKAGE_libopenssl=y
CONFIG_PACKAGE_libpcap=y
CONFIG_PACKAGE_libpcre=y
CONFIG_PACKAGE_libpcre2=y
CONFIG_PACKAGE_libpcsclite=y
CONFIG_PACKAGE_libpopt=y
CONFIG_PACKAGE_libreadline=y
CONFIG_PACKAGE_librt=y
CONFIG_PACKAGE_libslang2=y
CONFIG_PACKAGE_libss=y
CONFIG_PACKAGE_libtasn1=y
CONFIG_PACKAGE_libtirpc=y
CONFIG_PACKAGE_libubus-lua=y
CONFIG_PACKAGE_libusb-1.0=y
CONFIG_PACKAGE_libuuid=y
CONFIG_PACKAGE_lua=y
CONFIG_PACKAGE_luci=y
CONFIG_PACKAGE_luci-app-firewall=y
CONFIG_PACKAGE_luci-app-opkg=y
CONFIG_PACKAGE_luci-app-upnp=y
CONFIG_PACKAGE_luci-base=y
CONFIG_PACKAGE_luci-lib-ip=y
CONFIG_PACKAGE_luci-lib-jsonc=y
CONFIG_PACKAGE_luci-lib-nixio=y
CONFIG_PACKAGE_luci-mod-admin-full=y
CONFIG_PACKAGE_luci-mod-network=y
CONFIG_PACKAGE_luci-mod-status=y
CONFIG_PACKAGE_luci-mod-system=y
CONFIG_PACKAGE_luci-proto-ipv6=y
CONFIG_PACKAGE_luci-proto-ppp=y
CONFIG_PACKAGE_luci-theme-bootstrap=y
CONFIG_PACKAGE_miniupnpd=y
CONFIG_PACKAGE_rpcd=y
CONFIG_PACKAGE_rpcd-mod-file=y
CONFIG_PACKAGE_rpcd-mod-iwinfo=y
CONFIG_PACKAGE_rpcd-mod-luci=y
CONFIG_PACKAGE_rpcd-mod-rrdns=y
CONFIG_PACKAGE_samba4-libs=y
CONFIG_PACKAGE_samba4-server=y
CONFIG_PACKAGE_screen=y
CONFIG_PACKAGE_socat=y
CONFIG_PACKAGE_terminfo=y
CONFIG_PACKAGE_uclibcxx=y
CONFIG_PACKAGE_uhttpd=y
CONFIG_PACKAGE_uhttpd-mod-ubus=y
CONFIG_PACKAGE_vsftpd-tls=y
CONFIG_PACKAGE_wget=y
# CONFIG_PACKAGE_wpad-basic is not set
CONFIG_PACKAGE_wsdd2=y
CONFIG_PACKAGE_zlib=y
CONFIG_PCRE2_JIT_ENABLED=y
CONFIG_PCRE_JIT_ENABLED=y
CONFIG_SAMBA4_SERVER_NETBIOS=y
CONFIG_SOCAT_SSL=y
CONFIG_TARGET_OPTIMIZATION="-O2 -pipe -mfpu=neon-vfpv4 -mtune=cortex-a15 -ffunction-sections -fdata-sections -mfloat-abi=hard"
CONFIG_TARGET_OPTIONS=y
# CONFIG_TARGET_ROOTFS_INITRAMFS is not set
# CONFIG_PACKAGE_attr is not set
# CONFIG_PACKAGE_avahi-dbus-daemon is not set
# CONFIG_PACKAGE_dbus is not set
# CONFIG_PACKAGE_libavahi-client is not set
# CONFIG_PACKAGE_libavahi-dbus-support is not set
# CONFIG_PACKAGE_libdaemon is not set
# CONFIG_PACKAGE_libdbus is not set
# CONFIG_PACKAGE_libexpat is not set
# CONFIG_SAMBA4_SERVER_AVAHI is not set
# CONFIG_SAMBA4_SERVER_VFS is not set
1581

I only have two custom rules related to my Wireguard interface. These should not have any impact on lan behavior (at least in principle...).

1582

I'm not sure if it's relevant, as I'm not entirely sure what the rule is for, but one of the packages adds a file into /etc/firewall.d which, as far as I can tell, allows all traffic between various bridge interfaces. The script seems a little basic and ended up adding the rules in the wrong place or multiple times on my system, so I amended it as below

iptables -C FORWARD -m physdev --physdev-is-bridged -j ACCEPT > /dev/null 2>&1

if [ $? -eq 1 ]; then

 echo "Adding physdev NSS rule"
iptables -I FORWARD 1 -m physdev --physdev-is-bridged -j ACCEPT

else

  echo "Physdev rule exists"

fi

Possibly worth looking at as an alternative solution to the problem ?

1583

I had noticed a commit by @quarky that may have to do with this, but I'm not sure what it means either.

I'll take a closer look at it, thanks.

1584

Typing the command in the monitor:

iptables -I FORWARD 1 -m physdev --physdev-is-bridged -j ACCEPT

Results in:

Couldn't load match `physdev':No such file or directory

It seems to me that this rule has no effect...

1585

You're probably missing support for physdev in iptables and the relevant kernel modules, I seem to recall I had to go back and add them in to my build. kmod-ipt-physdev and iptables-mod-physdev are what you need...

If you don't have the modules then it may be why you couldn't ping different interfaces. I'm still not entirely sure why it's needed when nss is enabled, I was working on the assumption that the QCA guys had added it for a reason, but reading some of the comments about their coding standards, I'm not so sure now :slight_smile:

1586

I was unaware of those packages, thank you! I'll compile again and test it. Cheers.

EDIT: just tested and it solved the wifi pinging problem with all firewall rules the same as in the regular (non-nss) build.

However, I still can't figure out the Netflix/Ring connection problem. The only thing I saw unusual in the logs is "nameserver 127.0.0.53 refused to do a recursive query". I also tested regular DNS servers with dnsmasq and still no go. Weird... I'll try some other tests when possible.

1587

It seems wireless offload definitely buggy. I have observed some packet missing especially upload flow.

1588

Do see if you can see any error(s) logged in syslog.

You can turn off offload for mac80211 with the following:

echo 0 > /sys/module/mac80211/parameters/is_nss_enable

And finally, if you only set your NSS clock to 550/600MHz, I would suggest bumping it up to 730/800Mhz and see if the packet drops still occurs.

2 Likes
1589

@quarky - My friend has a problem with NSS acceleration all the time. Has a PPPoE connection on the WAN. The problem is strange because some pages are not working (http protocol error). It has qsdk10 by @Ansuel with my NSS clock lock. It's OK for me, but I have a DHCP connection on the WAN. All modules are loaded correctly, there are no errors in the LOG.
I found a new repository on CodeAurora where kernel 5.4 (similar to ours) is used. I tried to port the bonding driver and the whole ppp but screwed up completely because I don't know much about it. Can you have a look at the sources and verify that the NSS-CLIENTS patches are correct? I have no idea anymore.

As for the sources, I found such a manifest
I think from this manifest we can move the missing part of the code to our sources.

1590

Your friend’s issue may be caused by MSS clamping, or the lack of it. PPPoE’s connection uses MTU size of 1492 compared to the normal 1500. Intermittent issues with website appears to be a classic PMTU problem. Did your friend reconfigured the router’s firewall settings and removed the MSS clamping option for the WAN interface?

I would advice against messing with the kernel drivers. OpenWrt uses upstream drivers. When NSS takes overs it the drivers will not be used. I have yet to test PPPoE with QSDK 10 or QSDK 11 firmware but I doubt it’ll have any major issues as described by you. QSDK 6 firmware has been working fine for PPPoE, so I would think later NSS firmware should work fine as well.

The QSDK manifest you posted likely for QSDK 12, so it’s not going to be useful for any of us without the corresponding NSS firmware.

1591

It has a default configuration. Can you show an example what is to change?

1592

I'm using a build from Ansuels QSDK10 repo on a XR500 including PPPoE and i don't have any issues at all.
Neither with PPPoE nor WIFI offload.

In the firewall configuration in the WAN-zone MSS-Clamping is activated (this is the default i guess)

1593

It works for me too, but what if a friend has to leave NSS because it doesn't work for him.

1594

There a new commit that is trying to address a similar issue: https://git.openwrt.org/?p=openwrt/openwrt.git;a=commitdiff;h=afaa978b7474b4fc187598866534dee17d56179e. Turns out, mss clamping was not always working properly.

1595

If I understand the patch correctly, it will apply to cases where OpenWrt (i.e. the router) is acting as a server accepting requests from WAN, which has a lower MTU. For all traffic originating from LAN to WAN, MSS would have already been clamped to the WAN interface's MTU for the PPPoE case.

Also, folks running VPN service on their router will also see PMTU issue if it's not configured correctly. MSS clamping must also be done for TCP connections going out to the VPN tunnel as default gateway.

1596

@quarky can you take a look at the sources i indicated? I don't mean the qsdk drivers, because they are newer than ours, but it is possible that the kernel code has something we can use.
It's quite possible that we're missing something, because the patches you made are mostly from kernel 4.4, and we're using 5.4 (same as I indicated above).

There is a problem and it must be solved. It doesn't matter that it works for you and me. It doesn't work for everyone.
@Ansuel also downloaded the patches that fixed UDP from the latest sources CodeAurora - so it is possible.

1597

@sqter, studying Linux kernel code is not a simple tasks. As indicated in a few post earlier, it looks like PPPoE with NSS on QSDK 10 appears to work fine, as reported by @kirdes.

I would suggest that your friend try the same configuration in the same master build, but without the NSS active. I.e. just move the /lib/firmware/qca-* files somewhere and reboot. I suspect your friend will see the same issue even without NSS active.

1598

Hi all,

I'm trying to figure out if I can make further improvement to the NSS crypto performance and have some findings that I need help on to move further. Below are the logs generated when I try to encrypt a file using OpenSSL using the NSS crypto engine:

[  148.751346] crypto_done: len=4096, op=2, time=182(us), rate=180043(Kbps)
[  148.751671] crypto_done: len=4096, op=2, time=112(us), rate=292571(Kbps)
[  148.757205] crypto_done: len=4096, op=2, time=5417(us), rate=6049(Kbps)
[  148.763822] crypto_done: len=4096, op=2, time=6434(us), rate=5092(Kbps)
[  148.770311] crypto_done: len=4096, op=2, time=6373(us), rate=5141(Kbps)
[  148.776805] crypto_done: len=4096, op=2, time=6412(us), rate=5110(Kbps)
[  148.783393] crypto_done: len=4096, op=2, time=6473(us), rate=5062(Kbps)
[  148.790023] crypto_done: len=4096, op=2, time=6554(us), rate=4999(Kbps)
[  148.796640] crypto_done: len=4096, op=2, time=6375(us), rate=5140(Kbps)
[  148.803141] crypto_done: len=1392, op=2, time=6430(us), rate=1731(Kbps)
[  148.809828] crypto_done: len=16, op=2, time=6651(us), rate=19(Kbps)

The output above show each cipher rounds with the length of the message encrypted, the time taken(in microseconds) and the computed thruput as rate. The output is very consistent for each attempts I tried, in that the first two operations always complete in 100-300 us, but subsequent rounds balloons to 10 times longer, into the 5-6 ms range.

For each rounds of the cipher, general operations are:

1. prepare cipher data and send to NSS
2. send interrupt to NSS to start ciphering
3. receives interrupt from NSS upon completion and NAPI routine called
4. get cipher results and return as crypto request results

Would any of the resident experts here know anything about Linux's interrupt mechanism to help improve the timings above, specifically between 2. and 3.? I tried adjusting the NAPI budget from the standard value of 64 all the way down to 1, halving it for each attempts. It didn't help.

If the above issue can be resolved, we can make use of the NSS crypto engine for real world use, e.g. OpenVPN.

Ideally, if we can consistently make the duration down to within 500 us, it should be usable for real world crypto operations with low Krait CPU usage.

1 Like
1599

@quarky As it disables NSS it all works fine in the NSS image. Enabling NSS generates a problem.