Periodic problems in resolving DNS Requests with dnsmasq and stubby on OpenWrt (DoT, DNSSEC)

Tschabadu · October 6, 2022, 4:54pm

I followed the instructions from https://www.kuketz-blog.de/stubby-verschluesselte-dns-anfragen-openwrt-teil5/ (in German)

Setup is

OpenWRT: 20.03.0
Stubby: 0.4.0
Dnsmasq: 2.86

openwrt -> dnsmasq (53) -> stubby (5453) -> internet / resolvers (853)

Configurations

/etc/config/dhcp

config dnsmasq
	option domainneeded '1'
	option localise_queries '1'
	option boguspriv '1' # Changes this to on an off, without effect
	option filterwin2k '0'
	option rebind_protection '1'
	option rebind_localhost '1'
	option expandhosts '1'
	option authoritative '1'
	option readethers '1'
	option leasefile '/tmp/dhcp.leases'
	option localservice '1'
	option ednspacket_max '1232'
	option local '/lan/'
	option domain 'lan'
	list rebind_domain 'plex.direct'
	option confdir '/tmp/dnsmasq.d'
	list server '127.0.0.1#5453'
	option logqueries '1'
	option dnsseccheckunsigned '1'
	option noresolv '1'
	option dnssec '1'

/etc/config/stubby

config stubby 'global'
       option manual '0'
       option trigger 'wan'
       # option triggerdelay '2'
       list dns_transport 'GETDNS_TRANSPORT_TLS'
       option tls_authentication '1'
       option tls_query_padding_blocksize '128'
       # option tls_connection_retries '2'
       # option tls_backoff_time '3600'
       # option timeout '5000'
       # option dnssec_return_status '0'
       option appdata_dir '/var/lib/stubby'
       # option trust_anchors_backoff_time 2500
       # option dnssec_trust_anchors '/var/lib/stubby/getdns-root.key'
       option edns_client_subnet_private '1'
       option idle_timeout '10000'
       option round_robin_upstreams '1'
       list listen_address '127.0.0.1@5453'
       list listen_address '0::1@5453'
       option log_level '7'
       # option command_line_arguments ''
       # option tls_cipher_list 'EECDH+AESGCM:EECDH+CHACHA20'
       # option tls_ciphersuites 'TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256'
       option tls_min_version '1.2'
       # option tls_max_version '1.3'

# Upstream resolvers are specified using 'resolver' sections.
config resolver
        option address '80.241.218.68'
        option tls_auth_name 'fdns1.dismail.de'
        list spki 'sha256/MMi3E2HZr5A5GL+badqe3tzEPCB00+OmApZqJakbqUU='

config resolver
	option address '159.69.114.157'
	option tls_auth_name 'fdns2.dismail.de'
	list spki 'sha256/yJYDim2Wb6tbxUB3yA5ElU/FsRZZhyMXye8sXhKEd1w='

config resolver
        option address '5.9.164.112'
        option tls_auth_name 'dns3.digitalcourage.de'
        list spki 'sha256/2WFzfO2/56HpeR+v/l25NPf5dacfxLrudH5yZbWCfdo='

config resolver
	option address '185.95.218.42'
	option tls_auth_name 'dns.digitale-gesellschaft.ch'

I used the following links to verify and troubleshoot, without much success so far

And those links from the forum do not seem to be related to my issue, as my dns most of the times works fine.

Logread dnsmasq

When it is not working I found out that I get a lot of is BOGUS log entries.

Logread dnsmasq

Thu Oct  6 17:42:52 2022 daemon.info dnsmasq[1]: 123527 192.168.150.121/57910 validation forum.openwrt.org is BOGUS
Thu Oct  6 17:42:52 2022 daemon.info dnsmasq[1]: 123527 192.168.150.121/57910 reply error is SERVFAIL
Thu Oct  6 17:42:52 2022 daemon.info dnsmasq[1]: 123525 192.168.150.121/58554 reply query is duplicate
Thu Oct  6 17:42:52 2022 daemon.info dnsmasq[1]: 123523 192.168.150.121/46320 reply query is duplicate

Logread stubby

Thu Oct  6 18:22:53 2022 daemon.err stubby[10493]: [16:22:53.666172] STUBBY: 5.9.164.112                              : Upstream   : !Backing off TLS on this upstream    - Will retry again in 16s at Thu Oct  6 16:23:09 2022
Thu Oct  6 18:22:53 2022 daemon.err stubby[10493]:
Thu Oct  6 18:22:54 2022 daemon.err stubby[10493]: [16:22:54.465995] STUBBY: 185.95.218.42                            : Conn closed: TLS - *Failure*
Thu Oct  6 18:22:54 2022 daemon.err stubby[10493]:
Thu Oct  6 18:22:54 2022 daemon.err stubby[10493]: [16:22:54.466027] STUBBY: 185.95.218.42                            : Conn closed: TLS - *Failure*
Thu Oct  6 18:22:54 2022 daemon.err stubby[10493]:
Thu Oct  6 18:22:54 2022 daemon.err stubby[10493]: [16:22:54.466043] STUBBY: 185.95.218.42                            : Conn closed: TLS - *Failure*
Thu Oct  6 18:22:54 2022 daemon.err stubby[10493]:
Thu Oct  6 18:22:54 2022 daemon.err stubby[10493]: [16:22:54.466057] STUBBY: 185.95.218.42                            : Conn closed: TLS - *Failure*
Thu Oct  6 18:22:54 2022 daemon.err stubby[10493]:
Thu Oct  6 18:22:54 2022 daemon.err stubby[10493]: [16:22:54.466070] STUBBY: 185.95.218.42                            : Conn closed: TLS - Resps=     0, Timeouts  =     0, Curr_auth =   None, Keepalive(ms)=     0
Thu Oct  6 18:22:54 2022 daemon.err stubby[10493]:
Thu Oct  6 18:22:54 2022 daemon.err stubby[10493]: [16:22:54.466085] STUBBY: 185.95.218.42

Test sites

Sometimes I get a lot of BOGUS requests and a SERVFAIL...I think few of them would be normal...

The DNS interruption happens about once a week, after 5min wait time or a Router reboot, all works fine again for a while. At the moment, it happens at about every 5 min...not sure what changed, update from 21.03 to 22.03 was about 1 week ago. But I experienced Issues since I would say 4 weeks at least.

Has anybody any idea what the cause might be or how to better troubleshoot this issue?

Hint: At the moment I disabled DoT, DNSSEC and Stubby and just use normal DNS over Port 53 with dnsmasq and it seems to work fine.

trendy · October 6, 2022, 7:16pm

I am also using dnsmasq+stubby.
I do not see why you have enabled

in dnsmasq, as this is something that could trigger the SERVFAIL errors that you see.

Tschabadu · October 6, 2022, 7:55pm

Is a good question, however with it disabled it will not validate DNSSEC, however DoT will still work. I will try and see if errors at least vanish, but this just seems kind of a workaround and not a proper solution.

trendy · October 7, 2022, 1:25pm

But you are not doing recursive resolving, only forwarding the queries to an upstream resolver. As long as the communication to that resolver is encrypted, authenticated, and you trust the results it returns, I don't see why you need the DNSSEC validation.

Tschabadu · October 8, 2022, 1:52am

With DNSSEC disabled the is BOGUS log entries are gone, what is good. I had one last interruption for about 5min and got a lot of duplicate dnsmasq log entries, so I do not trust my results yet. I will test for a week and mark your answer as solved or continue troubleshooting if issues persist or return.

As a sidenote, could there also be problems with the resolvers itself...poor load balancing, service interruptions etc be a cause too?

Tschabadu · October 16, 2022, 4:33pm

The problems persist and I get random interruptions of the dns service and in the browser I see this message when it happens.

I updated OpenWrt now from 22.03 -> 22.03.1, but in the release notes it does not look like some related packages where updated which could help, so I think problem will persist.

Which means I am basically still on it with troubleshooting.

Anybody has more hints, let me know.

Sidenote: Stubby has a new release 0.4.2 according to https://github.com/getdnsapi/stubby, but OpenWrt servers only 0.4.0 at the moment. Is it possible to force push this latest release to add it to the repo package or manually install this?

trendy · October 17, 2022, 7:31am

Can you try to add Cloudflare nameservers to rule out malfunction of the nameservers you are using?

config resolver
        option address '2606:4700:4700::1111'
        option tls_auth_name 'cloudflare-dns.com'
        option tls_port '853'
        list spki 'sha256/yioEpqeR4WtDwE9YxNVnCEkTxIjx6EEIwFSQW+lJsbc='
        option tls_cipher_list 'EECDH+AESGCM:EECDH+CHACHA20'
        option tls_ciphersuites 'TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256'
        option tls_min_version '1.2'
        option tls_max_version '1.3'
        option disabled '1'

config resolver
        option address '2606:4700:4700::1001'
        option tls_auth_name 'cloudflare-dns.com'
        option tls_port '853'
        list spki 'sha256/yioEpqeR4WtDwE9YxNVnCEkTxIjx6EEIwFSQW+lJsbc='
        option tls_cipher_list 'EECDH+AESGCM:EECDH+CHACHA20'
        option tls_ciphersuites 'TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256'
        option tls_min_version '1.2'
        option tls_max_version '1.3'
        option disabled '1'

config resolver
        option tls_auth_name 'cloudflare-dns.com'
        option tls_port '853'
        list spki 'sha256/yioEpqeR4WtDwE9YxNVnCEkTxIjx6EEIwFSQW+lJsbc='
        list spki 'sha256/yioEpqeR4WtDwE9YxNVnCEkTxIjx6EEIwFSQW+lJsbc='
        option tls_cipher_list 'EECDH+AESGCM:EECDH+CHACHA20'
        option tls_ciphersuites 'TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256'
        option tls_min_version '1.2'
        option tls_max_version '1.3'
        option disabled '1'
        option address '1.0.0.1'

Tschabadu · October 18, 2022, 4:37pm

Yes, good idea, I will give it a try.

Tschabadu · October 18, 2022, 5:27pm

It seems to run smother and much less stubby log outputs, however see that list spki and certain other options are disabled. Copying yours I could not get stubby to run by the way (maybe formatting issues).

I used this one from the openwrt stubby default stubby-opkg under /etc/config:

/etc/config/stubby

# ....

# Upstream resolvers are specified using 'resolver' sections.
config resolver
       option address '2606:4700:4700::1111'
       option tls_auth_name 'cloudflare-dns.com'
       # option tls_port 853
       # list spki 'sha256/yioEpqeR4WtDwE9YxNVnCEkTxIjx6EEIwFSQW+lJsbc='
       # option tls_cipher_list 'EECDH+AESGCM:EECDH+CHACHA20'
       # option tls_ciphersuites 'TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256'
       # option tls_min_version '1.2'
       # option tls_max_version '1.3'

config resolver
       option address '2606:4700:4700::1001'
       option tls_auth_name 'cloudflare-dns.com'
       # option tls_port 853
       # list spki 'sha256/yioEpqeR4WtDwE9YxNVnCEkTxIjx6EEIwFSQW+lJsbc='
       # option tls_cipher_list 'EECDH+AESGCM:EECDH+CHACHA20'
       # option tls_ciphersuites 'TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256'
       # option tls_min_version '1.2'
       # option tls_max_version '1.3'

config resolver
       option address '1.1.1.1'
       option tls_auth_name 'cloudflare-dns.com'
       # option tls_port 853
       # list spki 'sha256/yioEpqeR4WtDwE9YxNVnCEkTxIjx6EEIwFSQW+lJsbc='
       # option tls_cipher_list 'EECDH+AESGCM:EECDH+CHACHA20'
       # option tls_ciphersuites 'TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256'
       # option tls_min_version '1.2'
       # option tls_max_version '1.3'

config resolver
       option address '1.0.0.1'
       option tls_auth_name 'cloudflare-dns.com'
       # option tls_port 853
       # list spki 'sha256/yioEpqeR4WtDwE9YxNVnCEkTxIjx6EEIwFSQW+lJsbc='
       # option tls_cipher_list 'EECDH+AESGCM:EECDH+CHACHA20'
       # option tls_ciphersuites 'TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256'
       # option tls_min_version '1.2'
       # option tls_max_version '1.3'

I will let it run for a while.

system · October 29, 2022, 2:15pm

This topic was automatically closed 10 days after the last reply. New replies are no longer allowed.