Unable to run Docker with nftables even after fixes

Installing and Using OpenWrt
1

Hey there!

Just reflashed from factory my copy of OpenWRT, now happily in version 22.03.4, but I seem to have run into some issues with Docker since the reflash.

I imported my network and firewall configurations manually from a backup I made before deleting everything, and even modified the dockerd init file to include both fixes outlined in this post (which had worked in my previous installation), but so far either I'm able to start the dockerd service and some containers, but have them isolated from the rest of the network, or have dockerd not start at all, stating that it cannot find the socket even though I can see it.

I'll attach the corresponding log files below just to be sure, but if there's a way to get back on track without having to move to firewall3, it'd be immensely appreciated.

OpenWRT_Troubleshooting.log

/etc/init.d/dockerd

#!/bin/sh /etc/rc.common

USE_PROCD=1
START=99

extra_command "uciadd" "<interface> <device> <zone> Add docker bridge configuration to network and firewall uci config"
extra_command "ucidel" "<interface> <device> <zone> Delete docker bridge configuration from network and firewall uci config"

DOCKER_CONF_DIR="/tmp/dockerd"
DOCKERD_CONF="${DOCKER_CONF_DIR}/daemon.json"

uci_quiet() {
	uci -q "${@}" >/dev/null
}

json_add_array_string() {
	json_add_string "" "${1}"
}

find_network_device() {
	local device="${1}"
	local device_section=""

	check_device() {
		local cfg="${1}"
		local device="${2}"

		local type name
		config_get type "${cfg}" type
		config_get name "${cfg}" name

		[ "${type}" = "bridge" ] && [ "${name}" = "${device}" ] \
			&& device_section="${cfg}"
	}

	config_load network
	config_foreach check_device device "${device}"

	echo "${device_section}"
}

boot() {
	uciadd
	rc_procd start_service
}

uciadd() {
	local iface="${1}"
	local device="${2}"
	local zone="${3}"

	[ -z "${iface}" ] && {
		iface="docker"
		device="docker0"
		zone="docker"
	}

	/etc/init.d/dockerd running && {
		echo "Please stop dockerd service first"
		exit 0
	}

	# Add network interface
	if ! uci_quiet get network.${iface}; then
		logger -t "dockerd-init" -p notice "Adding interface '${iface}' to network config"
		uci_quiet add network interface
		uci_quiet rename network.@interface[-1]="${iface}"
		uci_quiet set network.@interface[-1].device="${device}"
		uci_quiet set network.@interface[-1].proto="none"
		uci_quiet set network.@interface[-1].auto="0"
		uci_quiet commit network
	fi

	# Add docker bridge device
	if [ "$(find_network_device "$device")" = "" ]; then
		logger -t "dockerd-init" -p notice "Adding bridge device '${device}' to network config"
		uci_quiet add network device
		uci_quiet set network.@device[-1].type="bridge"
		uci_quiet set network.@device[-1].name="${device}"
		uci_quiet set network.@device[-1].bridge_empty='1' # <-- add this new bridge option
		uci_quiet commit network
	else
		logger -t "dockerd-init" -p notice "Bridge device '${device}' already defined in network config"
	fi

	# Add firewall zone
	if ! uci_quiet get firewall.${zone}; then
		logger -t "dockerd-init" -p notice "Adding firewall zone '${zone}' to firewall config"
		uci_quiet add firewall zone
		uci_quiet rename firewall.@zone[-1]="${zone}"
		uci_quiet set firewall.@zone[-1].input="ACCEPT"
		uci_quiet set firewall.@zone[-1].output="ACCEPT"
		uci_quiet set firewall.@zone[-1].forward="ACCEPT"
		uci_quiet set firewall.@zone[-1].name="${zone}"
		uci_quiet commit firewall
	fi

	# Add interface to firewall zone
	if uci_quiet get firewall.${zone}; then
		uci_quiet del_list firewall.${zone}.network="${iface}"
		uci_quiet add_list firewall.${zone}.network="${iface}"
		uci_quiet commit firewall
	fi

	reload_config
	ifup docker
}

ucidel() {
	local iface="${1}"
	local device="${2}"
	local zone="${3}"

	[ -z "${iface}" ] && {
		iface="docker"
		device="docker0"
		zone="docker"
	}

	/etc/init.d/dockerd running && {
		echo "Please stop dockerd service first"
		exit 0
	}

	# Remove network device
	if uci_quiet delete network.$(find_network_device "${device}"); then
		logger -t "dockerd-init" -p notice "Deleting bridge device '${device}' from network config"
		uci_quiet commit network
	fi

	# Remove network interface
	if uci_quiet get network.${iface}; then
		logger -t "dockerd-init" -p notice "Deleting interface '${iface}' from network config"
		uci_quiet delete network.${iface}
		uci_quiet commit network
	fi

	# Remove interface from firewall zone
	if uci_quiet get firewall.${zone}; then
		logger -t "dockerd-init" -p notice "Deleting network interface '${iface}' in zone '${zone}' from firewall config"
		uci_quiet del_list firewall.${zone}.network="${iface}"
		uci_quiet commit firewall
		# Remove Firewall zone if network is empty
		if ! uci_quiet get firewall.${zone}.network; then
			logger -t "dockerd-init" -p notice "Deleting firewall zone '${zone}' from firewall config"
			uci_quiet delete firewall.${zone}
		fi
		uci_quiet commit firewall
	fi

	reload_config
}

process_config() {
	local alt_config_file data_root log_level iptables bip

	[ -f /etc/config/dockerd ] || {
		# Use the daemon default configuration
		DOCKERD_CONF=""
		return 0
	}

	# reset configuration
	rm -fr "${DOCKER_CONF_DIR}"
	mkdir -p "${DOCKER_CONF_DIR}"

	config_load 'dockerd'
	config_get alt_config_file globals alt_config_file
	[ -n "${alt_config_file}" ] && [ -f "${alt_config_file}" ] && {
		ln -s "${alt_config_file}" "${DOCKERD_CONF}"
		return 0
	}

	config_get data_root globals data_root "/opt/docker/"
	config_get log_level globals log_level "warn"
	config_get_bool iptables globals iptables "1"

	# Don't add these options by default
	# omission == docker defaults
	config_get log_driver globals log_driver ""
	config_get bip globals bip ""
	config_get registry_mirrors globals registry_mirrors ""
	config_get hosts globals hosts ""
	config_get dns globals dns ""
	config_get_bool ipv6 globals ipv6 ""
	config_get ip globals ip ""
	config_get fixed_cidr globals fixed_cidr ""
	config_get fixed_cidr_v6 globals fixed_cidr_v6 ""

	. /usr/share/libubox/jshn.sh
	json_init
	json_add_string "data-root" "${data_root}"
	json_add_string "log-level" "${log_level}"
	json_add_boolean "iptables" "${iptables}"
	[ -z "${log_driver}" ] || json_add_string "log-driver" "${log_driver}"
	[ -z "${bip}" ] || json_add_string "bip" "${bip}"
	[ -z "${registry_mirrors}" ] || json_add_array "registry-mirrors"
	[ -z "${registry_mirrors}" ] || config_list_foreach globals registry_mirrors json_add_array_string
	[ -z "${registry_mirrors}" ] || json_close_array
	[ -z "${hosts}" ] || json_add_array "hosts"
	[ -z "${hosts}" ] || config_list_foreach globals hosts json_add_array_string
	[ -z "${hosts}" ] || json_close_array
	[ -z "${dns}" ] || json_add_array "dns"
	[ -z "${dns}" ] || config_list_foreach globals dns json_add_array_string
	[ -z "${dns}" ] || json_close_array
	[ -z "${ipv6}" ] || json_add_boolean "ipv6" "${ipv6}"
	[ -z "${ip}" ] || json_add_string "ip" "${ip}"
	[ -z "${fixed_cidr}" ] || json_add_string "fixed-cidr" "${fixed_cidr}"
	[ -z "${fixed_cidr_v6}" ] || json_add_string "fixed-cidr-v6" "${fixed_cidr_v6}"
	json_dump > "${DOCKERD_CONF}"

	[ "${iptables}" -eq "1" ] && config_foreach iptables_add_blocking_rule firewall
}

start_service() {
	local nofile=$(cat /proc/sys/fs/nr_open)

	process_config

	procd_open_instance
	procd_set_param stderr 1
	if [ -z "${DOCKERD_CONF}" ]; then
		procd_set_param command /usr/bin/dockerd
	else
		procd_set_param command /usr/bin/dockerd --config-file="${DOCKERD_CONF}"
	fi
	procd_set_param limits nofile="${nofile} ${nofile}"
	procd_close_instance
}

reload_service() {
	process_config
	procd_send_signal dockerd
}

service_triggers() {
	procd_add_reload_trigger 'dockerd'
}

iptables_add_blocking_rule() {
	local cfg="${1}"

	local device=""
	local extra_iptables_args=""

	handle_iptables_rule() {
		local interface="${1}"
		local outbound="${2}"
		local extra_iptables_args="${3}"

		local inbound=""

		. /lib/functions/network.sh
		network_get_physdev inbound "${interface}"

		[ -z "${inbound}" ] && {
			logger -t "dockerd-init" -p notice "Unable to get physical device for interface ${interface}"
			return
		}

		# Wait for a maximum of 10 second per command, retrying every millisecond
		local iptables_wait_args="--wait 10 --wait-interval 1000"

		# Ignore errors as it might already be present
		iptables ${iptables_wait_args} --table filter --new DOCKER-USER 2>/dev/null
		if ! iptables ${iptables_wait_args} --table filter --check DOCKER-USER --in-interface "${inbound}" --out-interface "${outbound}" ${extra_iptables_args} --jump REJECT 2>/dev/null; then
			logger -t "dockerd-init" -p notice "Drop traffic from ${inbound} to ${outbound}"
			iptables ${iptables_wait_args} --table filter --insert DOCKER-USER --in-interface "${inbound}" --out-interface "${outbound}" ${extra_iptables_args} --jump REJECT
		fi
	}

	config_get device "${cfg}" device

	[ -z "${device}" ] && {
		logger -t "dockerd-init" -p notice "No device configured for ${cfg}"
		return
	}

	config_get extra_iptables_args "${cfg}" extra_iptables_args
	config_list_foreach "${cfg}" blocked_interfaces handle_iptables_rule "${device}" "${extra_iptables_args}"
}

stop_service() {
	if /etc/init.d/dockerd running; then
		service_stop "/usr/bin/dockerd"
	fi
}

/etc/config/dockerd


config globals 'globals'
	option iptables '1'
	option remote_endpoint '0'
	option log_level 'fatal'
	list hosts 'unix:///var/run/docker.sock'
	option data_root '/usbstick/docker/root'
    option ipv6 '1'

config firewall 'firewall'
	option device 'docker0'
    option extra_iptables_args '--match conntrack ! --ctstate RELATED,ESTABLISHED' # allow outbound connections
2

Okay, turns out things were a bit more complicated than anticipated.

Running the dockerd executable directly lead me to find out I was missing some extra packages, of which among them I found out I needed the following:

iptables-zz-legacy
ip6tables-zz-legacy
kmod-ip6tables
kmod-ipt-core
kmod-ipt-extra
kmod-ipt-raw

I'm honestly not too sure why installing these packages was needed as they're not marked as a dependency, but fortunately combining both of the aforementioned fixes lets OpenWRT run Docker containers again under nftables.

3

This topic was automatically closed 10 days after the last reply. New replies are no longer allowed.