23.05 dnsmasq, ipsets and mwan3 incompatibility?

Installing and Using OpenWrt
36

Thanks for the expires fix. I've integrated it into the latest iteration. Which is now also an init Script and does not need to be run using cron anymore (although it's basically simply wrapping the script into a init-service):

#!/bin/bash /etc/rc.common
# Start before firewall and mwan3 which are at Prio 19
START=18
APP=nft2ipset
USE_PROCD=1
SCRIPTPATH="/tmp/nft2ipset"

write_script() {
cat > "$1" <<'EOT'
#!/bin/bash
#check if the script is already running
PID=$$
SCRIPT="$(basename $0)"
TMPDIR="/tmp"
MONITORPIDFILE="$TMPDIR/$SCRIPT-$$.nftmonitorpid"
MONITORFIFO="$TMPDIR/$SCRIPT-$$.nftmonitorfifo"
mkfifo "$MONITORFIFO"

cleanup () {
  # Cleanup nft monitor subprocess
  if [ -f "$MONITORPIDFILE" ]; then
    MONITORPID="$(cat "$MONITORPIDFILE")"
    if [ "$MONITORPID" -gt 1 ]; then
      kill "$MONITORPID"
    fi
  fi
  # Remove pid file and fifo
  rm "$MONITORFIFO" "$MONITORPIDFILE"
}
trap cleanup TERM INT EXIT

create_or_update_ipset() {
  # Determine ipset parameters
  local DEF="$1"
  local NAME="$(echo "$DEF" | cut -d' ' -f2)"
  local OPTS=""
  local FAMILY="inet"
  if echo "$DEF" | grep -q "ipv6_addr"; then
    FAMILY="inet6"
    OPTS="$OPTS family $FAMILY"
  fi 
  local TIMEOUT="$(echo "$DEF" | sed -r 's/.*timeout ([0-9]*)s.*/\1/; t; s/.*/0/')"
  if [ -n "$TIMEOUT" -a "$TIMEOUT" -gt 0 ]; then
    OPTS="$OPTS timeout $TIMEOUT"
  fi

  # Create or update ipset from nftables set
  if [ "$(ipset list -n "$NAME")" = "$NAME" ]; then
    CUR="$(ipset list -t "$NAME")"
    if ! ( echo "$CUR" | grep -q "family $FAMILY"); then
      ( ipset destroy "$NAME" 2>&1 | logger -t "$SCRIPT" ) || logger -t "$SCRIPT" "WARNING: Could not destroy ipset with family != $FAMILY"    
    elif ! ( echo "$CUR" | grep -q "timeout $TIMEOUT"); then
      # Swap current iteration of the ipset with a new iteration due to timeout mismatch
      ipset create "_$NAME" hash:ip $OPTS
      ipset swap "_$NAME" "$NAME"
      ipset destroy "_$NAME"
      logger -t "$SCRIPT" "Replaced ipset $NAME with new iteration with timeout $TIMEOUT"
    fi
  fi
  if [ "$(ipset list -n "$NAME")" != "$NAME" ]; then
    # Create a new ipset with options matching the nftables set
    ipset create "$NAME" hash:ip $OPTS
    # Restart mwan3 if this ipset is used by it, it is already running but the set name is not found in active rule output
    if [ $? = 0 ] && grep -q "option ipset '$NAME'" /etc/config/mwan3 2>/dev/null && ( service | grep mwan3 | grep running ) && ( ! (mwan3 rules | grep -q "match-set $NAME" ) ); then
      mwan3 restart
    fi
    logger -t "$SCRIPT" "Created new ipset $NAME with timeout $TIMEOUT"
  fi

  # Add already existing entries to the set
  echo "$DEF" | sed -re 's/.*elements = \{ ([^\}]+) \}.*/\1/g' | tr ',' '\n' | sed -re 's/^[ ]+//g;s/expires/timeout/g;s/s$//g' | while read LINE; do
    ipset -q add "$NAME" $LINE >/dev/null 2>&1 && logger -t "$SCRIPT" "Added $LINE to $NAME upon ipset creation/update"
  done
}

# Check if ipsets exist for all currently existing nftsets or create otherwise
nft -nT list sets | tr '\n' ' ' | awk '{$1=$1;print}' | sed -r 's/(set|table)/\n\1/g' | grep "^set" | while read DEF; do
  create_or_update_ipset "$DEF"
done

# Monitor nftables rule changes
nft -nT monitor > "$MONITORFIFO" 2>&1 &
echo $! > "$MONITORPIDFILE"
while read LINE; do
  if echo "$LINE" | grep -q "add element inet fw4"; then
    # Check if ipset exists or create otherwise
    NAME="$(echo "$LINE" | cut -d' ' -f 5)"
    if [ "$(ipset list -n $NAME)" != "$NAME" ]; then
      DEF="$(nft -tnT list sets | tr '\n' ' ' | awk '{$1=$1;print}' | sed -r 's/(set|table)/\n\1/g' | grep "^set $NAME")"
      create_or_update_ipset "$DEF"
    fi
    # Add element to ipset
    IP="$(echo "$LINE" | cut -d' ' -f 7)"
    EXPIRES="$(echo "$LINE" | cut -d' ' -f 9 | grep -Eo '[0-9]*')"
    ADDOPTS=""
    if [[ "$EXPIRES" =~ ^[0-9]+$ && "$EXPIRES" -gt 0 ]]; then
      ADDOPTS="timeout $EXPIRES"
    fi
    if ipset -q test "$NAME" "$IP"; then
      # Refresh the entry by deleting it first if already existing
      ipset -q del "$NAME" "$IP"
      ipset -q add "$NAME" "$IP" $ADDOPTS
    else
      ipset -q add "$NAME" "$IP" $ADDOPTS
      logger -t "$SCRIPT" "Added $IP to ipset $NAME"
    fi
  elif echo "$LINE" | grep -q "add set inet fw4"; then
    # Create or update ipset 
    NAME="$(echo "$LINE" | cut -d' ' -f 5)"
    DEF="$(nft -nT list sets | tr '\n' ' ' | awk '{$1=$1;print}' | sed -r 's/(set|table)/\n\1/g' | grep "^set $NAME")"
    create_or_update_ipset "$DEF"
  elif echo "$LINE" | grep -q "delete set inet fw4"; then
    # Clear and try to delete removed ipset (This will fail if it is in use by any iptables rule)
    NAME="$(echo "$LINE" | cut -d' ' -f 5)"
    ipset clear "$NAME"
    ipset destroy "$NAME" 2>&1 | logger -t "$SCRIPT"
  fi
done < "$MONITORFIFO"
EOT
}

start_service() {
  write_script "$SCRIPTPATH"
  chmod +x "$SCRIPTPATH"
  procd_open_instance
  procd_set_param command "$SCRIPTPATH"
  procd_set_param respawn
  procd_close_instance
}
service_stopped() {
  rm "$SCRIPTPATH"
}
# vim: ts=2 sw=2 et

For this to work you simply have to put the contents above into /etc/init.d/nft2ipset and enable + start the new service with:

chmod +x /etc/init.d/nft2ipset
service nft2ipset enable
service nft2ipset start

I've tested functionality and reboot safety so this should be as simple and portable as it gets to make mwan3 work on 23.05+ without it being adapted to nftables.

EDIT: The old cron scheduling and script can be safely removed after enabling/starting the new service.
P.S.: The new FIFO in the script is to work around some interesting way that bash handles subprocess pipes (nft monitor in our case) that would not properly cleanup on killing the script otherwise.

EDIT2: Added logging tag for messages of the script and missing note on making this executable

2 Likes