cat /etc/hotplug.d/iface/99-overlay-mods
#!/bin/sh
#
# Hotplug script for OpenWRT, sits in /etc/hotplug.d/iface/99-overlay-mods
#
# Processes: wan/ifup/ifupdate/reload and wan/ifdown
# aaisp/ifup/ifupdate/reload and aaisp/ifdown
# wan6/ifup/ifupdate/reload and wan6/ifdown
# aaisp6/ifup/ifupdate/reload and aaisp6/ifdown
#
# In ifup/wan:
# - Set up specific routes for the L2TP tunnel server over the wan interface
# - Start aaisp interfaces, which will start an L2TP tunnel over the routes we just created, with a higher
# default (IPv4) priority (ie, lower metric) than the wan interfaces
#
# In ifup/aaisp:
# - Create a default route to the underlay interface (wan) in the (pre-existing) underlay custom routing table
# - Create a rule that directs IPv4 packets marked with 0x10000000 to use the underlay custom routing table
# - Start the Wireguard interfaces, now that the default routing via aaisp is now in place
#
# In ifup/wan6 & aaisp6:
# - Create a default route to the underlay6 interface (wan6) in the (pre-existing) underlay6 custom routing table
# - Create a default route to the overlay6 interface (aaisp6) in the (pre-existing) overlay6 custom routing table
# - Create a rule that directs IPv6 packets marked with 0x10000000 to use the underlay custom routing table
# - Create a rule that directs IPv6 packets marked with 0x20000000 to use the overlay custom routing table
#
# The nftables element of this is injected by fw4 into a custom inet nftable via an include in /etc/config/firewall
# It marks Internet-bound IPv4 packets that need to be directed to the underlay; all the rest go via the default route, which is
# set to the overlay ISP.
# It also marks all Internet-bound IPv6 traffic (as IPv6 is not masqueraded) to direct it via one or the other of the
# underlay or overlay ISPs.
#
# All traffic marked with 0x10000000 will route to the underlay ISP (IPv4 and IPv6)
# All traffic marked with 0x20000000 will route to the overlay ISP (IPv6)
#
# Deliberately set up this way as Wireguard cannot be bound to a specific interface, and always uses the default
# gateway, so that default needs to be set to suit the desired functionality. In this case the wireguard tunnels
# must run over the overlay as its the only one that is externally resolveable, because the underlay is behind CGNAT
#
[ "$INTERFACE" = "aaisp" ] || [ "$INTERFACE" = "wan" ] || [ "$INTERFACE" = "wan6" ] || [ "$INTERFACE" = "aaisp6" ] || exit 0
[ "$ACTION" = "ifup" ] || [ "$ACTION" = "ifupdate" ] || [ "$ACTION" = "reload" ] || [ "$ACTION" = "ifdown" ] || exit 0
UNDERLAY_TABLE="underlay"
UNDERLAY6_TABLE="underlay6"
OVERLAY6_TABLE="overlay6"
UNDERLAY_IF="eth1"
OVERLAY_IF="l2tp-aaisp"
L2TP_HOST="l2tp.aa.net.uk"
SCRIPT="99-overlay-mods"
case "$INTERFACE" in
wan)
# Want to set up the routes to the AAISP L2TP tunnel server as part of the wan interface ifup so there is no
# chance that the L2TP tunnel is brought up over the wrong interface because of metric settings
case "$ACTION" in
ifup | ifupdate | reload)
logger -t hotplug "Setting up routes to $L2TP_HOST in $INTERFACE ifup hotplug script $SCRIPT"
# Add routes so L2TP tunnel continues to use wan interface
# Use nslookup to resolve IPv4 address(es) of L2TP endpoint in AAISP network (skip header and extract IPs only)
L2TP_IPS=$(nslookup "$L2TP_HOST" 127.0.0.1 | awk '/^Address: / {print $2}' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+')
logger -t hotplug "Resolved $L2TP_HOST to: $L2TP_IPS"
for ip in $L2TP_IPS; do
ip route replace "$ip" dev "$UNDERLAY_IF" table main
logger -t hotplug "Added route to $ip via $UNDERLAY_IF in main routing table"
done
# Cache IP addresses for teardown
TMPFILE="/tmp/l2tp_routes.txt"
echo "$L2TP_IPS" > "$TMPFILE"
# WAN is now up, L2TP routes are in place, so bring up the L2TP tunnel to AAISP
# If by any chance they are already up, this will actually take them down before bringing them back up
/sbin/ifup aaisp
/sbin/ifup aaisp6
;;
ifdown)
logger -t hotplug "Removing routes to $L2TP_HOST in $INTERFACE ifup hotplug script $SCRIPT"
if [ -f /tmp/l2tp_routes.txt ]; then
for ip in $(cat /tmp/l2tp_routes.txt); do
ip route delete "$ip" dev "$UNDERLAY_IF" table main 2>/dev/null
logger -t hotplug "Removed route to $ip via $UNDERLAY_IF in main table"
done
rm -f /tmp/l2tp_routes.txt
fi
;;
esac
;;
aaisp)
case "$ACTION" in
ifup | ifupdate | reload)
logger -t hotplug "Setting up $UNDERLAY_TABLE routing table in $INTERFACE ifup hotplug script $SCRIPT"
# Get WAN gateway IP dynamically (as it seems to vary a little)
for i in 1 2 3 4 5; do
WAN_GATEWAY_IP=$(ubus call network.interface.wan status | jsonfilter -e "@['route'][*]['nexthop']")
# WAN_GATEWAY_IP=$(ifstatus wan | grep nexthop | awk -F\" '{print $4}')
[ -n "$WAN_GATEWAY_IP" ] && break
sleep 1
done
if [ -z "$WAN_GATEWAY_IP" ]; then
logger -t hotplug "ERROR: Could not determine WAN gateway for $UNDERLAY_IF"
else
# Create default route in underlay table to Internet via WAN
ip route replace default via "$WAN_GATEWAY_IP" dev "$UNDERLAY_IF" table "$UNDERLAY_TABLE"
logger -t hotplug "Added default route via $WAN_GATEWAY_IP dev $UNDERLAY_IF table $UNDERLAY_TABLE"
# Tie fwmark to routing table with iproute2 rule
ip rule add fwmark 0x10000000/0x30000000 table "$UNDERLAY_TABLE" priority 20100 2>/dev/null
logger -t hotplug "Added ip rule so traffic fwmarked wih 0x10000000/0x30000000 uses custom routing table $UNDERLAY_TABLE"
fi
# At this point our environment is set up, so start the WG tunnels, which will establish on the overlay
# If by any chance they are already up, this will actually take them down before bringing them up, so its safe as it stands
/sbin/ifup site2site
/sbin/ifup mobiles
logger -t hotplug "$UNDERLAY_TABLE routing setup complete and all network interfaces started"
;;
ifdown)
logger -t hotplug "Tearing down $UNDERLAY_TABLE routing in $INTERFACE ifdown hotplug script $SCRIPT"
logger -t hotplug "Removing $UNDERLAY_TABLE routing table rules"
# Remove the rule that directs marked traffic to the underlay table
ip rule delete from all fwmark 0x10000000/0x30000000 lookup "$UNDERLAY_TABLE" 2>/dev/null
# Remove the default route in the underlay table
ip route delete default table "$UNDERLAY_TABLE" 2>/dev/null
logger -t hotplug "Teardown of $UNDERLAY_TABLE routing in $INTERFACE ifdown hotplug script $SCRIPT complete"
;;
esac
;;
wan6)
case "$ACTION" in
ifup | ifupdate | reload)
logger -t hotplug "Setting up $UNDERLAY6_TABLE routing table in $INTERFACE ifup hotplug script $SCRIPT"
# Get WAN6 nexthop
for i in 1 2 3 4 5; do
WAN_GATEWAY_IP=$(ubus call "network.interface.$INTERFACE" status | jsonfilter -e "@['route'][*]['nexthop']")
sleep 1
done
if [ -z "$WAN_GATEWAY_IP" ]; then
logger -t hotplug "ERROR: Could not determine $INTERFACE gateway for $UNDERLAY_IF"
else
# Create default route in underlay6 table to Internet via WAN6
ip -6 route replace default via "$WAN_GATEWAY_IP" dev "$UNDERLAY_IF" table "$UNDERLAY6_TABLE"
logger -t hotplug "Added default route via $WAN_GATEWAY_IP dev $UNDERLAY_IF table $UNDERLAY6_TABLE"
# Tie fwmark to routing table with iproute2 rule
ip -6 rule add fwmark 0x10000000/0x30000000 table "$UNDERLAY6_TABLE" priority 20099 2>/dev/null
logger -t hotplug "Added ip rule so traffic fwmarked wih 0x10000000/0x30000000 uses custom routing table $UNDERLAY6_TABLE"
fi
logger -t hotplug "$UNDERLAY6_TABLE routing setup complete"
;;
ifdown)
logger -t hotplug "Tearing down $UNDERLAY6_TABLE routing in $INTERFACE ifdown hotplug script $SCRIPT"
logger -t hotplug "Removing $UNDERLAY6_TABLE routing table rules"
# Remove the rule that directs marked traffic to the underlay table
ip -6 rule delete from all fwmark 0x10000000/0x30000000 lookup "$UNDERLAY6_TABLE" 2>/dev/null
# Remove the default route in the underlay table
ip -6 route delete default table "$UNDERLAY6_TABLE" 2>/dev/null
logger -t hotplug "Teardown of $UNDERLAY6_TABLE routing in $INTERFACE ifdown hotplug script $SCRIPT complete"
;;
esac
;;
aaisp6)
case "$ACTION" in
ifup | ifupdate | reload)
logger -t hotplug "Setting up $OVERLAY6_TABLE routing table in $INTERFACE ifup hotplug script $SCRIPT"
# Get AAISP6 nexthop
for i in 1 2 3 4 5; do
WAN_GATEWAY_IP=$(ubus call "network.interface.$INTERFACE" status | jsonfilter -e "@['route'][*]['nexthop']")
sleep 1
done
if [ -z "$WAN_GATEWAY_IP" ]; then
logger -t hotplug "ERROR: Could not determine $INTERFACE gateway for $OVERLAY_IF"
else
# Create default route in overlay6 table to Internet via aaisp6
ip -6 route replace default via "$WAN_GATEWAY_IP" dev "$OVERLAY_IF" table "$OVERLAY6_TABLE"
logger -t hotplug "Added default route via $WAN_GATEWAY_IP dev $OVERLAY_IF table $OVERLAY6_TABLE"
# Tie fwmark to routing table with iproute2 rule
ip -6 rule add fwmark 0x20000000/0x30000000 table "$OVERLAY6_TABLE" priority 20098 2>/dev/null
logger -t hotplug "Added ip rule so traffic fwmarked wih 0x20000000/0x30000000 uses custom routing table $OVERLAY6_TABLE"
fi
logger -t hotplug "$OVERLAY6_TABLE routing setup complete"
;;
ifdown)
logger -t hotplug "Tearing down $OVERLAY6_TABLE routing in $INTERFACE ifdown hotplug script $SCRIPT"
logger -t hotplug "Removing $OVERLAY6_TABLE routing table rules"
# Remove the rule that directs marked traffic to the overlay table
ip -6 rule delete from all fwmark 0x20000000/0x30000000 lookup "$OVERLAY6_TABLE" 2>/dev/null
# Remove the default route in the overlay table
ip -6 route delete default table "$OVERLAY6_TABLE" 2>/dev/null
logger -t hotplug "Teardown of $OVERLAY6_TABLE routing in $INTERFACE ifdown hotplug script $SCRIPT complete"
;;
esac
;;
esac
cat /etc/init.d/create_overlay_nftable
#!/bin/sh /etc/rc.common
# The nftable snippet that OpenWRT fw4 includes has to execute cleanly. But it also
# has to flush the table or delete it during reload or restart operations.
# Unfortunately these both create errors during startup when no table exists, and
# stops execution of the snippet, which prevents the marking rule being created, and
# prevents the startup of the OpenWRT firewall, which would be a disaster.
#
# The answer to that is to create the table on startup, before OpenWRTs firewall
# manager (fw4) starts, so the snippet always encounters an existing table, and
# can contain a flush command and not fail. This init.d script does that nftables
# table create.
#
# This script goes in /etc/init.d/create_overlay_nftable
#
# It includes some helper functions and the standard initd framework from /etc/rc.common
# (see top of this file). Customisation is all below, but we need to hook this into the
# init.d startup/shutdown process. This is done by calling the "enable" function that
# is included into this script, which causes links to be make to this script from
# /etc/rc.d/ that determine the timing of its execution during startup and shutdown,
# though for this script, shutdown is a NOP.
#
# To do that run the command "/etc/init.d/create_overlay_nftable enable" to link
# it into the startup process.
#
# For these changes to survive an attended sysupgrade, we need to put that command into the
# script that runs once, at the end of a sysupgrade.
#
START=15 # Must be before fw4 (which defaults to START=19)
USE_PROCD=1
start_service() {
logger -t overlay "Checking whether to create nftable 'overlay' during startup"
if nft list table inet overlay >/dev/null 2>&1; then
logger -t overlay "... nftable 'overlay' already exists, doing nothing."
else
nft 'add table inet overlay'
logger -t overlay "... nftable 'overlay' added, so fw4 can flush it during its startup"
fi
}
cat /etc/firewall-overlay-marking.nft
# There cannot be any errors returned from the execution of this snippet as it
# is included into the startup of OpenWRT by its firewall manager, fw4.
#
# The flush command is absolutely required to ensure there is no buildup of rules
# over time & repeated reload / restart operations of the network and firewall.
# However, on first boot there will be no nftables, and the flush (or a delete)
# of a non-existant table will fail with an error, causing the creation of the rest
# of the table/chain/rules to fail, breaking my overlay networking setup.
#
# To resolve this, there is a small script in /etc/init.d that creates the nftable
# just before fw4 starts up, so this snippet will always see an existing table.
# See /etc/init.d/create_overlay_nftable for more information on how that is created.
flush table inet overlay
table inet overlay {
set aaisp_voip4 {
type ipv4_addr;
flags interval;
elements = {
81.187.30.110/31,
81.187.30.112/29,
90.155.3.0/24,
90.155.103.0/24
}
}
set local4 {
type ipv4_addr;
flags interval;
elements = {
192.168.1.0/24,
192.168.43.0/24,
192.168.100.0/29,
192.168.101.0/24,
192.168.155.0/24,
192.168.252.0/22
}
}
set internal4 {
type ipv4_addr;
flags interval;
elements = { 192.168.0.0/16 }
}
set local6_lan {
type ipv6_addr;
flags interval;
elements = { 2a0e:cb01:17b:c900::/64 }
}
set local6_server {
type ipv6_addr;
flags interval;
elements = { 2001:8b0:a4ee:4d01::/64 }
}
set internal6 {
type ipv6_addr;
flags interval;
elements = {
2a0e:cb01:17b:c900::/64,
fdb4::/64,
2001:8b0:a4ee:4d01::/64,
fdb4:0:0:1::/64
}
}
chain marking {
type filter hook prerouting priority -300;
policy accept;
meta nfproto ipv4 goto ipv4_marking
meta nfproto ipv6 goto ipv6_marking
}
chain ipv4_marking {
ip daddr @internal4 return
ip daddr @aaisp_voip4 return
ip saddr @local4 ip daddr != @internal4 meta mark set meta mark | 0x10000000
}
chain ipv6_marking {
ip6 daddr @internal6 return
ip6 saddr @local6_lan ip6 daddr != @internal6 meta mark set meta mark | 0x10000000
ip6 saddr @local6_server ip6 daddr != @internal6 meta mark set meta mark | 0x20000000
}
}
cat /etc/iproute2/rt_tables
#
# reserved values
#
128 prelocal
255 local
254 main
253 default
0 unspec
#
# local
#
#1 inr.ruhep
#
# Custom Routing tables
#
146 underlay6
148 overlay6
150 underlay
The route to the L2TP tunnel server at my "overlay" ISP
ip route list match 194.4.172.12 table main
default via 81.187.81.187 dev l2tp-aaisp proto static metric 200
default via 100.68.0.1 dev eth1 proto static src 100.69.136.20 metric 300
194.4.172.12 dev eth1 scope link
194.4.172.12 via 100.68.0.1 dev eth1 proto static metric 300
ip -4 rule show
0: from all lookup local
20100: from all fwmark 0x10000000/0x30000000 lookup underlay
32766: from all lookup main
32767: from all lookup default
ip -6 rule show
0: from all lookup local
20098: from all fwmark 0x20000000/0x30000000 lookup overlay6
20099: from all fwmark 0x10000000/0x30000000 lookup underlay6
32766: from all lookup main
4200000000: from 2a0e:cb01:17b:c900::1/64 iif br-lan.1 unreachable
4200000000: from 2001:8b0:a4ee:4d01::1/64 iif br-lan.10 unreachable
main table for IPv4:
ip -4 route show table main
default via 81.187.81.187 dev l2tp-aaisp proto static metric 200
default via 100.68.0.1 dev eth1 proto static src 100.69.136.20 metric 300
81.187.81.187 dev l2tp-aaisp proto kernel scope link src 81.187.73.113
100.68.0.0/15 dev eth1 proto static scope link metric 300
192.168.1.1 dev site2site proto static scope link
192.168.43.0/24 dev site2site proto static scope link
192.168.100.0/29 dev site2site proto kernel scope link src 192.168.100.1
192.168.100.2 dev site2site proto static scope link
192.168.101.0/24 dev mobiles proto kernel scope link src 192.168.101.1
192.168.251.0/24 dev br-lan.10 proto kernel scope link src 192.168.251.1
192.168.252.0/22 dev br-lan.1 proto kernel scope link src 192.168.252.1
194.4.172.12 dev eth1 scope link
194.4.172.12 via 100.68.0.1 dev eth1 proto static metric 300
IPv4 underlay table:
ip -4 route show table underlay
default via 100.68.0.1 dev eth1
IPv6 main table:
ip -6 route show table main
2001:8b0:a4ee:4d01::/64 dev br-lan.10 proto static metric 1024 pref medium
unreachable 2001:8b0:a4ee:4d00::/60 dev lo proto static metric 2147483647 pref medium
2a0e:cb01:17b:c900::/64 dev br-lan.1 proto static metric 1024 pref medium
unreachable 2a0e:cb01:17b:c900::/56 dev lo proto static metric 2147483647 pref medium
fdb4::/64 dev br-lan.1 proto static metric 1024 pref medium
fdb4:0:0:1::/64 dev br-lan.10 proto static metric 1024 pref medium
unreachable fdb4::/48 dev lo proto static metric 2147483647 pref medium
fe80::29d9:88b8:cd10:6f81 dev l2tp-aaisp proto kernel metric 256 pref medium
fe80::9e89:1eff:fe3f:0 dev l2tp-aaisp proto kernel metric 256 pref medium
fe80::/64 dev eth0 proto kernel metric 256 pref medium
fe80::/64 dev br-lan proto kernel metric 256 pref medium
fe80::/64 dev br-lan.1 proto kernel metric 256 pref medium
fe80::/64 dev br-lan.10 proto kernel metric 256 pref medium
fe80::/64 dev eth1 proto kernel metric 256 pref medium
default via fe80::9e89:1eff:fe3f:0 dev l2tp-aaisp proto static metric 512 pref medium
IPv6 underlay table:
ip -6 route show table underlay6
default via fe80::429e:a4ff:fe3b:d62b dev eth1 metric 1024 pref medium
IPv6 overlay table:
ip -6 route show table overlay6
default via fe80::9e89:1eff:fe3f:0 dev l2tp-aaisp metric 1024 pref medium
NFT overlay marking rules:
nft list chain inet overlay marking
table inet overlay {
chain marking { # handle 1
type filter hook prerouting priority raw; policy accept;
meta nfproto ipv4 goto ipv4_marking # handle 90
meta nfproto ipv6 goto ipv6_marking # handle 91
}
}
nft list chain inet overlay ipv4_marking
table inet overlay {
chain ipv4_marking {
ip daddr @internal4 return
ip daddr @aaisp_voip4 return
ip saddr @local4 ip daddr != @internal4 meta mark set meta mark | 0x10000000
}
}
nft list chain inet overlay ipv6_marking
table inet overlay {
chain ipv6_marking {
ip6 daddr @internal6 return
ip6 saddr @local6_lan ip6 daddr != @internal6 meta mark set meta mark | 0x10000000
ip6 saddr @local6_server ip6 daddr != @internal6 meta mark set meta mark | 0x20000000
}
}
