source.openwrt.melmac.net
source.openwrt.melmac.net copied to clipboard
stangri
[PBR] Issue: Database /etc/iproute2/rt_tables is corrupted
OpenWrt 22.03.2 iproute2 5.15.0 PBR 0.9.9-32 (nft)
I have a nested VPN setup. WireGuard client connects through another OpenConnect tunnel to reach the internet.
After running pbr service, I have to restart the WireGuard interface manually because it initially tries to connect through the WAN interface (pbr service hasn't started just yet) and after pbr sets the policies, WireGuard still can't complete the handshake (this protocol is blocked in my region). Therefor I have to restart it manually for the traffic to go from the second tunnel. But after each restart, pbr service also restarts and messed up the ip rules. I have Database /etc/iproute2/rt_tables is corrupted in my logs. Deleting this file and restarting the service makes no difference.
config
soberia@XMR3G:~$ sudo cat /etc/config/pbr
config pbr 'config'
option enabled '1'
option verbosity '2'
option strict_enforcement '1'
option resolver_set 'none'
option ipv6_enabled '0'
option boot_timeout '30'
option rule_create_option 'add'
option procd_reload_delay '1'
option webui_show_ignore_target '1'
list webui_supported_protocol 'all'
list webui_supported_protocol 'tcp'
list webui_supported_protocol 'udp'
list webui_supported_protocol 'tcp udp'
list webui_supported_protocol 'icmp'
list supported_interface 'openvpn'
config include
option path '/usr/share/pbr/pbr.user.aws'
config include
option path '/usr/share/pbr/pbr.user.netflix'
config policy
option dest_addr '10.0.0.0/8'
option interface 'ignore'
option name 'ignore-local'
config policy
option name 'vpn_lan'
option src_addr '10.0.2.0/24'
option interface 'wg_client'
config policy
option name 'wg_server'
option src_addr '10.0.4.0/24'
option interface 'wg_client'
config policy
option name 'vpn_redirect'
option proto 'udp'
option chain 'output'
option interface 'openconnect'
option dest_port '59096 7468 51820'
service pbr status
soberia@XMR3G:~$ sudo service pbr status
============================================================
pbr - environment
pbr 0.9.9-32 running on OpenWrt 22.03.2. WAN (IPv4): wan/pppoe-wan/172.20.0.32.
============================================================
Dnsmasq version 2.86 Copyright (c) 2000-2021 Simon Kelley
Compile time options: IPv6 GNU-getopt no-DBus UBus no-i18n no-IDN DHCP no-DHCPv6 no-Lua TFTP no-conntrack no-ipset no-auth no-cryptohash no-DNSSEC no-ID loop-detect inotify dumpfile
============================================================
pbr chains - policies
chain pbr_forward {
}
chain pbr_input {
}
chain pbr_output {
udp dport { 7468, 51820, 59096 } goto pbr_mark_0x070000 comment "vpn_redirect"
}
chain pbr_prerouting {
ip daddr @pbr_wan_4_dst_ip_user goto pbr_mark_0x010000
ip saddr @pbr_wan_4_src_ip_user goto pbr_mark_0x010000
ether saddr @pbr_wan_4_src_mac_user goto pbr_mark_0x010000
ip daddr @pbr_wg_server_4_dst_ip_user goto pbr_mark_0x030000
ip saddr @pbr_wg_server_4_src_ip_user goto pbr_mark_0x030000
ether saddr @pbr_wg_server_4_src_mac_user goto pbr_mark_0x030000
ip daddr @pbr_wg_client_2_4_dst_ip_user goto pbr_mark_0x040000
ip saddr @pbr_wg_client_2_4_src_ip_user goto pbr_mark_0x040000
ether saddr @pbr_wg_client_2_4_src_mac_user goto pbr_mark_0x040000
ip daddr @pbr_wg_client_3_4_dst_ip_user goto pbr_mark_0x050000
ip saddr @pbr_wg_client_3_4_src_ip_user goto pbr_mark_0x050000
ether saddr @pbr_wg_client_3_4_src_mac_user goto pbr_mark_0x050000
ip daddr @pbr_openvpn_4_dst_ip_user goto pbr_mark_0x060000
ip saddr @pbr_openvpn_4_src_ip_user goto pbr_mark_0x060000
ether saddr @pbr_openvpn_4_src_mac_user goto pbr_mark_0x060000
ip daddr @pbr_openconnect_4_dst_ip_user goto pbr_mark_0x070000
ip saddr @pbr_openconnect_4_src_ip_user goto pbr_mark_0x070000
ether saddr @pbr_openconnect_4_src_mac_user goto pbr_mark_0x070000
ip daddr @pbr_pptp_4_dst_ip_user goto pbr_mark_0x080000
ip saddr @pbr_pptp_4_src_ip_user goto pbr_mark_0x080000
ether saddr @pbr_pptp_4_src_mac_user goto pbr_mark_0x080000
ip daddr @pbr_ignore_4_dst_ip_cfg046ff5 return comment "ignore-local"
ip saddr @pbr_wg_client_4_src_ip_cfg056ff5 goto pbr_mark_0x020000 comment "vpn_lan"
ip saddr @pbr_wg_client_4_src_ip_cfg066ff5 goto pbr_mark_0x020000 comment "wg_server"
}
chain pbr_postrouting {
}
============================================================
pbr chains - marking
chain pbr_mark_0x010000 {
counter packets 0 bytes 0 meta mark set meta mark & 0xff01ffff | 0x00010000
return
}
chain pbr_mark_0x020000 {
counter packets 2249 bytes 433764 meta mark set meta mark & 0xff02ffff | 0x00020000
return
}
chain pbr_mark_0x030000 {
counter packets 0 bytes 0 meta mark set meta mark & 0xff03ffff | 0x00030000
return
}
chain pbr_mark_0x040000 {
counter packets 0 bytes 0 meta mark set meta mark & 0xff04ffff | 0x00040000
return
}
chain pbr_mark_0x050000 {
counter packets 0 bytes 0 meta mark set meta mark & 0xff05ffff | 0x00050000
return
}
chain pbr_mark_0x060000 {
counter packets 0 bytes 0 meta mark set meta mark & 0xff06ffff | 0x00060000
return
}
chain pbr_mark_0x070000 {
counter packets 59 bytes 10384 meta mark set meta mark & 0xff07ffff | 0x00070000
return
}
chain pbr_mark_0x080000 {
counter packets 0 bytes 0 meta mark set meta mark & 0xff08ffff | 0x00080000
return
}
============================================================
pbr nft sets
set pbr_wan_4_dst_ip_user {
type ipv4_addr
policy memory
flags interval
auto-merge
comment ""
}
set pbr_wan_4_src_ip_user {
type ipv4_addr
policy memory
flags interval
auto-merge
comment ""
}
set pbr_wan_4_src_mac_user {
type ether_addr
policy memory
flags interval
auto-merge
comment ""
}
set pbr_wg_server_4_dst_ip_user {
type ipv4_addr
policy memory
flags interval
auto-merge
comment ""
}
set pbr_wg_server_4_src_ip_user {
type ipv4_addr
policy memory
flags interval
auto-merge
comment ""
}
set pbr_wg_server_4_src_mac_user {
type ether_addr
policy memory
flags interval
auto-merge
comment ""
}
set pbr_wg_client_2_4_dst_ip_user {
type ipv4_addr
policy memory
flags interval
auto-merge
comment ""
}
set pbr_wg_client_2_4_src_ip_user {
type ipv4_addr
policy memory
flags interval
auto-merge
comment ""
}
set pbr_wg_client_2_4_src_mac_user {
type ether_addr
policy memory
flags interval
auto-merge
comment ""
}
set pbr_wg_client_3_4_dst_ip_user {
type ipv4_addr
policy memory
flags interval
auto-merge
comment ""
}
set pbr_wg_client_3_4_src_ip_user {
type ipv4_addr
policy memory
flags interval
auto-merge
comment ""
}
set pbr_wg_client_3_4_src_mac_user {
type ether_addr
policy memory
flags interval
auto-merge
comment ""
}
set pbr_openvpn_4_dst_ip_user {
type ipv4_addr
policy memory
flags interval
auto-merge
comment ""
}
set pbr_openvpn_4_src_ip_user {
type ipv4_addr
policy memory
flags interval
auto-merge
comment ""
}
set pbr_openvpn_4_src_mac_user {
type ether_addr
policy memory
flags interval
auto-merge
comment ""
}
set pbr_openconnect_4_dst_ip_user {
type ipv4_addr
policy memory
flags interval
auto-merge
comment ""
}
set pbr_openconnect_4_src_ip_user {
type ipv4_addr
policy memory
flags interval
auto-merge
comment ""
}
set pbr_openconnect_4_src_mac_user {
type ether_addr
policy memory
flags interval
auto-merge
comment ""
}
set pbr_pptp_4_dst_ip_user {
type ipv4_addr
policy memory
flags interval
auto-merge
comment ""
}
set pbr_pptp_4_src_ip_user {
type ipv4_addr
policy memory
flags interval
auto-merge
comment ""
}
set pbr_pptp_4_src_mac_user {
type ether_addr
policy memory
flags interval
auto-merge
comment ""
}
set pbr_ignore_4_dst_ip_cfg046ff5 {
type ipv4_addr
flags interval
auto-merge
comment "ignore-local"
elements = { 10.0.0.0/8 }
}
set pbr_wg_client_4_src_ip_cfg056ff5 {
type ipv4_addr
flags interval
auto-merge
comment "vpn_lan"
elements = { 10.0.2.0/24 }
}
set pbr_wg_client_4_src_ip_cfg066ff5 {
type ipv4_addr
flags interval
auto-merge
comment "wg_server"
elements = { 10.0.4.0/24 }
}
============================================================
Database /etc/iproute2/rt_tables is corrupted at 28
IPv4 table 43 route: unreachable default
Database /etc/iproute2/rt_tables is corrupted at 28
IPv4 table 43 rule: 29997: from all fwmark 0x40000/0xff0000 lookup 43
Database /etc/iproute2/rt_tables is corrupted at 28
IPv4 table 44 route: unreachable default
Database /etc/iproute2/rt_tables is corrupted at 28
IPv4 table 44 rule: 29996: from all fwmark 0x50000/0xff0000 lookup 44
Database /etc/iproute2/rt_tables is corrupted at 28
IPv4 table 45 route: unreachable default
Database /etc/iproute2/rt_tables is corrupted at 28
IPv4 table 45 rule: 29997: from all fwmark 0x40000/0xff0000 lookup 45
Database /etc/iproute2/rt_tables is corrupted at 28
IPv4 table 46 route: unreachable default
Database /etc/iproute2/rt_tables is corrupted at 28
IPv4 table 46 rule: 29996: from all fwmark 0x50000/0xff0000 lookup 46
Database /etc/iproute2/rt_tables is corrupted at 28
IPv4 table 47 route: unreachable default
Database /etc/iproute2/rt_tables is corrupted at 28
IPv4 table 47 rule: 29997: from all fwmark 0x40000/0xff0000 lookup 47
Database /etc/iproute2/rt_tables is corrupted at 28
IPv4 table 48 route: unreachable default
Database /etc/iproute2/rt_tables is corrupted at 28
IPv4 table 48 rule: 29996: from all fwmark 0x50000/0xff0000 lookup 48
Database /etc/iproute2/rt_tables is corrupted at 28
IPv4 table 49 route: unreachable default
Database /etc/iproute2/rt_tables is corrupted at 28
IPv4 table 49 rule: 29997: from all fwmark 0x40000/0xff0000 lookup 49
Database /etc/iproute2/rt_tables is corrupted at 28
IPv4 table 50 route: unreachable default
Database /etc/iproute2/rt_tables is corrupted at 28
IPv4 table 50 rule: 29996: from all fwmark 0x50000/0xff0000 lookup 50
/usr/libexec/ip-full rule
soberia@XMR3G:~$ /usr/libexec/ip-full rule
Database /etc/iproute2/rt_tables is corrupted at 28
0: from all lookup local
29993: from all fwmark 0x80000/0xff0000 lookup 8
29994: from all fwmark 0x70000/0xff0000 lookup 7
29995: from all fwmark 0x60000/0xff0000 lookup 6
29996: from all fwmark 0x50000/0xff0000 lookup 32
29996: from all fwmark 0x50000/0xff0000 lookup 34
29996: from all fwmark 0x50000/0xff0000 lookup 36
29996: from all fwmark 0x50000/0xff0000 lookup 38
29996: from all fwmark 0x50000/0xff0000 lookup 40
29996: from all fwmark 0x50000/0xff0000 lookup 42
29996: from all fwmark 0x50000/0xff0000 lookup 44
29996: from all fwmark 0x50000/0xff0000 lookup 46
29996: from all fwmark 0x50000/0xff0000 lookup 48
29996: from all fwmark 0x50000/0xff0000 lookup 50
29997: from all fwmark 0x40000/0xff0000 lookup 31
29997: from all fwmark 0x40000/0xff0000 lookup 33
29997: from all fwmark 0x40000/0xff0000 lookup 35
29997: from all fwmark 0x40000/0xff0000 lookup 37
29997: from all fwmark 0x40000/0xff0000 lookup 39
29997: from all fwmark 0x40000/0xff0000 lookup 41
29997: from all fwmark 0x40000/0xff0000 lookup 43
29997: from all fwmark 0x40000/0xff0000 lookup 45
29997: from all fwmark 0x40000/0xff0000 lookup 47
29997: from all fwmark 0x40000/0xff0000 lookup 49
29998: from all fwmark 0x30000/0xff0000 lookup 3
30000: from all fwmark 0x10000/0xff0000 lookup 1
32766: from all lookup main
32767: from all lookup default
cat /etc/iproute2/rt_tables
soberia@XMR3G:~$ cat /etc/iproute2/rt_tables
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
1 pbr_wan
46
47
48 pbr_wg_client
3 pbr_wg_server
49 pbr_wg_client_2
50 pbr_wg_client_3
6 pbr_openvpn
7 pbr_openconnect
8 pbr_pptp
It looks like /etc/iproute2/rt_tables was molested before pbr has started modifying it. pbr can't fix what was broken before.
I had three different interfaces with wg_client in their names. ( wg_client, wg_client_1, wg_client_2)
After temporarily deleting the last two, table ids correctly mapped to the interface names in /etc/iproute2/rt_tables.
I'm not know much about the source, but I think regex-based tools like sed or grep should be leveraged more precisely when dealing with names. Like in get_rt_tables_id() and interface_routing().
@Soberia I believe I've addressed it in 1.0.1-12. The PR into official repo is still pending, you can install the package or build it yourself from my packages or source repos.