Good day friends, I have a problem with my chain:
appvm >sys dns > sys wireguard > sys vless > sys firewall.
I don’t understand well sing-box …
I tried to hide sys wg behind sys vless , But Vless works without a wg, but when I try to add a wg to the chain, everything breaks and turns into a loop. The problem is definitely with the Vless system.
my config sys vless:
/rw/config/sing-box/config.json
{
"log": {
"disabled": false,
"level": "warn",
"timestamp": true
},
"inbounds": [
{
"type": "tun",
"tag": "tun-in",
"interface_name": "tun0",
"address": [
"172.18.0.1/30"
],
"mtu": 1280,
"stack": "gvisor",
"auto_route": true,
"strict_route": true,
"sniff": false,
"route_exclude_address": [
"127.0.0.0/8",
"10.137.0.0/16",
"10.138.0.0/16",
"10.139.0.0/16",
"172.16.0.0/12",
"192.168.0.0/16",
"89.125.***.***/32"
]
}
],
"outbounds": [
{
"type": "vless",
"tag": "proxy",
"bind_interface": "eth0",
"server": "89.125.***.***",
"server_port": 443,
"uuid": "7***",
"flow": "xtls-rprx-vision",
"tls": {
"enabled": true,
"server_name": "www.cloudflare.com",
"utls": {
"enabled": true,
"fingerprint": "chrome"
},
"reality": {
"enabled": true,
"public_key": "0xu***",
"short_id": "a9***"
},
"alpn": [
"h2",
"http/1.1"
]
},
"packet_encoding": "xudp"
},
{
"type": "direct",
"tag": "direct"
}
],
"route": {
"auto_detect_interface": true,
"final": "proxy",
"rules": []
}
}
/rw/config/qubes-firewall-user-script
#!/bin/sh
set -eu
TUN_IF="tun0"
del_by_comment() {
chain="$1"; c="$2"
while :; do
h="$(nft -a list chain qubes "$chain" | awk -v C="$c" '$0 ~ C {print $NF; exit}')"
[ -z "$h" ] && break
nft delete rule qubes "$chain" handle "$h"
done
}
del_by_comment custom-forward "SYS_VLESS_FWD_BASE"
nft insert rule qubes custom-forward ct state invalid drop comment "SYS_VLESS_FWD_BASE"
nft insert rule qubes custom-forward ct state established,related accept comment "SYS_VLESS_FWD_BASE"
del_by_comment custom-forward "SYS_VLESS_KILL"
nft add rule qubes custom-forward iifname "vif*" oifname != "$TUN_IF" ct state new counter drop comment "SYS_VLESS_KILL"
nft add rule qubes custom-forward iifname "$TUN_IF" oifname "vif*" ct state new counter drop comment "SYS_VLESS_KILL"
nft add rule qubes custom-forward meta l4proto icmp icmp type destination-unreachable icmp code fragmentation-needed counter accept comment "SYS_VLESS_KILL"
/rw/config/rc.local
#!/bin/sh
rm -f /etc/systemd/system/sing-box.service
ln -s /rw/config/systemd/sing-box.service /etc/systemd/system/sing-box.service
systemctl daemon-reload
systemctl restart sing-box.service
/etc/systemd/system/sing-box.service
[Unit]
Description=sing-box
After=network-online.target
Wants=network-online.target
[Service]
User=sing-box
Group=sing-box
ExecStart=/usr/bin/sing-box run -c /rw/config/sing-box/config.json
Restart=always
RestartSec=3
NoNewPrivileges=yes
PrivateTmp=yes
ProtectSystem=strict
ProtectHome=yes
ProtectControlGroups=yes
ProtectKernelModules=yes
ProtectKernelTunables=yes
CapabilityBoundingSet=CAP_NET_ADMIN
AmbientCapabilities=CAP_NET_ADMIN
[Install]
WantedBy=multi-user.target
root@sys-vless6:~# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host noprefixroute
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group 1 qlen 1000
link/ether 00:16:3e:5e:6c:00 brd ff:ff:ff:ff:ff:ff
inet 10.137.0.38/32 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::216:3eff:fe5e:6c00/64 scope link proto kernel_ll
valid_lft forever preferred_lft forever
6: vif38.0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group 2 qlen 1000
link/ether fe:ff:ff:ff:ff:ff brd ff:ff:ff:ff:ff:ff
inet 10.137.0.38/32 scope global vif38.0
valid_lft forever preferred_lft forever
inet6 fe80::fcff:ffff:feff:ffff/64 scope link proto kernel_ll
valid_lft forever preferred_lft forever
8: tun0: <POINTOPOINT,MULTICAST,NOARP,UP,LOWER_UP> mtu 1280 qdisc fq_codel state UNKNOWN group default qlen 500
link/none
inet 172.18.0.1/30 brd 172.18.0.3 scope global tun0
valid_lft forever preferred_lft forever
inet6 fe80::b9f0:61e1:648c:82a0/64 scope link stable-privacy proto kernel_ll
valid_lft forever preferred_lft forever
root@sys-vless6:~# ip r
default via 10.138.15.240 dev eth0 onlink
10.137.0.39 dev vif38.0 scope link metric 32714
10.138.15.240 dev eth0 scope link
172.18.0.0/30 dev tun0 proto kernel scope link src 172.18.0.1
root@sys-vless6:~# ip rule
0: from all lookup local
9000: from all to 172.18.0.0/30 lookup 2022
9001: from all lookup 2022 suppress_prefixlength 0
9002: not from all dport 53 lookup main suppress_prefixlength 0
9002: from all iif tun0 goto 9010
9003: not from all iif lo lookup 2022
9003: from 0.0.0.0 iif lo lookup 2022
9003: from 172.18.0.0/30 iif lo lookup 2022
9010: from all nop
32766: from all lookup main
32767: from all lookup default
my config sys wg:
/rw/config/qubes-firewall-user-script-wg
#!/bin/sh
set -eu
WG_IF="wg0"
del_by_comment() {
chain="$1"; c="$2"
while :; do
h="$(nft -a list chain qubes "$chain" 2>/dev/null | awk -v C="$c" '$0 ~ C
{print $NF; exit}')"
[ -z "${h:-}" ] && break
nft delete rule qubes "$chain" handle "$h"
done
}
del_by_comment custom-forward "SYS_WG_KILL"
nft insert rule qubes custom-forward ct state invalid drop comment "SYS_WG_KILL"
nft insert rule qubes custom-forward ct state established,related accept
comment "SYS_WG_KILL"
nft add rule qubes custom-forward iifname "vif*" oifname "${WG_IF}" ct state
new counter accept comment "SYS_WG_KILL"
nft add rule qubes custom-forward iifname "vif*" oifname != "${WG_IF}" ct state
new counter drop comment "SYS_WG_KILL"
nft add rule qubes custom-forward iifname "${WG_IF}" oifname "vif*" ct state
new counter drop comment "SYS_WG_KILL"
nft add rule qubes custom-forward meta l4proto icmp icmp type destination-
unreachable icmp code fragmentation-needed counter accept comment "SYS_WG_KILL"
/rw/config/rc.local
#!/bin/sh
set -eu
WG_IF="wg0"
WAN_IF="eth0"
WG_ENDPOINT_IP="95.143.111.130"
WG_ENDPOINT_PORT="51820"
mkdir -p /etc/wireguard
ln -sf /rw/config/wireguard/wg0.conf /etc/wireguard/wg0.conf
wg-quick down "${WG_IF}" >/dev/null 2>&1 || true
wg-quick up "${WG_IF}"
GW="$(qubesdb-read /qubes-gateway || true)"
[ -n "${GW:-}" ] || exit 1
ip -4 route replace "${WG_ENDPOINT_IP}/32" via "${GW}" dev "${WAN_IF}" onlink
ip -4 route replace default dev "${WG_IF}"
/rw/config/wireguard/wg0.conf
[Interface]
Address = 10.10.0.4/32
PrivateKey = mOJ02***
MTU = 1280
Table = off
[Peer]
PublicKey = lNFdVnrK***
PresharedKey = GAtWd***
Endpoint = 95.143.***.***:51820
AllowedIPs = 0.0.0.0/0
PersistentKeepalive = 25
root@sys-wg2:~# ip r
default dev wg0 scope link
10.137.0.10 dev vif11.0 scope link metric 32741
10.137.0.38 dev eth0 scope link
95.143.***.*** via 10.137.0.38 dev eth0 onlink
root@sys-wg2:~# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host noprefixroute
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group 1 qlen 1000
link/ether 00:16:3e:5e:6c:00 brd ff:ff:ff:ff:ff:ff
inet 10.137.0.39/32 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::216:3eff:fe5e:6c00/64 scope link proto kernel_ll
valid_lft forever preferred_lft forever
3: vif11.0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group 2 qlen 1000
link/ether fe:ff:ff:ff:ff:ff brd ff:ff:ff:ff:ff:ff
inet 10.137.0.39/32 scope global vif11.0
valid_lft forever preferred_lft forever
inet6 fe80::fcff:ffff:feff:ffff/64 scope link proto kernel_ll
valid_lft forever preferred_lft forever
4: wg0: <POINTOPOINT,NOARP,UP,LOWER_UP> mtu 1280 qdisc noqueue state UNKNOWN group default qlen 1000
link/none
inet 10.10.0.4/32 scope global wg0
valid_lft forever preferred_lft forever
root@sys-wg2:~#
I configured sysctl -w net.ipv4.ip_forward=1 on the server and NAT
server for sys wg
cat /etc/nftables.conf
flush ruleset
table inet filter {
chain input {
type filter hook input priority 0; policy drop;
iif "lo" accept
ct state invalid drop
ct state established,related accept
ip protocol icmp accept
ip6 nexthdr icmpv6 accept
tcp dport 22 ct state new accept
tcp dport 443 ct state new accept
udp dport 51820 ct state new accept
}
chain forward {
type filter hook forward priority 0; policy drop;
ct state invalid drop
ct state established,related accept
iif "wg0" oif "wg0" drop
iif "wg0" oif "eth0" accept
}
chain output {
type filter hook output priority 0; policy accept;
}
}
table inet nat {
chain postrouting {
type nat hook postrouting priority 100; policy accept;
oif "eth0" ip saddr 10.10.0.0/24 masquerade
}
}
cat /etc/wireguard/wg0.conf
[Interface]
Address = 10.10.0.1/24
ListenPort = 51820
PrivateKey = uGsvf/R/yr9***
[Peer]
PublicKey = YSt70jg***
AllowedIPs = 10.10.0.4/32
PresharedKey = GAtWdV***
server for sys vless
cat /etc/sing-box/config.json
{
"log": {
"level": "info",
"timestamp": true
},
"inbounds": [
{
"type": "vless",
"tag": "vless-in",
"listen": "0.0.0.0",
"listen_port": 443,
"users": [
{
"uuid": "75c4***",
"flow": "xtls-rprx-vision"
}
],
"tls": {
"enabled": true,
"server_name": "www.cloudflare.com",
"reality": {
"enabled": true,
"handshake": {
"server": "www.cloudflare.com",
"server_port": 443
},
"private_key": "WHCFZ***",
"short_id": [
"a9f8***"
]
}
}
}
],
"outbounds": [
{
"type": "direct",
"tag": "direct"
}
]
}
no nft rules
sudo sysctl net.ipv4.ip_forward
net.ipv4.ip_forward = 0
I just changed it to net.ipv4.ip_forward = 1
but the problem isn’t solved.
Also, tell me how this problem will be solved. In order for there to be a network in AppVM, will I also need to do a masquerade on the sys wg system?