mirror of
https://github.com/outbackdingo/firezone.git
synced 2026-01-27 18:18:55 +00:00
90cf191a7c
## Context At present, we only have a single thread that reads and writes to the TUN device on all platforms. On Linux, it is possible to open the file descriptor of a TUN device multiple times by setting the `IFF_MULTI_QUEUE` option using `ioctl`. Using multi-queue, we can then spawn multiple threads that concurrently read and write to the TUN device. This is critical for achieving a better throughput. ## Solution `IFF_MULTI_QUEUE` is a Linux-only thing and therefore only applies to headless-client, GUI-client on Linux and the Gateway (it may also be possible on Android, I haven't tried). As such, we need to first change our internal abstractions a bit to move the creation of the TUN thread to the `Tun` abstraction itself. For this, we change the interface of `Tun` to the following: - `poll_recv_many`: An API, inspired by tokio's `mpsc::Receiver` where multiple items in a channel can be batch-received. - `poll_send_ready`: Mimics the API of `Sink` to check whether more items can be written. - `send`: Mimics the API of `Sink` to actually send an item. With these APIs in place, we can implement various (performance) improvements for the different platforms. - On Linux, this allows us to spawn multiple threads to read and write from the TUN device and send all packets into the same channel. The `Io` component of `connlib` then uses `poll_recv_many` to read batches of up to 100 packets at once. This ties in well with #7210 because we can then use GSO to send the encrypted packets in single syscalls to the OS. - On Windows, we already have a dedicated recv thread because `WinTun`'s most-convenient API uses blocking IO. As such, we can now also tie into that by batch-receiving from this channel. - In addition to using multiple threads, this API now also uses correct readiness checks on Linux, Darwin and Android to uphold backpressure in case we cannot write to the TUN device. ## Configuration Local testing has shown that 2 threads give the best performance for a local `iperf3` run. I suspect this is because there is only so much traffic that a single application (i.e. `iperf3`) can generate. With more than 2 threads, the throughput actually drops drastically because `connlib`'s main thread is too busy with lock-contention and triggering `Waker`s for the TUN threads (which mostly idle around if there are 4+ of them). I've made it configurable on the Gateway though so we can experiment with this during concurrent speedtests etc. In addition, switching `connlib` to a single-threaded tokio runtime further increased the throughput. I suspect due to less task / context switching. ## Results Local testing with `iperf3` shows some very promising results. We now achieve a throughput of 2+ Gbit/s. ``` Connecting to host 172.20.0.110, port 5201 Reverse mode, remote host 172.20.0.110 is sending [ 5] local 100.80.159.34 port 57040 connected to 172.20.0.110 port 5201 [ ID] Interval Transfer Bitrate [ 5] 0.00-1.00 sec 274 MBytes 2.30 Gbits/sec [ 5] 1.00-2.00 sec 279 MBytes 2.34 Gbits/sec [ 5] 2.00-3.00 sec 216 MBytes 1.82 Gbits/sec [ 5] 3.00-4.00 sec 224 MBytes 1.88 Gbits/sec [ 5] 4.00-5.00 sec 234 MBytes 1.96 Gbits/sec [ 5] 5.00-6.00 sec 238 MBytes 2.00 Gbits/sec [ 5] 6.00-7.00 sec 229 MBytes 1.92 Gbits/sec [ 5] 7.00-8.00 sec 222 MBytes 1.86 Gbits/sec [ 5] 8.00-9.00 sec 223 MBytes 1.87 Gbits/sec [ 5] 9.00-10.00 sec 217 MBytes 1.82 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-10.00 sec 2.30 GBytes 1.98 Gbits/sec 22247 sender [ 5] 0.00-10.00 sec 2.30 GBytes 1.98 Gbits/sec receiver iperf Done. ``` This is a pretty solid improvement over what is in `main`: ``` Connecting to host 172.20.0.110, port 5201 [ 5] local 100.65.159.3 port 56970 connected to 172.20.0.110 port 5201 [ ID] Interval Transfer Bitrate Retr Cwnd [ 5] 0.00-1.00 sec 90.4 MBytes 758 Mbits/sec 1800 106 KBytes [ 5] 1.00-2.00 sec 93.4 MBytes 783 Mbits/sec 1550 51.6 KBytes [ 5] 2.00-3.00 sec 92.6 MBytes 777 Mbits/sec 1350 76.8 KBytes [ 5] 3.00-4.00 sec 92.9 MBytes 779 Mbits/sec 1800 56.4 KBytes [ 5] 4.00-5.00 sec 93.4 MBytes 783 Mbits/sec 1650 69.6 KBytes [ 5] 5.00-6.00 sec 90.6 MBytes 760 Mbits/sec 1500 73.2 KBytes [ 5] 6.00-7.00 sec 87.6 MBytes 735 Mbits/sec 1400 76.8 KBytes [ 5] 7.00-8.00 sec 92.6 MBytes 777 Mbits/sec 1600 82.7 KBytes [ 5] 8.00-9.00 sec 91.1 MBytes 764 Mbits/sec 1500 70.8 KBytes [ 5] 9.00-10.00 sec 92.0 MBytes 771 Mbits/sec 1550 85.1 KBytes - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-10.00 sec 917 MBytes 769 Mbits/sec 15700 sender [ 5] 0.00-10.00 sec 916 MBytes 768 Mbits/sec receiver iperf Done. ```
74 lines
2.3 KiB
TOML
74 lines
2.3 KiB
TOML
[package]
|
|
name = "firezone-bin-shared"
|
|
version = "0.1.0"
|
|
edition = { workspace = true }
|
|
description = "Firezone-specific modules shared between binaries."
|
|
license = { workspace = true }
|
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
|
|
|
[dependencies]
|
|
anyhow = { workspace = true }
|
|
axum = { workspace = true, features = ["http1", "tokio"] }
|
|
clap = { workspace = true, features = ["derive", "env"] }
|
|
firezone-logging = { workspace = true }
|
|
futures = { workspace = true, features = ["std", "async-await"] }
|
|
hex-literal = { workspace = true }
|
|
ip-packet = { workspace = true }
|
|
ip_network = { workspace = true, features = ["serde"] }
|
|
socket-factory = { workspace = true }
|
|
thiserror = { workspace = true }
|
|
tokio = { workspace = true, features = ["io-util", "net", "rt", "sync"] }
|
|
tracing = { workspace = true }
|
|
tun = { workspace = true }
|
|
|
|
[dev-dependencies]
|
|
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
|
|
|
|
[target.'cfg(target_os = "linux")'.dependencies]
|
|
flume = { workspace = true }
|
|
libc = { workspace = true }
|
|
netlink-packet-core = { version = "0.7" }
|
|
netlink-packet-route = { version = "0.19" }
|
|
nix = { workspace = true, features = ["socket"] }
|
|
rtnetlink = { workspace = true }
|
|
zbus = "4.4" # Can't use `zbus`'s `tokio` feature here, or it will break toast popups all the way over in `gui-client`.
|
|
|
|
[target.'cfg(windows)'.dependencies]
|
|
known-folders = { workspace = true }
|
|
ring = "0.17"
|
|
uuid = { workspace = true, features = ["v4"] }
|
|
windows-core = "0.58.0"
|
|
windows-implement = "0.58.0"
|
|
wintun = "0.5.0"
|
|
winreg = { workspace = true }
|
|
|
|
[target.'cfg(windows)'.dependencies.windows]
|
|
workspace = true
|
|
features = [
|
|
# For implementing COM interfaces
|
|
"implement",
|
|
"Win32_Foundation",
|
|
# For listening for network change events
|
|
"Win32_Networking_NetworkListManager",
|
|
"Win32_NetworkManagement_IpHelper",
|
|
"Win32_NetworkManagement_Ndis",
|
|
"Win32_Networking_WinSock",
|
|
"Win32_Security",
|
|
# COM is needed to listen for network change events
|
|
"Win32_System_Com",
|
|
# Needed to listen for system DNS changes
|
|
"Win32_System_Registry",
|
|
"Win32_System_Threading",
|
|
]
|
|
|
|
[target.'cfg(windows)'.dev-dependencies]
|
|
ip-packet = { workspace = true }
|
|
tokio = { workspace = true, features = ["net", "time"] }
|
|
|
|
[lints]
|
|
workspace = true
|
|
|
|
[[bench]]
|
|
name = "tunnel"
|
|
harness = false
|