refactor(connlib): move Tun implementations out of firezone-tunnel (#5903)

The different implementations of `Tun` are the last platform-specific
code within `firezone-tunnel`. By introducing a dedicated crate and a
`Tun` trait, we can move this code into (platform-specific) leaf crates:

- `connlib-client-android`
- `connlib-client-apple`
- `firezone-bin-shared`

Related: #4473.

---------

Co-authored-by: Not Applicable <ReactorScram@users.noreply.github.com>

.github/actions/setup-rust/action.yml

@@ -22,13 +22,13 @@ outputs:
     value: ${{
       (runner.os == 'Linux' && '--help') ||
       (runner.os == 'macOS' && '--help') ||
-      (runner.os == 'Windows' && '-p firezone-tunnel') }}
+      (runner.os == 'Windows' && '-p firezone-bin-shared') }}
   packages:
     description: Compilable / testable packages for the current OS
     value: ${{
       (runner.os == 'Linux' && '--workspace') ||
       (runner.os == 'macOS' && '-p connlib-client-apple -p connlib-client-shared -p firezone-tunnel -p snownet') ||
-      (runner.os == 'Windows' && '-p connlib-client-shared -p firezone-headless-client -p firezone-gui-client -p firezone-tunnel -p gui-smoke-test -p snownet') }}
+      (runner.os == 'Windows' && '-p connlib-client-shared -p firezone-headless-client -p firezone-gui-client -p firezone-tunnel -p gui-smoke-test -p snownet -p firezone-bin-shared') }}
 
 runs:
   using: "composite"

rust/Cargo.lock

@@ -1041,6 +1041,7 @@ dependencies = [
  "connlib-shared",
  "ip_network",
  "jni 0.21.1",
+ "libc",
  "log",
  "phoenix-channel",
  "secrecy",
@@ -1051,6 +1052,7 @@ dependencies = [
  "tracing",
  "tracing-appender",
  "tracing-subscriber",
+ "tun",
  "url",
 ]
 
@@ -1075,6 +1077,7 @@ dependencies = [
  "tracing",
  "tracing-appender",
  "tracing-subscriber",
+ "tun",
  "url",
 ]
 
@@ -1102,6 +1105,7 @@ dependencies = [
  "tracing-appender",
  "tracing-stackdriver",
  "tracing-subscriber",
+ "tun",
  "url",
 ]
 
@@ -1828,9 +1832,10 @@ dependencies = [
  "anyhow",
  "clap",
  "connlib-shared",
- "firezone-tunnel",
  "futures",
+ "ip-packet",
  "ip_network",
+ "libc",
  "netlink-packet-core",
  "netlink-packet-route",
  "rtnetlink",
@@ -1838,8 +1843,11 @@ dependencies = [
  "tracing",
  "tracing-log",
  "tracing-subscriber",
+ "tun",
  "url",
+ "uuid",
  "windows 0.57.0",
+ "wintun",
 ]
 
 [[package]]
@@ -2023,7 +2031,6 @@ name = "firezone-tunnel"
 version = "0.1.0"
 dependencies = [
  "anyhow",
- "async-trait",
  "bimap",
  "boringtun",
  "bytes",
@@ -2031,7 +2038,6 @@ dependencies = [
  "connlib-shared",
  "derivative",
  "domain",
- "firezone-bin-shared",
  "firezone-relay",
  "futures",
  "futures-bounded",
@@ -2043,7 +2049,6 @@ dependencies = [
  "ip_network",
  "ip_network_table",
  "itertools 0.13.0",
- "libc",
  "proptest",
  "proptest-state-machine",
  "quinn-udp",
@@ -2061,9 +2066,7 @@ dependencies = [
  "tokio",
  "tracing",
  "tracing-subscriber",
- "uuid",
- "windows 0.57.0",
- "wintun",
+ "tun",
 ]
 
 [[package]]
@@ -6828,6 +6831,14 @@ version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
 
+[[package]]
+name = "tun"
+version = "0.1.0"
+dependencies = [
+ "libc",
+ "tokio",
+]
+
 [[package]]
 name = "tungstenite"
 version = "0.21.0"

rust/Cargo.toml

@@ -18,6 +18,7 @@ members = [
   "relay",
   "snownet-tests",
   "socket-factory",
+  "tun"
 ]
 
 resolver = "2"
@@ -55,6 +56,7 @@ phoenix-channel = { path = "phoenix-channel" }
 http-health-check = { path = "http-health-check" }
 ip-packet = { path = "ip-packet" }
 socket-factory = { path = "socket-factory" }
+tun = { path = "tun" }
 socket2 = { version = "0.5" }
 
 [workspace.lints.clippy]

rust/bin-shared/Cargo.toml

@@ -9,19 +9,28 @@ description = "Firezone-specific modules shared between binaries."
 anyhow = "1.0.82"
 clap = { version = "4.5", features = ["derive",  "env"] }
 connlib-shared = { workspace = true }
-firezone-tunnel = { workspace = true }
 futures = "0.3"
+ip-packet = { workspace = true }
 ip_network = { version = "0.4", default-features = false, features = ["serde"] }
-tokio = { workspace = true, features = ["rt"] }
+tokio = { workspace = true, features = ["rt", "sync"] }
 tracing = { workspace = true }
 tracing-log = "0.2"
 tracing-subscriber = { workspace = true, features = ["env-filter"] }
+tun = { workspace = true }
 url = { version = "2.3.1", default-features = false }
 
+[dev-dependencies]
+tokio = { workspace = true, features = ["macros"] }
+
 [target.'cfg(target_os = "linux")'.dependencies]
 netlink-packet-core = { version = "0.7", default-features = false }
 netlink-packet-route = { version = "0.19", default-features = false }
 rtnetlink = { workspace = true }
+libc = "0.2"
+
+[target.'cfg(target_os = "windows")'.dependencies]
+wintun = "0.4.0"
+uuid = { version = "1.7.0", features = ["v4"] }
 
 [target.'cfg(windows)'.dependencies.windows]
 version = "0.57.0"
@@ -34,3 +43,7 @@ features = [
 
 [lints]
 workspace = true
+
+[[bench]]
+name = "tunnel"
+harness = false

rust/bin-shared/benches/tunnel.rs

@@ -37,6 +37,7 @@ mod platform {
         net::UdpSocket,
         time::{timeout, Instant},
     };
+    use tun::Tun as _;
 
     pub(crate) async fn perf() -> Result<()> {
         // Install wintun so the test can run

rust/bin-shared/src/tun_device_manager.rs

@@ -12,3 +12,50 @@ pub use windows as platform;
 
 #[cfg(any(target_os = "linux", target_os = "windows"))]
 pub use platform::TunDeviceManager;
+
+#[cfg(test)]
+#[cfg(any(target_os = "linux", target_os = "windows"))]
+mod tests {
+    use super::*;
+    use tracing_subscriber::EnvFilter;
+
+    #[tokio::test]
+    #[ignore = "Needs admin / sudo"]
+    async fn tunnel() {
+        let _ = tracing_subscriber::fmt()
+            .with_env_filter(EnvFilter::from_default_env())
+            .with_test_writer()
+            .try_init();
+
+        #[cfg(target_os = "windows")]
+        {
+            // Install wintun so the test can run
+            let wintun_path = connlib_shared::windows::wintun_dll_path().unwrap();
+            tokio::fs::create_dir_all(wintun_path.parent().unwrap())
+                .await
+                .unwrap();
+            tokio::fs::write(&wintun_path, connlib_shared::windows::wintun_bytes())
+                .await
+                .unwrap();
+        }
+
+        // Run these tests in series since they would fight over the tunnel interface
+        // if they ran concurrently
+        create_tun();
+        tunnel_drop();
+    }
+
+    fn create_tun() {
+        let mut tun_device_manager = TunDeviceManager::new().unwrap();
+        let _tun = tun_device_manager.make_tun().unwrap();
+    }
+
+    /// Checks for regressions in issue #4765, un-initializing Wintun
+    /// Redundant but harmless on Linux.
+    fn tunnel_drop() {
+        // Each cycle takes about half a second, so this will take a fair bit to run.
+        for _ in 0..50 {
+            let _tun = platform::Tun::new().unwrap(); // This will panic if we don't correctly clean-up the wintun interface.
+        }
+    }
+}

rust/bin-shared/src/tun_device_manager/linux.rs

@@ -1,19 +1,37 @@
 //! Virtual network interface
 
+use crate::FIREZONE_MARK;
 use anyhow::{anyhow, Context as _, Result};
 use connlib_shared::DEFAULT_MTU;
-use firezone_tunnel::Tun;
 use futures::TryStreamExt;
 use ip_network::{IpNetwork, Ipv4Network, Ipv6Network};
+use libc::{close, fcntl, makedev, mknod, open, F_GETFL, F_SETFL, O_NONBLOCK, O_RDWR, S_IFCHR};
 use netlink_packet_route::route::{RouteProtocol, RouteScope};
 use netlink_packet_route::rule::RuleAction;
 use rtnetlink::{new_connection, Error::NetlinkError, Handle, RouteAddRequest, RuleAddRequest};
+use std::path::Path;
+use std::task::{Context, Poll};
 use std::{
     collections::HashSet,
     net::{Ipv4Addr, Ipv6Addr},
 };
+use std::{
+    ffi::CStr,
+    fs, io,
+    os::{
+        fd::{AsRawFd, RawFd},
+        unix::fs::PermissionsExt,
+    },
+};
+use tokio::io::unix::AsyncFd;
+use tun::ioctl;
 
-use crate::FIREZONE_MARK;
+const TUNSETIFF: libc::c_ulong = 0x4004_54ca;
+const TUN_DEV_MAJOR: u32 = 10;
+const TUN_DEV_MINOR: u32 = 200;
+
+// Safety: We know that this is a valid C string.
+const TUN_FILE: &CStr = unsafe { CStr::from_bytes_with_nul_unchecked(b"/dev/net/tun\0") };
 
 const FILE_ALREADY_EXISTS: i32 = -17;
 const FIREZONE_TABLE: u32 = 0x2021_fd00;
@@ -36,7 +54,7 @@ impl Drop for TunDeviceManager {
 }
 
 impl TunDeviceManager {
-    pub const IFACE_NAME: &'static str = "tun-firezone"; // Keep this synced with `Tun` until we fix the module dependencies (i.e. move `Tun` out of `firezone-tunnel`).
+    pub const IFACE_NAME: &'static str = "tun-firezone";
 
     /// Creates a new managed tunnel device.
     ///
@@ -87,7 +105,7 @@ impl TunDeviceManager {
         handle
             .link()
             .set(index)
-            .mtu(DEFAULT_MTU)
+            .mtu(DEFAULT_MTU as u32)
             .execute()
             .await
             .context("Failed to set default MTU")?;
@@ -257,3 +275,125 @@ async fn remove_route(route: &IpNetwork, idx: u32, handle: &Handle) -> Result<()
         .context("Failed to delete route")?;
     Ok(())
 }
+
+#[derive(Debug)]
+pub struct Tun {
+    fd: AsyncFd<RawFd>,
+}
+
+impl Tun {
+    pub fn new() -> io::Result<Self> {
+        create_tun_device()?;
+
+        let fd = match unsafe { open(TUN_FILE.as_ptr() as _, O_RDWR) } {
+            -1 => return Err(get_last_error()),
+            fd => fd,
+        };
+
+        // Safety: We just opened the file descriptor.
+        unsafe {
+            ioctl::exec(
+                fd,
+                TUNSETIFF,
+                &mut ioctl::Request::<ioctl::SetTunFlagsPayload>::new(TunDeviceManager::IFACE_NAME),
+            )?;
+        }
+
+        set_non_blocking(fd)?;
+
+        // Safety: We just opened the fd.
+        unsafe { Self::from_fd(fd) }
+    }
+
+    /// Create a new [`Tun`] from a raw file descriptor.
+    ///
+    /// # Safety
+    ///
+    /// The file descriptor must be open.
+    unsafe fn from_fd(fd: RawFd) -> io::Result<Self> {
+        Ok(Tun {
+            fd: AsyncFd::new(fd)?,
+        })
+    }
+}
+
+impl Drop for Tun {
+    fn drop(&mut self) {
+        unsafe { close(self.fd.as_raw_fd()) };
+    }
+}
+
+impl tun::Tun for Tun {
+    fn write4(&self, buf: &[u8]) -> io::Result<usize> {
+        write(self.fd.as_raw_fd(), buf)
+    }
+
+    fn write6(&self, buf: &[u8]) -> io::Result<usize> {
+        write(self.fd.as_raw_fd(), buf)
+    }
+
+    fn poll_read(&mut self, buf: &mut [u8], cx: &mut Context<'_>) -> Poll<io::Result<usize>> {
+        tun::unix::poll_raw_fd(&self.fd, |fd| read(fd, buf), cx)
+    }
+
+    fn name(&self) -> &str {
+        TunDeviceManager::IFACE_NAME
+    }
+}
+
+fn get_last_error() -> io::Error {
+    io::Error::last_os_error()
+}
+
+fn set_non_blocking(fd: RawFd) -> io::Result<()> {
+    match unsafe { fcntl(fd, F_GETFL) } {
+        -1 => Err(get_last_error()),
+        flags => match unsafe { fcntl(fd, F_SETFL, flags | O_NONBLOCK) } {
+            -1 => Err(get_last_error()),
+            _ => Ok(()),
+        },
+    }
+}
+
+fn create_tun_device() -> io::Result<()> {
+    let path = Path::new(TUN_FILE.to_str().expect("path is valid utf-8"));
+
+    if path.exists() {
+        return Ok(());
+    }
+
+    let parent_dir = path.parent().unwrap();
+    fs::create_dir_all(parent_dir)?;
+    let permissions = fs::Permissions::from_mode(0o751);
+    fs::set_permissions(parent_dir, permissions)?;
+    if unsafe {
+        mknod(
+            TUN_FILE.as_ptr() as _,
+            S_IFCHR,
+            makedev(TUN_DEV_MAJOR, TUN_DEV_MINOR),
+        )
+    } != 0
+    {
+        return Err(get_last_error());
+    }
+
+    Ok(())
+}
+
+/// Read from the given file descriptor in the buffer.
+fn read(fd: RawFd, dst: &mut [u8]) -> io::Result<usize> {
+    // Safety: Within this module, the file descriptor is always valid.
+    match unsafe { libc::read(fd, dst.as_mut_ptr() as _, dst.len()) } {
+        -1 => Err(io::Error::last_os_error()),
+        n => Ok(n as usize),
+    }
+}
+
+/// Write the buffer to the given file descriptor.
+fn write(fd: RawFd, buf: &[u8]) -> io::Result<usize> {
+    // Safety: Within this module, the file descriptor is always valid.
+    match unsafe { libc::write(fd, buf.as_ptr() as _, buf.len() as _) } {
+        -1 => Err(io::Error::last_os_error()),
+        n => Ok(n as usize),
+    }
+}

rust/bin-shared/src/tun_device_manager/windows.rs

@@ -1,17 +1,44 @@
 use anyhow::{Context as _, Result};
-use connlib_shared::windows::{CREATE_NO_WINDOW, TUNNEL_NAME};
-use firezone_tunnel::Tun;
-use ip_network::IpNetwork;
-use ip_network::{Ipv4Network, Ipv6Network};
+use connlib_shared::{
+    windows::{CREATE_NO_WINDOW, TUNNEL_NAME},
+    DEFAULT_MTU,
+};
+use ip_network::{IpNetwork, Ipv4Network, Ipv6Network};
 use std::{
     collections::HashSet,
+    io,
     net::{Ipv4Addr, Ipv6Addr, SocketAddrV4, SocketAddrV6},
     os::windows::process::CommandExt,
     process::{Command, Stdio},
+    str::FromStr,
+    sync::Arc,
+    task::{ready, Context, Poll},
 };
-use windows::Win32::NetworkManagement::IpHelper::{
-    CreateIpForwardEntry2, DeleteIpForwardEntry2, InitializeIpForwardEntry, MIB_IPFORWARD_ROW2,
+use tokio::sync::mpsc;
+use windows::Win32::{
+    NetworkManagement::{
+        IpHelper::{
+            CreateIpForwardEntry2, DeleteIpForwardEntry2, GetIpInterfaceEntry,
+            InitializeIpForwardEntry, SetIpInterfaceEntry, MIB_IPFORWARD_ROW2, MIB_IPINTERFACE_ROW,
+        },
+        Ndis::NET_LUID_LH,
+    },
+    Networking::WinSock::{AF_INET, AF_INET6},
 };
+use wintun::Adapter;
+
+// Not sure how this and `TUNNEL_NAME` differ
+const ADAPTER_NAME: &str = "Firezone";
+/// The ring buffer size used for Wintun.
+///
+/// Must be a power of two within a certain range <https://docs.rs/wintun/latest/wintun/struct.Adapter.html#method.start_session>
+/// 0x10_0000 is 1 MiB, which performs decently on the Cloudflare speed test.
+/// At 1 Gbps that's about 8 ms, so any delay where Firezone isn't scheduled by the OS
+/// onto a core for more than 8 ms would result in packet drops.
+///
+/// We think 1 MiB is similar to the buffer size on Linux / macOS but we're not sure
+/// where that is configured.
+const RING_BUFFER_SIZE: u32 = 0x10_0000;
 
 pub struct TunDeviceManager {
     iface_idx: Option<u32>,
@@ -147,3 +174,223 @@ fn forward_entry(route: IpNetwork, iface_idx: u32) -> MIB_IPFORWARD_ROW2 {
 
     row
 }
+
+// Must be public so the benchmark binary can find it
+pub struct Tun {
+    /// The index of our network adapter, we can use this when asking Windows to add / remove routes / DNS rules
+    /// It's stable across app restarts and I'm assuming across system reboots too.
+    iface_idx: u32,
+    packet_rx: mpsc::Receiver<wintun::Packet>,
+    recv_thread: Option<std::thread::JoinHandle<()>>,
+    session: Arc<wintun::Session>,
+}
+
+impl Drop for Tun {
+    fn drop(&mut self) {
+        tracing::debug!(
+            channel_capacity = self.packet_rx.capacity(),
+            "Shutting down packet channel..."
+        );
+        self.packet_rx.close(); // This avoids a deadlock when we join the worker thread, see PR 5571
+        if let Err(error) = self.session.shutdown() {
+            tracing::error!(?error, "wintun::Session::shutdown");
+        }
+        if let Err(error) = self
+            .recv_thread
+            .take()
+            .expect("`recv_thread` should always be `Some` until `Tun` drops")
+            .join()
+        {
+            tracing::error!(?error, "`Tun::recv_thread` panicked");
+        }
+    }
+}
+
+impl Tun {
+    #[tracing::instrument(level = "debug")]
+    pub fn new() -> Result<Self> {
+        const TUNNEL_UUID: &str = "e9245bc1-b8c1-44ca-ab1d-c6aad4f13b9c";
+
+        // SAFETY: we're loading a DLL from disk and it has arbitrary C code in it.
+        // The Windows client, in `wintun_install` hashes the DLL at startup, before calling connlib, so it's unlikely for the DLL to be accidentally corrupted by the time we get here.
+        let path = connlib_shared::windows::wintun_dll_path()?;
+        let wintun = unsafe { wintun::load_from_path(path) }?;
+
+        // Create wintun adapter
+        let uuid = uuid::Uuid::from_str(TUNNEL_UUID)
+            .expect("static UUID should always parse correctly")
+            .as_u128();
+        let adapter = &Adapter::create(&wintun, ADAPTER_NAME, TUNNEL_NAME, Some(uuid))?;
+        let iface_idx = adapter.get_adapter_index()?;
+
+        set_iface_config(adapter.get_luid(), DEFAULT_MTU as u32)?;
+
+        let session = Arc::new(adapter.start_session(RING_BUFFER_SIZE)?);
+        // 4 is a nice power of two. Wintun already queues packets for us, so we don't
+        // need much capacity here.
+        let (packet_tx, packet_rx) = mpsc::channel(4);
+        let recv_thread = start_recv_thread(packet_tx, Arc::clone(&session))?;
+
+        Ok(Self {
+            iface_idx,
+            recv_thread: Some(recv_thread),
+            packet_rx,
+            session: Arc::clone(&session),
+        })
+    }
+
+    pub fn iface_idx(&self) -> u32 {
+        self.iface_idx
+    }
+
+    // Moves packets from the Internet towards the user
+    #[allow(clippy::unnecessary_wraps)] // Fn signature must align with other platform implementations.
+    fn write(&self, bytes: &[u8]) -> io::Result<usize> {
+        let len = bytes
+            .len()
+            .try_into()
+            .expect("Packet length should fit into u16");
+
+        let Ok(mut pkt) = self.session.allocate_send_packet(len) else {
+            // Ring buffer is full, just drop the packet since we're at the IP layer
+            return Ok(0);
+        };
+
+        pkt.bytes_mut().copy_from_slice(bytes);
+        // `send_packet` cannot fail to enqueue the packet, since we already allocated
+        // space in the ring buffer.
+        self.session.send_packet(pkt);
+        Ok(bytes.len())
+    }
+}
+
+impl tun::Tun for Tun {
+    // Moves packets from the user towards the Internet
+    fn poll_read(&mut self, buf: &mut [u8], cx: &mut Context<'_>) -> Poll<io::Result<usize>> {
+        let pkt = ready!(self.packet_rx.poll_recv(cx));
+
+        match pkt {
+            Some(pkt) => {
+                let bytes = pkt.bytes();
+                let len = bytes.len();
+                if len > buf.len() {
+                    // This shouldn't happen now that we set IPv4 and IPv6 MTU
+                    // If it does, something is wrong.
+                    tracing::warn!("Packet is too long to read ({len} bytes)");
+                    return Poll::Ready(Ok(0));
+                }
+                buf[0..len].copy_from_slice(bytes);
+                Poll::Ready(Ok(len))
+            }
+            None => {
+                tracing::error!("error receiving packet from mpsc channel");
+                Poll::Ready(Err(std::io::ErrorKind::Other.into()))
+            }
+        }
+    }
+
+    fn name(&self) -> &str {
+        TUNNEL_NAME
+    }
+
+    fn write4(&self, bytes: &[u8]) -> io::Result<usize> {
+        self.write(bytes)
+    }
+
+    fn write6(&self, bytes: &[u8]) -> io::Result<usize> {
+        self.write(bytes)
+    }
+}
+
+// Moves packets from the user towards the Internet
+fn start_recv_thread(
+    packet_tx: mpsc::Sender<wintun::Packet>,
+    session: Arc<wintun::Session>,
+) -> io::Result<std::thread::JoinHandle<()>> {
+    std::thread::Builder::new()
+        .name("Firezone wintun worker".into())
+        .spawn(move || loop {
+            let pkt = match session.receive_blocking() {
+                Ok(pkt) => pkt,
+                Err(wintun::Error::ShuttingDown) => {
+                    tracing::info!(
+                        "Stopping outbound worker thread because Wintun is shutting down"
+                    );
+                    break;
+                }
+                Err(e) => {
+                    tracing::error!("wintun::Session::receive_blocking: {e:#?}");
+                    break;
+                }
+            };
+
+            // Use `blocking_send` so that if connlib is behind by a few packets,
+            // Wintun will queue up new packets in its ring buffer while we
+            // wait for our MPSC channel to clear.
+            // Unfortunately we don't know if Wintun is dropping packets, since
+            // it doesn't expose a sequence number or anything.
+            match packet_tx.blocking_send(pkt) {
+                Ok(()) => {}
+                Err(_) => {
+                    tracing::info!(
+                        "Stopping outbound worker thread because the packet channel closed"
+                    );
+                    break;
+                }
+            }
+        })
+}
+
+/// Sets MTU on the interface
+/// TODO: Set IP and other things in here too, so the code is more organized
+fn set_iface_config(luid: wintun::NET_LUID_LH, mtu: u32) -> Result<()> {
+    // SAFETY: Both NET_LUID_LH unions should be the same. We're just copying out
+    // the u64 value and re-wrapping it, since wintun doesn't refer to the windows
+    // crate's version of NET_LUID_LH.
+    let luid = NET_LUID_LH {
+        Value: unsafe { luid.Value },
+    };
+
+    // Set MTU for IPv4
+    {
+        let mut row = MIB_IPINTERFACE_ROW {
+            Family: AF_INET,
+            InterfaceLuid: luid,
+            ..Default::default()
+        };
+
+        // SAFETY: TODO
+        unsafe { GetIpInterfaceEntry(&mut row) }.ok()?;
+
+        // https://stackoverflow.com/questions/54857292/setipinterfaceentry-returns-error-invalid-parameter
+        row.SitePrefixLength = 0;
+
+        // Set MTU for IPv4
+        row.NlMtu = mtu;
+
+        // SAFETY: TODO
+        unsafe { SetIpInterfaceEntry(&mut row) }.ok()?;
+    }
+
+    // Set MTU for IPv6
+    {
+        let mut row = MIB_IPINTERFACE_ROW {
+            Family: AF_INET6,
+            InterfaceLuid: luid,
+            ..Default::default()
+        };
+
+        // SAFETY: TODO
+        unsafe { GetIpInterfaceEntry(&mut row) }.ok()?;
+
+        // https://stackoverflow.com/questions/54857292/setipinterfaceentry-returns-error-invalid-parameter
+        row.SitePrefixLength = 0;
+
+        // Set MTU for IPv4
+        row.NlMtu = mtu;
+
+        // SAFETY: TODO
+        unsafe { SetIpInterfaceEntry(&mut row) }.ok()?;
+    }
+    Ok(())
+}

rust/connlib/clients/android/Cargo.toml

@@ -18,6 +18,7 @@ connlib-client-shared = { workspace = true }
 connlib-shared = { workspace = true }
 ip_network = "0.4"
 jni = { version = "0.21.1", features = ["invocation"] }
+libc = "0.2"
 log = "0.4"
 phoenix-channel = { workspace = true }
 secrecy = { workspace = true }
@@ -28,6 +29,7 @@ tokio = { workspace = true, features = ["rt"] }
 tracing = { workspace = true, features = ["std", "attributes"] }
 tracing-appender = "0.2"
 tracing-subscriber = { workspace = true }
+tun = { workspace = true }
 url = "2.4.0"
 
 [target.'cfg(target_os = "android")'.dependencies]

rust/connlib/clients/android/src/lib.rs

@@ -3,10 +3,11 @@
 // However, this consideration has made it idiomatic for Java FFI in the Rust
 // ecosystem, so it's used here for consistency.
 
+use crate::tun::Tun;
 use backoff::ExponentialBackoffBuilder;
 use connlib_client_shared::{
     callbacks::ResourceDescription, file_logger, keypair, Callbacks, ConnectArgs, Error, LoginUrl,
-    LoginUrlError, Session, Tun, V4RouteList, V6RouteList,
+    LoginUrlError, Session, V4RouteList, V6RouteList,
 };
 use connlib_shared::get_user_agent;
 use ip_network::{Ipv4Network, Ipv6Network};
@@ -32,6 +33,7 @@ use tracing_subscriber::prelude::*;
 use tracing_subscriber::EnvFilter;
 
 mod make_writer;
+mod tun;
 
 /// The Android client doesn't use platform APIs to detect network connectivity changes,
 /// so we rely on connlib to do so. We have valid use cases for headless Android clients
@@ -527,7 +529,7 @@ pub unsafe extern "system" fn Java_dev_firezone_android_tunnel_ConnlibSession_se
         }
     };
 
-    session.inner.set_tun(tun);
+    session.inner.set_tun(Box::new(tun));
 }
 
 fn protected_tcp_socket_factory(

rust/connlib/clients/android/src/tun.rs

@@ -1,11 +1,10 @@
-use super::utils;
-use crate::device_channel::ioctl;
 use std::task::{Context, Poll};
 use std::{
     io,
     os::fd::{AsRawFd, RawFd},
 };
 use tokio::io::unix::AsyncFd;
+use tun::ioctl;
 
 #[derive(Debug)]
 pub struct Tun {
@@ -19,19 +18,25 @@ impl Drop for Tun {
     }
 }
 
+impl tun::Tun for Tun {
+    fn write4(&self, src: &[u8]) -> std::io::Result<usize> {
+        write(self.fd.as_raw_fd(), src)
+    }
+
+    fn write6(&self, src: &[u8]) -> std::io::Result<usize> {
+        write(self.fd.as_raw_fd(), src)
+    }
+
+    fn poll_read(&mut self, buf: &mut [u8], cx: &mut Context<'_>) -> Poll<io::Result<usize>> {
+        tun::unix::poll_raw_fd(&self.fd, |fd| read(fd, buf), cx)
+    }
+
+    fn name(&self) -> &str {
+        self.name.as_str()
+    }
+}
+
 impl Tun {
-    pub fn write4(&self, src: &[u8]) -> std::io::Result<usize> {
-        write(self.fd.as_raw_fd(), src)
-    }
-
-    pub fn write6(&self, src: &[u8]) -> std::io::Result<usize> {
-        write(self.fd.as_raw_fd(), src)
-    }
-
-    pub fn poll_read(&self, buf: &mut [u8], cx: &mut Context<'_>) -> Poll<io::Result<usize>> {
-        utils::poll_raw_fd(&self.fd, |fd| read(fd, buf), cx)
-    }
-
     /// Create a new [`Tun`] from a raw file descriptor.
     ///
     /// # Safety
@@ -45,10 +50,6 @@ impl Tun {
             name,
         })
     }
-
-    pub fn name(&self) -> &str {
-        self.name.as_str()
-    }
 }
 
 /// Retrieves the name of the interface pointed to by the provided file descriptor.
@@ -58,38 +59,13 @@ impl Tun {
 /// The file descriptor must be open.
 unsafe fn interface_name(fd: RawFd) -> io::Result<String> {
     const TUNGETIFF: libc::c_ulong = 0x800454d2;
-    let mut request = ioctl::Request::<GetInterfaceNamePayload>::new();
+    let mut request = tun::ioctl::Request::<tun::ioctl::GetInterfaceNamePayload>::new();
 
     ioctl::exec(fd, TUNGETIFF, &mut request)?;
 
     Ok(request.name().to_string())
 }
 
-impl ioctl::Request<GetInterfaceNamePayload> {
-    fn new() -> Self {
-        Self {
-            name: [0u8; libc::IF_NAMESIZE],
-            payload: Default::default(),
-        }
-    }
-
-    fn name(&self) -> std::borrow::Cow<'_, str> {
-        // Safety: The memory of `self.name` is always initialized.
-        let cstr = unsafe { std::ffi::CStr::from_ptr(self.name.as_ptr() as _) };
-
-        cstr.to_string_lossy()
-    }
-}
-
-#[derive(Default)]
-#[repr(C)]
-struct GetInterfaceNamePayload {
-    // Fixes a nasty alignment bug on 32-bit architectures on Android.
-    // The `name` field in `ioctl::Request` is only 16 bytes long and accessing it causes a NPE without this alignment.
-    // Why? Not sure. It seems to only happen in release mode which hints at an optimisation issue.
-    alignment: [std::ffi::c_uchar; 16],
-}
-
 /// Read from the given file descriptor in the buffer.
 fn read(fd: RawFd, dst: &mut [u8]) -> io::Result<usize> {
     // Safety: Within this module, the file descriptor is always valid.

rust/connlib/clients/apple/Cargo.toml

@@ -26,6 +26,7 @@ tokio = { workspace = true, features = ["rt"] }
 tracing = { workspace = true }
 tracing-appender = "0.2"
 tracing-subscriber = "0.3"
+tun = { workspace = true }
 url = "2.5.0"
 
 [target.'cfg(any(target_os = "macos", target_os = "ios"))'.dependencies]

rust/connlib/clients/apple/src/lib.rs

@@ -2,12 +2,13 @@
 #![allow(clippy::unnecessary_cast, improper_ctypes, non_camel_case_types)]
 
 mod make_writer;
+mod tun;
 
 use anyhow::Result;
 use backoff::ExponentialBackoffBuilder;
 use connlib_client_shared::{
     callbacks::ResourceDescription, file_logger, keypair, Callbacks, ConnectArgs, Error, LoginUrl,
-    Session, Tun, V4RouteList, V6RouteList,
+    Session, V4RouteList, V6RouteList,
 };
 use connlib_shared::get_user_agent;
 use ip_network::{Ipv4Network, Ipv6Network};
@@ -22,6 +23,7 @@ use std::{
 use tokio::runtime::Runtime;
 use tracing_subscriber::EnvFilter;
 use tracing_subscriber::{prelude::*, util::TryInitError};
+use tun::Tun;
 
 /// The Apple client implements reconnect logic in the upper layer using OS provided
 /// APIs to detect network connectivity changes. The reconnect timeout here only
@@ -214,7 +216,7 @@ impl WrappedSession {
             Arc::new(socket_factory::tcp),
         )?;
         let session = Session::connect(args, portal, runtime.handle().clone());
-        session.set_tun(Tun::new()?);
+        session.set_tun(Box::new(Tun::new()?));
 
         Ok(Self {
             inner: session,

rust/connlib/clients/apple/src/tun.rs

@@ -0,0 +1,225 @@
+use libc::{fcntl, iovec, msghdr, recvmsg, AF_INET, AF_INET6, F_GETFL, F_SETFL, O_NONBLOCK};
+use std::task::{Context, Poll};
+use std::{
+    io,
+    os::fd::{AsRawFd as _, RawFd},
+};
+use tokio::io::unix::AsyncFd;
+
+#[derive(Debug)]
+pub struct Tun {
+    name: String,
+    fd: AsyncFd<RawFd>,
+}
+
+impl Tun {
+    pub fn new() -> io::Result<Self> {
+        let fd = search_for_tun_fd()?;
+        set_non_blocking(fd)?;
+
+        let name = name(fd)?;
+
+        Ok(Self {
+            name,
+            fd: AsyncFd::new(fd)?,
+        })
+    }
+
+    fn write(&self, src: &[u8], af: u8) -> io::Result<usize> {
+        let mut hdr = [0, 0, 0, af];
+        let mut iov = [
+            iovec {
+                iov_base: hdr.as_mut_ptr() as _,
+                iov_len: hdr.len(),
+            },
+            iovec {
+                iov_base: src.as_ptr() as _,
+                iov_len: src.len(),
+            },
+        ];
+
+        let msg_hdr = msghdr {
+            msg_name: std::ptr::null_mut(),
+            msg_namelen: 0,
+            msg_iov: &mut iov[0],
+            msg_iovlen: iov.len() as _,
+            msg_control: std::ptr::null_mut(),
+            msg_controllen: 0,
+            msg_flags: 0,
+        };
+
+        match unsafe { libc::sendmsg(self.fd.as_raw_fd(), &msg_hdr, 0) } {
+            -1 => Err(io::Error::last_os_error()),
+            n => Ok(n as usize),
+        }
+    }
+}
+
+impl tun::Tun for Tun {
+    fn write4(&self, src: &[u8]) -> io::Result<usize> {
+        self.write(src, AF_INET as u8)
+    }
+
+    fn write6(&self, src: &[u8]) -> io::Result<usize> {
+        self.write(src, AF_INET6 as u8)
+    }
+
+    fn poll_read(&mut self, buf: &mut [u8], cx: &mut Context<'_>) -> Poll<io::Result<usize>> {
+        tun::unix::poll_raw_fd(&self.fd, |fd| read(fd, buf), cx)
+    }
+
+    fn name(&self) -> &str {
+        self.name.as_str()
+    }
+}
+
+fn get_last_error() -> io::Error {
+    io::Error::last_os_error()
+}
+
+fn set_non_blocking(fd: RawFd) -> io::Result<()> {
+    match unsafe { fcntl(fd, F_GETFL) } {
+        -1 => Err(get_last_error()),
+        flags => match unsafe { fcntl(fd, F_SETFL, flags | O_NONBLOCK) } {
+            -1 => Err(get_last_error()),
+            _ => Ok(()),
+        },
+    }
+}
+
+fn read(fd: RawFd, dst: &mut [u8]) -> io::Result<usize> {
+    let mut hdr = [0u8; 4];
+
+    let mut iov = [
+        iovec {
+            iov_base: hdr.as_mut_ptr() as _,
+            iov_len: hdr.len(),
+        },
+        iovec {
+            iov_base: dst.as_mut_ptr() as _,
+            iov_len: dst.len(),
+        },
+    ];
+
+    let mut msg_hdr = msghdr {
+        msg_name: std::ptr::null_mut(),
+        msg_namelen: 0,
+        msg_iov: &mut iov[0],
+        msg_iovlen: iov.len() as _,
+        msg_control: std::ptr::null_mut(),
+        msg_controllen: 0,
+        msg_flags: 0,
+    };
+
+    // Safety: Within this module, the file descriptor is always valid.
+    match unsafe { recvmsg(fd, &mut msg_hdr, 0) } {
+        -1 => Err(io::Error::last_os_error()),
+        0..=4 => Ok(0),
+        n => Ok((n - 4) as usize),
+    }
+}
+
+#[cfg(any(target_os = "macos", target_os = "ios"))]
+fn name(fd: RawFd) -> io::Result<String> {
+    use libc::{getsockopt, socklen_t, IF_NAMESIZE, SYSPROTO_CONTROL, UTUN_OPT_IFNAME};
+
+    let mut tunnel_name = [0u8; IF_NAMESIZE];
+    let mut tunnel_name_len = tunnel_name.len() as socklen_t;
+    if unsafe {
+        getsockopt(
+            fd,
+            SYSPROTO_CONTROL,
+            UTUN_OPT_IFNAME,
+            tunnel_name.as_mut_ptr() as _,
+            &mut tunnel_name_len,
+        )
+    } < 0
+        || tunnel_name_len == 0
+    {
+        return Err(get_last_error());
+    }
+
+    Ok(String::from_utf8_lossy(&tunnel_name[..(tunnel_name_len - 1) as usize]).to_string())
+}
+
+#[cfg(any(target_os = "macos", target_os = "ios"))]
+fn search_for_tun_fd() -> io::Result<RawFd> {
+    const CTL_NAME: &[u8] = b"com.apple.net.utun_control";
+
+    use libc::{ctl_info, getpeername, ioctl, sockaddr_ctl, socklen_t, AF_SYSTEM, CTLIOCGINFO};
+    use std::mem::size_of;
+
+    let mut info = ctl_info {
+        ctl_id: 0,
+        ctl_name: [0; 96],
+    };
+    info.ctl_name[..CTL_NAME.len()]
+        // SAFETY: We only care about maintaining the same byte value not the same value,
+        // meaning that the slice &[u8] here is just a blob of bytes for us, we need this conversion
+        // just because `c_char` is i8 (for some reason).
+        // One thing I don't like about this is that `ctl_name` is actually a nul-terminated string,
+        // which we are only getting because `CTRL_NAME` is less than 96 bytes long and we are 0-value
+        // initializing the array we should be using a CStr to be explicit... but this is slightly easier.
+        .copy_from_slice(unsafe { &*(CTL_NAME as *const [u8] as *const [i8]) });
+
+    // On Apple platforms, we must use a NetworkExtension for reading and writing
+    // packets if we want to be allowed in the iOS and macOS App Stores. This has the
+    // unfortunate side effect that we're not allowed to create or destroy the tunnel
+    // interface ourselves. The file descriptor should already be opened by the NetworkExtension for us
+    // by this point. So instead, we iterate through all file descriptors looking for the one corresponding
+    // to the utun interface we have access to read and write from.
+    //
+    // Credit to Jason Donenfeld (@zx2c4) for this technique. See docs/NOTICE.txt for attribution.
+    // https://github.com/WireGuard/wireguard-apple/blob/master/Sources/WireGuardKit/WireGuardAdapter.swift
+    for fd in 0..1024 {
+        tracing::debug!("Checking fd {}", fd);
+
+        // initialize empty sockaddr_ctl to be populated by getpeername
+        let mut addr = sockaddr_ctl {
+            sc_len: size_of::<sockaddr_ctl>() as u8,
+            sc_family: 0,
+            ss_sysaddr: 0,
+            sc_id: info.ctl_id,
+            sc_unit: 0,
+            sc_reserved: Default::default(),
+        };
+
+        let mut len = size_of::<sockaddr_ctl>() as u32;
+        let ret = unsafe {
+            getpeername(
+                fd,
+                &mut addr as *mut sockaddr_ctl as _,
+                &mut len as *mut socklen_t,
+            )
+        };
+        if ret != 0 || addr.sc_family != AF_SYSTEM as u8 {
+            continue;
+        }
+
+        if info.ctl_id == 0 {
+            let ret = unsafe { ioctl(fd, CTLIOCGINFO, &mut info as *mut ctl_info) };
+
+            if ret != 0 {
+                continue;
+            }
+        }
+
+        if addr.sc_id == info.ctl_id {
+            set_non_blocking(fd)?;
+
+            return Ok(fd);
+        }
+    }
+
+    Err(get_last_error())
+}
+
+#[cfg(not(any(target_os = "macos", target_os = "ios")))]
+fn search_for_tun_fd() -> io::Result<RawFd> {
+    unimplemented!("Stub")
+}
+
+#[cfg(not(any(target_os = "macos", target_os = "ios")))]
+fn name(_: RawFd) -> io::Result<String> {
+    unimplemented!("Stub")
+}

rust/connlib/clients/shared/Cargo.toml

@@ -25,6 +25,7 @@ tracing = { workspace = true }
 tracing-appender = { version = "0.2.2" }
 tracing-stackdriver = { version = "0.10.0" }
 tracing-subscriber = { workspace = true, features = ["env-filter"] }
+tun = { workspace = true }
 url = { version = "2.4.1", features = ["serde"] }
 
 [target.'cfg(target_os = "android")'.dependencies]

rust/connlib/clients/shared/src/eventloop.rs

@@ -10,13 +10,14 @@ use connlib_shared::{
     messages::{ConnectionAccepted, GatewayResponse, RelaysPresence, ResourceAccepted, ResourceId},
     Callbacks,
 };
-use firezone_tunnel::{ClientTunnel, Tun};
+use firezone_tunnel::ClientTunnel;
 use phoenix_channel::{ErrorReply, OutboundRequestId, PhoenixChannel};
 use std::{
     collections::{BTreeSet, HashMap},
     net::IpAddr,
     task::{Context, Poll},
 };
+use tun::Tun;
 
 pub struct Eventloop<C: Callbacks> {
     tunnel: ClientTunnel,
@@ -33,7 +34,7 @@ pub enum Command {
     Stop,
     Reconnect,
     SetDns(Vec<IpAddr>),
-    SetTun(Tun),
+    SetTun(Box<dyn Tun>),
 }
 
 impl<C: Callbacks> Eventloop<C> {

rust/connlib/clients/shared/src/lib.rs

@@ -5,7 +5,6 @@ pub use connlib_shared::{
     callbacks, keypair, Callbacks, Error, LoginUrl, LoginUrlError, StaticSecret,
 };
 pub use eventloop::Eventloop;
-pub use firezone_tunnel::Tun;
 pub use tracing_appender::non_blocking::WorkerGuard;
 
 use eventloop::Command;
@@ -18,6 +17,7 @@ use std::net::IpAddr;
 use std::sync::Arc;
 use tokio::sync::mpsc::UnboundedReceiver;
 use tokio::task::JoinHandle;
+use tun::Tun;
 
 mod eventloop;
 pub mod file_logger;
@@ -95,7 +95,7 @@ impl Session {
     }
 
     /// Sets a new [`Tun`] device handle.
-    pub fn set_tun(&self, new_tun: Tun) {
+    pub fn set_tun(&self, new_tun: Box<dyn Tun>) {
         let _ = self.channel.send(Command::SetTun(new_tun));
     }
 
@@ -177,68 +177,3 @@ where
         },
     }
 }
-
-#[cfg(test)]
-mod tests {
-    #[derive(Clone, Default)]
-    struct Callbacks {}
-    impl connlib_shared::Callbacks for Callbacks {}
-
-    #[cfg(target_os = "linux")]
-    #[tokio::test]
-    #[ignore = "Performs system-wide I/O, needs sudo"]
-    async fn device_linux() {
-        device_common().await;
-    }
-
-    #[cfg(target_os = "windows")]
-    #[tokio::test]
-    #[ignore = "Performs system-wide I/O, needs sudo"]
-    async fn device_windows() {
-        // Install wintun so the test can run
-        let wintun_path = connlib_shared::windows::wintun_dll_path().unwrap();
-        tokio::fs::create_dir_all(wintun_path.parent().unwrap())
-            .await
-            .unwrap();
-        tokio::fs::write(&wintun_path, connlib_shared::windows::wintun_bytes())
-            .await
-            .unwrap();
-
-        device_common().await;
-    }
-
-    #[cfg(any(target_os = "windows", target_os = "linux"))]
-    async fn device_common() {
-        use firezone_tunnel::Tun;
-        use std::{collections::HashMap, sync::Arc};
-
-        let (private_key, _public_key) = connlib_shared::keypair();
-        let mut tunnel = firezone_tunnel::ClientTunnel::new(
-            private_key,
-            Arc::new(socket_factory::tcp),
-            Arc::new(socket_factory::udp),
-            HashMap::new(),
-        )
-        .unwrap();
-        let upstream_dns = vec![([192, 168, 1, 1], 53).into()];
-        let interface = connlib_shared::messages::Interface {
-            ipv4: [100, 71, 96, 96].into(),
-            ipv6: [0xfd00, 0x2021, 0x1111, 0x0, 0x0, 0x0, 0x0019, 0x6538].into(),
-            upstream_dns,
-        };
-        tunnel.set_tun(Tun::new().unwrap());
-        tunnel.set_new_interface_config(interface).unwrap();
-
-        let tunnel = tokio::spawn(async move {
-            std::future::poll_fn(|cx| tunnel.poll_next_event(cx))
-                .await
-                .unwrap()
-        });
-
-        tokio::time::sleep(std::time::Duration::from_secs(5)).await;
-
-        if tunnel.is_finished() {
-            tunnel.await.unwrap();
-        }
-    }
-}

rust/connlib/shared/src/lib.rs

@@ -39,7 +39,7 @@ pub type DomainName = domain::base::Name<Vec<u8>>;
 /// <https://learn.microsoft.com/en-us/windows/configuration/find-the-application-user-model-id-of-an-installed-app>
 pub const BUNDLE_ID: &str = "dev.firezone.client";
 
-pub const DEFAULT_MTU: u32 = 1280;
+pub const DEFAULT_MTU: usize = 1280;
 
 const LIB_NAME: &str = "connlib";
 

rust/connlib/tunnel/Cargo.toml

@@ -5,7 +5,6 @@ edition = "2021"
 
 [dependencies]
 anyhow = "1.0"
-async-trait = { version = "0.1", default-features = false }
 bimap = "0.6"
 boringtun = { workspace = true }
 bytes = { version = "1.4", default-features = false, features = ["std"] }
@@ -22,7 +21,6 @@ ip-packet = { workspace = true }
 ip_network = { version = "0.4", default-features = false }
 ip_network_table = { version = "0.2", default-features = false }
 itertools = { version = "0.13", default-features = false, features = ["use_std"] }
-libc = { version = "0.2", default-features = false, features = ["std", "const-extern-fn", "extra_traits"] }
 proptest = { version = "1", optional = true }
 quinn-udp = { git = "https://github.com/quinn-rs/quinn", branch = "main" }
 rand_core = { version = "0.6", default-features = false, features = ["getrandom"] }
@@ -34,7 +32,8 @@ socket-factory = { workspace = true }
 socket2 = { workspace = true }
 thiserror = { version = "1.0", default-features = false }
 tokio = { workspace = true }
-tracing = { workspace = true }
+tracing = { workspace = true, features = ["attributes"] }
+tun = { workspace = true }
 
 [dev-dependencies]
 derivative = "2.2.0"
@@ -47,31 +46,8 @@ serde_json = "1.0"
 test-strategy = "0.3.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
-[target.'cfg(target_os = "windows")'.dev-dependencies]
-firezone-bin-shared = { workspace = true } # Required for benchmark.
-
-[[bench]]
-name = "tunnel"
-harness = false
-
 [features]
 proptest = ["dep:proptest", "connlib-shared/proptest"]
 
-# Windows tunnel dependencies
-[target.'cfg(target_os = "windows")'.dependencies]
-tokio = { workspace = true, features = ["sync"] }
-uuid = { version = "1.7.0", features = ["v4"] }
-wintun = "0.4.0"
-
-# Windows Win32 API
-[target.'cfg(target_os = "windows")'.dependencies.windows]
-version = "0.57.0"
-features = [
-  "Win32_Foundation",
-  "Win32_NetworkManagement_IpHelper",
-  "Win32_NetworkManagement_Ndis",
-  "Win32_Networking_WinSock",
-]
-
 [lints]
 workspace = true

rust/connlib/tunnel/src/client.rs

@@ -63,7 +63,7 @@ impl ClientTunnel {
             });
     }
 
-    pub fn set_tun(&mut self, tun: Tun) {
+    pub fn set_tun(&mut self, tun: Box<dyn Tun>) {
         self.io.device_mut().set_tun(tun);
     }
 

rust/connlib/tunnel/src/device_channel.rs

@@ -1,35 +1,10 @@
-#[cfg(any(target_os = "macos", target_os = "ios"))]
-mod tun_darwin;
-#[cfg(any(target_os = "macos", target_os = "ios"))]
-use tun_darwin as tun;
-
-#[cfg(target_os = "linux")]
-mod tun_linux;
-#[cfg(target_os = "linux")]
-use tun_linux as tun;
-
-#[cfg(target_os = "windows")]
-mod tun_windows;
-#[cfg(target_os = "windows")]
-use tun_windows as tun;
-
-// TODO: Android and linux are nearly identical; use a common tunnel module?
-#[cfg(target_os = "android")]
-mod tun_android;
-#[cfg(target_os = "android")]
-use tun_android as tun;
-
-#[cfg(target_family = "unix")]
-mod utils;
-
 use ip_packet::{IpPacket, MutableIpPacket, Packet as _};
 use std::io;
 use std::task::{Context, Poll, Waker};
-
-pub use tun::Tun;
+use tun::Tun;
 
 pub struct Device {
-    tun: Option<Tun>,
+    tun: Option<Box<dyn Tun>>,
     waker: Option<Waker>,
 }
 
@@ -41,7 +16,7 @@ impl Device {
         }
     }
 
-    pub(crate) fn set_tun(&mut self, tun: Tun) {
+    pub(crate) fn set_tun(&mut self, tun: Box<dyn Tun>) {
         tracing::info!(name = %tun.name(), "Initializing TUN device");
 
         self.tun = Some(tun);
@@ -128,45 +103,15 @@ impl Device {
         }
     }
 
-    fn tun(&self) -> io::Result<&Tun> {
-        self.tun.as_ref().ok_or_else(io_error_not_initialized)
+    fn tun(&self) -> io::Result<&dyn Tun> {
+        Ok(self
+            .tun
+            .as_ref()
+            .ok_or_else(io_error_not_initialized)?
+            .as_ref())
     }
 }
 
 fn io_error_not_initialized() -> io::Error {
     io::Error::new(io::ErrorKind::NotConnected, "device is not initialized yet")
 }
-
-#[cfg(any(target_os = "linux", target_os = "android"))]
-mod ioctl {
-    use super::*;
-    use std::os::fd::RawFd;
-
-    /// Executes the `ioctl` syscall on the given file descriptor with the provided request.
-    ///
-    /// # Safety
-    ///
-    /// The file descriptor must be open.
-    pub(crate) unsafe fn exec<P>(
-        fd: RawFd,
-        code: libc::c_ulong,
-        req: &mut Request<P>,
-    ) -> io::Result<()> {
-        let ret = unsafe { libc::ioctl(fd, code as _, req) };
-
-        if ret < 0 {
-            return Err(io::Error::last_os_error());
-        }
-
-        Ok(())
-    }
-
-    /// Represents a control request to an IO device, addresses by the device's name.
-    ///
-    /// The payload MUST also be `#[repr(C)]` and its layout depends on the particular request you are sending.
-    #[repr(C)]
-    pub(crate) struct Request<P> {
-        pub(crate) name: [std::ffi::c_uchar; libc::IF_NAMESIZE],
-        pub(crate) payload: P,
-    }
-}

rust/connlib/tunnel/src/device_channel/tun_darwin.rs

@@ -1,203 +0,0 @@
-use super::utils;
-use libc::{
-    ctl_info, fcntl, getpeername, getsockopt, ioctl, iovec, msghdr, recvmsg, sendmsg, sockaddr_ctl,
-    socklen_t, AF_INET, AF_INET6, AF_SYSTEM, CTLIOCGINFO, F_GETFL, F_SETFL, IF_NAMESIZE,
-    O_NONBLOCK, SYSPROTO_CONTROL, UTUN_OPT_IFNAME,
-};
-use std::task::{Context, Poll};
-use std::{
-    io,
-    mem::size_of,
-    os::fd::{AsRawFd, RawFd},
-};
-use tokio::io::unix::AsyncFd;
-
-const CTL_NAME: &[u8] = b"com.apple.net.utun_control";
-
-#[derive(Debug)]
-pub struct Tun {
-    name: String,
-    fd: AsyncFd<RawFd>,
-}
-
-impl Tun {
-    pub fn write4(&self, src: &[u8]) -> io::Result<usize> {
-        self.write(src, AF_INET as u8)
-    }
-
-    pub fn write6(&self, src: &[u8]) -> io::Result<usize> {
-        self.write(src, AF_INET6 as u8)
-    }
-
-    pub fn poll_read(&self, buf: &mut [u8], cx: &mut Context<'_>) -> Poll<io::Result<usize>> {
-        utils::poll_raw_fd(&self.fd, |fd| read(fd, buf), cx)
-    }
-
-    fn write(&self, src: &[u8], af: u8) -> io::Result<usize> {
-        let mut hdr = [0, 0, 0, af];
-        let mut iov = [
-            iovec {
-                iov_base: hdr.as_mut_ptr() as _,
-                iov_len: hdr.len(),
-            },
-            iovec {
-                iov_base: src.as_ptr() as _,
-                iov_len: src.len(),
-            },
-        ];
-
-        let msg_hdr = msghdr {
-            msg_name: std::ptr::null_mut(),
-            msg_namelen: 0,
-            msg_iov: &mut iov[0],
-            msg_iovlen: iov.len() as _,
-            msg_control: std::ptr::null_mut(),
-            msg_controllen: 0,
-            msg_flags: 0,
-        };
-
-        match unsafe { sendmsg(self.fd.as_raw_fd(), &msg_hdr, 0) } {
-            -1 => Err(io::Error::last_os_error()),
-            n => Ok(n as usize),
-        }
-    }
-
-    pub fn new() -> io::Result<Self> {
-        let mut info = ctl_info {
-            ctl_id: 0,
-            ctl_name: [0; 96],
-        };
-        info.ctl_name[..CTL_NAME.len()]
-            // SAFETY: We only care about maintaining the same byte value not the same value,
-            // meaning that the slice &[u8] here is just a blob of bytes for us, we need this conversion
-            // just because `c_char` is i8 (for some reason).
-            // One thing I don't like about this is that `ctl_name` is actually a nul-terminated string,
-            // which we are only getting because `CTRL_NAME` is less than 96 bytes long and we are 0-value
-            // initializing the array we should be using a CStr to be explicit... but this is slightly easier.
-            .copy_from_slice(unsafe { &*(CTL_NAME as *const [u8] as *const [i8]) });
-
-        // On Apple platforms, we must use a NetworkExtension for reading and writing
-        // packets if we want to be allowed in the iOS and macOS App Stores. This has the
-        // unfortunate side effect that we're not allowed to create or destroy the tunnel
-        // interface ourselves. The file descriptor should already be opened by the NetworkExtension for us
-        // by this point. So instead, we iterate through all file descriptors looking for the one corresponding
-        // to the utun interface we have access to read and write from.
-        //
-        // Credit to Jason Donenfeld (@zx2c4) for this technique. See docs/NOTICE.txt for attribution.
-        // https://github.com/WireGuard/wireguard-apple/blob/master/Sources/WireGuardKit/WireGuardAdapter.swift
-        for fd in 0..1024 {
-            tracing::debug!("Checking fd {}", fd);
-
-            // initialize empty sockaddr_ctl to be populated by getpeername
-            let mut addr = sockaddr_ctl {
-                sc_len: size_of::<sockaddr_ctl>() as u8,
-                sc_family: 0,
-                ss_sysaddr: 0,
-                sc_id: info.ctl_id,
-                sc_unit: 0,
-                sc_reserved: Default::default(),
-            };
-
-            let mut len = size_of::<sockaddr_ctl>() as u32;
-            let ret = unsafe {
-                getpeername(
-                    fd,
-                    &mut addr as *mut sockaddr_ctl as _,
-                    &mut len as *mut socklen_t,
-                )
-            };
-            if ret != 0 || addr.sc_family != AF_SYSTEM as u8 {
-                continue;
-            }
-
-            if info.ctl_id == 0 {
-                let ret = unsafe { ioctl(fd, CTLIOCGINFO, &mut info as *mut ctl_info) };
-
-                if ret != 0 {
-                    continue;
-                }
-            }
-
-            if addr.sc_id == info.ctl_id {
-                set_non_blocking(fd)?;
-
-                return Ok(Self {
-                    name: name(fd)?,
-                    fd: AsyncFd::new(fd)?,
-                });
-            }
-        }
-
-        Err(get_last_error())
-    }
-
-    pub fn name(&self) -> &str {
-        self.name.as_str()
-    }
-}
-
-fn get_last_error() -> io::Error {
-    io::Error::last_os_error()
-}
-
-fn set_non_blocking(fd: RawFd) -> io::Result<()> {
-    match unsafe { fcntl(fd, F_GETFL) } {
-        -1 => Err(get_last_error()),
-        flags => match unsafe { fcntl(fd, F_SETFL, flags | O_NONBLOCK) } {
-            -1 => Err(get_last_error()),
-            _ => Ok(()),
-        },
-    }
-}
-
-fn read(fd: RawFd, dst: &mut [u8]) -> io::Result<usize> {
-    let mut hdr = [0u8; 4];
-
-    let mut iov = [
-        iovec {
-            iov_base: hdr.as_mut_ptr() as _,
-            iov_len: hdr.len(),
-        },
-        iovec {
-            iov_base: dst.as_mut_ptr() as _,
-            iov_len: dst.len(),
-        },
-    ];
-
-    let mut msg_hdr = msghdr {
-        msg_name: std::ptr::null_mut(),
-        msg_namelen: 0,
-        msg_iov: &mut iov[0],
-        msg_iovlen: iov.len() as _,
-        msg_control: std::ptr::null_mut(),
-        msg_controllen: 0,
-        msg_flags: 0,
-    };
-
-    // Safety: Within this module, the file descriptor is always valid.
-    match unsafe { recvmsg(fd, &mut msg_hdr, 0) } {
-        -1 => Err(io::Error::last_os_error()),
-        0..=4 => Ok(0),
-        n => Ok((n - 4) as usize),
-    }
-}
-
-fn name(fd: RawFd) -> io::Result<String> {
-    let mut tunnel_name = [0u8; IF_NAMESIZE];
-    let mut tunnel_name_len = tunnel_name.len() as socklen_t;
-    if unsafe {
-        getsockopt(
-            fd,
-            SYSPROTO_CONTROL,
-            UTUN_OPT_IFNAME,
-            tunnel_name.as_mut_ptr() as _,
-            &mut tunnel_name_len,
-        )
-    } < 0
-        || tunnel_name_len == 0
-    {
-        return Err(get_last_error());
-    }
-
-    Ok(String::from_utf8_lossy(&tunnel_name[..(tunnel_name_len - 1) as usize]).to_string())
-}

rust/connlib/tunnel/src/device_channel/tun_linux.rs

@@ -1,167 +0,0 @@
-use super::utils;
-use crate::device_channel::ioctl;
-use libc::{
-    close, fcntl, makedev, mknod, open, F_GETFL, F_SETFL, IFF_NO_PI, IFF_TUN, O_NONBLOCK, O_RDWR,
-    S_IFCHR,
-};
-use std::path::Path;
-use std::task::{Context, Poll};
-use std::{
-    ffi::CStr,
-    fs, io,
-    os::{
-        fd::{AsRawFd, RawFd},
-        unix::fs::PermissionsExt,
-    },
-};
-use tokio::io::unix::AsyncFd;
-
-const TUNSETIFF: libc::c_ulong = 0x4004_54ca;
-const TUN_DEV_MAJOR: u32 = 10;
-const TUN_DEV_MINOR: u32 = 200;
-const IFACE_NAME: &str = "tun-firezone"; // Keep this synced with `TunDeviceManager` until we fix the module dependencies (i.e. move `Tun` out of `firezone-tunnel`).
-
-// Safety: We know that this is a valid C string.
-const TUN_FILE: &CStr = unsafe { CStr::from_bytes_with_nul_unchecked(b"/dev/net/tun\0") };
-
-#[derive(Debug)]
-pub struct Tun {
-    fd: AsyncFd<RawFd>,
-}
-
-impl Drop for Tun {
-    fn drop(&mut self) {
-        unsafe { close(self.fd.as_raw_fd()) };
-    }
-}
-
-impl Tun {
-    pub fn write4(&self, buf: &[u8]) -> io::Result<usize> {
-        write(self.fd.as_raw_fd(), buf)
-    }
-
-    pub fn write6(&self, buf: &[u8]) -> io::Result<usize> {
-        write(self.fd.as_raw_fd(), buf)
-    }
-
-    pub fn poll_read(&mut self, buf: &mut [u8], cx: &mut Context<'_>) -> Poll<io::Result<usize>> {
-        utils::poll_raw_fd(&self.fd, |fd| read(fd, buf), cx)
-    }
-
-    /// Create a new [`Tun`] from a raw file descriptor.
-    ///
-    /// # Safety
-    ///
-    /// The file descriptor must be open.
-    pub unsafe fn from_fd(fd: RawFd) -> io::Result<Self> {
-        Ok(Tun {
-            fd: AsyncFd::new(fd)?,
-        })
-    }
-
-    pub fn new() -> io::Result<Self> {
-        create_tun_device()?;
-
-        let fd = match unsafe { open(TUN_FILE.as_ptr() as _, O_RDWR) } {
-            -1 => return Err(get_last_error()),
-            fd => fd,
-        };
-
-        // Safety: We just opened the file descriptor.
-        unsafe {
-            ioctl::exec(
-                fd,
-                TUNSETIFF,
-                &mut ioctl::Request::<SetTunFlagsPayload>::new(),
-            )?;
-        }
-
-        set_non_blocking(fd)?;
-
-        // Safety: We just opened the fd.
-        unsafe { Self::from_fd(fd) }
-    }
-
-    pub fn name(&self) -> &str {
-        IFACE_NAME
-    }
-}
-
-fn get_last_error() -> io::Error {
-    io::Error::last_os_error()
-}
-
-fn set_non_blocking(fd: RawFd) -> io::Result<()> {
-    match unsafe { fcntl(fd, F_GETFL) } {
-        -1 => Err(get_last_error()),
-        flags => match unsafe { fcntl(fd, F_SETFL, flags | O_NONBLOCK) } {
-            -1 => Err(get_last_error()),
-            _ => Ok(()),
-        },
-    }
-}
-
-fn create_tun_device() -> io::Result<()> {
-    let path = Path::new(TUN_FILE.to_str().expect("path is valid utf-8"));
-
-    if path.exists() {
-        return Ok(());
-    }
-
-    let parent_dir = path.parent().unwrap();
-    fs::create_dir_all(parent_dir)?;
-    let permissions = fs::Permissions::from_mode(0o751);
-    fs::set_permissions(parent_dir, permissions)?;
-    if unsafe {
-        mknod(
-            TUN_FILE.as_ptr() as _,
-            S_IFCHR,
-            makedev(TUN_DEV_MAJOR, TUN_DEV_MINOR),
-        )
-    } != 0
-    {
-        return Err(get_last_error());
-    }
-
-    Ok(())
-}
-
-/// Read from the given file descriptor in the buffer.
-fn read(fd: RawFd, dst: &mut [u8]) -> io::Result<usize> {
-    // Safety: Within this module, the file descriptor is always valid.
-    match unsafe { libc::read(fd, dst.as_mut_ptr() as _, dst.len()) } {
-        -1 => Err(io::Error::last_os_error()),
-        n => Ok(n as usize),
-    }
-}
-
-/// Write the buffer to the given file descriptor.
-fn write(fd: RawFd, buf: &[u8]) -> io::Result<usize> {
-    // Safety: Within this module, the file descriptor is always valid.
-    match unsafe { libc::write(fd, buf.as_ptr() as _, buf.len() as _) } {
-        -1 => Err(io::Error::last_os_error()),
-        n => Ok(n as usize),
-    }
-}
-
-impl ioctl::Request<SetTunFlagsPayload> {
-    fn new() -> Self {
-        let name_as_bytes = IFACE_NAME.as_bytes();
-        debug_assert!(name_as_bytes.len() < libc::IF_NAMESIZE);
-
-        let mut name = [0u8; libc::IF_NAMESIZE];
-        name[..name_as_bytes.len()].copy_from_slice(name_as_bytes);
-
-        Self {
-            name,
-            payload: SetTunFlagsPayload {
-                flags: (IFF_TUN | IFF_NO_PI) as _,
-            },
-        }
-    }
-}
-
-#[repr(C)]
-struct SetTunFlagsPayload {
-    flags: std::ffi::c_short,
-}

rust/connlib/tunnel/src/device_channel/tun_windows.rs

@@ -1,268 +0,0 @@
-use crate::MTU;
-use connlib_shared::{windows::TUNNEL_NAME, Result};
-use std::{
-    io,
-    str::FromStr,
-    sync::Arc,
-    task::{ready, Context, Poll},
-};
-use tokio::sync::mpsc;
-use windows::Win32::{
-    NetworkManagement::{
-        IpHelper::{GetIpInterfaceEntry, SetIpInterfaceEntry, MIB_IPINTERFACE_ROW},
-        Ndis::NET_LUID_LH,
-    },
-    Networking::WinSock::{AF_INET, AF_INET6},
-};
-use wintun::Adapter;
-
-// Not sure how this and `TUNNEL_NAME` differ
-const ADAPTER_NAME: &str = "Firezone";
-/// The ring buffer size used for Wintun.
-///
-/// Must be a power of two within a certain range <https://docs.rs/wintun/latest/wintun/struct.Adapter.html#method.start_session>
-/// 0x10_0000 is 1 MiB, which performs decently on the Cloudflare speed test.
-/// At 1 Gbps that's about 8 ms, so any delay where Firezone isn't scheduled by the OS
-/// onto a core for more than 8 ms would result in packet drops.
-///
-/// We think 1 MiB is similar to the buffer size on Linux / macOS but we're not sure
-/// where that is configured.
-const RING_BUFFER_SIZE: u32 = 0x10_0000;
-
-// Must be public so the benchmark binary can find it
-pub struct Tun {
-    /// The index of our network adapter, we can use this when asking Windows to add / remove routes / DNS rules
-    /// It's stable across app restarts and I'm assuming across system reboots too.
-    iface_idx: u32,
-    packet_rx: mpsc::Receiver<wintun::Packet>,
-    recv_thread: Option<std::thread::JoinHandle<()>>,
-    session: Arc<wintun::Session>,
-}
-
-impl Drop for Tun {
-    fn drop(&mut self) {
-        tracing::debug!(
-            channel_capacity = self.packet_rx.capacity(),
-            "Shutting down packet channel..."
-        );
-        self.packet_rx.close(); // This avoids a deadlock when we join the worker thread, see PR 5571
-        if let Err(error) = self.session.shutdown() {
-            tracing::error!(?error, "wintun::Session::shutdown");
-        }
-        if let Err(error) = self
-            .recv_thread
-            .take()
-            .expect("`recv_thread` should always be `Some` until `Tun` drops")
-            .join()
-        {
-            tracing::error!(?error, "`Tun::recv_thread` panicked");
-        }
-    }
-}
-
-impl Tun {
-    #[tracing::instrument(level = "debug")]
-    pub fn new() -> Result<Self> {
-        const TUNNEL_UUID: &str = "e9245bc1-b8c1-44ca-ab1d-c6aad4f13b9c";
-
-        // SAFETY: we're loading a DLL from disk and it has arbitrary C code in it.
-        // The Windows client, in `wintun_install` hashes the DLL at startup, before calling connlib, so it's unlikely for the DLL to be accidentally corrupted by the time we get here.
-        let path = connlib_shared::windows::wintun_dll_path()?;
-        let wintun = unsafe { wintun::load_from_path(path) }?;
-
-        // Create wintun adapter
-        let uuid = uuid::Uuid::from_str(TUNNEL_UUID)
-            .expect("static UUID should always parse correctly")
-            .as_u128();
-        let adapter = &Adapter::create(&wintun, ADAPTER_NAME, TUNNEL_NAME, Some(uuid))?;
-        let iface_idx = adapter.get_adapter_index()?;
-
-        set_iface_config(adapter.get_luid(), MTU as u32)?;
-
-        let session = Arc::new(adapter.start_session(RING_BUFFER_SIZE)?);
-        // 4 is a nice power of two. Wintun already queues packets for us, so we don't
-        // need much capacity here.
-        let (packet_tx, packet_rx) = mpsc::channel(4);
-        let recv_thread = start_recv_thread(packet_tx, Arc::clone(&session))?;
-
-        Ok(Self {
-            iface_idx,
-            recv_thread: Some(recv_thread),
-            packet_rx,
-            session: Arc::clone(&session),
-        })
-    }
-
-    pub fn iface_idx(&self) -> u32 {
-        self.iface_idx
-    }
-
-    // Moves packets from the user towards the Internet
-    pub fn poll_read(&mut self, buf: &mut [u8], cx: &mut Context<'_>) -> Poll<io::Result<usize>> {
-        let pkt = ready!(self.packet_rx.poll_recv(cx));
-
-        match pkt {
-            Some(pkt) => {
-                let bytes = pkt.bytes();
-                let len = bytes.len();
-                if len > buf.len() {
-                    // This shouldn't happen now that we set IPv4 and IPv6 MTU
-                    // If it does, something is wrong.
-                    tracing::warn!("Packet is too long to read ({len} bytes)");
-                    return Poll::Ready(Ok(0));
-                }
-                buf[0..len].copy_from_slice(bytes);
-                Poll::Ready(Ok(len))
-            }
-            None => {
-                tracing::error!("error receiving packet from mpsc channel");
-                Poll::Ready(Err(std::io::ErrorKind::Other.into()))
-            }
-        }
-    }
-
-    pub fn name(&self) -> &str {
-        TUNNEL_NAME
-    }
-
-    pub fn write4(&self, bytes: &[u8]) -> io::Result<usize> {
-        self.write(bytes)
-    }
-
-    pub fn write6(&self, bytes: &[u8]) -> io::Result<usize> {
-        self.write(bytes)
-    }
-
-    // Moves packets from the Internet towards the user
-    #[allow(clippy::unnecessary_wraps)] // Fn signature must align with other platform implementations.
-    fn write(&self, bytes: &[u8]) -> io::Result<usize> {
-        let len = bytes
-            .len()
-            .try_into()
-            .expect("Packet length should fit into u16");
-
-        let Ok(mut pkt) = self.session.allocate_send_packet(len) else {
-            // Ring buffer is full, just drop the packet since we're at the IP layer
-            return Ok(0);
-        };
-
-        pkt.bytes_mut().copy_from_slice(bytes);
-        // `send_packet` cannot fail to enqueue the packet, since we already allocated
-        // space in the ring buffer.
-        self.session.send_packet(pkt);
-        Ok(bytes.len())
-    }
-}
-
-// Moves packets from the user towards the Internet
-fn start_recv_thread(
-    packet_tx: mpsc::Sender<wintun::Packet>,
-    session: Arc<wintun::Session>,
-) -> io::Result<std::thread::JoinHandle<()>> {
-    std::thread::Builder::new()
-        .name("Firezone wintun worker".into())
-        .spawn(move || loop {
-            let pkt = match session.receive_blocking() {
-                Ok(pkt) => pkt,
-                Err(wintun::Error::ShuttingDown) => {
-                    tracing::info!(
-                        "Stopping outbound worker thread because Wintun is shutting down"
-                    );
-                    break;
-                }
-                Err(e) => {
-                    tracing::error!("wintun::Session::receive_blocking: {e:#?}");
-                    break;
-                }
-            };
-
-            // Use `blocking_send` so that if connlib is behind by a few packets,
-            // Wintun will queue up new packets in its ring buffer while we
-            // wait for our MPSC channel to clear.
-            // Unfortunately we don't know if Wintun is dropping packets, since
-            // it doesn't expose a sequence number or anything.
-            match packet_tx.blocking_send(pkt) {
-                Ok(()) => {}
-                Err(_) => {
-                    tracing::info!(
-                        "Stopping outbound worker thread because the packet channel closed"
-                    );
-                    break;
-                }
-            }
-        })
-}
-
-/// Sets MTU on the interface
-/// TODO: Set IP and other things in here too, so the code is more organized
-fn set_iface_config(luid: wintun::NET_LUID_LH, mtu: u32) -> Result<()> {
-    // SAFETY: Both NET_LUID_LH unions should be the same. We're just copying out
-    // the u64 value and re-wrapping it, since wintun doesn't refer to the windows
-    // crate's version of NET_LUID_LH.
-    let luid = NET_LUID_LH {
-        Value: unsafe { luid.Value },
-    };
-
-    // Set MTU for IPv4
-    {
-        let mut row = MIB_IPINTERFACE_ROW {
-            Family: AF_INET,
-            InterfaceLuid: luid,
-            ..Default::default()
-        };
-
-        // SAFETY: TODO
-        unsafe { GetIpInterfaceEntry(&mut row) }.ok()?;
-
-        // https://stackoverflow.com/questions/54857292/setipinterfaceentry-returns-error-invalid-parameter
-        row.SitePrefixLength = 0;
-
-        // Set MTU for IPv4
-        row.NlMtu = mtu;
-
-        // SAFETY: TODO
-        unsafe { SetIpInterfaceEntry(&mut row) }.ok()?;
-    }
-
-    // Set MTU for IPv6
-    {
-        let mut row = MIB_IPINTERFACE_ROW {
-            Family: AF_INET6,
-            InterfaceLuid: luid,
-            ..Default::default()
-        };
-
-        // SAFETY: TODO
-        unsafe { GetIpInterfaceEntry(&mut row) }.ok()?;
-
-        // https://stackoverflow.com/questions/54857292/setipinterfaceentry-returns-error-invalid-parameter
-        row.SitePrefixLength = 0;
-
-        // Set MTU for IPv4
-        row.NlMtu = mtu;
-
-        // SAFETY: TODO
-        unsafe { SetIpInterfaceEntry(&mut row) }.ok()?;
-    }
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use tracing_subscriber::EnvFilter;
-
-    /// Checks for regressions in issue #4765, un-initializing Wintun
-    #[test]
-    #[ignore = "Needs admin privileges"]
-    fn tunnel_drop() {
-        let _ = tracing_subscriber::fmt()
-            .with_env_filter(EnvFilter::from_default_env())
-            .with_test_writer()
-            .try_init();
-        // Each cycle takes about half a second, so this will take a fair bit to run.
-        for _ in 0..50 {
-            let _tun = Tun::new().unwrap(); // This will panic if we don't correctly clean-up the wintun interface.
-        }
-    }
-}

rust/connlib/tunnel/src/gateway.rs

@@ -1,7 +1,7 @@
 use crate::peer::ClientOnGateway;
 use crate::peer_store::PeerStore;
 use crate::utils::earliest;
-use crate::{GatewayEvent, GatewayTunnel, Tun};
+use crate::{GatewayEvent, GatewayTunnel};
 use boringtun::x25519::PublicKey;
 use chrono::{DateTime, Utc};
 use connlib_shared::messages::{
@@ -15,11 +15,12 @@ use snownet::{RelaySocket, ServerNode};
 use std::collections::{BTreeMap, BTreeSet, VecDeque};
 use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
 use std::time::{Duration, Instant};
+use tun::Tun;
 
 const EXPIRE_RESOURCES_INTERVAL: Duration = Duration::from_secs(1);
 
 impl GatewayTunnel {
-    pub fn set_tun(&mut self, tun: Tun) {
+    pub fn set_tun(&mut self, tun: Box<dyn Tun>) {
         self.io.device_mut().set_tun(tun);
     }
 

rust/connlib/tunnel/src/lib.rs

@@ -3,14 +3,16 @@
 //! This is both the wireguard and ICE implementation that should work in tandem.
 //! [Tunnel] is the main entry-point for this crate.
 
+use bimap::BiMap;
 use boringtun::x25519::StaticSecret;
 use chrono::Utc;
 use connlib_shared::{
     callbacks,
     messages::{ClientId, GatewayId, Relay, RelayId, ResourceId, ReuseConnection},
-    DomainName, Result,
+    DomainName, Result, DEFAULT_MTU,
 };
 use io::Io;
+use ip_network::{Ipv4Network, Ipv6Network};
 use std::{
     collections::{BTreeSet, HashMap, HashSet},
     net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr},
@@ -18,10 +20,7 @@ use std::{
     task::{Context, Poll},
     time::Instant,
 };
-
-use bimap::BiMap;
-pub use client::{ClientState, Request};
-pub use gateway::GatewayState;
+use tun::Tun;
 use utils::turn;
 
 mod client;
@@ -34,20 +33,18 @@ mod peer_store;
 mod sockets;
 mod utils;
 
-pub use device_channel::Tun;
-use ip_network::{Ipv4Network, Ipv6Network};
-
 #[cfg(all(test, feature = "proptest"))]
 mod tests;
 
 const MAX_UDP_SIZE: usize = (1 << 16) - 1;
-const MTU: usize = 1280;
-
 const REALM: &str = "firezone";
 
 pub type GatewayTunnel = Tunnel<GatewayState>;
 pub type ClientTunnel = Tunnel<ClientState>;
 
+pub use client::{ClientState, Request};
+pub use gateway::GatewayState;
+
 /// [`Tunnel`] glues together connlib's [`Io`] component and the respective (pure) state of a client or gateway.
 ///
 /// Most of connlib's functionality is implemented as a pure state machine in [`ClientState`] and [`GatewayState`].
@@ -65,9 +62,9 @@ pub struct Tunnel<TRoleState> {
     ip6_read_buf: Box<[u8; MAX_UDP_SIZE]>,
 
     // We need an extra 16 bytes on top of the MTU for write_buf since boringtun copies the extra AEAD tag before decrypting it
-    write_buf: Box<[u8; MTU + 16 + 20]>,
+    write_buf: Box<[u8; DEFAULT_MTU + 16 + 20]>,
     // We have 20 extra bytes to be able to convert between ipv4 and ipv6
-    device_read_buf: Box<[u8; MTU + 20]>,
+    device_read_buf: Box<[u8; DEFAULT_MTU + 20]>,
 }
 
 impl ClientTunnel {
@@ -80,10 +77,10 @@ impl ClientTunnel {
         Ok(Self {
             io: Io::new(tcp_socket_factory, udp_socket_factory)?,
             role_state: ClientState::new(private_key, known_hosts),
-            write_buf: Box::new([0u8; MTU + 16 + 20]),
+            write_buf: Box::new([0u8; DEFAULT_MTU + 16 + 20]),
             ip4_read_buf: Box::new([0u8; MAX_UDP_SIZE]),
             ip6_read_buf: Box::new([0u8; MAX_UDP_SIZE]),
-            device_read_buf: Box::new([0u8; MTU + 20]),
+            device_read_buf: Box::new([0u8; DEFAULT_MTU + 20]),
         })
     }
 
@@ -174,10 +171,10 @@ impl GatewayTunnel {
         Ok(Self {
             io: Io::new(Arc::new(socket_factory::tcp), Arc::new(socket_factory::udp))?,
             role_state: GatewayState::new(private_key),
-            write_buf: Box::new([0u8; MTU + 20 + 16]),
+            write_buf: Box::new([0u8; DEFAULT_MTU + 20 + 16]),
             ip4_read_buf: Box::new([0u8; MAX_UDP_SIZE]),
             ip6_read_buf: Box::new([0u8; MAX_UDP_SIZE]),
-            device_read_buf: Box::new([0u8; MTU + 20]),
+            device_read_buf: Box::new([0u8; DEFAULT_MTU + 20]),
         })
     }
 

rust/gateway/src/main.rs

@@ -4,7 +4,7 @@ use backoff::ExponentialBackoffBuilder;
 use clap::Parser;
 use connlib_shared::{get_user_agent, keypair, messages::Interface, LoginUrl, StaticSecret};
 use firezone_bin_shared::{setup_global_subscriber, CommonArgs, TunDeviceManager};
-use firezone_tunnel::{GatewayTunnel, Tun};
+use firezone_tunnel::GatewayTunnel;
 
 use futures::channel::mpsc;
 use futures::{future, StreamExt, TryFutureExt};
@@ -113,8 +113,9 @@ async fn run(login: LoginUrl, private_key: StaticSecret) -> Result<Infallible> {
     )?;
 
     let (sender, receiver) = mpsc::channel::<Interface>(10);
-    let tun_device_manager = TunDeviceManager::new()?;
-    tunnel.set_tun(Tun::new()?);
+    let mut tun_device_manager = TunDeviceManager::new()?;
+    let tun = tun_device_manager.make_tun()?;
+    tunnel.set_tun(Box::new(tun));
 
     let update_device_task = update_device_task(tun_device_manager, receiver);
 

rust/headless-client/src/ipc_service.rs

@@ -369,7 +369,8 @@ impl<'a> Handler<'a> {
 
                 let new_session =
                     Session::connect(args, portal, tokio::runtime::Handle::try_current()?);
-                new_session.set_tun(self.tun_device.make_tun()?);
+                let tun = self.tun_device.make_tun()?;
+                new_session.set_tun(Box::new(tun));
                 new_session.set_dns(dns_control::system_resolvers().unwrap_or_default());
                 self.connlib = Some(new_session);
             }

rust/headless-client/src/standalone.rs

@@ -197,7 +197,8 @@ pub fn run_only_headless_client() -> Result<()> {
         let mut tun_device = TunDeviceManager::new()?;
         let mut cb_rx = ReceiverStream::new(cb_rx).fuse();
 
-        session.set_tun(tun_device.make_tun()?);
+        let tun = tun_device.make_tun()?;
+        session.set_tun(Box::new(tun));
         // TODO: DNS should be added dynamically
         session.set_dns(dns_control::system_resolvers().unwrap_or_default());
 

rust/tun/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "tun"
+version = "0.1.0"
+edition = "2021"
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+libc = "0.2"
+tokio = { workspace = true }
+
+[lints]
+workspace = true

rust/tun/src/ioctl.rs

@@ -0,0 +1,80 @@
+use std::{io, os::fd::RawFd};
+
+/// Executes the `ioctl` syscall on the given file descriptor with the provided request.
+///
+/// # Safety
+///
+/// The file descriptor must be open.
+pub unsafe fn exec<P>(fd: RawFd, code: libc::c_ulong, req: &mut Request<P>) -> io::Result<()> {
+    let ret = unsafe { libc::ioctl(fd, code as _, req) };
+
+    if ret < 0 {
+        return Err(io::Error::last_os_error());
+    }
+
+    Ok(())
+}
+
+/// Represents a control request to an IO device, addresses by the device's name.
+///
+/// The payload MUST also be `#[repr(C)]` and its layout depends on the particular request you are sending.
+#[repr(C)]
+pub struct Request<P> {
+    name: [std::ffi::c_uchar; libc::IF_NAMESIZE],
+    payload: P,
+}
+
+#[cfg(target_os = "linux")]
+impl Request<SetTunFlagsPayload> {
+    pub fn new(name: &str) -> Self {
+        let name_as_bytes = name.as_bytes();
+        debug_assert!(name_as_bytes.len() < libc::IF_NAMESIZE);
+
+        let mut name = [0u8; libc::IF_NAMESIZE];
+        name[..name_as_bytes.len()].copy_from_slice(name_as_bytes);
+
+        Self {
+            name,
+            payload: SetTunFlagsPayload {
+                flags: (libc::IFF_TUN | libc::IFF_NO_PI) as _,
+            },
+        }
+    }
+}
+
+impl Request<GetInterfaceNamePayload> {
+    pub fn new() -> Self {
+        Self {
+            name: [0u8; libc::IF_NAMESIZE],
+            payload: Default::default(),
+        }
+    }
+
+    pub fn name(&self) -> std::borrow::Cow<'_, str> {
+        // Safety: The memory of `self.name` is always initialized.
+        let cstr = unsafe { std::ffi::CStr::from_ptr(self.name.as_ptr() as _) };
+
+        cstr.to_string_lossy()
+    }
+}
+
+impl Default for Request<GetInterfaceNamePayload> {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[cfg(target_os = "linux")]
+#[repr(C)]
+pub struct SetTunFlagsPayload {
+    flags: std::ffi::c_short,
+}
+
+#[derive(Default)]
+#[repr(C)]
+pub struct GetInterfaceNamePayload {
+    // Fixes a nasty alignment bug on 32-bit architectures on Android.
+    // The `name` field in `ioctl::Request` is only 16 bytes long and accessing it causes a NPE without this alignment.
+    // Why? Not sure. It seems to only happen in release mode which hints at an optimisation issue.
+    alignment: [std::ffi::c_uchar; 16],
+}

rust/tun/src/lib.rs

@@ -0,0 +1,16 @@
+use std::{
+    io,
+    task::{Context, Poll},
+};
+
+#[cfg(target_family = "unix")]
+pub mod ioctl;
+#[cfg(target_family = "unix")]
+pub mod unix;
+
+pub trait Tun: Send + Sync + 'static {
+    fn write4(&self, buf: &[u8]) -> io::Result<usize>;
+    fn write6(&self, buf: &[u8]) -> io::Result<usize>;
+    fn poll_read(&mut self, buf: &mut [u8], cx: &mut Context<'_>) -> Poll<io::Result<usize>>;
+    fn name(&self) -> &str;
+}