From d07e32d91fab9ca4d166ae90f0874338a5006bdf Mon Sep 17 00:00:00 2001 From: Mariusz Klochowicz Date: Tue, 2 Sep 2025 16:25:53 +0930 Subject: [PATCH] chore: Build whole workspace on macos (#10228) - Add some macos stubs to gui-smoke-test. - Hide `ebpf-turn-router` binary functionality behind `#[cfg(target_arch = "bpf")]` Signed-off-by: Mariusz Klochowicz --- .github/actions/setup-rust/action.yml | 2 +- rust/relay/ebpf-turn-router/Cargo.toml | 5 + rust/relay/ebpf-turn-router/build.rs | 9 +- rust/relay/ebpf-turn-router/src/main.rs | 1488 +--------------- .../ebpf-turn-router/src/try_handle_turn.rs | 1497 +++++++++++++++++ rust/tests/gui-smoke-test/src/main.rs | 27 + 6 files changed, 1561 insertions(+), 1467 deletions(-) create mode 100644 rust/relay/ebpf-turn-router/src/try_handle_turn.rs diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml index 2ca8ad4ee..c0cd890d1 100644 --- a/.github/actions/setup-rust/action.yml +++ b/.github/actions/setup-rust/action.yml @@ -20,7 +20,7 @@ outputs: description: Compilable packages for the current OS value: ${{ (runner.os == 'Linux' && '--workspace') || - (runner.os == 'macOS' && '--workspace --exclude ebpf-turn-router --exclude gui-smoke-test --exclude client-ffi') || + (runner.os == 'macOS' && '--workspace') || (runner.os == 'Windows' && '--workspace --exclude ebpf-turn-router --exclude apple-client-ffi --exclude client-ffi') }} test-packages: description: Testable packages for the current OS diff --git a/rust/relay/ebpf-turn-router/Cargo.toml b/rust/relay/ebpf-turn-router/Cargo.toml index 2e2912212..249c07c96 100644 --- a/rust/relay/ebpf-turn-router/Cargo.toml +++ b/rust/relay/ebpf-turn-router/Cargo.toml @@ -4,6 +4,11 @@ version = "0.1.0" edition = { workspace = true } license = { workspace = true } +[package.metadata.cargo-udeps.ignore] +normal = ["aya-ebpf", "aya-log-ebpf", "ebpf-shared", "network-types"] +build = ["which"] +development = ["hex-literal", "ip-packet"] + [[bin]] name = "ebpf-turn-router-main" # This needs to be different from the package name otherwise the build-script fails to differentiate between the directory it is built in and the actual binary. path = "src/main.rs" diff --git a/rust/relay/ebpf-turn-router/build.rs b/rust/relay/ebpf-turn-router/build.rs index 3655e0aba..1bea27eb7 100644 --- a/rust/relay/ebpf-turn-router/build.rs +++ b/rust/relay/ebpf-turn-router/build.rs @@ -1,5 +1,3 @@ -use which::which; - /// Building this crate has an undeclared dependency on the `bpf-linker` binary. This would be /// better expressed by [artifact-dependencies][bindeps] but issues such as /// https://github.com/rust-lang/cargo/issues/12385 make their use impractical for the time being. @@ -11,7 +9,14 @@ use which::which; /// which would likely mean far too much cache invalidation. /// /// [bindeps]: https://doc.rust-lang.org/nightly/cargo/reference/unstable.html?highlight=feature#artifact-dependencies +#[cfg(target_os = "linux")] fn main() { + use which::which; let bpf_linker = which("bpf-linker").expect("bpf-linker not found in $PATH"); println!("cargo:rerun-if-changed={}", bpf_linker.to_str().unwrap()); } + +#[cfg(not(target_os = "linux"))] +fn main() { + println!("cargo:warning=eBPF is only supported on Linux, skipping bpf-linker check"); +} diff --git a/rust/relay/ebpf-turn-router/src/main.rs b/rust/relay/ebpf-turn-router/src/main.rs index 66e0a8fb2..5e7475300 100644 --- a/rust/relay/ebpf-turn-router/src/main.rs +++ b/rust/relay/ebpf-turn-router/src/main.rs @@ -1,81 +1,38 @@ #![cfg_attr(target_arch = "bpf", no_std)] #![cfg_attr(target_arch = "bpf", no_main)] -use crate::error::Error; -use aya_ebpf::{ - bindings::xdp_action, - helpers::bpf_xdp_adjust_head, - macros::{map, xdp}, - maps::{HashMap, PerCpuArray}, - programs::XdpContext, -}; -use aya_log_ebpf::*; -use channel_data::CdHdr; -use checksum::ChecksumUpdate; -use core::net::{Ipv4Addr, Ipv6Addr}; -use ebpf_shared::{ - ClientAndChannelV4, ClientAndChannelV6, InterfaceAddressV4, InterfaceAddressV6, PortAndPeerV4, - PortAndPeerV6, -}; -use error::SupportedChannel; -use network_types::{ - eth::{EthHdr, EtherType}, - ip::{IpProto, Ipv4Hdr, Ipv6Hdr}, - udp::UdpHdr, -}; -use ref_mut_at::ref_mut_at; +// For non-BPF targets: provide a stub main that exits with an error +#[cfg(not(target_arch = "bpf"))] +fn main() { + eprintln!("Error: This program is meant to be compiled as an eBPF program."); + eprintln!("Use --target bpf or --target bpfel-unknown-none to compile for eBPF."); + std::process::exit(1); +} +// Include modules only for BPF target +#[cfg(target_arch = "bpf")] mod channel_data; +#[cfg(target_arch = "bpf")] mod checksum; +#[cfg(target_arch = "bpf")] mod error; +#[cfg(target_arch = "bpf")] mod ref_mut_at; +#[cfg(target_arch = "bpf")] mod stats; +#[cfg(target_arch = "bpf")] +mod try_handle_turn; -const NUM_ENTRIES: u32 = 0x10000; -const LOWER_PORT: u16 = 49152; // Lower bound for TURN UDP ports -const UPPER_PORT: u16 = 65535; // Upper bound for TURN UDP ports -const CHAN_START: u16 = 0x4000; // Channel number start -const CHAN_END: u16 = 0x7FFF; // Channel number end +// Everything below is only for BPF target +#[cfg(target_arch = "bpf")] +#[aya_ebpf::macros::xdp] +// Per-CPU data structures to learn relay interface addresses +pub fn handle_turn(ctx: aya_ebpf::programs::XdpContext) -> u32 { + use aya_ebpf::bindings::xdp_action; + use aya_log_ebpf::{debug, warn}; + use error::Error; -// SAFETY: Testing has shown that these maps are safe to use as long as we aren't -// writing to them from multiple threads at the same time. Since we only update these -// from the single-threaded eventloop in userspace, we are ok. -// See https://github.com/firezone/firezone/issues/10138#issuecomment-3186074350 - -#[map] -static CHAN_TO_UDP_44: HashMap = - HashMap::with_max_entries(NUM_ENTRIES, 0); -#[map] -static UDP_TO_CHAN_44: HashMap = - HashMap::with_max_entries(NUM_ENTRIES, 0); -#[map] -static CHAN_TO_UDP_66: HashMap = - HashMap::with_max_entries(NUM_ENTRIES, 0); -#[map] -static UDP_TO_CHAN_66: HashMap = - HashMap::with_max_entries(NUM_ENTRIES, 0); -#[map] -static CHAN_TO_UDP_46: HashMap = - HashMap::with_max_entries(NUM_ENTRIES, 0); -#[map] -static UDP_TO_CHAN_46: HashMap = - HashMap::with_max_entries(NUM_ENTRIES, 0); -#[map] -static CHAN_TO_UDP_64: HashMap = - HashMap::with_max_entries(NUM_ENTRIES, 0); -#[map] -static UDP_TO_CHAN_64: HashMap = - HashMap::with_max_entries(NUM_ENTRIES, 0); - -// Per-CPU data structures to store relay interface addresses (configured from userspace) -#[map] -static INT_ADDR_V4: PerCpuArray = PerCpuArray::with_max_entries(1, 0); -#[map] -static INT_ADDR_V6: PerCpuArray = PerCpuArray::with_max_entries(1, 0); - -#[xdp] -pub fn handle_turn(ctx: XdpContext) -> u32 { - try_handle_turn(&ctx).unwrap_or_else(|e| match e { + try_handle_turn::try_handle_turn(&ctx).unwrap_or_else(|e| match e { Error::NotIp | Error::NotUdp => xdp_action::XDP_PASS, Error::InterfaceIpv4AddressAccessFailed @@ -107,1369 +64,6 @@ pub fn handle_turn(ctx: XdpContext) -> u32 { }) } -#[inline(always)] -fn try_handle_turn(ctx: &XdpContext) -> Result { - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let eth = unsafe { ref_mut_at::(ctx, 0)? }; - - match eth.ether_type { - EtherType::Ipv4 => try_handle_turn_ipv4(ctx)?, - EtherType::Ipv6 => try_handle_turn_ipv6(ctx)?, - _ => return Err(Error::NotIp), - }; - - // If we get to here, we modified the packet and need to send it back out again. - Ok(xdp_action::XDP_TX) -} - -#[inline(always)] -fn try_handle_turn_ipv4(ctx: &XdpContext) -> Result<(), Error> { - // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. - let ipv4 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; - - if ipv4.proto != IpProto::Udp { - return Err(Error::NotUdp); - } - - if ipv4.ihl() != 5 { - // IPv4 with options is not supported - return Err(Error::Ipv4PacketWithOptions); - } - - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN)? }; - let udp_payload_len = udp.len() - UdpHdr::LEN as u16; - - trace!( - ctx, - "New packet from {:i}:{} for {:i}:{} with UDP payload {}", - ipv4.src_addr(), - udp.source(), - ipv4.dst_addr(), - udp.dest(), - udp_payload_len - ); - - if (LOWER_PORT..=UPPER_PORT).contains(&udp.dest()) { - try_handle_ipv4_udp_to_channel_data(ctx)?; - stats::emit_data_relayed(ctx, udp_payload_len); - - return Ok(()); - } - - if udp.dest() == 3478 { - try_handle_ipv4_channel_data_to_udp(ctx)?; - stats::emit_data_relayed(ctx, udp_payload_len - CdHdr::LEN as u16); - - return Ok(()); - } - - Err(Error::NotTurn) -} - -#[inline(always)] -fn try_handle_turn_ipv6(ctx: &XdpContext) -> Result<(), Error> { - // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. - let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; - - if ipv6.next_hdr != IpProto::Udp { - return Err(Error::NotUdp); - } - - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; - let udp_payload_len = udp.len() - UdpHdr::LEN as u16; - - trace!( - ctx, - "New packet from {:i}:{} for {:i}:{} with UDP payload {}", - ipv6.src_addr(), - udp.source(), - ipv6.dst_addr(), - udp.dest(), - udp_payload_len - ); - - if (LOWER_PORT..=UPPER_PORT).contains(&udp.dest()) { - try_handle_ipv6_udp_to_channel_data(ctx)?; - stats::emit_data_relayed(ctx, udp_payload_len); - - return Ok(()); - } - - if udp.dest() == 3478 { - try_handle_ipv6_channel_data_to_udp(ctx)?; - stats::emit_data_relayed(ctx, udp_payload_len - CdHdr::LEN as u16); - - return Ok(()); - } - - Err(Error::NotTurn) -} - -#[inline(always)] -fn try_handle_ipv4_udp_to_channel_data(ctx: &XdpContext) -> Result<(), Error> { - // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. - let ipv4 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; - - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN)? }; - - let key = PortAndPeerV4::new(ipv4.src_addr(), udp.dest(), udp.source()); - - // SAFETY: We only write to these using a single thread in userspace. - if let Some(client_and_channel) = unsafe { UDP_TO_CHAN_44.get(&key) } { - handle_ipv4_udp_to_ipv4_channel(ctx, client_and_channel)?; - return Ok(()); - } - - // SAFETY: We only write to these using a single thread in userspace. - if let Some(client_and_channel) = unsafe { UDP_TO_CHAN_46.get(&key) } { - handle_ipv4_udp_to_ipv6_channel(ctx, client_and_channel)?; - return Ok(()); - } - - Err(Error::NoEntry(SupportedChannel::Udp4ToChan)) -} - -#[inline(always)] -fn try_handle_ipv4_channel_data_to_udp(ctx: &XdpContext) -> Result<(), Error> { - // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. - let ipv4 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; - - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN)? }; - - // SAFETY: The offset must point to the start of a valid `CdHdr`. - let cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN + UdpHdr::LEN)? }; - - let channel_number = u16::from_be_bytes(cd.number); - - if !(CHAN_START..=CHAN_END).contains(&channel_number) { - return Err(Error::NotAChannelDataMessage); - } - - let channel_data_len = u16::from_be_bytes(cd.length); - let expected_channel_data_len = udp.len() - UdpHdr::LEN as u16 - CdHdr::LEN as u16; - - // This can happen if we receive packets formed via GSO, like on a loopback interface. - if channel_data_len != expected_channel_data_len { - return Err(Error::BadChannelDataLength); - } - - let key = ClientAndChannelV4::new(ipv4.src_addr(), udp.source(), channel_number); - - // SAFETY: We only write to these using a single thread in userspace. - if let Some(port_and_peer) = unsafe { CHAN_TO_UDP_44.get(&key) } { - // IPv4 to IPv4 - existing logic - handle_ipv4_channel_to_ipv4_udp(ctx, port_and_peer)?; - return Ok(()); - } - - // SAFETY: We only write to these using a single thread in userspace. - if let Some(port_and_peer) = unsafe { CHAN_TO_UDP_46.get(&key) } { - handle_ipv4_channel_to_ipv6_udp(ctx, port_and_peer)?; - return Ok(()); - } - - Err(Error::NoEntry(SupportedChannel::Chan4ToUdp)) -} - -#[inline(always)] -fn try_handle_ipv6_udp_to_channel_data(ctx: &XdpContext) -> Result<(), Error> { - // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. - let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; - - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; - - let key = PortAndPeerV6::new(ipv6.src_addr(), udp.dest(), udp.source()); - - // SAFETY: We only write to these using a single thread in userspace. - if let Some(client_and_channel) = unsafe { UDP_TO_CHAN_66.get(&key) } { - handle_ipv6_udp_to_ipv6_channel(ctx, client_and_channel)?; - return Ok(()); - } - - // SAFETY: We only write to these using a single thread in userspace. - if let Some(client_and_channel) = unsafe { UDP_TO_CHAN_64.get(&key) } { - handle_ipv6_udp_to_ipv4_channel(ctx, client_and_channel)?; - return Ok(()); - } - - Err(Error::NoEntry(SupportedChannel::Udp6ToChan)) -} - -#[inline(always)] -fn try_handle_ipv6_channel_data_to_udp(ctx: &XdpContext) -> Result<(), Error> { - // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. - let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; - - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; - - // SAFETY: The offset must point to the start of a valid `CdHdr`. - let cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN + UdpHdr::LEN)? }; - - let channel_number = u16::from_be_bytes(cd.number); - - if !(CHAN_START..=CHAN_END).contains(&channel_number) { - return Err(Error::NotAChannelDataMessage); - } - - let channel_data_len = u16::from_be_bytes(cd.length); - let expected_channel_data_len = udp.len() - UdpHdr::LEN as u16 - CdHdr::LEN as u16; - - // This can happen if we receive packets formed via GSO, like on a loopback interface. - if channel_data_len != expected_channel_data_len { - return Err(Error::BadChannelDataLength); - } - - let key = ClientAndChannelV6::new(ipv6.src_addr(), udp.source(), u16::from_be_bytes(cd.number)); - - // SAFETY: We only write to these using a single thread in userspace. - if let Some(port_and_peer) = unsafe { CHAN_TO_UDP_66.get(&key) } { - handle_ipv6_channel_to_ipv6_udp(ctx, port_and_peer)?; - return Ok(()); - } - - // SAFETY: We only write to these using a single thread in userspace. - if let Some(port_and_peer) = unsafe { CHAN_TO_UDP_64.get(&key) } { - handle_ipv6_channel_to_ipv4_udp(ctx, port_and_peer)?; - return Ok(()); - } - - Err(Error::NoEntry(SupportedChannel::Chan6ToUdp)) -} - -#[inline(always)] -fn handle_ipv4_udp_to_ipv4_channel( - ctx: &XdpContext, - client_and_channel: &ClientAndChannelV4, -) -> Result<(), Error> { - const NET_EXPANSION: i32 = -(CdHdr::LEN as i32); - - adjust_head(ctx, NET_EXPANSION)?; - - // Now read the old packet data from its NEW location (shifted by 4 bytes) - let old_data_offset = -NET_EXPANSION as usize; - - let (old_eth_src, old_eth_dst, old_eth_type) = { - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let old_eth = unsafe { ref_mut_at::(ctx, old_data_offset)? }; - (old_eth.src_addr, old_eth.dst_addr, old_eth.ether_type) - }; - - let ( - old_ipv4_src, - old_ipv4_dst, - old_ipv4_len, - old_ipv4_check, - old_ipv4_tos, - old_ipv4_id, - old_ipv4_frag_off, - old_ipv4_ttl, - old_ipv4_proto, - ) = { - // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. - let old_ipv4 = unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN)? }; - ( - old_ipv4.src_addr(), - old_ipv4.dst_addr(), - old_ipv4.total_len(), - old_ipv4.checksum(), - old_ipv4.tos, - old_ipv4.id(), - old_ipv4.frag_off, - old_ipv4.ttl, - old_ipv4.proto, - ) - }; - - let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let old_udp = - unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN + Ipv4Hdr::LEN)? }; - ( - old_udp.len(), - old_udp.source(), - old_udp.dest(), - old_udp.check(), - ) - }; - - // - // 1. Ethernet header - // - - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let eth = unsafe { ref_mut_at::(ctx, 0)? }; - eth.src_addr = old_eth_dst; - eth.dst_addr = old_eth_src; - eth.ether_type = old_eth_type; - - // - // 2. IPv4 header - // - - let new_ipv4_src = old_ipv4_dst; - let new_ipv4_dst = client_and_channel.client_ip(); - let new_ipv4_len = old_ipv4_len + CdHdr::LEN as u16; - - // Check for packet loop - would we be sending to ourselves? - if new_ipv4_src == new_ipv4_dst { - return Err(Error::PacketLoop); - } - - // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. - let ipv4 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; - ipv4.set_version(4); // IPv4 - ipv4.set_ihl(5); // No options, 5 * 4 = 20 bytes - ipv4.tos = old_ipv4_tos; // Preserve TOS/DSCP - ipv4.set_total_len(new_ipv4_len); - ipv4.set_id(old_ipv4_id); // Preserve fragment ID - ipv4.frag_off = old_ipv4_frag_off; // Preserve fragment flags - ipv4.ttl = old_ipv4_ttl; // Preserve TTL exactly - ipv4.proto = old_ipv4_proto; // Protocol is UDP - ipv4.set_src_addr(new_ipv4_src); // Swap source and destination - ipv4.set_dst_addr(new_ipv4_dst); // Destination is the client IP - ipv4.set_checksum( - ChecksumUpdate::new(old_ipv4_check) - .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) - .remove_u16(old_ipv4_len) - .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) - .add_u16(new_ipv4_len) - .into_ip_checksum(), - ); - - // - // 3. UDP header - // - - let new_udp_src = 3478_u16; - let new_udp_dst = client_and_channel.client_port(); - let new_udp_len = old_udp_len + CdHdr::LEN as u16; - let channel_number = client_and_channel.channel(); - let channel_data_length = old_udp_len - UdpHdr::LEN as u16; - - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN)? }; - udp.set_source(new_udp_src); - udp.set_dest(new_udp_dst); - udp.set_len(new_udp_len); - - // Incrementally update UDP checksum - - if old_udp_check == 0 { - // No checksum is valid for UDP IPv4 - we didn't write it, but maybe a middlebox did - udp.set_check(0); - } else { - udp.set_check( - ChecksumUpdate::new(old_udp_check) - .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) - .remove_u16(old_udp_src) - .remove_u16(old_udp_dst) - .remove_u16(old_udp_len) - .remove_u16(old_udp_len) - .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) - .add_u16(new_udp_src) - .add_u16(new_udp_dst) - .add_u16(new_udp_len) - .add_u16(new_udp_len) - .add_u16(channel_number) - .add_u16(channel_data_length) - .into_udp_checksum(), - ); - } - - // - // 4. Channel data header - // - - // SAFETY: The offset must point to the start of a valid `CdHdr`. - let cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN + UdpHdr::LEN)? }; - cd.number = channel_number.to_be_bytes(); - cd.length = channel_data_length.to_be_bytes(); - - Ok(()) -} - -// Convert IPv4 to IPv6 and add channel data -#[inline(always)] -fn handle_ipv4_udp_to_ipv6_channel( - ctx: &XdpContext, - client_and_channel: &ClientAndChannelV6, -) -> Result<(), Error> { - // Expand the packet by 24 bytes for IPv6 header and channel data header - const NET_EXPANSION: i32 = -(Ipv6Hdr::LEN as i32 - Ipv4Hdr::LEN as i32 + CdHdr::LEN as i32); - - adjust_head(ctx, NET_EXPANSION)?; - - // Now read the old packet data from its NEW location (shifted by 24 bytes) - let old_data_offset = -NET_EXPANSION as usize; - - let (old_eth_src, old_eth_dst) = { - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let old_eth = unsafe { ref_mut_at::(ctx, old_data_offset)? }; - (old_eth.src_addr, old_eth.dst_addr) - }; - - let (old_ipv4_src, old_ipv4_dst, old_ipv4_len, old_ipv4_tos, old_ipv4_ttl, old_ipv4_proto) = { - // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. - let old_ipv4 = unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN)? }; - ( - old_ipv4.src_addr(), - old_ipv4.dst_addr(), - old_ipv4.total_len(), - old_ipv4.tos, - old_ipv4.ttl, - old_ipv4.proto, - ) - }; - - let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let old_udp = - unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN + Ipv4Hdr::LEN)? }; - ( - old_udp.len(), - old_udp.source(), - old_udp.dest(), - old_udp.check(), - ) - }; - - // Refuse to compute full UDP checksum. - // We forged these packets, so something's wrong if this is zero. - if old_udp_check == 0 { - return Err(Error::UdpChecksumMissing); - } - - // - // 1. Ethernet header - // - - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let eth = unsafe { ref_mut_at::(ctx, 0)? }; - eth.dst_addr = old_eth_src; // Swap source and destination - eth.src_addr = old_eth_dst; - eth.ether_type = EtherType::Ipv6; // Change to IPv6 - - // - // 2. IPv4 -> IPv6 header - // - - let new_ipv6_src = get_interface_ipv6_address()?; - let new_ipv6_dst = client_and_channel.client_ip(); - let new_ipv6_len = old_ipv4_len - Ipv4Hdr::LEN as u16 + CdHdr::LEN as u16; - - // Check for packet loop - would we be sending to ourselves? - if new_ipv6_dst == new_ipv6_src { - return Err(Error::PacketLoop); - } - - // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. - let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; - ipv6.set_version(6); - ipv6.set_priority(old_ipv4_tos); - ipv6.flow_label = [0, 0, 0]; // Default flow label - ipv6.set_payload_len(new_ipv6_len); - ipv6.next_hdr = old_ipv4_proto; - ipv6.hop_limit = old_ipv4_ttl; - ipv6.set_src_addr(new_ipv6_src); - ipv6.set_dst_addr(new_ipv6_dst); - - // - // 3. UDP header - // - - let new_udp_src = 3478_u16; - let new_udp_dst = client_and_channel.client_port(); - let new_udp_len = old_udp_len + CdHdr::LEN as u16; - - let channel_number = client_and_channel.channel(); - let channel_data_length = old_udp_len - UdpHdr::LEN as u16; - - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; - udp.set_source(new_udp_src); - udp.set_dest(new_udp_dst); - udp.set_len(new_udp_len); - - // Incrementally update UDP checksum - - udp.set_check( - ChecksumUpdate::new(old_udp_check) - .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) - .remove_u32(u32::from_be_bytes(old_ipv4_dst.octets())) - .remove_u16(old_udp_src) - .remove_u16(old_udp_dst) - .remove_u16(old_udp_len) - .remove_u16(old_udp_len) - .add_u128(u128::from_be_bytes(new_ipv6_src.octets())) - .add_u128(u128::from_be_bytes(new_ipv6_dst.octets())) - .add_u16(new_udp_src) - .add_u16(new_udp_dst) - .add_u16(new_udp_len) - .add_u16(new_udp_len) - .add_u16(channel_number) - .add_u16(channel_data_length) - .into_udp_checksum(), - ); - - // - // 4. Channel data header - // - - // SAFETY: The offset must point to the start of a valid `CdHdr`. - let cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN + UdpHdr::LEN)? }; - cd.number = channel_number.to_be_bytes(); - cd.length = channel_data_length.to_be_bytes(); - - Ok(()) -} - -#[inline(always)] -fn handle_ipv4_channel_to_ipv4_udp( - ctx: &XdpContext, - port_and_peer: &PortAndPeerV4, -) -> Result<(), Error> { - const NET_SHRINK: i32 = CdHdr::LEN as i32; // Shrink by 4 bytes for channel data header - - let (old_eth_src, old_eth_dst, old_eth_type) = { - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let old_eth = unsafe { ref_mut_at::(ctx, 0)? }; - (old_eth.src_addr, old_eth.dst_addr, old_eth.ether_type) - }; - - let ( - old_ipv4_src, - old_ipv4_dst, - old_ipv4_len, - old_ipv4_check, - old_ipv4_tos, - old_ipv4_id, - old_ipv4_frag_off, - old_ipv4_ttl, - old_ipv4_proto, - ) = { - // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. - let old_ipv4 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; - ( - old_ipv4.src_addr(), - old_ipv4.dst_addr(), - old_ipv4.total_len(), - old_ipv4.checksum(), - old_ipv4.tos, - old_ipv4.id(), - old_ipv4.frag_off, - old_ipv4.ttl, - old_ipv4.proto, - ) - }; - - let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let old_udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN)? }; - ( - old_udp.len(), - old_udp.source(), - old_udp.dest(), - old_udp.check(), - ) - }; - - let (channel_number, channel_data_length) = { - // SAFETY: The offset must point to the start of a valid `CdHdr`. - let old_cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN + UdpHdr::LEN)? }; - ( - u16::from_be_bytes(old_cd.number), - u16::from_be_bytes(old_cd.length), - ) - }; - - // - // 1. Ethernet header - // - - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let eth = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize)? }; - eth.dst_addr = old_eth_src; // Swap source and destination - eth.src_addr = old_eth_dst; - eth.ether_type = old_eth_type; - - // - // 2. IPv4 header - // - - let new_ipv4_src = old_ipv4_dst; // Swap source and destination - let new_ipv4_dst = port_and_peer.peer_ip(); - let new_ipv4_len = old_ipv4_len - CdHdr::LEN as u16; - - // Check for packet loop - would we be sending to ourselves? - if new_ipv4_src == new_ipv4_dst { - return Err(Error::PacketLoop); - } - - // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. - let ipv4 = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN)? }; - ipv4.set_version(4); // IPv4 - ipv4.set_ihl(5); // No options, 5 * 4 = 20 bytes - ipv4.tos = old_ipv4_tos; // Preserve TOS/DSCP - ipv4.set_total_len(new_ipv4_len); - ipv4.set_id(old_ipv4_id); // Preserve ID - ipv4.frag_off = old_ipv4_frag_off; // Preserve fragment flags - ipv4.ttl = old_ipv4_ttl; // Preserve TTL exactly - ipv4.proto = old_ipv4_proto; // Protocol is UDP - ipv4.set_src_addr(new_ipv4_src); - ipv4.set_dst_addr(new_ipv4_dst); - ipv4.set_checksum( - ChecksumUpdate::new(old_ipv4_check) - .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) - .remove_u16(old_ipv4_len) - .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) - .add_u16(new_ipv4_len) - .into_ip_checksum(), - ); - - // - // 3. UDP header - // - - let new_udp_src = port_and_peer.allocation_port(); - let new_udp_dst = port_and_peer.peer_port(); - let new_udp_len = old_udp_len - CdHdr::LEN as u16; - - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let udp = - unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN + Ipv4Hdr::LEN)? }; - udp.set_source(new_udp_src); - udp.set_dest(new_udp_dst); - udp.set_len(new_udp_len); - - // Incrementally update UDP checksum - - if old_udp_check == 0 { - // No checksum is valid for UDP IPv4 - we didn't write it, but maybe a middlebox did - udp.set_check(0); - } else { - udp.set_check( - ChecksumUpdate::new(old_udp_check) - .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) - .remove_u16(old_udp_src) - .remove_u16(old_udp_dst) - .remove_u16(old_udp_len) - .remove_u16(old_udp_len) - .remove_u16(channel_number) - .remove_u16(channel_data_length) - .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) - .add_u16(new_udp_src) - .add_u16(new_udp_dst) - .add_u16(new_udp_len) - .add_u16(new_udp_len) - .into_udp_checksum(), - ); - } - - adjust_head(ctx, NET_SHRINK)?; - - Ok(()) -} - -// Convert IPv4 to IPv6 and remove channel data header -#[inline(always)] -fn handle_ipv4_channel_to_ipv6_udp( - ctx: &XdpContext, - port_and_peer: &PortAndPeerV6, -) -> Result<(), Error> { - const NET_EXPANSION: i32 = Ipv4Hdr::LEN as i32 - Ipv6Hdr::LEN as i32 + CdHdr::LEN as i32; - - adjust_head(ctx, NET_EXPANSION)?; - - // Now read the old packet data from its NEW location - let old_data_offset = (-NET_EXPANSION) as usize; - - let (old_src_mac, old_dst_mac) = { - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let old_eth = unsafe { ref_mut_at::(ctx, old_data_offset)? }; - (old_eth.src_addr, old_eth.dst_addr) - }; - - let (old_ipv4_src, old_ipv4_dst, old_ipv4_tos, old_ipv4_ttl, old_ipv4_proto) = { - // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. - let old_ipv4 = unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN)? }; - ( - old_ipv4.src_addr(), - old_ipv4.dst_addr(), - old_ipv4.tos, - old_ipv4.ttl, - old_ipv4.proto, - ) - }; - - let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let old_udp = - unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN + Ipv4Hdr::LEN)? }; - ( - old_udp.len(), - old_udp.source(), - old_udp.dest(), - old_udp.check(), - ) - }; - - // Refuse to compute full UDP checksum. - // We forged these packets, so something's wrong if this is zero. - if old_udp_check == 0 { - return Err(Error::UdpChecksumMissing); - } - - let (channel_number, channel_data_length) = { - // SAFETY: The offset must point to the start of a valid `CdHdr`. - let old_cd = unsafe { - ref_mut_at::( - ctx, - old_data_offset + EthHdr::LEN + Ipv4Hdr::LEN + UdpHdr::LEN, - )? - }; - ( - u16::from_be_bytes(old_cd.number), - u16::from_be_bytes(old_cd.length), - ) - }; - - // - // 1. Ethernet header - // - - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let eth = unsafe { ref_mut_at::(ctx, 0)? }; - eth.dst_addr = old_src_mac; // Swap MACs - eth.src_addr = old_dst_mac; - eth.ether_type = EtherType::Ipv6; // Change to IPv6 - - // - // 2. IPv6 header - // - - let new_ipv6_src = get_interface_ipv6_address()?; - let new_ipv6_dst = port_and_peer.peer_ip(); - let new_udp_len = old_udp_len - CdHdr::LEN as u16; - - // Check for packet loop - would we be sending to ourselves? - if new_ipv6_src == new_ipv6_dst { - return Err(Error::PacketLoop); - } - - // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. - let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; - ipv6.set_version(6); // IPv6 - ipv6.set_priority(old_ipv4_tos); - ipv6.flow_label = [0, 0, 0]; - ipv6.set_payload_len(new_udp_len); - ipv6.next_hdr = old_ipv4_proto; - ipv6.hop_limit = old_ipv4_ttl; - ipv6.set_src_addr(new_ipv6_src); - ipv6.set_dst_addr(new_ipv6_dst); - - // - // 3. UDP header - // - - let new_udp_src = port_and_peer.allocation_port(); - let new_udp_dst = port_and_peer.peer_port(); - - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; - udp.set_source(new_udp_src); - udp.set_dest(new_udp_dst); - udp.set_len(new_udp_len); - - // Incrementally update UDP checksum - - udp.set_check( - ChecksumUpdate::new(old_udp_check) - .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) - .remove_u32(u32::from_be_bytes(old_ipv4_dst.octets())) - .remove_u16(old_udp_src) - .remove_u16(old_udp_dst) - .remove_u16(old_udp_len) - .remove_u16(old_udp_len) - .remove_u16(channel_number) - .remove_u16(channel_data_length) - .add_u128(u128::from_be_bytes(new_ipv6_src.octets())) - .add_u128(u128::from_be_bytes(new_ipv6_dst.octets())) - .add_u16(new_udp_src) - .add_u16(new_udp_dst) - .add_u16(new_udp_len) - .add_u16(new_udp_len) - .into_udp_checksum(), - ); - - Ok(()) -} - -#[inline(always)] -fn handle_ipv6_udp_to_ipv6_channel( - ctx: &XdpContext, - client_and_channel: &ClientAndChannelV6, -) -> Result<(), Error> { - // Expand by 4 bytes for channel data header - const NET_EXPANSION: i32 = -(CdHdr::LEN as i32); - - adjust_head(ctx, NET_EXPANSION)?; - - // Now read the old packet data from its NEW location (shifted by 4 bytes) - let old_data_offset = CdHdr::LEN; - - let (old_eth_src, old_eth_dst, old_eth_type) = { - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let old_eth = unsafe { ref_mut_at::(ctx, old_data_offset)? }; - (old_eth.src_addr, old_eth.dst_addr, old_eth.ether_type) - }; - - let ( - old_ipv6_src, - old_ipv6_dst, - old_ipv6_len, - old_ipv6_priority, - old_ipv6_flow_label, - old_ipv6_hop_limit, - old_ipv6_next_hdr, - ) = { - // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. - let old_ipv6 = unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN)? }; - ( - old_ipv6.src_addr(), - old_ipv6.dst_addr(), - old_ipv6.payload_len(), - old_ipv6.priority(), - old_ipv6.flow_label, - old_ipv6.hop_limit, - old_ipv6.next_hdr, - ) - }; - - let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let old_udp = - unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN + Ipv6Hdr::LEN)? }; - ( - old_udp.len(), - old_udp.source(), - old_udp.dest(), - old_udp.check(), - ) - }; - - // - // 1. Ethernet header - // - - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let eth = unsafe { ref_mut_at::(ctx, 0)? }; - eth.src_addr = old_eth_dst; // Swap source and destination - eth.dst_addr = old_eth_src; - eth.ether_type = old_eth_type; - - // - // 2. IPv6 header - // - - let new_ipv6_src = old_ipv6_dst; - let new_ipv6_dst = client_and_channel.client_ip(); - let new_ipv6_len = old_ipv6_len + CdHdr::LEN as u16; - - // Check for packet loop - would we be sending to ourselves? - if new_ipv6_src == new_ipv6_dst { - return Err(Error::PacketLoop); - } - - // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. - let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; - // Set fields explicitly to avoid reading potentially corrupted memory - ipv6.set_version(6); // IPv6 - ipv6.set_priority(old_ipv6_priority); - ipv6.flow_label = old_ipv6_flow_label; - ipv6.set_payload_len(new_ipv6_len); - ipv6.next_hdr = old_ipv6_next_hdr; - ipv6.hop_limit = old_ipv6_hop_limit; - ipv6.set_src_addr(new_ipv6_src); - ipv6.set_dst_addr(new_ipv6_dst); - - // - // 3. UDP header - // - - let channel_number = client_and_channel.channel(); - let channel_data_length = old_udp_len - UdpHdr::LEN as u16; - let new_udp_len = old_udp_len + CdHdr::LEN as u16; - let new_udp_src = 3478_u16; - let new_udp_dst = client_and_channel.client_port(); - - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; - udp.set_source(new_udp_src); - udp.set_dest(new_udp_dst); - udp.set_len(new_udp_len); - - // Incrementally update UDP checksum - - udp.set_check( - ChecksumUpdate::new(old_udp_check) - .remove_u128(u128::from_be_bytes(old_ipv6_src.octets())) - .remove_u16(old_udp_src) - .remove_u16(old_udp_dst) - .remove_u16(old_udp_len) - .remove_u16(old_udp_len) - .add_u128(u128::from_be_bytes(new_ipv6_dst.octets())) - .add_u16(new_udp_src) - .add_u16(new_udp_dst) - .add_u16(new_udp_len) - .add_u16(new_udp_len) - .add_u16(channel_number) - .add_u16(channel_data_length) - .into_udp_checksum(), - ); - - // - // 4. Channel data header - // - - // SAFETY: The offset must point to the start of a valid `CdHdr`. - let cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN + UdpHdr::LEN)? }; - cd.number = channel_number.to_be_bytes(); - cd.length = channel_data_length.to_be_bytes(); - - Ok(()) -} - -// Convert IPv6 to IPv4 and add channel data -#[inline(always)] -fn handle_ipv6_udp_to_ipv4_channel( - ctx: &XdpContext, - client_and_channel: &ClientAndChannelV4, -) -> Result<(), Error> { - // 40 - 20 - 4 = 16 bytes shrink - const NET_SHRINK: i32 = Ipv6Hdr::LEN as i32 - Ipv4Hdr::LEN as i32 - CdHdr::LEN as i32; - - let (old_eth_src, old_eth_dst) = { - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let old_eth = unsafe { ref_mut_at::(ctx, 0)? }; - (old_eth.src_addr, old_eth.dst_addr) - }; - - let (old_ipv6_src, old_ipv6_dst, old_ipv6_priority, old_ipv6_hop_limit, old_ipv6_next_hdr) = { - // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. - let old_ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; - ( - old_ipv6.src_addr(), - old_ipv6.dst_addr(), - old_ipv6.priority(), - old_ipv6.hop_limit, - old_ipv6.next_hdr, - ) - }; - - let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let old_udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; - ( - old_udp.len(), - old_udp.source(), - old_udp.dest(), - old_udp.check(), - ) - }; - - // - // 1. Ethernet header - // - - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let eth = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize)? }; - eth.src_addr = old_eth_dst; // Swap source and destination - eth.dst_addr = old_eth_src; - eth.ether_type = EtherType::Ipv4; // Change to IPv4 - - // - // 2. IPv6 -> IPv4 header - // - - let new_ipv4_src = get_interface_ipv4_address()?; - let new_ipv4_dst = client_and_channel.client_ip(); - let new_udp_len = old_udp_len + CdHdr::LEN as u16; - let new_ipv4_len = Ipv4Hdr::LEN as u16 + new_udp_len; - - // Check for packet loop - would we be sending to ourselves? - if new_ipv4_dst == new_ipv4_src { - return Err(Error::PacketLoop); - } - - // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. - let ipv4 = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN)? }; - ipv4.set_version(4); - ipv4.set_ihl(5); // No options - ipv4.tos = old_ipv6_priority; - ipv4.set_total_len(new_ipv4_len); - ipv4.set_id(0); // Default ID - ipv4.frag_off = 0x4000_u16.to_be_bytes(); // Don't fragment - ipv4.ttl = old_ipv6_hop_limit; // Preserve hop limit - ipv4.proto = old_ipv6_next_hdr; // Preserve protocol - ipv4.set_src_addr(new_ipv4_src); - ipv4.set_dst_addr(new_ipv4_dst); - - // Calculate fresh checksum - let check = checksum::new_ipv4(ipv4); - ipv4.set_checksum(check); - - // - // 3. UDP header - // - - let new_udp_src = 3478_u16; // Fixed source port for TURN - let new_udp_dst = client_and_channel.client_port(); - let channel_number = client_and_channel.channel(); - let channel_data_length = old_udp_len - UdpHdr::LEN as u16; - - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let udp = - unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN + Ipv4Hdr::LEN)? }; - udp.set_source(new_udp_src); - udp.set_dest(new_udp_dst); - udp.set_len(new_udp_len); - - // Incrementally update UDP checksum - - udp.set_check( - ChecksumUpdate::new(old_udp_check) - .remove_u128(u128::from_be_bytes(old_ipv6_src.octets())) - .remove_u128(u128::from_be_bytes(old_ipv6_dst.octets())) - .remove_u16(old_udp_src) - .remove_u16(old_udp_dst) - .remove_u16(old_udp_len) - .remove_u16(old_udp_len) - .add_u32(u32::from_be_bytes(new_ipv4_src.octets())) - .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) - .add_u16(new_udp_src) - .add_u16(new_udp_dst) - .add_u16(new_udp_len) - .add_u16(new_udp_len) - .add_u16(channel_number) - .add_u16(channel_data_length) - .into_udp_checksum(), - ); - - // - // 4. Channel data header - // - - // SAFETY: The offset must point to the start of a valid `CdHdr`. - let cd = unsafe { - ref_mut_at::( - ctx, - NET_SHRINK as usize + EthHdr::LEN + Ipv4Hdr::LEN + UdpHdr::LEN, - )? - }; - cd.number = channel_number.to_be_bytes(); - cd.length = channel_data_length.to_be_bytes(); - - adjust_head(ctx, NET_SHRINK)?; - - Ok(()) -} - -#[inline(always)] -fn handle_ipv6_channel_to_ipv6_udp( - ctx: &XdpContext, - port_and_peer: &PortAndPeerV6, -) -> Result<(), Error> { - const NET_SHRINK: i32 = CdHdr::LEN as i32; // Shrink by 4 bytes for channel data header - - let (old_eth_src, old_eth_dst, old_eth_type) = { - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let old_eth = unsafe { ref_mut_at::(ctx, 0)? }; - (old_eth.src_addr, old_eth.dst_addr, old_eth.ether_type) - }; - - let ( - old_ipv6_src, - old_ipv6_dst, - old_ipv6_len, - old_ipv6_priority, - old_ipv6_flow_label, - old_ipv6_hop_limit, - old_ipv6_next_hdr, - ) = { - // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. - let old_ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; - ( - old_ipv6.src_addr(), - old_ipv6.dst_addr(), - old_ipv6.payload_len(), - old_ipv6.priority(), - old_ipv6.flow_label, - old_ipv6.hop_limit, - old_ipv6.next_hdr, - ) - }; - - let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let old_udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; - ( - old_udp.len(), - old_udp.source(), - old_udp.dest(), - old_udp.check(), - ) - }; - - let (channel_number, channel_data_length) = { - // SAFETY: The offset must point to the start of a valid `CdHdr`. - let old_cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN + UdpHdr::LEN)? }; - ( - u16::from_be_bytes(old_cd.number), - u16::from_be_bytes(old_cd.length), - ) - }; - - // - // 1. Ethernet header - // - - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let eth = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize)? }; - eth.src_addr = old_eth_dst; // Swap source and destination - eth.dst_addr = old_eth_src; - eth.ether_type = old_eth_type; - - // - // 2. IPv6 header - // - - let new_ipv6_src = old_ipv6_dst; // Swap source and destination - let new_ipv6_dst = port_and_peer.peer_ip(); - let new_ipv6_len = old_ipv6_len - CdHdr::LEN as u16; - - // Check for packet loop - would we be sending to ourselves? - if new_ipv6_src == new_ipv6_dst { - return Err(Error::PacketLoop); - } - - // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. - let ipv6 = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN)? }; - ipv6.set_version(6); // IPv6 - ipv6.set_priority(old_ipv6_priority); - ipv6.flow_label = old_ipv6_flow_label; - ipv6.set_payload_len(new_ipv6_len); - ipv6.next_hdr = old_ipv6_next_hdr; - ipv6.hop_limit = old_ipv6_hop_limit; - ipv6.set_src_addr(new_ipv6_src); - ipv6.set_dst_addr(new_ipv6_dst); - - // - // 3. UDP header - // - - let new_udp_src = port_and_peer.allocation_port(); - let new_udp_dst = port_and_peer.peer_port(); - let new_udp_len = old_udp_len - CdHdr::LEN as u16; - - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let udp = - unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN + Ipv6Hdr::LEN)? }; - udp.set_source(new_udp_src); - udp.set_dest(new_udp_dst); - udp.set_len(new_udp_len); - - // Incrementally update UDP checksum - - udp.set_check( - ChecksumUpdate::new(old_udp_check) - .remove_u128(u128::from_be_bytes(old_ipv6_src.octets())) - .remove_u16(old_udp_src) - .remove_u16(old_udp_dst) - .remove_u16(old_udp_len) - .remove_u16(old_udp_len) - .remove_u16(channel_number) - .remove_u16(channel_data_length) - .add_u128(u128::from_be_bytes(new_ipv6_dst.octets())) - .add_u16(new_udp_src) - .add_u16(new_udp_dst) - .add_u16(new_udp_len) - .add_u16(new_udp_len) - .into_udp_checksum(), - ); - - adjust_head(ctx, NET_SHRINK)?; - - Ok(()) -} - -// Convert IPv6 to IPv4 and remove channel data -#[inline(always)] -fn handle_ipv6_channel_to_ipv4_udp( - ctx: &XdpContext, - port_and_peer: &PortAndPeerV4, -) -> Result<(), Error> { - // Shrink by 24 bytes: 20 for the IP header diff and 4 for the removed channel data header - const NET_SHRINK: i32 = Ipv6Hdr::LEN as i32 - Ipv4Hdr::LEN as i32 + CdHdr::LEN as i32; - - let (old_eth_src, old_eth_dst) = { - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let old_eth = unsafe { ref_mut_at::(ctx, 0)? }; - (old_eth.src_addr, old_eth.dst_addr) - }; - - let (old_ipv6_src, old_ipv6_dst, old_ipv6_priority, old_ipv6_hop_limit, old_ipv6_next_hdr) = { - // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. - let old_ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; - ( - old_ipv6.src_addr(), - old_ipv6.dst_addr(), - old_ipv6.priority(), - old_ipv6.hop_limit, - old_ipv6.next_hdr, - ) - }; - - let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let old_udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; - ( - old_udp.len(), - old_udp.source(), - old_udp.dest(), - old_udp.check(), - ) - }; - - let (channel_number, channel_data_length) = { - // SAFETY: The offset must point to the start of a valid `CdHdr`. - let old_cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN + UdpHdr::LEN)? }; - ( - u16::from_be_bytes(old_cd.number), - u16::from_be_bytes(old_cd.length), - ) - }; - - // - // 1. Ethernet header - // - - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let eth = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize)? }; - eth.src_addr = old_eth_dst; // Swap source and destination - eth.dst_addr = old_eth_src; - eth.ether_type = EtherType::Ipv4; // Change to IPv4 - - // - // 2. IPv6 -> IPv4 header - // - - let new_ipv4_src = get_interface_ipv4_address()?; - let new_ipv4_dst = port_and_peer.peer_ip(); - let new_ipv4_len = old_udp_len - CdHdr::LEN as u16 + Ipv4Hdr::LEN as u16; - - // Check for packet loop - would we be sending to ourselves? - if new_ipv4_src == new_ipv4_dst { - return Err(Error::PacketLoop); - } - - // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. - let ipv4 = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN)? }; - ipv4.set_version(4); - ipv4.set_ihl(5); // No options - ipv4.tos = old_ipv6_priority; // Copy TOS from IPv6 - ipv4.set_total_len(new_ipv4_len); - ipv4.set_id(0); // Default ID - ipv4.frag_off = 0x4000_u16.to_be_bytes(); // Don't fragment - ipv4.ttl = old_ipv6_hop_limit; // Preserve TTL - ipv4.proto = old_ipv6_next_hdr; // Copy protocol from IPv6 - ipv4.set_src_addr(new_ipv4_src); - ipv4.set_dst_addr(new_ipv4_dst); - - // Calculate fresh checksum - let check = checksum::new_ipv4(ipv4); - ipv4.set_checksum(check); - - // - // 3. UDP header - // - - let new_udp_src = port_and_peer.allocation_port(); - let new_udp_dst = port_and_peer.peer_port(); - let new_udp_len = old_udp_len - CdHdr::LEN as u16; - - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let udp = - unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN + Ipv4Hdr::LEN)? }; - udp.set_source(new_udp_src); - udp.set_dest(new_udp_dst); - udp.set_len(new_udp_len); - - // Incrementally update UDP checksum - - udp.set_check( - ChecksumUpdate::new(old_udp_check) - .remove_u128(u128::from_be_bytes(old_ipv6_src.octets())) - .remove_u128(u128::from_be_bytes(old_ipv6_dst.octets())) - .remove_u16(old_udp_src) - .remove_u16(old_udp_dst) - .remove_u16(old_udp_len) - .remove_u16(old_udp_len) - .remove_u16(channel_number) - .remove_u16(channel_data_length) - .add_u32(u32::from_be_bytes(new_ipv4_src.octets())) - .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) - .add_u16(new_udp_src) - .add_u16(new_udp_dst) - .add_u16(new_udp_len) - .add_u16(new_udp_len) - .into_udp_checksum(), - ); - - adjust_head(ctx, NET_SHRINK)?; - - Ok(()) -} - -#[inline(always)] -fn adjust_head(ctx: &XdpContext, size: i32) -> Result<(), Error> { - // SAFETY: The attach mode and NIC driver support headroom adjustment by `size` bytes. - let ret = unsafe { bpf_xdp_adjust_head(ctx.ctx, size) }; - if ret < 0 { - return Err(Error::XdpAdjustHeadFailed(ret)); - } - - Ok(()) -} - -#[inline(always)] -fn get_interface_ipv4_address() -> Result { - let interface_addr = INT_ADDR_V4 - .get_ptr_mut(0) - .ok_or(Error::InterfaceIpv4AddressAccessFailed)?; - - // SAFETY: This comes from a per-cpu data structure so we can safely access it. - let addr = unsafe { *interface_addr }; - - addr.get().ok_or(Error::InterfaceIpv4AddressNotConfigured) -} - -fn get_interface_ipv6_address() -> Result { - let interface_addr = INT_ADDR_V6 - .get_ptr_mut(0) - .ok_or(Error::InterfaceIpv6AddressAccessFailed)?; - - // SAFETY: This comes from a per-cpu data structure so we can safely access it. - let addr = unsafe { *interface_addr }; - - addr.get().ok_or(Error::InterfaceIpv6AddressNotConfigured) -} - /// Defines our panic handler. /// /// This doesn't do anything because we can never actually panic in eBPF. @@ -1479,37 +73,3 @@ fn get_interface_ipv6_address() -> Result { fn on_panic(_: &core::panic::PanicInfo) -> ! { loop {} } - -#[cfg(not(target_arch = "bpf"))] -fn main() { - panic!("This program is meant to be compiled as an eBPF program."); -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Memory overhead of an eBPF map. - /// - /// Determined empirically. - const HASH_MAP_OVERHEAD: f32 = 1.5; - - #[test] - fn hashmaps_are_less_than_11_mb() { - let ipv4_datatypes = - core::mem::size_of::() + core::mem::size_of::(); - let ipv6_datatypes = - core::mem::size_of::() + core::mem::size_of::(); - - let ipv4_map_size = ipv4_datatypes as f32 * NUM_ENTRIES as f32 * HASH_MAP_OVERHEAD; - let ipv6_map_size = ipv6_datatypes as f32 * NUM_ENTRIES as f32 * HASH_MAP_OVERHEAD; - - let total_map_size = (ipv4_map_size + ipv6_map_size) * 2_f32; - let total_map_size_mb = total_map_size / 1024_f32 / 1024_f32; - - assert!( - total_map_size_mb < 11_f32, - "Total map size = {total_map_size_mb} MB" - ); - } -} diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn.rs new file mode 100644 index 000000000..a8bc657f0 --- /dev/null +++ b/rust/relay/ebpf-turn-router/src/try_handle_turn.rs @@ -0,0 +1,1497 @@ +use crate::channel_data::CdHdr; +use crate::checksum::ChecksumUpdate; +use crate::error::Error; +use crate::error::SupportedChannel; +use crate::ref_mut_at::ref_mut_at; +use aya_ebpf::{ + bindings::xdp_action, + helpers::bpf_xdp_adjust_head, + macros::map, + maps::{HashMap, PerCpuArray}, + programs::XdpContext, +}; +use aya_log_ebpf::*; +use core::net::{Ipv4Addr, Ipv6Addr}; +use ebpf_shared::{ + ClientAndChannelV4, ClientAndChannelV6, InterfaceAddressV4, InterfaceAddressV6, PortAndPeerV4, + PortAndPeerV6, +}; +use network_types::{ + eth::{EthHdr, EtherType}, + ip::{IpProto, Ipv4Hdr, Ipv6Hdr}, + udp::UdpHdr, +}; + +const NUM_ENTRIES: u32 = 0x10000; +const LOWER_PORT: u16 = 49152; // Lower bound for TURN UDP ports +const UPPER_PORT: u16 = 65535; // Upper bound for TURN UDP ports +const CHAN_START: u16 = 0x4000; // Channel number start +const CHAN_END: u16 = 0x7FFF; // Channel number end + +// SAFETY: Testing has shown that these maps are safe to use as long as we aren't +// writing to them from multiple threads at the same time. Since we only update these +// from the single-threaded eventloop in userspace, we are ok. +// See https://github.com/firezone/firezone/issues/10138#issuecomment-3186074350 + +#[map] +static CHAN_TO_UDP_44: HashMap = + HashMap::with_max_entries(NUM_ENTRIES, 0); +#[map] +static UDP_TO_CHAN_44: HashMap = + HashMap::with_max_entries(NUM_ENTRIES, 0); +#[map] +static CHAN_TO_UDP_66: HashMap = + HashMap::with_max_entries(NUM_ENTRIES, 0); +#[map] +static UDP_TO_CHAN_66: HashMap = + HashMap::with_max_entries(NUM_ENTRIES, 0); +#[map] +static CHAN_TO_UDP_46: HashMap = + HashMap::with_max_entries(NUM_ENTRIES, 0); +#[map] +static UDP_TO_CHAN_46: HashMap = + HashMap::with_max_entries(NUM_ENTRIES, 0); +#[map] +static CHAN_TO_UDP_64: HashMap = + HashMap::with_max_entries(NUM_ENTRIES, 0); +#[map] +static UDP_TO_CHAN_64: HashMap = + HashMap::with_max_entries(NUM_ENTRIES, 0); + +// Per-CPU data structures to learn relay interface addresses +#[map] +static INT_ADDR_V4: PerCpuArray = PerCpuArray::with_max_entries(1, 0); +#[map] +static INT_ADDR_V6: PerCpuArray = PerCpuArray::with_max_entries(1, 0); + +#[inline(always)] +pub fn try_handle_turn(ctx: &XdpContext) -> Result { + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let eth = unsafe { ref_mut_at::(ctx, 0)? }; + + match eth.ether_type { + EtherType::Ipv4 => try_handle_turn_ipv4(ctx)?, + EtherType::Ipv6 => try_handle_turn_ipv6(ctx)?, + _ => return Err(Error::NotIp), + }; + + // If we get to here, we modified the packet and need to send it back out again. + Ok(xdp_action::XDP_TX) +} + +#[inline(always)] +fn try_handle_turn_ipv4(ctx: &XdpContext) -> Result<(), Error> { + // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. + let ipv4 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + + learn_interface_ipv4_address(ipv4)?; + + if ipv4.proto != IpProto::Udp { + return Err(Error::NotUdp); + } + + if ipv4.ihl() != 5 { + // IPv4 with options is not supported + return Err(Error::Ipv4PacketWithOptions); + } + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN)? }; + let udp_payload_len = udp.len() - UdpHdr::LEN as u16; + + trace!( + ctx, + "New packet from {:i}:{} for {:i}:{} with UDP payload {}", + ipv4.src_addr(), + udp.source(), + ipv4.dst_addr(), + udp.dest(), + udp_payload_len + ); + + if (LOWER_PORT..=UPPER_PORT).contains(&udp.dest()) { + try_handle_ipv4_udp_to_channel_data(ctx)?; + crate::stats::emit_data_relayed(ctx, udp_payload_len); + + return Ok(()); + } + + if udp.dest() == 3478 { + try_handle_ipv4_channel_data_to_udp(ctx)?; + crate::stats::emit_data_relayed(ctx, udp_payload_len - CdHdr::LEN as u16); + + return Ok(()); + } + + Err(Error::NotTurn) +} + +#[inline(always)] +fn try_handle_turn_ipv6(ctx: &XdpContext) -> Result<(), Error> { + // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. + let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + + learn_interface_ipv6_address(ipv6)?; + + if ipv6.next_hdr != IpProto::Udp { + return Err(Error::NotUdp); + } + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; + let udp_payload_len = udp.len() - UdpHdr::LEN as u16; + + trace!( + ctx, + "New packet from {:i}:{} for {:i}:{} with UDP payload {}", + ipv6.src_addr(), + udp.source(), + ipv6.dst_addr(), + udp.dest(), + udp_payload_len + ); + + if (LOWER_PORT..=UPPER_PORT).contains(&udp.dest()) { + try_handle_ipv6_udp_to_channel_data(ctx)?; + crate::stats::emit_data_relayed(ctx, udp_payload_len); + + return Ok(()); + } + + if udp.dest() == 3478 { + try_handle_ipv6_channel_data_to_udp(ctx)?; + crate::stats::emit_data_relayed(ctx, udp_payload_len - CdHdr::LEN as u16); + + return Ok(()); + } + + Err(Error::NotTurn) +} + +#[inline(always)] +fn learn_interface_ipv4_address(ipv4: &Ipv4Hdr) -> Result<(), Error> { + let interface_addr = INT_ADDR_V4 + .get_ptr_mut(0) + .ok_or(Error::InterfaceIpv4AddressAccessFailed)?; + + let dst_ip = ipv4.dst_addr(); + + // SAFETY: These are per-cpu maps so we don't need to worry about thread safety. + unsafe { + if (*interface_addr).get().is_none() { + (*interface_addr).set(dst_ip); + } + } + + Ok(()) +} + +#[inline(always)] +fn learn_interface_ipv6_address(ipv6: &Ipv6Hdr) -> Result<(), Error> { + let interface_addr = INT_ADDR_V6 + .get_ptr_mut(0) + .ok_or(Error::InterfaceIpv6AddressAccessFailed)?; + + let dst_ip = ipv6.dst_addr(); + + // SAFETY: These are per-cpu maps so we don't need to worry about thread safety. + unsafe { + if (*interface_addr).get().is_none() { + (*interface_addr).set(dst_ip); + } + } + + Ok(()) +} + +#[inline(always)] +fn try_handle_ipv4_udp_to_channel_data(ctx: &XdpContext) -> Result<(), Error> { + // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. + let ipv4 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN)? }; + + let key = PortAndPeerV4::new(ipv4.src_addr(), udp.dest(), udp.source()); + + // SAFETY: We only write to these using a single thread in userspace. + if let Some(client_and_channel) = unsafe { UDP_TO_CHAN_44.get(&key) } { + handle_ipv4_udp_to_ipv4_channel(ctx, client_and_channel)?; + return Ok(()); + } + + // SAFETY: We only write to these using a single thread in userspace. + if let Some(client_and_channel) = unsafe { UDP_TO_CHAN_46.get(&key) } { + handle_ipv4_udp_to_ipv6_channel(ctx, client_and_channel)?; + return Ok(()); + } + + Err(Error::NoEntry(SupportedChannel::Udp4ToChan)) +} + +#[inline(always)] +fn try_handle_ipv4_channel_data_to_udp(ctx: &XdpContext) -> Result<(), Error> { + // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. + let ipv4 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN)? }; + + // SAFETY: The offset must point to the start of a valid `CdHdr`. + let cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN + UdpHdr::LEN)? }; + + let channel_number = u16::from_be_bytes(cd.number); + + if !(CHAN_START..=CHAN_END).contains(&channel_number) { + return Err(Error::NotAChannelDataMessage); + } + + let channel_data_len = u16::from_be_bytes(cd.length); + let expected_channel_data_len = udp.len() - UdpHdr::LEN as u16 - CdHdr::LEN as u16; + + // This can happen if we receive packets formed via GSO, like on a loopback interface. + if channel_data_len != expected_channel_data_len { + return Err(Error::BadChannelDataLength); + } + + let key = ClientAndChannelV4::new(ipv4.src_addr(), udp.source(), channel_number); + + // SAFETY: We only write to these using a single thread in userspace. + if let Some(port_and_peer) = unsafe { CHAN_TO_UDP_44.get(&key) } { + // IPv4 to IPv4 - existing logic + handle_ipv4_channel_to_ipv4_udp(ctx, port_and_peer)?; + return Ok(()); + } + + // SAFETY: We only write to these using a single thread in userspace. + if let Some(port_and_peer) = unsafe { CHAN_TO_UDP_46.get(&key) } { + handle_ipv4_channel_to_ipv6_udp(ctx, port_and_peer)?; + return Ok(()); + } + + Err(Error::NoEntry(SupportedChannel::Chan4ToUdp)) +} + +#[inline(always)] +fn try_handle_ipv6_udp_to_channel_data(ctx: &XdpContext) -> Result<(), Error> { + // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. + let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; + + let key = PortAndPeerV6::new(ipv6.src_addr(), udp.dest(), udp.source()); + + // SAFETY: We only write to these using a single thread in userspace. + if let Some(client_and_channel) = unsafe { UDP_TO_CHAN_66.get(&key) } { + handle_ipv6_udp_to_ipv6_channel(ctx, client_and_channel)?; + return Ok(()); + } + + // SAFETY: We only write to these using a single thread in userspace. + if let Some(client_and_channel) = unsafe { UDP_TO_CHAN_64.get(&key) } { + handle_ipv6_udp_to_ipv4_channel(ctx, client_and_channel)?; + return Ok(()); + } + + Err(Error::NoEntry(SupportedChannel::Udp6ToChan)) +} + +#[inline(always)] +fn try_handle_ipv6_channel_data_to_udp(ctx: &XdpContext) -> Result<(), Error> { + // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. + let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; + + // SAFETY: The offset must point to the start of a valid `CdHdr`. + let cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN + UdpHdr::LEN)? }; + + let channel_number = u16::from_be_bytes(cd.number); + + if !(CHAN_START..=CHAN_END).contains(&channel_number) { + return Err(Error::NotAChannelDataMessage); + } + + let channel_data_len = u16::from_be_bytes(cd.length); + let expected_channel_data_len = udp.len() - UdpHdr::LEN as u16 - CdHdr::LEN as u16; + + // This can happen if we receive packets formed via GSO, like on a loopback interface. + if channel_data_len != expected_channel_data_len { + return Err(Error::BadChannelDataLength); + } + + let key = ClientAndChannelV6::new(ipv6.src_addr(), udp.source(), u16::from_be_bytes(cd.number)); + + // SAFETY: We only write to these using a single thread in userspace. + if let Some(port_and_peer) = unsafe { CHAN_TO_UDP_66.get(&key) } { + handle_ipv6_channel_to_ipv6_udp(ctx, port_and_peer)?; + return Ok(()); + } + + // SAFETY: We only write to these using a single thread in userspace. + if let Some(port_and_peer) = unsafe { CHAN_TO_UDP_64.get(&key) } { + handle_ipv6_channel_to_ipv4_udp(ctx, port_and_peer)?; + return Ok(()); + } + + Err(Error::NoEntry(SupportedChannel::Chan6ToUdp)) +} + +#[inline(always)] +fn handle_ipv4_udp_to_ipv4_channel( + ctx: &XdpContext, + client_and_channel: &ClientAndChannelV4, +) -> Result<(), Error> { + const NET_EXPANSION: i32 = -(CdHdr::LEN as i32); + + adjust_head(ctx, NET_EXPANSION)?; + + // Now read the old packet data from its NEW location (shifted by 4 bytes) + let old_data_offset = -NET_EXPANSION as usize; + + let (old_eth_src, old_eth_dst, old_eth_type) = { + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let old_eth = unsafe { ref_mut_at::(ctx, old_data_offset)? }; + (old_eth.src_addr, old_eth.dst_addr, old_eth.ether_type) + }; + + let ( + old_ipv4_src, + old_ipv4_dst, + old_ipv4_len, + old_ipv4_check, + old_ipv4_tos, + old_ipv4_id, + old_ipv4_frag_off, + old_ipv4_ttl, + old_ipv4_proto, + ) = { + // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. + let old_ipv4 = unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN)? }; + ( + old_ipv4.src_addr(), + old_ipv4.dst_addr(), + old_ipv4.total_len(), + old_ipv4.checksum(), + old_ipv4.tos, + old_ipv4.id(), + old_ipv4.frag_off, + old_ipv4.ttl, + old_ipv4.proto, + ) + }; + + let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let old_udp = + unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN + Ipv4Hdr::LEN)? }; + ( + old_udp.len(), + old_udp.source(), + old_udp.dest(), + old_udp.check(), + ) + }; + + // + // 1. Ethernet header + // + + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let eth = unsafe { ref_mut_at::(ctx, 0)? }; + eth.src_addr = old_eth_dst; + eth.dst_addr = old_eth_src; + eth.ether_type = old_eth_type; + + // + // 2. IPv4 header + // + + let new_ipv4_src = old_ipv4_dst; + let new_ipv4_dst = client_and_channel.client_ip(); + let new_ipv4_len = old_ipv4_len + CdHdr::LEN as u16; + + // Check for packet loop - would we be sending to ourselves? + if new_ipv4_src == new_ipv4_dst { + return Err(Error::PacketLoop); + } + + // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. + let ipv4 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + ipv4.set_version(4); // IPv4 + ipv4.set_ihl(5); // No options, 5 * 4 = 20 bytes + ipv4.tos = old_ipv4_tos; // Preserve TOS/DSCP + ipv4.set_total_len(new_ipv4_len); + ipv4.set_id(old_ipv4_id); // Preserve fragment ID + ipv4.frag_off = old_ipv4_frag_off; // Preserve fragment flags + ipv4.ttl = old_ipv4_ttl; // Preserve TTL exactly + ipv4.proto = old_ipv4_proto; // Protocol is UDP + ipv4.set_src_addr(new_ipv4_src); // Swap source and destination + ipv4.set_dst_addr(new_ipv4_dst); // Destination is the client IP + ipv4.set_checksum( + ChecksumUpdate::new(old_ipv4_check) + .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) + .remove_u16(old_ipv4_len) + .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) + .add_u16(new_ipv4_len) + .into_ip_checksum(), + ); + + // + // 3. UDP header + // + + let new_udp_src = 3478_u16; + let new_udp_dst = client_and_channel.client_port(); + let new_udp_len = old_udp_len + CdHdr::LEN as u16; + let channel_number = client_and_channel.channel(); + let channel_data_length = old_udp_len - UdpHdr::LEN as u16; + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN)? }; + udp.set_source(new_udp_src); + udp.set_dest(new_udp_dst); + udp.set_len(new_udp_len); + + // Incrementally update UDP checksum + + if old_udp_check == 0 { + // No checksum is valid for UDP IPv4 - we didn't write it, but maybe a middlebox did + udp.set_check(0); + } else { + udp.set_check( + ChecksumUpdate::new(old_udp_check) + .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) + .remove_u16(old_udp_src) + .remove_u16(old_udp_dst) + .remove_u16(old_udp_len) + .remove_u16(old_udp_len) + .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) + .add_u16(new_udp_src) + .add_u16(new_udp_dst) + .add_u16(new_udp_len) + .add_u16(new_udp_len) + .add_u16(channel_number) + .add_u16(channel_data_length) + .into_udp_checksum(), + ); + } + + // + // 4. Channel data header + // + + // SAFETY: The offset must point to the start of a valid `CdHdr`. + let cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN + UdpHdr::LEN)? }; + cd.number = channel_number.to_be_bytes(); + cd.length = channel_data_length.to_be_bytes(); + + Ok(()) +} + +// Convert IPv4 to IPv6 and add channel data +#[inline(always)] +fn handle_ipv4_udp_to_ipv6_channel( + ctx: &XdpContext, + client_and_channel: &ClientAndChannelV6, +) -> Result<(), Error> { + // Expand the packet by 24 bytes for IPv6 header and channel data header + const NET_EXPANSION: i32 = -(Ipv6Hdr::LEN as i32 - Ipv4Hdr::LEN as i32 + CdHdr::LEN as i32); + + adjust_head(ctx, NET_EXPANSION)?; + + // Now read the old packet data from its NEW location (shifted by 24 bytes) + let old_data_offset = -NET_EXPANSION as usize; + + let (old_eth_src, old_eth_dst) = { + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let old_eth = unsafe { ref_mut_at::(ctx, old_data_offset)? }; + (old_eth.src_addr, old_eth.dst_addr) + }; + + let (old_ipv4_src, old_ipv4_dst, old_ipv4_len, old_ipv4_tos, old_ipv4_ttl, old_ipv4_proto) = { + // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. + let old_ipv4 = unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN)? }; + ( + old_ipv4.src_addr(), + old_ipv4.dst_addr(), + old_ipv4.total_len(), + old_ipv4.tos, + old_ipv4.ttl, + old_ipv4.proto, + ) + }; + + let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let old_udp = + unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN + Ipv4Hdr::LEN)? }; + ( + old_udp.len(), + old_udp.source(), + old_udp.dest(), + old_udp.check(), + ) + }; + + // Refuse to compute full UDP checksum. + // We forged these packets, so something's wrong if this is zero. + if old_udp_check == 0 { + return Err(Error::UdpChecksumMissing); + } + + // + // 1. Ethernet header + // + + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let eth = unsafe { ref_mut_at::(ctx, 0)? }; + eth.dst_addr = old_eth_src; // Swap source and destination + eth.src_addr = old_eth_dst; + eth.ether_type = EtherType::Ipv6; // Change to IPv6 + + // + // 2. IPv4 -> IPv6 header + // + + let new_ipv6_src = get_interface_ipv6_address()?; + let new_ipv6_dst = client_and_channel.client_ip(); + let new_ipv6_len = old_ipv4_len - Ipv4Hdr::LEN as u16 + CdHdr::LEN as u16; + + // Check for packet loop - would we be sending to ourselves? + if new_ipv6_dst == new_ipv6_src { + return Err(Error::PacketLoop); + } + + // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. + let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + ipv6.set_version(6); + ipv6.set_priority(old_ipv4_tos); + ipv6.flow_label = [0, 0, 0]; // Default flow label + ipv6.set_payload_len(new_ipv6_len); + ipv6.next_hdr = old_ipv4_proto; + ipv6.hop_limit = old_ipv4_ttl; + ipv6.set_src_addr(new_ipv6_src); + ipv6.set_dst_addr(new_ipv6_dst); + + // + // 3. UDP header + // + + let new_udp_src = 3478_u16; + let new_udp_dst = client_and_channel.client_port(); + let new_udp_len = old_udp_len + CdHdr::LEN as u16; + + let channel_number = client_and_channel.channel(); + let channel_data_length = old_udp_len - UdpHdr::LEN as u16; + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; + udp.set_source(new_udp_src); + udp.set_dest(new_udp_dst); + udp.set_len(new_udp_len); + + // Incrementally update UDP checksum + + udp.set_check( + ChecksumUpdate::new(old_udp_check) + .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) + .remove_u32(u32::from_be_bytes(old_ipv4_dst.octets())) + .remove_u16(old_udp_src) + .remove_u16(old_udp_dst) + .remove_u16(old_udp_len) + .remove_u16(old_udp_len) + .add_u128(u128::from_be_bytes(new_ipv6_src.octets())) + .add_u128(u128::from_be_bytes(new_ipv6_dst.octets())) + .add_u16(new_udp_src) + .add_u16(new_udp_dst) + .add_u16(new_udp_len) + .add_u16(new_udp_len) + .add_u16(channel_number) + .add_u16(channel_data_length) + .into_udp_checksum(), + ); + + // + // 4. Channel data header + // + + // SAFETY: The offset must point to the start of a valid `CdHdr`. + let cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN + UdpHdr::LEN)? }; + cd.number = channel_number.to_be_bytes(); + cd.length = channel_data_length.to_be_bytes(); + + Ok(()) +} + +#[inline(always)] +fn handle_ipv4_channel_to_ipv4_udp( + ctx: &XdpContext, + port_and_peer: &PortAndPeerV4, +) -> Result<(), Error> { + const NET_SHRINK: i32 = CdHdr::LEN as i32; // Shrink by 4 bytes for channel data header + + let (old_eth_src, old_eth_dst, old_eth_type) = { + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let old_eth = unsafe { ref_mut_at::(ctx, 0)? }; + (old_eth.src_addr, old_eth.dst_addr, old_eth.ether_type) + }; + + let ( + old_ipv4_src, + old_ipv4_dst, + old_ipv4_len, + old_ipv4_check, + old_ipv4_tos, + old_ipv4_id, + old_ipv4_frag_off, + old_ipv4_ttl, + old_ipv4_proto, + ) = { + // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. + let old_ipv4 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + ( + old_ipv4.src_addr(), + old_ipv4.dst_addr(), + old_ipv4.total_len(), + old_ipv4.checksum(), + old_ipv4.tos, + old_ipv4.id(), + old_ipv4.frag_off, + old_ipv4.ttl, + old_ipv4.proto, + ) + }; + + let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let old_udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN)? }; + ( + old_udp.len(), + old_udp.source(), + old_udp.dest(), + old_udp.check(), + ) + }; + + let (channel_number, channel_data_length) = { + // SAFETY: The offset must point to the start of a valid `CdHdr`. + let old_cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN + UdpHdr::LEN)? }; + ( + u16::from_be_bytes(old_cd.number), + u16::from_be_bytes(old_cd.length), + ) + }; + + // + // 1. Ethernet header + // + + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let eth = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize)? }; + eth.dst_addr = old_eth_src; // Swap source and destination + eth.src_addr = old_eth_dst; + eth.ether_type = old_eth_type; + + // + // 2. IPv4 header + // + + let new_ipv4_src = old_ipv4_dst; // Swap source and destination + let new_ipv4_dst = port_and_peer.peer_ip(); + let new_ipv4_len = old_ipv4_len - CdHdr::LEN as u16; + + // Check for packet loop - would we be sending to ourselves? + if new_ipv4_src == new_ipv4_dst { + return Err(Error::PacketLoop); + } + + // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. + let ipv4 = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN)? }; + ipv4.set_version(4); // IPv4 + ipv4.set_ihl(5); // No options, 5 * 4 = 20 bytes + ipv4.tos = old_ipv4_tos; // Preserve TOS/DSCP + ipv4.set_total_len(new_ipv4_len); + ipv4.set_id(old_ipv4_id); // Preserve ID + ipv4.frag_off = old_ipv4_frag_off; // Preserve fragment flags + ipv4.ttl = old_ipv4_ttl; // Preserve TTL exactly + ipv4.proto = old_ipv4_proto; // Protocol is UDP + ipv4.set_src_addr(new_ipv4_src); + ipv4.set_dst_addr(new_ipv4_dst); + ipv4.set_checksum( + ChecksumUpdate::new(old_ipv4_check) + .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) + .remove_u16(old_ipv4_len) + .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) + .add_u16(new_ipv4_len) + .into_ip_checksum(), + ); + + // + // 3. UDP header + // + + let new_udp_src = port_and_peer.allocation_port(); + let new_udp_dst = port_and_peer.peer_port(); + let new_udp_len = old_udp_len - CdHdr::LEN as u16; + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = + unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN + Ipv4Hdr::LEN)? }; + udp.set_source(new_udp_src); + udp.set_dest(new_udp_dst); + udp.set_len(new_udp_len); + + // Incrementally update UDP checksum + + if old_udp_check == 0 { + // No checksum is valid for UDP IPv4 - we didn't write it, but maybe a middlebox did + udp.set_check(0); + } else { + udp.set_check( + ChecksumUpdate::new(old_udp_check) + .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) + .remove_u16(old_udp_src) + .remove_u16(old_udp_dst) + .remove_u16(old_udp_len) + .remove_u16(old_udp_len) + .remove_u16(channel_number) + .remove_u16(channel_data_length) + .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) + .add_u16(new_udp_src) + .add_u16(new_udp_dst) + .add_u16(new_udp_len) + .add_u16(new_udp_len) + .into_udp_checksum(), + ); + } + + adjust_head(ctx, NET_SHRINK)?; + + Ok(()) +} + +// Convert IPv4 to IPv6 and remove channel data header +#[inline(always)] +fn handle_ipv4_channel_to_ipv6_udp( + ctx: &XdpContext, + port_and_peer: &PortAndPeerV6, +) -> Result<(), Error> { + const NET_EXPANSION: i32 = Ipv4Hdr::LEN as i32 - Ipv6Hdr::LEN as i32 + CdHdr::LEN as i32; + + adjust_head(ctx, NET_EXPANSION)?; + + // Now read the old packet data from its NEW location + let old_data_offset = (-NET_EXPANSION) as usize; + + let (old_src_mac, old_dst_mac) = { + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let old_eth = unsafe { ref_mut_at::(ctx, old_data_offset)? }; + (old_eth.src_addr, old_eth.dst_addr) + }; + + let (old_ipv4_src, old_ipv4_dst, old_ipv4_tos, old_ipv4_ttl, old_ipv4_proto) = { + // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. + let old_ipv4 = unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN)? }; + ( + old_ipv4.src_addr(), + old_ipv4.dst_addr(), + old_ipv4.tos, + old_ipv4.ttl, + old_ipv4.proto, + ) + }; + + let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let old_udp = + unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN + Ipv4Hdr::LEN)? }; + ( + old_udp.len(), + old_udp.source(), + old_udp.dest(), + old_udp.check(), + ) + }; + + // Refuse to compute full UDP checksum. + // We forged these packets, so something's wrong if this is zero. + if old_udp_check == 0 { + return Err(Error::UdpChecksumMissing); + } + + let (channel_number, channel_data_length) = { + // SAFETY: The offset must point to the start of a valid `CdHdr`. + let old_cd = unsafe { + ref_mut_at::( + ctx, + old_data_offset + EthHdr::LEN + Ipv4Hdr::LEN + UdpHdr::LEN, + )? + }; + ( + u16::from_be_bytes(old_cd.number), + u16::from_be_bytes(old_cd.length), + ) + }; + + // + // 1. Ethernet header + // + + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let eth = unsafe { ref_mut_at::(ctx, 0)? }; + eth.dst_addr = old_src_mac; // Swap MACs + eth.src_addr = old_dst_mac; + eth.ether_type = EtherType::Ipv6; // Change to IPv6 + + // + // 2. IPv6 header + // + + let new_ipv6_src = get_interface_ipv6_address()?; + let new_ipv6_dst = port_and_peer.peer_ip(); + let new_udp_len = old_udp_len - CdHdr::LEN as u16; + + // Check for packet loop - would we be sending to ourselves? + if new_ipv6_src == new_ipv6_dst { + return Err(Error::PacketLoop); + } + + // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. + let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + ipv6.set_version(6); // IPv6 + ipv6.set_priority(old_ipv4_tos); + ipv6.flow_label = [0, 0, 0]; + ipv6.set_payload_len(new_udp_len); + ipv6.next_hdr = old_ipv4_proto; + ipv6.hop_limit = old_ipv4_ttl; + ipv6.set_src_addr(new_ipv6_src); + ipv6.set_dst_addr(new_ipv6_dst); + + // + // 3. UDP header + // + + let new_udp_src = port_and_peer.allocation_port(); + let new_udp_dst = port_and_peer.peer_port(); + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; + udp.set_source(new_udp_src); + udp.set_dest(new_udp_dst); + udp.set_len(new_udp_len); + + // Incrementally update UDP checksum + + udp.set_check( + ChecksumUpdate::new(old_udp_check) + .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) + .remove_u32(u32::from_be_bytes(old_ipv4_dst.octets())) + .remove_u16(old_udp_src) + .remove_u16(old_udp_dst) + .remove_u16(old_udp_len) + .remove_u16(old_udp_len) + .remove_u16(channel_number) + .remove_u16(channel_data_length) + .add_u128(u128::from_be_bytes(new_ipv6_src.octets())) + .add_u128(u128::from_be_bytes(new_ipv6_dst.octets())) + .add_u16(new_udp_src) + .add_u16(new_udp_dst) + .add_u16(new_udp_len) + .add_u16(new_udp_len) + .into_udp_checksum(), + ); + + Ok(()) +} + +#[inline(always)] +fn handle_ipv6_udp_to_ipv6_channel( + ctx: &XdpContext, + client_and_channel: &ClientAndChannelV6, +) -> Result<(), Error> { + // Expand by 4 bytes for channel data header + const NET_EXPANSION: i32 = -(CdHdr::LEN as i32); + + adjust_head(ctx, NET_EXPANSION)?; + + // Now read the old packet data from its NEW location (shifted by 4 bytes) + let old_data_offset = CdHdr::LEN; + + let (old_eth_src, old_eth_dst, old_eth_type) = { + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let old_eth = unsafe { ref_mut_at::(ctx, old_data_offset)? }; + (old_eth.src_addr, old_eth.dst_addr, old_eth.ether_type) + }; + + let ( + old_ipv6_src, + old_ipv6_dst, + old_ipv6_len, + old_ipv6_priority, + old_ipv6_flow_label, + old_ipv6_hop_limit, + old_ipv6_next_hdr, + ) = { + // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. + let old_ipv6 = unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN)? }; + ( + old_ipv6.src_addr(), + old_ipv6.dst_addr(), + old_ipv6.payload_len(), + old_ipv6.priority(), + old_ipv6.flow_label, + old_ipv6.hop_limit, + old_ipv6.next_hdr, + ) + }; + + let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let old_udp = + unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN + Ipv6Hdr::LEN)? }; + ( + old_udp.len(), + old_udp.source(), + old_udp.dest(), + old_udp.check(), + ) + }; + + // + // 1. Ethernet header + // + + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let eth = unsafe { ref_mut_at::(ctx, 0)? }; + eth.src_addr = old_eth_dst; // Swap source and destination + eth.dst_addr = old_eth_src; + eth.ether_type = old_eth_type; + + // + // 2. IPv6 header + // + + let new_ipv6_src = old_ipv6_dst; + let new_ipv6_dst = client_and_channel.client_ip(); + let new_ipv6_len = old_ipv6_len + CdHdr::LEN as u16; + + // Check for packet loop - would we be sending to ourselves? + if new_ipv6_src == new_ipv6_dst { + return Err(Error::PacketLoop); + } + + // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. + let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + // Set fields explicitly to avoid reading potentially corrupted memory + ipv6.set_version(6); // IPv6 + ipv6.set_priority(old_ipv6_priority); + ipv6.flow_label = old_ipv6_flow_label; + ipv6.set_payload_len(new_ipv6_len); + ipv6.next_hdr = old_ipv6_next_hdr; + ipv6.hop_limit = old_ipv6_hop_limit; + ipv6.set_src_addr(new_ipv6_src); + ipv6.set_dst_addr(new_ipv6_dst); + + // + // 3. UDP header + // + + let channel_number = client_and_channel.channel(); + let channel_data_length = old_udp_len - UdpHdr::LEN as u16; + let new_udp_len = old_udp_len + CdHdr::LEN as u16; + let new_udp_src = 3478_u16; + let new_udp_dst = client_and_channel.client_port(); + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; + udp.set_source(new_udp_src); + udp.set_dest(new_udp_dst); + udp.set_len(new_udp_len); + + // Incrementally update UDP checksum + + udp.set_check( + ChecksumUpdate::new(old_udp_check) + .remove_u128(u128::from_be_bytes(old_ipv6_src.octets())) + .remove_u16(old_udp_src) + .remove_u16(old_udp_dst) + .remove_u16(old_udp_len) + .remove_u16(old_udp_len) + .add_u128(u128::from_be_bytes(new_ipv6_dst.octets())) + .add_u16(new_udp_src) + .add_u16(new_udp_dst) + .add_u16(new_udp_len) + .add_u16(new_udp_len) + .add_u16(channel_number) + .add_u16(channel_data_length) + .into_udp_checksum(), + ); + + // + // 4. Channel data header + // + + // SAFETY: The offset must point to the start of a valid `CdHdr`. + let cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN + UdpHdr::LEN)? }; + cd.number = channel_number.to_be_bytes(); + cd.length = channel_data_length.to_be_bytes(); + + Ok(()) +} + +// Convert IPv6 to IPv4 and add channel data +#[inline(always)] +fn handle_ipv6_udp_to_ipv4_channel( + ctx: &XdpContext, + client_and_channel: &ClientAndChannelV4, +) -> Result<(), Error> { + // 40 - 20 - 4 = 16 bytes shrink + const NET_SHRINK: i32 = Ipv6Hdr::LEN as i32 - Ipv4Hdr::LEN as i32 - CdHdr::LEN as i32; + + let (old_eth_src, old_eth_dst) = { + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let old_eth = unsafe { ref_mut_at::(ctx, 0)? }; + (old_eth.src_addr, old_eth.dst_addr) + }; + + let (old_ipv6_src, old_ipv6_dst, old_ipv6_priority, old_ipv6_hop_limit, old_ipv6_next_hdr) = { + // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. + let old_ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + ( + old_ipv6.src_addr(), + old_ipv6.dst_addr(), + old_ipv6.priority(), + old_ipv6.hop_limit, + old_ipv6.next_hdr, + ) + }; + + let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let old_udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; + ( + old_udp.len(), + old_udp.source(), + old_udp.dest(), + old_udp.check(), + ) + }; + + // + // 1. Ethernet header + // + + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let eth = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize)? }; + eth.src_addr = old_eth_dst; // Swap source and destination + eth.dst_addr = old_eth_src; + eth.ether_type = EtherType::Ipv4; // Change to IPv4 + + // + // 2. IPv6 -> IPv4 header + // + + let new_ipv4_src = get_interface_ipv4_address()?; + let new_ipv4_dst = client_and_channel.client_ip(); + let new_udp_len = old_udp_len + CdHdr::LEN as u16; + let new_ipv4_len = Ipv4Hdr::LEN as u16 + new_udp_len; + + // Check for packet loop - would we be sending to ourselves? + if new_ipv4_dst == new_ipv4_src { + return Err(Error::PacketLoop); + } + + // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. + let ipv4 = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN)? }; + ipv4.set_version(4); + ipv4.set_ihl(5); // No options + ipv4.tos = old_ipv6_priority; + ipv4.set_total_len(new_ipv4_len); + ipv4.set_id(0); // Default ID + ipv4.frag_off = 0x4000_u16.to_be_bytes(); // Don't fragment + ipv4.ttl = old_ipv6_hop_limit; // Preserve hop limit + ipv4.proto = old_ipv6_next_hdr; // Preserve protocol + ipv4.set_src_addr(new_ipv4_src); + ipv4.set_dst_addr(new_ipv4_dst); + + // Calculate fresh checksum + let check = crate::checksum::new_ipv4(ipv4); + ipv4.set_checksum(check); + + // + // 3. UDP header + // + + let new_udp_src = 3478_u16; // Fixed source port for TURN + let new_udp_dst = client_and_channel.client_port(); + let channel_number = client_and_channel.channel(); + let channel_data_length = old_udp_len - UdpHdr::LEN as u16; + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = + unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN + Ipv4Hdr::LEN)? }; + udp.set_source(new_udp_src); + udp.set_dest(new_udp_dst); + udp.set_len(new_udp_len); + + // Incrementally update UDP checksum + + udp.set_check( + ChecksumUpdate::new(old_udp_check) + .remove_u128(u128::from_be_bytes(old_ipv6_src.octets())) + .remove_u128(u128::from_be_bytes(old_ipv6_dst.octets())) + .remove_u16(old_udp_src) + .remove_u16(old_udp_dst) + .remove_u16(old_udp_len) + .remove_u16(old_udp_len) + .add_u32(u32::from_be_bytes(new_ipv4_src.octets())) + .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) + .add_u16(new_udp_src) + .add_u16(new_udp_dst) + .add_u16(new_udp_len) + .add_u16(new_udp_len) + .add_u16(channel_number) + .add_u16(channel_data_length) + .into_udp_checksum(), + ); + + // + // 4. Channel data header + // + + // SAFETY: The offset must point to the start of a valid `CdHdr`. + let cd = unsafe { + ref_mut_at::( + ctx, + NET_SHRINK as usize + EthHdr::LEN + Ipv4Hdr::LEN + UdpHdr::LEN, + )? + }; + cd.number = channel_number.to_be_bytes(); + cd.length = channel_data_length.to_be_bytes(); + + adjust_head(ctx, NET_SHRINK)?; + + Ok(()) +} + +#[inline(always)] +fn handle_ipv6_channel_to_ipv6_udp( + ctx: &XdpContext, + port_and_peer: &PortAndPeerV6, +) -> Result<(), Error> { + const NET_SHRINK: i32 = CdHdr::LEN as i32; // Shrink by 4 bytes for channel data header + + let (old_eth_src, old_eth_dst, old_eth_type) = { + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let old_eth = unsafe { ref_mut_at::(ctx, 0)? }; + (old_eth.src_addr, old_eth.dst_addr, old_eth.ether_type) + }; + + let ( + old_ipv6_src, + old_ipv6_dst, + old_ipv6_len, + old_ipv6_priority, + old_ipv6_flow_label, + old_ipv6_hop_limit, + old_ipv6_next_hdr, + ) = { + // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. + let old_ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + ( + old_ipv6.src_addr(), + old_ipv6.dst_addr(), + old_ipv6.payload_len(), + old_ipv6.priority(), + old_ipv6.flow_label, + old_ipv6.hop_limit, + old_ipv6.next_hdr, + ) + }; + + let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let old_udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; + ( + old_udp.len(), + old_udp.source(), + old_udp.dest(), + old_udp.check(), + ) + }; + + let (channel_number, channel_data_length) = { + // SAFETY: The offset must point to the start of a valid `CdHdr`. + let old_cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN + UdpHdr::LEN)? }; + ( + u16::from_be_bytes(old_cd.number), + u16::from_be_bytes(old_cd.length), + ) + }; + + // + // 1. Ethernet header + // + + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let eth = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize)? }; + eth.src_addr = old_eth_dst; // Swap source and destination + eth.dst_addr = old_eth_src; + eth.ether_type = old_eth_type; + + // + // 2. IPv6 header + // + + let new_ipv6_src = old_ipv6_dst; // Swap source and destination + let new_ipv6_dst = port_and_peer.peer_ip(); + let new_ipv6_len = old_ipv6_len - CdHdr::LEN as u16; + + // Check for packet loop - would we be sending to ourselves? + if new_ipv6_src == new_ipv6_dst { + return Err(Error::PacketLoop); + } + + // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. + let ipv6 = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN)? }; + ipv6.set_version(6); // IPv6 + ipv6.set_priority(old_ipv6_priority); + ipv6.flow_label = old_ipv6_flow_label; + ipv6.set_payload_len(new_ipv6_len); + ipv6.next_hdr = old_ipv6_next_hdr; + ipv6.hop_limit = old_ipv6_hop_limit; + ipv6.set_src_addr(new_ipv6_src); + ipv6.set_dst_addr(new_ipv6_dst); + + // + // 3. UDP header + // + + let new_udp_src = port_and_peer.allocation_port(); + let new_udp_dst = port_and_peer.peer_port(); + let new_udp_len = old_udp_len - CdHdr::LEN as u16; + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = + unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN + Ipv6Hdr::LEN)? }; + udp.set_source(new_udp_src); + udp.set_dest(new_udp_dst); + udp.set_len(new_udp_len); + + // Incrementally update UDP checksum + + udp.set_check( + ChecksumUpdate::new(old_udp_check) + .remove_u128(u128::from_be_bytes(old_ipv6_src.octets())) + .remove_u16(old_udp_src) + .remove_u16(old_udp_dst) + .remove_u16(old_udp_len) + .remove_u16(old_udp_len) + .remove_u16(channel_number) + .remove_u16(channel_data_length) + .add_u128(u128::from_be_bytes(new_ipv6_dst.octets())) + .add_u16(new_udp_src) + .add_u16(new_udp_dst) + .add_u16(new_udp_len) + .add_u16(new_udp_len) + .into_udp_checksum(), + ); + + adjust_head(ctx, NET_SHRINK)?; + + Ok(()) +} + +// Convert IPv6 to IPv4 and remove channel data +#[inline(always)] +fn handle_ipv6_channel_to_ipv4_udp( + ctx: &XdpContext, + port_and_peer: &PortAndPeerV4, +) -> Result<(), Error> { + // Shrink by 24 bytes: 20 for the IP header diff and 4 for the removed channel data header + const NET_SHRINK: i32 = Ipv6Hdr::LEN as i32 - Ipv4Hdr::LEN as i32 + CdHdr::LEN as i32; + + let (old_eth_src, old_eth_dst) = { + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let old_eth = unsafe { ref_mut_at::(ctx, 0)? }; + (old_eth.src_addr, old_eth.dst_addr) + }; + + let (old_ipv6_src, old_ipv6_dst, old_ipv6_priority, old_ipv6_hop_limit, old_ipv6_next_hdr) = { + // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. + let old_ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + ( + old_ipv6.src_addr(), + old_ipv6.dst_addr(), + old_ipv6.priority(), + old_ipv6.hop_limit, + old_ipv6.next_hdr, + ) + }; + + let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let old_udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; + ( + old_udp.len(), + old_udp.source(), + old_udp.dest(), + old_udp.check(), + ) + }; + + let (channel_number, channel_data_length) = { + // SAFETY: The offset must point to the start of a valid `CdHdr`. + let old_cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN + UdpHdr::LEN)? }; + ( + u16::from_be_bytes(old_cd.number), + u16::from_be_bytes(old_cd.length), + ) + }; + + // + // 1. Ethernet header + // + + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let eth = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize)? }; + eth.src_addr = old_eth_dst; // Swap source and destination + eth.dst_addr = old_eth_src; + eth.ether_type = EtherType::Ipv4; // Change to IPv4 + + // + // 2. IPv6 -> IPv4 header + // + + let new_ipv4_src = get_interface_ipv4_address()?; + let new_ipv4_dst = port_and_peer.peer_ip(); + let new_ipv4_len = old_udp_len - CdHdr::LEN as u16 + Ipv4Hdr::LEN as u16; + + // Check for packet loop - would we be sending to ourselves? + if new_ipv4_src == new_ipv4_dst { + return Err(Error::PacketLoop); + } + + // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. + let ipv4 = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN)? }; + ipv4.set_version(4); + ipv4.set_ihl(5); // No options + ipv4.tos = old_ipv6_priority; // Copy TOS from IPv6 + ipv4.set_total_len(new_ipv4_len); + ipv4.set_id(0); // Default ID + ipv4.frag_off = 0x4000_u16.to_be_bytes(); // Don't fragment + ipv4.ttl = old_ipv6_hop_limit; // Preserve TTL + ipv4.proto = old_ipv6_next_hdr; // Copy protocol from IPv6 + ipv4.set_src_addr(new_ipv4_src); + ipv4.set_dst_addr(new_ipv4_dst); + + // Calculate fresh checksum + let check = crate::checksum::new_ipv4(ipv4); + ipv4.set_checksum(check); + + // + // 3. UDP header + // + + let new_udp_src = port_and_peer.allocation_port(); + let new_udp_dst = port_and_peer.peer_port(); + let new_udp_len = old_udp_len - CdHdr::LEN as u16; + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = + unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN + Ipv4Hdr::LEN)? }; + udp.set_source(new_udp_src); + udp.set_dest(new_udp_dst); + udp.set_len(new_udp_len); + + // Incrementally update UDP checksum + + udp.set_check( + ChecksumUpdate::new(old_udp_check) + .remove_u128(u128::from_be_bytes(old_ipv6_src.octets())) + .remove_u128(u128::from_be_bytes(old_ipv6_dst.octets())) + .remove_u16(old_udp_src) + .remove_u16(old_udp_dst) + .remove_u16(old_udp_len) + .remove_u16(old_udp_len) + .remove_u16(channel_number) + .remove_u16(channel_data_length) + .add_u32(u32::from_be_bytes(new_ipv4_src.octets())) + .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) + .add_u16(new_udp_src) + .add_u16(new_udp_dst) + .add_u16(new_udp_len) + .add_u16(new_udp_len) + .into_udp_checksum(), + ); + + adjust_head(ctx, NET_SHRINK)?; + + Ok(()) +} + +#[inline(always)] +fn adjust_head(ctx: &XdpContext, size: i32) -> Result<(), Error> { + // SAFETY: The attach mode and NIC driver support headroom adjustment by `size` bytes. + let ret = unsafe { bpf_xdp_adjust_head(ctx.ctx, size) }; + if ret < 0 { + return Err(Error::XdpAdjustHeadFailed(ret)); + } + + Ok(()) +} + +#[inline(always)] +fn get_interface_ipv4_address() -> Result { + let interface_addr = INT_ADDR_V4 + .get_ptr_mut(0) + .ok_or(Error::InterfaceIpv4AddressAccessFailed)?; + + // SAFETY: This comes from a per-cpu data structure so we can safely access it. + let addr = unsafe { *interface_addr }; + + addr.get().ok_or(Error::InterfaceIpv4AddressNotConfigured) +} + +fn get_interface_ipv6_address() -> Result { + let interface_addr = INT_ADDR_V6 + .get_ptr_mut(0) + .ok_or(Error::InterfaceIpv6AddressAccessFailed)?; + + // SAFETY: This comes from a per-cpu data structure so we can safely access it. + let addr = unsafe { *interface_addr }; + + addr.get().ok_or(Error::InterfaceIpv6AddressNotConfigured) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Memory overhead of an eBPF map. + /// + /// Determined empirically. + const HASH_MAP_OVERHEAD: f32 = 1.5; + + #[test] + fn hashmaps_are_less_than_11_mb() { + let ipv4_datatypes = + core::mem::size_of::() + core::mem::size_of::(); + let ipv6_datatypes = + core::mem::size_of::() + core::mem::size_of::(); + + let ipv4_map_size = ipv4_datatypes as f32 * NUM_ENTRIES as f32 * HASH_MAP_OVERHEAD; + let ipv6_map_size = ipv6_datatypes as f32 * NUM_ENTRIES as f32 * HASH_MAP_OVERHEAD; + + let total_map_size = (ipv4_map_size + ipv6_map_size) * 2_f32; + let total_map_size_mb = total_map_size / 1024_f32 / 1024_f32; + + assert!( + total_map_size_mb < 11_f32, + "Total map size = {total_map_size_mb} MB" + ); + } +} diff --git a/rust/tests/gui-smoke-test/src/main.rs b/rust/tests/gui-smoke-test/src/main.rs index cd445544a..31ef7bdf0 100644 --- a/rust/tests/gui-smoke-test/src/main.rs +++ b/rust/tests/gui-smoke-test/src/main.rs @@ -20,6 +20,9 @@ const TUNNEL_NAME: &str = "firezone-client-tunnel"; #[cfg(target_os = "linux")] const EXE_EXTENSION: &str = ""; +#[cfg(target_os = "macos")] +const EXE_EXTENSION: &str = ""; + #[cfg(target_os = "windows")] const EXE_EXTENSION: &str = "exe"; @@ -145,6 +148,20 @@ impl App { } } +#[cfg(target_os = "macos")] +impl App { + fn new() -> Result { + Ok(Self {}) + } + + fn gui_command(&self, args: &[&str]) -> Result { + Ok(Exec::cmd(gui_path()) + .arg("--no-deep-links") + .arg("--no-elevation-check") + .args(args)) + } +} + #[cfg(target_os = "windows")] impl App { fn new() -> Result { @@ -179,6 +196,11 @@ fn debug_db_path() -> PathBuf { Path::new("target").join("debug").join(GUI_NAME) } +#[cfg(target_os = "macos")] +fn debug_db_path() -> PathBuf { + Path::new("target").join("debug").join(GUI_NAME) +} + #[cfg(target_os = "windows")] fn debug_db_path() -> PathBuf { Path::new("target") @@ -202,6 +224,11 @@ fn tunnel_service_command() -> Exec { ]) } +#[cfg(target_os = "macos")] +fn tunnel_service_command() -> Exec { + Exec::cmd("sudo").arg(tunnel_path()) +} + #[cfg(target_os = "windows")] fn tunnel_service_command() -> Exec { Exec::cmd(tunnel_path())