diff --git a/.github/workflows/_integration_tests.yml b/.github/workflows/_integration_tests.yml index 8c9e503ea..d80a8ac05 100644 --- a/.github/workflows/_integration_tests.yml +++ b/.github/workflows/_integration_tests.yml @@ -73,7 +73,7 @@ env: jobs: integration-tests: - name: ${{ matrix.test.name }} + name: ${{ matrix.test.name || matrix.test.script }} runs-on: ubuntu-24.04 permissions: contents: read @@ -100,20 +100,27 @@ jobs: fail-fast: false matrix: test: - - name: direct-curl-api-down - - name: direct-curl-api-restart - - name: direct-curl-gateway-restart - - name: direct-curl-ecn - - name: direct-download-packet-loss - - name: direct-dns-api-down - - name: direct-dns-two-resources - - name: direct-dns - - name: direct-download-roaming-network + - script: curl-api-down + - script: curl-api-restart + - script: curl-ecn + - script: dns + - script: dns-api-down + - script: dns-nm + - script: dns-two-resources + - script: systemd/dns-systemd-resolved + - script: tcp-dns + # Setting both client and gateway to random masquerade will force relay-relay candidate pair + - name: download-double-symmetric-nat + script: download + client_masquerade: random + gateway_masquerade: random + rust_log: debug + stop_containers: relay-2 # Force single relay + - script: download-packet-loss + rust_log: debug + - script: download-roaming-network # Too noisy can cause flaky tests due to the amount of data rust_log: debug - - name: dns-nm - - name: tcp-dns - - name: systemd/dns-systemd-resolved steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: ./.github/actions/ghcr-docker-login @@ -131,6 +138,14 @@ jobs: export RUST_LOG="${{ matrix.test.rust_log }}" fi + if [[ -n "${{ matrix.test.client_masquerade }}" ]]; then + export CLIENT_MASQUERADE="${{ matrix.test.client_masquerade }}" + fi + + if [[ -n "${{ matrix.test.gateway_masquerade }}" ]]; then + export GATEWAY_MASQUERADE="${{ matrix.test.gateway_masquerade }}" + fi + docker compose build client-router gateway-router relay-1-router relay-2-router api-router # Start one-by-one to avoid variability in service startup order @@ -145,6 +160,10 @@ jobs: docker compose up -d client --no-build docker compose up veth-config + if [[ -n "${{ matrix.test.stop_containers }}" ]]; then + docker compose stop ${{ matrix.test.stop_containers }} + fi + # Wait a few seconds for the services to fully start. GH runners are # slow, so this gives the Client enough time to initialize its tun interface, # for example. @@ -162,7 +181,7 @@ jobs: docker compose exec -T gateway sh -c 'apk add --update --no-cache iproute2-tc' docker compose exec -T gateway sh -c 'tc qdisc add dev eth0 root netem delay 10ms' - - run: ./scripts/tests/${{ matrix.test.name }}.sh + - run: ./scripts/tests/${{ matrix.test.script }}.sh - name: Ensure Client emitted no warnings if: "!cancelled()" diff --git a/docker-compose.yml b/docker-compose.yml index ffacc757e..006e1cc67 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -746,9 +746,13 @@ services: echo "Done configuring $$(echo "$$VETHS" | wc -w) veth interfaces" depends_on: relay-1: - condition: "service_started" + condition: "service_healthy" relay-2: - condition: "service_started" + condition: "service_healthy" + relay-1-router: + condition: "service_healthy" + relay-2-router: + condition: "service_healthy" otel: image: otel/opentelemetry-collector:latest diff --git a/rust/Cargo.lock b/rust/Cargo.lock index ab9c06cde..5846e8489 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -2134,6 +2134,7 @@ name = "ebpf-shared" version = "0.1.0" dependencies = [ "aya", + "derive_more 2.0.1", ] [[package]] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index e416a0e50..6a68ae63b 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -67,7 +67,7 @@ client-shared = { path = "client-shared" } connlib-model = { path = "connlib/model" } crossbeam-queue = "0.3.12" dashmap = "6.1.0" -derive_more = "2.0.1" +derive_more = { version = "2.0.1", default-features = false } difference = "2.0.0" dirs = "6.0.0" divan = "0.1.21" diff --git a/rust/relay/ebpf-shared/Cargo.toml b/rust/relay/ebpf-shared/Cargo.toml index 979b4e164..9de11fd5b 100644 --- a/rust/relay/ebpf-shared/Cargo.toml +++ b/rust/relay/ebpf-shared/Cargo.toml @@ -7,5 +7,8 @@ license = { workspace = true } [features] std = ["aya"] +[dependencies] +derive_more = { workspace = true, features = ["from"] } + [target.'cfg(target_os = "linux")'.dependencies] aya = { workspace = true, optional = true } diff --git a/rust/relay/ebpf-shared/src/lib.rs b/rust/relay/ebpf-shared/src/lib.rs index 001daefec..8a2ac8f6f 100644 --- a/rust/relay/ebpf-shared/src/lib.rs +++ b/rust/relay/ebpf-shared/src/lib.rs @@ -5,7 +5,7 @@ #![cfg_attr(not(feature = "std"), no_std)] -use core::net::{Ipv4Addr, Ipv6Addr}; +use core::net::{IpAddr, Ipv4Addr, Ipv6Addr}; #[repr(C)] #[derive(Clone, Copy)] @@ -25,6 +25,37 @@ pub struct ClientAndChannelV6 { channel: [u8; 2], } +#[repr(C)] +#[derive(Clone, Copy, derive_more::From)] +#[cfg_attr(feature = "std", derive(Debug))] +pub enum ClientAndChannel { + V4(ClientAndChannelV4), + V6(ClientAndChannelV6), +} + +impl ClientAndChannel { + pub fn client_ip(&self) -> IpAddr { + match self { + ClientAndChannel::V4(cc) => cc.client_ip().into(), + ClientAndChannel::V6(cc) => cc.client_ip().into(), + } + } + + pub fn client_port(&self) -> u16 { + match self { + ClientAndChannel::V4(cc) => cc.client_port(), + ClientAndChannel::V6(cc) => cc.client_port(), + } + } + + pub fn channel(&self) -> u16 { + match self { + ClientAndChannel::V4(cc) => cc.channel(), + ClientAndChannel::V6(cc) => cc.channel(), + } + } +} + impl ClientAndChannelV4 { pub fn new(ipv4_address: Ipv4Addr, port: u16, channel: u16) -> Self { Self { @@ -96,6 +127,65 @@ pub struct PortAndPeerV6 { peer_port: [u8; 2], } +#[repr(C)] +#[derive(Clone, Copy, derive_more::From)] +#[cfg_attr(feature = "std", derive(Debug))] +pub enum PortAndPeer { + V4(PortAndPeerV4), + V6(PortAndPeerV6), +} + +impl PortAndPeer { + pub fn peer_ip(&self) -> IpAddr { + match self { + PortAndPeer::V4(pp) => pp.peer_ip().into(), + PortAndPeer::V6(pp) => pp.peer_ip().into(), + } + } + + pub fn peer_port(&self) -> u16 { + match self { + PortAndPeer::V4(pp) => pp.peer_port(), + PortAndPeer::V6(pp) => pp.peer_port(), + } + } + + pub fn allocation_port(&self) -> u16 { + match self { + PortAndPeer::V4(pp) => pp.allocation_port(), + PortAndPeer::V6(pp) => pp.allocation_port(), + } + } + + /// Flips the allocation and peer port. + /// + /// When sending out a packet: + /// - the allocation port is the source + /// - the peer port is the destination + /// + /// When receiving a packet: + /// - the allocation port is the destination + /// - the peer port is the source + /// + /// When sending a packet to ourselves, we therefore need to flip these ports. + /// 1. The allocation port becomes the source port of the packet. + /// 2. The peer port becomes the destination of the packet. + pub fn flip_ports(self) -> Self { + match self { + PortAndPeer::V4(pp) => PortAndPeer::V4(PortAndPeerV4 { + ipv4_address: pp.ipv4_address, + allocation_port: pp.peer_port, + peer_port: pp.allocation_port, + }), + PortAndPeer::V6(pp) => PortAndPeer::V6(PortAndPeerV6 { + ipv6_address: pp.ipv6_address, + allocation_port: pp.peer_port, + peer_port: pp.allocation_port, + }), + } + } +} + impl PortAndPeerV4 { pub fn new(ipv4_address: Ipv4Addr, allocation_port: u16, peer_port: u16) -> Self { Self { @@ -149,52 +239,6 @@ impl PortAndPeerV6 { } } -#[repr(C)] -#[derive(Clone, Copy, Default)] -#[cfg_attr(feature = "std", derive(Debug))] -pub struct InterfaceAddressV4 { - address: [u8; 4], -} - -impl InterfaceAddressV4 { - const ZERO: [u8; 4] = [0; 4]; - - pub fn set(&mut self, addr: Ipv4Addr) { - self.address = addr.octets(); - } - - pub fn get(&self) -> Option { - if self.address != Self::ZERO { - Some(self.address.into()) - } else { - None - } - } -} - -#[repr(C)] -#[derive(Clone, Copy, Default)] -#[cfg_attr(feature = "std", derive(Debug))] -pub struct InterfaceAddressV6 { - address: [u8; 16], -} - -impl InterfaceAddressV6 { - const ZERO: [u8; 16] = [0; 16]; - - pub fn set(&mut self, addr: Ipv6Addr) { - self.address = addr.octets(); - } - - pub fn get(&self) -> Option { - if self.address != Self::ZERO { - Some(self.address.into()) - } else { - None - } - } -} - #[repr(C)] #[derive(Clone, Copy)] #[cfg_attr(feature = "std", derive(Debug))] @@ -223,8 +267,4 @@ mod userspace { unsafe impl aya::Pod for ClientAndChannelV6 {} unsafe impl aya::Pod for PortAndPeerV6 {} - - unsafe impl aya::Pod for InterfaceAddressV4 {} - - unsafe impl aya::Pod for InterfaceAddressV6 {} } diff --git a/rust/relay/ebpf-turn-router/src/main.rs b/rust/relay/ebpf-turn-router/src/main.rs index 99c4ea6f3..91c3aba9c 100644 --- a/rust/relay/ebpf-turn-router/src/main.rs +++ b/rust/relay/ebpf-turn-router/src/main.rs @@ -16,42 +16,44 @@ mod try_handle_turn; #[aya_ebpf::macros::xdp] pub fn handle_turn(ctx: aya_ebpf::programs::XdpContext) -> u32 { use aya_ebpf::bindings::xdp_action; - use aya_log_ebpf::{debug, warn}; + use aya_log_ebpf::{debug, trace, warn}; use try_handle_turn::Error; - try_handle_turn::try_handle_turn(&ctx).unwrap_or_else(|e| match e { - Error::NotIp | Error::NotUdp => xdp_action::XDP_PASS, + match try_handle_turn::try_handle_turn(&ctx) { + Ok(()) => { + trace!(&ctx, target: "eBPF", "==> send packet"); - Error::InterfaceIpv4AddressAccessFailed - | Error::InterfaceIpv6AddressAccessFailed - | Error::PacketTooShort - | Error::NotTurn - | Error::NoEntry(_) - | Error::NotAChannelDataMessage - | Error::UdpChecksumMissing - | Error::Ipv4PacketWithOptions => { - debug!(&ctx, "Passing packet to the stack: {}", e); + xdp_action::XDP_TX + } + Err(Error::NotIp | Error::NotUdp) => xdp_action::XDP_PASS, + Err( + e @ (Error::PacketTooShort + | Error::NotTurn + | Error::NotAChannelDataMessage + | Error::UdpChecksumMissing + | Error::Ipv4PacketWithOptions), + ) => { + debug!(&ctx, target: "eBPF", "^^^ pass packet to userspace: {}", e); xdp_action::XDP_PASS } - - // TODO: Remove this when same-host relay-relay is supported. - Error::PacketLoop => { - debug!(&ctx, "Dropping packet: {}", e); + // In a double symmetric NAT setup, it is easily possible for packets to arrive from IPs that don't have channel bindings. + Err(e @ Error::NoEntry(_)) => { + debug!(&ctx,target: "eBPF", "XXX drop packet: {}", e); xdp_action::XDP_DROP } - - // These are exceptions and shouldn't happen in practice - WARN. - Error::BadChannelDataLength - | Error::InterfaceIpv4AddressNotConfigured - | Error::InterfaceIpv6AddressNotConfigured - | Error::XdpAdjustHeadFailed(_) => { - warn!(&ctx, "Dropping packet: {}", e); + Err( + e @ (Error::ArrayIndexOutOfBounds + | Error::IpAddrUnset + | Error::BadChannelDataLength + | Error::XdpAdjustHeadFailed(_)), + ) => { + warn!(&ctx,target: "eBPF", "XXX drop packet: {}", e); xdp_action::XDP_DROP } - }) + } } /// Defines our panic handler. diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn.rs index 6c1e6ae16..225f4589d 100644 --- a/rust/relay/ebpf-turn-router/src/try_handle_turn.rs +++ b/rust/relay/ebpf-turn-router/src/try_handle_turn.rs @@ -1,11 +1,28 @@ +mod adjust_head; +mod channel_data; +mod checksum; +mod config; +mod error; +mod from_ipv4_channel; +mod from_ipv4_udp; +mod from_ipv6_channel; +mod from_ipv6_udp; +mod ref_mut_at; +mod routing; +mod stats; + +use core::net::IpAddr; + +pub use adjust_head::adjust_head; pub use error::Error; -use aya_ebpf::{bindings::xdp_action, helpers::bpf_xdp_adjust_head, programs::XdpContext}; +use aya_ebpf::programs::XdpContext; use aya_log_ebpf::*; use channel_data::CdHdr; -use checksum::ChecksumUpdate; -use ebpf_shared::{ClientAndChannelV4, ClientAndChannelV6, PortAndPeerV4, PortAndPeerV6}; -use error::SupportedChannel; +use ebpf_shared::{ + ClientAndChannel, ClientAndChannelV4, ClientAndChannelV6, PortAndPeer, PortAndPeerV4, + PortAndPeerV6, +}; use network_types::{ eth::{EthHdr, EtherType}, ip::{IpProto, Ipv4Hdr, Ipv6Hdr}, @@ -13,14 +30,6 @@ use network_types::{ }; use ref_mut_at::ref_mut_at; -mod channel_data; -mod channel_maps; -mod checksum; -mod error; -mod interface; -mod ref_mut_at; -mod stats; - /// Lower bound for TURN UDP ports const LOWER_PORT: u16 = 49152; /// Upper bound for TURN UDP ports @@ -31,22 +40,22 @@ const CHAN_START: u16 = 0x4000; const CHAN_END: u16 = 0x7FFF; #[inline(always)] -pub fn try_handle_turn(ctx: &XdpContext) -> Result { +pub fn try_handle_turn(ctx: &XdpContext) -> Result<(), Error> { // SAFETY: The offset must point to the start of a valid `EthHdr`. let eth = unsafe { ref_mut_at::(ctx, 0)? }; - match eth.ether_type { + let num_bytes = match eth.ether_type { EtherType::Ipv4 => try_handle_turn_ipv4(ctx)?, EtherType::Ipv6 => try_handle_turn_ipv6(ctx)?, _ => return Err(Error::NotIp), }; + stats::emit_data_relayed(ctx, num_bytes); - // If we get to here, we modified the packet and need to send it back out again. - Ok(xdp_action::XDP_TX) + Ok(()) } #[inline(always)] -fn try_handle_turn_ipv4(ctx: &XdpContext) -> Result<(), Error> { +fn try_handle_turn_ipv4(ctx: &XdpContext) -> Result { // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. let ipv4 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; @@ -63,35 +72,23 @@ fn try_handle_turn_ipv4(ctx: &XdpContext) -> Result<(), Error> { let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN)? }; let udp_payload_len = udp.len() - UdpHdr::LEN as u16; - trace!( - ctx, - "New packet from {:i}:{} for {:i}:{} with UDP payload {}", - ipv4.src_addr(), - udp.source(), - ipv4.dst_addr(), - udp.dest(), - udp_payload_len - ); - if (LOWER_PORT..=UPPER_PORT).contains(&udp.dest()) { - try_handle_ipv4_udp_to_channel_data(ctx)?; - stats::emit_data_relayed(ctx, udp_payload_len); + try_handle_from_ipv4_udp(ctx)?; - return Ok(()); + return Ok(udp_payload_len); } if udp.dest() == 3478 { - try_handle_ipv4_channel_data_to_udp(ctx)?; - stats::emit_data_relayed(ctx, udp_payload_len - CdHdr::LEN as u16); + try_handle_from_ipv4_channel_data(ctx)?; - return Ok(()); + return Ok(udp_payload_len - CdHdr::LEN as u16); } Err(Error::NotTurn) } #[inline(always)] -fn try_handle_turn_ipv6(ctx: &XdpContext) -> Result<(), Error> { +fn try_handle_turn_ipv6(ctx: &XdpContext) -> Result { // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; @@ -103,60 +100,62 @@ fn try_handle_turn_ipv6(ctx: &XdpContext) -> Result<(), Error> { let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; let udp_payload_len = udp.len() - UdpHdr::LEN as u16; - trace!( - ctx, - "New packet from {:i}:{} for {:i}:{} with UDP payload {}", - ipv6.src_addr(), - udp.source(), - ipv6.dst_addr(), - udp.dest(), - udp_payload_len - ); - if (LOWER_PORT..=UPPER_PORT).contains(&udp.dest()) { - try_handle_ipv6_udp_to_channel_data(ctx)?; - stats::emit_data_relayed(ctx, udp_payload_len); + try_handle_from_ipv6_udp(ctx)?; - return Ok(()); + return Ok(udp_payload_len); } if udp.dest() == 3478 { - try_handle_ipv6_channel_data_to_udp(ctx)?; - stats::emit_data_relayed(ctx, udp_payload_len - CdHdr::LEN as u16); + try_handle_from_ipv6_channel_data(ctx)?; - return Ok(()); + return Ok(udp_payload_len - CdHdr::LEN as u16); } Err(Error::NotTurn) } #[inline(always)] -fn try_handle_ipv4_udp_to_channel_data(ctx: &XdpContext) -> Result<(), Error> { +fn try_handle_from_ipv4_udp(ctx: &XdpContext) -> Result<(), Error> { // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. let ipv4 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; // SAFETY: The offset must point to the start of a valid `UdpHdr`. let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN)? }; - let key = PortAndPeerV4::new(ipv4.src_addr(), udp.dest(), udp.source()); + let pp = PortAndPeerV4::new(ipv4.src_addr(), udp.dest(), udp.source()); - // SAFETY: We only write to these using a single thread in userspace. - if let Some(client_and_channel) = unsafe { channel_maps::UDP_TO_CHAN_44.get(&key) } { - handle_ipv4_udp_to_ipv4_channel(ctx, client_and_channel)?; - return Ok(()); + trace!( + ctx, + target: "eBPF", + "<== new packet from {:i}:{} on allocation {} with UDP payload {}", + pp.peer_ip(), + pp.peer_port(), + pp.allocation_port(), + udp.len() - UdpHdr::LEN as u16 + ); + + let cc = routing::get_client_and_channel(pp)?; + + aya_log_ebpf::trace!( + ctx, + target: "eBPF", + "Routing to {:i}:{} on channel {}", + cc.client_ip(), + cc.client_port(), + cc.channel() + ); + + match cc { + ClientAndChannel::V4(cc) => from_ipv4_udp::to_ipv4_channel(ctx, &cc)?, + ClientAndChannel::V6(cc) => from_ipv4_udp::to_ipv6_channel(ctx, &cc)?, } - // SAFETY: We only write to these using a single thread in userspace. - if let Some(client_and_channel) = unsafe { channel_maps::UDP_TO_CHAN_46.get(&key) } { - handle_ipv4_udp_to_ipv6_channel(ctx, client_and_channel)?; - return Ok(()); - } - - Err(Error::NoEntry(SupportedChannel::Udp4ToChan)) + Ok(()) } #[inline(always)] -fn try_handle_ipv4_channel_data_to_udp(ctx: &XdpContext) -> Result<(), Error> { +fn try_handle_from_ipv4_channel_data(ctx: &XdpContext) -> Result<(), Error> { // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. let ipv4 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; @@ -166,13 +165,13 @@ fn try_handle_ipv4_channel_data_to_udp(ctx: &XdpContext) -> Result<(), Error> { // SAFETY: The offset must point to the start of a valid `CdHdr`. let cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN + UdpHdr::LEN)? }; - let channel_number = u16::from_be_bytes(cd.number); + let channel_number = cd.number(); if !(CHAN_START..=CHAN_END).contains(&channel_number) { return Err(Error::NotAChannelDataMessage); } - let channel_data_len = u16::from_be_bytes(cd.length); + let channel_data_len = cd.length(); let expected_channel_data_len = udp.len() - UdpHdr::LEN as u16 - CdHdr::LEN as u16; // This can happen if we receive packets formed via GSO, like on a loopback interface. @@ -180,1192 +179,178 @@ fn try_handle_ipv4_channel_data_to_udp(ctx: &XdpContext) -> Result<(), Error> { return Err(Error::BadChannelDataLength); } - let key = ClientAndChannelV4::new(ipv4.src_addr(), udp.source(), channel_number); + let cc = ClientAndChannelV4::new(ipv4.src_addr(), udp.source(), channel_number); - // SAFETY: We only write to these using a single thread in userspace. - if let Some(port_and_peer) = unsafe { channel_maps::CHAN_TO_UDP_44.get(&key) } { - // IPv4 to IPv4 - existing logic - handle_ipv4_channel_to_ipv4_udp(ctx, port_and_peer)?; - return Ok(()); - } - - // SAFETY: We only write to these using a single thread in userspace. - if let Some(port_and_peer) = unsafe { channel_maps::CHAN_TO_UDP_46.get(&key) } { - handle_ipv4_channel_to_ipv6_udp(ctx, port_and_peer)?; - return Ok(()); - } - - Err(Error::NoEntry(SupportedChannel::Chan4ToUdp)) -} - -#[inline(always)] -fn try_handle_ipv6_udp_to_channel_data(ctx: &XdpContext) -> Result<(), Error> { - // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. - let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; - - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; - - let key = PortAndPeerV6::new(ipv6.src_addr(), udp.dest(), udp.source()); - - // SAFETY: We only write to these using a single thread in userspace. - if let Some(client_and_channel) = unsafe { channel_maps::UDP_TO_CHAN_66.get(&key) } { - handle_ipv6_udp_to_ipv6_channel(ctx, client_and_channel)?; - return Ok(()); - } - - // SAFETY: We only write to these using a single thread in userspace. - if let Some(client_and_channel) = unsafe { channel_maps::UDP_TO_CHAN_64.get(&key) } { - handle_ipv6_udp_to_ipv4_channel(ctx, client_and_channel)?; - return Ok(()); - } - - Err(Error::NoEntry(SupportedChannel::Udp6ToChan)) -} - -#[inline(always)] -fn try_handle_ipv6_channel_data_to_udp(ctx: &XdpContext) -> Result<(), Error> { - // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. - let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; - - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; - - // SAFETY: The offset must point to the start of a valid `CdHdr`. - let cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN + UdpHdr::LEN)? }; - - let channel_number = u16::from_be_bytes(cd.number); - - if !(CHAN_START..=CHAN_END).contains(&channel_number) { - return Err(Error::NotAChannelDataMessage); - } - - let channel_data_len = u16::from_be_bytes(cd.length); - let expected_channel_data_len = udp.len() - UdpHdr::LEN as u16 - CdHdr::LEN as u16; - - // This can happen if we receive packets formed via GSO, like on a loopback interface. - if channel_data_len != expected_channel_data_len { - return Err(Error::BadChannelDataLength); - } - - let key = ClientAndChannelV6::new(ipv6.src_addr(), udp.source(), u16::from_be_bytes(cd.number)); - - // SAFETY: We only write to these using a single thread in userspace. - if let Some(port_and_peer) = unsafe { channel_maps::CHAN_TO_UDP_66.get(&key) } { - handle_ipv6_channel_to_ipv6_udp(ctx, port_and_peer)?; - return Ok(()); - } - - // SAFETY: We only write to these using a single thread in userspace. - if let Some(port_and_peer) = unsafe { channel_maps::CHAN_TO_UDP_64.get(&key) } { - handle_ipv6_channel_to_ipv4_udp(ctx, port_and_peer)?; - return Ok(()); - } - - Err(Error::NoEntry(SupportedChannel::Chan6ToUdp)) -} - -#[inline(always)] -fn handle_ipv4_udp_to_ipv4_channel( - ctx: &XdpContext, - client_and_channel: &ClientAndChannelV4, -) -> Result<(), Error> { - const NET_EXPANSION: i32 = -(CdHdr::LEN as i32); - - adjust_head(ctx, NET_EXPANSION)?; - - // Now read the old packet data from its NEW location (shifted by 4 bytes) - let old_data_offset = -NET_EXPANSION as usize; - - let (old_eth_src, old_eth_dst, old_eth_type) = { - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let old_eth = unsafe { ref_mut_at::(ctx, old_data_offset)? }; - (old_eth.src_addr, old_eth.dst_addr, old_eth.ether_type) - }; - - let ( - old_ipv4_src, - old_ipv4_dst, - old_ipv4_len, - old_ipv4_check, - old_ipv4_tos, - old_ipv4_id, - old_ipv4_frag_off, - old_ipv4_ttl, - old_ipv4_proto, - ) = { - // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. - let old_ipv4 = unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN)? }; - ( - old_ipv4.src_addr(), - old_ipv4.dst_addr(), - old_ipv4.total_len(), - old_ipv4.checksum(), - old_ipv4.tos, - old_ipv4.id(), - old_ipv4.frag_off, - old_ipv4.ttl, - old_ipv4.proto, - ) - }; - - let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let old_udp = - unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN + Ipv4Hdr::LEN)? }; - ( - old_udp.len(), - old_udp.source(), - old_udp.dest(), - old_udp.check(), - ) - }; - - // - // 1. Ethernet header - // - - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let eth = unsafe { ref_mut_at::(ctx, 0)? }; - eth.src_addr = old_eth_dst; - eth.dst_addr = old_eth_src; - eth.ether_type = old_eth_type; - - // - // 2. IPv4 header - // - - let new_ipv4_src = old_ipv4_dst; - let new_ipv4_dst = client_and_channel.client_ip(); - let new_ipv4_len = old_ipv4_len + CdHdr::LEN as u16; - - // Check for packet loop - would we be sending to ourselves? - if new_ipv4_src == new_ipv4_dst { - return Err(Error::PacketLoop); - } - - // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. - let ipv4 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; - ipv4.set_version(4); // IPv4 - ipv4.set_ihl(5); // No options, 5 * 4 = 20 bytes - ipv4.tos = old_ipv4_tos; // Preserve TOS/DSCP - ipv4.set_total_len(new_ipv4_len); - ipv4.set_id(old_ipv4_id); // Preserve fragment ID - ipv4.frag_off = old_ipv4_frag_off; // Preserve fragment flags - ipv4.ttl = old_ipv4_ttl; // Preserve TTL exactly - ipv4.proto = old_ipv4_proto; // Protocol is UDP - ipv4.set_src_addr(new_ipv4_src); // Swap source and destination - ipv4.set_dst_addr(new_ipv4_dst); // Destination is the client IP - ipv4.set_checksum( - ChecksumUpdate::new(old_ipv4_check) - .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) - .remove_u16(old_ipv4_len) - .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) - .add_u16(new_ipv4_len) - .into_ip_checksum(), + trace!( + ctx, + target: "eBPF", + "<== new packet from {:i}:{} on channel {} with UDP payload {}", + cc.client_ip(), + cc.client_port(), + cc.channel(), + udp.len() - UdpHdr::LEN as u16 ); - // - // 3. UDP header - // + let pp = routing::get_port_and_peer(cc)?; - let new_udp_src = 3478_u16; - let new_udp_dst = client_and_channel.client_port(); - let new_udp_len = old_udp_len + CdHdr::LEN as u16; - let channel_number = client_and_channel.channel(); - let channel_data_length = old_udp_len - UdpHdr::LEN as u16; - - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN)? }; - udp.set_source(new_udp_src); - udp.set_dest(new_udp_dst); - udp.set_len(new_udp_len); - - // Incrementally update UDP checksum - - if old_udp_check == 0 { - // No checksum is valid for UDP IPv4 - we didn't write it, but maybe a middlebox did - udp.set_check(0); - } else { - udp.set_check( - ChecksumUpdate::new(old_udp_check) - .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) - .remove_u16(old_udp_src) - .remove_u16(old_udp_dst) - .remove_u16(old_udp_len) - .remove_u16(old_udp_len) - .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) - .add_u16(new_udp_src) - .add_u16(new_udp_dst) - .add_u16(new_udp_len) - .add_u16(new_udp_len) - .add_u16(channel_number) - .add_u16(channel_data_length) - .into_udp_checksum(), - ); - } - - // - // 4. Channel data header - // - - // SAFETY: The offset must point to the start of a valid `CdHdr`. - let cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN + UdpHdr::LEN)? }; - cd.number = channel_number.to_be_bytes(); - cd.length = channel_data_length.to_be_bytes(); - - Ok(()) -} - -// Convert IPv4 to IPv6 and add channel data -#[inline(always)] -fn handle_ipv4_udp_to_ipv6_channel( - ctx: &XdpContext, - client_and_channel: &ClientAndChannelV6, -) -> Result<(), Error> { - // Expand the packet by 24 bytes for IPv6 header and channel data header - const NET_EXPANSION: i32 = -(Ipv6Hdr::LEN as i32 - Ipv4Hdr::LEN as i32 + CdHdr::LEN as i32); - - adjust_head(ctx, NET_EXPANSION)?; - - // Now read the old packet data from its NEW location (shifted by 24 bytes) - let old_data_offset = -NET_EXPANSION as usize; - - let (old_eth_src, old_eth_dst) = { - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let old_eth = unsafe { ref_mut_at::(ctx, old_data_offset)? }; - (old_eth.src_addr, old_eth.dst_addr) - }; - - let (old_ipv4_src, old_ipv4_dst, old_ipv4_len, old_ipv4_tos, old_ipv4_ttl, old_ipv4_proto) = { - // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. - let old_ipv4 = unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN)? }; - ( - old_ipv4.src_addr(), - old_ipv4.dst_addr(), - old_ipv4.total_len(), - old_ipv4.tos, - old_ipv4.ttl, - old_ipv4.proto, - ) - }; - - let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let old_udp = - unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN + Ipv4Hdr::LEN)? }; - ( - old_udp.len(), - old_udp.source(), - old_udp.dest(), - old_udp.check(), - ) - }; - - // Refuse to compute full UDP checksum. - // We forged these packets, so something's wrong if this is zero. - if old_udp_check == 0 { - return Err(Error::UdpChecksumMissing); - } - - // - // 1. Ethernet header - // - - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let eth = unsafe { ref_mut_at::(ctx, 0)? }; - eth.dst_addr = old_eth_src; // Swap source and destination - eth.src_addr = old_eth_dst; - eth.ether_type = EtherType::Ipv6; // Change to IPv6 - - // - // 2. IPv4 -> IPv6 header - // - - let new_ipv6_src = interface::ipv6_address()?; - let new_ipv6_dst = client_and_channel.client_ip(); - let new_ipv6_len = old_ipv4_len - Ipv4Hdr::LEN as u16 + CdHdr::LEN as u16; - - // Check for packet loop - would we be sending to ourselves? - if new_ipv6_dst == new_ipv6_src { - return Err(Error::PacketLoop); - } - - // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. - let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; - ipv6.set_version(6); - ipv6.set_priority(old_ipv4_tos); - ipv6.flow_label = [0, 0, 0]; // Default flow label - ipv6.set_payload_len(new_ipv6_len); - ipv6.next_hdr = old_ipv4_proto; - ipv6.hop_limit = old_ipv4_ttl; - ipv6.set_src_addr(new_ipv6_src); - ipv6.set_dst_addr(new_ipv6_dst); - - // - // 3. UDP header - // - - let new_udp_src = 3478_u16; - let new_udp_dst = client_and_channel.client_port(); - let new_udp_len = old_udp_len + CdHdr::LEN as u16; - - let channel_number = client_and_channel.channel(); - let channel_data_length = old_udp_len - UdpHdr::LEN as u16; - - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; - udp.set_source(new_udp_src); - udp.set_dest(new_udp_dst); - udp.set_len(new_udp_len); - - // Incrementally update UDP checksum - - udp.set_check( - ChecksumUpdate::new(old_udp_check) - .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) - .remove_u32(u32::from_be_bytes(old_ipv4_dst.octets())) - .remove_u16(old_udp_src) - .remove_u16(old_udp_dst) - .remove_u16(old_udp_len) - .remove_u16(old_udp_len) - .add_u128(u128::from_be_bytes(new_ipv6_src.octets())) - .add_u128(u128::from_be_bytes(new_ipv6_dst.octets())) - .add_u16(new_udp_src) - .add_u16(new_udp_dst) - .add_u16(new_udp_len) - .add_u16(new_udp_len) - .add_u16(channel_number) - .add_u16(channel_data_length) - .into_udp_checksum(), + aya_log_ebpf::trace!( + ctx, + target: "eBPF", + "Routing packet to {:i}:{} on allocation {}", + pp.peer_ip(), + pp.peer_port(), + pp.allocation_port(), ); - // - // 4. Channel data header - // + if is_own_public_ip(pp.peer_ip())? { + let pp = pp.flip_ports(); + let cc = routing::get_client_and_channel(pp)?; - // SAFETY: The offset must point to the start of a valid `CdHdr`. - let cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN + UdpHdr::LEN)? }; - cd.number = channel_number.to_be_bytes(); - cd.length = channel_data_length.to_be_bytes(); - - Ok(()) -} - -#[inline(always)] -fn handle_ipv4_channel_to_ipv4_udp( - ctx: &XdpContext, - port_and_peer: &PortAndPeerV4, -) -> Result<(), Error> { - const NET_SHRINK: i32 = CdHdr::LEN as i32; // Shrink by 4 bytes for channel data header - - let (old_eth_src, old_eth_dst, old_eth_type) = { - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let old_eth = unsafe { ref_mut_at::(ctx, 0)? }; - (old_eth.src_addr, old_eth.dst_addr, old_eth.ether_type) - }; - - let ( - old_ipv4_src, - old_ipv4_dst, - old_ipv4_len, - old_ipv4_check, - old_ipv4_tos, - old_ipv4_id, - old_ipv4_frag_off, - old_ipv4_ttl, - old_ipv4_proto, - ) = { - // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. - let old_ipv4 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; - ( - old_ipv4.src_addr(), - old_ipv4.dst_addr(), - old_ipv4.total_len(), - old_ipv4.checksum(), - old_ipv4.tos, - old_ipv4.id(), - old_ipv4.frag_off, - old_ipv4.ttl, - old_ipv4.proto, - ) - }; - - let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let old_udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN)? }; - ( - old_udp.len(), - old_udp.source(), - old_udp.dest(), - old_udp.check(), - ) - }; - - let (channel_number, channel_data_length) = { - // SAFETY: The offset must point to the start of a valid `CdHdr`. - let old_cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN + UdpHdr::LEN)? }; - ( - u16::from_be_bytes(old_cd.number), - u16::from_be_bytes(old_cd.length), - ) - }; - - // - // 1. Ethernet header - // - - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let eth = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize)? }; - eth.dst_addr = old_eth_src; // Swap source and destination - eth.src_addr = old_eth_dst; - eth.ether_type = old_eth_type; - - // - // 2. IPv4 header - // - - let new_ipv4_src = old_ipv4_dst; // Swap source and destination - let new_ipv4_dst = port_and_peer.peer_ip(); - let new_ipv4_len = old_ipv4_len - CdHdr::LEN as u16; - - // Check for packet loop - would we be sending to ourselves? - if new_ipv4_src == new_ipv4_dst { - return Err(Error::PacketLoop); - } - - // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. - let ipv4 = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN)? }; - ipv4.set_version(4); // IPv4 - ipv4.set_ihl(5); // No options, 5 * 4 = 20 bytes - ipv4.tos = old_ipv4_tos; // Preserve TOS/DSCP - ipv4.set_total_len(new_ipv4_len); - ipv4.set_id(old_ipv4_id); // Preserve ID - ipv4.frag_off = old_ipv4_frag_off; // Preserve fragment flags - ipv4.ttl = old_ipv4_ttl; // Preserve TTL exactly - ipv4.proto = old_ipv4_proto; // Protocol is UDP - ipv4.set_src_addr(new_ipv4_src); - ipv4.set_dst_addr(new_ipv4_dst); - ipv4.set_checksum( - ChecksumUpdate::new(old_ipv4_check) - .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) - .remove_u16(old_ipv4_len) - .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) - .add_u16(new_ipv4_len) - .into_ip_checksum(), - ); - - // - // 3. UDP header - // - - let new_udp_src = port_and_peer.allocation_port(); - let new_udp_dst = port_and_peer.peer_port(); - let new_udp_len = old_udp_len - CdHdr::LEN as u16; - - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let udp = - unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN + Ipv4Hdr::LEN)? }; - udp.set_source(new_udp_src); - udp.set_dest(new_udp_dst); - udp.set_len(new_udp_len); - - // Incrementally update UDP checksum - - if old_udp_check == 0 { - // No checksum is valid for UDP IPv4 - we didn't write it, but maybe a middlebox did - udp.set_check(0); - } else { - udp.set_check( - ChecksumUpdate::new(old_udp_check) - .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) - .remove_u16(old_udp_src) - .remove_u16(old_udp_dst) - .remove_u16(old_udp_len) - .remove_u16(old_udp_len) - .remove_u16(channel_number) - .remove_u16(channel_data_length) - .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) - .add_u16(new_udp_src) - .add_u16(new_udp_dst) - .add_u16(new_udp_len) - .add_u16(new_udp_len) - .into_udp_checksum(), - ); - } - - adjust_head(ctx, NET_SHRINK)?; - - Ok(()) -} - -// Convert IPv4 to IPv6 and remove channel data header -#[inline(always)] -fn handle_ipv4_channel_to_ipv6_udp( - ctx: &XdpContext, - port_and_peer: &PortAndPeerV6, -) -> Result<(), Error> { - const NET_EXPANSION: i32 = Ipv4Hdr::LEN as i32 - Ipv6Hdr::LEN as i32 + CdHdr::LEN as i32; - - adjust_head(ctx, NET_EXPANSION)?; - - // Now read the old packet data from its NEW location - let old_data_offset = (-NET_EXPANSION) as usize; - - let (old_src_mac, old_dst_mac) = { - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let old_eth = unsafe { ref_mut_at::(ctx, old_data_offset)? }; - (old_eth.src_addr, old_eth.dst_addr) - }; - - let (old_ipv4_src, old_ipv4_dst, old_ipv4_tos, old_ipv4_ttl, old_ipv4_proto) = { - // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. - let old_ipv4 = unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN)? }; - ( - old_ipv4.src_addr(), - old_ipv4.dst_addr(), - old_ipv4.tos, - old_ipv4.ttl, - old_ipv4.proto, - ) - }; - - let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let old_udp = - unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN + Ipv4Hdr::LEN)? }; - ( - old_udp.len(), - old_udp.source(), - old_udp.dest(), - old_udp.check(), - ) - }; - - // Refuse to compute full UDP checksum. - // We forged these packets, so something's wrong if this is zero. - if old_udp_check == 0 { - return Err(Error::UdpChecksumMissing); - } - - let (channel_number, channel_data_length) = { - // SAFETY: The offset must point to the start of a valid `CdHdr`. - let old_cd = unsafe { - ref_mut_at::( - ctx, - old_data_offset + EthHdr::LEN + Ipv4Hdr::LEN + UdpHdr::LEN, - )? - }; - ( - u16::from_be_bytes(old_cd.number), - u16::from_be_bytes(old_cd.length), - ) - }; - - // - // 1. Ethernet header - // - - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let eth = unsafe { ref_mut_at::(ctx, 0)? }; - eth.dst_addr = old_src_mac; // Swap MACs - eth.src_addr = old_dst_mac; - eth.ether_type = EtherType::Ipv6; // Change to IPv6 - - // - // 2. IPv6 header - // - - let new_ipv6_src = interface::ipv6_address()?; - let new_ipv6_dst = port_and_peer.peer_ip(); - let new_udp_len = old_udp_len - CdHdr::LEN as u16; - - // Check for packet loop - would we be sending to ourselves? - if new_ipv6_src == new_ipv6_dst { - return Err(Error::PacketLoop); - } - - // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. - let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; - ipv6.set_version(6); // IPv6 - ipv6.set_priority(old_ipv4_tos); - ipv6.flow_label = [0, 0, 0]; - ipv6.set_payload_len(new_udp_len); - ipv6.next_hdr = old_ipv4_proto; - ipv6.hop_limit = old_ipv4_ttl; - ipv6.set_src_addr(new_ipv6_src); - ipv6.set_dst_addr(new_ipv6_dst); - - // - // 3. UDP header - // - - let new_udp_src = port_and_peer.allocation_port(); - let new_udp_dst = port_and_peer.peer_port(); - - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; - udp.set_source(new_udp_src); - udp.set_dest(new_udp_dst); - udp.set_len(new_udp_len); - - // Incrementally update UDP checksum - - udp.set_check( - ChecksumUpdate::new(old_udp_check) - .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) - .remove_u32(u32::from_be_bytes(old_ipv4_dst.octets())) - .remove_u16(old_udp_src) - .remove_u16(old_udp_dst) - .remove_u16(old_udp_len) - .remove_u16(old_udp_len) - .remove_u16(channel_number) - .remove_u16(channel_data_length) - .add_u128(u128::from_be_bytes(new_ipv6_src.octets())) - .add_u128(u128::from_be_bytes(new_ipv6_dst.octets())) - .add_u16(new_udp_src) - .add_u16(new_udp_dst) - .add_u16(new_udp_len) - .add_u16(new_udp_len) - .into_udp_checksum(), - ); - - Ok(()) -} - -#[inline(always)] -fn handle_ipv6_udp_to_ipv6_channel( - ctx: &XdpContext, - client_and_channel: &ClientAndChannelV6, -) -> Result<(), Error> { - // Expand by 4 bytes for channel data header - const NET_EXPANSION: i32 = -(CdHdr::LEN as i32); - - adjust_head(ctx, NET_EXPANSION)?; - - // Now read the old packet data from its NEW location (shifted by 4 bytes) - let old_data_offset = CdHdr::LEN; - - let (old_eth_src, old_eth_dst, old_eth_type) = { - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let old_eth = unsafe { ref_mut_at::(ctx, old_data_offset)? }; - (old_eth.src_addr, old_eth.dst_addr, old_eth.ether_type) - }; - - let ( - old_ipv6_src, - old_ipv6_dst, - old_ipv6_len, - old_ipv6_priority, - old_ipv6_flow_label, - old_ipv6_hop_limit, - old_ipv6_next_hdr, - ) = { - // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. - let old_ipv6 = unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN)? }; - ( - old_ipv6.src_addr(), - old_ipv6.dst_addr(), - old_ipv6.payload_len(), - old_ipv6.priority(), - old_ipv6.flow_label, - old_ipv6.hop_limit, - old_ipv6.next_hdr, - ) - }; - - let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let old_udp = - unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN + Ipv6Hdr::LEN)? }; - ( - old_udp.len(), - old_udp.source(), - old_udp.dest(), - old_udp.check(), - ) - }; - - // - // 1. Ethernet header - // - - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let eth = unsafe { ref_mut_at::(ctx, 0)? }; - eth.src_addr = old_eth_dst; // Swap source and destination - eth.dst_addr = old_eth_src; - eth.ether_type = old_eth_type; - - // - // 2. IPv6 header - // - - let new_ipv6_src = old_ipv6_dst; - let new_ipv6_dst = client_and_channel.client_ip(); - let new_ipv6_len = old_ipv6_len + CdHdr::LEN as u16; - - // Check for packet loop - would we be sending to ourselves? - if new_ipv6_src == new_ipv6_dst { - return Err(Error::PacketLoop); - } - - // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. - let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; - // Set fields explicitly to avoid reading potentially corrupted memory - ipv6.set_version(6); // IPv6 - ipv6.set_priority(old_ipv6_priority); - ipv6.flow_label = old_ipv6_flow_label; - ipv6.set_payload_len(new_ipv6_len); - ipv6.next_hdr = old_ipv6_next_hdr; - ipv6.hop_limit = old_ipv6_hop_limit; - ipv6.set_src_addr(new_ipv6_src); - ipv6.set_dst_addr(new_ipv6_dst); - - // - // 3. UDP header - // - - let channel_number = client_and_channel.channel(); - let channel_data_length = old_udp_len - UdpHdr::LEN as u16; - let new_udp_len = old_udp_len + CdHdr::LEN as u16; - let new_udp_src = 3478_u16; - let new_udp_dst = client_and_channel.client_port(); - - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; - udp.set_source(new_udp_src); - udp.set_dest(new_udp_dst); - udp.set_len(new_udp_len); - - // Incrementally update UDP checksum - - udp.set_check( - ChecksumUpdate::new(old_udp_check) - .remove_u128(u128::from_be_bytes(old_ipv6_src.octets())) - .remove_u16(old_udp_src) - .remove_u16(old_udp_dst) - .remove_u16(old_udp_len) - .remove_u16(old_udp_len) - .add_u128(u128::from_be_bytes(new_ipv6_dst.octets())) - .add_u16(new_udp_src) - .add_u16(new_udp_dst) - .add_u16(new_udp_len) - .add_u16(new_udp_len) - .add_u16(channel_number) - .add_u16(channel_data_length) - .into_udp_checksum(), - ); - - // - // 4. Channel data header - // - - // SAFETY: The offset must point to the start of a valid `CdHdr`. - let cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN + UdpHdr::LEN)? }; - cd.number = channel_number.to_be_bytes(); - cd.length = channel_data_length.to_be_bytes(); - - Ok(()) -} - -// Convert IPv6 to IPv4 and add channel data -#[inline(always)] -fn handle_ipv6_udp_to_ipv4_channel( - ctx: &XdpContext, - client_and_channel: &ClientAndChannelV4, -) -> Result<(), Error> { - // 40 - 20 - 4 = 16 bytes shrink - const NET_SHRINK: i32 = Ipv6Hdr::LEN as i32 - Ipv4Hdr::LEN as i32 - CdHdr::LEN as i32; - - let (old_eth_src, old_eth_dst) = { - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let old_eth = unsafe { ref_mut_at::(ctx, 0)? }; - (old_eth.src_addr, old_eth.dst_addr) - }; - - let (old_ipv6_src, old_ipv6_dst, old_ipv6_priority, old_ipv6_hop_limit, old_ipv6_next_hdr) = { - // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. - let old_ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; - ( - old_ipv6.src_addr(), - old_ipv6.dst_addr(), - old_ipv6.priority(), - old_ipv6.hop_limit, - old_ipv6.next_hdr, - ) - }; - - let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let old_udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; - ( - old_udp.len(), - old_udp.source(), - old_udp.dest(), - old_udp.check(), - ) - }; - - // - // 1. Ethernet header - // - - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let eth = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize)? }; - eth.src_addr = old_eth_dst; // Swap source and destination - eth.dst_addr = old_eth_src; - eth.ether_type = EtherType::Ipv4; // Change to IPv4 - - // - // 2. IPv6 -> IPv4 header - // - - let new_ipv4_src = interface::ipv4_address()?; - let new_ipv4_dst = client_and_channel.client_ip(); - let new_udp_len = old_udp_len + CdHdr::LEN as u16; - let new_ipv4_len = Ipv4Hdr::LEN as u16 + new_udp_len; - - // Check for packet loop - would we be sending to ourselves? - if new_ipv4_dst == new_ipv4_src { - return Err(Error::PacketLoop); - } - - // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. - let ipv4 = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN)? }; - ipv4.set_version(4); - ipv4.set_ihl(5); // No options - ipv4.tos = old_ipv6_priority; - ipv4.set_total_len(new_ipv4_len); - ipv4.set_id(0); // Default ID - ipv4.frag_off = 0x4000_u16.to_be_bytes(); // Don't fragment - ipv4.ttl = old_ipv6_hop_limit; // Preserve hop limit - ipv4.proto = old_ipv6_next_hdr; // Preserve protocol - ipv4.set_src_addr(new_ipv4_src); - ipv4.set_dst_addr(new_ipv4_dst); - - // Calculate fresh checksum - let check = checksum::new_ipv4(ipv4); - ipv4.set_checksum(check); - - // - // 3. UDP header - // - - let new_udp_src = 3478_u16; // Fixed source port for TURN - let new_udp_dst = client_and_channel.client_port(); - let channel_number = client_and_channel.channel(); - let channel_data_length = old_udp_len - UdpHdr::LEN as u16; - - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let udp = - unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN + Ipv4Hdr::LEN)? }; - udp.set_source(new_udp_src); - udp.set_dest(new_udp_dst); - udp.set_len(new_udp_len); - - // Incrementally update UDP checksum - - udp.set_check( - ChecksumUpdate::new(old_udp_check) - .remove_u128(u128::from_be_bytes(old_ipv6_src.octets())) - .remove_u128(u128::from_be_bytes(old_ipv6_dst.octets())) - .remove_u16(old_udp_src) - .remove_u16(old_udp_dst) - .remove_u16(old_udp_len) - .remove_u16(old_udp_len) - .add_u32(u32::from_be_bytes(new_ipv4_src.octets())) - .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) - .add_u16(new_udp_src) - .add_u16(new_udp_dst) - .add_u16(new_udp_len) - .add_u16(new_udp_len) - .add_u16(channel_number) - .add_u16(channel_data_length) - .into_udp_checksum(), - ); - - // - // 4. Channel data header - // - - // SAFETY: The offset must point to the start of a valid `CdHdr`. - let cd = unsafe { - ref_mut_at::( + aya_log_ebpf::trace!( ctx, - NET_SHRINK as usize + EthHdr::LEN + Ipv4Hdr::LEN + UdpHdr::LEN, - )? - }; - cd.number = channel_number.to_be_bytes(); - cd.length = channel_data_length.to_be_bytes(); + target: "eBPF", + "Routing packet to {:i}:{} on channel {}", + cc.client_ip(), + cc.client_port(), + cc.channel() + ); - adjust_head(ctx, NET_SHRINK)?; + match cc { + ClientAndChannel::V4(cc) => from_ipv4_channel::to_ipv4_channel(ctx, &cc)?, + ClientAndChannel::V6(cc) => from_ipv4_channel::to_ipv6_channel(ctx, &cc)?, + } + + return Ok(()); + } + + match pp { + PortAndPeer::V4(pp) => from_ipv4_channel::to_ipv4_udp(ctx, &pp)?, + PortAndPeer::V6(pp) => from_ipv4_channel::to_ipv6_udp(ctx, &pp)?, + } Ok(()) } #[inline(always)] -fn handle_ipv6_channel_to_ipv6_udp( - ctx: &XdpContext, - port_and_peer: &PortAndPeerV6, -) -> Result<(), Error> { - const NET_SHRINK: i32 = CdHdr::LEN as i32; // Shrink by 4 bytes for channel data header - - let (old_eth_src, old_eth_dst, old_eth_type) = { - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let old_eth = unsafe { ref_mut_at::(ctx, 0)? }; - (old_eth.src_addr, old_eth.dst_addr, old_eth.ether_type) - }; - - let ( - old_ipv6_src, - old_ipv6_dst, - old_ipv6_len, - old_ipv6_priority, - old_ipv6_flow_label, - old_ipv6_hop_limit, - old_ipv6_next_hdr, - ) = { - // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. - let old_ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; - ( - old_ipv6.src_addr(), - old_ipv6.dst_addr(), - old_ipv6.payload_len(), - old_ipv6.priority(), - old_ipv6.flow_label, - old_ipv6.hop_limit, - old_ipv6.next_hdr, - ) - }; - - let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let old_udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; - ( - old_udp.len(), - old_udp.source(), - old_udp.dest(), - old_udp.check(), - ) - }; - - let (channel_number, channel_data_length) = { - // SAFETY: The offset must point to the start of a valid `CdHdr`. - let old_cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN + UdpHdr::LEN)? }; - ( - u16::from_be_bytes(old_cd.number), - u16::from_be_bytes(old_cd.length), - ) - }; - - // - // 1. Ethernet header - // - - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let eth = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize)? }; - eth.src_addr = old_eth_dst; // Swap source and destination - eth.dst_addr = old_eth_src; - eth.ether_type = old_eth_type; - - // - // 2. IPv6 header - // - - let new_ipv6_src = old_ipv6_dst; // Swap source and destination - let new_ipv6_dst = port_and_peer.peer_ip(); - let new_ipv6_len = old_ipv6_len - CdHdr::LEN as u16; - - // Check for packet loop - would we be sending to ourselves? - if new_ipv6_src == new_ipv6_dst { - return Err(Error::PacketLoop); - } - +fn try_handle_from_ipv6_udp(ctx: &XdpContext) -> Result<(), Error> { // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. - let ipv6 = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN)? }; - ipv6.set_version(6); // IPv6 - ipv6.set_priority(old_ipv6_priority); - ipv6.flow_label = old_ipv6_flow_label; - ipv6.set_payload_len(new_ipv6_len); - ipv6.next_hdr = old_ipv6_next_hdr; - ipv6.hop_limit = old_ipv6_hop_limit; - ipv6.set_src_addr(new_ipv6_src); - ipv6.set_dst_addr(new_ipv6_dst); - - // - // 3. UDP header - // - - let new_udp_src = port_and_peer.allocation_port(); - let new_udp_dst = port_and_peer.peer_port(); - let new_udp_len = old_udp_len - CdHdr::LEN as u16; + let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let udp = - unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN + Ipv6Hdr::LEN)? }; - udp.set_source(new_udp_src); - udp.set_dest(new_udp_dst); - udp.set_len(new_udp_len); + let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; - // Incrementally update UDP checksum + let pp = PortAndPeerV6::new(ipv6.src_addr(), udp.dest(), udp.source()); - udp.set_check( - ChecksumUpdate::new(old_udp_check) - .remove_u128(u128::from_be_bytes(old_ipv6_src.octets())) - .remove_u16(old_udp_src) - .remove_u16(old_udp_dst) - .remove_u16(old_udp_len) - .remove_u16(old_udp_len) - .remove_u16(channel_number) - .remove_u16(channel_data_length) - .add_u128(u128::from_be_bytes(new_ipv6_dst.octets())) - .add_u16(new_udp_src) - .add_u16(new_udp_dst) - .add_u16(new_udp_len) - .add_u16(new_udp_len) - .into_udp_checksum(), + trace!( + ctx, + target: "eBPF", + "<== new packet from {:i}:{} on allocation {} with UDP payload {}", + pp.peer_ip(), + pp.peer_port(), + pp.allocation_port(), + udp.len() - UdpHdr::LEN as u16 ); - adjust_head(ctx, NET_SHRINK)?; + let cc = routing::get_client_and_channel(pp)?; - Ok(()) -} + aya_log_ebpf::trace!( + ctx, + target: "eBPF", + "Routing packet to {:i}:{} on channel {}", + cc.client_ip(), + cc.client_port(), + cc.channel() + ); -// Convert IPv6 to IPv4 and remove channel data -#[inline(always)] -fn handle_ipv6_channel_to_ipv4_udp( - ctx: &XdpContext, - port_and_peer: &PortAndPeerV4, -) -> Result<(), Error> { - // Shrink by 24 bytes: 20 for the IP header diff and 4 for the removed channel data header - const NET_SHRINK: i32 = Ipv6Hdr::LEN as i32 - Ipv4Hdr::LEN as i32 + CdHdr::LEN as i32; - - let (old_eth_src, old_eth_dst) = { - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let old_eth = unsafe { ref_mut_at::(ctx, 0)? }; - (old_eth.src_addr, old_eth.dst_addr) - }; - - let (old_ipv6_src, old_ipv6_dst, old_ipv6_priority, old_ipv6_hop_limit, old_ipv6_next_hdr) = { - // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. - let old_ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; - ( - old_ipv6.src_addr(), - old_ipv6.dst_addr(), - old_ipv6.priority(), - old_ipv6.hop_limit, - old_ipv6.next_hdr, - ) - }; - - let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let old_udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; - ( - old_udp.len(), - old_udp.source(), - old_udp.dest(), - old_udp.check(), - ) - }; - - let (channel_number, channel_data_length) = { - // SAFETY: The offset must point to the start of a valid `CdHdr`. - let old_cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN + UdpHdr::LEN)? }; - ( - u16::from_be_bytes(old_cd.number), - u16::from_be_bytes(old_cd.length), - ) - }; - - // - // 1. Ethernet header - // - - // SAFETY: The offset must point to the start of a valid `EthHdr`. - let eth = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize)? }; - eth.src_addr = old_eth_dst; // Swap source and destination - eth.dst_addr = old_eth_src; - eth.ether_type = EtherType::Ipv4; // Change to IPv4 - - // - // 2. IPv6 -> IPv4 header - // - - let new_ipv4_src = interface::ipv4_address()?; - let new_ipv4_dst = port_and_peer.peer_ip(); - let new_ipv4_len = old_udp_len - CdHdr::LEN as u16 + Ipv4Hdr::LEN as u16; - - // Check for packet loop - would we be sending to ourselves? - if new_ipv4_src == new_ipv4_dst { - return Err(Error::PacketLoop); + match cc { + ClientAndChannel::V4(cc) => from_ipv6_udp::to_ipv4_channel(ctx, &cc)?, + ClientAndChannel::V6(cc) => from_ipv6_udp::to_ipv6_channel(ctx, &cc)?, } - // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. - let ipv4 = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN)? }; - ipv4.set_version(4); - ipv4.set_ihl(5); // No options - ipv4.tos = old_ipv6_priority; // Copy TOS from IPv6 - ipv4.set_total_len(new_ipv4_len); - ipv4.set_id(0); // Default ID - ipv4.frag_off = 0x4000_u16.to_be_bytes(); // Don't fragment - ipv4.ttl = old_ipv6_hop_limit; // Preserve TTL - ipv4.proto = old_ipv6_next_hdr; // Copy protocol from IPv6 - ipv4.set_src_addr(new_ipv4_src); - ipv4.set_dst_addr(new_ipv4_dst); - - // Calculate fresh checksum - let check = checksum::new_ipv4(ipv4); - ipv4.set_checksum(check); - - // - // 3. UDP header - // - - let new_udp_src = port_and_peer.allocation_port(); - let new_udp_dst = port_and_peer.peer_port(); - let new_udp_len = old_udp_len - CdHdr::LEN as u16; - - // SAFETY: The offset must point to the start of a valid `UdpHdr`. - let udp = - unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN + Ipv4Hdr::LEN)? }; - udp.set_source(new_udp_src); - udp.set_dest(new_udp_dst); - udp.set_len(new_udp_len); - - // Incrementally update UDP checksum - - udp.set_check( - ChecksumUpdate::new(old_udp_check) - .remove_u128(u128::from_be_bytes(old_ipv6_src.octets())) - .remove_u128(u128::from_be_bytes(old_ipv6_dst.octets())) - .remove_u16(old_udp_src) - .remove_u16(old_udp_dst) - .remove_u16(old_udp_len) - .remove_u16(old_udp_len) - .remove_u16(channel_number) - .remove_u16(channel_data_length) - .add_u32(u32::from_be_bytes(new_ipv4_src.octets())) - .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) - .add_u16(new_udp_src) - .add_u16(new_udp_dst) - .add_u16(new_udp_len) - .add_u16(new_udp_len) - .into_udp_checksum(), - ); - - adjust_head(ctx, NET_SHRINK)?; - Ok(()) } #[inline(always)] -fn adjust_head(ctx: &XdpContext, size: i32) -> Result<(), Error> { - // SAFETY: The attach mode and NIC driver support headroom adjustment by `size` bytes. - let ret = unsafe { bpf_xdp_adjust_head(ctx.ctx, size) }; - if ret < 0 { - return Err(Error::XdpAdjustHeadFailed(ret)); +fn try_handle_from_ipv6_channel_data(ctx: &XdpContext) -> Result<(), Error> { + // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. + let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; + + // SAFETY: The offset must point to the start of a valid `CdHdr`. + let cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN + UdpHdr::LEN)? }; + + let channel_number = cd.number(); + + if !(CHAN_START..=CHAN_END).contains(&channel_number) { + return Err(Error::NotAChannelDataMessage); + } + + let channel_data_len = u16::from_be_bytes(cd.length); + let expected_channel_data_len = udp.len() - UdpHdr::LEN as u16 - CdHdr::LEN as u16; + + // This can happen if we receive packets formed via GSO, like on a loopback interface. + if channel_data_len != expected_channel_data_len { + return Err(Error::BadChannelDataLength); + } + + let cc = ClientAndChannelV6::new(ipv6.src_addr(), udp.source(), cd.number()); + + trace!( + ctx, + target: "eBPF", + "<== new packet from {:i}:{} on channel {} with UDP payload {}", + cc.client_ip(), + cc.client_port(), + cc.channel(), + udp.len() - UdpHdr::LEN as u16 + ); + + let pp = routing::get_port_and_peer(cc)?; + + aya_log_ebpf::trace!( + ctx, + target: "eBPF", + "Routing packet to {:i}:{} on allocation {}", + pp.peer_ip(), + pp.peer_port(), + pp.allocation_port(), + ); + + if is_own_public_ip(pp.peer_ip())? { + let pp = pp.flip_ports(); + let cc = routing::get_client_and_channel(pp)?; + + aya_log_ebpf::trace!( + ctx, + target: "eBPF", + "Routing packet to {:i}:{} on channel {}", + cc.client_ip(), + cc.client_port(), + cc.channel() + ); + + match cc { + ClientAndChannel::V4(cc) => from_ipv6_channel::to_ipv4_channel(ctx, &cc)?, + ClientAndChannel::V6(cc) => from_ipv6_channel::to_ipv6_channel(ctx, &cc)?, + } + + return Ok(()); + } + + match pp { + PortAndPeer::V4(pp) => from_ipv6_channel::to_ipv4_udp(ctx, &pp)?, + PortAndPeer::V6(pp) => from_ipv6_channel::to_ipv6_udp(ctx, &pp)?, } Ok(()) } + +#[inline(always)] +fn is_own_public_ip(ip: IpAddr) -> Result { + let ipv4_public = config::public_ipv4_address()?; + let ipv6_public = config::public_ipv6_address()?; + + Ok(ip == ipv4_public || ip == ipv6_public) +} diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn/adjust_head.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn/adjust_head.rs new file mode 100644 index 000000000..3b48ec68e --- /dev/null +++ b/rust/relay/ebpf-turn-router/src/try_handle_turn/adjust_head.rs @@ -0,0 +1,14 @@ +use aya_ebpf::{helpers::generated::bpf_xdp_adjust_head, programs::XdpContext}; + +use crate::try_handle_turn::Error; + +#[inline(always)] +pub fn adjust_head(ctx: &XdpContext, size: i32) -> Result<(), Error> { + // SAFETY: The attach mode and NIC driver support headroom adjustment by `size` bytes. + let ret = unsafe { bpf_xdp_adjust_head(ctx.ctx, size) }; + if ret < 0 { + return Err(Error::XdpAdjustHeadFailed(ret)); + } + + Ok(()) +} diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn/channel_data.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn/channel_data.rs index ad3617528..296ba2be7 100644 --- a/rust/relay/ebpf-turn-router/src/try_handle_turn/channel_data.rs +++ b/rust/relay/ebpf-turn-router/src/try_handle_turn/channel_data.rs @@ -7,4 +7,12 @@ pub struct CdHdr { impl CdHdr { pub const LEN: usize = core::mem::size_of::(); + + pub fn number(&self) -> u16 { + u16::from_be_bytes(self.number) + } + + pub fn length(&self) -> u16 { + u16::from_be_bytes(self.length) + } } diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn/channel_maps.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn/channel_maps.rs deleted file mode 100644 index 1f2fd35d9..000000000 --- a/rust/relay/ebpf-turn-router/src/try_handle_turn/channel_maps.rs +++ /dev/null @@ -1,65 +0,0 @@ -//! Houses all combinations of IPv4 <> IPv6 and Channel <> UDP mappings. -//! -//! Testing has shown that these maps are safe to use as long as we aren't -//! writing to them from multiple threads at the same time. Since we only update these -//! from the single-threaded eventloop in userspace, we are ok. -//! See . - -use aya_ebpf::{macros::map, maps::HashMap}; -use ebpf_shared::{ClientAndChannelV4, ClientAndChannelV6, PortAndPeerV4, PortAndPeerV6}; - -const NUM_ENTRIES: u32 = 0x10000; - -#[map] -pub static CHAN_TO_UDP_44: HashMap = - HashMap::with_max_entries(NUM_ENTRIES, 0); -#[map] -pub static UDP_TO_CHAN_44: HashMap = - HashMap::with_max_entries(NUM_ENTRIES, 0); -#[map] -pub static CHAN_TO_UDP_66: HashMap = - HashMap::with_max_entries(NUM_ENTRIES, 0); -#[map] -pub static UDP_TO_CHAN_66: HashMap = - HashMap::with_max_entries(NUM_ENTRIES, 0); -#[map] -pub static CHAN_TO_UDP_46: HashMap = - HashMap::with_max_entries(NUM_ENTRIES, 0); -#[map] -pub static UDP_TO_CHAN_46: HashMap = - HashMap::with_max_entries(NUM_ENTRIES, 0); -#[map] -pub static CHAN_TO_UDP_64: HashMap = - HashMap::with_max_entries(NUM_ENTRIES, 0); -#[map] -pub static UDP_TO_CHAN_64: HashMap = - HashMap::with_max_entries(NUM_ENTRIES, 0); - -#[cfg(test)] -mod tests { - use super::*; - - /// Memory overhead of an eBPF map. - /// - /// Determined empirically. - const HASH_MAP_OVERHEAD: f32 = 1.5; - - #[test] - fn hashmaps_are_less_than_11_mb() { - let ipv4_datatypes = - core::mem::size_of::() + core::mem::size_of::(); - let ipv6_datatypes = - core::mem::size_of::() + core::mem::size_of::(); - - let ipv4_map_size = ipv4_datatypes as f32 * NUM_ENTRIES as f32 * HASH_MAP_OVERHEAD; - let ipv6_map_size = ipv6_datatypes as f32 * NUM_ENTRIES as f32 * HASH_MAP_OVERHEAD; - - let total_map_size = (ipv4_map_size + ipv6_map_size) * 2_f32; - let total_map_size_mb = total_map_size / 1024_f32 / 1024_f32; - - assert!( - total_map_size_mb < 11_f32, - "Total map size = {total_map_size_mb} MB" - ); - } -} diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn/checksum.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn/checksum.rs index 8019edd07..09757786e 100644 --- a/rust/relay/ebpf-turn-router/src/try_handle_turn/checksum.rs +++ b/rust/relay/ebpf-turn-router/src/try_handle_turn/checksum.rs @@ -11,6 +11,19 @@ //! 3. Subtraction in one's complement arithmetic is implemented as the addition of the one's complement of the number to be subtracted. //! //! This allows us to e.g. take an existing IP header checksum and update it to account for just the destination address changing. +//! +//! To use this module, create a new instance of [`ChecksumUpdate`] from the old checksum and then chain the appropriate functions on it. +//! For example, if you are changing the source port of a UDP packet, you want to: +//! - [`remove_u16`](ChecksumUpdate::remove_u16) the old port +//! - [`add_u16`](ChecksumUpdate::add_u16) the new port +//! +//! This will adjust the checksum as if it would have been computed for the packet with the new port. +//! +//! When re-routing packets, we often send from our own address. +//! In other words, the previous destination becomes the new source. +//! In that case, it is common to optimise the checksum update by not removing the old destination at all. +//! Instead we are basically replacing the old source with the new destination. +//! Checksum computation is commutative, i.e. it doesn't care in which order the individual fields got added. use network_types::ip::Ipv4Hdr; diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn/config.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn/config.rs new file mode 100644 index 000000000..db69100f6 --- /dev/null +++ b/rust/relay/ebpf-turn-router/src/try_handle_turn/config.rs @@ -0,0 +1,49 @@ +//! Per-CPU data structures to store relay interface addresses. + +use core::net::{Ipv4Addr, Ipv6Addr}; + +use aya_ebpf::{macros::map, maps::PerCpuArray}; + +use crate::try_handle_turn::Error; + +#[map] +static INT_ADDR_V4: PerCpuArray<[u8; 4]> = PerCpuArray::with_max_entries(1, 0); +#[map] +static INT_ADDR_V6: PerCpuArray<[u8; 16]> = PerCpuArray::with_max_entries(1, 0); +#[map] +static PUBLIC_ADDR_V4: PerCpuArray<[u8; 4]> = PerCpuArray::with_max_entries(1, 0); +#[map] +static PUBLIC_ADDR_V6: PerCpuArray<[u8; 16]> = PerCpuArray::with_max_entries(1, 0); + +#[inline(always)] +pub fn interface_ipv4_address() -> Result { + get_ip(&INT_ADDR_V4) +} + +#[inline(always)] +pub fn interface_ipv6_address() -> Result { + get_ip(&INT_ADDR_V6) +} + +#[inline(always)] +pub fn public_ipv4_address() -> Result { + get_ip(&PUBLIC_ADDR_V4) +} + +#[inline(always)] +pub fn public_ipv6_address() -> Result { + get_ip(&PUBLIC_ADDR_V6) +} + +fn get_ip(array: &PerCpuArray<[u8; N]>) -> Result +where + T: From<[u8; N]>, +{ + let addr = *array.get(0).ok_or(Error::ArrayIndexOutOfBounds)?; + + if addr == [0u8; N] { + return Err(Error::IpAddrUnset); + } + + Ok(T::from(addr)) +} diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn/error.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn/error.rs index ac2c95db1..00ee9efa0 100644 --- a/rust/relay/ebpf-turn-router/src/try_handle_turn/error.rs +++ b/rust/relay/ebpf-turn-router/src/try_handle_turn/error.rs @@ -2,10 +2,8 @@ use core::num::NonZeroUsize; #[derive(Debug, Clone, Copy)] pub enum Error { - InterfaceIpv4AddressAccessFailed, - InterfaceIpv4AddressNotConfigured, - InterfaceIpv6AddressAccessFailed, - InterfaceIpv6AddressNotConfigured, + ArrayIndexOutOfBounds, + IpAddrUnset, UdpChecksumMissing, PacketTooShort, NotUdp, @@ -16,7 +14,6 @@ pub enum Error { BadChannelDataLength, NoEntry(SupportedChannel), XdpAdjustHeadFailed(i64), - PacketLoop, } #[derive(Debug, Clone, Copy)] @@ -32,14 +29,8 @@ impl aya_log_ebpf::WriteToBuf for Error { fn write(self, buf: &mut [u8]) -> Option { // Use a simpler match structure to help the verifier let msg = match self { - Error::InterfaceIpv4AddressAccessFailed => { - "Failed to get pointer to interface IPv4 address map" - } - Error::InterfaceIpv4AddressNotConfigured => "Interface IPv4 address not configured", - Error::InterfaceIpv6AddressAccessFailed => { - "Failed to get pointer to interface IPv6 address map" - } - Error::InterfaceIpv6AddressNotConfigured => "Interface IPv6 address not configured", + Error::ArrayIndexOutOfBounds => "Array index is out of bounds", + Error::IpAddrUnset => "IP address has not been configured", Error::UdpChecksumMissing => "UDP checksum is missing", Error::PacketTooShort => "Packet is too short", Error::NotUdp => "Not a UDP packet", @@ -48,7 +39,6 @@ impl aya_log_ebpf::WriteToBuf for Error { Error::Ipv4PacketWithOptions => "IPv4 packet has options", Error::NotAChannelDataMessage => "Not a channel data message", Error::BadChannelDataLength => "Channel data length does not match packet length", - Error::PacketLoop => "Packet loop detected", Error::NoEntry(ch) => match ch { SupportedChannel::Udp4ToChan => "No entry in UDPv4 to channel IPv4 or IPv6 map", SupportedChannel::Chan4ToUdp => "No entry in channel IPv4 to UDPv4 or UDPv6 map", diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv4_channel.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv4_channel.rs new file mode 100644 index 000000000..cc8e51b2d --- /dev/null +++ b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv4_channel.rs @@ -0,0 +1,9 @@ +mod to_ipv4_channel; +mod to_ipv4_udp; +mod to_ipv6_channel; +mod to_ipv6_udp; + +pub use to_ipv4_channel::*; +pub use to_ipv4_udp::*; +pub use to_ipv6_channel::*; +pub use to_ipv6_udp::*; diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv4_channel/to_ipv4_channel.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv4_channel/to_ipv4_channel.rs new file mode 100644 index 000000000..f0f1c7772 --- /dev/null +++ b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv4_channel/to_ipv4_channel.rs @@ -0,0 +1,121 @@ +use aya_ebpf::programs::XdpContext; +use ebpf_shared::ClientAndChannelV4; +use network_types::{eth::EthHdr, ip::Ipv4Hdr, udp::UdpHdr}; + +use crate::try_handle_turn::{Error, channel_data::CdHdr, checksum::ChecksumUpdate, ref_mut_at}; + +#[inline(always)] +pub fn to_ipv4_channel( + ctx: &XdpContext, + client_and_channel: &ClientAndChannelV4, +) -> Result<(), Error> { + let (old_eth_src, old_eth_dst, old_eth_type) = { + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let old_eth = unsafe { ref_mut_at::(ctx, 0)? }; + (old_eth.src_addr, old_eth.dst_addr, old_eth.ether_type) + }; + + let (old_ipv4_src, old_ipv4_dst, _, old_ipv4_check, ..) = { + // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. + let old_ipv4 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + ( + old_ipv4.src_addr(), + old_ipv4.dst_addr(), + old_ipv4.total_len(), + old_ipv4.checksum(), + old_ipv4.tos, + old_ipv4.id(), + old_ipv4.frag_off, + old_ipv4.ttl, + old_ipv4.proto, + ) + }; + + let (_, old_udp_src, old_udp_dst, old_udp_check) = { + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let old_udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN)? }; + ( + old_udp.len(), + old_udp.source(), + old_udp.dest(), + old_udp.check(), + ) + }; + + let (old_channel_number, _) = { + let old_cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN + UdpHdr::LEN)? }; + + (old_cd.number(), old_cd.length()) + }; + + // + // 1. Ethernet header + // + + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let eth = unsafe { ref_mut_at::(ctx, 0)? }; + eth.src_addr = old_eth_dst; + eth.dst_addr = old_eth_src; + eth.ether_type = old_eth_type; + + // + // 2. IPv4 header + // + + let new_ipv4_src = old_ipv4_dst; + let new_ipv4_dst = client_and_channel.client_ip(); + + // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. + let ipv4 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + ipv4.set_src_addr(new_ipv4_src); // Swap source and destination + ipv4.set_dst_addr(new_ipv4_dst); // Destination is the client IP + ipv4.set_checksum( + ChecksumUpdate::new(old_ipv4_check) + .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) + .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) + .into_ip_checksum(), + ); + + // + // 3. UDP header + // + + let new_udp_src = 3478_u16; + let new_udp_dst = client_and_channel.client_port(); + let channel_number = client_and_channel.channel(); + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN)? }; + udp.set_source(new_udp_src); + udp.set_dest(new_udp_dst); + + // Incrementally update UDP checksum + + if old_udp_check == 0 { + // No checksum is valid for UDP IPv4 - we didn't write it, but maybe a middlebox did + udp.set_check(0); + } else { + udp.set_check( + ChecksumUpdate::new(old_udp_check) + .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) + .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) + .remove_u16(old_udp_src) + .add_u16(new_udp_src) + .remove_u16(old_udp_dst) + .add_u16(new_udp_dst) + .remove_u16(old_channel_number) + .add_u16(channel_number) + .into_udp_checksum(), + ); + } + + // + // 4. Channel data header + // + + // SAFETY: The offset must point to the start of a valid `CdHdr`. + let cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN + UdpHdr::LEN)? }; + cd.number = channel_number.to_be_bytes(); + + Ok(()) +} diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv4_channel/to_ipv4_udp.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv4_channel/to_ipv4_udp.rs new file mode 100644 index 000000000..b609bd078 --- /dev/null +++ b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv4_channel/to_ipv4_udp.rs @@ -0,0 +1,143 @@ +use aya_ebpf::programs::XdpContext; +use ebpf_shared::PortAndPeerV4; +use network_types::{eth::EthHdr, ip::Ipv4Hdr, udp::UdpHdr}; + +use crate::try_handle_turn::{ + Error, adjust_head, channel_data::CdHdr, checksum::ChecksumUpdate, ref_mut_at, +}; + +#[inline(always)] +pub fn to_ipv4_udp(ctx: &XdpContext, port_and_peer: &PortAndPeerV4) -> Result<(), Error> { + const NET_SHRINK: i32 = CdHdr::LEN as i32; // Shrink by 4 bytes for channel data header + + let (old_eth_src, old_eth_dst, old_eth_type) = { + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let old_eth = unsafe { ref_mut_at::(ctx, 0)? }; + (old_eth.src_addr, old_eth.dst_addr, old_eth.ether_type) + }; + + let ( + old_ipv4_src, + old_ipv4_dst, + old_ipv4_len, + old_ipv4_check, + old_ipv4_tos, + old_ipv4_id, + old_ipv4_frag_off, + old_ipv4_ttl, + old_ipv4_proto, + ) = { + // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. + let old_ipv4 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + ( + old_ipv4.src_addr(), + old_ipv4.dst_addr(), + old_ipv4.total_len(), + old_ipv4.checksum(), + old_ipv4.tos, + old_ipv4.id(), + old_ipv4.frag_off, + old_ipv4.ttl, + old_ipv4.proto, + ) + }; + + let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let old_udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN)? }; + ( + old_udp.len(), + old_udp.source(), + old_udp.dest(), + old_udp.check(), + ) + }; + + let (channel_number, channel_data_length) = { + // SAFETY: The offset must point to the start of a valid `CdHdr`. + let old_cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN + UdpHdr::LEN)? }; + (old_cd.number(), old_cd.length()) + }; + + // + // 1. Ethernet header + // + + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let eth = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize)? }; + eth.dst_addr = old_eth_src; // Swap source and destination + eth.src_addr = old_eth_dst; + eth.ether_type = old_eth_type; + + // + // 2. IPv4 header + // + + let new_ipv4_src = old_ipv4_dst; // Swap source and destination + let new_ipv4_dst = port_and_peer.peer_ip(); + let new_ipv4_len = old_ipv4_len - CdHdr::LEN as u16; + + // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. + let ipv4 = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN)? }; + ipv4.set_version(4); // IPv4 + ipv4.set_ihl(5); // No options, 5 * 4 = 20 bytes + ipv4.tos = old_ipv4_tos; // Preserve TOS/DSCP + ipv4.set_total_len(new_ipv4_len); + ipv4.set_id(old_ipv4_id); // Preserve ID + ipv4.frag_off = old_ipv4_frag_off; // Preserve fragment flags + ipv4.ttl = old_ipv4_ttl; // Preserve TTL exactly + ipv4.proto = old_ipv4_proto; // Protocol is UDP + ipv4.set_src_addr(new_ipv4_src); + ipv4.set_dst_addr(new_ipv4_dst); + ipv4.set_checksum( + ChecksumUpdate::new(old_ipv4_check) + .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) + .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) + .remove_u16(old_ipv4_len) + .add_u16(new_ipv4_len) + .into_ip_checksum(), + ); + + // + // 3. UDP header + // + + let new_udp_src = port_and_peer.allocation_port(); + let new_udp_dst = port_and_peer.peer_port(); + let new_udp_len = old_udp_len - CdHdr::LEN as u16; + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = + unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN + Ipv4Hdr::LEN)? }; + udp.set_source(new_udp_src); + udp.set_dest(new_udp_dst); + udp.set_len(new_udp_len); + + // Incrementally update UDP checksum + + if old_udp_check == 0 { + // No checksum is valid for UDP IPv4 - we didn't write it, but maybe a middlebox did + udp.set_check(0); + } else { + udp.set_check( + ChecksumUpdate::new(old_udp_check) + .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) + .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) + .remove_u16(old_udp_src) + .add_u16(new_udp_src) + .remove_u16(old_udp_dst) + .add_u16(new_udp_dst) + .remove_u16(old_udp_len) + .add_u16(new_udp_len) + .remove_u16(old_udp_len) + .add_u16(new_udp_len) + .remove_u16(channel_number) + .remove_u16(channel_data_length) + .into_udp_checksum(), + ); + } + + adjust_head(ctx, NET_SHRINK)?; + + Ok(()) +} diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv4_channel/to_ipv6_channel.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv4_channel/to_ipv6_channel.rs new file mode 100644 index 000000000..96e31cf62 --- /dev/null +++ b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv4_channel/to_ipv6_channel.rs @@ -0,0 +1,140 @@ +use aya_ebpf::programs::XdpContext; +use ebpf_shared::ClientAndChannelV6; +use network_types::{ + eth::{EthHdr, EtherType}, + ip::{Ipv4Hdr, Ipv6Hdr}, + udp::UdpHdr, +}; + +use crate::try_handle_turn::{ + Error, adjust_head, channel_data::CdHdr, checksum::ChecksumUpdate, config, ref_mut_at, +}; + +#[inline(always)] +pub fn to_ipv6_channel( + ctx: &XdpContext, + client_and_channel: &ClientAndChannelV6, +) -> Result<(), Error> { + // Expand the packet by 20 bytes for IPv6 header + const NET_EXPANSION: i32 = -(Ipv6Hdr::LEN as i32 - Ipv4Hdr::LEN as i32); + + adjust_head(ctx, NET_EXPANSION)?; + + // Now read the old packet data from its NEW location (shifted by 24 bytes) + let old_data_offset = -NET_EXPANSION as usize; + + let (old_eth_src, old_eth_dst) = { + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let old_eth = unsafe { ref_mut_at::(ctx, old_data_offset)? }; + (old_eth.src_addr, old_eth.dst_addr) + }; + + let (old_ipv4_src, old_ipv4_dst, old_ipv4_len, old_ipv4_tos, old_ipv4_ttl, old_ipv4_proto) = { + // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. + let old_ipv4 = unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN)? }; + ( + old_ipv4.src_addr(), + old_ipv4.dst_addr(), + old_ipv4.total_len(), + old_ipv4.tos, + old_ipv4.ttl, + old_ipv4.proto, + ) + }; + + let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let old_udp = + unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN + Ipv4Hdr::LEN)? }; + ( + old_udp.len(), + old_udp.source(), + old_udp.dest(), + old_udp.check(), + ) + }; + + let (old_channel_number, old_channel_data_length) = { + let old_cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN + UdpHdr::LEN)? }; + + (old_cd.number(), old_cd.length()) + }; + + // Refuse to compute full UDP checksum. + // We forged these packets, so something's wrong if this is zero. + if old_udp_check == 0 { + return Err(Error::UdpChecksumMissing); + } + + // + // 1. Ethernet header + // + + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let eth = unsafe { ref_mut_at::(ctx, 0)? }; + eth.dst_addr = old_eth_src; // Swap source and destination + eth.src_addr = old_eth_dst; + eth.ether_type = EtherType::Ipv6; // Change to IPv6 + + // + // 2. IPv4 -> IPv6 header + // + + let new_ipv6_src = config::interface_ipv6_address()?; + let new_ipv6_dst = client_and_channel.client_ip(); + let new_ipv6_len = old_ipv4_len - Ipv4Hdr::LEN as u16; + + // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. + let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + ipv6.set_version(6); + ipv6.set_priority(old_ipv4_tos); + ipv6.flow_label = [0, 0, 0]; // Default flow label + ipv6.set_payload_len(new_ipv6_len); + ipv6.next_hdr = old_ipv4_proto; + ipv6.hop_limit = old_ipv4_ttl; + ipv6.set_src_addr(new_ipv6_src); + ipv6.set_dst_addr(new_ipv6_dst); + + // + // 3. UDP header + // + + let new_udp_src = 3478_u16; + let new_udp_dst = client_and_channel.client_port(); + + let channel_number = client_and_channel.channel(); + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; + udp.set_source(new_udp_src); + udp.set_dest(new_udp_dst); + udp.set_len(old_udp_len); + + // Incrementally update UDP checksum + + udp.set_check( + ChecksumUpdate::new(old_udp_check) + .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) + .add_u128(u128::from_be_bytes(new_ipv6_src.octets())) + .remove_u32(u32::from_be_bytes(old_ipv4_dst.octets())) + .add_u128(u128::from_be_bytes(new_ipv6_dst.octets())) + .remove_u16(old_udp_src) + .add_u16(new_udp_src) + .remove_u16(old_udp_dst) + .add_u16(new_udp_dst) + .remove_u16(old_channel_number) + .add_u16(channel_number) + .into_udp_checksum(), + ); + + // + // 4. Channel data header + // + + // SAFETY: The offset must point to the start of a valid `CdHdr`. + let cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN + UdpHdr::LEN)? }; + cd.number = channel_number.to_be_bytes(); + cd.length = old_channel_data_length.to_be_bytes(); + + Ok(()) +} diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv4_channel/to_ipv6_udp.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv4_channel/to_ipv6_udp.rs new file mode 100644 index 000000000..5b4c643ed --- /dev/null +++ b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv4_channel/to_ipv6_udp.rs @@ -0,0 +1,133 @@ +use aya_ebpf::programs::XdpContext; +use ebpf_shared::PortAndPeerV6; +use network_types::{ + eth::{EthHdr, EtherType}, + ip::{Ipv4Hdr, Ipv6Hdr}, + udp::UdpHdr, +}; + +use crate::try_handle_turn::{ + Error, adjust_head, channel_data::CdHdr, checksum::ChecksumUpdate, config, ref_mut_at, +}; + +#[inline(always)] +pub fn to_ipv6_udp(ctx: &XdpContext, port_and_peer: &PortAndPeerV6) -> Result<(), Error> { + const NET_EXPANSION: i32 = Ipv4Hdr::LEN as i32 - Ipv6Hdr::LEN as i32 + CdHdr::LEN as i32; + + adjust_head(ctx, NET_EXPANSION)?; + + // Now read the old packet data from its NEW location + let old_data_offset = (-NET_EXPANSION) as usize; + + let (old_src_mac, old_dst_mac) = { + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let old_eth = unsafe { ref_mut_at::(ctx, old_data_offset)? }; + (old_eth.src_addr, old_eth.dst_addr) + }; + + let (old_ipv4_src, old_ipv4_dst, old_ipv4_tos, old_ipv4_ttl, old_ipv4_proto) = { + // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. + let old_ipv4 = unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN)? }; + ( + old_ipv4.src_addr(), + old_ipv4.dst_addr(), + old_ipv4.tos, + old_ipv4.ttl, + old_ipv4.proto, + ) + }; + + let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let old_udp = + unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN + Ipv4Hdr::LEN)? }; + ( + old_udp.len(), + old_udp.source(), + old_udp.dest(), + old_udp.check(), + ) + }; + + // Refuse to compute full UDP checksum. + // We forged these packets, so something's wrong if this is zero. + if old_udp_check == 0 { + return Err(Error::UdpChecksumMissing); + } + + let (channel_number, channel_data_length) = { + // SAFETY: The offset must point to the start of a valid `CdHdr`. + let old_cd = unsafe { + ref_mut_at::( + ctx, + old_data_offset + EthHdr::LEN + Ipv4Hdr::LEN + UdpHdr::LEN, + )? + }; + (old_cd.number(), old_cd.length()) + }; + + // + // 1. Ethernet header + // + + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let eth = unsafe { ref_mut_at::(ctx, 0)? }; + eth.dst_addr = old_src_mac; // Swap MACs + eth.src_addr = old_dst_mac; + eth.ether_type = EtherType::Ipv6; // Change to IPv6 + + // + // 2. IPv6 header + // + + let new_ipv6_src = config::interface_ipv6_address()?; + let new_ipv6_dst = port_and_peer.peer_ip(); + let new_udp_len = old_udp_len - CdHdr::LEN as u16; + + // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. + let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + ipv6.set_version(6); // IPv6 + ipv6.set_priority(old_ipv4_tos); + ipv6.flow_label = [0, 0, 0]; + ipv6.set_payload_len(new_udp_len); + ipv6.next_hdr = old_ipv4_proto; + ipv6.hop_limit = old_ipv4_ttl; + ipv6.set_src_addr(new_ipv6_src); + ipv6.set_dst_addr(new_ipv6_dst); + + // + // 3. UDP header + // + + let new_udp_src = port_and_peer.allocation_port(); + let new_udp_dst = port_and_peer.peer_port(); + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; + udp.set_source(new_udp_src); + udp.set_dest(new_udp_dst); + udp.set_len(new_udp_len); + + // Incrementally update UDP checksum + + udp.set_check( + ChecksumUpdate::new(old_udp_check) + .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) + .add_u128(u128::from_be_bytes(new_ipv6_src.octets())) + .remove_u32(u32::from_be_bytes(old_ipv4_dst.octets())) + .add_u128(u128::from_be_bytes(new_ipv6_dst.octets())) + .remove_u16(old_udp_src) + .add_u16(new_udp_src) + .remove_u16(old_udp_dst) + .add_u16(new_udp_dst) + .remove_u16(old_udp_len) + .add_u16(new_udp_len) + .remove_u16(old_udp_len) + .add_u16(new_udp_len) + .remove_u16(channel_number) + .remove_u16(channel_data_length) + .into_udp_checksum(), + ); + + Ok(()) +} diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv4_udp.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv4_udp.rs new file mode 100644 index 000000000..db8e90e5d --- /dev/null +++ b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv4_udp.rs @@ -0,0 +1,5 @@ +mod to_ipv4_channel; +mod to_ipv6_channel; + +pub use to_ipv4_channel::*; +pub use to_ipv6_channel::*; diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv4_udp/to_ipv4_channel.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv4_udp/to_ipv4_channel.rs new file mode 100644 index 000000000..cb9836c98 --- /dev/null +++ b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv4_udp/to_ipv4_channel.rs @@ -0,0 +1,154 @@ +use aya_ebpf::programs::XdpContext; +use ebpf_shared::ClientAndChannelV4; +use network_types::{eth::EthHdr, ip::Ipv4Hdr, udp::UdpHdr}; + +use crate::try_handle_turn::{ + Error, adjust_head, channel_data::CdHdr, checksum::ChecksumUpdate, ref_mut_at, +}; + +#[inline(always)] +pub fn to_ipv4_channel( + ctx: &XdpContext, + client_and_channel: &ClientAndChannelV4, +) -> Result<(), Error> { + const NET_EXPANSION: i32 = -(CdHdr::LEN as i32); + + adjust_head(ctx, NET_EXPANSION)?; + + // Now read the old packet data from its NEW location (shifted by 4 bytes) + let old_data_offset = -NET_EXPANSION as usize; + + let (old_eth_src, old_eth_dst, old_eth_type) = { + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let old_eth = unsafe { ref_mut_at::(ctx, old_data_offset)? }; + (old_eth.src_addr, old_eth.dst_addr, old_eth.ether_type) + }; + + let ( + old_ipv4_src, + old_ipv4_dst, + old_ipv4_len, + old_ipv4_check, + old_ipv4_tos, + old_ipv4_id, + old_ipv4_frag_off, + old_ipv4_ttl, + old_ipv4_proto, + ) = { + // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. + let old_ipv4 = unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN)? }; + ( + old_ipv4.src_addr(), + old_ipv4.dst_addr(), + old_ipv4.total_len(), + old_ipv4.checksum(), + old_ipv4.tos, + old_ipv4.id(), + old_ipv4.frag_off, + old_ipv4.ttl, + old_ipv4.proto, + ) + }; + + let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let old_udp = + unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN + Ipv4Hdr::LEN)? }; + ( + old_udp.len(), + old_udp.source(), + old_udp.dest(), + old_udp.check(), + ) + }; + + // + // 1. Ethernet header + // + + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let eth = unsafe { ref_mut_at::(ctx, 0)? }; + eth.src_addr = old_eth_dst; + eth.dst_addr = old_eth_src; + eth.ether_type = old_eth_type; + + // + // 2. IPv4 header + // + + let new_ipv4_src = old_ipv4_dst; + let new_ipv4_dst = client_and_channel.client_ip(); + let new_ipv4_len = old_ipv4_len + CdHdr::LEN as u16; + + // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. + let ipv4 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + ipv4.set_version(4); // IPv4 + ipv4.set_ihl(5); // No options, 5 * 4 = 20 bytes + ipv4.tos = old_ipv4_tos; // Preserve TOS/DSCP + ipv4.set_total_len(new_ipv4_len); + ipv4.set_id(old_ipv4_id); // Preserve fragment ID + ipv4.frag_off = old_ipv4_frag_off; // Preserve fragment flags + ipv4.ttl = old_ipv4_ttl; // Preserve TTL exactly + ipv4.proto = old_ipv4_proto; // Protocol is UDP + ipv4.set_src_addr(new_ipv4_src); // Swap source and destination + ipv4.set_dst_addr(new_ipv4_dst); // Destination is the client IP + ipv4.set_checksum( + ChecksumUpdate::new(old_ipv4_check) + .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) + .remove_u16(old_ipv4_len) + .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) + .add_u16(new_ipv4_len) + .into_ip_checksum(), + ); + + // + // 3. UDP header + // + + let new_udp_src = 3478_u16; + let new_udp_dst = client_and_channel.client_port(); + let new_udp_len = old_udp_len + CdHdr::LEN as u16; + let channel_number = client_and_channel.channel(); + let channel_data_length = old_udp_len - UdpHdr::LEN as u16; + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN)? }; + udp.set_source(new_udp_src); + udp.set_dest(new_udp_dst); + udp.set_len(new_udp_len); + + // Incrementally update UDP checksum + + if old_udp_check == 0 { + // No checksum is valid for UDP IPv4 - we didn't write it, but maybe a middlebox did + udp.set_check(0); + } else { + udp.set_check( + ChecksumUpdate::new(old_udp_check) + .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) + .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) + .remove_u16(old_udp_src) + .add_u16(new_udp_src) + .remove_u16(old_udp_dst) + .add_u16(new_udp_dst) + .remove_u16(old_udp_len) + .add_u16(new_udp_len) + .remove_u16(old_udp_len) + .add_u16(new_udp_len) + .add_u16(channel_number) + .add_u16(channel_data_length) + .into_udp_checksum(), + ); + } + + // + // 4. Channel data header + // + + // SAFETY: The offset must point to the start of a valid `CdHdr`. + let cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv4Hdr::LEN + UdpHdr::LEN)? }; + cd.number = channel_number.to_be_bytes(); + cd.length = channel_data_length.to_be_bytes(); + + Ok(()) +} diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv4_udp/to_ipv6_channel.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv4_udp/to_ipv6_channel.rs new file mode 100644 index 000000000..c23ee1372 --- /dev/null +++ b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv4_udp/to_ipv6_channel.rs @@ -0,0 +1,140 @@ +use aya_ebpf::programs::XdpContext; +use ebpf_shared::ClientAndChannelV6; +use network_types::{ + eth::{EthHdr, EtherType}, + ip::{Ipv4Hdr, Ipv6Hdr}, + udp::UdpHdr, +}; + +use crate::try_handle_turn::{ + Error, adjust_head, channel_data::CdHdr, checksum::ChecksumUpdate, config, ref_mut_at, +}; + +#[inline(always)] +pub fn to_ipv6_channel( + ctx: &XdpContext, + client_and_channel: &ClientAndChannelV6, +) -> Result<(), Error> { + // Expand the packet by 24 bytes for IPv6 header and channel data header + const NET_EXPANSION: i32 = -(Ipv6Hdr::LEN as i32 - Ipv4Hdr::LEN as i32 + CdHdr::LEN as i32); + + adjust_head(ctx, NET_EXPANSION)?; + + // Now read the old packet data from its NEW location (shifted by 24 bytes) + let old_data_offset = -NET_EXPANSION as usize; + + let (old_eth_src, old_eth_dst) = { + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let old_eth = unsafe { ref_mut_at::(ctx, old_data_offset)? }; + (old_eth.src_addr, old_eth.dst_addr) + }; + + let (old_ipv4_src, old_ipv4_dst, old_ipv4_len, old_ipv4_tos, old_ipv4_ttl, old_ipv4_proto) = { + // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. + let old_ipv4 = unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN)? }; + ( + old_ipv4.src_addr(), + old_ipv4.dst_addr(), + old_ipv4.total_len(), + old_ipv4.tos, + old_ipv4.ttl, + old_ipv4.proto, + ) + }; + + let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let old_udp = + unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN + Ipv4Hdr::LEN)? }; + ( + old_udp.len(), + old_udp.source(), + old_udp.dest(), + old_udp.check(), + ) + }; + + // Refuse to compute full UDP checksum. + // We forged these packets, so something's wrong if this is zero. + if old_udp_check == 0 { + return Err(Error::UdpChecksumMissing); + } + + // + // 1. Ethernet header + // + + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let eth = unsafe { ref_mut_at::(ctx, 0)? }; + eth.dst_addr = old_eth_src; // Swap source and destination + eth.src_addr = old_eth_dst; + eth.ether_type = EtherType::Ipv6; // Change to IPv6 + + // + // 2. IPv4 -> IPv6 header + // + + let new_ipv6_src = config::interface_ipv6_address()?; + let new_ipv6_dst = client_and_channel.client_ip(); + let new_ipv6_len = old_ipv4_len - Ipv4Hdr::LEN as u16 + CdHdr::LEN as u16; + + // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. + let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + ipv6.set_version(6); + ipv6.set_priority(old_ipv4_tos); + ipv6.flow_label = [0, 0, 0]; // Default flow label + ipv6.set_payload_len(new_ipv6_len); + ipv6.next_hdr = old_ipv4_proto; + ipv6.hop_limit = old_ipv4_ttl; + ipv6.set_src_addr(new_ipv6_src); + ipv6.set_dst_addr(new_ipv6_dst); + + // + // 3. UDP header + // + + let new_udp_src = 3478_u16; + let new_udp_dst = client_and_channel.client_port(); + let new_udp_len = old_udp_len + CdHdr::LEN as u16; + + let channel_number = client_and_channel.channel(); + let channel_data_length = old_udp_len - UdpHdr::LEN as u16; + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; + udp.set_source(new_udp_src); + udp.set_dest(new_udp_dst); + udp.set_len(new_udp_len); + + // Incrementally update UDP checksum + + udp.set_check( + ChecksumUpdate::new(old_udp_check) + .remove_u32(u32::from_be_bytes(old_ipv4_src.octets())) + .add_u128(u128::from_be_bytes(new_ipv6_src.octets())) + .remove_u32(u32::from_be_bytes(old_ipv4_dst.octets())) + .add_u128(u128::from_be_bytes(new_ipv6_dst.octets())) + .remove_u16(old_udp_src) + .add_u16(new_udp_src) + .remove_u16(old_udp_dst) + .add_u16(new_udp_dst) + .remove_u16(old_udp_len) + .add_u16(new_udp_len) + .remove_u16(old_udp_len) + .add_u16(new_udp_len) + .add_u16(channel_number) + .add_u16(channel_data_length) + .into_udp_checksum(), + ); + + // + // 4. Channel data header + // + + // SAFETY: The offset must point to the start of a valid `CdHdr`. + let cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN + UdpHdr::LEN)? }; + cd.number = channel_number.to_be_bytes(); + cd.length = channel_data_length.to_be_bytes(); + + Ok(()) +} diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv6_channel.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv6_channel.rs new file mode 100644 index 000000000..cc8e51b2d --- /dev/null +++ b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv6_channel.rs @@ -0,0 +1,9 @@ +mod to_ipv4_channel; +mod to_ipv4_udp; +mod to_ipv6_channel; +mod to_ipv6_udp; + +pub use to_ipv4_channel::*; +pub use to_ipv4_udp::*; +pub use to_ipv6_channel::*; +pub use to_ipv6_udp::*; diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv6_channel/to_ipv4_channel.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv6_channel/to_ipv4_channel.rs new file mode 100644 index 000000000..9b0ef414a --- /dev/null +++ b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv6_channel/to_ipv4_channel.rs @@ -0,0 +1,143 @@ +use aya_ebpf::programs::XdpContext; + +use ebpf_shared::ClientAndChannelV4; +use network_types::{ + eth::{EthHdr, EtherType}, + ip::{Ipv4Hdr, Ipv6Hdr}, + udp::UdpHdr, +}; + +use crate::try_handle_turn::{ + Error, adjust_head, + channel_data::CdHdr, + checksum::{self, ChecksumUpdate}, + config, ref_mut_at, +}; + +#[inline(always)] +pub fn to_ipv4_channel( + ctx: &XdpContext, + client_and_channel: &ClientAndChannelV4, +) -> Result<(), Error> { + const NET_SHRINK: i32 = Ipv6Hdr::LEN as i32 - Ipv4Hdr::LEN as i32; + + let (old_eth_src, old_eth_dst) = { + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let old_eth = unsafe { ref_mut_at::(ctx, 0)? }; + (old_eth.src_addr, old_eth.dst_addr) + }; + + let (old_ipv6_src, old_ipv6_dst, old_ipv6_priority, old_ipv6_hop_limit, old_ipv6_next_hdr) = { + // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. + let old_ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + ( + old_ipv6.src_addr(), + old_ipv6.dst_addr(), + old_ipv6.priority(), + old_ipv6.hop_limit, + old_ipv6.next_hdr, + ) + }; + + let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let old_udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; + ( + old_udp.len(), + old_udp.source(), + old_udp.dest(), + old_udp.check(), + ) + }; + + let (old_channel_number, old_channel_data_length) = { + let old_cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN + UdpHdr::LEN)? }; + + (old_cd.number(), old_cd.length()) + }; + + // + // 1. Ethernet header + // + + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let eth = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize)? }; + eth.src_addr = old_eth_dst; // Swap source and destination + eth.dst_addr = old_eth_src; + eth.ether_type = EtherType::Ipv4; // Change to IPv4 + + // + // 2. IPv6 -> IPv4 header + // + + let new_ipv4_src = config::interface_ipv4_address()?; + let new_ipv4_dst = client_and_channel.client_ip(); + let new_ipv4_len = Ipv4Hdr::LEN as u16 + old_udp_len; + + // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. + let ipv4 = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN)? }; + ipv4.set_version(4); + ipv4.set_ihl(5); // No options + ipv4.tos = old_ipv6_priority; + ipv4.set_total_len(new_ipv4_len); + ipv4.set_id(0); // Default ID + ipv4.frag_off = 0x4000_u16.to_be_bytes(); // Don't fragment + ipv4.ttl = old_ipv6_hop_limit; // Preserve hop limit + ipv4.proto = old_ipv6_next_hdr; // Preserve protocol + ipv4.set_src_addr(new_ipv4_src); + ipv4.set_dst_addr(new_ipv4_dst); + + // Calculate fresh checksum + let check = checksum::new_ipv4(ipv4); + ipv4.set_checksum(check); + + // + // 3. UDP header + // + + let new_udp_src = 3478_u16; // Fixed source port for TURN + let new_udp_dst = client_and_channel.client_port(); + let channel_number = client_and_channel.channel(); + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = + unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN + Ipv4Hdr::LEN)? }; + udp.set_source(new_udp_src); + udp.set_dest(new_udp_dst); + udp.set_len(old_udp_len); + + // Incrementally update UDP checksum + + udp.set_check( + ChecksumUpdate::new(old_udp_check) + .remove_u128(u128::from_be_bytes(old_ipv6_src.octets())) + .add_u32(u32::from_be_bytes(new_ipv4_src.octets())) + .remove_u128(u128::from_be_bytes(old_ipv6_dst.octets())) + .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) + .remove_u16(old_udp_src) + .add_u16(new_udp_src) + .remove_u16(old_udp_dst) + .add_u16(new_udp_dst) + .remove_u16(old_channel_number) + .add_u16(channel_number) + .into_udp_checksum(), + ); + + // + // 4. Channel data header + // + + // SAFETY: The offset must point to the start of a valid `CdHdr`. + let cd = unsafe { + ref_mut_at::( + ctx, + NET_SHRINK as usize + EthHdr::LEN + Ipv4Hdr::LEN + UdpHdr::LEN, + )? + }; + cd.number = channel_number.to_be_bytes(); + cd.length = old_channel_data_length.to_be_bytes(); + + adjust_head(ctx, NET_SHRINK)?; + + Ok(()) +} diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv6_channel/to_ipv4_udp.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv6_channel/to_ipv4_udp.rs new file mode 100644 index 000000000..8d877d220 --- /dev/null +++ b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv6_channel/to_ipv4_udp.rs @@ -0,0 +1,130 @@ +use aya_ebpf::programs::XdpContext; +use ebpf_shared::PortAndPeerV4; +use network_types::{ + eth::{EthHdr, EtherType}, + ip::{Ipv4Hdr, Ipv6Hdr}, + udp::UdpHdr, +}; + +use crate::try_handle_turn::{ + Error, adjust_head, + channel_data::CdHdr, + checksum::{self, ChecksumUpdate}, + config, ref_mut_at, +}; + +#[inline(always)] +pub fn to_ipv4_udp(ctx: &XdpContext, port_and_peer: &PortAndPeerV4) -> Result<(), Error> { + // Shrink by 24 bytes: 20 for the IP header diff and 4 for the removed channel data header + const NET_SHRINK: i32 = Ipv6Hdr::LEN as i32 - Ipv4Hdr::LEN as i32 + CdHdr::LEN as i32; + + let (old_eth_src, old_eth_dst) = { + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let old_eth = unsafe { ref_mut_at::(ctx, 0)? }; + (old_eth.src_addr, old_eth.dst_addr) + }; + + let (old_ipv6_src, old_ipv6_dst, old_ipv6_priority, old_ipv6_hop_limit, old_ipv6_next_hdr) = { + // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. + let old_ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + ( + old_ipv6.src_addr(), + old_ipv6.dst_addr(), + old_ipv6.priority(), + old_ipv6.hop_limit, + old_ipv6.next_hdr, + ) + }; + + let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let old_udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; + ( + old_udp.len(), + old_udp.source(), + old_udp.dest(), + old_udp.check(), + ) + }; + + let (channel_number, channel_data_length) = { + // SAFETY: The offset must point to the start of a valid `CdHdr`. + let old_cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN + UdpHdr::LEN)? }; + (old_cd.number(), old_cd.length()) + }; + + // + // 1. Ethernet header + // + + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let eth = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize)? }; + eth.src_addr = old_eth_dst; // Swap source and destination + eth.dst_addr = old_eth_src; + eth.ether_type = EtherType::Ipv4; // Change to IPv4 + + // + // 2. IPv6 -> IPv4 header + // + + let new_ipv4_src = config::interface_ipv4_address()?; + let new_ipv4_dst = port_and_peer.peer_ip(); + let new_ipv4_len = old_udp_len - CdHdr::LEN as u16 + Ipv4Hdr::LEN as u16; + + // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. + let ipv4 = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN)? }; + ipv4.set_version(4); + ipv4.set_ihl(5); // No options + ipv4.tos = old_ipv6_priority; // Copy TOS from IPv6 + ipv4.set_total_len(new_ipv4_len); + ipv4.set_id(0); // Default ID + ipv4.frag_off = 0x4000_u16.to_be_bytes(); // Don't fragment + ipv4.ttl = old_ipv6_hop_limit; // Preserve TTL + ipv4.proto = old_ipv6_next_hdr; // Copy protocol from IPv6 + ipv4.set_src_addr(new_ipv4_src); + ipv4.set_dst_addr(new_ipv4_dst); + + // Calculate fresh checksum + let check = checksum::new_ipv4(ipv4); + ipv4.set_checksum(check); + + // + // 3. UDP header + // + + let new_udp_src = port_and_peer.allocation_port(); + let new_udp_dst = port_and_peer.peer_port(); + let new_udp_len = old_udp_len - CdHdr::LEN as u16; + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = + unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN + Ipv4Hdr::LEN)? }; + udp.set_source(new_udp_src); + udp.set_dest(new_udp_dst); + udp.set_len(new_udp_len); + + // Incrementally update UDP checksum + + udp.set_check( + ChecksumUpdate::new(old_udp_check) + .remove_u128(u128::from_be_bytes(old_ipv6_src.octets())) + .add_u32(u32::from_be_bytes(new_ipv4_src.octets())) + .remove_u128(u128::from_be_bytes(old_ipv6_dst.octets())) + .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) + .remove_u16(old_udp_src) + .add_u16(new_udp_src) + .remove_u16(old_udp_dst) + .add_u16(new_udp_dst) + .remove_u16(old_udp_len) + .add_u16(new_udp_len) + .remove_u16(old_udp_len) + .add_u16(new_udp_len) + .remove_u16(channel_number) + .remove_u16(channel_data_length) + .into_udp_checksum(), + ); + + adjust_head(ctx, NET_SHRINK)?; + + Ok(()) +} diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv6_channel/to_ipv6_channel.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv6_channel/to_ipv6_channel.rs new file mode 100644 index 000000000..ce037a8f8 --- /dev/null +++ b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv6_channel/to_ipv6_channel.rs @@ -0,0 +1,108 @@ +use aya_ebpf::programs::XdpContext; +use ebpf_shared::ClientAndChannelV6; +use network_types::{eth::EthHdr, ip::Ipv6Hdr, udp::UdpHdr}; + +use crate::try_handle_turn::{Error, channel_data::CdHdr, checksum::ChecksumUpdate, ref_mut_at}; + +#[inline(always)] +pub fn to_ipv6_channel( + ctx: &XdpContext, + client_and_channel: &ClientAndChannelV6, +) -> Result<(), Error> { + let (old_eth_src, old_eth_dst, old_eth_type) = { + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let old_eth = unsafe { ref_mut_at::(ctx, 0)? }; + (old_eth.src_addr, old_eth.dst_addr, old_eth.ether_type) + }; + + let (old_ipv6_src, old_ipv6_dst, ..) = { + // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. + let old_ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + ( + old_ipv6.src_addr(), + old_ipv6.dst_addr(), + old_ipv6.payload_len(), + old_ipv6.priority(), + old_ipv6.flow_label, + old_ipv6.hop_limit, + old_ipv6.next_hdr, + ) + }; + + let (_, old_udp_src, old_udp_dst, old_udp_check) = { + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let old_udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; + ( + old_udp.len(), + old_udp.source(), + old_udp.dest(), + old_udp.check(), + ) + }; + + let (old_channel_number, _) = { + let old_cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN + UdpHdr::LEN)? }; + + (old_cd.number(), old_cd.length()) + }; + + // + // 1. Ethernet header + // + + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let eth = unsafe { ref_mut_at::(ctx, 0)? }; + eth.src_addr = old_eth_dst; // Swap source and destination + eth.dst_addr = old_eth_src; + eth.ether_type = old_eth_type; + + // + // 2. IPv6 header + // + + let new_ipv6_src = old_ipv6_dst; + let new_ipv6_dst = client_and_channel.client_ip(); + + // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. + let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + ipv6.set_src_addr(new_ipv6_src); + ipv6.set_dst_addr(new_ipv6_dst); + + // + // 3. UDP header + // + + let channel_number = client_and_channel.channel(); + let new_udp_src = 3478_u16; + let new_udp_dst = client_and_channel.client_port(); + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; + udp.set_source(new_udp_src); + udp.set_dest(new_udp_dst); + + // Incrementally update UDP checksum + + udp.set_check( + ChecksumUpdate::new(old_udp_check) + .remove_u128(u128::from_be_bytes(old_ipv6_src.octets())) + .add_u128(u128::from_be_bytes(new_ipv6_dst.octets())) + .remove_u16(old_udp_src) + .add_u16(new_udp_src) + .remove_u16(old_udp_dst) + .add_u16(new_udp_dst) + .remove_u16(old_channel_number) + .add_u16(channel_number) + .into_udp_checksum(), + ); + + // + // 4. Channel data header + // + + // SAFETY: The offset must point to the start of a valid `CdHdr`. + let cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN + UdpHdr::LEN)? }; + cd.number = channel_number.to_be_bytes(); + + Ok(()) +} diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv6_channel/to_ipv6_udp.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv6_channel/to_ipv6_udp.rs new file mode 100644 index 000000000..d487028c1 --- /dev/null +++ b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv6_channel/to_ipv6_udp.rs @@ -0,0 +1,124 @@ +use aya_ebpf::programs::XdpContext; +use ebpf_shared::PortAndPeerV6; +use network_types::{eth::EthHdr, ip::Ipv6Hdr, udp::UdpHdr}; + +use crate::try_handle_turn::{ + Error, adjust_head, channel_data::CdHdr, checksum::ChecksumUpdate, ref_mut_at, +}; + +#[inline(always)] +pub fn to_ipv6_udp(ctx: &XdpContext, port_and_peer: &PortAndPeerV6) -> Result<(), Error> { + const NET_SHRINK: i32 = CdHdr::LEN as i32; // Shrink by 4 bytes for channel data header + + let (old_eth_src, old_eth_dst, old_eth_type) = { + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let old_eth = unsafe { ref_mut_at::(ctx, 0)? }; + (old_eth.src_addr, old_eth.dst_addr, old_eth.ether_type) + }; + + let ( + old_ipv6_src, + old_ipv6_dst, + old_ipv6_len, + old_ipv6_priority, + old_ipv6_flow_label, + old_ipv6_hop_limit, + old_ipv6_next_hdr, + ) = { + // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. + let old_ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + ( + old_ipv6.src_addr(), + old_ipv6.dst_addr(), + old_ipv6.payload_len(), + old_ipv6.priority(), + old_ipv6.flow_label, + old_ipv6.hop_limit, + old_ipv6.next_hdr, + ) + }; + + let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let old_udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; + ( + old_udp.len(), + old_udp.source(), + old_udp.dest(), + old_udp.check(), + ) + }; + + let (channel_number, channel_data_length) = { + // SAFETY: The offset must point to the start of a valid `CdHdr`. + let old_cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN + UdpHdr::LEN)? }; + (old_cd.number(), old_cd.length()) + }; + + // + // 1. Ethernet header + // + + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let eth = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize)? }; + eth.src_addr = old_eth_dst; // Swap source and destination + eth.dst_addr = old_eth_src; + eth.ether_type = old_eth_type; + + // + // 2. IPv6 header + // + + let new_ipv6_src = old_ipv6_dst; // Swap source and destination + let new_ipv6_dst = port_and_peer.peer_ip(); + let new_ipv6_len = old_ipv6_len - CdHdr::LEN as u16; + + // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. + let ipv6 = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN)? }; + ipv6.set_version(6); // IPv6 + ipv6.set_priority(old_ipv6_priority); + ipv6.flow_label = old_ipv6_flow_label; + ipv6.set_payload_len(new_ipv6_len); + ipv6.next_hdr = old_ipv6_next_hdr; + ipv6.hop_limit = old_ipv6_hop_limit; + ipv6.set_src_addr(new_ipv6_src); + ipv6.set_dst_addr(new_ipv6_dst); + + // + // 3. UDP header + // + + let new_udp_src = port_and_peer.allocation_port(); + let new_udp_dst = port_and_peer.peer_port(); + let new_udp_len = old_udp_len - CdHdr::LEN as u16; + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = + unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN + Ipv6Hdr::LEN)? }; + udp.set_source(new_udp_src); + udp.set_dest(new_udp_dst); + udp.set_len(new_udp_len); + + // Incrementally update UDP checksum + + udp.set_check( + ChecksumUpdate::new(old_udp_check) + .remove_u128(u128::from_be_bytes(old_ipv6_src.octets())) + .add_u128(u128::from_be_bytes(new_ipv6_dst.octets())) + .remove_u16(old_udp_src) + .add_u16(new_udp_src) + .remove_u16(old_udp_dst) + .add_u16(new_udp_dst) + .remove_u16(old_udp_len) + .add_u16(new_udp_len) + .remove_u16(old_udp_len) + .add_u16(new_udp_len) + .remove_u16(channel_number) + .remove_u16(channel_data_length) + .into_udp_checksum(), + ); + + adjust_head(ctx, NET_SHRINK)?; + + Ok(()) +} diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv6_udp.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv6_udp.rs new file mode 100644 index 000000000..db8e90e5d --- /dev/null +++ b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv6_udp.rs @@ -0,0 +1,5 @@ +mod to_ipv4_channel; +mod to_ipv6_channel; + +pub use to_ipv4_channel::*; +pub use to_ipv6_channel::*; diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv6_udp/to_ipv4_channel.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv6_udp/to_ipv4_channel.rs new file mode 100644 index 000000000..3935ed26e --- /dev/null +++ b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv6_udp/to_ipv4_channel.rs @@ -0,0 +1,143 @@ +use aya_ebpf::programs::XdpContext; +use ebpf_shared::ClientAndChannelV4; +use network_types::{ + eth::{EthHdr, EtherType}, + ip::{Ipv4Hdr, Ipv6Hdr}, + udp::UdpHdr, +}; + +use crate::try_handle_turn::{ + Error, adjust_head, + channel_data::CdHdr, + checksum::{self, ChecksumUpdate}, + config, ref_mut_at, +}; + +#[inline(always)] +pub fn to_ipv4_channel( + ctx: &XdpContext, + client_and_channel: &ClientAndChannelV4, +) -> Result<(), Error> { + // 40 - 20 - 4 = 16 bytes shrink + const NET_SHRINK: i32 = Ipv6Hdr::LEN as i32 - Ipv4Hdr::LEN as i32 - CdHdr::LEN as i32; + + let (old_eth_src, old_eth_dst) = { + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let old_eth = unsafe { ref_mut_at::(ctx, 0)? }; + (old_eth.src_addr, old_eth.dst_addr) + }; + + let (old_ipv6_src, old_ipv6_dst, old_ipv6_priority, old_ipv6_hop_limit, old_ipv6_next_hdr) = { + // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. + let old_ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + ( + old_ipv6.src_addr(), + old_ipv6.dst_addr(), + old_ipv6.priority(), + old_ipv6.hop_limit, + old_ipv6.next_hdr, + ) + }; + + let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let old_udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; + ( + old_udp.len(), + old_udp.source(), + old_udp.dest(), + old_udp.check(), + ) + }; + + // + // 1. Ethernet header + // + + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let eth = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize)? }; + eth.src_addr = old_eth_dst; // Swap source and destination + eth.dst_addr = old_eth_src; + eth.ether_type = EtherType::Ipv4; // Change to IPv4 + + // + // 2. IPv6 -> IPv4 header + // + + let new_ipv4_src = config::interface_ipv4_address()?; + let new_ipv4_dst = client_and_channel.client_ip(); + let new_udp_len = old_udp_len + CdHdr::LEN as u16; + let new_ipv4_len = Ipv4Hdr::LEN as u16 + new_udp_len; + + // SAFETY: The offset must point to the start of a valid `Ipv4Hdr`. + let ipv4 = unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN)? }; + ipv4.set_version(4); + ipv4.set_ihl(5); // No options + ipv4.tos = old_ipv6_priority; + ipv4.set_total_len(new_ipv4_len); + ipv4.set_id(0); // Default ID + ipv4.frag_off = 0x4000_u16.to_be_bytes(); // Don't fragment + ipv4.ttl = old_ipv6_hop_limit; // Preserve hop limit + ipv4.proto = old_ipv6_next_hdr; // Preserve protocol + ipv4.set_src_addr(new_ipv4_src); + ipv4.set_dst_addr(new_ipv4_dst); + + // Calculate fresh checksum + let check = checksum::new_ipv4(ipv4); + ipv4.set_checksum(check); + + // + // 3. UDP header + // + + let new_udp_src = 3478_u16; // Fixed source port for TURN + let new_udp_dst = client_and_channel.client_port(); + let channel_number = client_and_channel.channel(); + let channel_data_length = old_udp_len - UdpHdr::LEN as u16; + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = + unsafe { ref_mut_at::(ctx, NET_SHRINK as usize + EthHdr::LEN + Ipv4Hdr::LEN)? }; + udp.set_source(new_udp_src); + udp.set_dest(new_udp_dst); + udp.set_len(new_udp_len); + + // Incrementally update UDP checksum + + udp.set_check( + ChecksumUpdate::new(old_udp_check) + .remove_u128(u128::from_be_bytes(old_ipv6_src.octets())) + .add_u32(u32::from_be_bytes(new_ipv4_src.octets())) + .remove_u128(u128::from_be_bytes(old_ipv6_dst.octets())) + .add_u32(u32::from_be_bytes(new_ipv4_dst.octets())) + .remove_u16(old_udp_src) + .add_u16(new_udp_src) + .remove_u16(old_udp_dst) + .add_u16(new_udp_dst) + .remove_u16(old_udp_len) + .add_u16(new_udp_len) + .remove_u16(old_udp_len) + .add_u16(new_udp_len) + .add_u16(channel_number) + .add_u16(channel_data_length) + .into_udp_checksum(), + ); + + // + // 4. Channel data header + // + + // SAFETY: The offset must point to the start of a valid `CdHdr`. + let cd = unsafe { + ref_mut_at::( + ctx, + NET_SHRINK as usize + EthHdr::LEN + Ipv4Hdr::LEN + UdpHdr::LEN, + )? + }; + cd.number = channel_number.to_be_bytes(); + cd.length = channel_data_length.to_be_bytes(); + + adjust_head(ctx, NET_SHRINK)?; + + Ok(()) +} diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv6_udp/to_ipv6_channel.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv6_udp/to_ipv6_channel.rs new file mode 100644 index 000000000..7a6abda84 --- /dev/null +++ b/rust/relay/ebpf-turn-router/src/try_handle_turn/from_ipv6_udp/to_ipv6_channel.rs @@ -0,0 +1,137 @@ +use aya_ebpf::programs::XdpContext; +use ebpf_shared::ClientAndChannelV6; +use network_types::{eth::EthHdr, ip::Ipv6Hdr, udp::UdpHdr}; + +use crate::try_handle_turn::{ + Error, adjust_head, channel_data::CdHdr, checksum::ChecksumUpdate, ref_mut_at, +}; + +#[inline(always)] +pub fn to_ipv6_channel( + ctx: &XdpContext, + client_and_channel: &ClientAndChannelV6, +) -> Result<(), Error> { + // Expand by 4 bytes for channel data header + const NET_EXPANSION: i32 = -(CdHdr::LEN as i32); + + adjust_head(ctx, NET_EXPANSION)?; + + // Now read the old packet data from its NEW location (shifted by 4 bytes) + let old_data_offset = CdHdr::LEN; + + let (old_eth_src, old_eth_dst, old_eth_type) = { + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let old_eth = unsafe { ref_mut_at::(ctx, old_data_offset)? }; + (old_eth.src_addr, old_eth.dst_addr, old_eth.ether_type) + }; + + let ( + old_ipv6_src, + old_ipv6_dst, + old_ipv6_len, + old_ipv6_priority, + old_ipv6_flow_label, + old_ipv6_hop_limit, + old_ipv6_next_hdr, + ) = { + // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. + let old_ipv6 = unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN)? }; + ( + old_ipv6.src_addr(), + old_ipv6.dst_addr(), + old_ipv6.payload_len(), + old_ipv6.priority(), + old_ipv6.flow_label, + old_ipv6.hop_limit, + old_ipv6.next_hdr, + ) + }; + + let (old_udp_len, old_udp_src, old_udp_dst, old_udp_check) = { + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let old_udp = + unsafe { ref_mut_at::(ctx, old_data_offset + EthHdr::LEN + Ipv6Hdr::LEN)? }; + ( + old_udp.len(), + old_udp.source(), + old_udp.dest(), + old_udp.check(), + ) + }; + + // + // 1. Ethernet header + // + + // SAFETY: The offset must point to the start of a valid `EthHdr`. + let eth = unsafe { ref_mut_at::(ctx, 0)? }; + eth.src_addr = old_eth_dst; // Swap source and destination + eth.dst_addr = old_eth_src; + eth.ether_type = old_eth_type; + + // + // 2. IPv6 header + // + + let new_ipv6_src = old_ipv6_dst; + let new_ipv6_dst = client_and_channel.client_ip(); + let new_ipv6_len = old_ipv6_len + CdHdr::LEN as u16; + + // SAFETY: The offset must point to the start of a valid `Ipv6Hdr`. + let ipv6 = unsafe { ref_mut_at::(ctx, EthHdr::LEN)? }; + // Set fields explicitly to avoid reading potentially corrupted memory + ipv6.set_version(6); // IPv6 + ipv6.set_priority(old_ipv6_priority); + ipv6.flow_label = old_ipv6_flow_label; + ipv6.set_payload_len(new_ipv6_len); + ipv6.next_hdr = old_ipv6_next_hdr; + ipv6.hop_limit = old_ipv6_hop_limit; + ipv6.set_src_addr(new_ipv6_src); + ipv6.set_dst_addr(new_ipv6_dst); + + // + // 3. UDP header + // + + let channel_number = client_and_channel.channel(); + let channel_data_length = old_udp_len - UdpHdr::LEN as u16; + let new_udp_len = old_udp_len + CdHdr::LEN as u16; + let new_udp_src = 3478_u16; + let new_udp_dst = client_and_channel.client_port(); + + // SAFETY: The offset must point to the start of a valid `UdpHdr`. + let udp = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN)? }; + udp.set_source(new_udp_src); + udp.set_dest(new_udp_dst); + udp.set_len(new_udp_len); + + // Incrementally update UDP checksum + + udp.set_check( + ChecksumUpdate::new(old_udp_check) + .remove_u128(u128::from_be_bytes(old_ipv6_src.octets())) + .add_u128(u128::from_be_bytes(new_ipv6_dst.octets())) + .remove_u16(old_udp_src) + .add_u16(new_udp_src) + .remove_u16(old_udp_dst) + .add_u16(new_udp_dst) + .remove_u16(old_udp_len) + .add_u16(new_udp_len) + .remove_u16(old_udp_len) + .add_u16(new_udp_len) + .add_u16(channel_number) + .add_u16(channel_data_length) + .into_udp_checksum(), + ); + + // + // 4. Channel data header + // + + // SAFETY: The offset must point to the start of a valid `CdHdr`. + let cd = unsafe { ref_mut_at::(ctx, EthHdr::LEN + Ipv6Hdr::LEN + UdpHdr::LEN)? }; + cd.number = channel_number.to_be_bytes(); + cd.length = channel_data_length.to_be_bytes(); + + Ok(()) +} diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn/interface.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn/interface.rs deleted file mode 100644 index 3df7a6139..000000000 --- a/rust/relay/ebpf-turn-router/src/try_handle_turn/interface.rs +++ /dev/null @@ -1,37 +0,0 @@ -//! Per-CPU data structures to store relay interface addresses. - -use core::net::{Ipv4Addr, Ipv6Addr}; - -use aya_ebpf::{macros::map, maps::PerCpuArray}; -use ebpf_shared::{InterfaceAddressV4, InterfaceAddressV6}; - -use crate::try_handle_turn::Error; - -#[map] -static INT_ADDR_V4: PerCpuArray = PerCpuArray::with_max_entries(1, 0); -#[map] -static INT_ADDR_V6: PerCpuArray = PerCpuArray::with_max_entries(1, 0); - -#[inline(always)] -pub fn ipv4_address() -> Result { - let interface_addr = INT_ADDR_V4 - .get_ptr_mut(0) - .ok_or(Error::InterfaceIpv4AddressAccessFailed)?; - - // SAFETY: This comes from a per-cpu data structure so we can safely access it. - let addr = unsafe { *interface_addr }; - - addr.get().ok_or(Error::InterfaceIpv4AddressNotConfigured) -} - -#[inline(always)] -pub fn ipv6_address() -> Result { - let interface_addr = INT_ADDR_V6 - .get_ptr_mut(0) - .ok_or(Error::InterfaceIpv6AddressAccessFailed)?; - - // SAFETY: This comes from a per-cpu data structure so we can safely access it. - let addr = unsafe { *interface_addr }; - - addr.get().ok_or(Error::InterfaceIpv6AddressNotConfigured) -} diff --git a/rust/relay/ebpf-turn-router/src/try_handle_turn/routing.rs b/rust/relay/ebpf-turn-router/src/try_handle_turn/routing.rs new file mode 100644 index 000000000..e49699600 --- /dev/null +++ b/rust/relay/ebpf-turn-router/src/try_handle_turn/routing.rs @@ -0,0 +1,140 @@ +//! Houses all combinations of IPv4 <> IPv6 and Channel <> UDP mappings. +//! +//! Testing has shown that these maps are safe to use as long as we aren't +//! writing to them from multiple threads at the same time. Since we only update these +//! from the single-threaded eventloop in userspace, we are ok. +//! See . + +use aya_ebpf::{macros::map, maps::HashMap}; +use ebpf_shared::{ + ClientAndChannel, ClientAndChannelV4, ClientAndChannelV6, PortAndPeer, PortAndPeerV4, + PortAndPeerV6, +}; + +use crate::try_handle_turn::Error; + +use super::error::SupportedChannel; + +const NUM_ENTRIES: u32 = 0x10000; + +#[map] +static CHAN_TO_UDP_44: HashMap = + HashMap::with_max_entries(NUM_ENTRIES, 0); +#[map] +static UDP_TO_CHAN_44: HashMap = + HashMap::with_max_entries(NUM_ENTRIES, 0); +#[map] +static CHAN_TO_UDP_66: HashMap = + HashMap::with_max_entries(NUM_ENTRIES, 0); +#[map] +static UDP_TO_CHAN_66: HashMap = + HashMap::with_max_entries(NUM_ENTRIES, 0); +#[map] +static CHAN_TO_UDP_46: HashMap = + HashMap::with_max_entries(NUM_ENTRIES, 0); +#[map] +static UDP_TO_CHAN_46: HashMap = + HashMap::with_max_entries(NUM_ENTRIES, 0); +#[map] +static CHAN_TO_UDP_64: HashMap = + HashMap::with_max_entries(NUM_ENTRIES, 0); +#[map] +static UDP_TO_CHAN_64: HashMap = + HashMap::with_max_entries(NUM_ENTRIES, 0); + +#[inline(always)] +pub fn get_client_and_channel(key: impl Into) -> Result { + let key = key.into(); + + let maybe_v4 = get_client_and_channel_v4(key).map(ClientAndChannel::V4); + let maybe_v6 = get_client_and_channel_v6(key).map(ClientAndChannel::V6); + + maybe_v4.or(maybe_v6) +} + +#[inline(always)] +pub fn get_port_and_peer(key: impl Into) -> Result { + let key = key.into(); + + let maybe_v4 = get_port_and_peer_v4(key).map(PortAndPeer::V4); + let maybe_v6 = get_port_and_peer_v6(key).map(PortAndPeer::V6); + + maybe_v4.or(maybe_v6) +} + +#[inline(always)] +fn get_client_and_channel_v4(key: impl Into) -> Result { + match key.into() { + PortAndPeer::V4(pp) => unsafe { UDP_TO_CHAN_44.get(&pp) } + .copied() + .ok_or(Error::NoEntry(SupportedChannel::Udp4ToChan)), + PortAndPeer::V6(pp) => unsafe { UDP_TO_CHAN_64.get(&pp) } + .copied() + .ok_or(Error::NoEntry(SupportedChannel::Udp6ToChan)), + } +} + +#[inline(always)] +fn get_client_and_channel_v6(key: impl Into) -> Result { + match key.into() { + PortAndPeer::V4(pp) => unsafe { UDP_TO_CHAN_46.get(&pp) } + .copied() + .ok_or(Error::NoEntry(SupportedChannel::Udp4ToChan)), + PortAndPeer::V6(pp) => unsafe { UDP_TO_CHAN_66.get(&pp) } + .copied() + .ok_or(Error::NoEntry(SupportedChannel::Udp6ToChan)), + } +} + +#[inline(always)] +fn get_port_and_peer_v4(key: impl Into) -> Result { + match key.into() { + ClientAndChannel::V4(pp) => unsafe { CHAN_TO_UDP_44.get(&pp) } + .copied() + .ok_or(Error::NoEntry(SupportedChannel::Chan4ToUdp)), + ClientAndChannel::V6(pp) => unsafe { CHAN_TO_UDP_64.get(&pp) } + .copied() + .ok_or(Error::NoEntry(SupportedChannel::Chan6ToUdp)), + } +} + +#[inline(always)] +fn get_port_and_peer_v6(key: impl Into) -> Result { + match key.into() { + ClientAndChannel::V4(pp) => unsafe { CHAN_TO_UDP_46.get(&pp) } + .copied() + .ok_or(Error::NoEntry(SupportedChannel::Chan4ToUdp)), + ClientAndChannel::V6(pp) => unsafe { CHAN_TO_UDP_66.get(&pp) } + .copied() + .ok_or(Error::NoEntry(SupportedChannel::Chan6ToUdp)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Memory overhead of an eBPF map. + /// + /// Determined empirically. + const HASH_MAP_OVERHEAD: f32 = 1.5; + + #[test] + fn hashmaps_are_less_than_11_mb() { + let ipv4_datatypes = + core::mem::size_of::() + core::mem::size_of::(); + let ipv6_datatypes = + core::mem::size_of::() + core::mem::size_of::(); + + let ipv4_map_size = ipv4_datatypes as f32 * NUM_ENTRIES as f32 * HASH_MAP_OVERHEAD; + let ipv6_map_size = ipv6_datatypes as f32 * NUM_ENTRIES as f32 * HASH_MAP_OVERHEAD; + + let total_map_size = (ipv4_map_size + ipv6_map_size) * 2_f32; + let total_map_size_mb = total_map_size / 1024_f32 / 1024_f32; + + assert!( + total_map_size_mb < 11_f32, + "Total map size = {total_map_size_mb} MB" + ); + } +} diff --git a/rust/relay/server/src/ebpf/linux.rs b/rust/relay/server/src/ebpf/linux.rs index 749d7051a..065604a30 100644 --- a/rust/relay/server/src/ebpf/linux.rs +++ b/rust/relay/server/src/ebpf/linux.rs @@ -9,8 +9,7 @@ use aya::{ use aya_log::EbpfLogger; use bytes::BytesMut; use ebpf_shared::{ - ClientAndChannelV4, ClientAndChannelV6, InterfaceAddressV4, InterfaceAddressV6, PortAndPeerV4, - PortAndPeerV6, StatsEvent, + ClientAndChannelV4, ClientAndChannelV6, PortAndPeerV4, PortAndPeerV6, StatsEvent, }; use stun_codec::rfc5766::attributes::ChannelNumber; @@ -36,8 +35,10 @@ impl Program { pub fn try_load( interface: &str, attach_mode: AttachMode, - ipv4_addr: Option, - ipv6_addr: Option, + ipv4_interface_addr: Option, + ipv6_interface_addr: Option, + ipv4_public_addr: Option, + ipv6_public_addr: Option, ) -> Result { let mut ebpf = aya::Ebpf::load(aya::include_bytes_aligned!(concat!( env!("OUT_DIR"), @@ -121,13 +122,21 @@ impl Program { } // Set interface addresses if provided - if let Some(ipv4) = ipv4_addr { + if let Some(ipv4) = ipv4_interface_addr { set_interface_ipv4_address(&mut ebpf, ipv4)?; } - if let Some(ipv6) = ipv6_addr { + if let Some(ipv6) = ipv6_interface_addr { set_interface_ipv6_address(&mut ebpf, ipv6)?; } + // Set public addresses if provided + if let Some(ipv4) = ipv4_public_addr { + set_public_ipv4_address(&mut ebpf, ipv4)?; + } + if let Some(ipv6) = ipv6_public_addr { + set_public_ipv6_address(&mut ebpf, ipv6)?; + } + tracing::info!("eBPF TURN router loaded and attached to interface {interface}"); Ok(Self { ebpf, stats }) @@ -301,10 +310,7 @@ impl Program { } fn set_interface_ipv4_address(ebpf: &mut aya::Ebpf, addr: Ipv4Addr) -> Result<()> { - let mut interface_addr = InterfaceAddressV4::default(); - interface_addr.set(addr); - - set_per_cpu_map(ebpf, "INT_ADDR_V4", interface_addr) + set_per_cpu_map(ebpf, "INT_ADDR_V4", addr.octets()) .context("Failed to set IPv4 interface address")?; tracing::info!(%addr, "Set eBPF interface IPv4 address"); @@ -312,16 +318,29 @@ fn set_interface_ipv4_address(ebpf: &mut aya::Ebpf, addr: Ipv4Addr) -> Result<() } fn set_interface_ipv6_address(ebpf: &mut aya::Ebpf, addr: Ipv6Addr) -> Result<()> { - let mut interface_addr = InterfaceAddressV6::default(); - interface_addr.set(addr); - - set_per_cpu_map(ebpf, "INT_ADDR_V6", interface_addr) + set_per_cpu_map(ebpf, "INT_ADDR_V6", addr.octets()) .context("Failed to set IPv6 interface address")?; tracing::info!(%addr, "Set eBPF interface IPv6 address"); Ok(()) } +fn set_public_ipv4_address(ebpf: &mut aya::Ebpf, addr: Ipv4Addr) -> Result<()> { + set_per_cpu_map(ebpf, "PUBLIC_ADDR_V4", addr.octets()) + .context("Failed to set IPv4 public address")?; + + tracing::info!(%addr, "Set eBPF public IPv4 address"); + Ok(()) +} + +fn set_public_ipv6_address(ebpf: &mut aya::Ebpf, addr: Ipv6Addr) -> Result<()> { + set_per_cpu_map(ebpf, "PUBLIC_ADDR_V6", addr.octets()) + .context("Failed to set IPv6 public address")?; + + tracing::info!(%addr, "Set eBPF public IPv6 address"); + Ok(()) +} + fn set_per_cpu_map(ebpf: &mut aya::Ebpf, map_name: &str, value: T) -> Result<()> where T: Pod + Clone, diff --git a/rust/relay/server/src/ebpf/stub.rs b/rust/relay/server/src/ebpf/stub.rs index 0ae6f0a93..468f6dcd9 100644 --- a/rust/relay/server/src/ebpf/stub.rs +++ b/rust/relay/server/src/ebpf/stub.rs @@ -18,6 +18,8 @@ impl Program { _: AttachMode, _: Option, _: Option, + _: Option, + _: Option, ) -> Result { Err(anyhow::anyhow!("Platform not supported")) } diff --git a/rust/relay/server/src/main.rs b/rust/relay/server/src/main.rs index ec8be3f90..e6d5964bd 100644 --- a/rust/relay/server/src/main.rs +++ b/rust/relay/server/src/main.rs @@ -192,6 +192,8 @@ async fn try_main(args: Args) -> Result<()> { args.ebpf_attach_mode, args.ebpf_int4_addr, args.ebpf_int6_addr, + args.public_ip4_addr, + args.public_ip6_addr, ) .context("Failed to load eBPF TURN router")?, ) diff --git a/scripts/tests/direct-curl-api-down.sh b/scripts/tests/curl-api-down.sh similarity index 100% rename from scripts/tests/direct-curl-api-down.sh rename to scripts/tests/curl-api-down.sh diff --git a/scripts/tests/direct-curl-api-restart.sh b/scripts/tests/curl-api-restart.sh similarity index 100% rename from scripts/tests/direct-curl-api-restart.sh rename to scripts/tests/curl-api-restart.sh diff --git a/scripts/tests/direct-curl-ecn.sh b/scripts/tests/curl-ecn.sh similarity index 100% rename from scripts/tests/direct-curl-ecn.sh rename to scripts/tests/curl-ecn.sh diff --git a/scripts/tests/direct-dns-api-down.sh b/scripts/tests/dns-api-down.sh similarity index 100% rename from scripts/tests/direct-dns-api-down.sh rename to scripts/tests/dns-api-down.sh diff --git a/scripts/tests/direct-dns-two-resources.sh b/scripts/tests/dns-two-resources.sh similarity index 100% rename from scripts/tests/direct-dns-two-resources.sh rename to scripts/tests/dns-two-resources.sh diff --git a/scripts/tests/direct-dns.sh b/scripts/tests/dns.sh similarity index 100% rename from scripts/tests/direct-dns.sh rename to scripts/tests/dns.sh diff --git a/scripts/tests/direct-download-packet-loss.sh b/scripts/tests/download-packet-loss.sh similarity index 100% rename from scripts/tests/direct-download-packet-loss.sh rename to scripts/tests/download-packet-loss.sh diff --git a/scripts/tests/direct-download-roaming-network.sh b/scripts/tests/download-roaming-network.sh similarity index 100% rename from scripts/tests/direct-download-roaming-network.sh rename to scripts/tests/download-roaming-network.sh diff --git a/scripts/tests/download.sh b/scripts/tests/download.sh new file mode 100755 index 000000000..63acf8343 --- /dev/null +++ b/scripts/tests/download.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +source "./scripts/tests/lib.sh" + +client sh -c "curl --fail --max-time 5 --output download.file http://download.httpbin/bytes?num=10000000" & + +DOWNLOAD_PID=$! + +wait $DOWNLOAD_PID || { + echo "Download process failed" + exit 1 +} + +known_checksum="f5e02aa71e67f41d79023a128ca35bad86cf7b6656967bfe0884b3a3c4325eaf" +computed_checksum=$(client sha256sum download.file | awk '{ print $1 }') + +if [[ "$computed_checksum" != "$known_checksum" ]]; then + echo "Checksum of downloaded file does not match" + exit 1 +fi