use std::{ collections::BTreeSet, io, net::{IpAddr, Ipv4Addr, Ipv6Addr}, pin::Pin, sync::{Arc, Weak}, task::{Context, Poll}, }; use crate::{ common::{ error::Error, global_ctx::{ArcGlobalCtx, GlobalCtxEvent}, ifcfg::{IfConfiger, IfConfiguerTrait}, log, }, instance::proxy_cidrs_monitor::ProxyCidrsMonitor, peers::{PacketRecvChanReceiver, peer_manager::PeerManager, recv_packet_from_chan}, tunnel::{ StreamItem, Tunnel, TunnelError, ZCPacketSink, ZCPacketStream, common::{FramedWriter, TunnelWrapper, ZCPacketToBytes, reserve_buf}, packet_def::{TAIL_RESERVED_SIZE, ZCPacket, ZCPacketType}, }, }; use byteorder::WriteBytesExt as _; use bytes::{BufMut, BytesMut}; use cidr::{Ipv4Inet, Ipv6Inet}; use futures::{SinkExt, Stream, StreamExt, lock::BiLock, ready}; use pin_project_lite::pin_project; use pnet::packet::{ipv4::Ipv4Packet, ipv6::Ipv6Packet}; use tokio::{ io::{AsyncRead, AsyncWrite, ReadBuf}, sync::{Mutex, Notify}, task::JoinSet, }; use tokio_util::bytes::Bytes; use tun::{AbstractDevice, AsyncDevice, Configuration, Layer}; use zerocopy::{NativeEndian, NetworkEndian}; #[cfg(target_os = "windows")] use crate::common::ifcfg::RegistryManager; pin_project! { pub struct TunStream { #[pin] l: BiLock, cur_buf: BytesMut, has_packet_info: bool, payload_offset: usize, } } impl TunStream { pub fn new(l: BiLock, has_packet_info: bool) -> Self { let mut payload_offset = ZCPacketType::NIC.get_packet_offsets().payload_offset; if has_packet_info { payload_offset -= 4; } Self { l, cur_buf: BytesMut::new(), has_packet_info, payload_offset, } } } impl Stream for TunStream { type Item = StreamItem; fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let self_mut = self.project(); let mut g = ready!(self_mut.l.poll_lock(cx)); reserve_buf(self_mut.cur_buf, 2500, 4 * 1024); if self_mut.cur_buf.is_empty() { unsafe { self_mut.cur_buf.set_len(*self_mut.payload_offset); } } let buf = self_mut.cur_buf.chunk_mut().as_mut_ptr(); let buf = unsafe { std::slice::from_raw_parts_mut(buf, 2500) }; let mut buf = ReadBuf::new(buf); let ret = ready!(g.as_pin_mut().poll_read(cx, &mut buf)); let len = buf.filled().len(); if len == 0 { return Poll::Ready(None); } unsafe { self_mut.cur_buf.advance_mut(len + TAIL_RESERVED_SIZE) }; let mut ret_buf = self_mut.cur_buf.split(); let cur_len = ret_buf.len(); ret_buf.truncate(cur_len - TAIL_RESERVED_SIZE); match ret { Ok(_) => Poll::Ready(Some(Ok(ZCPacket::new_from_buf(ret_buf, ZCPacketType::NIC)))), Err(err) => { log::error!("tun stream error: {:?}", err); Poll::Ready(None) } } } } #[derive(Debug, Clone, Copy, Default)] enum PacketProtocol { #[default] IPv4, IPv6, Other(u8), } // Note: the protocol in the packet information header is platform dependent. impl PacketProtocol { #[cfg(any(target_os = "linux", target_os = "android", target_env = "ohos"))] fn into_pi_field(self) -> Result { use nix::libc; match self { PacketProtocol::IPv4 => Ok(libc::ETH_P_IP as u16), PacketProtocol::IPv6 => Ok(libc::ETH_P_IPV6 as u16), PacketProtocol::Other(_) => Err(io::Error::other("neither an IPv4 nor IPv6 packet")), } } #[cfg(any(target_os = "macos", target_os = "ios", target_os = "freebsd"))] fn into_pi_field(self) -> Result { use nix::libc; match self { PacketProtocol::IPv4 => Ok(libc::PF_INET as u16), PacketProtocol::IPv6 => Ok(libc::PF_INET6 as u16), PacketProtocol::Other(_) => Err(io::Error::other("neither an IPv4 nor IPv6 packet")), } } #[cfg(target_os = "windows")] fn into_pi_field(self) -> Result { unimplemented!() } } /// Infer the protocol based on the first nibble in the packet buffer. fn infer_proto(buf: &[u8]) -> PacketProtocol { match buf[0] >> 4 { 4 => PacketProtocol::IPv4, 6 => PacketProtocol::IPv6, p => PacketProtocol::Other(p), } } struct TunZCPacketToBytes { has_packet_info: bool, } impl TunZCPacketToBytes { pub fn new(has_packet_info: bool) -> Self { Self { has_packet_info } } pub fn fill_packet_info( &self, mut buf: &mut [u8], proto: PacketProtocol, ) -> Result<(), io::Error> { // flags is always 0 buf.write_u16::(0)?; // write the protocol as network byte order buf.write_u16::(proto.into_pi_field()?)?; Ok(()) } } impl ZCPacketToBytes for TunZCPacketToBytes { fn zcpacket_into_bytes(&self, zc_packet: ZCPacket) -> Result { let payload_offset = zc_packet.payload_offset(); let mut inner = zc_packet.inner(); // we have peer manager header, so payload offset must larger than 4 assert!(payload_offset >= 4); let ret = if self.has_packet_info { let mut inner = inner.split_off(payload_offset - 4); let proto = infer_proto(&inner[4..]); self.fill_packet_info(&mut inner[0..4], proto)?; inner } else { inner.split_off(payload_offset) }; tracing::debug!(?ret, ?payload_offset, "convert zc packet to tun packet"); Ok(ret.into()) } } pin_project! { pub struct TunAsyncWrite { #[pin] l: BiLock, } } impl AsyncWrite for TunAsyncWrite { fn poll_write( self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8], ) -> Poll> { let self_mut = self.project(); let mut g = ready!(self_mut.l.poll_lock(cx)); g.as_pin_mut().poll_write(cx, buf) } fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let self_mut = self.project(); let mut g = ready!(self_mut.l.poll_lock(cx)); g.as_pin_mut().poll_flush(cx) } fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let self_mut = self.project(); let mut g = ready!(self_mut.l.poll_lock(cx)); g.as_pin_mut().poll_shutdown(cx) } fn poll_write_vectored( self: Pin<&mut Self>, cx: &mut Context<'_>, bufs: &[io::IoSlice<'_>], ) -> Poll> { let self_mut = self.project(); let mut g = ready!(self_mut.l.poll_lock(cx)); g.as_pin_mut().poll_write_vectored(cx, bufs) } fn is_write_vectored(&self) -> bool { true } } pub struct VirtualNic { global_ctx: ArcGlobalCtx, ifname: Option, ifcfg: Box, } impl Drop for VirtualNic { fn drop(&mut self) { #[cfg(target_os = "windows")] { if let Some(ref ifname) = self.ifname { // Try to clean up firewall rules, but don't panic in destructor if let Err(error) = crate::arch::windows::remove_interface_firewall_rules(ifname) { log::warn!( %error, "failed to remove firewall rules for interface {}", ifname ); } } } } } impl VirtualNic { pub fn new(global_ctx: ArcGlobalCtx) -> Self { Self { global_ctx, ifname: None, ifcfg: Box::new(IfConfiger {}), } } /// Check and create TUN device node if necessary on Linux systems #[cfg(target_os = "linux")] async fn ensure_tun_device_node() { const TUN_DEV_PATH: &str = "/dev/net/tun"; const TUN_DIR_PATH: &str = "/dev/net"; // Check if /dev/net/tun already exists if tokio::fs::metadata(TUN_DEV_PATH).await.is_ok() { tracing::debug!("TUN device node {} already exists", TUN_DEV_PATH); return; } tracing::info!( "TUN device node {} not found, attempting to create", TUN_DEV_PATH ); // Check if TUN kernel module is available let tun_module_available = tokio::fs::metadata("/proc/net/dev").await.is_ok() && (tokio::fs::read_to_string("/proc/modules").await) .map(|content| content.contains("tun")) .unwrap_or(false); if !tun_module_available { log::warn!("TUN kernel module may not be available."); log::warn!("\tYou may need to load it with: sudo modprobe tun."); } // Try to create /dev/net directory if it doesn't exist if tokio::fs::metadata(TUN_DIR_PATH).await.is_err() { if let Err(error) = tokio::fs::create_dir_all(TUN_DIR_PATH).await { log::warn!( ?error, "Failed to create directory {}. TUN device creation may fail. Continuing anyway.", TUN_DIR_PATH ); log::warn!( "\tYou may need to run with root privileges or manually create the TUN device." ); Self::print_troubleshooting_info(); return; } tracing::info!("Created directory {}", TUN_DIR_PATH); } // Try to create the TUN device node // Major number 10, minor number 200 for /dev/net/tun let dev_node = nix::sys::stat::makedev(10, 200); match nix::sys::stat::mknod( TUN_DEV_PATH, nix::sys::stat::SFlag::S_IFCHR, nix::sys::stat::Mode::from_bits(0o600).unwrap(), dev_node, ) { Ok(_) => { log::info!("Successfully created TUN device node {}", TUN_DEV_PATH); } Err(error) => { tracing::warn!( %error, "Failed to create TUN device node {}. Continuing anyway.", TUN_DEV_PATH, ); Self::print_troubleshooting_info(); } } } /// Print troubleshooting information for TUN device issues #[cfg(target_os = "linux")] fn print_troubleshooting_info() { log::info!( "Possible solutions:\ \n\t1. Run with root privileges: sudo ./easytier-core [options]\ \n\t2. Manually create TUN device: sudo mkdir -p /dev/net && sudo mknod /dev/net/tun c 10 200\ \n\t3. Load TUN kernel module: sudo modprobe tun\ \n\t4. Use --no-tun flag if TUN functionality is not needed\ \n\t5. Check if your system/container supports TUN devices\ \nNote: TUN functionality may still work if the kernel supports dynamic device creation." ); } /// For non-Linux systems, this is a no-op #[cfg(not(target_os = "linux"))] async fn ensure_tun_device_node() -> Result<(), Error> { Ok(()) } /// FreeBSD specific: Rename a TUN interface #[cfg(target_os = "freebsd")] async fn rename_tun_interface(old_name: &str, new_name: &str) -> Result<(), Error> { let output = tokio::process::Command::new("ifconfig") .arg(old_name) .arg("name") .arg(new_name) .output() .await?; if output.status.success() { tracing::info!( "Successfully renamed interface {} to {}", old_name, new_name ); Ok(()) } else { let stderr = String::from_utf8_lossy(&output.stderr); tracing::warn!( "Failed to rename interface {} to {}: {}", old_name, new_name, stderr ); // Return Ok even if rename fails, as it's not critical Ok(()) } } /// FreeBSD specific: List all TUN interface names #[cfg(target_os = "freebsd")] async fn list_tun_names() -> Result, Error> { let output = tokio::process::Command::new("ifconfig") .arg("-g") .arg("tun") .output() .await?; if output.status.success() { let stdout = String::from_utf8_lossy(&output.stdout); let tun_names: Vec = stdout .trim() .split_whitespace() .map(|s| s.to_string()) .collect(); tracing::debug!("Found TUN interfaces: {:?}", tun_names); Ok(tun_names) } else { let stderr = String::from_utf8_lossy(&output.stderr); tracing::warn!("Failed to list TUN interfaces: {}", stderr); Ok(Vec::new()) } } /// FreeBSD specific: Get interface information #[cfg(target_os = "freebsd")] async fn get_interface_info(ifname: &str) -> Result { let output = tokio::process::Command::new("ifconfig") .arg("-v") .arg(ifname) .output() .await?; if output.status.success() { Ok(String::from_utf8_lossy(&output.stdout).to_string()) } else { let stderr = String::from_utf8_lossy(&output.stderr); Err( anyhow::anyhow!("Failed to get interface details for {}: {}", ifname, stderr) .into(), ) } } /// FreeBSD specific: Extract original name from interface information #[cfg(target_os = "freebsd")] fn extract_original_name(ifinfo: &str) -> Option { ifinfo .lines() .find(|line| line.trim().starts_with("drivername:")) .and_then(|line| line.trim().split_whitespace().nth(1)) .map(|name| name.to_string()) } /// FreeBSD specific: Check if interface is used by any process #[cfg(target_os = "freebsd")] fn is_interface_used(ifinfo: &str) -> bool { ifinfo.contains("Opened by PID") } /// FreeBSD specific: Restore TUN interface name to its original value #[cfg(target_os = "freebsd")] async fn restore_tun_name(dev_name: &str) -> Result<(), Error> { let tun_names = Self::list_tun_names().await?; // Check if desired dev_name is in use if tun_names.iter().any(|name| name == dev_name) { tracing::debug!( "Desired dev_name {} is in TUN interfaces list, checking if it can be renamed", dev_name ); let ifinfo = Self::get_interface_info(dev_name).await?; // Check if interface is not occupied if !Self::is_interface_used(&ifinfo) { // Extract original name if let Some(orig_name) = Self::extract_original_name(&ifinfo) { if orig_name != dev_name { tracing::info!( "Restoring dev_name {} to original name {}", dev_name, orig_name ); // Rename interface Self::rename_tun_interface(dev_name, &orig_name).await?; } } } else { tracing::debug!( "Interface {} is opened by a process, skipping rename", dev_name ); } } Ok(()) } async fn create_tun(&self) -> Result { let mut config = Configuration::default(); config.layer(Layer::L3); // FreeBSD specific: Check and restore TUN interfaces before creating new one #[cfg(target_os = "freebsd")] { let dev_name = self.global_ctx.get_flags().dev_name; if !dev_name.is_empty() { // Restore TUN interface name if needed, ignoring errors as it's not critical let _ = Self::restore_tun_name(&dev_name).await; } } #[cfg(target_os = "linux")] { // Check and create TUN device node if necessary (Linux only) Self::ensure_tun_device_node().await; let dev_name = self.global_ctx.get_flags().dev_name; if !dev_name.is_empty() { config.tun_name(&dev_name); } } #[cfg(all(target_os = "macos", not(feature = "macos-ne")))] config.platform_config(|config| { // disable packet information so we can process the header by ourselves, see tun2 impl for more details config.packet_information(false); }); #[cfg(target_os = "windows")] { let dev_name = self.global_ctx.get_flags().dev_name; match crate::arch::windows::add_self_to_firewall_allowlist() { Ok(_) => tracing::info!("add_self_to_firewall_allowlist successful!"), Err(error) => { log::warn!(%error, "Failed to add Easytier to firewall allowlist, Subnet proxy and KCP proxy may not work properly."); log::warn!( "You can add firewall rules manually, or use --use-smoltcp to run with user-space TCP/IP stack." ); } } match RegistryManager::reg_delete_obsoleted_items(&dev_name) { Ok(_) => tracing::trace!("delete successful!"), Err(e) => tracing::error!("An error occurred: {}", e), } if !dev_name.is_empty() { config.tun_name(&dev_name); } else { use rand::distributions::Distribution as _; let c = crate::arch::windows::interface_count()?; let mut rng = rand::thread_rng(); let s: String = rand::distributions::Alphanumeric .sample_iter(&mut rng) .take(4) .map(char::from) .collect::() .to_lowercase(); let random_dev_name = format!("et_{}_{}", c, s); config.tun_name(random_dev_name.clone()); let mut flags = self.global_ctx.get_flags(); flags.dev_name = random_dev_name.clone(); self.global_ctx.set_flags(flags); } config.platform_config(|config| { config.skip_config(true); config.ring_cap(Some(std::cmp::min( config.min_ring_cap() * 32, config.max_ring_cap(), ))); }); } config.up(); let _g = self.global_ctx.net_ns.guard(); Ok(tun::create(&config)?) } #[cfg(mobile)] pub async fn create_dev_for_mobile( &mut self, tun_fd: std::os::fd::RawFd, ) -> Result, Error> { log::debug!(%tun_fd); let mut config = Configuration::default(); config.layer(Layer::L3); #[cfg(any(target_os = "ios", all(target_os = "macos", feature = "macos-ne")))] config.platform_config(|config| { // disable packet information so we can process the header by ourselves, see tun2 impl for more details config.packet_information(false); }); config.raw_fd(tun_fd); config.close_fd_on_drop(false); config.up(); let has_packet_info = cfg!(any( target_os = "ios", all(target_os = "macos", feature = "macos-ne") )); let dev = tun::create(&config)?; let dev = AsyncDevice::new(dev)?; let (a, b) = BiLock::new(dev); let ft = TunnelWrapper::new( TunStream::new(a, has_packet_info), FramedWriter::new_with_converter( TunAsyncWrite { l: b }, TunZCPacketToBytes::new(has_packet_info), ), None, ); self.ifname = Some(format!("tunfd_{}", tun_fd)); Ok(Box::new(ft)) } pub async fn create_dev(&mut self) -> Result, Error> { let dev = self.create_tun().await?; #[cfg(not(target_os = "freebsd"))] let ifname = dev.tun_name()?; #[cfg(target_os = "freebsd")] let mut ifname = dev.tun_name()?; self.ifcfg.wait_interface_show(ifname.as_str()).await?; // FreeBSD TUN interface rename functionality #[cfg(target_os = "freebsd")] { let dev_name = self.global_ctx.get_flags().dev_name; if !dev_name.is_empty() && dev_name != ifname { // Use ifconfig to rename the TUN interface if Self::rename_tun_interface(&ifname, &dev_name).await.is_ok() { ifname = dev_name; } } } #[cfg(target_os = "windows")] { if let Ok(guid) = RegistryManager::find_interface_guid(&ifname) { if let Err(e) = RegistryManager::disable_dynamic_updates(&guid) { tracing::error!( "Failed to disable dhcp for interface {} {}: {}", ifname, guid, e ); } // Disable NetBIOS over TCP/IP if let Err(e) = RegistryManager::disable_netbios(&guid) { tracing::error!( "Failed to disable netbios for interface {} {}: {}", ifname, guid, e ); } } } let dev = AsyncDevice::new(dev)?; let flags = self.global_ctx.config.get_flags(); let mut mtu_in_config = flags.mtu; if flags.enable_encryption { mtu_in_config -= 20; } { // set mtu by ourselves, rust-tun does not handle it correctly on windows let _g = self.global_ctx.net_ns.guard(); self.ifcfg.set_mtu(ifname.as_str(), mtu_in_config).await?; } let has_packet_info = cfg!(all(target_os = "macos", not(feature = "macos-ne"))); let (a, b) = BiLock::new(dev); let ft = TunnelWrapper::new( TunStream::new(a, has_packet_info), FramedWriter::new_with_converter( TunAsyncWrite { l: b }, TunZCPacketToBytes::new(has_packet_info), ), None, ); self.ifname = Some(ifname.to_owned()); #[cfg(target_os = "windows")] { // Add firewall rules for virtual NIC interface to allow all traffic match crate::arch::windows::add_interface_to_firewall_allowlist(&ifname) { Ok(_) => { tracing::info!( "Successfully configured Windows Firewall for interface: {}", ifname ); tracing::info!( "All protocols (TCP/UDP/ICMP) are now allowed on interface: {}", ifname ); } Err(error) => { log::warn!(%error, "Failed to configure Windows Firewall for interface {}\ \n\tThis may cause connectivity issues with ping and other network functions.\ \n\tPlease run as Administrator or manually configure Windows Firewall.\ \n\tAlternatively, you can disable Windows Firewall for testing purposes.", ifname); } } } Ok(Box::new(ft)) } pub fn ifname(&self) -> &str { self.ifname.as_ref().unwrap().as_str() } pub async fn link_up(&self) -> Result<(), Error> { let _g = self.global_ctx.net_ns.guard(); self.ifcfg.set_link_status(self.ifname(), true).await?; Ok(()) } pub async fn add_route(&self, address: Ipv4Addr, cidr: u8) -> Result<(), Error> { let _g = self.global_ctx.net_ns.guard(); self.ifcfg .add_ipv4_route(self.ifname(), address, cidr, None) .await?; Ok(()) } pub async fn add_ipv6_route(&self, address: Ipv6Addr, cidr: u8) -> Result<(), Error> { let _g = self.global_ctx.net_ns.guard(); self.ifcfg .add_ipv6_route(self.ifname(), address, cidr, None) .await?; Ok(()) } pub async fn remove_ip(&self, ip: Option) -> Result<(), Error> { let _g = self.global_ctx.net_ns.guard(); self.ifcfg.remove_ip(self.ifname(), ip).await?; Ok(()) } pub async fn remove_ipv6(&self, ip: Option) -> Result<(), Error> { let _g = self.global_ctx.net_ns.guard(); self.ifcfg.remove_ipv6(self.ifname(), ip).await?; Ok(()) } pub async fn add_ip(&self, ip: Ipv4Addr, cidr: i32) -> Result<(), Error> { let _g = self.global_ctx.net_ns.guard(); self.ifcfg .add_ipv4_ip(self.ifname(), ip, cidr as u8) .await?; Ok(()) } pub async fn add_ipv6(&self, ip: Ipv6Addr, cidr: i32) -> Result<(), Error> { let _g = self.global_ctx.net_ns.guard(); self.ifcfg .add_ipv6_ip(self.ifname(), ip, cidr as u8) .await?; Ok(()) } pub fn get_ifcfg(&self) -> impl IfConfiguerTrait + use<> { IfConfiger {} } } pub struct NicCtx { global_ctx: ArcGlobalCtx, peer_mgr: Weak, peer_packet_receiver: Arc>, close_notifier: Arc, nic: Arc>, tasks: JoinSet<()>, } impl NicCtx { pub fn new( global_ctx: ArcGlobalCtx, peer_manager: &Arc, peer_packet_receiver: Arc>, close_notifier: Arc, ) -> Self { NicCtx { global_ctx: global_ctx.clone(), peer_mgr: Arc::downgrade(peer_manager), peer_packet_receiver, close_notifier, nic: Arc::new(Mutex::new(VirtualNic::new(global_ctx))), tasks: JoinSet::new(), } } pub async fn ifname(&self) -> Option { let nic = self.nic.lock().await; nic.ifname.as_ref().map(|s| s.to_owned()) } pub async fn assign_ipv4_to_tun_device(&self, ipv4_addr: cidr::Ipv4Inet) -> Result<(), Error> { let nic = self.nic.lock().await; nic.link_up().await?; nic.remove_ip(None).await?; nic.add_ip(ipv4_addr.address(), ipv4_addr.network_length() as i32) .await?; #[cfg(any( all(target_os = "macos", not(feature = "macos-ne")), target_os = "freebsd" ))] { nic.add_route(ipv4_addr.first_address(), ipv4_addr.network_length()) .await?; } Ok(()) } pub async fn assign_ipv6_to_tun_device(&self, ipv6_addr: cidr::Ipv6Inet) -> Result<(), Error> { let nic = self.nic.lock().await; nic.link_up().await?; nic.remove_ipv6(None).await?; nic.add_ipv6(ipv6_addr.address(), ipv6_addr.network_length() as i32) .await?; #[cfg(any( all(target_os = "macos", not(feature = "macos-ne")), target_os = "freebsd" ))] { nic.add_ipv6_route(ipv6_addr.first_address(), ipv6_addr.network_length()) .await?; } Ok(()) } async fn do_forward_nic_to_peers_ipv4(ret: ZCPacket, mgr: &PeerManager) { if let Some(ipv4) = Ipv4Packet::new(ret.payload()) { if ipv4.get_version() != 4 { tracing::info!("[USER_PACKET] not ipv4 packet: {:?}", ipv4); return; } let dst_ipv4 = ipv4.get_destination(); let src_ipv4 = ipv4.get_source(); let my_ipv4 = mgr.get_global_ctx().get_ipv4().map(|x| x.address()); tracing::trace!( ?ret, ?src_ipv4, ?dst_ipv4, "[USER_PACKET] recv new packet from tun device and forward to peers." ); // Subnet A is proxied as 10.0.0.0/24, and Subnet B is also proxied as 10.0.0.0/24. // // Subnet A has received a route advertised by Subnet B. As a result, A can reach // the physical subnet 10.0.0.0/24 directly and has also added a virtual route for // the same subnet 10.0.0.0/24. However, the physical route has a higher priority // (lower metric) than the virtual one. // // When A sends a UDP packet to a non-existent IP within this subnet, the packet // cannot be delivered on the physical network and is instead routed to the virtual // network interface. // // The virtual interface receives the packet and forwards it to itself, which triggers // the subnet proxy logic. The subnet proxy then attempts to send another packet to // the same destination address, causing the same process to repeat and creating an // infinite loop. Therefore, we must avoid re-sending packets back to ourselves // when the subnet proxy itself is the originator of the packet. // // However, there is a special scenario to consider: when A acts as a gateway, // packets from devices behind A may be forwarded by the OS to the ET (e.g., an // eBPF or tunneling component), which happens to proxy the subnet. In this case, // the packet’s source IP is not A’s own IP, and we must allow such packets to be // sent to the virtual interface (i.e., "sent to ourselves") to maintain correct // forwarding behavior. Thus, loop prevention should only apply when the source IP // belongs to the local host. let send_ret = mgr .send_msg_by_ip(ret, IpAddr::V4(dst_ipv4), Some(src_ipv4) == my_ipv4) .await; if send_ret.is_err() { tracing::trace!(?send_ret, "[USER_PACKET] send_msg failed") } } else { tracing::warn!(?ret, "[USER_PACKET] not ipv4 packet"); } } async fn do_forward_nic_to_peers_ipv6(ret: ZCPacket, mgr: &PeerManager) { if let Some(ipv6) = Ipv6Packet::new(ret.payload()) { if ipv6.get_version() != 6 { tracing::info!("[USER_PACKET] not ipv6 packet: {:?}", ipv6); return; } let src_ipv6 = ipv6.get_source(); let dst_ipv6 = ipv6.get_destination(); let my_ipv6 = mgr.get_global_ctx().get_ipv6().map(|x| x.address()); tracing::trace!( ?ret, ?src_ipv6, ?dst_ipv6, "[USER_PACKET] recv new packet from tun device and forward to peers." ); if src_ipv6.is_unicast_link_local() && Some(src_ipv6) != my_ipv6 { // do not route link local packet to other nodes unless the address is assigned by user return; } // TODO: use zero-copy let send_ret = mgr .send_msg_by_ip(ret, IpAddr::V6(dst_ipv6), Some(src_ipv6) == my_ipv6) .await; if send_ret.is_err() { tracing::trace!(?send_ret, "[USER_PACKET] send_msg failed") } } else { tracing::warn!(?ret, "[USER_PACKET] not ipv6 packet"); } } async fn do_forward_nic_to_peers(ret: ZCPacket, mgr: &PeerManager) { let payload = ret.payload(); if payload.is_empty() { return; } match payload[0] >> 4 { 4 => Self::do_forward_nic_to_peers_ipv4(ret, mgr).await, 6 => Self::do_forward_nic_to_peers_ipv6(ret, mgr).await, _ => { tracing::warn!(?ret, "[USER_PACKET] unknown IP version"); } } } fn do_forward_nic_to_peers_task( &mut self, mut stream: Pin>, ) -> Result<(), Error> { // read from nic and write to corresponding tunnel let Some(mgr) = self.peer_mgr.upgrade() else { return Err(anyhow::anyhow!("peer manager not available").into()); }; let close_notifier = self.close_notifier.clone(); self.tasks.spawn(async move { while let Some(ret) = stream.next().await { if ret.is_err() { tracing::error!("read from nic failed: {:?}", ret); break; } Self::do_forward_nic_to_peers(ret.unwrap(), mgr.as_ref()).await; } close_notifier.notify_one(); tracing::error!("nic closed when recving from it"); }); Ok(()) } fn do_forward_peers_to_nic(&mut self, mut sink: Pin>) { let channel = self.peer_packet_receiver.clone(); let close_notifier = self.close_notifier.clone(); self.tasks.spawn(async move { // unlock until coroutine finished let mut channel = channel.lock().await; while let Ok(packet) = recv_packet_from_chan(&mut channel).await { tracing::trace!( "[USER_PACKET] forward packet from peers to nic. packet: {:?}", packet ); let ret = sink.send(packet).await; if ret.is_err() { tracing::error!(?ret, "do_forward_tunnel_to_nic sink error"); } } close_notifier.notify_one(); tracing::error!("nic closed when sending to it"); }); } async fn apply_route_changes( ifcfg: &impl IfConfiguerTrait, ifname: &str, net_ns: &crate::common::netns::NetNS, cur_proxy_cidrs: &mut BTreeSet, added: Vec, removed: Vec, ) { tracing::debug!(?added, ?removed, "applying proxy_cidrs route changes"); // Remove routes for cidr in removed { if !cur_proxy_cidrs.contains(&cidr) { continue; } let _g = net_ns.guard(); let ret = ifcfg .remove_ipv4_route(ifname, cidr.first_address(), cidr.network_length()) .await; if ret.is_err() { tracing::trace!( cidr = ?cidr, err = ?ret, "remove route failed.", ); } cur_proxy_cidrs.remove(&cidr); } // Add routes for cidr in added { if cur_proxy_cidrs.contains(&cidr) { continue; } let _g = net_ns.guard(); let ret = ifcfg .add_ipv4_route(ifname, cidr.first_address(), cidr.network_length(), None) .await; if ret.is_err() { tracing::trace!( cidr = ?cidr, err = ?ret, "add route failed.", ); } cur_proxy_cidrs.insert(cidr); } } async fn run_proxy_cidrs_route_updater(&mut self) -> Result<(), Error> { let Some(peer_mgr) = self.peer_mgr.upgrade() else { return Err(anyhow::anyhow!("peer manager not available").into()); }; let global_ctx = self.global_ctx.clone(); let net_ns = self.global_ctx.net_ns.clone(); let nic = self.nic.lock().await; let ifcfg = nic.get_ifcfg(); let ifname = nic.ifname().to_owned(); let mut event_receiver = global_ctx.subscribe(); self.tasks.spawn(async move { let mut cur_proxy_cidrs = BTreeSet::::new(); // Initial sync: get current proxy_cidrs state and apply routes let (_, added, removed) = ProxyCidrsMonitor::diff_proxy_cidrs( peer_mgr.as_ref(), &global_ctx, &cur_proxy_cidrs, ) .await; Self::apply_route_changes( &ifcfg, &ifname, &net_ns, &mut cur_proxy_cidrs, added, removed, ) .await; loop { let event = match event_receiver.recv().await { Ok(event) => event, Err(tokio::sync::broadcast::error::RecvError::Closed) => { tracing::debug!("event bus closed, stopping proxy_cidrs route updater"); break; } Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => { tracing::warn!( "event bus lagged in proxy_cidrs route updater, doing full sync" ); event_receiver = event_receiver.resubscribe(); // Full sync after lagged to recover consistent state let (_, added, removed) = ProxyCidrsMonitor::diff_proxy_cidrs( peer_mgr.as_ref(), &global_ctx, &cur_proxy_cidrs, ) .await; GlobalCtxEvent::ProxyCidrsUpdated(added, removed) } }; // Only handle ProxyCidrsUpdated events let (added, removed) = match event { GlobalCtxEvent::ProxyCidrsUpdated(added, removed) => (added, removed), _ => continue, }; Self::apply_route_changes( &ifcfg, &ifname, &net_ns, &mut cur_proxy_cidrs, added, removed, ) .await; } }); Ok(()) } pub async fn run( &mut self, ipv4_addr: Option, ipv6_addr: Option, ) -> Result<(), Error> { let tunnel = { let mut nic = self.nic.lock().await; match nic.create_dev().await { Ok(ret) => { #[cfg(target_os = "windows")] { let dev_name = self.global_ctx.get_flags().dev_name; let _ = RegistryManager::reg_change_catrgory_in_profile(&dev_name); } #[cfg(any( all(target_os = "macos", not(feature = "macos-ne")), target_os = "freebsd" ))] { // remove the 10.0.0.0/24 route (which is added by rust-tun by default) let _ = nic .ifcfg .remove_ipv4_route(nic.ifname(), "10.0.0.0".parse().unwrap(), 24) .await; } self.global_ctx .issue_event(GlobalCtxEvent::TunDeviceReady(nic.ifname().to_string())); ret } Err(err) => { self.global_ctx .issue_event(GlobalCtxEvent::TunDeviceError(err.to_string())); return Err(err); } } }; let (stream, sink) = tunnel.split(); self.do_forward_nic_to_peers_task(stream)?; self.do_forward_peers_to_nic(sink); // Assign IPv4 address if provided if let Some(ipv4_addr) = ipv4_addr { self.assign_ipv4_to_tun_device(ipv4_addr).await?; } // Assign IPv6 address if provided if let Some(ipv6_addr) = ipv6_addr { self.assign_ipv6_to_tun_device(ipv6_addr).await?; } self.run_proxy_cidrs_route_updater().await?; Ok(()) } #[cfg(mobile)] pub async fn run_for_mobile(&mut self, tun_fd: std::os::fd::RawFd) -> Result<(), Error> { let tunnel = { let mut nic = self.nic.lock().await; match nic.create_dev_for_mobile(tun_fd).await { Ok(ret) => { self.global_ctx .issue_event(GlobalCtxEvent::TunDeviceReady(nic.ifname().to_string())); ret } Err(err) => { self.global_ctx .issue_event(GlobalCtxEvent::TunDeviceError(err.to_string())); return Err(err); } } }; let (stream, sink) = tunnel.split(); self.do_forward_nic_to_peers_task(stream)?; self.do_forward_peers_to_nic(sink); Ok(()) } } #[cfg(test)] mod tests { use crate::common::{error::Error, global_ctx::tests::get_mock_global_ctx}; use super::VirtualNic; async fn run_test_helper() -> Result { let mut dev = VirtualNic::new(get_mock_global_ctx()); let _tunnel = dev.create_dev().await?; tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; dev.link_up().await?; dev.remove_ip(None).await?; dev.add_ip("10.144.111.1".parse().unwrap(), 24).await?; Ok(dev) } #[tokio::test] async fn tun_test() { let _dev = run_test_helper().await.unwrap(); // let mut stream = nic.pin_recv_stream(); // while let Some(item) = stream.next().await { // println!("item: {:?}", item); // } // let framed = dev.into_framed(); // let (mut s, mut b) = framed.split(); // loop { // let tmp = b.next().await.unwrap().unwrap(); // let tmp = EthernetPacket::new(tmp.get_bytes()); // println!("ret: {:?}", tmp.unwrap()); // } } }