Set UDP socket buffer sizes to prevent receive overflow

Under high-throughput forwarding, the kernel default 212KB receive
buffer overflows, silently dropping ~11% of UDP datagrams at transit
nodes. Add configurable recv_buf_size and send_buf_size to UdpConfig
(default 2MB each) using socket2 for pre-bind buffer configuration.
Startup log now reports actual buffer sizes granted by the kernel.

Requires net.core.rmem_max >= 2097152 on the host for the full 2MB
to take effect; otherwise the kernel silently clamps.
This commit is contained in:
Johnathan Corgan
2026-02-19 05:51:36 +00:00
parent 8cc9532bad
commit ac82e61c77
7 changed files with 83 additions and 4 deletions

22
Cargo.lock generated
View File

@@ -546,6 +546,7 @@ dependencies = [
"serde_yaml",
"sha2",
"simple-dns",
"socket2 0.5.10",
"tempfile",
"thiserror 2.0.18",
"tokio",
@@ -1402,6 +1403,16 @@ version = "1.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
[[package]]
name = "socket2"
version = "0.5.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"
dependencies = [
"libc",
"windows-sys 0.52.0",
]
[[package]]
name = "socket2"
version = "0.6.2"
@@ -1518,7 +1529,7 @@ dependencies = [
"mio",
"pin-project-lite",
"signal-hook-registry",
"socket2",
"socket2 0.6.2",
"tokio-macros",
"windows-sys 0.61.2",
]
@@ -1770,6 +1781,15 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
[[package]]
name = "windows-sys"
version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-sys"
version = "0.59.0"

View File

@@ -24,6 +24,7 @@ rtnetlink = "0.14"
tokio = { version = "1", features = ["rt", "macros", "signal", "sync", "net", "time"] }
futures = "0.3"
simple-dns = "0.9"
socket2 = { version = "0.5", features = ["all"] }
[dev-dependencies]
tempfile = "3.15"

View File

@@ -517,6 +517,7 @@ transports: {}
let single = TransportInstances::Single(UdpConfig {
bind_addr: Some("0.0.0.0:4000".to_string()),
mtu: None,
..Default::default()
});
let items: Vec<_> = single.iter().collect();
assert_eq!(items.len(), 1);

View File

@@ -13,6 +13,12 @@ const DEFAULT_UDP_BIND_ADDR: &str = "0.0.0.0:4000";
/// Default UDP MTU (IPv6 minimum).
const DEFAULT_UDP_MTU: u16 = 1280;
/// Default UDP receive buffer size (2 MB).
const DEFAULT_UDP_RECV_BUF: usize = 2 * 1024 * 1024;
/// Default UDP send buffer size (2 MB).
const DEFAULT_UDP_SEND_BUF: usize = 2 * 1024 * 1024;
/// UDP transport instance configuration.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
@@ -24,6 +30,14 @@ pub struct UdpConfig {
/// UDP MTU (`mtu`). Defaults to 1280 (IPv6 minimum).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub mtu: Option<u16>,
/// UDP receive buffer size in bytes (`recv_buf_size`). Defaults to 2 MB.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub recv_buf_size: Option<usize>,
/// UDP send buffer size in bytes (`send_buf_size`). Defaults to 2 MB.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub send_buf_size: Option<usize>,
}
impl UdpConfig {
@@ -36,6 +50,16 @@ impl UdpConfig {
pub fn mtu(&self) -> u16 {
self.mtu.unwrap_or(DEFAULT_UDP_MTU)
}
/// Get the receive buffer size, using default if not configured.
pub fn recv_buf_size(&self) -> usize {
self.recv_buf_size.unwrap_or(DEFAULT_UDP_RECV_BUF)
}
/// Get the send buffer size, using default if not configured.
pub fn send_buf_size(&self) -> usize {
self.send_buf_size.unwrap_or(DEFAULT_UDP_SEND_BUF)
}
}
/// Transport instances - either a single config or named instances.

View File

@@ -20,6 +20,7 @@ async fn test_two_node_handshake_udp() {
let udp_config = UdpConfig {
bind_addr: Some("127.0.0.1:0".to_string()),
mtu: Some(1280),
..Default::default()
};
let (packet_tx_a, mut packet_rx_a) = packet_channel(64);
@@ -255,6 +256,7 @@ async fn test_run_rx_loop_handshake() {
let udp_config = UdpConfig {
bind_addr: Some("127.0.0.1:0".to_string()),
mtu: Some(1280),
..Default::default()
};
let (packet_tx_a, packet_rx_a) = packet_channel(64);
@@ -445,6 +447,7 @@ async fn test_cross_connection_both_initiate() {
let udp_config = UdpConfig {
bind_addr: Some("127.0.0.1:0".to_string()),
mtu: Some(1280),
..Default::default()
};
let (packet_tx_a, mut packet_rx_a) = packet_channel(64);

View File

@@ -25,6 +25,7 @@ pub(super) async fn make_test_node() -> TestNode {
let udp_config = UdpConfig {
bind_addr: Some("127.0.0.1:0".to_string()),
mtu: Some(1280),
..Default::default()
};
let (packet_tx, packet_rx) = packet_channel(256);

View File

@@ -7,6 +7,7 @@ use super::{
TransportId, TransportState, TransportType,
};
use crate::config::UdpConfig;
use socket2::{Domain, Protocol, Socket, Type};
use std::net::SocketAddr;
use std::sync::Arc;
use tokio::net::UdpSocket;
@@ -89,11 +90,33 @@ impl UdpTransport {
.parse()
.map_err(|e| TransportError::StartFailed(format!("invalid bind address: {}", e)))?;
// Bind socket
let socket = UdpSocket::bind(bind_addr)
.await
// Create socket via socket2 for buffer size control
let domain = if bind_addr.is_ipv4() { Domain::IPV4 } else { Domain::IPV6 };
let sock2 = Socket::new(domain, Type::DGRAM, Some(Protocol::UDP))
.map_err(|e| TransportError::StartFailed(format!("socket create failed: {}", e)))?;
sock2.set_nonblocking(true)
.map_err(|e| TransportError::StartFailed(format!("set nonblocking failed: {}", e)))?;
sock2.bind(&bind_addr.into())
.map_err(|e| TransportError::StartFailed(format!("bind failed: {}", e)))?;
// Set socket buffer sizes
let recv_buf = self.config.recv_buf_size();
let send_buf = self.config.send_buf_size();
sock2.set_recv_buffer_size(recv_buf)
.map_err(|e| TransportError::StartFailed(format!("set recv buffer: {}", e)))?;
sock2.set_send_buffer_size(send_buf)
.map_err(|e| TransportError::StartFailed(format!("set send buffer: {}", e)))?;
let actual_recv = sock2.recv_buffer_size()
.map_err(|e| TransportError::StartFailed(format!("get recv buffer: {}", e)))?;
let actual_send = sock2.send_buffer_size()
.map_err(|e| TransportError::StartFailed(format!("get send buffer: {}", e)))?;
// Convert to tokio UdpSocket
let std_socket: std::net::UdpSocket = sock2.into();
let socket = UdpSocket::from_std(std_socket)
.map_err(|e| TransportError::StartFailed(format!("tokio socket failed: {}", e)))?;
self.local_addr = Some(
socket
.local_addr()
@@ -119,11 +142,15 @@ impl UdpTransport {
info!(
name = %name,
local_addr = %self.local_addr.unwrap(),
recv_buf = actual_recv,
send_buf = actual_send,
"UDP transport started"
);
} else {
info!(
local_addr = %self.local_addr.unwrap(),
recv_buf = actual_recv,
send_buf = actual_send,
"UDP transport started"
);
}
@@ -309,6 +336,8 @@ mod tests {
UdpConfig {
bind_addr: Some(format!("127.0.0.1:{}", port)),
mtu: Some(1280),
recv_buf_size: None,
send_buf_size: None,
}
}