changed to new message.proto, which eliminates the pkt_id

Makefile

@@ -8,7 +8,7 @@ aarch64: libtun-so-aarch64
 	RUSTFLAGS="-L ." cargo build --release --target aarch64-unknown-linux-gnu
 
 
-linux-tun:
+linux-tun: libtun-so
 	RUSTFLAGS="-L ." cargo build  --features "tun" --release
 
 win:

proto/empty.proto

@@ -0,0 +1,6 @@
+syntax = "proto3";
+
+package message;
+
+message SDLEmpty {
+}

proto/message.proto

@@ -28,92 +28,81 @@ message SDLWelcome {
     uint32 heartbeat_sec = 4;
 }
 
-// quic 通讯消息
-message SDLEmpty {
-    uint32 pkt_id = 1;
-}
-
 // 这里修改成了扁平的结构, 否则有些字段不好找放的位置
 message SDLRegisterSuper {
-    uint32 pkt_id = 1;
-    string client_id = 2;
+    string client_id = 1;
     // 网络地址信息已经有https请求分配了
     // 注册的时候需要带上(network_id, mac, ip, mask_len, hostname)
-    uint32 network_id = 3;
-    bytes mac = 4;
-    uint32 ip = 5;
-    uint32 mask_len = 6;
-    string hostname = 7;
+    uint32 network_id = 2;
+    bytes mac = 3;
+    uint32 ip = 4;
+    uint32 mask_len = 5;
+    string hostname = 6;
 
-    string pub_key = 8;
+    string pub_key = 7;
     // 客户端使用http协议请求后端，通过token或者账号密码登录时, 统一返回一个access_token;
     // RegisterSuper的时候，验证凭证是否合法 (access_token)
-    string access_token = 9;
+    string access_token = 8;
 }
 
 // 客户端的升级逻辑，在https的接口里面去完成
 // 部分逻辑会脱离quic去通讯，增加session_token校验
 message SDLRegisterSuperAck {
-    uint32 pkt_id = 1;
-    // 目前支持aes， chacha20
-    string algorithm = 2;
-    bytes key = 3;
+    // 目前支持aes, chacha20
+    string algorithm = 1;
+    bytes key = 2;
     // 逻辑分段，chacha20加密算法需要使用该字段
-    uint32 region_id = 4;
-    bytes session_token = 5;
+    uint32 region_id = 3;
+    bytes session_token = 4;
 }
 
 message SDLRegisterSuperNak {
-    uint32 pkt_id = 1;
-    uint32 error_code = 2;
-    string error_message = 3;
+    uint32 error_code = 1;
+    string error_message = 2;
 }
 
 // 网络地址查询
-
 message SDLQueryInfo {
-    uint32 pkt_id = 1;
-    bytes dst_mac = 2;
+    bytes dst_mac = 1;
 }
 
 message SDLPeerInfo {
-    uint32 pkt_id = 1;
-    bytes dst_mac = 2;
-    SDLV4Info v4_info = 3;
-    optional SDLV6Info v6_info = 4;
+    bytes dst_mac = 1;
+    optional SDLV4Info v4_info = 2;
+    optional SDLV6Info v6_info = 3;
 }
 
 // ARP查询相关
 // 真实的arp请求是通过广播的形式获取到的，但是针对于macos这种tun的实现；是能够分析出arp请求包的；
 // 对于当前网络来说，服务端是知道mac对应的ip地址的，因此没有必要广播；直接通过服务器端返回
 message SDLArpRequest {
-    uint32 pkt_id = 1;
-    uint32 target_ip = 2;
+    uint32 target_ip = 1;
+    uint32 origin_ip = 2;
+    bytes context = 3;
 }
 
 message SDLArpResponse {
-    uint32 pkt_id = 1;
-    uint32 target_ip = 2;
-    bytes target_mac = 3;
+    uint32 target_ip = 1;
+    bytes target_mac = 2;
+    uint32 origin_ip = 3;
+    bytes context = 4;
 }
 
 // 权限请求查询相关
 message SDLPolicyRequest {
-    uint32 pkt_id = 1;
-    uint32 src_identity_id = 2;
-    uint32 dst_identity_id = 3;
-    uint32 version = 4;
+    uint32 src_identity_id = 1;
+    uint32 dst_identity_id = 2;
+    uint32 version = 3;
 }
 
 // 基于quic通讯，rules部分已经没有了长度限制
 message SDLPolicyResponse {
-    uint32 pkt_id = 1;
-    uint32 src_identity_id = 2;
-    uint32 dst_identity_id = 3;
+    uint32 src_identity_id = 1;
+    uint32 dst_identity_id = 2;
     // 版本号，客户端需要比较版本号确定是否覆盖; 请求端自己去管理版本号，服务端只是原样回写
-    uint32 version = 4;
-    // 4+1+2 的稀疏序列化规则
-    bytes rules = 5;
+    uint32 version = 3;
+    // 1 + 2稀疏序列化规则, 按照: <<Proto:8, Port:16>> 这个格式序列号所有的规则信息; 下发的数据默认都是allow，deny规则的服务器端已经屏蔽
+    bytes rules = 4;
 }
 
 // 事件定义
@@ -139,18 +128,16 @@ message SDLNetworkShutdownEvent {
 
 // client和stun之间的心跳包，客户端需要和super的udp之间的存活逻辑
 message SDLStunRequest {
-    uint32 cookie = 1;
-    string client_id = 2;
-    uint32 network_id = 3;
-    bytes mac = 4;
-    uint32 ip = 5;
-    uint32 nat_type = 6;
-    optional SDLV6Info v6_info = 7;
-    bytes session_token = 8;
+    string client_id = 1;
+    uint32 network_id = 2;
+    bytes mac = 3;
+    uint32 ip = 4;
+    uint32 nat_type = 5;
+    optional SDLV6Info v6_info = 6;
+    bytes session_token = 7;
 }
 
 message SDLStunReply {
-    uint32 cookie = 1;
 }
 
 message SDLData {

src/bin/build_pb/main.rs

@@ -4,6 +4,6 @@ fn main() {
         .bytes(&[".message.SDLData.data"])
         // .out_dir("../tcp_mock/pb")
         .protoc_arg("--experimental_allow_proto3_optional")
-        .compile_protos(&["message.proto"], &["."])
+        .compile_protos(&["message.proto", "empty.proto"], &["./proto"])
         .unwrap();
 }

src/bin/punchnet/main.rs

@@ -106,9 +106,12 @@ fn parse_login_result(res: Result<LoginResponse>) -> LoginData {
 }
 
 async fn daemonize_me(
+    allow_routing: bool,
     connect_info: ConnectData, 
     remembered: CachedLoginInfo,
     client_id: String,
+    route_file: String,
+    route_str: String,
     mac: Mac,
 ) {
     let _guard = log::init_log(&format!("{}/.output", get_base_dir()));
@@ -158,7 +161,7 @@ async fn daemonize_me(
             nat_server1: server.clone() +":1365",
             // nat_server2: "47.98.178.3:1265".to_owned(),
             nat_server2: server.clone() +":1366",
-            _allow_routing: true,
+            allow_routing: allow_routing,
             _drop_multicast: true,
             register_ttl: 1,
             mtu: 1400,
@@ -168,6 +171,8 @@ async fn daemonize_me(
             // token: cmd.token.clone(),
             // network_code: cmd.network_code.clone(),
             allow_p2p: true,
+            route_file,
+            route_str,
         },
         tx,
         &punchnet::get_install_channel(),
@@ -411,7 +416,7 @@ fn run_it(cmd: CommandLineInput2, client_id: String, mac: Mac, system: &str, ver
                 let connect_info = parse_connect_result(
                     connect(TEST_PREFIX, &client_id, &remembered.access_token).await
                 );
-                daemonize_me(connect_info, remembered, client_id, mac).await;
+                daemonize_me(rtinfo.allow_routing, connect_info, remembered, client_id, rtinfo.route_file.clone(), rtinfo.route.clone(), mac).await;
             })
         }
         Commands::AutoRun(tk) => {
@@ -436,10 +441,10 @@ fn run_it(cmd: CommandLineInput2, client_id: String, mac: Mac, system: &str, ver
                 let connect_info = parse_connect_result(
                     connect(TEST_PREFIX, &client_id, &remembered.access_token).await
                 );
-                daemonize_me(connect_info, remembered, client_id, mac).await;
+                daemonize_me(tk.allow_routing, connect_info, remembered, client_id, tk.route_file.clone(), tk.route.clone(), mac).await;
             })
-
         }
+
         other => {
             eprintln!("should not comes here");
             process::exit(-1);

src/lib.rs

@@ -12,7 +12,7 @@ use std::net::{SocketAddr, ToSocketAddrs};
 
 pub use network::get_edge;
 pub use network::get_install_channel;
-pub use network::{async_main, init_arp, init_edge, NodeConfig, restore_dns};
+pub use network::{async_main, init_edge, NodeConfig, restore_dns};
 use sdlan_sn_rs::utils::{Mac, save_to_file};
 use serde::{Deserialize, Serialize};
 use tokio::net::UdpSocket;
@@ -64,7 +64,6 @@ pub async fn run_sdlan(
     let hostfile = format!("{}/.host", get_base_dir());
     let host = create_or_load_uuid(&hostfile, Some(8))?;
 
-    init_arp();
 
     let hostname = hostname.unwrap_or(host);
     let _ = save_to_file(&hostfile, &hostname);
@@ -74,11 +73,12 @@ pub async fn run_sdlan(
     if let Err(e) = init_edge(
         // &args.token,
         // &args.network_code,
+        &args,
         mac,
         node_conf,
-        args.tos,
+        // args.tos,
         start_stop_sender,
-        args.mtu,
+        // args.mtu,
         connecting_chan.clone(),
         sock,
         hostname,
@@ -154,7 +154,7 @@ async fn parse_config(uuid: String, args: &CommandLine) -> Result<NodeConfig> {
 
     let node_conf = NodeConfig {
         name: args.name.to_owned(),
-        allow_routing: true,
+        allow_routing: args.allow_routing,
         _drop_multicast: true,
         allow_p2p: args.allow_p2p,
         mtu: args.mtu,

src/network/arp.rs

@@ -2,21 +2,26 @@
 
 use std::{
     collections::HashMap,
-    sync::atomic::{AtomicU8, Ordering},
-    time::Duration,
+    sync::{Arc, atomic::{AtomicU8, Ordering}},
+    time::{Duration, Instant},
 };
 
-use tracing::error;
+use bytes::{Bytes, BytesMut};
+use dashmap::DashMap;
+use tracing::{debug, error};
 
 use once_cell::sync::OnceCell;
-use sdlan_sn_rs::utils::{BROADCAST_MAC, MULTICAST_MAC, Mac, ip_to_string, net_bit_len_to_mask};
+use sdlan_sn_rs::{config::SDLAN_DEFAULT_TTL, utils::{BROADCAST_MAC, MULTICAST_MAC, Mac, get_current_timestamp, ip_to_string, mac_to_string, net_bit_len_to_mask}};
 use tokio::sync::{
     mpsc::{channel, Receiver, Sender},
     oneshot,
 };
 
-use super::{get_edge, init_arp_wait_list};
+use crate::{network::{form_ethernet_packet, send_packet_to_net}, pb::{SdlData, encode_to_udp_message}, tcp::PacketType};
 
+use super::{get_edge};
+
+/* 
 static GLOBAL_ARP: OnceCell<ArpActor> = OnceCell::new();
 pub fn init_arp() {
     init_arp_wait_list();
@@ -27,6 +32,7 @@ pub fn init_arp() {
 pub fn get_arp() -> &'static ArpActor {
     GLOBAL_ARP.get().unwrap()
 }
+*/
 
 const ETHER_TYPE_ARP: u16 = 0x0806;
 const ETHER_TYPE_IP: u16 = 0x0800;
@@ -141,9 +147,9 @@ const BROADCAST_IPADDR: u32 = 0xffffffff;
 #[derive(Debug)]
 #[allow(unused)]
 pub struct ArpEntry {
-    ip_addr: u32,
-    arptime: u8,
-    hw_addr: [u8; 6],
+    // ip_addr: u32,
+    last_seen: Instant,
+    hw_addr: Mac,
 }
 
 /*
@@ -158,6 +164,62 @@ impl ArpEntry {
 }
 */
 
+
+
+pub struct ArpTable {
+    entries: Arc<DashMap<u32, ArpEntry>>,
+    ttl: Duration,
+    pending_packet_buffer: ArpWaitList,
+}
+
+impl ArpTable {
+    pub fn new() -> Self {
+        let entries = Arc::new(DashMap::new());
+        let res = Self {
+            entries: entries.clone(),
+            ttl: Duration::from_secs(60),
+            pending_packet_buffer: ArpWaitList::new(),
+        };
+
+        let mut bytes = BytesMut::with_capacity(1024+20);
+
+        let ttl = res.ttl;
+        tokio::spawn(async move {
+            loop {
+                tokio::time::sleep(Duration::from_secs(30)).await;
+                let now = Instant::now();
+                entries.retain(|_, entry| now.duration_since(entry.last_seen) < ttl);
+            }
+        });
+
+        res
+    }
+
+    pub async fn arp_arrived(&self, ip: u32, mac: Mac) {
+        self.pending_packet_buffer.arp_arrived(ip, mac).await;
+    }
+
+    pub fn add_to_arp_wait_list(&self, ip: u32, origin_data: BytesMut) {
+        self.pending_packet_buffer.add_to_wait_list(ip, origin_data);
+    }
+
+    #[inline]
+    pub fn get(&self, ip: u32) -> Option<Mac> {
+        self.entries.get(&ip).map(|entry| entry.hw_addr)
+    }
+
+    pub fn set(&self, ip: u32, mac: Mac) {
+        self.entries.insert(
+            ip,
+            ArpEntry {
+                last_seen: Instant::now(),
+                hw_addr: mac,
+            },
+        );
+    }
+}
+
+/* 
 pub struct ArpInfo {
     // host_ip: AtomicU32,
     // ip representation of mask
@@ -302,7 +364,9 @@ impl ArpInfo {
         */
     }
 }
+*/
 
+/*
 pub enum ArpRequestInfo {
     Lookup { ip: u32 },
     Set { ip: u32, mac: Mac },
@@ -392,6 +456,7 @@ pub async fn send_arp_request(data: ArpRequestInfo) -> ArpResponse {
         Err(e) => ArpResponse::ArpRespError { msg: e.to_string() },
     }
 }
+    */
 
 pub fn generate_arp_request(srcmac: [u8; 6], dstip: u32, srcip: u32) -> Vec<u8> {
     let mut arphdr = ArpHdr::new();
@@ -418,3 +483,88 @@ pub fn generate_arp_request(srcmac: [u8; 6], dstip: u32, srcip: u32) -> Vec<u8>
 
     arphdr.marshal_to_bytes()
 }
+
+
+#[derive(Debug)]
+pub struct ArpWaitInfo {
+    timestamp: u64,
+    // origin data is from the tun or tap device
+    origin_data: BytesMut,
+}
+
+
+pub const MAX_WAIT_PACKETS: usize = 100;
+
+#[derive(Debug)]
+pub struct ArpWaitList {
+    content: DashMap<u32, Vec<ArpWaitInfo>>,
+}
+
+impl ArpWaitList {
+    fn new() -> Self {
+        Self {
+            content: DashMap::new(),
+        }
+    }
+
+
+    fn add_to_wait_list(&self, ip: u32, origin_data: BytesMut) {
+        let mut entry = self.content.entry(ip).or_insert(vec![]);
+        if entry.len() < MAX_WAIT_PACKETS {
+            entry.push(ArpWaitInfo {
+                timestamp: get_current_timestamp(),
+                origin_data,
+            })
+        }
+    }
+
+    async fn arp_arrived(&self, ip: u32, mac: Mac) {
+        debug!(
+            "arp for {} arrived: {}",
+            ip_to_string(&ip),
+            mac_to_string(&mac)
+        );
+        let Some(items) = self.content.remove(&ip) else {
+            return;
+        };
+        let edge = get_edge();
+        // just remove the items
+        if !edge.is_authorized() {
+            return;
+        }
+
+        // let encrypt_key = edge.get_encrypt_key();
+        let network_id = edge.network_id.load(Ordering::Relaxed);
+
+        let src_mac = edge.device_config.get_mac();
+        let now = get_current_timestamp();
+        for item in items.1 {
+            if (now - item.timestamp) > 5 {
+                continue;
+            }
+            let packet = form_ethernet_packet(src_mac, mac, item.origin_data);
+
+            let pkt_size = packet.len();
+
+            let Ok(encrypted) = edge.encryptor.load().encrypt(&packet) else {
+            // let Ok(encrypted) = edge.encryptor.read().unwrap().encrypt(&packet) else {
+            // let Ok(encrypted) = aes_encrypt(&encrypt_key, &packet) else {
+                error!("failed to encrypt packet request");
+                return;
+            };
+            let data_bytes = Bytes::from(encrypted);
+            let data = SdlData {
+                is_p2p: true,
+                network_id,
+                ttl: SDLAN_DEFAULT_TTL as u32,
+                src_mac: Vec::from(src_mac),
+                dst_mac: Vec::from(mac),
+                data: data_bytes,
+                identity_id: edge.identity_id.load(),
+                session_token: edge.session_token.get(),
+            };
+            let msg = encode_to_udp_message(Some(data), PacketType::Data as u8).unwrap();
+            send_packet_to_net(edge, mac, &msg, pkt_size as u64).await;
+        }
+    }
+}

src/network/async_main.rs

@@ -10,6 +10,7 @@ use crate::network::{
 use crate::tcp::{init_quic_conn, send_stun_request};
 use crate::utils::{send_to_sock, CommandLine};
 use crate::{ConnectionInfo};
+use bytes::BytesMut;
 use sdlan_sn_rs::peer::{SdlanSock};
 use sdlan_sn_rs::utils::{get_current_timestamp, is_multi_broadcast};
 use sdlan_sn_rs::utils::{Mac, Result};
@@ -253,15 +254,39 @@ async fn loop_tap(eee: &'static Node, cancel: CancellationToken) {
     error!("loop_tap exited");
 }
 
-async fn get_tun_flow(eee: &'static Node, tx: Sender<Vec<u8>>) {
-    loop {
-        let buf = tokio::task::spawn_blocking(|| {
-            let mut buf = vec![0; 1500];
+#[cfg(feature = "tun")]
+fn get_data_from_tun_with_layer2_zeroed(eee: &Node) -> BytesMut {
+    let mut temp = BytesMut::zeroed(1514);
+    // let mut temp = BytesMut::with_capacity(1514);
+    let mut data_buf = temp.split_off(14);
+
+    let Ok(size) = eee.device.recv(&mut data_buf) else {
+        return BytesMut::new();
+    };
+    data_buf.truncate(size);
+    temp.unsplit(data_buf);
+    temp
+}
+
+#[cfg(not(feature = "tun"))]
+fn get_data_from_tap_with_layer2(eee: &Node) -> BytesMut {
+    let mut buf = BytesMut::zeroed(1514);
     let Ok(size) = eee.device.recv(&mut buf) else {
-                return vec![];
+        return BytesMut::new();
     };
     buf.truncate(size);
     buf
+}
+
+async fn get_tun_flow(eee: &'static Node, tx: Sender<BytesMut>) {
+    loop {
+        let buf = tokio::task::spawn_blocking(|| {
+            #[cfg(feature = "tun")]
+            let data = get_data_from_tun_with_layer2_zeroed(eee);
+            #[cfg(not(feature = "tun"))]
+            let data = get_data_from_tap_with_layer2(eee);
+
+            data
         })
         .await
         .unwrap();
@@ -276,7 +301,7 @@ async fn get_tun_flow(eee: &'static Node, tx: Sender<Vec<u8>>) {
     }
 }
 
-async fn read_and_parse_tun_packet(eee: &'static Node, buf: Vec<u8>) {
+async fn read_and_parse_tun_packet(eee: &'static Node, buf: BytesMut) {
     /*
     if !eee.is_authorized() {
         debug!("drop packet before authorized");
@@ -294,7 +319,7 @@ async fn read_and_parse_tun_packet(eee: &'static Node, buf: Vec<u8>) {
     edge_send_packet_to_net(eee, buf).await;
 }
 
-async fn edge_send_packet_to_net(eee: &Node, data: Vec<u8>) {
+async fn edge_send_packet_to_net(eee: &Node, data: BytesMut) {
     // debug!("edge send packet to net({} bytes): {:?}", data.len(), data);
 
     /* 

src/network/device.rs

@@ -1,13 +1,19 @@
+use std::{net::Ipv4Addr, sync::{Arc, atomic::{AtomicU32, Ordering}}};
+
 use crate::utils::mac_to_string;
-use sdlan_sn_rs::{peer::IpSubnet, utils::Mac};
+use arc_swap::ArcSwap;
+use ipnet::Ipv4Net;
+use sdlan_sn_rs::{peer::IpSubnet, utils::{Mac, Result, SDLanError}};
 use tracing::debug;
 
 use crate::utils::generate_mac_address;
 
 pub struct DeviceConfig {
     pub mtu: u32,
-    pub mac: Mac,
-    pub ip: IpSubnet,
+    mac: Mac,
+    ip: IpSubnet,
+
+    self_ip_net: ArcSwap<Ipv4Net>,
 
     pub dns_mac: Mac,
 }
@@ -23,6 +29,7 @@ impl DeviceConfig {
             mac: mac,
             ip: IpSubnet::new(0, 0),
             dns_mac,
+            self_ip_net: ArcSwap::new(Arc::new(Ipv4Net::new_assert(Ipv4Addr::from(0), 32))),
         }
     }
 
@@ -37,6 +44,20 @@ impl DeviceConfig {
     }
     */
 
+    pub fn contains(&self, ip: &Ipv4Addr) -> bool {
+        self.self_ip_net.load().contains(ip)
+    }
+
+    pub fn set_ip(&self, ip_net_addr: u32, ip_net_bit_len: u8) -> Result<()>{
+        if ip_net_bit_len > 32 {
+            return Err(SDLanError::NormalError("invalid ip net bit len"));
+        }   
+        self.ip.net_addr.store(ip_net_addr, Ordering::Relaxed);
+        self.ip.net_bit_len.store(ip_net_bit_len, Ordering::Relaxed);
+        self.self_ip_net.store(Arc::new(Ipv4Net::new(Ipv4Addr::from(ip_net_addr), ip_net_bit_len).unwrap()));
+        Ok(())
+    }
+
     pub fn get_ip(&self) -> u32 {
         self.ip.net_addr()
     }

src/network/mod.rs

@@ -23,6 +23,6 @@ pub const DNS_IP: u32 = (100<<24) + (100<<16) + (100<<8) + 100;
 #[cfg_attr(target_os = "linux", path = "tun_linux.rs")]
 #[cfg_attr(target_os = "windows", path = "tun_win.rs")]
 mod tun;
-pub use tun::{get_install_channel, restore_dns};
+pub use tun::{get_install_channel, restore_dns, arp_reply_arrived};
 
 mod device;

src/network/node.rs

@@ -15,11 +15,11 @@ use tokio::io::AsyncReadExt;
 use tokio::sync::mpsc::Sender;
 use tracing::{debug, error, warn};
 
-use crate::network::{ARP_REPLY, ArpHdr, EthHdr, RouteTable2};
+use crate::network::{ArpTable, RouteTable2};
 use crate::quic::quic_init;
-use crate::{ConnectionInfo, MyEncryptor, RuleCache, get_base_dir};
+use crate::{CommandLine, ConnectionInfo, MyEncryptor, RuleCache, get_base_dir};
 use crate::pb::{
-    SdlArpRequest, SdlArpResponse, SdlEmpty, SdlStunProbe, SdlStunProbeReply, encode_to_tcp_message, encode_to_udp_message
+    SdlArpRequest, SdlEmpty, SdlStunProbe, SdlStunProbeReply, encode_to_tcp_message, encode_to_udp_message
 };
 use crate::tcp::{NatType, PacketType, StunProbeAttr, get_quic_write_conn};
 use crate::utils::{Socket};
@@ -32,7 +32,7 @@ use super::device::{DeviceConfig, Mode};
 use super::tun::{new_iface, Iface};
 use tokio::fs::File;
 
-use sdlan_sn_rs::utils::{Mac, gen_rsa_keys, ip_to_string, load_private_key_file, save_to_file};
+use sdlan_sn_rs::utils::{Mac, gen_rsa_keys, load_private_key_file, save_to_file};
 use sdlan_sn_rs::utils::{Result, SDLanError};
 
 static EDGE: OnceCell<Node> = OnceCell::new();
@@ -40,11 +40,12 @@ static EDGE: OnceCell<Node> = OnceCell::new();
 pub async fn init_edge(
     // token: &str,
     // network_code: &str,
+    args: &CommandLine,
     mac: Mac,
     node_conf: NodeConfig,
-    tos: u32,
+    // tos: u32,
     start_stop: Sender<StartStopInfo>,
-    mtu: u32,
+    // mtu: u32,
     connecting_chan: Option<Sender<ConnectionInfo>>,
     udpsock_for_dns: Arc<UdpSocket>,
     hostname: String,
@@ -67,7 +68,7 @@ pub async fn init_edge(
     // let edge_uuid = create_or_load_uuid("")?;
     //let node_conf = parse_config(edge_uuid, &args).await?;
 
-    let Ok(sock_v4) = Socket::build(node_conf._local_port, true, false, tos).await else {
+    let Ok(sock_v4) = Socket::build(node_conf._local_port, true, false, args.tos).await else {
         panic!("failed to build sock for sock v4");
     };
 
@@ -92,13 +93,15 @@ pub async fn init_edge(
         privatekey,
         tcp_pong.clone(),
         start_stop,
-        mtu,
+        args.mtu,
         connecting_chan,
         hostname,
         udpsock_for_dns,
         server_ip,
         install_channel,
     );
+
+    edge.route_table.parse_and_add_route(&args.route_file, &args.route_str);
     do_init_edge(edge)?;
 
     Ok(())
@@ -173,7 +176,10 @@ pub struct Node {
 
     pub identity_id: IdentityID,
 
+    // rule is identity-to-identity ok
     pub rule_cache: RuleCache,
+
+    // route_table stores the routes
     pub route_table: RouteTable2,
 
     pub access_token: StringToken<String>,
@@ -212,6 +218,10 @@ pub struct Node {
 
     // store pending, and known peers
     pub pending_peers: PeerMap,
+
+    #[cfg(feature = "tun")]
+    pub arp_table: ArpTable,
+
     pub known_peers: PeerMap,
 
     // pub tcp_sock_v4: TCPSocket,
@@ -270,8 +280,12 @@ impl Node {
         }
 
         self.access_token.set(access_token);
-        self.device_config.ip.net_addr.store(ip_net, Ordering::Relaxed);
-        self.device_config.ip.net_bit_len.store(ip_net_bit_len, Ordering::Relaxed);
+
+        if let Err(_e) = self.device_config.set_ip(ip_net, ip_net_bit_len) {
+            error!("failed to set ip");
+        }
+        // self.device_config.ip.net_addr.store(ip_net, Ordering::Relaxed);
+        // self.device_config.ip.net_bit_len.store(ip_net_bit_len, Ordering::Relaxed);
         // *self.device_config.mac.write().unwrap() = create_or_load_mac();
         self.network_id.store(network_id, Ordering::Relaxed);
         self.network_domain.write().unwrap().clone_from(network_domain);
@@ -313,8 +327,9 @@ impl Node {
         }
 
         self.access_token.set(access_token);
-        self.device_config.ip.net_addr.store(ip_net, Ordering::Relaxed);
-        self.device_config.ip.net_bit_len.store(ip_net_bit_len, Ordering::Relaxed);
+        if let Err(_e) = self.device_config.set_ip(ip_net, ip_net_bit_len) {
+            error!("failed to set ip");
+        }
         // *self.device_config.mac.write().unwrap() = create_or_load_mac();
         self.network_id.store(network_id, Ordering::Relaxed);
         self.network_domain.write().unwrap().clone_from(network_domain);
@@ -325,7 +340,7 @@ impl Node {
         let id = self.get_next_packet_id();
 
         let res = self.cookie_match.do_action_and_wait_for(
-            id, 
+            0, 
             || async {
                 let _ = self
                     .start_stop_sender
@@ -384,6 +399,10 @@ impl Node {
         };
 
         Self {
+
+            #[cfg(feature = "tun")]
+            arp_table: ArpTable::new(),
+
             packet_id: AtomicU32::new(1),
             encryptor: ArcSwap::from(Arc::new(MyEncryptor::new())),
             // encryptor: RwLock::new(MyEncryptor::new()),
@@ -530,69 +549,17 @@ impl Node {
     }
     */
 
-    pub async fn send_and_wait_for_arp_reply(&self, gw_ip: u32, real_ip: u32) -> Result<()>{
-        let id = self.get_next_packet_id();
-        let res = self.cookie_match.do_action_and_wait_for(
-            id,
-            || async {
+    pub async fn send_arp_request(&self, gw_ip: u32, real_ip: u32) -> Result<()>{
         let arp_request = SdlArpRequest {
-                    pkt_id: id,
+            // pkt_id: id,
             target_ip: gw_ip,
+            origin_ip: real_ip,
+            context: Vec::new(),
         };
-                warn!("arp request: {:?}", arp_request);
         let msg = encode_to_tcp_message(Some(arp_request), PacketType::ArpRequest as u8).unwrap();
         let conn = get_quic_write_conn();
         let _ = conn.send(msg).await;
-            },
-            Duration::from_secs(3)).await?;
-
-        match res.downcast() {
-            Ok(res) => {
-                let res: SdlArpResponse = *res;
-
-                debug!("got arp response: {:?}", res);
-                if res.target_mac.len() != 6 {
-                    // invalid target_mac
-                    error!("invalid target_mac: {:?}, ip={}", res.target_mac, ip_to_string(&res.target_ip));
-                    return Ok(());
-                }
-
-                // TODO: construct the arp reply, and write to tun;
-                let src_mac = res.target_mac.try_into().unwrap();
-                let dst_mac = self.device_config.get_mac();
-                let dst_ip = self.device_config.get_ip();
-
-                let hdr = ArpHdr{
-                    ethhdr: EthHdr {
-                        dest: dst_mac,
-                        src: src_mac,
-                        eth_type: 0x0806,
-                    },
-                    hwtype: 0x0001,
-                    protocol: 0x0800,
-                    hwlen: 6,
-                    protolen: 4,
-                    opcode: ARP_REPLY,
-                    shwaddr: src_mac,
-                    sipaddr: [((real_ip >> 16) as u16) & 0xffff, (real_ip as u16) & 0xffff],
-                    dhwaddr: dst_mac,
-                    dipaddr: [((dst_ip >> 16) & 0x0000ffff) as u16, (dst_ip & 0x0000ffff) as u16]
-                };
-
-                let data = hdr.marshal_to_bytes();
-                if let Err(_e) = self.device.send(&data) {
-                    error!("failed to write arp response to device");
-                }
-
         Ok(())
-
-            }
-            Err(e ) => {
-
-                error!("failed to convert to ArpResponse: {:?}", e);
-                Err(SDLanError::ConvertError("failed to convert to ArpResponse".to_owned()))
-            }
-        }
     }
 
     pub async fn send_nat_probe_reply(&self, cookie: u32, buf: SdlStunProbeReply) {
@@ -885,7 +852,7 @@ impl Queryer {
 
     pub fn write_feedback(&self, id: u32, data: BoxedProstMessage) {
         if let Some((_, tx)) = self.mailbox.remove(&id) {
-            if let Err(e) = tx.send(data) {
+            if let Err(_e) = tx.send(data) {
                 error!("failed to write feedback");
             }
         }
@@ -945,7 +912,6 @@ impl Queryer {
                 Err(SDLanError::IOError("timed out".to_string()))
             }
         }
-
     }
 
     pub async fn send_message_to_quic_and_wait_for<T: Message>(&self, id: u32, message: T, packet_type: u8, timeout: Duration) -> Result<BoxedProstMessage> {

src/network/packet.rs

@@ -14,6 +14,7 @@ use crate::{
     tcp::{PacketType},
     utils::{send_to_sock, Socket},
 };
+use bytes::BytesMut;
 use etherparse::{Ethernet2Header, IpNumber, PacketHeaders, ip_number};
 use prost::Message;
 use sdlan_sn_rs::utils::{BROADCAST_MAC};
@@ -823,7 +824,7 @@ pub fn print_hex(key: &[u8]) {
 
 async fn renew_identity_request(eee: &Node, identity: u32) {
     let policy_request = SdlPolicyRequest {
-        pkt_id: eee.get_next_packet_id(),
+        // pkt_id: eee.get_next_packet_id(),
         src_identity_id: identity,
         dst_identity_id: eee.identity_id.load(),
         version: 1,
@@ -1228,7 +1229,7 @@ async fn send_query_peer(eee: &Node, dst_mac: Mac) -> Result<()> {
     }
     let query = SdlQueryInfo {
         dst_mac: Vec::from(dst_mac),
-        pkt_id: 0,
+        // pkt_id: 0,
     };
 
     let Ok(content) = encode_to_tcp_message(
@@ -1292,13 +1293,19 @@ pub async fn update_supernode_reg(eee: &Node) {
 */
 
 #[allow(unused)]
-pub fn form_ethernet_packet(src_mac: Mac, dst_mac: Mac, data: &[u8]) -> Vec<u8> {
+pub fn form_ethernet_packet(src_mac: Mac, dst_mac: Mac, mut data_with_zeroed_layer2: BytesMut) -> BytesMut {
     let mut etherheader = Ethernet2Header::default();
     etherheader.destination = dst_mac;
     etherheader.ether_type = etherparse::EtherType::IPV4;
     etherheader.source = src_mac;
-    let mut packet = Vec::with_capacity(14 + data.len() + 4);
-    packet.extend_from_slice(&etherheader.to_bytes()[..]);
-    packet.extend_from_slice(&data);
-    packet
+
+    let data = data_with_zeroed_layer2.split_off(14);
+    data_with_zeroed_layer2.copy_from_slice(&etherheader.to_bytes()[..]);
+
+    data_with_zeroed_layer2.unsplit(data);
+    data_with_zeroed_layer2
+    // let mut packet = Vec::with_capacity(14 + data.len() + 4);
+    // packet.extend_from_slice(&etherheader.to_bytes()[..]);
+    // packet.extend_from_slice(&data);
+    // packet
 }

src/network/route.rs

@@ -1,14 +1,14 @@
-use std::{collections::HashMap, net::Ipv4Addr, sync::atomic::{AtomicBool, Ordering}, time::Duration};
+use std::{collections::HashMap, fs, io::{BufRead, BufReader}, net::Ipv4Addr, sync::atomic::{AtomicBool, Ordering}, time::Duration};
 
 use ahash::RandomState;
 use dashmap::{DashMap};
 use ipnet::Ipv4Net;
 use sdlan_sn_rs::utils::{Result, SDLanError};
 
-use tokio::sync::oneshot::{Receiver, Sender, channel};
+use tokio::{io::ReadBuf, sync::oneshot::{Receiver, Sender, channel}};
 use tracing::{debug, error};
 
-use crate::{RouteTableTrie, network::tun::add_route, pb::{SdlArpResponse, SdlStunReply}};
+use crate::{RouteTableTrie, network::tun::{add_route, del_route}, pb::{SdlArpResponse, SdlStunReply}};
 
 
 pub struct RouteTable2 {
@@ -24,8 +24,18 @@ impl RouteTable2 {
         }
     }
 
-    pub fn parse_and_add_route(&self, route_str: &str) -> Result<()> {
-        let routes = parse_route(route_str);
+    pub fn parse_and_add_route(&self, route_file: &str, route_str: &str) -> Result<()> {
+        let routes = match true {
+            _ if route_str.len() != 0 => {
+                parse_route(route_str)
+            }
+            _ if route_file.len() != 0 => {
+                load_route_file(route_file)
+            }
+            _ => {
+                Ok(HashMap::new())
+            }
+        }?;
         for route in routes.keys() {
             if self.cache_table.get(route).is_some() {
                 error!("route {} {} has been added", route.0.to_string(), route.1);
@@ -42,42 +52,65 @@ impl RouteTable2 {
 
     pub fn apply_system(&self) {
         for route in &self.cache_table {
-            let origin = route.fetch_or(true, Ordering::Relaxed);
-            if !origin {
+            let (net, gw) = route.key();
+            if let Err(e) = del_route(net, gw) {
+                error!("failed to del route: {}", e.as_str());
+            }
                 // should add to system
-                add_route(route.key().0, route.key().1);
+            if let Err(e) = add_route(net, gw) {
+                error!("failed to add route: {}", e.as_str());
             }
         }
     }
 }
 
-// ip, mask, gateway, cidr gateway,cidr2 gateway2
-pub fn parse_route(route: &str) -> HashMap<(Ipv4Net, Ipv4Addr), bool> {
+/// 路由文件，应该是以下格式，一行一条
+///     network-cidr gw
+///     network-cidr gw
+pub fn load_route_file(filename: &str) -> Result<HashMap<(Ipv4Net, Ipv4Addr), bool>> {
     let mut result = HashMap::new();
-    // let routes: Vec<_> = route.split(",").collect();
+    let fp = fs::File::open(filename)?;
     
-    for route in route.trim().split(",") {
+    let buf = BufReader::new(fp);
+    for line in buf.lines() {
+        let line = line?;
+        if let Ok(rt) = parse_a_route(&line) {
+            result.insert(rt, true);
+        }
+    }
+    Ok(result)
+}
+
+pub fn parse_a_route(route: &str) -> Result<(Ipv4Net, Ipv4Addr)> {
     let route_info: Vec<_> = route.trim().split_whitespace().collect();
     if route_info.len() != 2 {
         error!("route info format error: {}", route);
-            continue;
+        return Err(SDLanError::IOError("route format error".to_string()));
     }
     debug!("got route info: {:?}", route_info);
 
     let Ok(gateway) = route_info[1].parse::<Ipv4Addr>() else {
         error!("failed to parse gw: {}", route_info[1]);
-            continue;
+        return Err(SDLanError::IOError("failed to parse gw".to_string()));
     };
 
     let cidr = route_info[0];
     let Ok(net )= cidr.parse::<Ipv4Net>()  else {
         error!("failed to parse cidr: {}, skipping", cidr);
-            continue;
+        return Err(SDLanError::IOError("failed to parse cidr".to_string()));
     };
-        let origin = result.insert((net, gateway), true);
-        if origin.is_some() {
-            error!("{} {} already added", net.to_string(), gateway.to_string());
-        }
-    }
-    result
+    Ok((net, gateway))
+}
+
+// ip, mask, gateway, cidr gateway,cidr2 gateway2
+pub fn parse_route(route: &str) -> Result<HashMap<(Ipv4Net, Ipv4Addr), bool>> {
+    let mut result = HashMap::new();
+    // let routes: Vec<_> = route.split(",").collect();
+
+    for route in route.trim().split(",") {
+        if let Ok(rt) = parse_a_route(route) {
+            result.insert(rt, true);
+        }
+    }
+    Ok(result)
 }

src/network/tun_linux.rs

@@ -1,3 +1,7 @@
+#[cfg(not(feature = "tun"))]
+use bytes::BytesMut;
+#[cfg(feature = "tun")]
+use bytes::{Bytes, BytesMut};
 use etherparse::{Ethernet2Header};
 use ipnet::Ipv4Net;
 use sdlan_sn_rs::config::SDLAN_DEFAULT_TTL;
@@ -21,8 +25,16 @@ use std::process::Command;
 
 use tracing::{debug, error, info};
 
+#[cfg(feature = "tun")]
+use crate::caculate_crc;
 use crate::get_edge;
-use crate::network::send_packet_to_net;
+#[cfg(not(feature = "tun"))]
+use crate::network::{ARP_REPLY, ArpHdr, EthHdr};
+use crate::network::{Node, send_packet_to_net};
+#[cfg(not(feature = "tun"))]
+use crate::pb::SdlArpResponse;
+#[cfg(feature = "tun")]
+use crate::pb::SdlArpResponse;
 use crate::pb::{encode_to_udp_message, SdlData};
 use crate::tcp::PacketType;
 
@@ -113,7 +125,7 @@ impl Iface {
         }
     }
 
-    pub fn reload_config(&self, device_config: &DeviceConfig, network_domain: &str) {
+    pub fn reload_config(&self, node: &Node, device_config: &DeviceConfig, network_domain: &str) {
         let netbit = device_config.get_net_bit();
         let ip = device_config.get_ip();
         if netbit == 0 || ip == 0 {
@@ -156,6 +168,12 @@ impl Iface {
                 }
             }
 
+            node.route_table.apply_system();
+
+            if node.config.allow_routing {
+                set_allow_routing();
+            }
+
             // TODO: set dns should be opened
             /* 
             if let Err(e) = set_dns(self, &self.name, network_domain, &ip_to_string(&default_gw)) {
@@ -212,7 +230,7 @@ impl TunTapPacketHandler for Iface {
 
     async fn handle_packet_from_device(
         &self,
-        data: Vec<u8>,
+        data: BytesMut,
         // encrypt_key: &[u8],
     ) -> std::io::Result<()> {
         use etherparse::PacketHeaders;
@@ -237,10 +255,16 @@ impl TunTapPacketHandler for Iface {
                     let arp = ArpHdr::from_slice(&data);
                     match arp.opcode {
                         ARP_REQUEST => {
-                            use crate::{network::ArpRequest, pb::{SdlArpRequest, encode_to_tcp_message}, tcp::get_quic_write_conn};
                             let dest_ip = ((arp.dipaddr[0] as u32) << 16) + arp.dipaddr[1] as u32;
 
-                            edge.send_and_wait_for_arp_reply(dest_ip, dest_ip).await;
+                            if edge.device_config.contains(&Ipv4Addr::from_bits(dest_ip)) {
+                                let _ = edge.send_arp_request(dest_ip,  dest_ip).await;
+                            } else {
+                                if let Some((_, real_ip)) = edge.route_table.route_table.lookup(dest_ip) {
+                                    let real_ip = u32::from_be_bytes(real_ip.octets());
+                                    let _ = edge.send_arp_request(dest_ip, real_ip).await;
+                                }
+                            }
                             /* 
                             let request = SdlArpRequest {
                                 pkt_id: edge.get_next_packet_id(),
@@ -416,6 +440,11 @@ impl TunTapPacketHandler for Iface {
                                 use bytes::Bytes;
                                 use sdlan_sn_rs::utils::mac_to_string;
 
+                                use crate::network::ARP_REPLY;
+
+                                edge.arp_table.set(from_ip, arp.shwaddr);
+
+                                /* 
                                 use crate::network::{ARP_REPLY, ArpRequestInfo, send_arp_request};
 
                                 send_arp_request(ArpRequestInfo::Set {
@@ -423,6 +452,7 @@ impl TunTapPacketHandler for Iface {
                                     mac: arp.shwaddr,
                                 })
                                 .await;
+                                */
 
                                 // target to us
                                 arp.opcode = ARP_REPLY;
@@ -438,7 +468,7 @@ impl TunTapPacketHandler for Iface {
 
                                 let data = arp.marshal_to_bytes();
                                 // let Ok(encrypted) = aes_encrypt(key, &data) else {
-                                let Ok(encrypted) = edge.encryptor.read().unwrap().encrypt(&data) else {
+                                let Ok(encrypted) = edge.encryptor.load().encrypt(&data) else {
                                     error!("failed to encrypt arp reply");
                                     return Ok(());
                                 };
@@ -472,6 +502,7 @@ impl TunTapPacketHandler for Iface {
 
                             debug!("mac {:?} is at {:?}", arp.shwaddr, from_ip.to_be_bytes());
                             if dest_ip == self_ip {
+                                /* 
                                 use crate::network::{ArpRequestInfo, arp_arrived, send_arp_request};
 
                                 send_arp_request(ArpRequestInfo::Set {
@@ -479,7 +510,11 @@ impl TunTapPacketHandler for Iface {
                                     mac: arp.shwaddr,
                                 })
                                 .await;
-                                arp_arrived(from_ip, arp.shwaddr).await;
+                                */
+
+                                // use crate::network::arp_arrived;
+                                edge.arp_table.set(from_ip, arp.shwaddr);
+                                edge.arp_table.arp_arrived(from_ip, arp.shwaddr).await;
                             }
                         }
                         _other => {
@@ -498,9 +533,10 @@ impl TunTapPacketHandler for Iface {
                             let ip = u32::from_be_bytes(ipv4.0.source);
                             let mac = hdr.source;
                             if !is_multi_broadcast(&mac) {
-                                use crate::network::{ArpRequestInfo, send_arp_request};
+                                //use crate::network::{ArpRequestInfo, send_arp_request};
 
-                                send_arp_request(ArpRequestInfo::Set { ip, mac }).await;
+                                edge.arp_table.set(ip, mac);
+                                // send_arp_request(ArpRequestInfo::Set { ip, mac }).await;
                             }
                         }
                         Err(_) => {
@@ -533,7 +569,7 @@ impl TunTapPacketHandler for Iface {
 
     async fn handle_packet_from_device(
         &self,
-        data: Vec<u8>,
+        mut header: BytesMut,
     ) -> std::io::Result<()> {
         use etherparse::IpHeaders;
 
@@ -541,9 +577,11 @@ impl TunTapPacketHandler for Iface {
 
         let src_mac = eee.device_config.get_mac();
 
+        let data = header.split_off(14);
+
         match IpHeaders::from_slice(&data) {
             Ok((iphdr, _payload)) => {
-                use crate::network::{ArpRequestInfo, ArpResponse, send_arp_request};
+                //use crate::network::{ArpRequestInfo, ArpResponse, send_arp_request};
 
                 let Some(ipv4hdr) = iphdr.ipv4() else {
                     debug!("ipv6 packet ignored");
@@ -573,75 +611,36 @@ impl TunTapPacketHandler for Iface {
                     }
                     return Ok(());
                 }
-                match send_arp_request(ArpRequestInfo::Lookup { ip: dstip }).await {
-                    ArpResponse::LookupResp {
-                        mac,
-                        ip,
-                        do_arp_request,
-                    } => {
-                        use bytes::Bytes;
+                match eee.arp_table.get(dstip) {
+                    Some(mac) => {
 
-                        use crate::utils::caculate_crc;
-
-                        if do_arp_request {
-                            use sdlan_sn_rs::utils::BROADCAST_MAC;
-
-                            use crate::network::{add_to_arp_wait_list, generate_arp_request};
-
-                            add_to_arp_wait_list(dstip, data);
-                            debug!(
-                                "find ip: {:?} => {:?}",
-                                src.to_be_bytes(),
-                                dstip.to_be_bytes()
-                            );
-                            let arp_msg =
-                                generate_arp_request(src_mac, ip, eee.device_config.get_ip());
-                            // let Ok(encrypted) = aes_encrypt(&encrypt_key, &arp_msg) else {
-                            let Ok(encrypted) = eee.encryptor.read().unwrap().encrypt(&arp_msg) else {
-                                error!("failed to encrypt arp request");
-                                return Ok(());
-                            };
-                            // println!("arp_msg: {:?}", arp_msg);
-                            let data = SdlData {
-                                network_id: eee.network_id.load(Ordering::Relaxed),
-                                src_mac: Vec::from(src_mac),
-                                dst_mac: Vec::from([0xff; 6]),
-                                is_p2p: true,
-                                ttl: SDLAN_DEFAULT_TTL as u32,
-                                data: Bytes::from(encrypted),
-                                session_token: eee.session_token.get(),
-                                identity_id: eee.identity_id.load(),
-                            };
-                            let data =
-                                encode_to_udp_message(Some(data), PacketType::Data as u8).unwrap();
-                            debug!("sending arp");
-                            // let data = marshal_message(&data);
-                            send_packet_to_net(eee, BROADCAST_MAC, &data, arp_msg.len() as u64)
-                                .await;
-                            // edge.sock.send(data).await;
-                            // println!("should send arp");
-                            return Ok(());
-                        }
-
-                        // prepend the ether header
+                        let pkt_size = data.len() + 14;
                         let mut etherheader = Ethernet2Header::default();
                         etherheader.destination = mac;
                         etherheader.ether_type = etherparse::EtherType::IPV4;
                         etherheader.source = src_mac;
-                        let mut packet = Vec::with_capacity(14 + data.len() + 4);
-                        packet.extend_from_slice(&etherheader.to_bytes()[..]);
-                        packet.extend_from_slice(&data);
-                        let crc = caculate_crc(&packet);
-                        packet.extend_from_slice(&crc.to_be_bytes());
+                        // let mut packet = Vec::with_capacity(14 + data.len() + 4);
 
-                        let pkt_size = packet.len();
+                        header.copy_from_slice(&etherheader.to_bytes()[..]);
+
+                        let crc = caculate_crc(&data);
+                        header.unsplit(data);
+
+
+                        // packet.extend_from_slice(&etherheader.to_bytes()[..]);
+                        // packet.extend_from_slice(&data);
+                        header.extend_from_slice(&crc.to_be_bytes());
+                        // packet.extend_from_slice(&crc.to_be_bytes());
+
+                        // let pkt_size = packet.len();
                         // println!("sending data with mac");
 
                         // let Ok(encrypted) = aes_encrypt(&encrypt_key, &packet) else {
-                        let Ok(encrypted) = eee.encryptor.read().unwrap().encrypt(&packet) else {
+                        let Ok(encrypted) = eee.encryptor.load().encrypt(&header) else {
                             error!("failed to encrypt packet request");
                             return Ok(());
                         };
+
                         let data = SdlData {
                             is_p2p: true,
                             network_id: eee.network_id.load(Ordering::Relaxed),
@@ -656,11 +655,31 @@ impl TunTapPacketHandler for Iface {
                             encode_to_udp_message(Some(data), PacketType::Data as u8).unwrap();
                         let size = msg.len();
                         send_packet_to_net(eee, mac, &msg, pkt_size as u64).await;
-                        // let dstip = u32::from_be_bytes(ipv4hdr.0.destination);
                     }
-                    _ => {}
+                    None => {
+                        header.unsplit(data);
+                        eee.arp_table.add_to_arp_wait_list(dstip, header);
+                        debug!(
+                            "find ip: {:?} => {:?}",
+                            src.to_be_bytes(),
+                            dstip.to_be_bytes()
+                        );
+                        debug!("no mac found for ip {:?}, sending arp request", dstip.to_be_bytes());
+                        // let _ = eee.send_arp_request(dstip, dstip).await;
+
+                        if eee.device_config.contains(&Ipv4Addr::from_bits(dstip)) {
+                            let _ = eee.send_arp_request(dstip,  dstip).await;
+                        } else {
+                            if let Some((_, real_ip)) = eee.route_table.route_table.lookup(dstip) {
+                                let real_ip = u32::from_be_bytes(real_ip.octets());
+                                let _ = eee.send_arp_request(dstip, real_ip).await;
                             }
                         }
+
+                    }
+                }
+
+            }
             Err(e) => {
                 error!("failed to parse ip packet: {}", e.to_string());
             }
@@ -889,7 +908,20 @@ fn restore_resolv_conf() -> Result<()> {
     Ok(())
 }
 
-pub fn add_route(net: Ipv4Net, gw: Ipv4Addr) -> Result<()> {
+pub fn del_route(net: &Ipv4Net, gw: &Ipv4Addr) -> Result<()> {
+    let res = Command::new("route")
+        .arg("del")
+        .arg("-net")
+        .arg(net.to_string())
+        .arg("gw")
+        .arg(gw.to_string())
+        .output()?;
+
+    Ok(())
+}
+
+
+pub fn add_route(net: &Ipv4Net, gw: &Ipv4Addr) -> Result<()> {
     let res = Command::new("route")
         .arg("add")
         .arg("-net")
@@ -901,6 +933,82 @@ pub fn add_route(net: Ipv4Net, gw: Ipv4Addr) -> Result<()> {
     Ok(())
 }
 
-pub fn del_route() -> Result<()> {
-    Ok(())
+
+fn set_allow_routing() {
+    let _ = Command::new("sysctl")
+        .arg("-w")
+        .arg("net.ipv4.ip_forward=1")
+        .output();
+
+    let _ = Command::new("iptables")
+        .arg("-t")
+        .arg("nat")
+        .arg("-D")
+        .arg("POSTROUTING")
+        .arg("-j")
+        .arg("MASQUERADE")
+        .output();
+
+    let _ = Command::new("iptables")
+        .arg("-t")
+        .arg("nat")
+        .arg("-A")
+        .arg("POSTROUTING")
+        .arg("-j")
+        .arg("MASQUERADE")
+        .output();
+}
+
+#[cfg(feature = "tun")]
+pub async fn arp_reply_arrived(edge: &Node, data: SdlArpResponse) {
+    debug!("got arp response: {:?}", data);
+    if data.target_mac.len() != 6 {
+        // invalid target_mac
+        error!("invalid target_mac: {:?}, ip={}", data.target_mac, ip_to_string(&data.target_ip));
+        return;
+    }
+
+    let ip = data.target_ip;
+    let mac = data.target_mac.try_into().unwrap();
+    
+    edge.arp_table.arp_arrived(ip, mac).await;
+}
+
+#[cfg(not(feature = "tun"))]
+pub async fn arp_reply_arrived(edge: &Node, data: SdlArpResponse) {
+    debug!("got arp response: {:?}", data);
+    if data.target_mac.len() != 6 {
+        // invalid target_mac
+        error!("invalid target_mac: {:?}, ip={}", data.target_mac, ip_to_string(&data.target_ip));
+        return;
+    }
+
+    // TODO: construct the arp reply, and write to tun;
+    let src_mac = data.target_mac.try_into().unwrap();
+    let dst_mac = edge.device_config.get_mac();
+    let dst_ip = edge.device_config.get_ip();
+
+    let src_ip = data.origin_ip;
+
+    let hdr = ArpHdr{
+        ethhdr: EthHdr {
+            dest: dst_mac,
+            src: src_mac,
+            eth_type: 0x0806,
+        },
+        hwtype: 0x0001,
+        protocol: 0x0800,
+        hwlen: 6,
+        protolen: 4,
+        opcode: ARP_REPLY,
+        shwaddr: src_mac,
+        sipaddr: [((src_ip >> 16) as u16) & 0xffff, (src_ip as u16) & 0xffff],
+        dhwaddr: dst_mac,
+        dipaddr: [((dst_ip >> 16) & 0x0000ffff) as u16, (dst_ip & 0x0000ffff) as u16]
+    };
+
+    let data = hdr.marshal_to_bytes();
+    if let Err(_e) = edge.device.send(&data) {
+        error!("failed to write arp response to device");
+    }
 }

src/network/tun_win.rs

@@ -1,11 +1,13 @@
 use bytes::Bytes;
 use etherparse::ether_type::ARP;
 use etherparse::{Ethernet2Header, IpHeaders};
+use ipnet::Ipv4Net;
 use sdlan_sn_rs::config::SDLAN_DEFAULT_TTL;
 use sdlan_sn_rs::utils::{
-    aes_encrypt, ip_to_string, is_multi_broadcast, net_bit_len_to_mask, BROADCAST_MAC,
+    BROADCAST_MAC, Result, aes_encrypt, ip_to_string, is_multi_broadcast, net_bit_len_to_mask
 };
 use std::io::{Error, ErrorKind};
+use std::net::Ipv4Addr;
 use std::os::windows::process::CommandExt;
 use std::process::Command;
 use std::sync::atomic::Ordering;
@@ -15,7 +17,7 @@ use wintun;
 
 use crate::get_edge;
 use crate::network::{
-    ARP_REPLY, ARP_REQUEST, ArpHdr, ArpRequestInfo, ArpResponse, DNS_IP, add_to_arp_wait_list, arp_arrived, form_ethernet_packet, generate_arp_request, send_arp_request, send_packet_to_net
+    ARP_REPLY, ARP_REQUEST, ArpHdr, ArpRequestInfo, ArpResponse, DNS_IP, Node, add_to_arp_wait_list, arp_arrived, form_ethernet_packet, generate_arp_request, send_arp_request, send_packet_to_net
 };
 use crate::pb::{encode_to_udp_message, SdlData};
 use crate::tcp::PacketType;
@@ -58,7 +60,7 @@ impl Iface {
         Ok(content.len())
     }
 
-    pub fn reload_config(&self, device_config: &DeviceConfig, network_domain: &str) {
+    pub fn reload_config(&self, node: &Node, device_config: &DeviceConfig, network_domain: &str) {
         let netbit = device_config.get_net_bit();
         let ip = device_config.get_ip();
         if netbit == 0 || ip == 0 {
@@ -202,7 +204,7 @@ impl TunTapPacketHandler for Iface {
 
 
                                 // let Ok(encrypted) = aes_encrypt(key, &data) else {
-                                let Ok(encrypted) = edge.encryptor.read().unwrap().encrypt(&data) else {
+                                let Ok(encrypted) = edge.encryptor.load().encrypt(&data) else {
                                     error!("failed to encrypt arp reply");
                                     return Ok(());
                                 };
@@ -340,7 +342,7 @@ impl TunTapPacketHandler for Iface {
                             let arp_msg =
                                 generate_arp_request(src_mac, ip, eee.device_config.get_ip());
                             
-                            let Ok(encrypted) = eee.encryptor.read().unwrap().encrypt(&arp_msg) else {
+                            let Ok(encrypted) = eee.encryptor.load().encrypt(&arp_msg) else {
                             // let Ok(encrypted) = aes_encrypt(&encrypt_key, &arp_msg) else {
                                 error!("failed to encrypt arp request");
                                 return Ok(());
@@ -386,7 +388,7 @@ impl TunTapPacketHandler for Iface {
                         // println!("sending data with mac");
 
                         // let Ok(encrypted) = aes_encrypt(&encrypt_key, &packet) else {
-                        let Ok(encrypted) = eee.encryptor.read().unwrap().encrypt(&packet) else {
+                        let Ok(encrypted) = eee.encryptor.load().encrypt(&packet) else {
                             error!("failed to encrypt packet request");
                             return Ok(());
                         };
@@ -480,3 +482,28 @@ pub fn set_dns(name: &str, _network_domain: &str, gw: &str, ifidx: u32) -> std::
 pub fn restore_dns() {
 
 }
+
+pub fn del_route(net: &Ipv4Net, gw: &Ipv4Addr) -> Result<()> {
+    let mask = net.netmask().to_string();
+    let network = net.network().to_string();
+    let res = Command::new("route")
+        .arg("delete")
+        .arg(network)
+        .arg(mask)
+        .arg(gw.to_string())
+        .output()?;
+    Ok(())
+
+}
+
+pub fn add_route(net: &Ipv4Net, gw: &Ipv4Addr) -> Result<()> {
+    let mask = net.netmask().to_string();
+    let network = net.network().to_string();
+    let res = Command::new("route")
+        .arg("add")
+        .arg(network)
+        .arg(mask)
+        .arg(gw.to_string())
+        .output()?;
+    Ok(())
+}

src/network/tuntap.rs

@@ -1,6 +1,7 @@
+
 use std::sync::atomic::Ordering;
 
-use bytes::Bytes;
+use bytes::{Bytes, BytesMut};
 use dashmap::DashMap;
 use once_cell::sync::OnceCell;
 use sdlan_sn_rs::{
@@ -25,9 +26,10 @@ pub const MAX_WAIT_PACKETS: usize = 100;
 
 pub trait TunTapPacketHandler {
     async fn handle_packet_from_net(&self, data: &[u8]) -> std::io::Result<()>;
-    async fn handle_packet_from_device(&self, data: Vec<u8>) -> std::io::Result<()>;
+    async fn handle_packet_from_device(&self, data: BytesMut) -> std::io::Result<()>;
 }
 
+/* 
 static ARP_WAIT_LIST: OnceCell<ArpWaitList> = OnceCell::new();
 
 pub fn init_arp_wait_list() {
@@ -41,7 +43,7 @@ pub fn init_arp_wait_list() {
 pub struct ArpWaitInfo {
     timestamp: u64,
     // origin data is from the tun or tap device
-    origin_data: Vec<u8>,
+    origin_data: BytesMut,
 }
 
 #[derive(Debug)]
@@ -50,12 +52,12 @@ pub struct ArpWaitList {
 }
 
 impl ArpWaitList {
-    fn add_to_wait_list(&self, ip: u32, origin_data: Vec<u8>) {
+    fn add_to_wait_list(&self, ip: u32, origin_data_with_zeroed_layer2: BytesMut) {
         let mut entry = self.content.entry(ip).or_insert(vec![]);
         if entry.len() < MAX_WAIT_PACKETS {
             entry.push(ArpWaitInfo {
                 timestamp: get_current_timestamp(),
-                origin_data,
+                origin_data: origin_data_with_zeroed_layer2,
             })
         }
     }
@@ -84,7 +86,7 @@ impl ArpWaitList {
             if (now - item.timestamp) > 5 {
                 continue;
             }
-            let packet = form_ethernet_packet(src_mac, mac, &item.origin_data);
+            let packet = form_ethernet_packet(src_mac, mac, item.origin_data);
 
             let pkt_size = packet.len();
 
@@ -111,12 +113,12 @@ impl ArpWaitList {
     }
 }
 
-pub fn add_to_arp_wait_list(ip: u32, origin_data: Vec<u8>) {
+pub fn add_to_arp_wait_list(ip: u32, origin_data_with_zeroed_layer2: BytesMut) {
     let waitlist = ARP_WAIT_LIST
         .get()
         .expect("ARP_WAIT_LIST has not been inited");
 
-    waitlist.add_to_wait_list(ip, origin_data);
+    waitlist.add_to_wait_list(ip, origin_data_with_zeroed_layer2);
 }
 
 pub async fn arp_arrived(ip: u32, mac: Mac) {
@@ -126,3 +128,4 @@ pub async fn arp_arrived(ip: u32, mac: Mac) {
 
     waitlist.arp_arrived(ip, mac).await;
 }
+*/

src/pb/message.rs

@@ -32,38 +32,29 @@ pub struct SdlWelcome {
     #[prost(uint32, tag = "4")]
     pub heartbeat_sec: u32,
 }
-/// quic 通讯消息
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct SdlEmpty {
-    #[prost(uint32, tag = "1")]
-    pub pkt_id: u32,
-}
 /// 这里修改成了扁平的结构, 否则有些字段不好找放的位置
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct SdlRegisterSuper {
-    #[prost(uint32, tag = "1")]
-    pub pkt_id: u32,
-    #[prost(string, tag = "2")]
+    #[prost(string, tag = "1")]
     pub client_id: ::prost::alloc::string::String,
     /// 网络地址信息已经有https请求分配了
     /// 注册的时候需要带上(network_id, mac, ip, mask_len, hostname)
-    #[prost(uint32, tag = "3")]
+    #[prost(uint32, tag = "2")]
     pub network_id: u32,
-    #[prost(bytes = "vec", tag = "4")]
+    #[prost(bytes = "vec", tag = "3")]
     pub mac: ::prost::alloc::vec::Vec<u8>,
-    #[prost(uint32, tag = "5")]
+    #[prost(uint32, tag = "4")]
     pub ip: u32,
-    #[prost(uint32, tag = "6")]
+    #[prost(uint32, tag = "5")]
     pub mask_len: u32,
-    #[prost(string, tag = "7")]
+    #[prost(string, tag = "6")]
     pub hostname: ::prost::alloc::string::String,
-    #[prost(string, tag = "8")]
+    #[prost(string, tag = "7")]
     pub pub_key: ::prost::alloc::string::String,
     /// 客户端使用http协议请求后端，通过token或者账号密码登录时, 统一返回一个access_token;
     /// RegisterSuper的时候，验证凭证是否合法 (access_token)
-    #[prost(string, tag = "9")]
+    #[prost(string, tag = "8")]
     pub access_token: ::prost::alloc::string::String,
 }
 /// 客户端的升级逻辑，在https的接口里面去完成
@@ -71,47 +62,40 @@ pub struct SdlRegisterSuper {
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct SdlRegisterSuperAck {
-    #[prost(uint32, tag = "1")]
-    pub pkt_id: u32,
-    /// 目前支持aes， chacha20
-    #[prost(string, tag = "2")]
+    /// 目前支持aes, chacha20
+    #[prost(string, tag = "1")]
     pub algorithm: ::prost::alloc::string::String,
-    #[prost(bytes = "vec", tag = "3")]
+    #[prost(bytes = "vec", tag = "2")]
     pub key: ::prost::alloc::vec::Vec<u8>,
     /// 逻辑分段，chacha20加密算法需要使用该字段
-    #[prost(uint32, tag = "4")]
+    #[prost(uint32, tag = "3")]
     pub region_id: u32,
-    #[prost(bytes = "vec", tag = "5")]
+    #[prost(bytes = "vec", tag = "4")]
     pub session_token: ::prost::alloc::vec::Vec<u8>,
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct SdlRegisterSuperNak {
     #[prost(uint32, tag = "1")]
-    pub pkt_id: u32,
-    #[prost(uint32, tag = "2")]
     pub error_code: u32,
-    #[prost(string, tag = "3")]
+    #[prost(string, tag = "2")]
     pub error_message: ::prost::alloc::string::String,
 }
+/// 网络地址查询
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct SdlQueryInfo {
-    #[prost(uint32, tag = "1")]
-    pub pkt_id: u32,
-    #[prost(bytes = "vec", tag = "2")]
+    #[prost(bytes = "vec", tag = "1")]
     pub dst_mac: ::prost::alloc::vec::Vec<u8>,
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct SdlPeerInfo {
-    #[prost(uint32, tag = "1")]
-    pub pkt_id: u32,
-    #[prost(bytes = "vec", tag = "2")]
+    #[prost(bytes = "vec", tag = "1")]
     pub dst_mac: ::prost::alloc::vec::Vec<u8>,
-    #[prost(message, optional, tag = "3")]
+    #[prost(message, optional, tag = "2")]
     pub v4_info: ::core::option::Option<Sdlv4Info>,
-    #[prost(message, optional, tag = "4")]
+    #[prost(message, optional, tag = "3")]
     pub v6_info: ::core::option::Option<Sdlv6Info>,
 }
 /// ARP查询相关
@@ -121,31 +105,33 @@ pub struct SdlPeerInfo {
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct SdlArpRequest {
     #[prost(uint32, tag = "1")]
-    pub pkt_id: u32,
-    #[prost(uint32, tag = "2")]
     pub target_ip: u32,
+    #[prost(uint32, tag = "2")]
+    pub origin_ip: u32,
+    #[prost(bytes = "vec", tag = "3")]
+    pub context: ::prost::alloc::vec::Vec<u8>,
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct SdlArpResponse {
     #[prost(uint32, tag = "1")]
-    pub pkt_id: u32,
-    #[prost(uint32, tag = "2")]
     pub target_ip: u32,
-    #[prost(bytes = "vec", tag = "3")]
+    #[prost(bytes = "vec", tag = "2")]
     pub target_mac: ::prost::alloc::vec::Vec<u8>,
+    #[prost(uint32, tag = "3")]
+    pub origin_ip: u32,
+    #[prost(bytes = "vec", tag = "4")]
+    pub context: ::prost::alloc::vec::Vec<u8>,
 }
 /// 权限请求查询相关
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct SdlPolicyRequest {
     #[prost(uint32, tag = "1")]
-    pub pkt_id: u32,
-    #[prost(uint32, tag = "2")]
     pub src_identity_id: u32,
-    #[prost(uint32, tag = "3")]
+    #[prost(uint32, tag = "2")]
     pub dst_identity_id: u32,
-    #[prost(uint32, tag = "4")]
+    #[prost(uint32, tag = "3")]
     pub version: u32,
 }
 /// 基于quic通讯，rules部分已经没有了长度限制
@@ -153,16 +139,14 @@ pub struct SdlPolicyRequest {
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct SdlPolicyResponse {
     #[prost(uint32, tag = "1")]
-    pub pkt_id: u32,
-    #[prost(uint32, tag = "2")]
     pub src_identity_id: u32,
-    #[prost(uint32, tag = "3")]
+    #[prost(uint32, tag = "2")]
     pub dst_identity_id: u32,
     /// 版本号，客户端需要比较版本号确定是否覆盖; 请求端自己去管理版本号，服务端只是原样回写
-    #[prost(uint32, tag = "4")]
+    #[prost(uint32, tag = "3")]
     pub version: u32,
-    /// 4+1+2 的稀疏序列化规则
-    #[prost(bytes = "vec", tag = "5")]
+    /// 1 + 2稀疏序列化规则, 按照: <<Proto:8, Port:16>> 这个格式序列号所有的规则信息; 下发的数据默认都是allow，deny规则的服务器端已经屏蔽
+    #[prost(bytes = "vec", tag = "4")]
     pub rules: ::prost::alloc::vec::Vec<u8>,
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
@@ -197,29 +181,24 @@ pub struct SdlNetworkShutdownEvent {
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct SdlStunRequest {
-    #[prost(uint32, tag = "1")]
-    pub cookie: u32,
-    #[prost(string, tag = "2")]
+    #[prost(string, tag = "1")]
     pub client_id: ::prost::alloc::string::String,
-    #[prost(uint32, tag = "3")]
+    #[prost(uint32, tag = "2")]
     pub network_id: u32,
-    #[prost(bytes = "vec", tag = "4")]
+    #[prost(bytes = "vec", tag = "3")]
     pub mac: ::prost::alloc::vec::Vec<u8>,
-    #[prost(uint32, tag = "5")]
+    #[prost(uint32, tag = "4")]
     pub ip: u32,
-    #[prost(uint32, tag = "6")]
+    #[prost(uint32, tag = "5")]
     pub nat_type: u32,
-    #[prost(message, optional, tag = "7")]
+    #[prost(message, optional, tag = "6")]
     pub v6_info: ::core::option::Option<Sdlv6Info>,
-    #[prost(bytes = "vec", tag = "8")]
+    #[prost(bytes = "vec", tag = "7")]
     pub session_token: ::prost::alloc::vec::Vec<u8>,
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
-pub struct SdlStunReply {
-    #[prost(uint32, tag = "1")]
-    pub cookie: u32,
-}
+pub struct SdlStunReply {}
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct SdlData {
@@ -286,3 +265,6 @@ pub struct SdlRegisterAck {
     #[prost(bytes = "vec", tag = "3")]
     pub dst_mac: ::prost::alloc::vec::Vec<u8>,
 }
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct SdlEmpty {}

src/tcp/quic.rs

@@ -9,7 +9,7 @@ use tokio::{sync::mpsc::{Receiver, Sender, channel}, time::sleep};
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, warn};
 
-use crate::{AesEncryptor, Chacha20Encryptor, ConnectionInfo, ConnectionState, MyEncryptor, RuleFromServer, config::{NULL_MAC, TCP_PING_TIME}, get_edge, network::{ARP_REPLY, ArpHdr, EthHdr, Node, RegisterSuperFeedback, StartStopInfo, check_peer_registration_needed, handle_packet_peer_info}, pb::{SdlArpResponse, SdlPolicyResponse, SdlRegisterSuper, SdlRegisterSuperAck, SdlRegisterSuperNak, SdlSendRegisterEvent, encode_to_tcp_message}, tcp::{EventType, NakMsgCode, NatType, PacketType, SdlanTcp, read_a_packet, send_stun_request}};
+use crate::{AesEncryptor, Chacha20Encryptor, ConnectionInfo, ConnectionState, MyEncryptor, RuleFromServer, config::{NULL_MAC, TCP_PING_TIME}, get_edge, network::{ARP_REPLY, ArpHdr, EthHdr, Node, RegisterSuperFeedback, StartStopInfo, arp_reply_arrived, check_peer_registration_needed, handle_packet_peer_info}, pb::{SdlArpResponse, SdlPolicyResponse, SdlRegisterSuper, SdlRegisterSuperAck, SdlRegisterSuperNak, SdlSendRegisterEvent, encode_to_tcp_message}, tcp::{EventType, NakMsgCode, NatType, PacketType, SdlanTcp, read_a_packet, send_stun_request}};
 
 static GLOBAL_QUIC_HANDLE: OnceLock<ReadWriterHandle> = OnceLock::new();
 
@@ -118,7 +118,7 @@ async fn handle_tcp_message(msg: SdlanTcp) {
             };
 
             edge.send_register_super_feedback(
-                ack.pkt_id,
+                0,
                 RegisterSuperFeedback {
                     result: 0,
                     message: "".to_owned(),
@@ -183,7 +183,7 @@ async fn handle_tcp_message(msg: SdlanTcp) {
             edge.network_id.store(dev.network_id, Ordering::Relaxed);
             */
             // edge.device.reload_config(&edge.device_config, &dev.network_domain);
-            edge.device.reload_config(&edge.device_config, &edge.network_domain.read().unwrap().clone());
+            edge.device.reload_config(edge, &edge.device_config, &edge.network_domain.read().unwrap().clone());
 
             edge.set_authorized(true);
 
@@ -195,13 +195,14 @@ async fn handle_tcp_message(msg: SdlanTcp) {
             });
         }
         PacketType::ArpResponse => {
-            let Ok(resp) = SdlArpResponse::decode(&msg.current_packet[..]) else {
+            let Ok(res) = SdlArpResponse::decode(&msg.current_packet[..]) else {
                 error!("failed to decode ARP RESPONSE");
                 return;
             };
 
-            warn!("got arp response: {:?}", resp);
-            edge.cookie_match.write_feedback(resp.pkt_id, Box::new(resp));
+
+            arp_reply_arrived(edge, res).await;
+            return;
         }
         PacketType::PolicyReply => {
             let Ok(policy) = SdlPolicyResponse::decode(&msg.current_packet[..]) else {
@@ -246,12 +247,12 @@ async fn handle_tcp_message(msg: SdlanTcp) {
             };
 
             error!("got nak: {:?}", _nak);
-            let pkt_id = _nak.pkt_id;
+            // let pkt_id = _nak.pkt_id;
 
             let Ok(error_code) = NakMsgCode::try_from(_nak.error_code as u8) else {
                 edge.send_register_super_feedback(
                     //msg._packet_id,
-                    pkt_id,
+                    0,
                     RegisterSuperFeedback {
                         result: 2,
                         message: "error_code not recognized".to_owned(),
@@ -264,7 +265,7 @@ async fn handle_tcp_message(msg: SdlanTcp) {
                 NakMsgCode::InvalidToken => {
                     edge.send_register_super_feedback(
                         // msg._packet_id,
-                        pkt_id,
+                        0,
                         RegisterSuperFeedback {
                             result: 3,
                             message: "invalid token".to_owned(),
@@ -276,7 +277,7 @@ async fn handle_tcp_message(msg: SdlanTcp) {
                 NakMsgCode::NodeDisabled => {
                     edge.send_register_super_feedback(
                         // msg._packet_id,
-                        pkt_id,
+                        0,
                         RegisterSuperFeedback {
                             result: 4,
                             message: "Node is disabled".to_owned(),
@@ -288,7 +289,7 @@ async fn handle_tcp_message(msg: SdlanTcp) {
                 _other => {
                     edge.send_register_super_feedback(
                         // msg._packet_id,
-                        pkt_id,
+                        0,
                         RegisterSuperFeedback {
                             result: 0,
                             message: "".to_owned(),
@@ -471,7 +472,7 @@ impl ReadWriteActor {
         let edge = get_edge();
 
         let mut started = false;
-        let mut start_pkt_id = None;
+        // let mut start_pkt_id = None;
         loop {
             if let Some(ref connecting_chan) = self.connecting_chan {
                 let state = ConnectionInfo::ConnState(ConnectionState::NotConnected);
@@ -485,7 +486,7 @@ impl ReadWriteActor {
                     if let Some(m) = start_or_stop {
                         if m.is_start {
                             started = true;
-                            start_pkt_id = m.pkt_id;
+                            // start_pkt_id = m.pkt_id;
                             break;
                         }
                     } else {
@@ -566,7 +567,7 @@ impl ReadWriteActor {
             debug!("connected");
 
             sleep(Duration::from_millis(200)).await;
-            on_connected_callback(local_ip, &mut send, start_pkt_id.take()).await;
+            on_connected_callback(local_ip, &mut send).await;
 
             if let Some(ref connecting_chan) = self.connecting_chan {
                 let state = ConnectionInfo::ConnState(ConnectionState::Connected);
@@ -669,7 +670,7 @@ async fn on_disconnected_callback() {
     edge.encryptor.store(Arc::new(MyEncryptor::Invalid));
 }
 
-async fn on_connected_callback(local_ip: Option<IpAddr>, stream: &mut SendStream, _pkt_id: Option<u32>) {
+async fn on_connected_callback(local_ip: Option<IpAddr>, stream: &mut SendStream) {
     let edge = get_edge();
     // let installed_channel = install_channel.to_owned();
 
@@ -691,7 +692,7 @@ async fn on_connected_callback(local_ip: Option<IpAddr>, stream: &mut SendStream
     }
     let register_super = SdlRegisterSuper {
         mac: Vec::from(edge.device_config.get_mac()),
-        pkt_id: edge.get_next_packet_id(),
+        // pkt_id: edge.get_next_packet_id(),
         network_id: edge.network_id.load(Ordering::Relaxed),
         ip: edge.device_config.get_ip(),
         mask_len: edge.device_config.get_net_bit() as u32,
@@ -704,6 +705,7 @@ async fn on_connected_callback(local_ip: Option<IpAddr>, stream: &mut SendStream
     };
 
     println!("register super: {:?}", register_super);
+
     // debug!("send register super: {:?}", register_super);
     // let packet_id = edge.get_next_packet_id();
     let data = encode_to_tcp_message(

src/tcp/tcp_codec.rs

@@ -114,7 +114,7 @@ pub async fn send_stun_request(eee: &Node) {
     };
     let req = SdlStunRequest {
         session_token: Vec::from(eee.session_token.get()),
-        cookie: 0,
+        // cookie: 0,
         client_id: eee.config.node_uuid.clone(),
         network_id: eee.network_id.load(Ordering::Relaxed),
         ip: eee.device_config.get_ip(),

src/utils/acl_session.rs

@@ -1,10 +1,10 @@
-use std::{net::IpAddr, sync::{Arc, atomic::{AtomicU64, Ordering}}, time::{Duration, Instant, SystemTime, UNIX_EPOCH}};
+use std::{net::IpAddr, sync::{Arc, atomic::{AtomicU64, Ordering}}, time::{Duration, SystemTime, UNIX_EPOCH}};
 
 use ahash::RandomState;
 use dashmap::{DashMap, DashSet};
 use tracing::{debug, error};
 
-const RuleValidTimeInSecs: u64 = 60;
+const RULE_VALID_TIME_IN_SECS: u64 = 60;
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] 
 pub struct FiveTuple {
@@ -115,7 +115,7 @@ impl RuleCache {
         });
         Self {
             rule_info: DashMap::with_hasher(RandomState::new()),
-            rule_valid_secs: RuleValidTimeInSecs,
+            rule_valid_secs: RULE_VALID_TIME_IN_SECS,
             session_table,
         }
     }

src/utils/command.rs

@@ -18,11 +18,11 @@ pub enum Commands {
 
     /// if logined in, just start,
     /// else, use the token to login, and start
-    AutoRun(TokenLogin),
+    AutoRun(AutoRunArgument),
 
     /// after login, we can use start to 
     /// connect to the remote
-    Start(RouteCmdInfo),
+    Start(StartArguments),
 
     Info,
 
@@ -34,6 +34,18 @@ pub enum Commands {
     Stop,
 }
 
+#[derive(Args, Debug)]
+pub struct StartArguments {
+    #[arg(short, long, default_value="")]
+    pub route: String,
+
+    #[arg(short, long, default_value="")]
+    pub route_file: String,
+
+    #[arg(short, long, default_value_t=false)]
+    pub allow_routing: bool,
+}
+
 #[derive(Args, Debug)]
 pub struct RouteCmdInfo {
     #[arg(short, long, default_value="")]
@@ -58,6 +70,21 @@ pub struct AutoRunTokenLogin {
     pub route: String,
 }
 
+#[derive(Args, Debug)]
+pub struct AutoRunArgument {
+    #[arg(long, env=APP_TOKEN_ENV_NAME, required=false)]
+    pub token: String,
+
+    #[arg(short, long, default_value_t=false)]
+    pub allow_routing: bool,
+
+    #[arg(short, long, default_value="")]
+    pub route: String,
+
+    #[arg(short, long, default_value="")]
+    pub route_file: String,
+}
+
 #[derive(Args, Debug)]
 pub struct TokenLogin {
     #[arg(long, env=APP_TOKEN_ENV_NAME, required=false)]
@@ -100,7 +127,7 @@ pub struct CommandLine {
     pub nat_server2: String,
 
     #[structopt(short = "r")]
-    pub _allow_routing: bool,
+    pub allow_routing: bool,
 
     #[structopt(short = "dm")]
     pub _drop_multicast: bool,
@@ -132,6 +159,12 @@ pub struct CommandLine {
 
     #[structopt(short = "l")]
     pub local_port: u16,
+
+    #[structopt(long, default_value="")]
+    pub route_str: String,
+
+    #[structopt(long, default_value="")]
+    pub route_file: String,
 }
 
 impl Clone for CommandLine {
@@ -139,7 +172,7 @@ impl Clone for CommandLine {
         Self {
             sn: self.sn.clone(),
             quic: self.quic.clone(),
-            _allow_routing: self._allow_routing,
+            allow_routing: self.allow_routing,
             _drop_multicast: self._drop_multicast,
             register_ttl: self.register_ttl,
             mtu: self.mtu,
@@ -151,6 +184,8 @@ impl Clone for CommandLine {
             allow_p2p: self.allow_p2p,
             nat_server1: self.nat_server1.clone(),
             nat_server2: self.nat_server2.clone(),
+            route_str: self.route_str.clone(),
+            route_file: self.route_file.clone(),
         }
     }
 }

src/utils/system_action.rs

@@ -2,8 +2,6 @@ use std::{net::Ipv4Addr, sync::Arc};
 
 use arc_swap::ArcSwap;
 
-use crate::network::Node;
-
 #[derive(Default, Clone)]
 pub struct TrieNode {
     child: [Option<Box<TrieNode>>; 2],