changed to new message.proto, which eliminates the pkt_id

Cargo.lock

@@ -155,6 +155,15 @@ version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d67af77d68a931ecd5cbd8a3b5987d63a1d1d1278f7f6a60ae33db485cdebb69"
 
+[[package]]
+name = "arc-swap"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6"
+dependencies = [
+ "rustversion",
+]
+
 [[package]]
 name = "arrayvec"
 version = "0.7.6"
@@ -2134,6 +2143,8 @@ dependencies = [
 name = "punchnet"
 version = "1.0.3"
 dependencies = [
+ "ahash",
+ "arc-swap",
  "bytes",
  "cargo-deb",
  "chacha20poly1305",
@@ -2147,6 +2158,7 @@ dependencies = [
  "futures-util",
  "hex",
  "hmac",
+ "ipnet",
  "libc",
  "local-ip-address",
  "md-5",

Cargo.toml

@@ -39,6 +39,9 @@ chacha20poly1305 = "0.10.1"
 hmac = "0.12.1"
 md-5 = "0.10.6"
 hex = "0.4.3"
+ahash = "0.8.12"
+ipnet = "2.12.0"
+arc-swap = "1.9.0"
 # rolling-file = { path = "../rolling-file" }
 
 [target.'cfg(unix)'.dependencies]

Makefile

@@ -1,4 +1,4 @@
-self:
+self: libtun-so
 	RUSTFLAGS="-L ." cargo build --release
 
 linux: libtun-so
@@ -8,7 +8,7 @@ aarch64: libtun-so-aarch64
 	RUSTFLAGS="-L ." cargo build --release --target aarch64-unknown-linux-gnu
 
 
-linux-tun:
+linux-tun: libtun-so
 	RUSTFLAGS="-L ." cargo build  --features "tun" --release
 
 win:

proto/empty.proto

@@ -0,0 +1,6 @@
+syntax = "proto3";
+
+package message;
+
+message SDLEmpty {
+}

proto/message.proto

@@ -28,92 +28,81 @@ message SDLWelcome {
     uint32 heartbeat_sec = 4;
 }
 
-// quic 通讯消息
-message SDLEmpty {
-    uint32 pkt_id = 1;
-}
-
 // 这里修改成了扁平的结构, 否则有些字段不好找放的位置
 message SDLRegisterSuper {
-    uint32 pkt_id = 1;
-    string client_id = 2;
+    string client_id = 1;
     // 网络地址信息已经有https请求分配了
     // 注册的时候需要带上(network_id, mac, ip, mask_len, hostname)
-    uint32 network_id = 3;
-    bytes mac = 4;
-    uint32 ip = 5;
-    uint32 mask_len = 6;
-    string hostname = 7;
+    uint32 network_id = 2;
+    bytes mac = 3;
+    uint32 ip = 4;
+    uint32 mask_len = 5;
+    string hostname = 6;
 
-    string pub_key = 8;
+    string pub_key = 7;
     // 客户端使用http协议请求后端，通过token或者账号密码登录时, 统一返回一个access_token;
     // RegisterSuper的时候，验证凭证是否合法 (access_token)
-    string access_token = 9;
+    string access_token = 8;
 }
 
 // 客户端的升级逻辑，在https的接口里面去完成
 // 部分逻辑会脱离quic去通讯，增加session_token校验
 message SDLRegisterSuperAck {
-    uint32 pkt_id = 1;
-    // 目前支持aes， chacha20
-    string algorithm = 2;
-    bytes key = 3;
+    // 目前支持aes, chacha20
+    string algorithm = 1;
+    bytes key = 2;
     // 逻辑分段，chacha20加密算法需要使用该字段
-    uint32 region_id = 4;
-    bytes session_token = 5;
+    uint32 region_id = 3;
+    bytes session_token = 4;
 }
 
 message SDLRegisterSuperNak {
-    uint32 pkt_id = 1;
-    uint32 error_code = 2;
-    string error_message = 3;
+    uint32 error_code = 1;
+    string error_message = 2;
 }
 
 // 网络地址查询
-
 message SDLQueryInfo {
-    uint32 pkt_id = 1;
-    bytes dst_mac = 2;
+    bytes dst_mac = 1;
 }
 
 message SDLPeerInfo {
-    uint32 pkt_id = 1;
-    bytes dst_mac = 2;
-    SDLV4Info v4_info = 3;
-    optional SDLV6Info v6_info = 4;
+    bytes dst_mac = 1;
+    optional SDLV4Info v4_info = 2;
+    optional SDLV6Info v6_info = 3;
 }
 
 // ARP查询相关
 // 真实的arp请求是通过广播的形式获取到的，但是针对于macos这种tun的实现；是能够分析出arp请求包的；
 // 对于当前网络来说，服务端是知道mac对应的ip地址的，因此没有必要广播；直接通过服务器端返回
 message SDLArpRequest {
-    uint32 pkt_id = 1;
-    uint32 target_ip = 2;
+    uint32 target_ip = 1;
+    uint32 origin_ip = 2;
+    bytes context = 3;
 }
 
 message SDLArpResponse {
-    uint32 pkt_id = 1;
-    uint32 target_ip = 2;
-    bytes target_mac = 3;
+    uint32 target_ip = 1;
+    bytes target_mac = 2;
+    uint32 origin_ip = 3;
+    bytes context = 4;
 }
 
 // 权限请求查询相关
 message SDLPolicyRequest {
-    uint32 pkt_id = 1;
-    uint32 src_identity_id = 2;
-    uint32 dst_identity_id = 3;
-    uint32 version = 4;
+    uint32 src_identity_id = 1;
+    uint32 dst_identity_id = 2;
+    uint32 version = 3;
 }
 
 // 基于quic通讯，rules部分已经没有了长度限制
 message SDLPolicyResponse {
-    uint32 pkt_id = 1;
-    uint32 src_identity_id = 2;
-    uint32 dst_identity_id = 3;
+    uint32 src_identity_id = 1;
+    uint32 dst_identity_id = 2;
     // 版本号，客户端需要比较版本号确定是否覆盖; 请求端自己去管理版本号，服务端只是原样回写
-    uint32 version = 4;
-    // 4+1+2 的稀疏序列化规则
-    bytes rules = 5;
+    uint32 version = 3;
+    // 1 + 2稀疏序列化规则, 按照: <<Proto:8, Port:16>> 这个格式序列号所有的规则信息; 下发的数据默认都是allow，deny规则的服务器端已经屏蔽
+    bytes rules = 4;
 }
 
 // 事件定义
@@ -139,18 +128,16 @@ message SDLNetworkShutdownEvent {
 
 // client和stun之间的心跳包，客户端需要和super的udp之间的存活逻辑
 message SDLStunRequest {
-    uint32 cookie = 1;
-    string client_id = 2;
-    uint32 network_id = 3;
-    bytes mac = 4;
-    uint32 ip = 5;
-    uint32 nat_type = 6;
-    optional SDLV6Info v6_info = 7;
-    bytes session_token = 8;
+    string client_id = 1;
+    uint32 network_id = 2;
+    bytes mac = 3;
+    uint32 ip = 4;
+    uint32 nat_type = 5;
+    optional SDLV6Info v6_info = 6;
+    bytes session_token = 7;
 }
 
 message SDLStunReply {
-    uint32 cookie = 1;
 }
 
 message SDLData {

src/bin/build_pb/main.rs

@@ -4,6 +4,6 @@ fn main() {
         .bytes(&[".message.SDLData.data"])
         // .out_dir("../tcp_mock/pb")
         .protoc_arg("--experimental_allow_proto3_optional")
-        .compile_protos(&["message.proto"], &["."])
+        .compile_protos(&["message.proto", "empty.proto"], &["./proto"])
         .unwrap();
 }

src/bin/punchnet/main.rs

@@ -106,9 +106,12 @@ fn parse_login_result(res: Result<LoginResponse>) -> LoginData {
 }
 
 async fn daemonize_me(
+    allow_routing: bool,
     connect_info: ConnectData, 
     remembered: CachedLoginInfo,
     client_id: String,
+    route_file: String,
+    route_str: String,
     mac: Mac,
 ) {
     let _guard = log::init_log(&format!("{}/.output", get_base_dir()));
@@ -117,7 +120,7 @@ async fn daemonize_me(
 
     let (tx, rx) = std::sync::mpsc::channel();
 
-    let hostname = "118.178.229.213".to_owned();
+    let hostname = "root.punchsky.com".to_owned();
     let host = format!("{}:80", hostname);
     let mut server = String::new();
     if let Ok(addrs) = host.to_socket_addrs() {
@@ -158,7 +161,7 @@ async fn daemonize_me(
             nat_server1: server.clone() +":1365",
             // nat_server2: "47.98.178.3:1265".to_owned(),
             nat_server2: server.clone() +":1366",
-            _allow_routing: true,
+            allow_routing: allow_routing,
             _drop_multicast: true,
             register_ttl: 1,
             mtu: 1400,
@@ -168,6 +171,8 @@ async fn daemonize_me(
             // token: cmd.token.clone(),
             // network_code: cmd.network_code.clone(),
             allow_p2p: true,
+            route_file,
+            route_str,
         },
         tx,
         &punchnet::get_install_channel(),
@@ -398,7 +403,7 @@ fn main() {
 fn run_it(cmd: CommandLineInput2, client_id: String, mac: Mac, system: &str, version: &str) {
     let rt = Runtime::new().unwrap();
     match &cmd.cmd {
-        Commands::Start => {
+        Commands::Start(rtinfo) => {
             rt.block_on(async move {
                 let remembered_token = get_access_token();
                 if remembered_token.is_none() {
@@ -411,7 +416,7 @@ fn run_it(cmd: CommandLineInput2, client_id: String, mac: Mac, system: &str, ver
                 let connect_info = parse_connect_result(
                     connect(TEST_PREFIX, &client_id, &remembered.access_token).await
                 );
-                daemonize_me(connect_info, remembered, client_id, mac).await;
+                daemonize_me(rtinfo.allow_routing, connect_info, remembered, client_id, rtinfo.route_file.clone(), rtinfo.route.clone(), mac).await;
             })
         }
         Commands::AutoRun(tk) => {
@@ -436,10 +441,10 @@ fn run_it(cmd: CommandLineInput2, client_id: String, mac: Mac, system: &str, ver
                 let connect_info = parse_connect_result(
                     connect(TEST_PREFIX, &client_id, &remembered.access_token).await
                 );
-                daemonize_me(connect_info, remembered, client_id, mac).await;
+                daemonize_me(tk.allow_routing, connect_info, remembered, client_id, tk.route_file.clone(), tk.route.clone(), mac).await;
             })
-
         }
+
         other => {
             eprintln!("should not comes here");
             process::exit(-1);

src/lib.rs

@@ -12,7 +12,7 @@ use std::net::{SocketAddr, ToSocketAddrs};
 
 pub use network::get_edge;
 pub use network::get_install_channel;
-pub use network::{async_main, init_arp, init_edge, NodeConfig, restore_dns};
+pub use network::{async_main, init_edge, NodeConfig, restore_dns};
 use sdlan_sn_rs::utils::{Mac, save_to_file};
 use serde::{Deserialize, Serialize};
 use tokio::net::UdpSocket;
@@ -64,7 +64,6 @@ pub async fn run_sdlan(
     let hostfile = format!("{}/.host", get_base_dir());
     let host = create_or_load_uuid(&hostfile, Some(8))?;
 
-    init_arp();
 
     let hostname = hostname.unwrap_or(host);
     let _ = save_to_file(&hostfile, &hostname);
@@ -74,11 +73,12 @@ pub async fn run_sdlan(
     if let Err(e) = init_edge(
         // &args.token,
         // &args.network_code,
+        &args,
         mac,
         node_conf,
-        args.tos,
+        // args.tos,
         start_stop_sender,
-        args.mtu,
+        // args.mtu,
         connecting_chan.clone(),
         sock,
         hostname,
@@ -154,7 +154,7 @@ async fn parse_config(uuid: String, args: &CommandLine) -> Result<NodeConfig> {
 
     let node_conf = NodeConfig {
         name: args.name.to_owned(),
-        allow_routing: true,
+        allow_routing: args.allow_routing,
         _drop_multicast: true,
         allow_p2p: args.allow_p2p,
         mtu: args.mtu,

src/network/arp.rs

@@ -2,24 +2,28 @@
 
 use std::{
     collections::HashMap,
-    sync::atomic::{AtomicU8, Ordering},
-    time::Duration,
+    sync::{Arc, atomic::{AtomicU8, Ordering}},
+    time::{Duration, Instant},
 };
 
-use tracing::error;
+use bytes::{Bytes, BytesMut};
+use dashmap::DashMap;
+use tracing::{debug, error};
 
 use once_cell::sync::OnceCell;
-use sdlan_sn_rs::utils::{BROADCAST_MAC, MULTICAST_MAC, Mac, ip_to_string, net_bit_len_to_mask};
+use sdlan_sn_rs::{config::SDLAN_DEFAULT_TTL, utils::{BROADCAST_MAC, MULTICAST_MAC, Mac, get_current_timestamp, ip_to_string, mac_to_string, net_bit_len_to_mask}};
 use tokio::sync::{
     mpsc::{channel, Receiver, Sender},
     oneshot,
 };
 
-use super::{get_edge, get_route_table, init_arp_wait_list, init_route};
+use crate::{network::{form_ethernet_packet, send_packet_to_net}, pb::{SdlData, encode_to_udp_message}, tcp::PacketType};
 
+use super::{get_edge};
+
+/* 
 static GLOBAL_ARP: OnceCell<ArpActor> = OnceCell::new();
 pub fn init_arp() {
-    init_route();
     init_arp_wait_list();
     let actor = ArpActor::new();
     GLOBAL_ARP.set(actor).unwrap();
@@ -28,6 +32,7 @@ pub fn init_arp() {
 pub fn get_arp() -> &'static ArpActor {
     GLOBAL_ARP.get().unwrap()
 }
+*/
 
 const ETHER_TYPE_ARP: u16 = 0x0806;
 const ETHER_TYPE_IP: u16 = 0x0800;
@@ -142,9 +147,9 @@ const BROADCAST_IPADDR: u32 = 0xffffffff;
 #[derive(Debug)]
 #[allow(unused)]
 pub struct ArpEntry {
-    ip_addr: u32,
-    arptime: u8,
-    hw_addr: [u8; 6],
+    // ip_addr: u32,
+    last_seen: Instant,
+    hw_addr: Mac,
 }
 
 /*
@@ -159,6 +164,62 @@ impl ArpEntry {
 }
 */
 
+
+
+pub struct ArpTable {
+    entries: Arc<DashMap<u32, ArpEntry>>,
+    ttl: Duration,
+    pending_packet_buffer: ArpWaitList,
+}
+
+impl ArpTable {
+    pub fn new() -> Self {
+        let entries = Arc::new(DashMap::new());
+        let res = Self {
+            entries: entries.clone(),
+            ttl: Duration::from_secs(60),
+            pending_packet_buffer: ArpWaitList::new(),
+        };
+
+        let mut bytes = BytesMut::with_capacity(1024+20);
+
+        let ttl = res.ttl;
+        tokio::spawn(async move {
+            loop {
+                tokio::time::sleep(Duration::from_secs(30)).await;
+                let now = Instant::now();
+                entries.retain(|_, entry| now.duration_since(entry.last_seen) < ttl);
+            }
+        });
+
+        res
+    }
+
+    pub async fn arp_arrived(&self, ip: u32, mac: Mac) {
+        self.pending_packet_buffer.arp_arrived(ip, mac).await;
+    }
+
+    pub fn add_to_arp_wait_list(&self, ip: u32, origin_data: BytesMut) {
+        self.pending_packet_buffer.add_to_wait_list(ip, origin_data);
+    }
+
+    #[inline]
+    pub fn get(&self, ip: u32) -> Option<Mac> {
+        self.entries.get(&ip).map(|entry| entry.hw_addr)
+    }
+
+    pub fn set(&self, ip: u32, mac: Mac) {
+        self.entries.insert(
+            ip,
+            ArpEntry {
+                last_seen: Instant::now(),
+                hw_addr: mac,
+            },
+        );
+    }
+}
+
+/* 
 pub struct ArpInfo {
     // host_ip: AtomicU32,
     // ip representation of mask
@@ -198,9 +259,10 @@ impl ArpInfo {
         }
 
         if target_ip == 0 {
-            let route_table = get_route_table();
-            if let Some(gateway_ip) = route_table.get_gateway_ip(ip) {
-                target_ip = gateway_ip;
+            // let route_table = get_route_table();
+            if let Some((_prefix, gateway_ip)) = edge.route_table.route_table.lookup(ip) {
+            // if let Some(gateway_ip) = route_table.get_gateway_ip(ip) {
+                target_ip = gateway_ip.into();
             }
         }
         if target_ip == 0 {
@@ -302,7 +364,9 @@ impl ArpInfo {
         */
     }
 }
+*/
 
+/*
 pub enum ArpRequestInfo {
     Lookup { ip: u32 },
     Set { ip: u32, mac: Mac },
@@ -392,6 +456,7 @@ pub async fn send_arp_request(data: ArpRequestInfo) -> ArpResponse {
         Err(e) => ArpResponse::ArpRespError { msg: e.to_string() },
     }
 }
+    */
 
 pub fn generate_arp_request(srcmac: [u8; 6], dstip: u32, srcip: u32) -> Vec<u8> {
     let mut arphdr = ArpHdr::new();
@@ -418,3 +483,88 @@ pub fn generate_arp_request(srcmac: [u8; 6], dstip: u32, srcip: u32) -> Vec<u8>
 
     arphdr.marshal_to_bytes()
 }
+
+
+#[derive(Debug)]
+pub struct ArpWaitInfo {
+    timestamp: u64,
+    // origin data is from the tun or tap device
+    origin_data: BytesMut,
+}
+
+
+pub const MAX_WAIT_PACKETS: usize = 100;
+
+#[derive(Debug)]
+pub struct ArpWaitList {
+    content: DashMap<u32, Vec<ArpWaitInfo>>,
+}
+
+impl ArpWaitList {
+    fn new() -> Self {
+        Self {
+            content: DashMap::new(),
+        }
+    }
+
+
+    fn add_to_wait_list(&self, ip: u32, origin_data: BytesMut) {
+        let mut entry = self.content.entry(ip).or_insert(vec![]);
+        if entry.len() < MAX_WAIT_PACKETS {
+            entry.push(ArpWaitInfo {
+                timestamp: get_current_timestamp(),
+                origin_data,
+            })
+        }
+    }
+
+    async fn arp_arrived(&self, ip: u32, mac: Mac) {
+        debug!(
+            "arp for {} arrived: {}",
+            ip_to_string(&ip),
+            mac_to_string(&mac)
+        );
+        let Some(items) = self.content.remove(&ip) else {
+            return;
+        };
+        let edge = get_edge();
+        // just remove the items
+        if !edge.is_authorized() {
+            return;
+        }
+
+        // let encrypt_key = edge.get_encrypt_key();
+        let network_id = edge.network_id.load(Ordering::Relaxed);
+
+        let src_mac = edge.device_config.get_mac();
+        let now = get_current_timestamp();
+        for item in items.1 {
+            if (now - item.timestamp) > 5 {
+                continue;
+            }
+            let packet = form_ethernet_packet(src_mac, mac, item.origin_data);
+
+            let pkt_size = packet.len();
+
+            let Ok(encrypted) = edge.encryptor.load().encrypt(&packet) else {
+            // let Ok(encrypted) = edge.encryptor.read().unwrap().encrypt(&packet) else {
+            // let Ok(encrypted) = aes_encrypt(&encrypt_key, &packet) else {
+                error!("failed to encrypt packet request");
+                return;
+            };
+            let data_bytes = Bytes::from(encrypted);
+            let data = SdlData {
+                is_p2p: true,
+                network_id,
+                ttl: SDLAN_DEFAULT_TTL as u32,
+                src_mac: Vec::from(src_mac),
+                dst_mac: Vec::from(mac),
+                data: data_bytes,
+                identity_id: edge.identity_id.load(),
+                session_token: edge.session_token.get(),
+            };
+            let msg = encode_to_udp_message(Some(data), PacketType::Data as u8).unwrap();
+            send_packet_to_net(edge, mac, &msg, pkt_size as u64).await;
+        }
+    }
+}

src/network/async_main.rs

@@ -7,9 +7,10 @@ use crate::network::ipv6::run_ipv6;
 use crate::network::{
     get_edge, ping_to_sn, read_and_parse_packet, TunTapPacketHandler,
 };
-use crate::tcp::{init_identity_cache, init_quic_conn, send_stun_request};
+use crate::tcp::{init_quic_conn, send_stun_request};
 use crate::utils::{send_to_sock, CommandLine};
 use crate::{ConnectionInfo};
+use bytes::BytesMut;
 use sdlan_sn_rs::peer::{SdlanSock};
 use sdlan_sn_rs::utils::{get_current_timestamp, is_multi_broadcast};
 use sdlan_sn_rs::utils::{Mac, Result};
@@ -31,8 +32,6 @@ pub async fn async_main(
     // let _ = PidRecorder::new(".pid");
     let edge = get_edge();
 
-    init_identity_cache();
-
     // let token = args.token.clone();
     let cancel_tcp = cancel.clone();
     let (ipv6_network_restarter, rx) = channel(10);
@@ -252,18 +251,42 @@ async fn loop_tap(eee: &'static Node, cancel: CancellationToken) {
             }
         }
     }
-    debug!("loop_tap exited");
+    error!("loop_tap exited");
 }
 
-async fn get_tun_flow(eee: &'static Node, tx: Sender<Vec<u8>>) {
+#[cfg(feature = "tun")]
+fn get_data_from_tun_with_layer2_zeroed(eee: &Node) -> BytesMut {
+    let mut temp = BytesMut::zeroed(1514);
+    // let mut temp = BytesMut::with_capacity(1514);
+    let mut data_buf = temp.split_off(14);
+
+    let Ok(size) = eee.device.recv(&mut data_buf) else {
+        return BytesMut::new();
+    };
+    data_buf.truncate(size);
+    temp.unsplit(data_buf);
+    temp
+}
+
+#[cfg(not(feature = "tun"))]
+fn get_data_from_tap_with_layer2(eee: &Node) -> BytesMut {
+    let mut buf = BytesMut::zeroed(1514);
+    let Ok(size) = eee.device.recv(&mut buf) else {
+        return BytesMut::new();
+    };
+    buf.truncate(size);
+    buf
+}
+
+async fn get_tun_flow(eee: &'static Node, tx: Sender<BytesMut>) {
     loop {
         let buf = tokio::task::spawn_blocking(|| {
-            let mut buf = vec![0; 1500];
-            let Ok(size) = eee.device.recv(&mut buf) else {
-                return vec![];
-            };
-            buf.truncate(size);
-            buf
+            #[cfg(feature = "tun")]
+            let data = get_data_from_tun_with_layer2_zeroed(eee);
+            #[cfg(not(feature = "tun"))]
+            let data = get_data_from_tap_with_layer2(eee);
+
+            data
         })
         .await
         .unwrap();
@@ -278,7 +301,7 @@ async fn get_tun_flow(eee: &'static Node, tx: Sender<Vec<u8>>) {
     }
 }
 
-async fn read_and_parse_tun_packet(eee: &'static Node, buf: Vec<u8>) {
+async fn read_and_parse_tun_packet(eee: &'static Node, buf: BytesMut) {
     /*
     if !eee.is_authorized() {
         debug!("drop packet before authorized");
@@ -296,7 +319,7 @@ async fn read_and_parse_tun_packet(eee: &'static Node, buf: Vec<u8>) {
     edge_send_packet_to_net(eee, buf).await;
 }
 
-async fn edge_send_packet_to_net(eee: &Node, data: Vec<u8>) {
+async fn edge_send_packet_to_net(eee: &Node, data: BytesMut) {
     // debug!("edge send packet to net({} bytes): {:?}", data.len(), data);
 
     /* 

src/network/device.rs

@@ -1,13 +1,19 @@
+use std::{net::Ipv4Addr, sync::{Arc, atomic::{AtomicU32, Ordering}}};
+
 use crate::utils::mac_to_string;
-use sdlan_sn_rs::{peer::IpSubnet, utils::Mac};
+use arc_swap::ArcSwap;
+use ipnet::Ipv4Net;
+use sdlan_sn_rs::{peer::IpSubnet, utils::{Mac, Result, SDLanError}};
 use tracing::debug;
 
 use crate::utils::generate_mac_address;
 
 pub struct DeviceConfig {
     pub mtu: u32,
-    pub mac: Mac,
-    pub ip: IpSubnet,
+    mac: Mac,
+    ip: IpSubnet,
+
+    self_ip_net: ArcSwap<Ipv4Net>,
 
     pub dns_mac: Mac,
 }
@@ -23,6 +29,7 @@ impl DeviceConfig {
             mac: mac,
             ip: IpSubnet::new(0, 0),
             dns_mac,
+            self_ip_net: ArcSwap::new(Arc::new(Ipv4Net::new_assert(Ipv4Addr::from(0), 32))),
         }
     }
 
@@ -37,6 +44,20 @@ impl DeviceConfig {
     }
     */
 
+    pub fn contains(&self, ip: &Ipv4Addr) -> bool {
+        self.self_ip_net.load().contains(ip)
+    }
+
+    pub fn set_ip(&self, ip_net_addr: u32, ip_net_bit_len: u8) -> Result<()>{
+        if ip_net_bit_len > 32 {
+            return Err(SDLanError::NormalError("invalid ip net bit len"));
+        }   
+        self.ip.net_addr.store(ip_net_addr, Ordering::Relaxed);
+        self.ip.net_bit_len.store(ip_net_bit_len, Ordering::Relaxed);
+        self.self_ip_net.store(Arc::new(Ipv4Net::new(Ipv4Addr::from(ip_net_addr), ip_net_bit_len).unwrap()));
+        Ok(())
+    }
+
     pub fn get_ip(&self) -> u32 {
         self.ip.net_addr()
     }

src/network/mod.rs

@@ -23,6 +23,6 @@ pub const DNS_IP: u32 = (100<<24) + (100<<16) + (100<<8) + 100;
 #[cfg_attr(target_os = "linux", path = "tun_linux.rs")]
 #[cfg_attr(target_os = "windows", path = "tun_win.rs")]
 mod tun;
-pub use tun::{get_install_channel, restore_dns};
+pub use tun::{get_install_channel, restore_dns, arp_reply_arrived};
 
 mod device;

src/network/node.rs

@@ -1,21 +1,25 @@
+use arc_swap::ArcSwap;
 use dashmap::DashMap;
+use prost::Message;
 use quinn::Endpoint;
 use rsa::RsaPrivateKey;
 use sdlan_sn_rs::config::{AF_INET, AF_INET6};
 use tokio::net::UdpSocket;
+use std::any::Any;
+use std::future::Future;
 use std::net::SocketAddr;
 use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicU8, Ordering};
 use std::sync::{Arc, Mutex, RwLock};
 use std::time::Duration;
 use tokio::io::AsyncReadExt;
 use tokio::sync::mpsc::Sender;
-use tokio::sync::oneshot;
-use tracing::{debug, error};
+use tracing::{debug, error, warn};
 
+use crate::network::{ArpTable, RouteTable2};
 use crate::quic::quic_init;
-use crate::{ConnectionInfo, MyEncryptor, get_base_dir};
+use crate::{CommandLine, ConnectionInfo, MyEncryptor, RuleCache, get_base_dir};
 use crate::pb::{
-    encode_to_tcp_message, encode_to_udp_message, SdlEmpty, SdlStunProbe, SdlStunProbeReply,
+    SdlArpRequest, SdlEmpty, SdlStunProbe, SdlStunProbeReply, encode_to_tcp_message, encode_to_udp_message
 };
 use crate::tcp::{NatType, PacketType, StunProbeAttr, get_quic_write_conn};
 use crate::utils::{Socket};
@@ -36,11 +40,12 @@ static EDGE: OnceCell<Node> = OnceCell::new();
 pub async fn init_edge(
     // token: &str,
     // network_code: &str,
+    args: &CommandLine,
     mac: Mac,
     node_conf: NodeConfig,
-    tos: u32,
+    // tos: u32,
     start_stop: Sender<StartStopInfo>,
-    mtu: u32,
+    // mtu: u32,
     connecting_chan: Option<Sender<ConnectionInfo>>,
     udpsock_for_dns: Arc<UdpSocket>,
     hostname: String,
@@ -63,7 +68,7 @@ pub async fn init_edge(
     // let edge_uuid = create_or_load_uuid("")?;
     //let node_conf = parse_config(edge_uuid, &args).await?;
 
-    let Ok(sock_v4) = Socket::build(node_conf._local_port, true, false, tos).await else {
+    let Ok(sock_v4) = Socket::build(node_conf._local_port, true, false, args.tos).await else {
         panic!("failed to build sock for sock v4");
     };
 
@@ -88,13 +93,15 @@ pub async fn init_edge(
         privatekey,
         tcp_pong.clone(),
         start_stop,
-        mtu,
+        args.mtu,
         connecting_chan,
         hostname,
         udpsock_for_dns,
         server_ip,
         install_channel,
     );
+
+    edge.route_table.parse_and_add_route(&args.route_file, &args.route_str);
     do_init_edge(edge)?;
 
     Ok(())
@@ -162,13 +169,19 @@ impl IdentityID {
 pub struct Node {
     packet_id: AtomicU32,
 
-    pub encryptor: RwLock<MyEncryptor>,
+    pub encryptor: ArcSwap<MyEncryptor>,
 
     pub network_id: AtomicU32,
     pub network_domain: RwLock<String>,
 
     pub identity_id: IdentityID,
 
+    // rule is identity-to-identity ok
+    pub rule_cache: RuleCache,
+
+    // route_table stores the routes
+    pub route_table: RouteTable2,
+
     pub access_token: StringToken<String>,
     pub session_token: StringToken<Vec<u8>>,
     
@@ -205,6 +218,10 @@ pub struct Node {
 
     // store pending, and known peers
     pub pending_peers: PeerMap,
+
+    #[cfg(feature = "tun")]
+    pub arp_table: ArpTable,
+
     pub known_peers: PeerMap,
 
     // pub tcp_sock_v4: TCPSocket,
@@ -228,23 +245,18 @@ pub struct Node {
     nat_type: Mutex<NatType>,
 
     nat_cookie: AtomicU32,
-    cookie_match: DashMap<u32, oneshot::Sender<SdlStunProbeReply>>,
 
-    packet_id_match: DashMap<u32, oneshot::Sender<RegisterSuperFeedback>>,
+    //cookie_match: DashMap<u32, oneshot::Sender<SdlStunProbeReply>>,
+    pub cookie_match: Queryer,
+
+    // packet_id_match: DashMap<u32, oneshot::Sender<RegisterSuperFeedback>>,
 }
 
 unsafe impl Sync for Node {}
 
 impl Node {
     pub fn send_register_super_feedback(&self, pktid: u32, feed: RegisterSuperFeedback) {
-        match self.packet_id_match.remove(&pktid) {
-            Some(sender) => {
-                let _ = sender.1.send(feed);
-            }
-            None => {
-                return;
-            }
-        }
+        self.cookie_match.write_feedback(pktid, Box::new(feed));
     }
 
     pub fn get_nat_type(&self) -> NatType {
@@ -268,8 +280,12 @@ impl Node {
         }
 
         self.access_token.set(access_token);
-        self.device_config.ip.net_addr.store(ip_net, Ordering::Relaxed);
-        self.device_config.ip.net_bit_len.store(ip_net_bit_len, Ordering::Relaxed);
+
+        if let Err(_e) = self.device_config.set_ip(ip_net, ip_net_bit_len) {
+            error!("failed to set ip");
+        }
+        // self.device_config.ip.net_addr.store(ip_net, Ordering::Relaxed);
+        // self.device_config.ip.net_bit_len.store(ip_net_bit_len, Ordering::Relaxed);
         // *self.device_config.mac.write().unwrap() = create_or_load_mac();
         self.network_id.store(network_id, Ordering::Relaxed);
         self.network_domain.write().unwrap().clone_from(network_domain);
@@ -311,8 +327,9 @@ impl Node {
         }
 
         self.access_token.set(access_token);
-        self.device_config.ip.net_addr.store(ip_net, Ordering::Relaxed);
-        self.device_config.ip.net_bit_len.store(ip_net_bit_len, Ordering::Relaxed);
+        if let Err(_e) = self.device_config.set_ip(ip_net, ip_net_bit_len) {
+            error!("failed to set ip");
+        }
         // *self.device_config.mac.write().unwrap() = create_or_load_mac();
         self.network_id.store(network_id, Ordering::Relaxed);
         self.network_domain.write().unwrap().clone_from(network_domain);
@@ -320,30 +337,27 @@ impl Node {
         self.identity_id.store(identity_id);
         // *self._token.lock().unwrap() = token;
         // *self.network_code.lock().unwrap() = network_code;
-        let (tx, rx) = oneshot::channel();
         let id = self.get_next_packet_id();
-        self.packet_id_match.insert(id, tx);
-        let _ = self
-            .start_stop_sender
-            .send(StartStopInfo {
-                is_start: true,
-                pkt_id: Some(id),
-            })
-            .await;
-        debug!("start with feedback");
 
-        tokio::select! {
-            rx_info = rx => {
-                if let Ok(result) = rx_info {
-                    self.packet_id_match.remove(&id);
-                    Ok(result)
-                } else {
-                    Err(SDLanError::NormalError("rx closed"))
-                }
-            }
-            _ = tokio::time::sleep(timeout) => {
-                Err(SDLanError::NormalError("timed out"))
-            }
+        let res = self.cookie_match.do_action_and_wait_for(
+            0, 
+            || async {
+                let _ = self
+                    .start_stop_sender
+                    .send(StartStopInfo {
+                        is_start: true,
+                        pkt_id: Some(id),
+                    })
+                    .await;
+                debug!("start with feedback");
+            },
+            timeout
+        ).await?;
+
+        if let Ok(res) = res.downcast() {
+            Ok(*res)
+        } else {
+            Err(SDLanError::ConvertError("failed to convert feedback to RSFeedback".to_owned()))
         }
     }
 
@@ -385,12 +399,21 @@ impl Node {
         };
 
         Self {
+
+            #[cfg(feature = "tun")]
+            arp_table: ArpTable::new(),
+
             packet_id: AtomicU32::new(1),
-            encryptor: RwLock::new(MyEncryptor::new()),
+            encryptor: ArcSwap::from(Arc::new(MyEncryptor::new())),
+            // encryptor: RwLock::new(MyEncryptor::new()),
 
             network_id: AtomicU32::new(0),
             hostname: RwLock::new(hostname),
 
+            rule_cache: RuleCache::new(),
+
+            route_table: RouteTable2::new(),
+
             network_domain: RwLock::new(String::new()),
 
             udp_sock_for_dns: udpsock_for_dns,
@@ -443,9 +466,9 @@ impl Node {
             stats: NodeStats::new(),
             _last_register_req: AtomicU64::new(0),
 
-            packet_id_match: DashMap::new(),
+            // packet_id_match: DashMap::new(),
             nat_cookie: AtomicU32::new(1),
-            cookie_match: DashMap::new(),
+            cookie_match: Queryer::new(),
             server_ip,
             install_channel,
         }
@@ -526,12 +549,21 @@ impl Node {
     }
     */
 
+    pub async fn send_arp_request(&self, gw_ip: u32, real_ip: u32) -> Result<()>{
+        let arp_request = SdlArpRequest {
+            // pkt_id: id,
+            target_ip: gw_ip,
+            origin_ip: real_ip,
+            context: Vec::new(),
+        };
+        let msg = encode_to_tcp_message(Some(arp_request), PacketType::ArpRequest as u8).unwrap();
+        let conn = get_quic_write_conn();
+        let _ = conn.send(msg).await;
+        Ok(())
+    }
+
     pub async fn send_nat_probe_reply(&self, cookie: u32, buf: SdlStunProbeReply) {
-        if let Some((_key, chan)) = self.cookie_match.remove(&cookie) {
-            let _ = chan.send(buf);
-            return;
-        }
-        error!("failed to get such cookie stun probe");
+        self.cookie_match.write_feedback(cookie, Box::new(buf));
     }
 
     pub async fn probe_nat_type(&self) -> NatType {
@@ -614,33 +646,14 @@ impl Node {
             cookie,
             step: 0,
         };
+
+        let result = self.cookie_match.send_message_to_udp_and_wait_for(&self.udp_sock_v4, cookie, probe, PacketType::StunProbe as u8, to_server, Duration::from_secs(3)).await?;
+
+        if let Ok(res) = result.downcast() {
+            return Ok(*res);
+        }
+        return Err(SDLanError::ConvertError("failed to convert to StunprobeReply".to_owned()))
         // println!("==> sending probe request: {:?}", probe);
-
-        let (tx, rx) = oneshot::channel();
-        self.cookie_match.insert(cookie, tx);
-        // let cookie = msg.cookie;
-        let msg = encode_to_udp_message(Some(probe), PacketType::StunProbe as u8).unwrap();
-        if let Err(_e) = self.udp_sock_v4.send_to(&msg, to_server).await {
-            self.cookie_match.remove(&cookie);
-            return Err(SDLanError::NormalError("send error"));
-        }
-
-        tokio::select! {
-            _ = tokio::time::sleep(Duration::from_secs(3)) => {
-                self.cookie_match.remove(&cookie);
-                return Err(SDLanError::NormalError("timed out"));
-            }
-            reply = rx => {
-                self.cookie_match.remove(&cookie);
-                if let Ok(reply) = reply {
-                    // reply received,
-                    return Ok(reply);
-                    // println!("got nat ip: {}:{}", ip_to_string(&reply.ip), reply.port);
-                }
-                return Err(SDLanError::NormalError("reply recv error"));
-                // step 1 received
-            }
-        }
     }
 }
 
@@ -823,3 +836,106 @@ impl EdgePeer {
         self.nat_type
     }
 }
+
+type BoxedProstMessage = Box<dyn Any + Send + Sync + 'static>;
+
+pub struct Queryer {
+    pub mailbox: DashMap<u32, tokio::sync::oneshot::Sender<BoxedProstMessage>>,
+}
+
+impl Queryer {
+    pub fn new() -> Self {
+        Self {
+            mailbox: DashMap::new(),
+        }
+    }
+
+    pub fn write_feedback(&self, id: u32, data: BoxedProstMessage) {
+        if let Some((_, tx)) = self.mailbox.remove(&id) {
+            if let Err(_e) = tx.send(data) {
+                error!("failed to write feedback");
+            }
+        }
+    }
+
+    pub async fn send_message_to_udp_and_wait_for<T: Message>(&self, sock: &Socket, id: u32, message: T, packet_type: u8, to_server: &SocketAddr, timeout: Duration) -> Result<BoxedProstMessage> {
+        let (tx, rx) = tokio::sync::oneshot::channel();
+        self.mailbox.insert(id, tx);
+
+        let content = encode_to_udp_message(Some(message), packet_type)?;
+
+        if let Err(_e) = sock.send_to(&content, to_server).await {
+            self.mailbox.remove(&id);
+            return Err(SDLanError::NormalError("send error"));
+        }
+
+        tokio::select! {
+            data = rx => {
+                if let Ok(data) = data {
+                    self.mailbox.remove(&id);
+                    Ok(data)
+                } else {
+                    self.mailbox.remove(&id);
+                    Err(SDLanError::IOError("rx receive failed".to_string()))
+                }
+            }
+            _ = tokio::time::sleep(timeout) => {
+                self.mailbox.remove(&id);
+                Err(SDLanError::IOError("timed out".to_string()))
+            }
+        }
+
+    }
+
+    pub async fn do_action_and_wait_for<T, F>(&self, id: u32, action: T, timeout: Duration) -> Result<BoxedProstMessage> 
+    where 
+        F: Future<Output = ()>,
+        T: Fn() -> F,
+    {
+        let (tx, rx) = tokio::sync::oneshot::channel();
+        self.mailbox.insert(id, tx);
+
+        action().await;
+
+        tokio::select! {
+            data = rx => {
+                if let Ok(data) = data {
+                    self.mailbox.remove(&id);
+                    Ok(data)
+                } else {
+                    self.mailbox.remove(&id);
+                    Err(SDLanError::IOError("rx receive failed".to_string()))
+                }
+            }
+            _ = tokio::time::sleep(timeout) => {
+                self.mailbox.remove(&id);
+                Err(SDLanError::IOError("timed out".to_string()))
+            }
+        }
+    }
+
+    pub async fn send_message_to_quic_and_wait_for<T: Message>(&self, id: u32, message: T, packet_type: u8, timeout: Duration) -> Result<BoxedProstMessage> {
+        let (tx, rx) = tokio::sync::oneshot::channel();
+        self.mailbox.insert(id, tx);
+
+        let content = encode_to_tcp_message(Some(message), packet_type)?;
+        let quic_conn = get_quic_write_conn();
+        quic_conn.send(content).await?;
+
+        tokio::select! {
+            data = rx => {
+                if let Ok(data) = data {
+                    self.mailbox.remove(&id);
+                    Ok(data)
+                } else {
+                    self.mailbox.remove(&id);
+                    Err(SDLanError::IOError("rx receive failed".to_string()))
+                }
+            }
+            _ = tokio::time::sleep(timeout) => {
+                self.mailbox.remove(&id);
+                Err(SDLanError::IOError("timed out".to_string()))
+            }
+        }
+    }
+}

src/network/packet.rs

@@ -1,7 +1,8 @@
 use std::{net::SocketAddr, sync::atomic::Ordering, time::Duration};
 
+use crate::FiveTuple;
 use crate::pb::SdlPolicyRequest;
-use crate::tcp::{NatType, get_quic_write_conn, is_identity_ok};
+use crate::tcp::{NatType, get_quic_write_conn};
 use crate::{network::TunTapPacketHandler, utils::mac_to_string};
 
 use crate::{
@@ -13,7 +14,8 @@ use crate::{
     tcp::{PacketType},
     utils::{send_to_sock, Socket},
 };
-use etherparse::{Ethernet2Header, PacketHeaders, ip_number};
+use bytes::BytesMut;
+use etherparse::{Ethernet2Header, IpNumber, PacketHeaders, ip_number};
 use prost::Message;
 use sdlan_sn_rs::utils::{BROADCAST_MAC};
 use sdlan_sn_rs::{
@@ -822,7 +824,7 @@ pub fn print_hex(key: &[u8]) {
 
 async fn renew_identity_request(eee: &Node, identity: u32) {
     let policy_request = SdlPolicyRequest {
-        pkt_id: eee.get_next_packet_id(),
+        // pkt_id: eee.get_next_packet_id(),
         src_identity_id: identity,
         dst_identity_id: eee.identity_id.load(),
         version: 1,
@@ -843,38 +845,6 @@ async fn renew_identity_request(eee: &Node, identity: u32) {
     }
 }
 
-async fn check_identity_is_ok(eee: &Node, identity: u32, protocol: u8, port: u16) -> bool{
-    true
-}
-
-async fn check_identity_is_ok2(eee: &Node, identity: u32, protocol: u8, port: u16) -> bool{
-    let result = is_identity_ok(identity, protocol, port);
-    if result.1 {
-        renew_identity_request(eee, identity).await;
-    }
-    match result.0 {
-        Some(true) => {
-            // identity is ok
-            true
-        }
-        Some(false) => {
-            // identity is not allowed
-            warn!("identity is not allowed for protocol={:?}, port={}", protocol, port);
-            false
-        }
-        None => {
-            if !result.1 {
-                renew_identity_request(eee, identity).await;
-            } else {
-                // has been sent
-            }
-            false
-            // no such identity, should request for it
-        }
-    }
-
-}
-
 async fn handle_tun_packet(
     eee: &Node,
     _from_sock: &SdlanSock,
@@ -892,7 +862,8 @@ async fn handle_tun_packet(
 
     // test_aes(key.as_slice());
 
-    let origin = eee.encryptor.read().unwrap().decrypt(&payload);
+    let origin = eee.encryptor.load().decrypt(&payload);
+    // let origin = eee.encryptor.read().unwrap().decrypt(&payload);
     // let origin = aes_decrypt(&payload);
     if let Err(_e) = origin {
         error!("failed to decrypt original data");
@@ -918,17 +889,35 @@ async fn handle_tun_packet(
                 match protocol {
                     ip_number::TCP => {
                         let tcp_header = headers.transport.unwrap().tcp().unwrap();
-                        let port = tcp_header.destination_port;
-                        let src_port = tcp_header.source_port;
-                        println!("tcp srcport={}, dstport={}", src_port, port);
-                        if !check_identity_is_ok(eee, pkt.identity_id, protocol.0, port).await {
+                        let five_tuple = FiveTuple {
+                            src_ip: ipv4.destination.into(),
+                            dst_ip: ipv4.source.into(),
+                            src_port: tcp_header.destination_port,
+                            dst_port: tcp_header.source_port,
+                            proto:IpNumber::TCP.0,
+                        };
+                        let (valid, need_refresh) = eee.rule_cache.is_identity_ok(pkt.identity_id, five_tuple);
+                        if need_refresh {
+                            renew_identity_request(eee, pkt.identity_id).await;
+                        }
+                        if !valid {
                             return;
                         }
                     }
                     ip_number::UDP => {
                         let udp_header = headers.transport.unwrap().udp().unwrap();
-                        let port = udp_header.destination_port;
-                        if !check_identity_is_ok(eee, pkt.identity_id, protocol.0, port).await {
+                        let five_tuple = FiveTuple {
+                            src_ip: ipv4.destination.into(),
+                            dst_ip: ipv4.source.into(),
+                            src_port: udp_header.destination_port,
+                            dst_port: udp_header.source_port,
+                            proto:IpNumber::UDP.0,
+                        };
+                        let (valid, need_refresh) = eee.rule_cache.is_identity_ok(pkt.identity_id, five_tuple);
+                        if need_refresh {
+                            renew_identity_request(eee, pkt.identity_id).await;
+                        }
+                        if !valid {
                             return;
                         }
                     }
@@ -1240,7 +1229,7 @@ async fn send_query_peer(eee: &Node, dst_mac: Mac) -> Result<()> {
     }
     let query = SdlQueryInfo {
         dst_mac: Vec::from(dst_mac),
-        pkt_id: 0,
+        // pkt_id: 0,
     };
 
     let Ok(content) = encode_to_tcp_message(
@@ -1304,13 +1293,19 @@ pub async fn update_supernode_reg(eee: &Node) {
 */
 
 #[allow(unused)]
-pub fn form_ethernet_packet(src_mac: Mac, dst_mac: Mac, data: &[u8]) -> Vec<u8> {
+pub fn form_ethernet_packet(src_mac: Mac, dst_mac: Mac, mut data_with_zeroed_layer2: BytesMut) -> BytesMut {
     let mut etherheader = Ethernet2Header::default();
     etherheader.destination = dst_mac;
     etherheader.ether_type = etherparse::EtherType::IPV4;
     etherheader.source = src_mac;
-    let mut packet = Vec::with_capacity(14 + data.len() + 4);
-    packet.extend_from_slice(&etherheader.to_bytes()[..]);
-    packet.extend_from_slice(&data);
-    packet
+
+    let data = data_with_zeroed_layer2.split_off(14);
+    data_with_zeroed_layer2.copy_from_slice(&etherheader.to_bytes()[..]);
+
+    data_with_zeroed_layer2.unsplit(data);
+    data_with_zeroed_layer2
+    // let mut packet = Vec::with_capacity(14 + data.len() + 4);
+    // packet.extend_from_slice(&etherheader.to_bytes()[..]);
+    // packet.extend_from_slice(&data);
+    // packet
 }

src/network/route.rs

@@ -1,127 +1,116 @@
-use std::{net::Ipv4Addr, sync::RwLock};
+use std::{collections::HashMap, fs, io::{BufRead, BufReader}, net::Ipv4Addr, sync::atomic::{AtomicBool, Ordering}, time::Duration};
 
-use once_cell::sync::OnceCell;
-use sdlan_sn_rs::utils::net_bit_len_to_mask;
+use ahash::RandomState;
+use dashmap::{DashMap};
+use ipnet::Ipv4Net;
+use sdlan_sn_rs::utils::{Result, SDLanError};
 
+use tokio::{io::ReadBuf, sync::oneshot::{Receiver, Sender, channel}};
 use tracing::{debug, error};
 
-#[derive(Debug)]
-pub struct RouteTable {
-    content: RwLock<Vec<RouteInfo>>,
+use crate::{RouteTableTrie, network::tun::{add_route, del_route}, pb::{SdlArpResponse, SdlStunReply}};
+
+
+pub struct RouteTable2 {
+    pub cache_table: DashMap<(Ipv4Net, Ipv4Addr), AtomicBool, RandomState>,
+    pub route_table: RouteTableTrie,
 }
 
-static ROUTETABLE: OnceCell<RouteTable> = OnceCell::new();
-
-pub fn init_route() {
-    let rt = RouteTable::new();
-    ROUTETABLE.set(rt).unwrap();
-}
-
-pub fn get_route_table() -> &'static RouteTable {
-    ROUTETABLE.get().unwrap()
-}
-
-impl RouteTable {
+impl RouteTable2 {
     pub fn new() -> Self {
         Self {
-            content: RwLock::new(Vec::new()),
+            cache_table: DashMap::with_hasher(RandomState::new()),
+            route_table: RouteTableTrie::new(),
         }
     }
 
-    pub fn get_gateway_ip(&self, net_ip: u32) -> Option<u32> {
-        let routes = self.content.read().unwrap();
-        for route in &*routes {
-            debug!("route: {:?}", route.to_string());
-            if (route.net_ip & route.net_mask) == (net_ip & route.net_mask) {
-                // found
-                return Some(route.gateway_ip);
+    pub fn parse_and_add_route(&self, route_file: &str, route_str: &str) -> Result<()> {
+        let routes = match true {
+            _ if route_str.len() != 0 => {
+                parse_route(route_str)
+            }
+            _ if route_file.len() != 0 => {
+                load_route_file(route_file)
+            }
+            _ => {
+                Ok(HashMap::new())
+            }
+        }?;
+        for route in routes.keys() {
+            if self.cache_table.get(route).is_some() {
+                error!("route {} {} has been added", route.0.to_string(), route.1);
+                return Err(SDLanError::IOError(format!("route {} already added", route.0.to_string())));
             }
         }
-        None
-    }
 
-    pub fn del_route(&self, net_ip: u32, net_mask: u32) {
-        let mut routes = self.content.write().unwrap();
-        let mut remove_idx = routes.len();
-        for i in 0..routes.len() {
-            let route = &routes[i];
-            if route.net_ip == net_ip && route.net_mask == net_mask {
-                remove_idx = i;
-                break;
-            }
+        for route in routes.keys() {
+            self.cache_table.insert(*route, AtomicBool::new(false));
+            self.route_table.insert(route.0.addr().into(), route.0.prefix_len(), route.1);
         }
-        if remove_idx < routes.len() {
-            routes.remove(remove_idx);
-        }
-    }
-
-    pub fn add_route(&self, net_ip: u32, net_mask: u32, gateway_ip: u32) -> Result<(), String> {
-        {
-            let cnt = self.content.read().unwrap();
-            let net = net_ip & net_mask;
-            for route in &*cnt {
-                if (route.net_ip & route.net_mask) == net {
-                    return Err("route exists".to_owned());
-                }
-            }
-        }
-        {
-            let mut routes = self.content.write().unwrap();
-            routes.push(RouteInfo {
-                net_ip,
-                net_mask,
-                gateway_ip,
-            })
-        }
-
         Ok(())
     }
-}
 
-#[derive(Debug)]
-pub struct RouteInfo {
-    pub net_ip: u32,
-    pub net_mask: u32,
-    pub gateway_ip: u32,
-}
-
-impl RouteInfo {
-    pub fn to_string(&self) -> String {
-        format!(
-            "{:?} mask={:?}, gateway={:?}",
-            self.net_ip.to_be_bytes(),
-            self.net_mask.to_be_bytes(),
-            self.gateway_ip.to_be_bytes()
-        )
+    pub fn apply_system(&self) {
+        for route in &self.cache_table {
+            let (net, gw) = route.key();
+            if let Err(e) = del_route(net, gw) {
+                error!("failed to del route: {}", e.as_str());
+            }
+                // should add to system
+            if let Err(e) = add_route(net, gw) {
+                error!("failed to add route: {}", e.as_str());
+            }
+        }
     }
 }
 
-// ip, mask, gateway, cidr;gateway,cidr2;gateway2
-pub fn parse_route(route: String) -> Vec<(u32, u32, u32)> {
-    let mut result = Vec::new();
-    let routes: Vec<_> = route.split(",").collect();
-    for route in routes {
-        let route_info: Vec<_> = route.split(";").collect();
-        debug!("got route info: {:?}", route_info);
-        if route_info.len() != 2 {
-
-            error!("route info format error");
-            continue;
+/// 路由文件，应该是以下格式，一行一条
+///     network-cidr gw
+///     network-cidr gw
+pub fn load_route_file(filename: &str) -> Result<HashMap<(Ipv4Net, Ipv4Addr), bool>> {
+    let mut result = HashMap::new();
+    let fp = fs::File::open(filename)?;
+    
+    let buf = BufReader::new(fp);
+    for line in buf.lines() {
+        let line = line?;
+        if let Ok(rt) = parse_a_route(&line) {
+            result.insert(rt, true);
         }
-        let cidr = route_info[0];
-        let gateway = route_info[1].parse::<Ipv4Addr>().unwrap();
-        let ip_and_mask: Vec<_> = cidr.split("/").collect();
-        if ip_and_mask.len() != 2 {
-            error!("route info ip/bit error");
-            continue;
-        }
-        let ip = ip_and_mask[0].parse::<Ipv4Addr>().unwrap();
-        let maskbit = ip_and_mask[1].parse::<u8>().unwrap();
-        result.push((
-            u32::from_be_bytes(ip.octets()),
-            net_bit_len_to_mask(maskbit),
-            u32::from_be_bytes(gateway.octets()),
-        ));
     }
-    result
+    Ok(result)
+}
+
+pub fn parse_a_route(route: &str) -> Result<(Ipv4Net, Ipv4Addr)> {
+    let route_info: Vec<_> = route.trim().split_whitespace().collect();
+    if route_info.len() != 2 {
+        error!("route info format error: {}", route);
+        return Err(SDLanError::IOError("route format error".to_string()));
+    }
+    debug!("got route info: {:?}", route_info);
+
+    let Ok(gateway) = route_info[1].parse::<Ipv4Addr>() else {
+        error!("failed to parse gw: {}", route_info[1]);
+        return Err(SDLanError::IOError("failed to parse gw".to_string()));
+    };
+
+    let cidr = route_info[0];
+    let Ok(net )= cidr.parse::<Ipv4Net>()  else {
+        error!("failed to parse cidr: {}, skipping", cidr);
+        return Err(SDLanError::IOError("failed to parse cidr".to_string()));
+    };
+    Ok((net, gateway))
+}
+
+// ip, mask, gateway, cidr gateway,cidr2 gateway2
+pub fn parse_route(route: &str) -> Result<HashMap<(Ipv4Net, Ipv4Addr), bool>> {
+    let mut result = HashMap::new();
+    // let routes: Vec<_> = route.split(",").collect();
+
+    for route in route.trim().split(",") {
+        if let Ok(rt) = parse_a_route(route) {
+            result.insert(rt, true);
+        }
+    }
+    Ok(result)
 }

src/network/tun_linux.rs

@@ -1,4 +1,9 @@
+#[cfg(not(feature = "tun"))]
+use bytes::BytesMut;
+#[cfg(feature = "tun")]
+use bytes::{Bytes, BytesMut};
 use etherparse::{Ethernet2Header};
+use ipnet::Ipv4Net;
 use sdlan_sn_rs::config::SDLAN_DEFAULT_TTL;
 use sdlan_sn_rs::utils::{
     ip_to_string, is_ipv6_multicast, net_bit_len_to_mask,
@@ -7,6 +12,7 @@ use sdlan_sn_rs::utils::{
 use std::ffi::CStr;
 use std::ffi::{c_char, c_int};
 use std::fs::{self, OpenOptions};
+use std::net::Ipv4Addr;
 use std::os::unix::fs::{MetadataExt, PermissionsExt};
 use std::path::Path;
 use std::ptr::null_mut;
@@ -19,8 +25,16 @@ use std::process::Command;
 
 use tracing::{debug, error, info};
 
+#[cfg(feature = "tun")]
+use crate::caculate_crc;
 use crate::get_edge;
-use crate::network::send_packet_to_net;
+#[cfg(not(feature = "tun"))]
+use crate::network::{ARP_REPLY, ArpHdr, EthHdr};
+use crate::network::{Node, send_packet_to_net};
+#[cfg(not(feature = "tun"))]
+use crate::pb::SdlArpResponse;
+#[cfg(feature = "tun")]
+use crate::pb::SdlArpResponse;
 use crate::pb::{encode_to_udp_message, SdlData};
 use crate::tcp::PacketType;
 
@@ -111,7 +125,7 @@ impl Iface {
         }
     }
 
-    pub fn reload_config(&self, device_config: &DeviceConfig, network_domain: &str) {
+    pub fn reload_config(&self, node: &Node, device_config: &DeviceConfig, network_domain: &str) {
         let netbit = device_config.get_net_bit();
         let ip = device_config.get_ip();
         if netbit == 0 || ip == 0 {
@@ -154,6 +168,12 @@ impl Iface {
                 }
             }
 
+            node.route_table.apply_system();
+
+            if node.config.allow_routing {
+                set_allow_routing();
+            }
+
             // TODO: set dns should be opened
             /* 
             if let Err(e) = set_dns(self, &self.name, network_domain, &ip_to_string(&default_gw)) {
@@ -210,7 +230,7 @@ impl TunTapPacketHandler for Iface {
 
     async fn handle_packet_from_device(
         &self,
-        data: Vec<u8>,
+        data: BytesMut,
         // encrypt_key: &[u8],
     ) -> std::io::Result<()> {
         use etherparse::PacketHeaders;
@@ -235,9 +255,17 @@ impl TunTapPacketHandler for Iface {
                     let arp = ArpHdr::from_slice(&data);
                     match arp.opcode {
                         ARP_REQUEST => {
-                            use crate::{network::ArpRequest, pb::{SdlArpRequest, encode_to_tcp_message}, tcp::get_quic_write_conn};
                             let dest_ip = ((arp.dipaddr[0] as u32) << 16) + arp.dipaddr[1] as u32;
 
+                            if edge.device_config.contains(&Ipv4Addr::from_bits(dest_ip)) {
+                                let _ = edge.send_arp_request(dest_ip,  dest_ip).await;
+                            } else {
+                                if let Some((_, real_ip)) = edge.route_table.route_table.lookup(dest_ip) {
+                                    let real_ip = u32::from_be_bytes(real_ip.octets());
+                                    let _ = edge.send_arp_request(dest_ip, real_ip).await;
+                                }
+                            }
+                            /* 
                             let request = SdlArpRequest {
                                 pkt_id: edge.get_next_packet_id(),
                                 target_ip: dest_ip,
@@ -247,6 +275,7 @@ impl TunTapPacketHandler for Iface {
                             let conn = get_quic_write_conn();
                             debug!("sending arp request");
                             let _ = conn.send(req).await;
+                            */
                             return Ok(());
                         }
                         _other => {
@@ -260,10 +289,40 @@ impl TunTapPacketHandler for Iface {
                         etherparse::NetHeaders::Ipv4(ipv4, _) => {
                             use etherparse::ip_number::{self, ICMP};
 
-
+                            use crate::FiveTuple;
+                            use etherparse::IpNumber;
+                            
                             if let Some(transport) = headers.transport {
-                                if let Some(tcp) = transport.tcp() {
+                                match ipv4.protocol {
+                                    IpNumber::TCP => {
+                                        let tcp = transport.tcp().unwrap();
+
+                                        let out_five_tuple = FiveTuple {
+                                            src_ip: ipv4.source.into(),
+                                            dst_ip: ipv4.destination.into(),
+                                            src_port: tcp.source_port,
+                                            dst_port: tcp.destination_port,
+                                            proto: IpNumber::TCP.0,
+                                        };
+
+                                        edge.rule_cache.touch_packet(out_five_tuple);
                                     // is tcp
+                                    }
+                                    IpNumber::UDP => {
+                                        let udp = transport.udp().unwrap();
+
+                                        let out_five_tuple = FiveTuple {
+                                            src_ip: ipv4.source.into(),
+                                            dst_ip: ipv4.destination.into(),
+                                            src_port: udp.source_port,
+                                            dst_port: udp.destination_port,
+                                            proto: IpNumber::UDP.0,
+                                        };
+                                        edge.rule_cache.touch_packet(out_five_tuple);
+                                    }
+                                    _other => {
+
+                                    }
                                 }
                             }
 
@@ -289,7 +348,8 @@ impl TunTapPacketHandler for Iface {
                 }
                 let size = data.len();
 
-                let Ok(encrypted) = edge.encryptor.read().unwrap().encrypt(&data) else {
+                let Ok(encrypted) = edge.encryptor.load().encrypt(&data) else {
+                // let Ok(encrypted) = edge.encryptor.read().unwrap().encrypt(&data) else {
                 // let Ok(encrypted) = aes_encrypt(encrypt_key, &data) else {
                     error!("failed to encrypt packet request");
                     return Ok(());
@@ -380,6 +440,11 @@ impl TunTapPacketHandler for Iface {
                                 use bytes::Bytes;
                                 use sdlan_sn_rs::utils::mac_to_string;
 
+                                use crate::network::ARP_REPLY;
+
+                                edge.arp_table.set(from_ip, arp.shwaddr);
+
+                                /* 
                                 use crate::network::{ARP_REPLY, ArpRequestInfo, send_arp_request};
 
                                 send_arp_request(ArpRequestInfo::Set {
@@ -387,6 +452,7 @@ impl TunTapPacketHandler for Iface {
                                     mac: arp.shwaddr,
                                 })
                                 .await;
+                                */
 
                                 // target to us
                                 arp.opcode = ARP_REPLY;
@@ -402,7 +468,7 @@ impl TunTapPacketHandler for Iface {
 
                                 let data = arp.marshal_to_bytes();
                                 // let Ok(encrypted) = aes_encrypt(key, &data) else {
-                                let Ok(encrypted) = edge.encryptor.read().unwrap().encrypt(&data) else {
+                                let Ok(encrypted) = edge.encryptor.load().encrypt(&data) else {
                                     error!("failed to encrypt arp reply");
                                     return Ok(());
                                 };
@@ -436,6 +502,7 @@ impl TunTapPacketHandler for Iface {
 
                             debug!("mac {:?} is at {:?}", arp.shwaddr, from_ip.to_be_bytes());
                             if dest_ip == self_ip {
+                                /* 
                                 use crate::network::{ArpRequestInfo, arp_arrived, send_arp_request};
 
                                 send_arp_request(ArpRequestInfo::Set {
@@ -443,7 +510,11 @@ impl TunTapPacketHandler for Iface {
                                     mac: arp.shwaddr,
                                 })
                                 .await;
-                                arp_arrived(from_ip, arp.shwaddr).await;
+                                */
+
+                                // use crate::network::arp_arrived;
+                                edge.arp_table.set(from_ip, arp.shwaddr);
+                                edge.arp_table.arp_arrived(from_ip, arp.shwaddr).await;
                             }
                         }
                         _other => {
@@ -462,9 +533,10 @@ impl TunTapPacketHandler for Iface {
                             let ip = u32::from_be_bytes(ipv4.0.source);
                             let mac = hdr.source;
                             if !is_multi_broadcast(&mac) {
-                                use crate::network::{ArpRequestInfo, send_arp_request};
+                                //use crate::network::{ArpRequestInfo, send_arp_request};
 
-                                send_arp_request(ArpRequestInfo::Set { ip, mac }).await;
+                                edge.arp_table.set(ip, mac);
+                                // send_arp_request(ArpRequestInfo::Set { ip, mac }).await;
                             }
                         }
                         Err(_) => {
@@ -497,7 +569,7 @@ impl TunTapPacketHandler for Iface {
 
     async fn handle_packet_from_device(
         &self,
-        data: Vec<u8>,
+        mut header: BytesMut,
     ) -> std::io::Result<()> {
         use etherparse::IpHeaders;
 
@@ -505,9 +577,11 @@ impl TunTapPacketHandler for Iface {
 
         let src_mac = eee.device_config.get_mac();
 
+        let data = header.split_off(14);
+
         match IpHeaders::from_slice(&data) {
             Ok((iphdr, _payload)) => {
-                use crate::network::{ArpRequestInfo, ArpResponse, send_arp_request};
+                //use crate::network::{ArpRequestInfo, ArpResponse, send_arp_request};
 
                 let Some(ipv4hdr) = iphdr.ipv4() else {
                     debug!("ipv6 packet ignored");
@@ -537,75 +611,36 @@ impl TunTapPacketHandler for Iface {
                     }
                     return Ok(());
                 }
-                match send_arp_request(ArpRequestInfo::Lookup { ip: dstip }).await {
-                    ArpResponse::LookupResp {
-                        mac,
-                        ip,
-                        do_arp_request,
-                    } => {
-                        use bytes::Bytes;
+                match eee.arp_table.get(dstip) {
+                    Some(mac) => {
 
-                        use crate::utils::caculate_crc;
-
-                        if do_arp_request {
-                            use sdlan_sn_rs::utils::BROADCAST_MAC;
-
-                            use crate::network::{add_to_arp_wait_list, generate_arp_request};
-
-                            add_to_arp_wait_list(dstip, data);
-                            debug!(
-                                "find ip: {:?} => {:?}",
-                                src.to_be_bytes(),
-                                dstip.to_be_bytes()
-                            );
-                            let arp_msg =
-                                generate_arp_request(src_mac, ip, eee.device_config.get_ip());
-                            // let Ok(encrypted) = aes_encrypt(&encrypt_key, &arp_msg) else {
-                            let Ok(encrypted) = eee.encryptor.read().unwrap().encrypt(&arp_msg) else {
-                                error!("failed to encrypt arp request");
-                                return Ok(());
-                            };
-                            // println!("arp_msg: {:?}", arp_msg);
-                            let data = SdlData {
-                                network_id: eee.network_id.load(Ordering::Relaxed),
-                                src_mac: Vec::from(src_mac),
-                                dst_mac: Vec::from([0xff; 6]),
-                                is_p2p: true,
-                                ttl: SDLAN_DEFAULT_TTL as u32,
-                                data: Bytes::from(encrypted),
-                                session_token: eee.session_token.get(),
-                                identity_id: eee.identity_id.load(),
-                            };
-                            let data =
-                                encode_to_udp_message(Some(data), PacketType::Data as u8).unwrap();
-                            debug!("sending arp");
-                            // let data = marshal_message(&data);
-                            send_packet_to_net(eee, BROADCAST_MAC, &data, arp_msg.len() as u64)
-                                .await;
-                            // edge.sock.send(data).await;
-                            // println!("should send arp");
-                            return Ok(());
-                        }
-
-                        // prepend the ether header
+                        let pkt_size = data.len() + 14;
                         let mut etherheader = Ethernet2Header::default();
                         etherheader.destination = mac;
                         etherheader.ether_type = etherparse::EtherType::IPV4;
                         etherheader.source = src_mac;
-                        let mut packet = Vec::with_capacity(14 + data.len() + 4);
-                        packet.extend_from_slice(&etherheader.to_bytes()[..]);
-                        packet.extend_from_slice(&data);
-                        let crc = caculate_crc(&packet);
-                        packet.extend_from_slice(&crc.to_be_bytes());
+                        // let mut packet = Vec::with_capacity(14 + data.len() + 4);
 
-                        let pkt_size = packet.len();
+                        header.copy_from_slice(&etherheader.to_bytes()[..]);
+
+                        let crc = caculate_crc(&data);
+                        header.unsplit(data);
+
+
+                        // packet.extend_from_slice(&etherheader.to_bytes()[..]);
+                        // packet.extend_from_slice(&data);
+                        header.extend_from_slice(&crc.to_be_bytes());
+                        // packet.extend_from_slice(&crc.to_be_bytes());
+
+                        // let pkt_size = packet.len();
                         // println!("sending data with mac");
 
                         // let Ok(encrypted) = aes_encrypt(&encrypt_key, &packet) else {
-                        let Ok(encrypted) = eee.encryptor.read().unwrap().encrypt(&packet) else {
+                        let Ok(encrypted) = eee.encryptor.load().encrypt(&header) else {
                             error!("failed to encrypt packet request");
                             return Ok(());
                         };
+
                         let data = SdlData {
                             is_p2p: true,
                             network_id: eee.network_id.load(Ordering::Relaxed),
@@ -620,10 +655,30 @@ impl TunTapPacketHandler for Iface {
                             encode_to_udp_message(Some(data), PacketType::Data as u8).unwrap();
                         let size = msg.len();
                         send_packet_to_net(eee, mac, &msg, pkt_size as u64).await;
-                        // let dstip = u32::from_be_bytes(ipv4hdr.0.destination);
                     }
-                    _ => {}
+                    None => {
+                        header.unsplit(data);
+                        eee.arp_table.add_to_arp_wait_list(dstip, header);
+                        debug!(
+                            "find ip: {:?} => {:?}",
+                            src.to_be_bytes(),
+                            dstip.to_be_bytes()
+                        );
+                        debug!("no mac found for ip {:?}, sending arp request", dstip.to_be_bytes());
+                        // let _ = eee.send_arp_request(dstip, dstip).await;
+
+                        if eee.device_config.contains(&Ipv4Addr::from_bits(dstip)) {
+                            let _ = eee.send_arp_request(dstip,  dstip).await;
+                        } else {
+                            if let Some((_, real_ip)) = eee.route_table.route_table.lookup(dstip) {
+                                let real_ip = u32::from_be_bytes(real_ip.octets());
+                                let _ = eee.send_arp_request(dstip, real_ip).await;
+                            }
+                        }
+
+                    }
                 }
+
             }
             Err(e) => {
                 error!("failed to parse ip packet: {}", e.to_string());
@@ -851,4 +906,109 @@ fn restore_resolv_conf() -> Result<()> {
     }
 
     Ok(())
+}
+
+pub fn del_route(net: &Ipv4Net, gw: &Ipv4Addr) -> Result<()> {
+    let res = Command::new("route")
+        .arg("del")
+        .arg("-net")
+        .arg(net.to_string())
+        .arg("gw")
+        .arg(gw.to_string())
+        .output()?;
+
+    Ok(())
+}
+
+
+pub fn add_route(net: &Ipv4Net, gw: &Ipv4Addr) -> Result<()> {
+    let res = Command::new("route")
+        .arg("add")
+        .arg("-net")
+        .arg(net.to_string())
+        .arg("gw")
+        .arg(gw.to_string())
+        .output()?;
+
+    Ok(())
+}
+
+
+fn set_allow_routing() {
+    let _ = Command::new("sysctl")
+        .arg("-w")
+        .arg("net.ipv4.ip_forward=1")
+        .output();
+
+    let _ = Command::new("iptables")
+        .arg("-t")
+        .arg("nat")
+        .arg("-D")
+        .arg("POSTROUTING")
+        .arg("-j")
+        .arg("MASQUERADE")
+        .output();
+
+    let _ = Command::new("iptables")
+        .arg("-t")
+        .arg("nat")
+        .arg("-A")
+        .arg("POSTROUTING")
+        .arg("-j")
+        .arg("MASQUERADE")
+        .output();
+}
+
+#[cfg(feature = "tun")]
+pub async fn arp_reply_arrived(edge: &Node, data: SdlArpResponse) {
+    debug!("got arp response: {:?}", data);
+    if data.target_mac.len() != 6 {
+        // invalid target_mac
+        error!("invalid target_mac: {:?}, ip={}", data.target_mac, ip_to_string(&data.target_ip));
+        return;
+    }
+
+    let ip = data.target_ip;
+    let mac = data.target_mac.try_into().unwrap();
+    
+    edge.arp_table.arp_arrived(ip, mac).await;
+}
+
+#[cfg(not(feature = "tun"))]
+pub async fn arp_reply_arrived(edge: &Node, data: SdlArpResponse) {
+    debug!("got arp response: {:?}", data);
+    if data.target_mac.len() != 6 {
+        // invalid target_mac
+        error!("invalid target_mac: {:?}, ip={}", data.target_mac, ip_to_string(&data.target_ip));
+        return;
+    }
+
+    // TODO: construct the arp reply, and write to tun;
+    let src_mac = data.target_mac.try_into().unwrap();
+    let dst_mac = edge.device_config.get_mac();
+    let dst_ip = edge.device_config.get_ip();
+
+    let src_ip = data.origin_ip;
+
+    let hdr = ArpHdr{
+        ethhdr: EthHdr {
+            dest: dst_mac,
+            src: src_mac,
+            eth_type: 0x0806,
+        },
+        hwtype: 0x0001,
+        protocol: 0x0800,
+        hwlen: 6,
+        protolen: 4,
+        opcode: ARP_REPLY,
+        shwaddr: src_mac,
+        sipaddr: [((src_ip >> 16) as u16) & 0xffff, (src_ip as u16) & 0xffff],
+        dhwaddr: dst_mac,
+        dipaddr: [((dst_ip >> 16) & 0x0000ffff) as u16, (dst_ip & 0x0000ffff) as u16]
+    };
+
+    let data = hdr.marshal_to_bytes();
+    if let Err(_e) = edge.device.send(&data) {
+        error!("failed to write arp response to device");
+    }
 }

src/network/tun_win.rs

@@ -1,11 +1,13 @@
 use bytes::Bytes;
 use etherparse::ether_type::ARP;
 use etherparse::{Ethernet2Header, IpHeaders};
+use ipnet::Ipv4Net;
 use sdlan_sn_rs::config::SDLAN_DEFAULT_TTL;
 use sdlan_sn_rs::utils::{
-    aes_encrypt, ip_to_string, is_multi_broadcast, net_bit_len_to_mask, BROADCAST_MAC,
+    BROADCAST_MAC, Result, aes_encrypt, ip_to_string, is_multi_broadcast, net_bit_len_to_mask
 };
 use std::io::{Error, ErrorKind};
+use std::net::Ipv4Addr;
 use std::os::windows::process::CommandExt;
 use std::process::Command;
 use std::sync::atomic::Ordering;
@@ -15,7 +17,7 @@ use wintun;
 
 use crate::get_edge;
 use crate::network::{
-    ARP_REPLY, ARP_REQUEST, ArpHdr, ArpRequestInfo, ArpResponse, DNS_IP, add_to_arp_wait_list, arp_arrived, form_ethernet_packet, generate_arp_request, send_arp_request, send_packet_to_net
+    ARP_REPLY, ARP_REQUEST, ArpHdr, ArpRequestInfo, ArpResponse, DNS_IP, Node, add_to_arp_wait_list, arp_arrived, form_ethernet_packet, generate_arp_request, send_arp_request, send_packet_to_net
 };
 use crate::pb::{encode_to_udp_message, SdlData};
 use crate::tcp::PacketType;
@@ -58,7 +60,7 @@ impl Iface {
         Ok(content.len())
     }
 
-    pub fn reload_config(&self, device_config: &DeviceConfig, network_domain: &str) {
+    pub fn reload_config(&self, node: &Node, device_config: &DeviceConfig, network_domain: &str) {
         let netbit = device_config.get_net_bit();
         let ip = device_config.get_ip();
         if netbit == 0 || ip == 0 {
@@ -202,7 +204,7 @@ impl TunTapPacketHandler for Iface {
 
 
                                 // let Ok(encrypted) = aes_encrypt(key, &data) else {
-                                let Ok(encrypted) = edge.encryptor.read().unwrap().encrypt(&data) else {
+                                let Ok(encrypted) = edge.encryptor.load().encrypt(&data) else {
                                     error!("failed to encrypt arp reply");
                                     return Ok(());
                                 };
@@ -340,7 +342,7 @@ impl TunTapPacketHandler for Iface {
                             let arp_msg =
                                 generate_arp_request(src_mac, ip, eee.device_config.get_ip());
                             
-                            let Ok(encrypted) = eee.encryptor.read().unwrap().encrypt(&arp_msg) else {
+                            let Ok(encrypted) = eee.encryptor.load().encrypt(&arp_msg) else {
                             // let Ok(encrypted) = aes_encrypt(&encrypt_key, &arp_msg) else {
                                 error!("failed to encrypt arp request");
                                 return Ok(());
@@ -386,7 +388,7 @@ impl TunTapPacketHandler for Iface {
                         // println!("sending data with mac");
 
                         // let Ok(encrypted) = aes_encrypt(&encrypt_key, &packet) else {
-                        let Ok(encrypted) = eee.encryptor.read().unwrap().encrypt(&packet) else {
+                        let Ok(encrypted) = eee.encryptor.load().encrypt(&packet) else {
                             error!("failed to encrypt packet request");
                             return Ok(());
                         };
@@ -479,4 +481,29 @@ pub fn set_dns(name: &str, _network_domain: &str, gw: &str, ifidx: u32) -> std::
 
 pub fn restore_dns() {
 
+}
+
+pub fn del_route(net: &Ipv4Net, gw: &Ipv4Addr) -> Result<()> {
+    let mask = net.netmask().to_string();
+    let network = net.network().to_string();
+    let res = Command::new("route")
+        .arg("delete")
+        .arg(network)
+        .arg(mask)
+        .arg(gw.to_string())
+        .output()?;
+    Ok(())
+
+}
+
+pub fn add_route(net: &Ipv4Net, gw: &Ipv4Addr) -> Result<()> {
+    let mask = net.netmask().to_string();
+    let network = net.network().to_string();
+    let res = Command::new("route")
+        .arg("add")
+        .arg(network)
+        .arg(mask)
+        .arg(gw.to_string())
+        .output()?;
+    Ok(())
 }

src/network/tuntap.rs

@@ -1,6 +1,7 @@
+
 use std::sync::atomic::Ordering;
 
-use bytes::Bytes;
+use bytes::{Bytes, BytesMut};
 use dashmap::DashMap;
 use once_cell::sync::OnceCell;
 use sdlan_sn_rs::{
@@ -25,9 +26,10 @@ pub const MAX_WAIT_PACKETS: usize = 100;
 
 pub trait TunTapPacketHandler {
     async fn handle_packet_from_net(&self, data: &[u8]) -> std::io::Result<()>;
-    async fn handle_packet_from_device(&self, data: Vec<u8>) -> std::io::Result<()>;
+    async fn handle_packet_from_device(&self, data: BytesMut) -> std::io::Result<()>;
 }
 
+/* 
 static ARP_WAIT_LIST: OnceCell<ArpWaitList> = OnceCell::new();
 
 pub fn init_arp_wait_list() {
@@ -41,7 +43,7 @@ pub fn init_arp_wait_list() {
 pub struct ArpWaitInfo {
     timestamp: u64,
     // origin data is from the tun or tap device
-    origin_data: Vec<u8>,
+    origin_data: BytesMut,
 }
 
 #[derive(Debug)]
@@ -50,12 +52,12 @@ pub struct ArpWaitList {
 }
 
 impl ArpWaitList {
-    fn add_to_wait_list(&self, ip: u32, origin_data: Vec<u8>) {
+    fn add_to_wait_list(&self, ip: u32, origin_data_with_zeroed_layer2: BytesMut) {
         let mut entry = self.content.entry(ip).or_insert(vec![]);
         if entry.len() < MAX_WAIT_PACKETS {
             entry.push(ArpWaitInfo {
                 timestamp: get_current_timestamp(),
-                origin_data,
+                origin_data: origin_data_with_zeroed_layer2,
             })
         }
     }
@@ -84,11 +86,12 @@ impl ArpWaitList {
             if (now - item.timestamp) > 5 {
                 continue;
             }
-            let packet = form_ethernet_packet(src_mac, mac, &item.origin_data);
+            let packet = form_ethernet_packet(src_mac, mac, item.origin_data);
 
             let pkt_size = packet.len();
 
-            let Ok(encrypted) = edge.encryptor.read().unwrap().encrypt(&packet) else {
+            let Ok(encrypted) = edge.encryptor.load().encrypt(&packet) else {
+            // let Ok(encrypted) = edge.encryptor.read().unwrap().encrypt(&packet) else {
             // let Ok(encrypted) = aes_encrypt(&encrypt_key, &packet) else {
                 error!("failed to encrypt packet request");
                 return;
@@ -110,12 +113,12 @@ impl ArpWaitList {
     }
 }
 
-pub fn add_to_arp_wait_list(ip: u32, origin_data: Vec<u8>) {
+pub fn add_to_arp_wait_list(ip: u32, origin_data_with_zeroed_layer2: BytesMut) {
     let waitlist = ARP_WAIT_LIST
         .get()
         .expect("ARP_WAIT_LIST has not been inited");
 
-    waitlist.add_to_wait_list(ip, origin_data);
+    waitlist.add_to_wait_list(ip, origin_data_with_zeroed_layer2);
 }
 
 pub async fn arp_arrived(ip: u32, mac: Mac) {
@@ -125,3 +128,4 @@ pub async fn arp_arrived(ip: u32, mac: Mac) {
 
     waitlist.arp_arrived(ip, mac).await;
 }
+*/

src/pb/message.rs

@@ -32,38 +32,29 @@ pub struct SdlWelcome {
     #[prost(uint32, tag = "4")]
     pub heartbeat_sec: u32,
 }
-/// quic 通讯消息
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct SdlEmpty {
-    #[prost(uint32, tag = "1")]
-    pub pkt_id: u32,
-}
 /// 这里修改成了扁平的结构, 否则有些字段不好找放的位置
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct SdlRegisterSuper {
-    #[prost(uint32, tag = "1")]
-    pub pkt_id: u32,
-    #[prost(string, tag = "2")]
+    #[prost(string, tag = "1")]
     pub client_id: ::prost::alloc::string::String,
     /// 网络地址信息已经有https请求分配了
     /// 注册的时候需要带上(network_id, mac, ip, mask_len, hostname)
-    #[prost(uint32, tag = "3")]
+    #[prost(uint32, tag = "2")]
     pub network_id: u32,
-    #[prost(bytes = "vec", tag = "4")]
+    #[prost(bytes = "vec", tag = "3")]
     pub mac: ::prost::alloc::vec::Vec<u8>,
-    #[prost(uint32, tag = "5")]
+    #[prost(uint32, tag = "4")]
     pub ip: u32,
-    #[prost(uint32, tag = "6")]
+    #[prost(uint32, tag = "5")]
     pub mask_len: u32,
-    #[prost(string, tag = "7")]
+    #[prost(string, tag = "6")]
     pub hostname: ::prost::alloc::string::String,
-    #[prost(string, tag = "8")]
+    #[prost(string, tag = "7")]
     pub pub_key: ::prost::alloc::string::String,
     /// 客户端使用http协议请求后端，通过token或者账号密码登录时, 统一返回一个access_token;
     /// RegisterSuper的时候，验证凭证是否合法 (access_token)
-    #[prost(string, tag = "9")]
+    #[prost(string, tag = "8")]
     pub access_token: ::prost::alloc::string::String,
 }
 /// 客户端的升级逻辑，在https的接口里面去完成
@@ -71,47 +62,40 @@ pub struct SdlRegisterSuper {
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct SdlRegisterSuperAck {
-    #[prost(uint32, tag = "1")]
-    pub pkt_id: u32,
-    /// 目前支持aes， chacha20
-    #[prost(string, tag = "2")]
+    /// 目前支持aes, chacha20
+    #[prost(string, tag = "1")]
     pub algorithm: ::prost::alloc::string::String,
-    #[prost(bytes = "vec", tag = "3")]
+    #[prost(bytes = "vec", tag = "2")]
     pub key: ::prost::alloc::vec::Vec<u8>,
     /// 逻辑分段，chacha20加密算法需要使用该字段
-    #[prost(uint32, tag = "4")]
+    #[prost(uint32, tag = "3")]
     pub region_id: u32,
-    #[prost(bytes = "vec", tag = "5")]
+    #[prost(bytes = "vec", tag = "4")]
     pub session_token: ::prost::alloc::vec::Vec<u8>,
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct SdlRegisterSuperNak {
     #[prost(uint32, tag = "1")]
-    pub pkt_id: u32,
-    #[prost(uint32, tag = "2")]
     pub error_code: u32,
-    #[prost(string, tag = "3")]
+    #[prost(string, tag = "2")]
     pub error_message: ::prost::alloc::string::String,
 }
+/// 网络地址查询
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct SdlQueryInfo {
-    #[prost(uint32, tag = "1")]
-    pub pkt_id: u32,
-    #[prost(bytes = "vec", tag = "2")]
+    #[prost(bytes = "vec", tag = "1")]
     pub dst_mac: ::prost::alloc::vec::Vec<u8>,
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct SdlPeerInfo {
-    #[prost(uint32, tag = "1")]
-    pub pkt_id: u32,
-    #[prost(bytes = "vec", tag = "2")]
+    #[prost(bytes = "vec", tag = "1")]
     pub dst_mac: ::prost::alloc::vec::Vec<u8>,
-    #[prost(message, optional, tag = "3")]
+    #[prost(message, optional, tag = "2")]
     pub v4_info: ::core::option::Option<Sdlv4Info>,
-    #[prost(message, optional, tag = "4")]
+    #[prost(message, optional, tag = "3")]
     pub v6_info: ::core::option::Option<Sdlv6Info>,
 }
 /// ARP查询相关
@@ -121,31 +105,33 @@ pub struct SdlPeerInfo {
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct SdlArpRequest {
     #[prost(uint32, tag = "1")]
-    pub pkt_id: u32,
-    #[prost(uint32, tag = "2")]
     pub target_ip: u32,
+    #[prost(uint32, tag = "2")]
+    pub origin_ip: u32,
+    #[prost(bytes = "vec", tag = "3")]
+    pub context: ::prost::alloc::vec::Vec<u8>,
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct SdlArpResponse {
     #[prost(uint32, tag = "1")]
-    pub pkt_id: u32,
-    #[prost(uint32, tag = "2")]
     pub target_ip: u32,
-    #[prost(bytes = "vec", tag = "3")]
+    #[prost(bytes = "vec", tag = "2")]
     pub target_mac: ::prost::alloc::vec::Vec<u8>,
+    #[prost(uint32, tag = "3")]
+    pub origin_ip: u32,
+    #[prost(bytes = "vec", tag = "4")]
+    pub context: ::prost::alloc::vec::Vec<u8>,
 }
 /// 权限请求查询相关
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct SdlPolicyRequest {
     #[prost(uint32, tag = "1")]
-    pub pkt_id: u32,
-    #[prost(uint32, tag = "2")]
     pub src_identity_id: u32,
-    #[prost(uint32, tag = "3")]
+    #[prost(uint32, tag = "2")]
     pub dst_identity_id: u32,
-    #[prost(uint32, tag = "4")]
+    #[prost(uint32, tag = "3")]
     pub version: u32,
 }
 /// 基于quic通讯，rules部分已经没有了长度限制
@@ -153,16 +139,14 @@ pub struct SdlPolicyRequest {
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct SdlPolicyResponse {
     #[prost(uint32, tag = "1")]
-    pub pkt_id: u32,
-    #[prost(uint32, tag = "2")]
     pub src_identity_id: u32,
-    #[prost(uint32, tag = "3")]
+    #[prost(uint32, tag = "2")]
     pub dst_identity_id: u32,
     /// 版本号，客户端需要比较版本号确定是否覆盖; 请求端自己去管理版本号，服务端只是原样回写
-    #[prost(uint32, tag = "4")]
+    #[prost(uint32, tag = "3")]
     pub version: u32,
-    /// 4+1+2 的稀疏序列化规则
-    #[prost(bytes = "vec", tag = "5")]
+    /// 1 + 2稀疏序列化规则, 按照: <<Proto:8, Port:16>> 这个格式序列号所有的规则信息; 下发的数据默认都是allow，deny规则的服务器端已经屏蔽
+    #[prost(bytes = "vec", tag = "4")]
     pub rules: ::prost::alloc::vec::Vec<u8>,
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
@@ -197,29 +181,24 @@ pub struct SdlNetworkShutdownEvent {
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct SdlStunRequest {
-    #[prost(uint32, tag = "1")]
-    pub cookie: u32,
-    #[prost(string, tag = "2")]
+    #[prost(string, tag = "1")]
     pub client_id: ::prost::alloc::string::String,
-    #[prost(uint32, tag = "3")]
+    #[prost(uint32, tag = "2")]
     pub network_id: u32,
-    #[prost(bytes = "vec", tag = "4")]
+    #[prost(bytes = "vec", tag = "3")]
     pub mac: ::prost::alloc::vec::Vec<u8>,
-    #[prost(uint32, tag = "5")]
+    #[prost(uint32, tag = "4")]
     pub ip: u32,
-    #[prost(uint32, tag = "6")]
+    #[prost(uint32, tag = "5")]
     pub nat_type: u32,
-    #[prost(message, optional, tag = "7")]
+    #[prost(message, optional, tag = "6")]
     pub v6_info: ::core::option::Option<Sdlv6Info>,
-    #[prost(bytes = "vec", tag = "8")]
+    #[prost(bytes = "vec", tag = "7")]
     pub session_token: ::prost::alloc::vec::Vec<u8>,
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
-pub struct SdlStunReply {
-    #[prost(uint32, tag = "1")]
-    pub cookie: u32,
-}
+pub struct SdlStunReply {}
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct SdlData {
@@ -286,3 +265,6 @@ pub struct SdlRegisterAck {
     #[prost(bytes = "vec", tag = "3")]
     pub dst_mac: ::prost::alloc::vec::Vec<u8>,
 }
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct SdlEmpty {}

src/tcp/mod.rs

@@ -2,11 +2,8 @@ mod tcp_codec;
 // mod tcp_conn;
 mod quic;
 
-mod identity_cache;
-
 pub use tcp_codec::*;
 pub use quic::*;
-pub use identity_cache::*;
 
 
 // pub use tcp_conn::*;

src/tcp/quic.rs

@@ -9,7 +9,7 @@ use tokio::{sync::mpsc::{Receiver, Sender, channel}, time::sleep};
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, warn};
 
-use crate::{AesEncryptor, Chacha20Encryptor, ConnectionInfo, ConnectionState, MyEncryptor, config::{NULL_MAC, TCP_PING_TIME}, get_edge, network::{ARP_REPLY, ArpHdr, EthHdr, Node, RegisterSuperFeedback, StartStopInfo, check_peer_registration_needed, handle_packet_peer_info}, pb::{SdlArpResponse, SdlPolicyResponse, SdlRegisterSuper, SdlRegisterSuperAck, SdlRegisterSuperNak, SdlSendRegisterEvent, encode_to_tcp_message}, tcp::{EventType, NakMsgCode, NatType, PacketType, RuleInfo, SdlanTcp, read_a_packet, send_stun_request, set_identity_cache}};
+use crate::{AesEncryptor, Chacha20Encryptor, ConnectionInfo, ConnectionState, MyEncryptor, RuleFromServer, config::{NULL_MAC, TCP_PING_TIME}, get_edge, network::{ARP_REPLY, ArpHdr, EthHdr, Node, RegisterSuperFeedback, StartStopInfo, arp_reply_arrived, check_peer_registration_needed, handle_packet_peer_info}, pb::{SdlArpResponse, SdlPolicyResponse, SdlRegisterSuper, SdlRegisterSuperAck, SdlRegisterSuperNak, SdlSendRegisterEvent, encode_to_tcp_message}, tcp::{EventType, NakMsgCode, NatType, PacketType, SdlanTcp, read_a_packet, send_stun_request}};
 
 static GLOBAL_QUIC_HANDLE: OnceLock<ReadWriterHandle> = OnceLock::new();
 
@@ -36,7 +36,7 @@ impl ReadWriterHandle {
         Ok(())
     }
 
-    fn new<>(
+    fn new(
         cancel: CancellationToken,
         addr: &str,
         // on_connected: OnConnectedCallback<'a>,
@@ -80,6 +80,7 @@ impl ReadWriterHandle {
                     handle_tcp_message(msg).await;
                 } else {
                     error!("data from tcp exited");
+                    println!("data from tcp exited");
                     // eprintln!("data from tcp exited");
                     return;
                 }
@@ -117,7 +118,7 @@ async fn handle_tcp_message(msg: SdlanTcp) {
             };
 
             edge.send_register_super_feedback(
-                ack.pkt_id,
+                0,
                 RegisterSuperFeedback {
                     result: 0,
                     message: "".to_owned(),
@@ -133,10 +134,12 @@ async fn handle_tcp_message(msg: SdlanTcp) {
 
             match ack.algorithm.to_ascii_lowercase().as_str() {
                 "chacha20" => {
-                    *edge.encryptor.write().unwrap() = MyEncryptor::ChaChao20(Chacha20Encryptor::new(key, ack.region_id));
+                    edge.encryptor.store(Arc::new(MyEncryptor::ChaChao20(Chacha20Encryptor::new(key, ack.region_id))))
+                    // *edge.encryptor.write().unwrap() = MyEncryptor::ChaChao20(Chacha20Encryptor::new(key, ack.region_id));
                 }
                 "aes" => {
-                    *edge.encryptor.write().unwrap() = MyEncryptor::Aes(AesEncryptor::new(key));
+                    edge.encryptor.store(Arc::new(MyEncryptor::Aes(AesEncryptor::new(key))))
+                    // *edge.encryptor.write().unwrap() = MyEncryptor::Aes(AesEncryptor::new(key));
                 }
                 _other => {
 
@@ -180,7 +183,7 @@ async fn handle_tcp_message(msg: SdlanTcp) {
             edge.network_id.store(dev.network_id, Ordering::Relaxed);
             */
             // edge.device.reload_config(&edge.device_config, &dev.network_domain);
-            edge.device.reload_config(&edge.device_config, &edge.network_domain.read().unwrap().clone());
+            edge.device.reload_config(edge, &edge.device_config, &edge.network_domain.read().unwrap().clone());
 
             edge.set_authorized(true);
 
@@ -192,42 +195,14 @@ async fn handle_tcp_message(msg: SdlanTcp) {
             });
         }
         PacketType::ArpResponse => {
-            let Ok(resp) = SdlArpResponse::decode(&msg.current_packet[..]) else {
+            let Ok(res) = SdlArpResponse::decode(&msg.current_packet[..]) else {
                 error!("failed to decode ARP RESPONSE");
                 return;
             };
-            if resp.target_mac.len() != 6 {
-                // invalid target_mac
-                error!("invalid target_mac: {:?}, ip={}", resp.target_mac, ip_to_string(&resp.target_ip));
-                return;
-            }
 
-            // TODO: construct the arp reply, and write to tun;
-            let src_mac = resp.target_mac.try_into().unwrap();
-            let dst_mac = edge.device_config.get_mac();
-            let dst_ip = edge.device_config.get_ip();
 
-            let hdr = ArpHdr{
-                ethhdr: EthHdr {
-                    dest: dst_mac,
-                    src: src_mac,
-                    eth_type: 0x0806,
-                },
-                hwtype: 0x0001,
-                protocol: 0x0800,
-                hwlen: 6,
-                protolen: 4,
-                opcode: ARP_REPLY,
-                shwaddr: src_mac,
-                sipaddr: [((resp.target_ip >> 16) as u16) & 0xffff, (resp.target_ip as u16) & 0xffff],
-                dhwaddr: dst_mac,
-                dipaddr: [((dst_ip >> 16) & 0x0000ffff) as u16, (dst_ip & 0x0000ffff) as u16]
-            };
-
-            let data = hdr.marshal_to_bytes();
-            if let Err(_e) = edge.device.send(&data) {
-                error!("failed to write arp response to device");
-            }
+            arp_reply_arrived(edge, res).await;
+            return;
         }
         PacketType::PolicyReply => {
             let Ok(policy) = SdlPolicyResponse::decode(&msg.current_packet[..]) else {
@@ -245,13 +220,13 @@ async fn handle_tcp_message(msg: SdlanTcp) {
                 let port = u16::from_be_bytes([policy.rules[start+1], policy.rules[start+2]]);
                 start += 3;
 
-                infos.push(RuleInfo{
+                infos.push(RuleFromServer{
                     proto,
                     port,
                 });
             }
 
-            set_identity_cache(identity, infos);
+            edge.rule_cache.set_identity_cache(identity, infos);
         }
 
         PacketType::RegisterSuperNAK => {
@@ -272,12 +247,12 @@ async fn handle_tcp_message(msg: SdlanTcp) {
             };
 
             error!("got nak: {:?}", _nak);
-            let pkt_id = _nak.pkt_id;
+            // let pkt_id = _nak.pkt_id;
 
             let Ok(error_code) = NakMsgCode::try_from(_nak.error_code as u8) else {
                 edge.send_register_super_feedback(
                     //msg._packet_id,
-                    pkt_id,
+                    0,
                     RegisterSuperFeedback {
                         result: 2,
                         message: "error_code not recognized".to_owned(),
@@ -290,7 +265,7 @@ async fn handle_tcp_message(msg: SdlanTcp) {
                 NakMsgCode::InvalidToken => {
                     edge.send_register_super_feedback(
                         // msg._packet_id,
-                        pkt_id,
+                        0,
                         RegisterSuperFeedback {
                             result: 3,
                             message: "invalid token".to_owned(),
@@ -302,7 +277,7 @@ async fn handle_tcp_message(msg: SdlanTcp) {
                 NakMsgCode::NodeDisabled => {
                     edge.send_register_super_feedback(
                         // msg._packet_id,
-                        pkt_id,
+                        0,
                         RegisterSuperFeedback {
                             result: 4,
                             message: "Node is disabled".to_owned(),
@@ -314,7 +289,7 @@ async fn handle_tcp_message(msg: SdlanTcp) {
                 _other => {
                     edge.send_register_super_feedback(
                         // msg._packet_id,
-                        pkt_id,
+                        0,
                         RegisterSuperFeedback {
                             result: 0,
                             message: "".to_owned(),
@@ -330,7 +305,8 @@ async fn handle_tcp_message(msg: SdlanTcp) {
             });
             */
             edge.set_authorized(false);
-            *edge.encryptor.write().unwrap() = MyEncryptor::Invalid;
+            edge.encryptor.store(Arc::new(MyEncryptor::Invalid));
+            // *edge.encryptor.write().unwrap() = MyEncryptor::Invalid;
             // std::process::exit(0);
         }
         PacketType::Command => {
@@ -496,7 +472,7 @@ impl ReadWriteActor {
         let edge = get_edge();
 
         let mut started = false;
-        let mut start_pkt_id = None;
+        // let mut start_pkt_id = None;
         loop {
             if let Some(ref connecting_chan) = self.connecting_chan {
                 let state = ConnectionInfo::ConnState(ConnectionState::NotConnected);
@@ -510,11 +486,12 @@ impl ReadWriteActor {
                     if let Some(m) = start_or_stop {
                         if m.is_start {
                             started = true;
-                            start_pkt_id = m.pkt_id;
+                            // start_pkt_id = m.pkt_id;
                             break;
                         }
                     } else {
                         // None, just return
+                        println!("start or stop is None");
                         return;
                     }
                 }
@@ -590,7 +567,7 @@ impl ReadWriteActor {
             debug!("connected");
 
             sleep(Duration::from_millis(200)).await;
-            on_connected_callback(local_ip, &mut send, start_pkt_id.take()).await;
+            on_connected_callback(local_ip, &mut send).await;
 
             if let Some(ref connecting_chan) = self.connecting_chan {
                 let state = ConnectionInfo::ConnState(ConnectionState::Connected);
@@ -622,6 +599,7 @@ impl ReadWriteActor {
 
             let write_to_tcp = async {
                 while let Some(data) = to_tcp.recv().await {
+                    debug!("data size = {}", data.len());
                     match send.write(&data).await {
                         Ok(size) => {
                             debug!("{} bytes sent to tcp", size);
@@ -689,10 +667,10 @@ impl ReadWriteActor {
 async fn on_disconnected_callback() {
     let edge = get_edge();
     edge.set_authorized(false);
-    *edge.encryptor.write().unwrap() = MyEncryptor::Invalid;
+    edge.encryptor.store(Arc::new(MyEncryptor::Invalid));
 }
 
-async fn on_connected_callback(local_ip: Option<IpAddr>, stream: &mut SendStream, _pkt_id: Option<u32>) {
+async fn on_connected_callback(local_ip: Option<IpAddr>, stream: &mut SendStream) {
     let edge = get_edge();
     // let installed_channel = install_channel.to_owned();
 
@@ -714,7 +692,7 @@ async fn on_connected_callback(local_ip: Option<IpAddr>, stream: &mut SendStream
     }
     let register_super = SdlRegisterSuper {
         mac: Vec::from(edge.device_config.get_mac()),
-        pkt_id: edge.get_next_packet_id(),
+        // pkt_id: edge.get_next_packet_id(),
         network_id: edge.network_id.load(Ordering::Relaxed),
         ip: edge.device_config.get_ip(),
         mask_len: edge.device_config.get_net_bit() as u32,
@@ -727,6 +705,7 @@ async fn on_connected_callback(local_ip: Option<IpAddr>, stream: &mut SendStream
     };
 
     println!("register super: {:?}", register_super);
+
     // debug!("send register super: {:?}", register_super);
     // let packet_id = edge.get_next_packet_id();
     let data = encode_to_tcp_message(

src/tcp/tcp_codec.rs

@@ -114,7 +114,7 @@ pub async fn send_stun_request(eee: &Node) {
     };
     let req = SdlStunRequest {
         session_token: Vec::from(eee.session_token.get()),
-        cookie: 0,
+        // cookie: 0,
         client_id: eee.config.node_uuid.clone(),
         network_id: eee.network_id.load(Ordering::Relaxed),
         ip: eee.device_config.get_ip(),

src/utils/acl_session.rs

@@ -0,0 +1,163 @@
+use std::{net::IpAddr, sync::{Arc, atomic::{AtomicU64, Ordering}}, time::{Duration, SystemTime, UNIX_EPOCH}};
+
+use ahash::RandomState;
+use dashmap::{DashMap, DashSet};
+use tracing::{debug, error};
+
+const RULE_VALID_TIME_IN_SECS: u64 = 60;
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] 
+pub struct FiveTuple {
+    pub src_ip: IpAddr,
+    pub dst_ip: IpAddr,
+    pub src_port: u16,
+    pub dst_port: u16,
+    pub proto: u8,
+}
+
+impl FiveTuple {
+    pub fn reverse(&self) -> Self {
+        Self {
+            src_ip: self.dst_ip,
+            dst_ip: self.src_ip,
+            dst_port: self.src_port,
+            src_port: self.dst_port,
+            proto: self.proto,
+        }
+    }
+}
+
+pub struct SessionInfo {
+    pub last_active: AtomicU64,
+}
+
+pub struct SessionTable {
+    table: Arc<DashMap<FiveTuple, SessionInfo, RandomState>>,
+    timeout_secs: u64,
+}
+
+fn now_secs() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_secs()
+}
+
+impl SessionTable {
+    pub fn new(timeout_secs: u64) -> Self {
+        Self {
+            table: Arc::new(DashMap::with_hasher(RandomState::new())),
+            timeout_secs,
+        }
+    }
+
+    pub fn add_session_info(&self, key: FiveTuple) {
+        if let Some(info) = self.table.get(&key) {
+            info.last_active.store(now_secs(), Ordering::Relaxed);
+            return;
+        } 
+        self.table.insert(key, SessionInfo { last_active: AtomicU64::new(now_secs()) });
+    }
+
+    pub fn process_packet(&self, key: &FiveTuple) -> bool {
+        if let Some(session) = self.table.get_mut(&key) {
+            session.last_active.store(now_secs(), Ordering::Relaxed);
+            return true;
+        }
+
+        false
+    }
+
+    pub fn retain(&self) {
+        let now = now_secs();
+        debug!("retain session");
+        self.table.retain(|_, info|{
+            let last = info.last_active.load(Ordering::Relaxed);
+            now-last < self.timeout_secs
+        });
+    }
+}
+
+// ip's u32 representation
+type IdentityID = u32;
+// port representation
+type Port = u16;
+// proto, like icmp = 1, udp=16
+type Proto = u8;
+
+type RuleInfo = (Port, Proto);
+
+
+
+#[derive(Debug)]
+pub struct RuleFromServer {
+    pub proto: Proto,
+    pub port: Port,
+}
+
+pub struct RuleCache {
+    pub rule_info: DashMap<IdentityID, (AtomicU64, DashSet<RuleInfo,RandomState>), RandomState>,
+    pub rule_valid_secs: u64,
+    pub session_table: Arc<SessionTable>,
+}
+
+type ShouldRenew = bool;
+
+impl RuleCache {
+    pub fn new() -> Self {
+        let session_table= Arc::new(SessionTable::new(300));
+        let table_cleaner = Arc::clone(&session_table);
+        tokio::spawn(async move {
+            loop {
+                tokio::time::sleep(Duration::from_secs(20)).await;
+                table_cleaner.retain();
+            }
+        });
+        Self {
+            rule_info: DashMap::with_hasher(RandomState::new()),
+            rule_valid_secs: RULE_VALID_TIME_IN_SECS,
+            session_table,
+        }
+    }
+
+    pub fn set_identity_cache(&self, identity: IdentityID, infos: Vec<RuleFromServer>) {
+        debug!("setting identity cache for identity={}, infos: {:?}", identity, infos);
+
+        let now = now_secs();
+
+        let now_sets = DashSet::with_hasher(RandomState::new());
+        for info in &infos {
+            // let mut protomap = HashSet::new();
+            now_sets.insert((info.port, info.proto));
+        }
+        self.rule_info.insert(identity, (AtomicU64::new(now) , now_sets));
+    }
+
+    pub fn touch_packet(&self, info: FiveTuple) {
+        error!("touch a packet: {:?}", info);
+        self.session_table.add_session_info(info);
+    }
+
+    pub fn is_identity_ok(&self, identity: IdentityID, info: FiveTuple) -> (bool, ShouldRenew) {
+        error!("is identity ok? {:?}", info);
+        if self.session_table.process_packet(&info) {
+            error!("identity is ok");
+            return (true, false);
+        }
+
+        let now = now_secs();
+        if let Some(sets_info) = self.rule_info.get(&identity) {
+            let last_set_time = sets_info.0.load(Ordering::Relaxed);
+            if sets_info.1.contains(&(info.dst_port, info.proto)) {
+                if (now - last_set_time) > self.rule_valid_secs {
+                    return (true, true);
+                }
+                return (true, false);
+            }
+        }
+
+        self.rule_info.insert(identity, (AtomicU64::new(now), DashSet::with_hasher(RandomState::new())));
+
+        (false, true)
+    }
+}

src/utils/command.rs

@@ -18,18 +18,40 @@ pub enum Commands {
 
     /// if logined in, just start,
     /// else, use the token to login, and start
-    AutoRun(TokenLogin),
+    AutoRun(AutoRunArgument),
 
     /// after login, we can use start to 
     /// connect to the remote
-    Start,
+    Start(StartArguments),
 
     Info,
 
+    RouteAdd(RouteCmdInfo),
+    RouteDel(RouteCmdInfo),
+    RouteList,
+
     /// exits the 
     Stop,
 }
 
+#[derive(Args, Debug)]
+pub struct StartArguments {
+    #[arg(short, long, default_value="")]
+    pub route: String,
+
+    #[arg(short, long, default_value="")]
+    pub route_file: String,
+
+    #[arg(short, long, default_value_t=false)]
+    pub allow_routing: bool,
+}
+
+#[derive(Args, Debug)]
+pub struct RouteCmdInfo {
+    #[arg(short, long, default_value="")]
+    pub route: String,
+}
+
 #[derive(Args, Debug)]
 pub struct UserLogin {
     #[arg(short, long, env = APP_USER_ENV_NAME)]
@@ -39,6 +61,29 @@ pub struct UserLogin {
     pub password: String,
 }
 
+#[derive(Args, Debug)]
+pub struct AutoRunTokenLogin {
+    #[arg(long, env=APP_TOKEN_ENV_NAME, required=false)]
+    pub token: String,
+
+    #[arg(short, long, default_value="")]
+    pub route: String,
+}
+
+#[derive(Args, Debug)]
+pub struct AutoRunArgument {
+    #[arg(long, env=APP_TOKEN_ENV_NAME, required=false)]
+    pub token: String,
+
+    #[arg(short, long, default_value_t=false)]
+    pub allow_routing: bool,
+
+    #[arg(short, long, default_value="")]
+    pub route: String,
+
+    #[arg(short, long, default_value="")]
+    pub route_file: String,
+}
 
 #[derive(Args, Debug)]
 pub struct TokenLogin {
@@ -82,7 +127,7 @@ pub struct CommandLine {
     pub nat_server2: String,
 
     #[structopt(short = "r")]
-    pub _allow_routing: bool,
+    pub allow_routing: bool,
 
     #[structopt(short = "dm")]
     pub _drop_multicast: bool,
@@ -114,6 +159,12 @@ pub struct CommandLine {
 
     #[structopt(short = "l")]
     pub local_port: u16,
+
+    #[structopt(long, default_value="")]
+    pub route_str: String,
+
+    #[structopt(long, default_value="")]
+    pub route_file: String,
 }
 
 impl Clone for CommandLine {
@@ -121,7 +172,7 @@ impl Clone for CommandLine {
         Self {
             sn: self.sn.clone(),
             quic: self.quic.clone(),
-            _allow_routing: self._allow_routing,
+            allow_routing: self.allow_routing,
             _drop_multicast: self._drop_multicast,
             register_ttl: self.register_ttl,
             mtu: self.mtu,
@@ -133,6 +184,8 @@ impl Clone for CommandLine {
             allow_p2p: self.allow_p2p,
             nat_server1: self.nat_server1.clone(),
             nat_server2: self.nat_server2.clone(),
+            route_str: self.route_str.clone(),
+            route_file: self.route_file.clone(),
         }
     }
 }

src/utils/mod.rs

@@ -1,10 +1,14 @@
 mod command;
+mod acl_session;
 mod encrypter;
+mod system_action;
 
 use std::{fs::OpenOptions, io::Write, net::Ipv4Addr, path::Path};
 
 pub use encrypter::*;
 pub use command::*;
+pub use acl_session::*;
+pub use system_action::*;
 
 mod socks;
 use rand::Rng;

src/utils/system_action.rs

@@ -0,0 +1,89 @@
+use std::{net::Ipv4Addr, sync::Arc};
+
+use arc_swap::ArcSwap;
+
+#[derive(Default, Clone)]
+pub struct TrieNode {
+    child: [Option<Box<TrieNode>>; 2],
+    prefix_len: u8,
+    nexthop: Option<Ipv4Addr>,
+}
+
+#[derive(Default, Clone)]
+struct IpTrie {
+    root: TrieNode,
+}
+
+impl IpTrie {
+    fn new() -> Self {
+        Self {
+            root: TrieNode::default()
+        }
+    }
+
+    fn insert(&mut self, prefix: u32, prefix_len: u8, nexthop: Ipv4Addr) {
+        if prefix_len > 32 {
+            return;
+        }
+
+        let mut node = &mut self.root;
+        for i in 0..prefix_len {
+            let bit = ((prefix >> (31-i)) & 1) as usize;
+            node = node.child[bit].get_or_insert_with(|| Box::new(TrieNode::default()));
+
+        }
+        if prefix_len > node.prefix_len {
+            node.prefix_len = prefix_len;
+            node.nexthop = Some(nexthop);
+        }
+    }
+
+    fn lookup(&self, ip: u32) -> Option<(u8, Ipv4Addr)>{
+        let mut node = &self.root;
+        let mut best = None;
+
+        for i in 0..32 {
+            if node.nexthop.is_some() {
+                best = Some((node.prefix_len, node.nexthop.unwrap()));
+            }
+            let bit = ((ip>>(31-i)) & 1) as usize;
+            match &node.child[bit] {
+                Some(child) => {
+                    node = child;
+                }
+                None => {
+                    break;
+                }
+            }
+        }
+        if node.nexthop.is_some() {
+            best = Some((node.prefix_len, node.nexthop.unwrap()));
+        }
+        best
+    }
+}
+
+pub struct RouteTableTrie {
+    trie: ArcSwap<IpTrie>,
+}
+
+impl RouteTableTrie {
+    pub fn new() -> Self {
+        Self {
+            trie: ArcSwap::new(Arc::new(IpTrie::default()))
+        }
+    }
+
+    pub fn lookup(&self, ip: u32) -> Option<(u8, Ipv4Addr)> {
+        let trie = self.trie.load();
+        trie.lookup(ip)
+    }
+
+    pub fn insert(&self, prefix: u32, prefix_len: u8, nexthop: Ipv4Addr) {
+        let old = self.trie.load();
+        let mut new_trie = (*(*old)).clone();
+
+        new_trie.insert(prefix, prefix_len, nexthop);
+        self.trie.store(Arc::new(new_trie));
+    }
+}