add L2 reliability: seq + cumulative ACK + retransmit (protocol v2)

DATA frames now carry seq:4 and ack_seq:4 in a 16-byte extended header. Both sides maintain per-session send/recv state: Sender: - Monotonic seq counter, retransmit buffer (seq -> frame bytes) - Retransmit timer: 5ms timeout, 10 max retries -> CLOSE - Window advances on cumulative ACK Receiver: - In-order delivery to TCP socket (expected_seq) - Out-of-order buffering (SortedList by seq) - Duplicate detection (seq < expected -> discard + re-ACK) - Pure ACK frames (empty-payload DATA) for duplicate/OOO responses This prevents lost Ethernet frames from permanently corrupting TCP sessions, which was the key v1 limitation. The local kernel TCP stack ACKs data before we chunk it into DATA frames; without L2 reliability a dropped frame creates an unrecoverable gap. Version bumped to 2. Both sides must speak v2; no negotiation. Updated: PROTOCOL.md (full v2 spec), README.md, Rust frame.rs/ session.rs/main.rs, C# Frame.cs/SessionManager.cs/TunnelLink.cs.
2026-08-13 09:08:05 +00:00
parent 2f61f2bb1b
commit eb8994d1e1
8 changed files with 721 additions and 102 deletions
@@ -6,8 +6,9 @@ static class Proto
 {
    public const ushort EtherType = 0x6969;
    public const int EthHeaderLen = 14;
-    public const byte Version = 1;
+    public const byte Version = 2;
    public const int HeaderLen = 8;
+    public const int DataHeaderLen = 16; // 8 common + 4 seq + 4 ack_seq
    public const int MaxPayload = 1480;

    public const byte TypeDiscover = 0x01;
@@ -28,6 +29,7 @@ static class Proto
    public const byte ReasonConnectFailed = 2;
    public const byte ReasonOversize = 3;
    public const byte ReasonUnknownSession = 4;
+    public const byte ReasonMaxRetries = 5;
 }

 readonly record struct UpstreamEntry(
@@ -46,7 +48,7 @@ abstract record Frame
    internal record Open(byte UpstreamId) : Frame;
    internal record OpenAck(uint SessionId, byte UpstreamId) : Frame;
    internal record OpenNak(byte UpstreamId, byte Reason) : Frame;
-    internal record Data(uint SessionId, byte[] Payload) : Frame;
+    internal record Data(uint SessionId, uint Seq, uint AckSeq, byte[] Payload) : Frame;
    internal record Close(uint SessionId, byte? Reason) : Frame;
    internal record Ping(ulong Nonce) : Frame;
    internal record Pong(ulong Nonce) : Frame;
@@ -54,8 +56,7 @@ abstract record Frame

 static class FrameCodec
 {
-    /// Build the 8-byte header + payload. payload_len records the exact
-    /// payload length so the receiver can ignore Ethernet padding.
+    /// Build a non-DATA frame: 8-byte common header + payload.
    static byte[] Build(byte type, uint sessionId, byte[] payload)
    {
        var buf = new byte[Proto.HeaderLen + payload.Length];
@@ -71,6 +72,31 @@ static class FrameCodec
        return buf;
    }

+    /// Build a DATA frame: 8-byte common header + seq + ack_seq + payload.
+    /// payload_len counts only the raw bytes, not seq/ack_seq.
+    static byte[] BuildData(uint sessionId, uint seq, uint ackSeq, byte[] payload)
+    {
+        var buf = new byte[Proto.DataHeaderLen + payload.Length];
+        buf[0] = Proto.Version;
+        buf[1] = Proto.TypeData;
+        buf[2] = (byte)(sessionId >> 24);
+        buf[3] = (byte)(sessionId >> 16);
+        buf[4] = (byte)(sessionId >> 8);
+        buf[5] = (byte)(sessionId & 0xFF);
+        buf[6] = (byte)(payload.Length >> 8);
+        buf[7] = (byte)(payload.Length & 0xFF);
+        buf[8] = (byte)(seq >> 24);
+        buf[9] = (byte)(seq >> 16);
+        buf[10] = (byte)(seq >> 8);
+        buf[11] = (byte)(seq & 0xFF);
+        buf[12] = (byte)(ackSeq >> 24);
+        buf[13] = (byte)(ackSeq >> 16);
+        buf[14] = (byte)(ackSeq >> 8);
+        buf[15] = (byte)(ackSeq & 0xFF);
+        Buffer.BlockCopy(payload, 0, buf, Proto.DataHeaderLen, payload.Length);
+        return buf;
+    }
+
    public static byte[] Encode(Frame frame)
    {
        return frame switch
@@ -86,7 +112,7 @@ static class FrameCodec
            Frame.OpenNak nak =>
                Build(Proto.TypeOpenNak, 0, [nak.UpstreamId, nak.Reason]),
            Frame.Data data =>
-                Build(Proto.TypeData, data.SessionId, data.Payload),
+                BuildData(data.SessionId, data.Seq, data.AckSeq, data.Payload),
            Frame.Close close =>
                Build(Proto.TypeClose, close.SessionId,
                    close.Reason.HasValue ? [close.Reason.Value] : []),
@@ -130,27 +156,38 @@ static class FrameCodec
        var type = buf[1];
        var sessionId = (uint)(buf[2] << 24 | buf[3] << 16 | buf[4] << 8 | buf[5]);
        var payloadLen = (ushort)(buf[6] << 8 | buf[7]);
+
+        // DATA frames have seq + ack_seq after the common header.
+        if (type == Proto.TypeData)
+        {
+            if (buf.Length < Proto.DataHeaderLen + payloadLen)
+                return null;
+            var seq = (uint)(buf[8] << 24 | buf[9] << 16 | buf[10] << 8 | buf[11]);
+            var ackSeq = (uint)(buf[12] << 24 | buf[13] << 16 | buf[14] << 8 | buf[15]);
+            var payload = buf.Slice(Proto.DataHeaderLen, payloadLen);
+            if (payload.Length > Proto.MaxPayload)
+                return null;
+            return new Frame.Data(sessionId, seq, ackSeq, payload.ToArray());
+        }
+
        if (buf.Length < Proto.HeaderLen + payloadLen)
            return null;
-        // Slice exactly payloadLen bytes, ignoring any trailing Ethernet padding.
-        var payload = buf.Slice(Proto.HeaderLen, payloadLen);
+        var payload2 = buf.Slice(Proto.HeaderLen, payloadLen);

        return type switch
        {
-            Proto.TypeManifest => ParseManifest(payload),
-            Proto.TypeOpenAck when payload.Length == 1 =>
-                new Frame.OpenAck(sessionId, payload[0]),
-            Proto.TypeOpenNak when payload.Length == 2 =>
-                new Frame.OpenNak(payload[0], payload[1]),
-            Proto.TypeData when payload.Length <= Proto.MaxPayload =>
-                new Frame.Data(sessionId, payload.ToArray()),
-            Proto.TypeClose when payload.Length is 0 or 1 =>
-                new Frame.Close(sessionId, payload.Length == 1 ? payload[0] : null),
-            Proto.TypePong when payload.Length == 8 =>
-                new Frame.Pong(ParseNonce(payload)),
-            Proto.TypePing when payload.Length == 8 =>
-                new Frame.Ping(ParseNonce(payload)),
-            Proto.TypeDiscover when payload.Length == 0 =>
+            Proto.TypeManifest => ParseManifest(payload2),
+            Proto.TypeOpenAck when payload2.Length == 1 =>
+                new Frame.OpenAck(sessionId, payload2[0]),
+            Proto.TypeOpenNak when payload2.Length == 2 =>
+                new Frame.OpenNak(payload2[0], payload2[1]),
+            Proto.TypeClose when payload2.Length is 0 or 1 =>
+                new Frame.Close(sessionId, payload2.Length == 1 ? payload2[0] : null),
+            Proto.TypePong when payload2.Length == 8 =>
+                new Frame.Pong(ParseNonce(payload2)),
+            Proto.TypePing when payload2.Length == 8 =>
+                new Frame.Ping(ParseNonce(payload2)),
+            Proto.TypeDiscover when payload2.Length == 0 =>
                new Frame.Discover(),
            _ => null,
        };
@@ -92,7 +92,7 @@ sealed class SessionManager : IDisposable

            case Frame.Data data:
                if (_sessions.TryGetValue(data.SessionId, out var session))
-                    session.Deliver(data.Payload);
+                    session.HandleData(data);
                else if (_link != null && _serverMac != null)
                    _link.SendTo(_serverMac,
                        new Frame.Close(data.SessionId, Proto.ReasonUnknownSession));
@@ -279,6 +279,105 @@ sealed class PendingOpen(TcpClient client, byte upstreamId)
    public byte UpstreamId { get; } = upstreamId;
 }

+/// L2 reliability: sender-side state.
+class SendState
+{
+    public uint SendSeq;
+    public uint AckedSeq;
+    // seq -> (frame_bytes, send_time_ticks, retry_count)
+    public readonly SortedList<uint, (byte[], long, uint)> RetransmitBuffer = new();
+
+    public uint NextSeq()
+    {
+        var s = SendSeq;
+        SendSeq++;
+        return s;
+    }
+
+    public void RecordSent(uint seq, byte[] frameBytes)
+    {
+        RetransmitBuffer[seq] = (frameBytes, Environment.TickCount64, 0);
+    }
+
+    public void ProcessAck(uint ackSeq)
+    {
+        var toRemove = RetransmitBuffer.Keys
+            .Where(k => k <= ackSeq)
+            .ToList();
+        foreach (var k in toRemove)
+            RetransmitBuffer.Remove(k);
+        if (ackSeq > AckedSeq)
+            AckedSeq = ackSeq;
+    }
+
+    /// Returns frames to retransmit and whether the session should close.
+    public (List<byte[]> resend, bool shouldClose) CheckRetransmit()
+    {
+        var resend = new List<byte[]>();
+        var shouldClose = false;
+        var now = Environment.TickCount64;
+        const int timeoutMs = 5;
+        const uint maxRetries = 10;
+
+        foreach (var kv in RetransmitBuffer.ToList())
+        {
+            if (now - kv.Value.Item2 > timeoutMs)
+            {
+                if (kv.Value.Item3 >= maxRetries)
+                {
+                    shouldClose = true;
+                    break;
+                }
+                RetransmitBuffer[kv.Key] = (kv.Value.Item1, now, kv.Value.Item3 + 1);
+                resend.Add(kv.Value.Item1);
+            }
+        }
+        return (resend, shouldClose);
+    }
+}
+
+/// L2 reliability: receiver-side state.
+class RecvState
+{
+    public uint ExpectedSeq;
+    public uint DeliverSeq;
+    // seq -> payload (out-of-order buffer)
+    public readonly SortedList<uint, byte[]> ReceiveBuffer = new();
+
+    /// Process an incoming DATA frame. Returns (payloads to deliver, needAck).
+    public (List<byte[]> deliver, bool needAck) ProcessData(uint seq, byte[] payload)
+    {
+        if (seq < ExpectedSeq)
+        {
+            // Duplicate.
+            return ([], true);
+        }
+        if (seq == ExpectedSeq)
+        {
+            // In-order: deliver and drain buffer.
+            var deliver = new List<byte[]> { payload };
+            ExpectedSeq++;
+            DeliverSeq = ExpectedSeq - 1;
+            while (ReceiveBuffer.Remove(ExpectedSeq, out var buffered))
+            {
+                deliver.Add(buffered);
+                ExpectedSeq++;
+                DeliverSeq = ExpectedSeq - 1;
+            }
+            return (deliver, false);
+        }
+        else
+        {
+            // Out-of-order: buffer.
+            ReceiveBuffer[seq] = payload;
+            return ([], true);
+        }
+    }
+
+    /// The ack_seq to report in outgoing DATA frames.
+    public uint CurrentAckSeq => ExpectedSeq == 0 ? uint.MaxValue : ExpectedSeq - 1;
+}
+
 sealed class Session(
    uint sessionId,
    TcpClient client,
@@ -288,20 +387,45 @@ sealed class Session(
    Action<string>? log) : IDisposable
 {
    readonly CancellationTokenSource _cts = new();
-    readonly Channel<byte[]> _incoming = Channel.CreateBounded<byte[]>(256);
+    readonly SendState _send = new();
+    readonly RecvState _recv = new();
+    readonly object _sendLock = new();
+    readonly object _recvLock = new();

    public void Start()
    {
        _ = PumpSocketToTunnel();
        _ = PumpTunnelToSocket();
+        _ = RetransmitTimer();
    }

-    public void Deliver(byte[] payload)
+    /// Handle a DATA frame from the tunnel: process ack, deliver in-order.
+    public void HandleData(Frame.Data data)
    {
-        if (!_incoming.Writer.TryWrite(payload))
-            log?.Invoke($"session {sessionId}: incoming channel full");
+        // Process ack_seq to advance send window.
+        lock (_sendLock)
+            _send.ProcessAck(data.AckSeq);
+
+        // Process seq for in-order delivery.
+        List<byte[]> deliver;
+        bool needAck;
+        lock (_recvLock)
+            (deliver, needAck) = _recv.ProcessData(data.Seq, data.Payload);
+
+        // Deliver to the TCP socket via the channel.
+        foreach (var chunk in deliver)
+        {
+            if (!_deliverChannel.Writer.TryWrite(chunk))
+                log?.Invoke($"session {sessionId}: deliver channel full");
+        }
+
+        // Send pure ACK if duplicate or out-of-order.
+        if (needAck)
+            SendPureAck();
    }

+    readonly Channel<byte[]> _deliverChannel = Channel.CreateBounded<byte[]>(256);
+
    async Task PumpSocketToTunnel()
    {
        try
@@ -313,7 +437,23 @@ sealed class Session(
            {
                var n = await stream.ReadAsync(buf, _cts.Token);
                if (n == 0) break;
-                link.SendTo(serverMac, new Frame.Data(sessionId, buf[..n]));
+
+                uint seq;
+                uint ackSeq;
+                lock (_sendLock)
+                {
+                    seq = _send.NextSeq();
+                    lock (_recvLock)
+                        ackSeq = _recv.CurrentAckSeq;
+                }
+
+                var frame = new Frame.Data(sessionId, seq, ackSeq, buf[..n]);
+                var frameBytes = FrameCodec.Encode(frame);
+
+                lock (_sendLock)
+                    _send.RecordSent(seq, frameBytes);
+
+                link.SendTo(serverMac, frame);
            }
        }
        catch { }
@@ -326,18 +466,65 @@ sealed class Session(
        try
        {
            var stream = client.GetStream();
-            await foreach (var payload in _incoming.Reader.ReadAllAsync(_cts.Token))
+            await foreach (var payload in _deliverChannel.Reader.ReadAllAsync(_cts.Token))
                await stream.WriteAsync(payload, _cts.Token);
        }
        catch { }
    }

+    async Task RetransmitTimer()
+    {
+        using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(1));
+        try
+        {
+            while (!_cts.IsCancellationRequested)
+            {
+                await timer.WaitForNextTickAsync(_cts.Token);
+
+                List<byte[]>? resend = null;
+                bool shouldClose = false;
+                lock (_sendLock)
+                    (resend, shouldClose) = _send.CheckRetransmit();
+
+                if (resend != null)
+                {
+                    foreach (var frameBytes in resend)
+                    {
+                        // Send raw bytes directly (already encoded).
+                        link.SendRaw(serverMac, frameBytes);
+                    }
+                }
+
+                if (shouldClose)
+                {
+                    link.SendTo(serverMac, new Frame.Close(sessionId, Proto.ReasonMaxRetries));
+                    onClosed();
+                    break;
+                }
+            }
+        }
+        catch { }
+    }
+
+    void SendPureAck()
+    {
+        uint seq, ackSeq;
+        lock (_sendLock)
+        {
+            seq = _send.NextSeq();
+            lock (_recvLock)
+                ackSeq = _recv.CurrentAckSeq;
+        }
+        // Pure ACK: empty payload. Not stored in retransmit buffer.
+        link.SendTo(serverMac, new Frame.Data(sessionId, seq, ackSeq, []));
+    }
+
    void SendClose() => link.SendTo(serverMac, new Frame.Close(sessionId, null));

    public void Dispose()
    {
        _cts.Cancel();
-        _incoming.Writer.TryComplete();
+        _deliverChannel.Writer.TryComplete();
        SendClose();
        try { client.Dispose(); } catch { }
        _cts.Dispose();
@@ -72,7 +72,9 @@ sealed class TunnelLink : IDisposable
        SendRaw(dstMac, FrameCodec.Encode(frame));
    }

-    void SendRaw(byte[] dstMac, byte[] payload)
+    /// Send a pre-encoded protocol payload (for retransmit, where we already
+    /// have the exact bytes and want to avoid re-encoding).
+    public void SendRaw(byte[] dstMac, byte[] payload)
    {
        var frame = new byte[Proto.EthHeaderLen + payload.Length];
        Buffer.BlockCopy(dstMac, 0, frame, 0, 6);