From d453f9406e6475175561c4201d6c011521fd2188 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Mon, 12 May 2025 19:50:44 -0500 Subject: [PATCH 01/60] Working on refactoring `TcpConnection` --- src/core/Akka/IO/Tcp.cs | 29 +- src/core/Akka/IO/TcpConnection.cs | 1019 ++++++--------------- src/core/Akka/IO/TcpOutgoingConnection.cs | 34 +- 3 files changed, 322 insertions(+), 760 deletions(-) diff --git a/src/core/Akka/IO/Tcp.cs b/src/core/Akka/IO/Tcp.cs index 4e12196408d..bb5747d38a7 100644 --- a/src/core/Akka/IO/Tcp.cs +++ b/src/core/Akka/IO/Tcp.cs @@ -50,25 +50,7 @@ public override TcpExt CreateExtension(ExtendedActorSystem system) internal abstract class SocketCompleted : INoSerializationVerificationNeeded, IDeadLetterSuppression { } - - internal sealed class SocketSent : SocketCompleted - { - public static readonly SocketSent Instance = new(); - private SocketSent() { } - } - - internal sealed class SocketReceived : SocketCompleted - { - public static readonly SocketReceived Instance = new(); - private SocketReceived() { } - } - - internal sealed class SocketAccepted : SocketCompleted - { - public static readonly SocketAccepted Instance = new(); - private SocketAccepted() { } - } - + internal sealed class SocketConnected : SocketCompleted { public static readonly SocketConnected Instance = new(); @@ -378,6 +360,11 @@ public CompoundWrite Prepend(SimpleWriteCommand other) { return new CompoundWrite(other, this); } + + /// + /// The number of bytes that will be written to the socket. + /// + public abstract long Bytes { get; } /// /// Prepend a group of writes before this one. @@ -489,6 +476,8 @@ public static Write Create(ByteString data, Event ack) { return new Write(data, ack); } + + public override long Bytes => Data.Count; } /// @@ -540,6 +529,8 @@ private IEnumerable Enumerable() public override string ToString() => $"CompoundWrite({Head}, {TailCommand})"; + + public override long Bytes => Head.Bytes + TailCommand.Bytes; } /// diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 38a15b79768..cbdf45f5635 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -6,7 +6,7 @@ //----------------------------------------------------------------------- using System; -using System.Collections.Concurrent; +using System.Buffers; using System.Collections.Generic; using System.Collections.Immutable; using System.IO; @@ -16,11 +16,12 @@ using Akka.Actor; using Akka.Dispatch; using Akka.Event; -using Akka.IO.Buffers; using Akka.Pattern; using Akka.Util; using Akka.Util.Internal; +#nullable enable + namespace Akka.IO { using static Akka.IO.Tcp; @@ -48,88 +49,107 @@ namespace Akka.IO /// internal abstract class TcpConnection : ActorBase, IRequiresMessageQueue { - [Flags] - enum ConnectionStatus + #region Ack‑aware SAEA + + private sealed class AckSocketAsyncEventArgs : SocketAsyncEventArgs { - /// - /// Marks that connection has invoked Socket.ReceiveAsync and that - /// are currently trying to receive data. - /// - Receiving = 1, + public readonly List<(IActorRef Commander, object Ack)> PendingAcks = new(8); + public void ClearAcks() => PendingAcks.Clear(); + } - /// - /// Marks that connection has invoked Socket.SendAsync and that - /// are currently sending data. It's important as - /// will throw exception if another socket operations will - /// be called over it as it's performing send request. For that reason we cannot release send args - /// back to pool if it's sending (another connection actor could acquire that buffer and try to - /// use it while it's sending the data). - /// - Sending = 1 << 1, + #endregion - /// - /// Marks that current connection has suspended reading. - /// - ReadingSuspended = 1 << 2, + #region completion msgs - /// - /// Marks that current connection has suspended writing. - /// - WritingSuspended = 1 << 3, + private sealed class SocketReceiveCompleted(int bytes, SocketError error) : INoSerializationVerificationNeeded + { + public int Bytes { get; } = bytes; + public SocketError Error { get; } = error; + } - /// - /// Marks that current connection has been requested for shutdown. It may not occur immediatelly, - /// i.e. because underlying is actually sending the data. - /// - ShutdownRequested = 1 << 4 + private sealed class SocketSendCompleted : INoSerializationVerificationNeeded + { + public static readonly SocketSendCompleted Instance = new(); } - private ConnectionStatus _status; + #endregion + protected readonly TcpExt Tcp; protected readonly Socket Socket; - protected SocketAsyncEventArgs ReceiveArgs; - protected SocketAsyncEventArgs SendArgs; + protected ILoggingAdapter Log { get; } = Context.GetLogger(); - protected readonly ILoggingAdapter Log = Context.GetLogger(); - private readonly bool _pullMode; - private readonly PendingSimpleWritesQueue _writeCommandsQueue; - private readonly bool _traceLogging; + private readonly ArrayPool _bufferPool = ArrayPool.Shared; - private bool _isOutputShutdown; + private readonly Queue<(WriteCommand Cmd, IActorRef Sender)> _pendingWrites; - private readonly ConcurrentQueue<(IActorRef Commander, object Ack)> _pendingAcks = new(); - private bool _peerClosed; - private IActorRef _interestedInResume; - private CloseInformation _closedMessage; // for ConnectionClosed message in postStop + private readonly byte[] _receiveBuffer; + private SocketAsyncEventArgs _receiveArgs; + private AckSocketAsyncEventArgs _sendArgs; + private IActorRef _handler = ActorRefs.Nobody; private IActorRef _watchedActor = Context.System.DeadLetters; + private readonly int _maxWriteCapacity; - private readonly IOException droppingWriteBecauseWritingIsSuspendedException = new("Dropping write because writing is suspended"); + private volatile bool _sending; + private volatile bool _closingRequested; + private volatile bool _peerClosed; - private readonly IOException droppingWriteBecauseQueueIsFullException = new("Dropping write because queue is full"); + private readonly bool _traceLogging; - protected TcpConnection(TcpExt tcp, Socket socket, bool pullMode, Option writeCommandsBufferMaxSize) - { - if (socket == null) throw new ArgumentNullException(nameof(socket)); + private bool _isOutputShutdown; + + private CloseInformation _closedMessage; // for ConnectionClosed message in postStop + + private readonly IOException _droppingWriteBecauseWritingIsSuspendedException = + new("Dropping write because writing is suspended"); + + private readonly IOException _droppingWriteBecauseQueueIsFullException = + new("Dropping write because queue is full"); - _pullMode = pullMode; - _writeCommandsQueue = new PendingSimpleWritesQueue(Log, writeCommandsBufferMaxSize); + protected TcpConnection(TcpExt tcp, Socket socket, Option writeCommandsBufferMaxSize) + { + _maxWriteCapacity = writeCommandsBufferMaxSize.GetOrElse(tcp.Settings.WriteCommandsQueueMaxSize); + _pendingWrites = new Queue<(WriteCommand Cmd, IActorRef Sender)>(_maxWriteCapacity); _traceLogging = tcp.Settings.TraceLogging; Tcp = tcp; - Socket = socket; - - if (pullMode) SetStatus(ConnectionStatus.ReadingSuspended); + Socket = socket ?? throw new ArgumentNullException(nameof(socket)); + const int DefaultBufferSize = 64 * 1024; // 64 KiB – matches legacy DirectBufferSize + _receiveBuffer = _bufferPool.Rent(DefaultBufferSize); + InitSocketEventArgs(); } - - private bool IsWritePending + private void InitSocketEventArgs() { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - get { return !HasStatus(ConnectionStatus.Sending) && !_writeCommandsQueue.IsEmpty; } + _receiveArgs = new SocketAsyncEventArgs(); + _receiveArgs.SetBuffer(_receiveBuffer, 0, _receiveBuffer.Length); + _receiveArgs.UserToken = Self; + _receiveArgs.Completed += OnCompleted; + + _sendArgs = new AckSocketAsyncEventArgs(); + _sendArgs.UserToken = Self; + _sendArgs.Completed += OnCompleted; } - private Option GetAllowedPendingWrite() => IsWritePending ? GetNextWrite() : Option.None; + private static void OnCompleted(object? sender, SocketAsyncEventArgs e) + { + if (e.UserToken is not IActorRef self) return; + switch (e.LastOperation) + { + case SocketAsyncOperation.Receive: + self.Tell(new SocketReceiveCompleted(e.BytesTransferred, e.SocketError)); + break; + case SocketAsyncOperation.Send: + self.Tell(SocketSendCompleted.Instance); + break; + case SocketAsyncOperation.Connect: // TODO: need to anchor this to the `TcpOutGoingConnection` implementation + self.Tell(SocketConnected.Instance); + break; + default: + self.Tell(new ErrorClosed($"Unexpected socket op {e.LastOperation}")); + break; + } + } protected void SignDeathPact(IActorRef actor) { @@ -143,8 +163,131 @@ protected void UnsignDeathPact() if (!ReferenceEquals(_watchedActor, Context.System.DeadLetters)) Context.Unwatch(_watchedActor); } + private void IssueReceive() + { + if (!Socket.ReceiveAsync(_receiveArgs)) + Self.Tell(new SocketReceiveCompleted(_receiveArgs.BytesTransferred, _receiveArgs.SocketError)); + } + + private void HandleRead(SocketReceiveCompleted rc) + { + if (rc.Error != SocketError.Success) + { + Log.Error("Closing connection due to IO error {0}", rc.Error); + _handler.Tell(new ErrorClosed(rc.Error.ToString())); + Context.Stop(Self); + return; + } + + if (rc.Bytes == 0) + { + _peerClosed = true; + TryCloseIfDone(); + return; + } + + var bs = ByteString.CopyFrom(_receiveBuffer, 0, rc.Bytes); + _handler.Tell(new Received(bs)); + IssueReceive(); + } + + private void IssueSend(IList> buffers) + { + _sendArgs.BufferList = buffers; + if (!Socket.SendAsync(_sendArgs)) + Self.Tell(SocketSendCompleted.Instance); + } + + private void TrySendNext() + { + if (_sending || _pendingWrites.Count == 0) return; + + var maxBytes = _receiveBuffer.Length; + var accumulated = 0; + var batch = new List(); + _sendArgs.ClearAcks(); + + while (_pendingWrites.Count > 0 && accumulated < maxBytes) + { + var (cmd, snd) = _pendingWrites.Peek(); + switch (cmd) + { + case Write w when !w.Data.IsEmpty: + int wouldBe = accumulated + w.Data.Count; + if (wouldBe > maxBytes && batch.Count > 0) goto done; + _pendingWrites.Dequeue(); + batch.Add(w.Data); + accumulated = wouldBe; + if (!Equals(w.Ack, NoAck.Instance)) + _sendArgs.PendingAcks.Add((snd, w.Ack)); + break; + case Write w: + _pendingWrites.Dequeue(); + if (w.WantsAck) snd.Tell(w.Ack); + break; + default: + _pendingWrites.Dequeue(); + snd.Tell(new CommandFailed(cmd)); + break; + } + } + + done: + if (batch.Count == 0) + { + TrySendNext(); + return; + } + + _sending = true; + var payload = FlattenByteStrings(batch); + IssueSend(payload); + } + + private void HandleSendCompleted() + { + _sending = false; + foreach (var (c, ack) in _sendArgs.PendingAcks) + c.Tell(ack); + _sendArgs.ClearAcks(); + _sendArgs.BufferList.Clear(); + TrySendNext(); + TryCloseIfDone(); + } + + private void TryCloseIfDone() + { + if (!_closingRequested) return; + if (_sending || _pendingWrites.Count > 0) return; + if (!_peerClosed) return; + _handler.Tell(ConfirmedClosed.Instance); + Context.Stop(Self); + } + + private static IList> FlattenByteStrings(List parts) + { + if (parts.Count == 1) + return parts[0].Buffers; + + return parts.SelectMany(c => c.Buffers).ToArray(); + } + // STATES + private bool TryBuffer(WriteCommand cmd, IActorRef sender) + { + if (_pendingWrites.Count < _maxWriteCapacity) + { + _pendingWrites.Enqueue((cmd, sender)); + return true; + } + else + { + // buffer is full + return false; + } + } + /// /// Connection established, waiting for registration from user handler. /// @@ -162,38 +305,30 @@ private Receive WaitingForRegistration(IActorRef commander) if (_traceLogging) Log.Debug("[{0}] registered as connection handler", register.Handler); - var registerInfo = new ConnectionInfo(register.Handler, register.KeepOpenOnPeerClosed, register.UseResumeWriting); + var registerInfo = new ConnectionInfo(register.Handler, register.KeepOpenOnPeerClosed, + register.UseResumeWriting); Context.SetReceiveTimeout(null); Context.Become(Connected(registerInfo)); - - // if we are in push mode or already have resumed reading in pullMode while waiting for Register then read - if (!_pullMode || !HasStatus(ConnectionStatus.ReadingSuspended)) ResumeReading(); - // If there is something buffered before we got Register message - put it all to the socket - var bufferedWrite = GetNextWrite(); - if (bufferedWrite.HasValue) - { - SetStatus(ConnectionStatus.Sending); - DoWrite(registerInfo, bufferedWrite.Value); - } - + TrySendNext(); + // start reading + IssueReceive(); return true; - case ResumeReading _: ClearStatus(ConnectionStatus.ReadingSuspended); return true; - case SuspendReading _: SetStatus(ConnectionStatus.ReadingSuspended); return true; case CloseCommand cmd: var info = new ConnectionInfo(commander, keepOpenOnPeerClosed: false, useResumeWriting: false); HandleClose(info, Sender, cmd.Event); return true; - case ReceiveTimeout _: + case ReceiveTimeout: // after sending `Register` user should watch this actor to make sure // it didn't die because of the timeout - Log.Debug("Configured registration timeout of [{0}] expired, stopping", Tcp.Settings.RegisterTimeout); + Log.Debug("Configured registration timeout of [{0}] expired, stopping", + Tcp.Settings.RegisterTimeout); Context.Stop(Self); return true; case WriteCommand write: - // When getting Write before regestered handler, have to buffer writes until registration - var buffered = _writeCommandsQueue.EnqueueSimpleWrites(write, Sender, out var commandSize); + // Have to buffer writes until registration + var buffered = TryBuffer(write, Sender); if (!buffered) { var writerInfo = new ConnectionInfo(Sender, false, false); @@ -202,7 +337,8 @@ private Receive WaitingForRegistration(IActorRef commander) else { Log.Warning("Received Write command before Register command. " + - "It will be buffered until Register will be received (buffered write size is {0} bytes)", commandSize); + "It will be buffered until Register will be received (buffered write size is {0} bytes)", + write.Bytes); } return true; @@ -216,16 +352,38 @@ private Receive WaitingForRegistration(IActorRef commander) /// private Receive Connected(ConnectionInfo info) { - var handleWrite = HandleWriteMessages(info); return message => { - if (handleWrite(message)) return true; switch (message) { - case SuspendReading _: SuspendReading(); return true; - case ResumeReading _: ResumeReading(); return true; - case SocketReceived _: DoRead(info, null); return true; - case CloseCommand cmd: HandleClose(info, Sender, cmd.Event); return true; + case SocketReceiveCompleted r: + HandleRead(r); + return true; + case WriteCommand write: + var buffered = TryBuffer(write, Sender); + if (!buffered) + { + var writerInfo = new ConnectionInfo(Sender, false, false); + DropWrite(writerInfo, write); + } + else + { + Log.Warning("Received Write command before Register command. " + + "It will be buffered until Register will be received (buffered write size is {0} bytes)", + write.Bytes); + } + + return true; + case SocketSendCompleted: + HandleSendCompleted(); + return true; + case CloseCommand cmd: + HandleClose(info, Sender, cmd.Event); + return true; + case SuspendReading: + case ResumeReading: + // no-ops + return true; default: return false; } }; @@ -246,6 +404,7 @@ private Receive PeerSentEOF(ConnectionInfo info) HandleClose(info, Sender, cmd.Event); return true; } + if (message is ResumeReading) return true; return false; }; @@ -254,15 +413,22 @@ private Receive PeerSentEOF(ConnectionInfo info) /// /// Connection is closing but a write has to be finished first /// - private Receive ClosingWithPendingWrite(ConnectionInfo info, IActorRef closeCommander, ConnectionClosed closedEvent) + private Receive ClosingWithPendingWrite(ConnectionInfo info, IActorRef closeCommander, + ConnectionClosed closedEvent) { return message => { switch (message) { - case SuspendReading _: SuspendReading(); return true; - case ResumeReading _: ResumeReading(); return true; - case SocketReceived _: DoRead(info, closeCommander); return true; + case SuspendReading _: + SuspendReading(); + return true; + case ResumeReading _: + ResumeReading(); + return true; + case SocketReceived _: + DoRead(info, closeCommander); + return true; case SocketSent _: AcknowledgeSent(); if (IsWritePending) @@ -279,8 +445,12 @@ private Receive ClosingWithPendingWrite(ConnectionInfo info, IActorRef closeComm else HandleClose(info, closeCommander, closedEvent); return true; - case WriteFileFailed fail: HandleError(info.Handler, fail.Cause); return true; - case Abort _: HandleClose(info, Sender, Aborted.Instance); return true; + case WriteFileFailed fail: + HandleError(info.Handler, fail.Cause); + return true; + case Abort _: + HandleClose(info, Sender, Aborted.Instance); + return true; default: return false; } }; @@ -293,113 +463,15 @@ private Receive Closing(ConnectionInfo info, IActorRef closeCommander) { switch (message) { - case SuspendReading _: SuspendReading(); return true; - case ResumeReading _: ResumeReading(); return true; - case SocketReceived _: DoRead(info, closeCommander); return true; - case Abort _: HandleClose(info, Sender, Aborted.Instance); return true; - default: return false; - } - }; - } - - private Receive HandleWriteMessages(ConnectionInfo info) - { - return message => - { - switch (message) - { - case SocketSent _: - // Send ack to sender - AcknowledgeSent(); - - // If there is something to send - send it - var pendingWrite = GetAllowedPendingWrite(); - if (pendingWrite.HasValue) - { - SetStatus(ConnectionStatus.Sending); - DoWrite(info, pendingWrite); - } - - // If message is fully sent, notify sender who sent ResumeWriting command - if (!IsWritePending && _interestedInResume != null) - { - _interestedInResume.Tell(WritingResumed.Instance); - _interestedInResume = null; - } - - return true; - case WriteCommand write: - if (HasStatus(ConnectionStatus.WritingSuspended)) - { - if (_traceLogging) Log.Debug("Dropping write because writing is suspended"); - Sender.Tell(write.FailureMessage.WithCause(droppingWriteBecauseWritingIsSuspendedException)); - } - - if (HasStatus(ConnectionStatus.Sending)) - { - // If we are sending something right now, just enqueue incoming write - if (!_writeCommandsQueue.EnqueueSimpleWrites(write, Sender)) - { - DropWrite(info, write); - return true; - } - } - else - { - Option nextWrite; - if (_writeCommandsQueue.IsEmpty) - { - // If writes queue is empty, do not enqueue first write - we will send it immidiately - if (!_writeCommandsQueue.EnqueueSimpleWritesExceptFirst(write, Sender, out var simpleWriteCommand)) - { - DropWrite(info, write); - return true; - } - - nextWrite = GetNextWrite(headCommands: new []{ (simpleWriteCommand, Sender) }); - } - else - { - _writeCommandsQueue.EnqueueSimpleWrites(write, Sender); - nextWrite = GetNextWrite(); - } - - // If there is something to send and we are allowed to, lets put the next command on the wire - if (nextWrite.HasValue) - { - SetStatus(ConnectionStatus.Sending); - DoWrite(info, nextWrite.Value); - } - } - - return true; - case ResumeWriting _: - /* - * If more than one actor sends Writes then the first to send this - * message might resume too early for the second, leading to a Write of - * the second to go through although it has not been resumed yet; there - * is nothing we can do about this apart from all actors needing to - * register themselves and us keeping track of them, which sounds bad. - * - * Thus it is documented that useResumeWriting is incompatible with - * multiple writers. But we fail as gracefully as we can. - */ - ClearStatus(ConnectionStatus.WritingSuspended); - if (IsWritePending) - { - if (_interestedInResume == null) _interestedInResume = Sender; - else Sender.Tell(new CommandFailed(ResumeWriting.Instance)); - } - else Sender.Tell(WritingResumed.Instance); + case SocketReceived _: + DoRead(info, closeCommander); return true; - case UpdatePendingWriteAndThen updatePendingWrite: - var updatedWrite = updatePendingWrite.RemainingWrite; - updatePendingWrite.Work(); - if (updatedWrite.HasValue) - DoWrite(info, updatedWrite.Value); + case Abort _: + HandleClose(info, Sender, Aborted.Instance); return true; - case WriteFileFailed fail: - HandleError(info.Handler, fail.Cause); + case SuspendReading _: + case ResumeReading _: + // no-ops return true; default: return false; } @@ -409,8 +481,7 @@ private Receive HandleWriteMessages(ConnectionInfo info) private void DropWrite(ConnectionInfo info, WriteCommand write) { if (_traceLogging) Log.Debug("Dropping write because queue is full"); - Sender.Tell(write.FailureMessage.WithCause(droppingWriteBecauseQueueIsFullException)); - if (info.UseResumeWriting) SetStatus(ConnectionStatus.WritingSuspended); + Sender.Tell(write.FailureMessage.WithCause(_droppingWriteBecauseQueueIsFullException)); } // AUXILIARIES and IMPLEMENTATION @@ -441,106 +512,6 @@ protected void CompleteConnect(IActorRef commander, IEnumerable 0) - { - //var maxBufferSpace = Math.Min(_tcp.Settings.DirectBufferSize, remainingLimit); - var readBytes = ea.BytesTransferred; - - if (_traceLogging) Log.Debug("Read [{0}] bytes.", readBytes); - if (ea.SocketError == SocketError.Success && readBytes > 0) - info.Handler.Tell(new Received(ByteString.CopyFrom(ea.Buffer, ea.Offset, ea.BytesTransferred))); - - //if (ea.SocketError == SocketError.ConnectionReset) return ReadResult.EndOfStream; - if (ea.SocketError != SocketError.Success) return new ReadResult(ReadResultType.ReadError, ea.SocketError); - if (readBytes > 0) return ReadResult.AllRead; - if (readBytes == 0) return ReadResult.EndOfStream; - - throw new IllegalStateException($"Unexpected value returned from read: {readBytes}"); - } - return ReadResult.AllRead; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void DoWrite(ConnectionInfo info, Option write) - { - if (!write.HasValue) - return; - - // Enqueue all acks assigned to this write to be sent once write is finished - foreach (var pendingAck in write.Value.PendingAcks.Where(ackInfo => !ackInfo.Ack.Equals(NoAck.Instance))) - { - _pendingAcks.Enqueue(pendingAck); - } - - write.Value.DoWrite(info); - } - private void HandleClose(ConnectionInfo info, IActorRef closeCommander, ConnectionClosed closedEvent) { SetStatus(ConnectionStatus.ShutdownRequested); @@ -558,7 +529,7 @@ private void HandleClose(ConnectionInfo info, IActorRef closeCommander, Connecti _peerClosed = true; Context.Become(PeerSentEOF(info)); } - else if (IsWritePending) // finish writing first + else if (IsWritePending) // finish writing first { UnsignDeathPact(); if (_traceLogging) Log.Debug("Got Close command but write is still pending."); @@ -601,7 +572,8 @@ private void DoCloseConnection(ConnectionInfo info, IActorRef closeCommander, Co private void HandleError(IActorRef handler, SocketException exception) { Log.Debug("Closing connection due to IO error {0}", exception); - StopWith(new CloseInformation(new HashSet(new[] { handler }), new ErrorClosed(exception.Message))); + StopWith( + new CloseInformation(new HashSet(new[] { handler }), new ErrorClosed(exception.Message))); } private bool SafeShutdownOutput() @@ -618,73 +590,7 @@ private bool SafeShutdownOutput() } } - protected void AcquireSocketAsyncEventArgs() - { - if (ReceiveArgs != null) throw new InvalidOperationException("Cannot acquire receive SocketAsyncEventArgs. It's already has been initialized"); - if (SendArgs != null) throw new InvalidOperationException("Cannot acquire send SocketAsyncEventArgs. It's already has been initialized"); - - ReceiveArgs = CreateSocketEventArgs(Self); - var buffer = Tcp.BufferPool.Rent(); - ReceiveArgs.SetBuffer(buffer.Array, buffer.Offset, buffer.Count); - - SendArgs = CreateSocketEventArgs(Self); - } - - private void ReleaseSocketAsyncEventArgs() - { - if (ReceiveArgs != null) - { - var buffer = new ByteBuffer(ReceiveArgs.Buffer, ReceiveArgs.Offset, ReceiveArgs.Count); - ReleaseSocketEventArgs(ReceiveArgs); - // TODO: When using DirectBufferPool as a pool impelementation, there is potential risk, - // that socket was working while released. In that case releasing buffer may not be safe. - Tcp.BufferPool.Release(buffer); - ReceiveArgs = null; - } - - if (SendArgs != null) - { - ReleaseSocketEventArgs(SendArgs); - SendArgs = null; - } - } - - protected SocketAsyncEventArgs CreateSocketEventArgs(IActorRef onCompleteNotificationsReceiver) - { - SocketCompleted ResolveMessage(SocketAsyncEventArgs e) - { - switch (e.LastOperation) - { - case SocketAsyncOperation.Receive: - case SocketAsyncOperation.ReceiveFrom: - case SocketAsyncOperation.ReceiveMessageFrom: - return SocketReceived.Instance; - case SocketAsyncOperation.Send: - case SocketAsyncOperation.SendTo: - case SocketAsyncOperation.SendPackets: - return SocketSent.Instance; - case SocketAsyncOperation.Accept: - return SocketAccepted.Instance; - case SocketAsyncOperation.Connect: - return SocketConnected.Instance; - default: - throw new NotSupportedException($"Socket operation {e.LastOperation} is not supported"); - } - } - - var args = new SocketAsyncEventArgs(); - args.UserToken = onCompleteNotificationsReceiver; - args.Completed += (_, e) => - { - var actorRef = e.UserToken as IActorRef; - var completeMsg = ResolveMessage(e); - actorRef?.Tell(completeMsg); - }; - - return args; - } - - protected void ReleaseSocketEventArgs(SocketAsyncEventArgs e) + protected static void ReleaseSocketEventArgs(SocketAsyncEventArgs e) { e.UserToken = null; e.AcceptSocket = null; @@ -696,25 +602,18 @@ protected void ReleaseSocketEventArgs(SocketAsyncEventArgs e) e.BufferList = null; } // it can be that for some reason socket is in use and haven't closed yet - catch (InvalidOperationException) { } + catch (InvalidOperationException) + { + } e.Dispose(); - - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void CloseSocket() - { - Socket.Dispose(); - _isOutputShutdown = true; - ReleaseSocketAsyncEventArgs(); } - + private void Abort() { try { - Socket.LingerState = new LingerOption(true, 0); // causes the following close() to send TCP RST + Socket.LingerState = new LingerOption(true, 0); // causes the following close() to send TCP RST } catch (Exception e) { @@ -731,52 +630,14 @@ protected void StopWith(CloseInformation closeInfo) Context.Stop(Self); } - private void ReceiveAsync() - { - if (!HasStatus(ConnectionStatus.Receiving)) - { - if (!Socket.ReceiveAsync(ReceiveArgs)) - Self.Tell(SocketReceived.Instance); - - SetStatus(ConnectionStatus.Receiving); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool HasStatus(ConnectionStatus connectionStatus) - { - // don't use Enum.HasFlag - it's using reflection underneat - var s = (int)connectionStatus; - return ((int)_status & s) == s; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void SetStatus(ConnectionStatus connectionStatus) => _status |= connectionStatus; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void ClearStatus(ConnectionStatus connectionStatus) => _status &= ~connectionStatus; - protected override void PostStop() { - if (Socket.Connected) Abort(); - else CloseSocket(); - - // We do never store pending writes between messages anymore, so nothing is acquired and nothing to release - - // always try to release SocketAsyncEventArgs to avoid memory leaks - ReleaseSocketAsyncEventArgs(); - - if (_closedMessage != null) - { - var interestedInClose = _writeCommandsQueue.TryGetNext(out var pending) - ? _closedMessage.NotificationsTo.Union(_writeCommandsQueue.DequeueAll().Select(cmd => cmd.Sender)) - : _closedMessage.NotificationsTo; - - foreach (var listener in interestedInClose) - { - listener.Tell(_closedMessage.ClosedEvent); - } - } + try { Socket.Shutdown(SocketShutdown.Both); } catch { /* ignore */ } + Socket.Dispose(); + _receiveArgs.Dispose(); + _sendArgs.Dispose(); + _bufferPool.Return(_receiveBuffer); + base.PostStop(); } protected override void PostRestart(Exception reason) @@ -784,69 +645,6 @@ protected override void PostRestart(Exception reason) throw new IllegalStateException("Restarting not supported for connection actors."); } - private Option GetNextWrite(IEnumerable<(SimpleWriteCommand Command, IActorRef Sender)> headCommands = null) - { - headCommands = headCommands ?? ImmutableList<(SimpleWriteCommand Command, IActorRef Sender)>.Empty; - var writeCommands = new List<(Write Command, IActorRef Sender)>(_writeCommandsQueue.ItemsCount); - foreach (var commandInfo in headCommands.Concat(_writeCommandsQueue.DequeueAll())) - { - switch (commandInfo.Command) - { - case Write w when !w.Data.IsEmpty: - // Have real write - go on and put it to the wire - writeCommands.Add((w, commandInfo.Sender)); - break; - case Write w: - // Write command is empty, so just sending Ask if required - if (w.WantsAck) commandInfo.Sender.Tell(w.Ack); - break; - default: - //TODO: there's no TransmitFile API - .NET Core doesn't support it at all - throw new InvalidOperationException("Non reachable code"); - } - } - - if (writeCommands.Count > 0) - { - return CreatePendingBufferWrite(writeCommands); - } - - // No more writes out there - return Option.None; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private PendingWrite CreatePendingBufferWrite(List<(Write Command, IActorRef Sender)> writes) - { - var acks = writes.Select(w => (w.Sender, (object)w.Command.Ack)).ToImmutableList(); - var dataList = writes.Select(w => w.Command.Data); - return new PendingBufferWrite(this, SendArgs, Self, acks, dataList, Tcp.BufferPool); - } - - //TODO: Port File IO - currently .NET Core doesn't support TransmitFile API - - private enum ReadResultType - { - EndOfStream, - AllRead, - ReadError, - } - - private struct ReadResult - { - public static readonly ReadResult EndOfStream = new(ReadResultType.EndOfStream, SocketError.Success); - public static readonly ReadResult AllRead = new(ReadResultType.AllRead, SocketError.Success); - - public readonly ReadResultType Type; - public readonly SocketError Error; - - public ReadResult(ReadResultType type, SocketError error) - { - Type = type; - Error = error; - } - } - /// /// Used to transport information to the postStop method to notify /// interested party about a connection close. @@ -857,6 +655,7 @@ protected sealed class CloseInformation /// TBD /// public ISet NotificationsTo { get; } + public Tcp.Event ClosedEvent { get; } public CloseInformation(ISet notificationsTo, Tcp.Event closedEvent) @@ -865,255 +664,5 @@ public CloseInformation(ISet notificationsTo, Tcp.Event closedEvent) ClosedEvent = closedEvent; } } - - /// - /// Groups required connection-related data that are only available once the connection has been fully established. - /// - private sealed class ConnectionInfo - { - public readonly IActorRef Handler; - public readonly bool KeepOpenOnPeerClosed; - public readonly bool UseResumeWriting; - - public ConnectionInfo(IActorRef handler, bool keepOpenOnPeerClosed, bool useResumeWriting) - { - Handler = handler; - KeepOpenOnPeerClosed = keepOpenOnPeerClosed; - UseResumeWriting = useResumeWriting; - } - } - - // INTERNAL MESSAGES - private sealed class UpdatePendingWriteAndThen : INoSerializationVerificationNeeded - { - public Option RemainingWrite { get; } - public Action Work { get; } - - public UpdatePendingWriteAndThen(Option remainingWrite, Action work) - { - RemainingWrite = remainingWrite; - Work = work; - } - } - - private sealed class WriteFileFailed - { - public SocketException Cause { get; } - - public WriteFileFailed(SocketException cause) - { - Cause = cause; - } - } - - private abstract class PendingWrite - { - public IImmutableList<(IActorRef Commander, object Ack)> PendingAcks { get; } - - protected PendingWrite(IImmutableList<(IActorRef Commander, object Ack)> pendingAcks) - { - PendingAcks = pendingAcks; - } - - public abstract void DoWrite(ConnectionInfo info); - } - - private sealed class PendingBufferWrite : PendingWrite - { - private readonly TcpConnection _connection; - private readonly IActorRef _self; - private readonly IEnumerable _dataToSend; - private readonly IBufferPool _bufferPool; - private readonly SocketAsyncEventArgs _sendArgs; - - public PendingBufferWrite( - TcpConnection connection, - SocketAsyncEventArgs sendArgs, - IActorRef self, - IImmutableList<(IActorRef Commander, object Ack)> acks, - IEnumerable dataToSend, - IBufferPool bufferPool) : base(acks) - { - _connection = connection; - _sendArgs = sendArgs; - _self = self; - _dataToSend = dataToSend; - _bufferPool = bufferPool; - } - - public override void DoWrite(ConnectionInfo info) - { - try - { - _sendArgs.SetBuffer(_dataToSend); - if (!_connection.Socket.SendAsync(_sendArgs)) - _self.Tell(SocketSent.Instance); - } - catch (SocketException e) - { - _connection.HandleError(info.Handler, e); - } - } - } - - public class PendingSimpleWritesQueue - { - private readonly ILoggingAdapter _log; - private readonly Option _maxQueueSizeInBytes; - private readonly Queue<(SimpleWriteCommand Command, IActorRef Commander, int Size)> _queue; - private int _totalSizeInBytes = 0; - - public PendingSimpleWritesQueue(ILoggingAdapter log, Option maxQueueSizeInBytes) - { - _log = log; - _maxQueueSizeInBytes = maxQueueSizeInBytes; - _queue = new Queue<(SimpleWriteCommand Command, IActorRef Commander, int Size)>(); - } - - /// - /// Gets total number of items in queue - /// - public int ItemsCount => _queue.Count; - - /// - /// Adds all subcommands stored in provided command. - /// Performs buffer size checks - /// - /// - /// Thrown when data to buffer is larger then allowed - /// - public bool EnqueueSimpleWrites(WriteCommand command, IActorRef sender) - { - return EnqueueSimpleWrites(command, sender, out _); - } - - /// - /// Adds all subcommands stored in provided command. - /// Performs buffer size checks - /// - /// - /// Thrown when data to buffer is larger then allowed - /// - public bool EnqueueSimpleWrites(WriteCommand command, IActorRef sender, out int bufferedSize) - { - bufferedSize = 0; - - foreach (var writeInfo in ExtractFromCommand(command)) - { - var sizeAfterAppending = _totalSizeInBytes + writeInfo.DataSize; - if (_maxQueueSizeInBytes.HasValue && _maxQueueSizeInBytes.Value < sizeAfterAppending) - { - _log.Warning("Could not receive write command of size {0} bytes, " + - "because buffer limit is {1} bytes and " + - "it is already {2} bytes", writeInfo.DataSize, _maxQueueSizeInBytes, _totalSizeInBytes); - return false; - } - - _totalSizeInBytes = sizeAfterAppending; - _queue.Enqueue((writeInfo.Command, sender, writeInfo.DataSize)); - bufferedSize += writeInfo.DataSize; - } - - return true; - } - - /// - /// Adds all subcommands stored in provided command. - /// Performs buffer size checks for all, except first one, that is not buffered - /// - /// - /// Not buffered (and not checked) first - /// - /// - /// Thrown when data to buffer is larger then allowed - /// - public bool EnqueueSimpleWritesExceptFirst(WriteCommand command, IActorRef sender, out SimpleWriteCommand first) - { - first = null; - foreach (var writeInfo in ExtractFromCommand(command)) - { - if (first == null) - { - first = writeInfo.Command; - continue; - } - - var sizeAfterAppending = _totalSizeInBytes + writeInfo.DataSize; - if (_maxQueueSizeInBytes.HasValue && _maxQueueSizeInBytes.Value < sizeAfterAppending) - { - _log.Warning("Could not receive write command of size {0} bytes, " + - "because buffer limit is {1} bytes and " + - "it is already {2} bytes", writeInfo.DataSize, _maxQueueSizeInBytes, _totalSizeInBytes); - return false; - } - - _totalSizeInBytes = sizeAfterAppending; - _queue.Enqueue((writeInfo.Command, sender, writeInfo.DataSize)); - } - - return true; - } - - /// - /// Gets next command from the queue, if any - /// - public (SimpleWriteCommand, IActorRef Sender) Dequeue() - { - if (_queue.Count == 0) - throw new InvalidOperationException("Write commands queue is empty"); - - var (command, sender, size) = _queue.Dequeue(); - _totalSizeInBytes -= size; - return (command, sender); - } - - /// - /// Dequeue all elements one by one - /// - /// - public IEnumerable<(SimpleWriteCommand Command, IActorRef Sender)> DequeueAll() - { - while (TryGetNext(out var command)) - yield return command; - } - - /// - /// Gets next command from the queue, if any - /// - public bool TryGetNext(out (SimpleWriteCommand Command, IActorRef Sender) command) - { - command = default; - if (_queue.Count == 0) - return false; - - command = Dequeue(); - return true; - } - - /// - /// Checks if commands queue is empty - /// - public bool IsEmpty => _totalSizeInBytes == 0; - - private IEnumerable<(SimpleWriteCommand Command, int DataSize)> ExtractFromCommand(WriteCommand command) - { - switch (command) - { - case Write write: - yield return (write, write.Data.Count); - break; - case CompoundWrite compoundWrite: - var extractedFromHead = ExtractFromCommand(compoundWrite.Head); - var extractedFromTail = ExtractFromCommand(compoundWrite.TailCommand); - foreach (var extractedSimple in extractedFromHead.Concat(extractedFromTail)) - { - yield return extractedSimple; - } - break; - default: - throw new ArgumentException($"Trying to calculate size of unknown write type: {command.GetType().FullName}"); - } - } - } } -} +} \ No newline at end of file diff --git a/src/core/Akka/IO/TcpOutgoingConnection.cs b/src/core/Akka/IO/TcpOutgoingConnection.cs index 075caa4ee82..54a1cd65136 100644 --- a/src/core/Akka/IO/TcpOutgoingConnection.cs +++ b/src/core/Akka/IO/TcpOutgoingConnection.cs @@ -29,7 +29,7 @@ internal sealed class TcpOutgoingConnection : TcpConnection private SocketAsyncEventArgs _connectArgs; - private readonly ConnectException finishConnectNeverReturnedTrueException = + private readonly ConnectException _finishConnectNeverReturnedTrueException = new("Could not establish connection because finishConnect never returned true"); public TcpOutgoingConnection(TcpExt tcp, IActorRef commander, Tcp.Connect connect) @@ -38,7 +38,6 @@ public TcpOutgoingConnection(TcpExt tcp, IActorRef commander, Tcp.Connect connec tcp.Settings.OutgoingSocketForceIpv4 ? new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp) { Blocking = false } : new Socket(SocketType.Stream, ProtocolType.Tcp) { Blocking = false }, - connect.PullMode, tcp.Settings.WriteCommandsQueueMaxSize >= 0 ? tcp.Settings.WriteCommandsQueueMaxSize : Option.None) { _commander = commander; @@ -99,7 +98,7 @@ protected override void PreStart() if (resolved == null) Become(Resolving(remoteAddress)); else if (resolved.Ipv4.Any() && resolved.Ipv6.Any()) // one of both families - Register(new IPEndPoint(resolved.Ipv4.FirstOrDefault(), remoteAddress.Port), new IPEndPoint(resolved.Ipv6.FirstOrDefault(), remoteAddress.Port)); + Register(new IPEndPoint(resolved.Ipv4.First(), remoteAddress.Port), new IPEndPoint(resolved.Ipv6.First(), remoteAddress.Port)); else // one or the other Register(new IPEndPoint(resolved.Addr, remoteAddress.Port), null); } @@ -133,8 +132,8 @@ private Receive Resolving(DnsEndPoint remoteAddress) if (resolved.Ipv4.Any() && resolved.Ipv6.Any()) // multiple addresses { ReportConnectFailure(() => Register( - new IPEndPoint(resolved.Ipv4.FirstOrDefault(), remoteAddress.Port), - new IPEndPoint(resolved.Ipv6.FirstOrDefault(), remoteAddress.Port))); + new IPEndPoint(resolved.Ipv4.First(), remoteAddress.Port), + new IPEndPoint(resolved.Ipv6.First(), remoteAddress.Port))); } else // only one address family. No fallbacks. { @@ -148,6 +147,29 @@ private Receive Resolving(DnsEndPoint remoteAddress) }; } + private static SocketAsyncEventArgs CreateSocketEventArgs(IActorRef onCompleteNotificationsReceiver) + { + var args = new SocketAsyncEventArgs(); + args.UserToken = onCompleteNotificationsReceiver; + args.Completed += (_, e) => + { + var actorRef = e.UserToken as IActorRef; + var completeMsg = ResolveMessage(e); + actorRef?.Tell(completeMsg); + }; + + return args; + + Tcp.SocketCompleted ResolveMessage(SocketAsyncEventArgs e) + { + return e.LastOperation switch + { + SocketAsyncOperation.Connect => IO.Tcp.SocketConnected.Instance, + _ => throw new NotSupportedException($"Socket operation {e.LastOperation} is not supported") + }; + } + } + private void Register(IPEndPoint address, IPEndPoint fallbackAddress) { ReportConnectFailure(() => @@ -205,7 +227,7 @@ private Receive Connecting(int remainingFinishConnectRetries, SocketAsyncEventAr else { Log.Debug("Could not establish connection because finishConnect never returned true (consider increasing akka.io.tcp.finish-connect-retries)"); - Stop(finishConnectNeverReturnedTrueException); + Stop(_finishConnectNeverReturnedTrueException); } return true; } From 03b27d8f780bb1afb9f6e6a4d6a27771a36c64b8 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 13 May 2025 12:33:36 -0500 Subject: [PATCH 02/60] removing more garbage --- src/core/Akka/IO/TcpConnection.cs | 355 ++++++++++++---------- src/core/Akka/IO/TcpOutgoingConnection.cs | 15 +- 2 files changed, 200 insertions(+), 170 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index cbdf45f5635..bba3a91516c 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -67,9 +67,10 @@ private sealed class SocketReceiveCompleted(int bytes, SocketError error) : INoS public SocketError Error { get; } = error; } - private sealed class SocketSendCompleted : INoSerializationVerificationNeeded + private sealed class SocketSendCompleted(int bytes, SocketError error) : INoSerializationVerificationNeeded { - public static readonly SocketSendCompleted Instance = new(); + public int Bytes { get; } = bytes; + public SocketError Error { get; } = error; } #endregion @@ -85,8 +86,7 @@ private sealed class SocketSendCompleted : INoSerializationVerificationNeeded private readonly byte[] _receiveBuffer; private SocketAsyncEventArgs _receiveArgs; private AckSocketAsyncEventArgs _sendArgs; - - private IActorRef _handler = ActorRefs.Nobody; + private IActorRef _watchedActor = Context.System.DeadLetters; private readonly int _maxWriteCapacity; @@ -96,14 +96,15 @@ private sealed class SocketSendCompleted : INoSerializationVerificationNeeded private readonly bool _traceLogging; - private bool _isOutputShutdown; - - private CloseInformation _closedMessage; // for ConnectionClosed message in postStop + private CloseInformation? _closedMessage; // for ConnectionClosed message in postStop + + private static readonly IOException DroppingWriteBecauseClosingException = + new("Dropping write because the connection is closing"); - private readonly IOException _droppingWriteBecauseWritingIsSuspendedException = + private static readonly IOException DroppingWriteBecauseWritingIsSuspendedException = new("Dropping write because writing is suspended"); - private readonly IOException _droppingWriteBecauseQueueIsFullException = + private static readonly IOException DroppingWriteBecauseQueueIsFullException = new("Dropping write because queue is full"); protected TcpConnection(TcpExt tcp, Socket socket, Option writeCommandsBufferMaxSize) @@ -140,7 +141,7 @@ private static void OnCompleted(object? sender, SocketAsyncEventArgs e) self.Tell(new SocketReceiveCompleted(e.BytesTransferred, e.SocketError)); break; case SocketAsyncOperation.Send: - self.Tell(SocketSendCompleted.Instance); + self.Tell(new SocketSendCompleted(e.BytesTransferred, e.SocketError)); break; case SocketAsyncOperation.Connect: // TODO: need to anchor this to the `TcpOutGoingConnection` implementation self.Tell(SocketConnected.Instance); @@ -150,6 +151,11 @@ private static void OnCompleted(object? sender, SocketAsyncEventArgs e) break; } } + + /// + /// Returns true if a write is in-progress over the wire or if we have writes pending in the queue. + /// + public bool IsWritePending => _sending || _pendingWrites.Count > 0; protected void SignDeathPact(IActorRef actor) { @@ -169,25 +175,29 @@ private void IssueReceive() Self.Tell(new SocketReceiveCompleted(_receiveArgs.BytesTransferred, _receiveArgs.SocketError)); } - private void HandleRead(SocketReceiveCompleted rc) + private void HandleRead(IActorRef handler, SocketReceiveCompleted rc) { + if(_traceLogging) + Log.Debug("Received {0} bytes from {1}", rc.Bytes, Socket.RemoteEndPoint); + if (rc.Error != SocketError.Success) { Log.Error("Closing connection due to IO error {0}", rc.Error); - _handler.Tell(new ErrorClosed(rc.Error.ToString())); - Context.Stop(Self); + Self.Tell(new ErrorClosed(rc.Error.ToString())); return; } if (rc.Bytes == 0) { _peerClosed = true; - TryCloseIfDone(); + + // signal to the handler that the peer has closed the connection + Self.Tell(PeerClosed.Instance); return; } var bs = ByteString.CopyFrom(_receiveBuffer, 0, rc.Bytes); - _handler.Tell(new Received(bs)); + handler.Tell(new Received(bs)); IssueReceive(); } @@ -195,12 +205,12 @@ private void IssueSend(IList> buffers) { _sendArgs.BufferList = buffers; if (!Socket.SendAsync(_sendArgs)) - Self.Tell(SocketSendCompleted.Instance); + Self.Tell(new SocketSendCompleted(_sendArgs.BytesTransferred, _sendArgs.SocketError)); } private void TrySendNext() { - if (_sending || _pendingWrites.Count == 0) return; + if (IsWritePending) return; var maxBytes = _receiveBuffer.Length; var accumulated = 0; @@ -243,24 +253,54 @@ private void TrySendNext() var payload = FlattenByteStrings(batch); IssueSend(payload); } + + private void FailWritesWithAck(IEnumerable<(IActorRef sender, object Ack)> acks, Exception cause) + { + foreach (var (sender, ack) in acks) + { + sender.Tell(new CommandFailed()); + } + } - private void HandleSendCompleted() + private void HandleSendCompleted(SocketSendCompleted socketSendCompleted) { _sending = false; + + if(_traceLogging) + Log.Debug("Sent {0} bytes to {1}", socketSendCompleted.Bytes, Socket.RemoteEndPoint); + + // check for errors + if (socketSendCompleted.Error != SocketError.Success) + { + Log.Error("Closing connection due to IO error {0} received during send", socketSendCompleted.Error); + Self.Tell(new ErrorClosed(socketSendCompleted.Error.ToString())); + return; + } + foreach (var (c, ack) in _sendArgs.PendingAcks) c.Tell(ack); _sendArgs.ClearAcks(); _sendArgs.BufferList.Clear(); + TrySendNext(); TryCloseIfDone(); } + + private void DeliverCloseMessages() + { + if (_closedMessage == null) return; + foreach (var handler in _closedMessage.NotificationsTo) + { + handler.Tell(_closedMessage.ClosedEvent); + } + } private void TryCloseIfDone() { if (!_closingRequested) return; if (_sending || _pendingWrites.Count > 0) return; if (!_peerClosed) return; - _handler.Tell(ConfirmedClosed.Instance); + DeliverCloseMessages(); Context.Stop(Self); } @@ -281,11 +321,9 @@ private bool TryBuffer(WriteCommand cmd, IActorRef sender) _pendingWrites.Enqueue((cmd, sender)); return true; } - else - { - // buffer is full - return false; - } + + // buffer is full + return false; } /// @@ -317,7 +355,7 @@ private Receive WaitingForRegistration(IActorRef commander) return true; case CloseCommand cmd: var info = new ConnectionInfo(commander, keepOpenOnPeerClosed: false, useResumeWriting: false); - HandleClose(info, Sender, cmd.Event); + HandleCloseEvent(info, Sender, cmd.Event); return true; case ReceiveTimeout: // after sending `Register` user should watch this actor to make sure @@ -331,8 +369,7 @@ private Receive WaitingForRegistration(IActorRef commander) var buffered = TryBuffer(write, Sender); if (!buffered) { - var writerInfo = new ConnectionInfo(Sender, false, false); - DropWrite(writerInfo, write); + DropWrite(write); } else { @@ -342,6 +379,22 @@ private Receive WaitingForRegistration(IActorRef commander) } return true; + case Terminated t: + { + // if the handler dies before registration, we need to stop + if (t.ActorRef.Equals(commander)) + { + Log.Debug("Handler [{0}] died before registration, stopping", t.ActorRef); + Context.Stop(Self); + } + else + { + // ignore + Log.Debug("Handler [{0}] died before registration, ignoring", t.ActorRef); + } + + return true; + } default: return false; } }; @@ -357,14 +410,13 @@ private Receive Connected(ConnectionInfo info) switch (message) { case SocketReceiveCompleted r: - HandleRead(r); + HandleRead(info.Handler, r); return true; case WriteCommand write: var buffered = TryBuffer(write, Sender); if (!buffered) { - var writerInfo = new ConnectionInfo(Sender, false, false); - DropWrite(writerInfo, write); + DropWrite(write); } else { @@ -374,11 +426,14 @@ private Receive Connected(ConnectionInfo info) } return true; - case SocketSendCompleted: - HandleSendCompleted(); + case SocketSendCompleted sendCompleted: + HandleSendCompleted(sendCompleted); return true; - case CloseCommand cmd: - HandleClose(info, Sender, cmd.Event); + case CloseCommand cmd: // we are trying to close the socket first + HandleCloseCommand(info, Sender, cmd); + return true; + case ConnectionClosed closed: // peer is closing the socket + HandleCloseEvent(info, Sender, closed); return true; case SuspendReading: case ResumeReading: @@ -389,99 +444,75 @@ private Receive Connected(ConnectionInfo info) }; } - /// - /// The peer sent EOF first, but we may still want to send - /// - private Receive PeerSentEOF(ConnectionInfo info) + private void HandleCloseCommand(ConnectionInfo info, IActorRef sender, CloseCommand cmd) { - var handleWrite = HandleWriteMessages(info); - return message => - { - if (handleWrite(message)) return true; - var cmd = message as CloseCommand; - if (cmd != null) - { - HandleClose(info, Sender, cmd.Event); - return true; - } - - if (message is ResumeReading) return true; - return false; - }; + } /// /// Connection is closing but a write has to be finished first /// - private Receive ClosingWithPendingWrite(ConnectionInfo info, IActorRef closeCommander, + private Receive Closing(ConnectionInfo info, IActorRef closeCommander, ConnectionClosed closedEvent) { return message => { switch (message) { - case SuspendReading _: - SuspendReading(); - return true; - case ResumeReading _: - ResumeReading(); - return true; - case SocketReceived _: - DoRead(info, closeCommander); + case SocketReceiveCompleted r: + HandleRead(info.Handler, r); return true; - case SocketSent _: - AcknowledgeSent(); - if (IsWritePending) - DoWrite(info, GetAllowedPendingWrite()); - else - HandleClose(info, closeCommander, closedEvent); + case SocketSendCompleted s: + HandleSendCompleted(s); return true; - case UpdatePendingWriteAndThen updatePendingWrite: - var nextWrite = updatePendingWrite.RemainingWrite; - updatePendingWrite.Work(); - - if (nextWrite.HasValue) - DoWrite(info, nextWrite); - else - HandleClose(info, closeCommander, closedEvent); + case WriteCommand write: + DropWrite(write); return true; - case WriteFileFailed fail: - HandleError(info.Handler, fail.Cause); + case SuspendReading: + case ResumeReading: + // no-ops return true; case Abort _: - HandleClose(info, Sender, Aborted.Instance); + HandleCloseEvent(info, Sender, Aborted.Instance); return true; default: return false; } }; } - /** connection is closed on our side and we're waiting from confirmation from the other side */ - private Receive Closing(ConnectionInfo info, IActorRef closeCommander) + private enum DropReason { - return message => + QueueFull = 1, + Closing = 2, + WritingSuspended = 3 + } + + private static string GetDropReasonMessage(DropReason reason) + { + return reason switch { - switch (message) - { - case SocketReceived _: - DoRead(info, closeCommander); - return true; - case Abort _: - HandleClose(info, Sender, Aborted.Instance); - return true; - case SuspendReading _: - case ResumeReading _: - // no-ops - return true; - default: return false; - } + DropReason.QueueFull => "queue is full", + DropReason.Closing => "connection is closing", + DropReason.WritingSuspended => "writing is suspended", + _ => throw new ArgumentOutOfRangeException(nameof(reason), reason, null) + }; + } + + private static IOException GetDropMessageException(DropReason reason) + { + return reason switch + { + DropReason.QueueFull => DroppingWriteBecauseQueueIsFullException, + DropReason.Closing => DroppingWriteBecauseClosingException, + DropReason.WritingSuspended => DroppingWriteBecauseWritingIsSuspendedException, + _ => throw new ArgumentOutOfRangeException(nameof(reason), reason, null) }; } - private void DropWrite(ConnectionInfo info, WriteCommand write) + private void DropWrite(WriteCommand write, DropReason reason = DropReason.QueueFull) { - if (_traceLogging) Log.Debug("Dropping write because queue is full"); - Sender.Tell(write.FailureMessage.WithCause(_droppingWriteBecauseQueueIsFullException)); + if (_traceLogging) Log.Debug("Dropping write because {0}", GetDropReasonMessage(reason)); + Sender.Tell(write.FailureMessage.WithCause(GetDropMessageException(reason))); } // AUXILIARIES and IMPLEMENTATION @@ -498,7 +529,7 @@ protected void CompleteConnect(IActorRef commander, IEnumerable(); - if (info.Handler != null) notifications.Add(info.Handler); - if (closeCommander != null) notifications.Add(closeCommander); + var notifications = new HashSet { info.Handler, closeCommander }; StopWith(new CloseInformation(notifications, closedEvent)); } - private void HandleError(IActorRef handler, SocketException exception) - { - Log.Debug("Closing connection due to IO error {0}", exception); - StopWith( - new CloseInformation(new HashSet(new[] { handler }), new ErrorClosed(exception.Message))); - } - private bool SafeShutdownOutput() { try { Socket.Shutdown(SocketShutdown.Send); - _isOutputShutdown = true; return true; } catch (SocketException) @@ -589,25 +611,6 @@ private bool SafeShutdownOutput() return false; } } - - protected static void ReleaseSocketEventArgs(SocketAsyncEventArgs e) - { - e.UserToken = null; - e.AcceptSocket = null; - - try - { - e.SetBuffer(null, 0, 0); - if (e.BufferList != null) - e.BufferList = null; - } - // it can be that for some reason socket is in use and haven't closed yet - catch (InvalidOperationException) - { - } - - e.Dispose(); - } private void Abort() { @@ -644,6 +647,23 @@ protected override void PostRestart(Exception reason) { throw new IllegalStateException("Restarting not supported for connection actors."); } + + /// + /// Groups required connection-related data that are only available once the connection has been fully established. + /// + private sealed class ConnectionInfo + { + public readonly IActorRef Handler; + public readonly bool KeepOpenOnPeerClosed; + public readonly bool UseResumeWriting; + + public ConnectionInfo(IActorRef handler, bool keepOpenOnPeerClosed, bool useResumeWriting) + { + Handler = handler; + KeepOpenOnPeerClosed = keepOpenOnPeerClosed; + UseResumeWriting = useResumeWriting; + } + } /// /// Used to transport information to the postStop method to notify @@ -651,9 +671,6 @@ protected override void PostRestart(Exception reason) /// protected sealed class CloseInformation { - /// - /// TBD - /// public ISet NotificationsTo { get; } public Tcp.Event ClosedEvent { get; } diff --git a/src/core/Akka/IO/TcpOutgoingConnection.cs b/src/core/Akka/IO/TcpOutgoingConnection.cs index 54a1cd65136..42c5df5510d 100644 --- a/src/core/Akka/IO/TcpOutgoingConnection.cs +++ b/src/core/Akka/IO/TcpOutgoingConnection.cs @@ -61,7 +61,20 @@ private void ReleaseConnectionSocketArgs() { if (_connectArgs != null) { - ReleaseSocketEventArgs(_connectArgs); + _connectArgs.UserToken = null; + _connectArgs.AcceptSocket = null; + + try + { + _connectArgs.SetBuffer(null, 0, 0); + _connectArgs.BufferList = null; + } + // it can be that for some reason socket is in use and haven't closed yet + catch (InvalidOperationException) + { + } + + _connectArgs.Dispose(); _connectArgs = null; } } From fc375840934a7768822897c734d05382538aaf43 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 13 May 2025 13:29:48 -0500 Subject: [PATCH 03/60] fixed compilation errors --- src/core/Akka/IO/TcpConnection.cs | 215 ++++++++++++++-------- src/core/Akka/IO/TcpIncomingConnection.cs | 4 +- src/core/Akka/IO/TcpOutgoingConnection.cs | 1 - 3 files changed, 143 insertions(+), 77 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index bba3a91516c..e3964746559 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -95,6 +95,9 @@ private sealed class SocketSendCompleted(int bytes, SocketError error) : INoSeri private volatile bool _peerClosed; private readonly bool _traceLogging; + + // so we don't try to close the socket a second time during PostStop + private bool _socketAlreadyClosed; private CloseInformation? _closedMessage; // for ConnectionClosed message in postStop @@ -143,9 +146,6 @@ private static void OnCompleted(object? sender, SocketAsyncEventArgs e) case SocketAsyncOperation.Send: self.Tell(new SocketSendCompleted(e.BytesTransferred, e.SocketError)); break; - case SocketAsyncOperation.Connect: // TODO: need to anchor this to the `TcpOutGoingConnection` implementation - self.Tell(SocketConnected.Instance); - break; default: self.Tell(new ErrorClosed($"Unexpected socket op {e.LastOperation}")); break; @@ -171,6 +171,7 @@ protected void UnsignDeathPact() private void IssueReceive() { + if(_peerClosed) return; // can't read if peer is closed if (!Socket.ReceiveAsync(_receiveArgs)) Self.Tell(new SocketReceiveCompleted(_receiveArgs.BytesTransferred, _receiveArgs.SocketError)); } @@ -254,12 +255,17 @@ private void TrySendNext() IssueSend(payload); } - private void FailWritesWithAck(IEnumerable<(IActorRef sender, object Ack)> acks, Exception cause) + /// + /// Called when the socket closes before we have processed all pending writes. + /// + private void FailUnprocessedPendingWrites(Exception cause) { - foreach (var (sender, ack) in acks) + foreach (var (cmd, ack) in _pendingWrites) { - sender.Tell(new CommandFailed()); + var failure = cmd.FailureMessage.WithCause(cause); + ack.Tell(failure); } + _pendingWrites.Clear(); } private void HandleSendCompleted(SocketSendCompleted socketSendCompleted) @@ -300,7 +306,6 @@ private void TryCloseIfDone() if (!_closingRequested) return; if (_sending || _pendingWrites.Count > 0) return; if (!_peerClosed) return; - DeliverCloseMessages(); Context.Stop(Self); } @@ -345,6 +350,11 @@ private Receive WaitingForRegistration(IActorRef commander) var registerInfo = new ConnectionInfo(register.Handler, register.KeepOpenOnPeerClosed, register.UseResumeWriting); + + // we set a default close message here in case the actor dies before we get a close message + // this will prevent close messages from going missing + _closedMessage = + new CloseInformation(new HashSet([register.Handler]), Aborted.Instance); Context.SetReceiveTimeout(null); Context.Become(Connected(registerInfo)); @@ -355,7 +365,7 @@ private Receive WaitingForRegistration(IActorRef commander) return true; case CloseCommand cmd: var info = new ConnectionInfo(commander, keepOpenOnPeerClosed: false, useResumeWriting: false); - HandleCloseEvent(info, Sender, cmd.Event); + HandleCloseCommand(info, Sender, cmd); return true; case ReceiveTimeout: // after sending `Register` user should watch this actor to make sure @@ -424,7 +434,6 @@ private Receive Connected(ConnectionInfo info) "It will be buffered until Register will be received (buffered write size is {0} bytes)", write.Bytes); } - return true; case SocketSendCompleted sendCompleted: HandleSendCompleted(sendCompleted); @@ -439,21 +448,21 @@ private Receive Connected(ConnectionInfo info) case ResumeReading: // no-ops return true; + case Terminated t: // handler died + { + Log.Debug("Handler [{0}] died, stopping", t.ActorRef); + Context.Stop(Self); + return true; + } default: return false; } }; } - private void HandleCloseCommand(ConnectionInfo info, IActorRef sender, CloseCommand cmd) - { - - } - /// /// Connection is closing but a write has to be finished first /// - private Receive Closing(ConnectionInfo info, IActorRef closeCommander, - ConnectionClosed closedEvent) + private Receive Closing(ConnectionInfo info, bool confirmClose) { return message => { @@ -464,6 +473,14 @@ private Receive Closing(ConnectionInfo info, IActorRef closeCommander, return true; case SocketSendCompleted s: HandleSendCompleted(s); + if (confirmClose && !IsWritePending) + { + // done writing, so we can now half-close the socket + if (_traceLogging) Log.Debug("Running in close-confirm mode, half-closing socket for writes"); + + // We will need to get an EOF + Socket.Shutdown(SocketShutdown.Send); + } return true; case WriteCommand write: DropWrite(write); @@ -472,10 +489,25 @@ private Receive Closing(ConnectionInfo info, IActorRef closeCommander, case ResumeReading: // no-ops return true; - case Abort _: - HandleCloseEvent(info, Sender, Aborted.Instance); + case Abort a: + HandleCloseCommand(info, Sender, a); return true; - default: return false; + case ConnectionClosed: + _peerClosed = true; + if (!IsWritePending) + { + Log.Debug("Peer closed connection, stopping");; + Context.Stop(Self); + } + return true; + case Terminated t: // handler died + { + Log.Debug("Handler [{0}] died, stopping", t.ActorRef); + Context.Stop(Self); + return true; + } + default: + return false; } }; } @@ -542,74 +574,103 @@ protected void CompleteConnect(IActorRef commander, IEnumerable + /// We are in the driver's seat and want to close the connection. + /// + private void HandleCloseCommand(ConnectionInfo info, IActorRef sender, CloseCommand cmd) { - switch (closedEvent) + // we are closing the connection, so set the hook now. + _closedMessage = new CloseInformation(new HashSet { info.Handler, sender }, cmd.Event); + + switch (cmd) { - case Aborted: + case Abort _: { if (_traceLogging) Log.Debug("Got Abort command. RESETing connection."); - _peerClosed = true; - _closingRequested = true; - DoCloseConnection(info, closeCommander, closedEvent); + Abort(); break; } - default: + case Close: { - if (IsWritePending) // finish writing first - { - UnsignDeathPact(); - if (_traceLogging) Log.Debug("Got Close command but write is still pending."); - Context.Become(ClosingWithPendingWrite(info, closeCommander, closedEvent)); - } - else if (closedEvent is ConfirmedClosed) // shutdown output and wait for confirmation + _closingRequested = true; + if (IsWritePending) // if we have writes pending, we need to send them { - if (_traceLogging) Log.Debug("Got ConfirmedClose command, sending FIN."); - - // If peer closed first, the socket is now fully closed. - // Also, if shutdownOutput threw an exception we expect this to be an indication - // that the peer closed first or concurrently with this code running. - if (_peerClosed || !SafeShutdownOutput()) - DoCloseConnection(info, closeCommander, closedEvent); - else Context.Become(Closing(info, closeCommander)); + if(_traceLogging) Log.Debug("Got Close command but writes are still pending."); + Become(Closing(info, false)); } - // close gracefully now else { + // if we are not writing, we can close the socket right away if (_traceLogging) Log.Debug("Got Close command, closing connection."); - Socket.Shutdown(SocketShutdown.Both); - DoCloseConnection(info, closeCommander, closedEvent); + CloseSocket(); + Context.Stop(Self); } - break; } + case ConfirmedClose: + { + _closingRequested = true; + if(_traceLogging) Log.Debug("Got ConfirmedClose command - waiting for peer to terminate."); + Become(Closing(info, true)); + break; + } + default: + throw new ArgumentOutOfRangeException(nameof(cmd), cmd, "Unknown close command"); } } - private void DoCloseConnection(ConnectionInfo info, IActorRef closeCommander, ConnectionClosed closedEvent) + /// + /// Someone else is closing the connection, so we need to handle it. + /// + private void HandleCloseEvent(ConnectionInfo info, IActorRef closeCommander, ConnectionClosed closedEvent) { - if (closedEvent is Aborted) Abort(); - else + _closedMessage = new CloseInformation(new HashSet { info.Handler, closeCommander }, closedEvent); + _closingRequested = true; + + switch (closedEvent) { - CloseSocket(); + case Aborted: + { + if (_traceLogging) Log.Debug("Got Aborted event. RESETing connection."); + Context.Stop(Self); + break; + } + case PeerClosed: + { + if (_traceLogging) Log.Debug("Got PeerClosed event. Closing connection."); + _peerClosed = true; + if (info.KeepOpenOnPeerClosed) + { + // we are not closing the socket, but we need to stop reading + Context.Become(Closing(info, false)); + } + else + { + // we are closing the socket + CloseSocket(); + Context.Stop(Self); + } + break; + } + default: + { + // log a warning - someone sent us the wrong message type + Log.Warning("Received unexpected ConnectionClosed event type [{0}]", closedEvent.GetType()); + + // closing connection anyway, I guess + Become(Closing(info, false)); + break; + } } - - var notifications = new HashSet { info.Handler, closeCommander }; - StopWith(new CloseInformation(notifications, closedEvent)); } - - private bool SafeShutdownOutput() + + /* Mostly called from outside */ + protected void StopWith(CloseInformation closeInfo) { - try - { - Socket.Shutdown(SocketShutdown.Send); - return true; - } - catch (SocketException) - { - return false; - } + _closedMessage = closeInfo; + UnsignDeathPact(); + Context.Stop(Self); } private void Abort() @@ -622,24 +683,32 @@ private void Abort() { if (_traceLogging) Log.Debug("setSoLinger(true, 0) failed with [{0}]", e); } - - CloseSocket(); + Context.Stop(Self); } - protected void StopWith(CloseInformation closeInfo) + private void CloseSocket() { - _closedMessage = closeInfo; - UnsignDeathPact(); - Context.Stop(Self); + if (_socketAlreadyClosed) return; + _socketAlreadyClosed = true; + try { Socket.Shutdown(SocketShutdown.Both); } catch { /* ignore */ } + try{ Socket.Dispose(); } catch { /* ignore */ } } protected override void PostStop() { - try { Socket.Shutdown(SocketShutdown.Both); } catch { /* ignore */ } - Socket.Dispose(); + if (_traceLogging) Log.Debug("Stopping connection actor [{0}]", Self); + CloseSocket(); // just in case we didn't shut ourselves down gracefully first _receiveArgs.Dispose(); _sendArgs.Dispose(); _bufferPool.Return(_receiveBuffer); + + FailUnprocessedPendingWrites(DroppingWriteBecauseClosingException); + if(_closedMessage != null) + { + // if we have a close message, we need to deliver it + DeliverCloseMessages(); + } + base.PostStop(); } @@ -673,7 +742,7 @@ protected sealed class CloseInformation { public ISet NotificationsTo { get; } - public Tcp.Event ClosedEvent { get; } + public Event ClosedEvent { get; } public CloseInformation(ISet notificationsTo, Tcp.Event closedEvent) { diff --git a/src/core/Akka/IO/TcpIncomingConnection.cs b/src/core/Akka/IO/TcpIncomingConnection.cs index 7774e4c2430..9d78de589ef 100644 --- a/src/core/Akka/IO/TcpIncomingConnection.cs +++ b/src/core/Akka/IO/TcpIncomingConnection.cs @@ -34,7 +34,7 @@ public TcpIncomingConnection(TcpExt tcp, IActorRef bindHandler, IEnumerable options, bool readThrottling) - : base(tcp, socket, readThrottling, Option.None) + : base(tcp, socket, Option.None) { _bindHandler = bindHandler; _options = options; @@ -44,8 +44,6 @@ public TcpIncomingConnection(TcpExt tcp, protected override void PreStart() { - AcquireSocketAsyncEventArgs(); - CompleteConnect(_bindHandler, _options); } diff --git a/src/core/Akka/IO/TcpOutgoingConnection.cs b/src/core/Akka/IO/TcpOutgoingConnection.cs index 42c5df5510d..2418d4f5419 100644 --- a/src/core/Akka/IO/TcpOutgoingConnection.cs +++ b/src/core/Akka/IO/TcpOutgoingConnection.cs @@ -211,7 +211,6 @@ private Receive Connecting(int remainingFinishConnectRetries, SocketAsyncEventAr Log.Debug("Connection established to [{0}]", _connect.RemoteAddress); ReleaseConnectionSocketArgs(); - AcquireSocketAsyncEventArgs(); CompleteConnect(_commander, _connect.Options); } From 0b5d391070e597f232b71c018324094c5f9f62a6 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 13 May 2025 13:49:19 -0500 Subject: [PATCH 04/60] fixed all compilation errors --- src/core/Akka.Tests/IO/TcpIntegrationSpec.cs | 4 +- src/core/Akka/IO/TcpConnection.cs | 12 +++-- src/core/Akka/IO/TcpOutgoingConnection.cs | 51 +++++++++++--------- 3 files changed, 37 insertions(+), 30 deletions(-) diff --git a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs index 0d9cf70d698..ebed72498a5 100644 --- a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs +++ b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs @@ -519,8 +519,8 @@ await AwaitConditionNoThrowAsync(() => private async Task ChitChat(TestSetup.ConnectionDetail actors, int rounds = 100) { - var testData = ByteString.FromBytes(new byte[] {(byte) 0}); - for (int i = 0; i < rounds; i++) + var testData = ByteString.FromBytes([0]); + for (var i = 0; i < rounds; i++) { actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(testData)); await actors.ServerHandler.ExpectMsgAsync(x => x.Data.Count == 1 && x.Data[0] == 0, hint: $"server didn't received at {i} round"); diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index e3964746559..2c85795254e 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -113,8 +113,10 @@ private sealed class SocketSendCompleted(int bytes, SocketError error) : INoSeri protected TcpConnection(TcpExt tcp, Socket socket, Option writeCommandsBufferMaxSize) { _maxWriteCapacity = writeCommandsBufferMaxSize.GetOrElse(tcp.Settings.WriteCommandsQueueMaxSize); - _pendingWrites = new Queue<(WriteCommand Cmd, IActorRef Sender)>(_maxWriteCapacity); - _traceLogging = tcp.Settings.TraceLogging; + _pendingWrites = _maxWriteCapacity > 0 + ? new Queue<(WriteCommand Cmd, IActorRef Sender)>(_maxWriteCapacity) + : new Queue<(WriteCommand Cmd, IActorRef Sender)>(); // unbounded +; _traceLogging = tcp.Settings.TraceLogging; Tcp = tcp; Socket = socket ?? throw new ArgumentNullException(nameof(socket)); @@ -246,7 +248,6 @@ private void TrySendNext() done: if (batch.Count == 0) { - TrySendNext(); return; } @@ -321,12 +322,13 @@ private static IList> FlattenByteStrings(List par private bool TryBuffer(WriteCommand cmd, IActorRef sender) { - if (_pendingWrites.Count < _maxWriteCapacity) + // buffer is unlimited OR we're below the max write capacity + if (_maxWriteCapacity < 0 || _pendingWrites.Count < _maxWriteCapacity) { _pendingWrites.Enqueue((cmd, sender)); return true; } - + // buffer is full return false; } diff --git a/src/core/Akka/IO/TcpOutgoingConnection.cs b/src/core/Akka/IO/TcpOutgoingConnection.cs index 2418d4f5419..ff21f3203cf 100644 --- a/src/core/Akka/IO/TcpOutgoingConnection.cs +++ b/src/core/Akka/IO/TcpOutgoingConnection.cs @@ -214,32 +214,37 @@ private Receive Connecting(int remainingFinishConnectRetries, SocketAsyncEventAr CompleteConnect(_commander, _connect.Options); } - else if (remainingFinishConnectRetries > 0 && fallbackAddress != null) // used only when we've resolved a DNS endpoint. + else switch (remainingFinishConnectRetries) { - var self = Self; - var previousAddress = (IPEndPoint)args.RemoteEndPoint; - args.RemoteEndPoint = fallbackAddress; - Context.System.Scheduler.Advanced.ScheduleOnce(TimeSpan.FromMilliseconds(1), () => + // used only when we've resolved a DNS endpoint. + case > 0 when fallbackAddress != null: { - if (!Socket.ConnectAsync(args)) - self.Tell(IO.Tcp.SocketConnected.Instance); - }); - Context.Become(Connecting(remainingFinishConnectRetries - 1, args, previousAddress)); - } - else if (remainingFinishConnectRetries > 0) - { - var self = Self; - Context.System.Scheduler.Advanced.ScheduleOnce(TimeSpan.FromMilliseconds(1), () => + var self = Self; + var previousAddress = (IPEndPoint)args.RemoteEndPoint; + args.RemoteEndPoint = fallbackAddress; + Context.System.Scheduler.Advanced.ScheduleOnce(TimeSpan.FromMilliseconds(1), () => + { + if (!Socket.ConnectAsync(args)) + self.Tell(IO.Tcp.SocketConnected.Instance); + }); + Context.Become(Connecting(remainingFinishConnectRetries - 1, args, previousAddress)); + break; + } + case > 0: { - if (!Socket.ConnectAsync(args)) - self.Tell(IO.Tcp.SocketConnected.Instance); - }); - Context.Become(Connecting(remainingFinishConnectRetries - 1, args, null)); - } - else - { - Log.Debug("Could not establish connection because finishConnect never returned true (consider increasing akka.io.tcp.finish-connect-retries)"); - Stop(_finishConnectNeverReturnedTrueException); + var self = Self; + Context.System.Scheduler.Advanced.ScheduleOnce(TimeSpan.FromMilliseconds(1), () => + { + if (!Socket.ConnectAsync(args)) + self.Tell(IO.Tcp.SocketConnected.Instance); + }); + Context.Become(Connecting(remainingFinishConnectRetries - 1, args, null)); + break; + } + default: + Log.Debug("Could not establish connection because finishConnect never returned true (consider increasing akka.io.tcp.finish-connect-retries)"); + Stop(_finishConnectNeverReturnedTrueException); + break; } return true; } From 49018691186cd95600507e2d61bc1475cb4c78bc Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 13 May 2025 14:22:21 -0500 Subject: [PATCH 05/60] fixes --- src/core/Akka/IO/TcpConnection.cs | 15 +++++++++++---- src/core/Akka/IO/TcpIncomingConnection.cs | 17 ++--------------- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 2c85795254e..77afd64bc58 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -121,6 +121,7 @@ protected TcpConnection(TcpExt tcp, Socket socket, Option writeCommandsBuff Tcp = tcp; Socket = socket ?? throw new ArgumentNullException(nameof(socket)); const int DefaultBufferSize = 64 * 1024; // 64 KiB – matches legacy DirectBufferSize + // TODO: need to set the actual syscall buffer sizes to match _receiveBuffer = _bufferPool.Rent(DefaultBufferSize); InitSocketEventArgs(); } @@ -183,14 +184,18 @@ private void HandleRead(IActorRef handler, SocketReceiveCompleted rc) if(_traceLogging) Log.Debug("Received {0} bytes from {1}", rc.Bytes, Socket.RemoteEndPoint); + // todo: need to harden our SocketError handling if (rc.Error != SocketError.Success) { Log.Error("Closing connection due to IO error {0}", rc.Error); Self.Tell(new ErrorClosed(rc.Error.ToString())); return; } + + // TODO: what about a closed for writing event handler? + // When the peer says "I'm not sending you any more data, but you should send me some?" - if (rc.Bytes == 0) + if (rc.Bytes == 0) // CLOSED FOR READING { _peerClosed = true; @@ -213,7 +218,8 @@ private void IssueSend(IList> buffers) private void TrySendNext() { - if (IsWritePending) return; + // already sending or no writes to send + if (_sending || _pendingWrites.Count == 0) return; var maxBytes = _receiveBuffer.Length; var accumulated = 0; @@ -235,6 +241,7 @@ private void TrySendNext() _sendArgs.PendingAcks.Add((snd, w.Ack)); break; case Write w: + // empty write, discard and ACK if needed - can't send a 0-length message _pendingWrites.Dequeue(); if (w.WantsAck) snd.Tell(w.Ack); break; @@ -354,7 +361,8 @@ private Receive WaitingForRegistration(IActorRef commander) register.UseResumeWriting); // we set a default close message here in case the actor dies before we get a close message - // this will prevent close messages from going missing + // this will prevent close messages from going missing + // part of the fix for https://github.com/akkadotnet/akka.net/issues/7634 _closedMessage = new CloseInformation(new HashSet([register.Handler]), Aborted.Instance); @@ -389,7 +397,6 @@ private Receive WaitingForRegistration(IActorRef commander) "It will be buffered until Register will be received (buffered write size is {0} bytes)", write.Bytes); } - return true; case Terminated t: { diff --git a/src/core/Akka/IO/TcpIncomingConnection.cs b/src/core/Akka/IO/TcpIncomingConnection.cs index 9d78de589ef..1c0ba7a1536 100644 --- a/src/core/Akka/IO/TcpIncomingConnection.cs +++ b/src/core/Akka/IO/TcpIncomingConnection.cs @@ -20,15 +20,7 @@ internal sealed class TcpIncomingConnection : TcpConnection { private readonly IActorRef _bindHandler; private readonly IEnumerable _options; - - /// - /// TBD - /// - /// TBD - /// TBD - /// TBD - /// TBD - /// TBD + public TcpIncomingConnection(TcpExt tcp, Socket socket, IActorRef bindHandler, @@ -46,12 +38,7 @@ protected override void PreStart() { CompleteConnect(_bindHandler, _options); } - - /// - /// TBD - /// - /// TBD - /// TBD + protected override bool Receive(object message) { throw new NotSupportedException(); From 2e5fd78ca7c65314e065179fae089ae8b97e926e Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 13 May 2025 14:28:22 -0500 Subject: [PATCH 06/60] fixed write-send loop --- src/core/Akka/IO/TcpConnection.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 77afd64bc58..eb48d62e37c 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -294,7 +294,7 @@ private void HandleSendCompleted(SocketSendCompleted socketSendCompleted) foreach (var (c, ack) in _sendArgs.PendingAcks) c.Tell(ack); _sendArgs.ClearAcks(); - _sendArgs.BufferList.Clear(); + _sendArgs.BufferList = null; TrySendNext(); TryCloseIfDone(); @@ -439,9 +439,7 @@ private Receive Connected(ConnectionInfo info) } else { - Log.Warning("Received Write command before Register command. " + - "It will be buffered until Register will be received (buffered write size is {0} bytes)", - write.Bytes); + TrySendNext(); } return true; case SocketSendCompleted sendCompleted: From a75b0c8caec589e0f637259d540aaa51a716cf9a Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 13 May 2025 14:59:07 -0500 Subject: [PATCH 07/60] suppressed dead letters on `SocketReceiveCompleted` and `SocketSendCompleted` --- src/core/Akka.Tests/IO/TcpIntegrationSpec.cs | 2 +- src/core/Akka/IO/TcpConnection.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs index ebed72498a5..f30c79e888a 100644 --- a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs +++ b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs @@ -395,7 +395,7 @@ public async Task Should_fail_writing_when_buffer_is_filled() await AwaitAssertAsync(async () => { // try sending overflow - actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(overflowData)); // this is sent immidiately + actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(overflowData)); // this is sent immediately actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(overflowData)); // this will try to buffer await actors.ClientHandler.ExpectMsgAsync(TimeSpan.FromSeconds(20)); diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index eb48d62e37c..8bddb1c1b7f 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -61,13 +61,13 @@ private sealed class AckSocketAsyncEventArgs : SocketAsyncEventArgs #region completion msgs - private sealed class SocketReceiveCompleted(int bytes, SocketError error) : INoSerializationVerificationNeeded + private sealed class SocketReceiveCompleted(int bytes, SocketError error) : INoSerializationVerificationNeeded, IDeadLetterSuppression { public int Bytes { get; } = bytes; public SocketError Error { get; } = error; } - private sealed class SocketSendCompleted(int bytes, SocketError error) : INoSerializationVerificationNeeded + private sealed class SocketSendCompleted(int bytes, SocketError error) : INoSerializationVerificationNeeded, IDeadLetterSuppression { public int Bytes { get; } = bytes; public SocketError Error { get; } = error; From 1e8705aa1092e57d5413399e06fd628b4b597937 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 13 May 2025 15:09:39 -0500 Subject: [PATCH 08/60] fixed a number of unit tests --- src/core/Akka.Tests/IO/TcpIntegrationSpec.cs | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs index f30c79e888a..b2735894cc7 100644 --- a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs +++ b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs @@ -60,7 +60,7 @@ public async Task The_TCP_transport_implementation_should_properly_bind_a_test_s await new TestSetup(this).RunAsync(async _ => await Task.CompletedTask); } - [Fact(Skip="FIXME .net core / linux")] + [Fact] public async Task The_TCP_transport_implementation_should_allow_connecting_to_and_disconnecting_from_the_test_server() { await new TestSetup(this).RunAsync(async x => @@ -75,7 +75,7 @@ public async Task The_TCP_transport_implementation_should_allow_connecting_to_an }); } - [Fact(Skip="FIXME .net core / linux")] + [Fact] public async Task The_TCP_transport_implementation_should_properly_handle_connection_abort_from_client_side() { await new TestSetup(this).RunAsync(async x => @@ -89,7 +89,7 @@ public async Task The_TCP_transport_implementation_should_properly_handle_connec }); } - [Fact(Skip="FIXME .net core / linux")] + [Fact] public async Task The_TCP_transport_implementation_should_properly_handle_connection_abort_from_client_side_after_chit_chat() { await new TestSetup(this).RunAsync(async x => @@ -114,7 +114,8 @@ public async Task The_TCP_transport_implementation_should_properly_handle_connec actors.ClientHandler.Send(actors.ClientConnection, PoisonPill.Instance); await VerifyActorTermination(actors.ClientConnection); - await actors.ServerHandler.ExpectMsgAsync(); + // PeerClosed because the PostStop of the client connection actor will have it send a graceful termination + await actors.ServerHandler.ExpectMsgAsync(); await VerifyActorTermination(actors.ServerConnection); }); } @@ -130,7 +131,7 @@ public async Task The_TCP_transport_implementation_should_properly_handle_connec actors.ClientHandler.Send(actors.ClientConnection, PoisonPill.Instance); await VerifyActorTermination(actors.ClientConnection); - await actors.ServerHandler.ExpectMsgAsync(); + await actors.ServerHandler.ExpectMsgAsync(); await VerifyActorTermination(actors.ServerConnection); }); } @@ -144,7 +145,7 @@ public async Task The_TCP_transport_implementation_should_properly_handle_connec actors.ServerHandler.Send(actors.ServerConnection, PoisonPill.Instance); await VerifyActorTermination(actors.ServerConnection); - await actors.ClientHandler.ExpectMsgAsync(); + await actors.ClientHandler.ExpectMsgAsync(); await VerifyActorTermination(actors.ClientConnection); }); } @@ -160,7 +161,7 @@ public async Task The_TCP_transport_implementation_should_properly_handle_connec actors.ServerHandler.Send(actors.ServerConnection, PoisonPill.Instance); await VerifyActorTermination(actors.ServerConnection); - await actors.ClientHandler.ExpectMsgAsync(); + await actors.ClientHandler.ExpectMsgAsync(); await VerifyActorTermination(actors.ClientConnection); }); } @@ -170,11 +171,6 @@ public async Task The_TCP_transport_implementation_should_properly_handle_connec [Theory] public async Task The_TCP_transport_implementation_should_properly_support_connecting_to_DNS_endpoints(AddressFamily family) { - // Aaronontheweb, 9/2/2017 - POSIX-based OSES are still having trouble with IPV6 DNS resolution - if(!RuntimeInformation - .IsOSPlatform(OSPlatform.Windows) && family == AddressFamily.InterNetworkV6) - return; - var serverHandler = CreateTestProbe(); var bindCommander = CreateTestProbe(); bindCommander.Send(Sys.Tcp(), new Tcp.Bind(serverHandler.Ref, new IPEndPoint(family == AddressFamily.InterNetwork ? IPAddress.Loopback From 37b8d0a885e7589b1e85f9e298f61725716705a5 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 13 May 2025 15:46:36 -0500 Subject: [PATCH 09/60] need a solution for computing max frame size and max buffer size --- src/core/Akka/IO/TcpConnection.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 8bddb1c1b7f..1bf4c811cc8 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -95,6 +95,8 @@ private sealed class SocketSendCompleted(int bytes, SocketError error) : INoSeri private volatile bool _peerClosed; private readonly bool _traceLogging; + + private long _pendingOutboundBytes; // so we don't try to close the socket a second time during PostStop private bool _socketAlreadyClosed; @@ -156,7 +158,7 @@ private static void OnCompleted(object? sender, SocketAsyncEventArgs e) } /// - /// Returns true if a write is in-progress over the wire or if we have writes pending in the queue. + /// Returns true if write is in-progress over the wire or if we have writes pending in the queue. /// public bool IsWritePending => _sending || _pendingWrites.Count > 0; @@ -333,6 +335,7 @@ private bool TryBuffer(WriteCommand cmd, IActorRef sender) if (_maxWriteCapacity < 0 || _pendingWrites.Count < _maxWriteCapacity) { _pendingWrites.Enqueue((cmd, sender)); + _pendingOutboundBytes += cmd.Bytes; return true; } From 187b26786c841035a0c502fc10d9ba2b9cbe7add Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Wed, 14 May 2025 19:47:31 -0500 Subject: [PATCH 10/60] redid `TcpSettings` to include max-frame-size and buffer sizes --- src/core/Akka.Tests/IO/TcpSettingsSpec.cs | 8 +- src/core/Akka/Configuration/Config.cs | 2 + src/core/Akka/Configuration/akka.conf | 48 +++------- src/core/Akka/IO/Tcp.cs | 41 +-------- src/core/Akka/IO/TcpSettings.cs | 107 +++++++++++++++++----- 5 files changed, 103 insertions(+), 103 deletions(-) diff --git a/src/core/Akka.Tests/IO/TcpSettingsSpec.cs b/src/core/Akka.Tests/IO/TcpSettingsSpec.cs index b9b55059787..e256707dd18 100644 --- a/src/core/Akka.Tests/IO/TcpSettingsSpec.cs +++ b/src/core/Akka.Tests/IO/TcpSettingsSpec.cs @@ -25,18 +25,16 @@ public void TcpSettings_should_parse_all_akka_io_tcp_config_values_correctly() var settings = TcpSettings.Create(tcpConfig); // Assert: all values match akka.conf reference - settings.BufferPoolConfigPath.Should().Be("akka.io.tcp.disabled-buffer-pool"); - settings.InitialSocketAsyncEventArgs.Should().Be(32); settings.TraceLogging.Should().BeFalse(); settings.BatchAcceptLimit.Should().Be(Environment.ProcessorCount * 2); settings.RegisterTimeout.Should().Be(TimeSpan.FromSeconds(5)); - settings.ReceivedMessageSizeLimit.Should().Be(int.MaxValue); settings.ManagementDispatcher.Should().Be("akka.actor.internal-dispatcher"); - settings.FileIODispatcher.Should().Be("akka.actor.default-blocking-io-dispatcher"); - settings.TransferToLimit.Should().Be(524288); // 512 KiB settings.FinishConnectRetries.Should().Be(5); settings.OutgoingSocketForceIpv4.Should().BeFalse(); settings.WriteCommandsQueueMaxSize.Should().Be(-1); + settings.SendBufferSize.Should().Be(8192); + settings.ReceiveBufferSize.Should().Be(8192); + settings.MaxFrameSizeBytes.Should().Be(4096); } } } \ No newline at end of file diff --git a/src/core/Akka/Configuration/Config.cs b/src/core/Akka/Configuration/Config.cs index 9141a781012..78d77d1e5b0 100644 --- a/src/core/Akka/Configuration/Config.cs +++ b/src/core/Akka/Configuration/Config.cs @@ -7,6 +7,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Linq; using Akka.Configuration.Hocon; using Akka.Util.Internal; @@ -149,6 +150,7 @@ public virtual bool GetBoolean(string path, bool @default = false) /// Default return value if none provided. /// This exception is thrown if the current node is undefined. /// The long value defined in the specified path. + [return: NotNullIfNotNull(nameof(def))] public virtual long? GetByteSize(string path, long? def = null) { HoconValue value = GetNode(path); diff --git a/src/core/Akka/Configuration/akka.conf b/src/core/Akka/Configuration/akka.conf index 602d7c8d4ef..df5704d8f94 100644 --- a/src/core/Akka/Configuration/akka.conf +++ b/src/core/Akka/Configuration/akka.conf @@ -773,57 +773,33 @@ akka { # its commander before aborting the connection. register-timeout = 5s - # The maximum number of bytes delivered by a `Received` message. Before - # more data is read from the network the connection actor will try to - # do other work. - # The purpose of this setting is to impose a smaller limit than the - # configured receive buffer size. When using value 'unlimited' it will - # try to read all from the receive buffer. - max-received-message-size = unlimited + # The maximum size of a message that can be flushed onto the wire at once. + maximum-frame-size = 4k + + # The size of the system send buffer. Should be at least 2x the maximum + # frame size. The default value is 8kB, which is the default value for + # the underlying Socket.SendBufferSize property on Windows. + send-buffer-size = 8k + + # The size of the system receive buffer. Should be at least 2x the maximum + # frame size. The default value is 8kB, which is the default value for + # the underlying Socket.ReceiveBufferSize property on Windows. + receive-buffer-size = 8k # Enable fine grained logging of what goes on inside the implementation. # Be aware that this may log more than once per message sent to the actors # of the tcp implementation. trace-logging = off - # Fully qualified config path which holds the dispatcher configuration - # to be used for running the select() calls in the selectors - selector-dispatcher = "akka.io.pinned-dispatcher" - - # Fully qualified config path which holds the dispatcher configuration - # for the read/write worker actors - worker-dispatcher = "akka.actor.internal-dispatcher" - # Fully qualified config path which holds the dispatcher configuration # for the selector management actors management-dispatcher = "akka.actor.internal-dispatcher" - # Fully qualified config path which holds the dispatcher configuration - # on which file IO tasks are scheduled - file-io-dispatcher = "akka.actor.default-blocking-io-dispatcher" - - # The maximum number of bytes (or "unlimited") to transfer in one batch - # when using `WriteFile` command which uses `FileChannel.transferTo` to - # pipe files to a TCP socket. On some OS like Linux `FileChannel.transferTo` - # may block for a long time when network IO is faster than file IO. - # Decreasing the value may improve fairness while increasing may improve - # throughput. - file-io-transferTo-limit = 524288 # 512 KiB - # The number of times to retry the `finishConnect` call after being notified about # OP_CONNECT. Retries are needed if the OP_CONNECT notification doesn't imply that # `finishConnect` will succeed, which is the case on Android. finish-connect-retries = 5 - # On Windows connection aborts are not reliably detected unless an OP_READ is - # registered on the selector _after_ the connection has been reset. This - # workaround enables an OP_CONNECT which forces the abort to be visible on Windows. - # Enabling this setting on other platforms than Windows will cause various failures - # and undefined behavior. - # Possible values of this key are on, off and auto where auto will enable the - # workaround if Windows is detected automatically. - windows-connection-abort-workaround-enabled = off - # Enforce outgoing socket connection to use IPv4 address family. Required in # scenario when IPv6 is not available, for example in Azure Web App sandbox. # When set to true it is required to set akka.io.dns.inet-address.use-ipv6 to false diff --git a/src/core/Akka/IO/Tcp.cs b/src/core/Akka/IO/Tcp.cs index bb5747d38a7..67dd9cbde06 100644 --- a/src/core/Akka/IO/Tcp.cs +++ b/src/core/Akka/IO/Tcp.cs @@ -891,14 +891,7 @@ public TcpExt(ExtendedActorSystem system) : this(system, TcpSettings.Create(syst internal TcpExt(ExtendedActorSystem system, TcpSettings settings) { - var bufferPoolConfig = system.Settings.Config.GetConfig(settings.BufferPoolConfigPath); - - if (bufferPoolConfig.IsNullOrEmpty()) - throw new ConfigurationException($"Cannot retrieve TCP buffer pool configuration: {settings.BufferPoolConfigPath} configuration node not found"); - Settings = settings; - FileIoDispatcher = system.Dispatchers.Lookup(Settings.FileIODispatcher); - BufferPool = CreateBufferPool(system, bufferPoolConfig); Manager = system.SystemActorOf( props: Props.Create(() => new TcpManager(this)).WithDispatcher(Settings.ManagementDispatcher).WithDeploy(Deploy.Local), name: "IO-TCP"); @@ -908,43 +901,11 @@ internal TcpExt(ExtendedActorSystem system, TcpSettings settings) /// Gets reference to a TCP manager actor. /// public override IActorRef Manager { get; } - - /// - /// A buffer pool used by current plugin. - /// - public IBufferPool BufferPool { get; } - + /// /// The settings used by this extension. /// public TcpSettings Settings { get; } - - /// - /// TBD - /// - internal MessageDispatcher FileIoDispatcher { get; } - - private IBufferPool CreateBufferPool(ExtendedActorSystem system, Config config) - { - if (config.IsNullOrEmpty()) - throw ConfigurationException.NullOrEmptyConfig(); - - var type = Type.GetType(config.GetString("class", null), true); - - if (!typeof(IBufferPool).IsAssignableFrom(type)) - throw new ArgumentException($"Buffer pool of type {type} doesn't implement {nameof(IBufferPool)} interface"); - - try - { - // try to construct via `BufferPool(ExtendedActorSystem, Config)` ctor - return (IBufferPool)Activator.CreateInstance(type, system, config); - } - catch - { - // try to construct via `BufferPool(ExtendedActorSystem)` ctor - return (IBufferPool)Activator.CreateInstance(type, system); - } - } } /// diff --git a/src/core/Akka/IO/TcpSettings.cs b/src/core/Akka/IO/TcpSettings.cs index 33c19e0316c..48a4f8a87dd 100644 --- a/src/core/Akka/IO/TcpSettings.cs +++ b/src/core/Akka/IO/TcpSettings.cs @@ -12,9 +12,9 @@ namespace Akka.IO { /// - /// TBD + /// Settings for Akka.IO.Tcp's outbound and inbound connection acvtors. /// - public class TcpSettings + public sealed record TcpSettings { /// /// Creates a new instance of class @@ -29,7 +29,7 @@ public static TcpSettings Create(ActorSystem system) ConfigurationException .NullOrEmptyConfig< TcpSettings>( - "akka.io.tcp"); //($"Failed to create {typeof(TcpSettings)}: akka.io.tcp configuration node not found"); + "akka.io.tcp"); return Create(config); } @@ -38,38 +38,72 @@ public static TcpSettings Create(ActorSystem system) /// Creates a new instance of class /// and fills it with values parsed from provided HOCON config. /// - /// TBD + /// The HOCON path that contains the `akka.io.tcp` section. public static TcpSettings Create(Config config) { if (config.IsNullOrEmpty()) throw ConfigurationException.NullOrEmptyConfig(); return new TcpSettings( - bufferPoolConfigPath: config.GetString("buffer-pool", "akka.io.tcp.disabled-buffer-pool"), - initialSocketAsyncEventArgs: config.GetInt("nr-of-socket-async-event-args", 32), traceLogging: config.GetBoolean("trace-logging", false), batchAcceptLimit: config.GetString("batch-accept-limit") == "scale-to-cpus" ? DefaultAcceptLimit : config.GetInt("batch-accept-limit", DefaultAcceptLimit), registerTimeout: config.GetTimeSpan("register-timeout", TimeSpan.FromSeconds(5)), - receivedMessageSizeLimit: config.GetString("max-received-message-size", "unlimited") == "unlimited" - ? int.MaxValue - : config.GetInt("max-received-message-size", 0), + maxFrameSizeBytes: config.GetByteSize("maximum-frame-size", 4096).Value, + receiveBufferSize: config.GetByteSize("receive-buffer-size", 8192).Value, + sendBufferSize: config.GetByteSize("send-buffer-size", 8192).Value, managementDispatcher: config.GetString("management-dispatcher", "akka.actor.default-dispatcher"), - fileIoDispatcher: config.GetString("file-io-dispatcher", "akka.actor.default-dispatcher"), - transferToLimit: config.GetString("file-io-transferTo-limit", null) == "unlimited" - ? int.MaxValue - : config.GetInt("file-io-transferTo-limit", 512 * 1024), finishConnectRetries: config.GetInt("finish-connect-retries", 5), outgoingSocketForceIpv4: config.GetBoolean("outgoing-socket-force-ipv4", false), writeCommandsQueueMaxSize: config.GetInt("write-commands-queue-max-size", -1)); } + + + // private so we can change the constructor in the future + private TcpSettings( + bool traceLogging, + int batchAcceptLimit, + TimeSpan? registerTimeout, + long maxFrameSizeBytes, + long sendBufferSize, + long receiveBufferSize, + string managementDispatcher, + int finishConnectRetries, + bool outgoingSocketForceIpv4, + int writeCommandsQueueMaxSize) + { + TraceLogging = traceLogging; + BatchAcceptLimit = batchAcceptLimit; + RegisterTimeout = registerTimeout; + MaxFrameSizeBytes = maxFrameSizeBytes; + SendBufferSize = sendBufferSize; + ReceiveBufferSize = receiveBufferSize; + + // fail if send/receive buffer sizes are smaller than max frame size + if (SendBufferSize < MaxFrameSizeBytes) + throw new ArgumentException($"SendBufferSize ({SendBufferSize}) must be at least 2x the size of the maximum frame size ({MaxFrameSizeBytes})"); + if (ReceiveBufferSize < MaxFrameSizeBytes) + throw new ArgumentException($"ReceiveBufferSize ({ReceiveBufferSize}) must be at least 2x the size of the maximum frame size ({MaxFrameSizeBytes})"); + + // fail if the max frame size is negative + if (MaxFrameSizeBytes < 0) + throw new ArgumentException($"MaxFrameSizeBytes ({MaxFrameSizeBytes}) must be a positive number"); + + FinishConnectRetries = finishConnectRetries; + OutgoingSocketForceIpv4 = outgoingSocketForceIpv4; + WriteCommandsQueueMaxSize = writeCommandsQueueMaxSize; + ManagementDispatcher = managementDispatcher; + } + + /// /// Default size of the SAEA pool /// internal static readonly int DefaultAcceptLimit = Environment.ProcessorCount * 2; + [Obsolete("Many of these options are no longer used. Use the TcpSettings.Create method instead.")] public TcpSettings(string bufferPoolConfigPath, int initialSocketAsyncEventArgs, bool traceLogging, @@ -88,7 +122,12 @@ public TcpSettings(string bufferPoolConfigPath, TraceLogging = traceLogging; BatchAcceptLimit = batchAcceptLimit; RegisterTimeout = registerTimeout; - ReceivedMessageSizeLimit = receivedMessageSizeLimit; + MaxFrameSizeBytes = receivedMessageSizeLimit; + + // have to manually set these + SendBufferSize = receivedMessageSizeLimit * 2; + ReceiveBufferSize = receivedMessageSizeLimit * 2; + ManagementDispatcher = managementDispatcher; FileIODispatcher = fileIoDispatcher; TransferToLimit = transferToLimit; @@ -102,12 +141,14 @@ public TcpSettings(string bufferPoolConfigPath, /// Buffer pools are used to mitigate GC-pressure made by potential allocation /// and deallocation of byte buffers used for writing/receiving data from sockets. /// + [Obsolete("This property is unused")] public string BufferPoolConfigPath { get; } /// /// The initial number of SocketAsyncEventArgs to be preallocated. This value /// will grow infinitely if needed. /// + [Obsolete("This property is unused")] public int InitialSocketAsyncEventArgs { get; } /// @@ -115,20 +156,36 @@ public TcpSettings(string bufferPoolConfigPath, /// Be aware that this may log more than once per message sent to the /// actors of the tcp implementation. /// - public bool TraceLogging { get; } + public bool TraceLogging { get; init; } /// /// The maximum number of connection that are accepted in one go, higher /// numbers decrease latency, lower numbers increase fairness on the /// worker-dispatcher /// - public int BatchAcceptLimit { get; } + public int BatchAcceptLimit { get; init; } /// /// The duration a connection actor waits for a `Register` message from /// its commander before aborting the connection. /// - public TimeSpan? RegisterTimeout { get; } + public TimeSpan? RegisterTimeout { get; init; } + + /// + /// The maximum frame size we will accept when reading or writing to a socket. + /// + + public long MaxFrameSizeBytes { get; init; } + + /// + /// Should be at least 2x the size of the maximum frame size. + /// + public long ReceiveBufferSize { get; init; } + + /// + /// Should be at least 2x the size of the maximum frame size. + /// + public long SendBufferSize { get; init; } /// /// The maximum number of bytes delivered by a `Received` message. Before @@ -138,7 +195,8 @@ public TcpSettings(string bufferPoolConfigPath, /// configured receive buffer size. When using value 'unlimited' it will /// try to read all from the receive buffer. /// - public int ReceivedMessageSizeLimit { get; } + [Obsolete("This property is now MaxFrameSizeBytes")] + public long ReceivedMessageSizeLimit => MaxFrameSizeBytes; /// /// Fully qualified config path which holds the dispatcher configuration @@ -150,6 +208,7 @@ public TcpSettings(string bufferPoolConfigPath, /// Fully qualified config path which holds the dispatcher configuration /// on which file IO tasks are scheduled /// + [Obsolete("This property is unused")] public string FileIODispatcher { get; } /// @@ -160,6 +219,7 @@ public TcpSettings(string bufferPoolConfigPath, /// Decreasing the value may improve fairness while increasing may improve /// throughput. /// + [Obsolete("This property is unused")] public int TransferToLimit { get; set; } /// @@ -167,21 +227,24 @@ public TcpSettings(string bufferPoolConfigPath, /// OP_CONNECT. Retries are needed if the OP_CONNECT notification doesn't imply that /// `finishConnect` will succeed, which is the case on Android. /// - public int FinishConnectRetries { get; } + public int FinishConnectRetries { get; init; } /// /// Enforce outgoing socket connection to use IPv4 address family. Required in - /// scenario when IPv6 is not available, for example in Azure Web App sandbox. + /// a scenario when IPv6 is not available, for example in Azure Web App sandbox. /// When set to true it is required to set akka.io.dns.inet-address.use-ipv6 to false /// in cases when DnsEndPoint is used to describe the remote address /// - public bool OutgoingSocketForceIpv4 { get; } + public bool OutgoingSocketForceIpv4 { get; init; } /// /// Limits maximum size of internal queue, used in connection actor /// to store pending write commands. /// To allow unlimited size, set to -1. /// - public int WriteCommandsQueueMaxSize { get; } + /// + /// This setting defines the maximum number of messages, not the maximum size in bytes. + /// + public int WriteCommandsQueueMaxSize { get; init; } } } \ No newline at end of file From 5a7a24e4f4a05792542a92ef0285573ff5a4c43a Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Wed, 14 May 2025 19:53:17 -0500 Subject: [PATCH 11/60] added API approvals --- ...oreAPISpec.ApproveCore.DotNet.verified.txt | 35 ++++++++++++++----- .../CoreAPISpec.ApproveCore.Net.verified.txt | 34 +++++++++++++----- src/core/Akka/IO/Tcp.cs | 14 ++++++++ 3 files changed, 65 insertions(+), 18 deletions(-) diff --git a/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCore.DotNet.verified.txt b/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCore.DotNet.verified.txt index 779a8153cac..dd6804edcb4 100644 --- a/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCore.DotNet.verified.txt +++ b/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCore.DotNet.verified.txt @@ -2269,6 +2269,7 @@ namespace Akka.Configuration public virtual System.Collections.Generic.IList GetBooleanList(string path) { } public virtual System.Collections.Generic.IList GetByteList(string path) { } public virtual System.Nullable GetByteSize(string path) { } + [return: System.Diagnostics.CodeAnalysis.NotNullIfNotNullAttribute("def")] public virtual System.Nullable GetByteSize(string path, System.Nullable def = null) { } public virtual Akka.Configuration.Config GetConfig(string path) { } public virtual decimal GetDecimal(string path, [System.Runtime.CompilerServices.DecimalConstantAttribute(0, 0, 0u, 0u, 0u)] decimal @default) { } @@ -4001,6 +4002,8 @@ namespace Akka.IO public System.Net.EndPoint LocalAddress { get; } public System.Collections.Generic.IEnumerable Options { get; } public bool PullMode { get; } + [System.Runtime.CompilerServices.NullableAttribute(2)] + public Akka.IO.TcpSettings TcpSettings { get; set; } public override string ToString() { } } public sealed class Bound : Akka.IO.Tcp.Event @@ -4043,6 +4046,7 @@ namespace Akka.IO public sealed class CompoundWrite : Akka.IO.Tcp.WriteCommand, System.Collections.Generic.IEnumerable, System.Collections.IEnumerable { public CompoundWrite(Akka.IO.Tcp.SimpleWriteCommand head, Akka.IO.Tcp.WriteCommand tailCommand) { } + public override long Bytes { get; } public Akka.IO.Tcp.SimpleWriteCommand Head { get; } public Akka.IO.Tcp.WriteCommand TailCommand { get; } public System.Collections.Generic.IEnumerator GetEnumerator() { } @@ -4065,6 +4069,8 @@ namespace Akka.IO public System.Collections.Generic.IEnumerable Options { get; } public bool PullMode { get; } public System.Net.EndPoint RemoteAddress { get; } + [System.Runtime.CompilerServices.NullableAttribute(2)] + public Akka.IO.TcpSettings TcpSettings { get; set; } public System.Nullable Timeout { get; } public override string ToString() { } } @@ -4184,6 +4190,7 @@ namespace Akka.IO { public static readonly Akka.IO.Tcp.Write Empty; public override Akka.IO.Tcp.Event Ack { get; } + public override long Bytes { get; } public Akka.IO.ByteString Data { get; } public static Akka.IO.Tcp.Write Create(Akka.IO.ByteString data) { } public static Akka.IO.Tcp.Write Create(Akka.IO.ByteString data, Akka.IO.Tcp.Event ack) { } @@ -4192,6 +4199,7 @@ namespace Akka.IO public abstract class WriteCommand : Akka.IO.Tcp.Command { protected WriteCommand() { } + public abstract long Bytes { get; } public static Akka.IO.Tcp.WriteCommand Create(System.Collections.Generic.IEnumerable writes) { } public static Akka.IO.Tcp.WriteCommand Create(params WriteCommand[] writes) { } public Akka.IO.Tcp.CompoundWrite Prepend(Akka.IO.Tcp.SimpleWriteCommand other) { } @@ -4205,7 +4213,6 @@ namespace Akka.IO public sealed class TcpExt : Akka.IO.IOExtension { public TcpExt(Akka.Actor.ExtendedActorSystem system) { } - public Akka.IO.Buffers.IBufferPool BufferPool { get; } public override Akka.Actor.IActorRef Manager { get; } public Akka.IO.TcpSettings Settings { get; } } @@ -4231,21 +4238,31 @@ namespace Akka.IO public static Akka.IO.Tcp.Command Unbind() { } public static Akka.IO.Tcp.Command Write(Akka.IO.ByteString data, Akka.IO.Tcp.Event ack = null) { } } - public class TcpSettings + public sealed class TcpSettings : System.IEquatable { + [System.ObsoleteAttribute("Many of these options are no longer used. Use the TcpSettings.Create method inste" + + "ad.")] public TcpSettings(string bufferPoolConfigPath, int initialSocketAsyncEventArgs, bool traceLogging, int batchAcceptLimit, System.Nullable registerTimeout, int receivedMessageSizeLimit, string managementDispatcher, string fileIoDispatcher, int transferToLimit, int finishConnectRetries, bool outgoingSocketForceIpv4, int writeCommandsQueueMaxSize) { } - public int BatchAcceptLimit { get; } + public int BatchAcceptLimit { get; set; } + [System.ObsoleteAttribute("This property is unused")] public string BufferPoolConfigPath { get; } + [System.ObsoleteAttribute("This property is unused")] public string FileIODispatcher { get; } - public int FinishConnectRetries { get; } + public int FinishConnectRetries { get; set; } + [System.ObsoleteAttribute("This property is unused")] public int InitialSocketAsyncEventArgs { get; } public string ManagementDispatcher { get; } - public bool OutgoingSocketForceIpv4 { get; } - public int ReceivedMessageSizeLimit { get; } - public System.Nullable RegisterTimeout { get; } - public bool TraceLogging { get; } + public long MaxFrameSizeBytes { get; set; } + public bool OutgoingSocketForceIpv4 { get; set; } + public long ReceiveBufferSize { get; set; } + [System.ObsoleteAttribute("This property is now MaxFrameSizeBytes")] + public long ReceivedMessageSizeLimit { get; } + public System.Nullable RegisterTimeout { get; set; } + public long SendBufferSize { get; set; } + public bool TraceLogging { get; set; } + [System.ObsoleteAttribute("This property is unused")] public int TransferToLimit { get; set; } - public int WriteCommandsQueueMaxSize { get; } + public int WriteCommandsQueueMaxSize { get; set; } public static Akka.IO.TcpSettings Create(Akka.Actor.ActorSystem system) { } public static Akka.IO.TcpSettings Create(Akka.Configuration.Config config) { } } diff --git a/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCore.Net.verified.txt b/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCore.Net.verified.txt index 52408af9289..03a679d93e4 100644 --- a/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCore.Net.verified.txt +++ b/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCore.Net.verified.txt @@ -3991,6 +3991,8 @@ namespace Akka.IO public System.Net.EndPoint LocalAddress { get; } public System.Collections.Generic.IEnumerable Options { get; } public bool PullMode { get; } + [System.Runtime.CompilerServices.NullableAttribute(2)] + public Akka.IO.TcpSettings TcpSettings { get; set; } public override string ToString() { } } public sealed class Bound : Akka.IO.Tcp.Event @@ -4033,6 +4035,7 @@ namespace Akka.IO public sealed class CompoundWrite : Akka.IO.Tcp.WriteCommand, System.Collections.Generic.IEnumerable, System.Collections.IEnumerable { public CompoundWrite(Akka.IO.Tcp.SimpleWriteCommand head, Akka.IO.Tcp.WriteCommand tailCommand) { } + public override long Bytes { get; } public Akka.IO.Tcp.SimpleWriteCommand Head { get; } public Akka.IO.Tcp.WriteCommand TailCommand { get; } public System.Collections.Generic.IEnumerator GetEnumerator() { } @@ -4055,6 +4058,8 @@ namespace Akka.IO public System.Collections.Generic.IEnumerable Options { get; } public bool PullMode { get; } public System.Net.EndPoint RemoteAddress { get; } + [System.Runtime.CompilerServices.NullableAttribute(2)] + public Akka.IO.TcpSettings TcpSettings { get; set; } public System.Nullable Timeout { get; } public override string ToString() { } } @@ -4174,6 +4179,7 @@ namespace Akka.IO { public static readonly Akka.IO.Tcp.Write Empty; public override Akka.IO.Tcp.Event Ack { get; } + public override long Bytes { get; } public Akka.IO.ByteString Data { get; } public static Akka.IO.Tcp.Write Create(Akka.IO.ByteString data) { } public static Akka.IO.Tcp.Write Create(Akka.IO.ByteString data, Akka.IO.Tcp.Event ack) { } @@ -4182,6 +4188,7 @@ namespace Akka.IO public abstract class WriteCommand : Akka.IO.Tcp.Command { protected WriteCommand() { } + public abstract long Bytes { get; } public static Akka.IO.Tcp.WriteCommand Create(System.Collections.Generic.IEnumerable writes) { } public static Akka.IO.Tcp.WriteCommand Create(params WriteCommand[] writes) { } public Akka.IO.Tcp.CompoundWrite Prepend(Akka.IO.Tcp.SimpleWriteCommand other) { } @@ -4195,7 +4202,6 @@ namespace Akka.IO public sealed class TcpExt : Akka.IO.IOExtension { public TcpExt(Akka.Actor.ExtendedActorSystem system) { } - public Akka.IO.Buffers.IBufferPool BufferPool { get; } public override Akka.Actor.IActorRef Manager { get; } public Akka.IO.TcpSettings Settings { get; } } @@ -4221,21 +4227,31 @@ namespace Akka.IO public static Akka.IO.Tcp.Command Unbind() { } public static Akka.IO.Tcp.Command Write(Akka.IO.ByteString data, Akka.IO.Tcp.Event ack = null) { } } - public class TcpSettings + public sealed class TcpSettings : System.IEquatable { + [System.ObsoleteAttribute("Many of these options are no longer used. Use the TcpSettings.Create method inste" + + "ad.")] public TcpSettings(string bufferPoolConfigPath, int initialSocketAsyncEventArgs, bool traceLogging, int batchAcceptLimit, System.Nullable registerTimeout, int receivedMessageSizeLimit, string managementDispatcher, string fileIoDispatcher, int transferToLimit, int finishConnectRetries, bool outgoingSocketForceIpv4, int writeCommandsQueueMaxSize) { } - public int BatchAcceptLimit { get; } + public int BatchAcceptLimit { get; set; } + [System.ObsoleteAttribute("This property is unused")] public string BufferPoolConfigPath { get; } + [System.ObsoleteAttribute("This property is unused")] public string FileIODispatcher { get; } - public int FinishConnectRetries { get; } + public int FinishConnectRetries { get; set; } + [System.ObsoleteAttribute("This property is unused")] public int InitialSocketAsyncEventArgs { get; } public string ManagementDispatcher { get; } - public bool OutgoingSocketForceIpv4 { get; } - public int ReceivedMessageSizeLimit { get; } - public System.Nullable RegisterTimeout { get; } - public bool TraceLogging { get; } + public long MaxFrameSizeBytes { get; set; } + public bool OutgoingSocketForceIpv4 { get; set; } + public long ReceiveBufferSize { get; set; } + [System.ObsoleteAttribute("This property is now MaxFrameSizeBytes")] + public long ReceivedMessageSizeLimit { get; } + public System.Nullable RegisterTimeout { get; set; } + public long SendBufferSize { get; set; } + public bool TraceLogging { get; set; } + [System.ObsoleteAttribute("This property is unused")] public int TransferToLimit { get; set; } - public int WriteCommandsQueueMaxSize { get; } + public int WriteCommandsQueueMaxSize { get; set; } public static Akka.IO.TcpSettings Create(Akka.Actor.ActorSystem system) { } public static Akka.IO.TcpSettings Create(Akka.Configuration.Config config) { } } diff --git a/src/core/Akka/IO/Tcp.cs b/src/core/Akka/IO/Tcp.cs index 67dd9cbde06..0dbf8db3979 100644 --- a/src/core/Akka/IO/Tcp.cs +++ b/src/core/Akka/IO/Tcp.cs @@ -164,6 +164,13 @@ public Connect(EndPoint remoteAddress, public TimeSpan? Timeout { get; } public bool PullMode { get; } + + /// + /// Optional - allows you to specify TCP settings for the connection. + /// + /// Otherwise the system defaults will be used. + /// + public TcpSettings? TcpSettings { get; set; } public override string ToString() => $"Connect(remote: {RemoteAddress}, local: {LocalAddress}, timeout: {Timeout}, pullMode: {PullMode})"; @@ -209,6 +216,13 @@ public Bind(IActorRef handler, public IEnumerable Options { get; } public bool PullMode { get; } + + /// + /// Optional - allows you to specify TCP settings for the connection. + /// + /// Otherwise the system defaults will be used. + /// + public TcpSettings? TcpSettings { get; set; } public override string ToString() => $"Bind(addr: {LocalAddress}, handler: {Handler}, backlog: {Backlog}, pullMode: {PullMode})"; From cd6c64cdfc7e3463c2a7b4e5b394c86925f8b5e2 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Wed, 14 May 2025 20:41:01 -0500 Subject: [PATCH 12/60] ensure that `TcpSettings` get propagated --- src/core/Akka/IO/Tcp.cs | 12 ++++++++++-- src/core/Akka/IO/TcpConnection.cs | 23 +++++++++++++---------- src/core/Akka/IO/TcpIncomingConnection.cs | 4 ++-- src/core/Akka/IO/TcpListener.cs | 2 +- src/core/Akka/IO/TcpOutgoingConnection.cs | 11 +++++------ src/core/Akka/IO/TcpSettings.cs | 18 +++++++++--------- 6 files changed, 40 insertions(+), 30 deletions(-) diff --git a/src/core/Akka/IO/Tcp.cs b/src/core/Akka/IO/Tcp.cs index 0dbf8db3979..94a91aa878b 100644 --- a/src/core/Akka/IO/Tcp.cs +++ b/src/core/Akka/IO/Tcp.cs @@ -168,8 +168,12 @@ public Connect(EndPoint remoteAddress, /// /// Optional - allows you to specify TCP settings for the connection. /// - /// Otherwise the system defaults will be used. + /// Otherwise, the system defaults will be used. /// + /// + /// var tcpSettings = TcpSettings.Create(ActorSystem); + /// var tcpSettingsWithDifferentBufferSizes = tcpSettings with { SendBufferSize = 8192, ReceiveBufferSize = 8192 }; + /// public TcpSettings? TcpSettings { get; set; } public override string ToString() => @@ -220,8 +224,12 @@ public Bind(IActorRef handler, /// /// Optional - allows you to specify TCP settings for the connection. /// - /// Otherwise the system defaults will be used. + /// Otherwise, the system defaults will be used. /// + /// + /// var tcpSettings = TcpSettings.Create(ActorSystem); + /// var tcpSettingsWithDifferentBufferSizes = tcpSettings with { SendBufferSize = 8192, ReceiveBufferSize = 8192 }; + /// public TcpSettings? TcpSettings { get; set; } public override string ToString() => diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 1bf4c811cc8..f035159c5fe 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -75,7 +75,7 @@ private sealed class SocketSendCompleted(int bytes, SocketError error) : INoSeri #endregion - protected readonly TcpExt Tcp; + protected readonly TcpSettings Settings; protected readonly Socket Socket; protected ILoggingAdapter Log { get; } = Context.GetLogger(); @@ -112,19 +112,17 @@ private sealed class SocketSendCompleted(int bytes, SocketError error) : INoSeri private static readonly IOException DroppingWriteBecauseQueueIsFullException = new("Dropping write because queue is full"); - protected TcpConnection(TcpExt tcp, Socket socket, Option writeCommandsBufferMaxSize) + protected TcpConnection(TcpSettings settings, Socket socket) { - _maxWriteCapacity = writeCommandsBufferMaxSize.GetOrElse(tcp.Settings.WriteCommandsQueueMaxSize); + Settings = settings; + _maxWriteCapacity = settings.WriteCommandsQueueMaxSize; _pendingWrites = _maxWriteCapacity > 0 ? new Queue<(WriteCommand Cmd, IActorRef Sender)>(_maxWriteCapacity) : new Queue<(WriteCommand Cmd, IActorRef Sender)>(); // unbounded -; _traceLogging = tcp.Settings.TraceLogging; +; _traceLogging = Settings.TraceLogging; - Tcp = tcp; Socket = socket ?? throw new ArgumentNullException(nameof(socket)); - const int DefaultBufferSize = 64 * 1024; // 64 KiB – matches legacy DirectBufferSize - // TODO: need to set the actual syscall buffer sizes to match - _receiveBuffer = _bufferPool.Rent(DefaultBufferSize); + _receiveBuffer = _bufferPool.Rent(settings.MaxFrameSizeBytes); InitSocketEventArgs(); } @@ -384,7 +382,7 @@ private Receive WaitingForRegistration(IActorRef commander) // after sending `Register` user should watch this actor to make sure // it didn't die because of the timeout Log.Debug("Configured registration timeout of [{0}] expired, stopping", - Tcp.Settings.RegisterTimeout); + Settings.RegisterTimeout); Context.Stop(Self); return true; case WriteCommand write: @@ -573,6 +571,11 @@ protected void CompleteConnect(IActorRef commander, IEnumerable _options; - public TcpIncomingConnection(TcpExt tcp, + public TcpIncomingConnection(TcpSettings settings, Socket socket, IActorRef bindHandler, IEnumerable options, bool readThrottling) - : base(tcp, socket, Option.None) + : base(settings, socket) { _bindHandler = bindHandler; _options = options; diff --git a/src/core/Akka/IO/TcpListener.cs b/src/core/Akka/IO/TcpListener.cs index 198ec16e480..29a677d67d1 100644 --- a/src/core/Akka/IO/TcpListener.cs +++ b/src/core/Akka/IO/TcpListener.cs @@ -268,7 +268,7 @@ private void HandleAccept(SocketAsyncEventArgs saea) var accepted = saea.AcceptSocket!; saea.AcceptSocket = null; // ready for re‑use var incomingConnection = Context.ActorOf(Props - .Create(_tcp, accepted, _bind.Handler, _bind.Options, _bind.PullMode) + .Create(_bind.TcpSettings ?? _tcp.Settings, accepted, _bind.Handler, _bind.Options, _bind.PullMode) .WithDeploy(Deploy.Local)); // set up the watch for monitoring purposes diff --git a/src/core/Akka/IO/TcpOutgoingConnection.cs b/src/core/Akka/IO/TcpOutgoingConnection.cs index ff21f3203cf..d9eb7f8e760 100644 --- a/src/core/Akka/IO/TcpOutgoingConnection.cs +++ b/src/core/Akka/IO/TcpOutgoingConnection.cs @@ -34,11 +34,10 @@ internal sealed class TcpOutgoingConnection : TcpConnection public TcpOutgoingConnection(TcpExt tcp, IActorRef commander, Tcp.Connect connect) : base( - tcp, - tcp.Settings.OutgoingSocketForceIpv4 + (connect.TcpSettings ?? tcp.Settings), + (connect.TcpSettings ?? tcp.Settings).OutgoingSocketForceIpv4 ? new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp) { Blocking = false } - : new Socket(SocketType.Stream, ProtocolType.Tcp) { Blocking = false }, - tcp.Settings.WriteCommandsQueueMaxSize >= 0 ? tcp.Settings.WriteCommandsQueueMaxSize : Option.None) + : new Socket(SocketType.Stream, ProtocolType.Tcp) { Blocking = false }) { _commander = commander; _connect = connect; @@ -83,7 +82,7 @@ private void Stop(Exception cause) { ReleaseConnectionSocketArgs(); - StopWith(new CloseInformation(new HashSet(new[] {_commander}), _connect.FailureMessage.WithCause(cause))); + StopWith(new CloseInformation(new HashSet([_commander]), _connect.FailureMessage.WithCause(cause))); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -195,7 +194,7 @@ private void Register(IPEndPoint address, IPEndPoint fallbackAddress) if (!Socket.ConnectAsync(_connectArgs)) Self.Tell(IO.Tcp.SocketConnected.Instance); - Become(Connecting(Tcp.Settings.FinishConnectRetries, _connectArgs, fallbackAddress)); + Become(Connecting(Settings.FinishConnectRetries, _connectArgs, fallbackAddress)); }); } diff --git a/src/core/Akka/IO/TcpSettings.cs b/src/core/Akka/IO/TcpSettings.cs index 48a4f8a87dd..c712d55e3bb 100644 --- a/src/core/Akka/IO/TcpSettings.cs +++ b/src/core/Akka/IO/TcpSettings.cs @@ -50,9 +50,9 @@ public static TcpSettings Create(Config config) ? DefaultAcceptLimit : config.GetInt("batch-accept-limit", DefaultAcceptLimit), registerTimeout: config.GetTimeSpan("register-timeout", TimeSpan.FromSeconds(5)), - maxFrameSizeBytes: config.GetByteSize("maximum-frame-size", 4096).Value, - receiveBufferSize: config.GetByteSize("receive-buffer-size", 8192).Value, - sendBufferSize: config.GetByteSize("send-buffer-size", 8192).Value, + maxFrameSizeBytes: (int)config.GetByteSize("maximum-frame-size", 4096).Value, + receiveBufferSize: (int)config.GetByteSize("receive-buffer-size", 8192).Value, + sendBufferSize: (int)config.GetByteSize("send-buffer-size", 8192).Value, managementDispatcher: config.GetString("management-dispatcher", "akka.actor.default-dispatcher"), finishConnectRetries: config.GetInt("finish-connect-retries", 5), outgoingSocketForceIpv4: config.GetBoolean("outgoing-socket-force-ipv4", false), @@ -65,9 +65,9 @@ private TcpSettings( bool traceLogging, int batchAcceptLimit, TimeSpan? registerTimeout, - long maxFrameSizeBytes, - long sendBufferSize, - long receiveBufferSize, + int maxFrameSizeBytes, + int sendBufferSize, + int receiveBufferSize, string managementDispatcher, int finishConnectRetries, bool outgoingSocketForceIpv4, @@ -175,17 +175,17 @@ public TcpSettings(string bufferPoolConfigPath, /// The maximum frame size we will accept when reading or writing to a socket. /// - public long MaxFrameSizeBytes { get; init; } + public int MaxFrameSizeBytes { get; init; } /// /// Should be at least 2x the size of the maximum frame size. /// - public long ReceiveBufferSize { get; init; } + public int ReceiveBufferSize { get; init; } /// /// Should be at least 2x the size of the maximum frame size. /// - public long SendBufferSize { get; init; } + public int SendBufferSize { get; init; } /// /// The maximum number of bytes delivered by a `Received` message. Before From 41af0d622dcb717a3b23f395301c8a6c220f35d0 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Wed, 14 May 2025 20:46:05 -0500 Subject: [PATCH 13/60] more cleanup and fixes --- src/core/Akka.Tests/IO/TcpIntegrationSpec.cs | 18 +++++++++--------- src/core/Akka/IO/Tcp.cs | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs index b2735894cc7..2785898f004 100644 --- a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs +++ b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs @@ -191,10 +191,10 @@ public async Task The_TCP_transport_implementation_should_properly_support_conne var testData = ByteString.FromString(str); clientEp.Tell(Tcp.Write.Create(testData, Ack.Instance), clientHandler); await clientHandler.ExpectMsgAsync(); - var received = await serverHandler.ReceiveWhileAsync(o => - { - return o as Tcp.Received; - }, RemainingOrDefault, TimeSpan.FromSeconds(0.5)).ToListAsync(); + var received = await serverHandler + .ReceiveWhileAsync(o => o as Tcp.Received, + RemainingOrDefault, + TimeSpan.FromSeconds(0.5)).ToListAsync(); received.Sum(s => s.Data.Count).Should().Be(testData.Count); } @@ -332,12 +332,12 @@ public async Task When_multiple_concurrent_writing_clients_All_acks_should_be_re // Setup multiple clients var actors = await x.EstablishNewClientConnectionAsync(); - // Each client sends his index to server + // Each client sends their index to server var indexRange = Enumerable.Range(0, clientsCount).ToList(); var clients = indexRange.Select(i => (Index: i, Probe: CreateTestProbe($"test-client-{i}"))).ToArray(); Parallel.ForEach(clients, client => { - var msg = ByteString.FromBytes(new byte[] {(byte) 0}); + var msg = ByteString.FromBytes([0]); client.Probe.Send(actors.ClientConnection, Tcp.Write.Create(msg, AckWithValue.Create(client.Index))); }); @@ -359,7 +359,7 @@ public async Task When_multiple_writing_clients_Should_receive_messages_in_order // Setup multiple clients var actors = await x.EstablishNewClientConnectionAsync(); - // Each client sends his index to server + // Each client sends their index to server var clients = Enumerable.Range(0, clientsCount).Select(i => (Index: i, Probe: CreateTestProbe($"test-client-{i}"))).ToArray(); var contentBuilder = new StringBuilder(); clients.ForEach(client => @@ -441,8 +441,8 @@ public async Task The_TCP_transport_implementation_should_support_waiting_for_wr { await new TestSetup(this).RunAsync(async x => { - x.BindOptions = new[] {new Inet.SO.SendBufferSize(1024)}; - x.ConnectOptions = new[] {new Inet.SO.SendBufferSize(1024)}; + x.BindOptions = [new Inet.SO.SendBufferSize(1024)]; + x.ConnectOptions = [new Inet.SO.SendBufferSize(1024)]; var actors = await x.EstablishNewClientConnectionAsync(); diff --git a/src/core/Akka/IO/Tcp.cs b/src/core/Akka/IO/Tcp.cs index 94a91aa878b..6b5a1cc3b46 100644 --- a/src/core/Akka/IO/Tcp.cs +++ b/src/core/Akka/IO/Tcp.cs @@ -269,7 +269,7 @@ public override string ToString() => } /// - /// In order to close down a listening socket, send this message to that socket’s + /// To close down a listening socket, send this message to that socket’s /// actor (that is the actor which previously had sent the message). The /// listener socket actor will reply with a message. /// From 5940b72c2fc50a695b4a8358b41f47ac5ab35f64 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Thu, 15 May 2025 16:08:20 -0500 Subject: [PATCH 14/60] fix compilation errors --- src/core/Akka/IO/TcpConnection.cs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index c5b82ebee2b..f035159c5fe 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -657,12 +657,6 @@ private void HandleCloseEvent(ConnectionInfo info, IActorRef closeCommander, Con { // we are not closing the socket, but we need to stop reading Context.Become(Closing(info, false)); - { - } - catch (SocketException e) - { - Log.Error("Socket shutdown failed with [{0}]", e); - } } else { From 2d44cc41fda4917ae51758f179f3045c71cb06a6 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Thu, 15 May 2025 16:22:38 -0500 Subject: [PATCH 15/60] added max-frame-size enforcement --- src/core/Akka.Tests/IO/TcpIntegrationSpec.cs | 4 ++-- src/core/Akka/IO/TcpConnection.cs | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs index f4c71e3137c..297abbe338c 100644 --- a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs +++ b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs @@ -560,8 +560,8 @@ class TestSetup public TestSetup(AkkaSpec spec, bool shouldBindServer = true) { - BindOptions = Enumerable.Empty(); - ConnectOptions = Enumerable.Empty(); + BindOptions = []; + ConnectOptions = []; _spec = spec; _shouldBindServer = shouldBindServer; _bindHandler = _spec.CreateTestProbe("bind-handler-probe"); diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index f035159c5fe..fb2e587753b 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -111,6 +111,9 @@ private sealed class SocketSendCompleted(int bytes, SocketError error) : INoSeri private static readonly IOException DroppingWriteBecauseQueueIsFullException = new("Dropping write because queue is full"); + + private static readonly IOException DroppingWriteBecauseExceededMaxFrameSizeException = + new("Dropping write because it exceeds the max frame size"); protected TcpConnection(TcpSettings settings, Socket socket) { @@ -386,6 +389,11 @@ private Receive WaitingForRegistration(IActorRef commander) Context.Stop(Self); return true; case WriteCommand write: + if (write.Bytes > Settings.MaxFrameSizeBytes) + { + DropWrite(write, ); + } + // Have to buffer writes until registration var buffered = TryBuffer(write, Sender); if (!buffered) @@ -524,7 +532,8 @@ private enum DropReason { QueueFull = 1, Closing = 2, - WritingSuspended = 3 + WritingSuspended = 3, + ExceededMaxFrameSize = 4 } private static string GetDropReasonMessage(DropReason reason) @@ -534,6 +543,7 @@ private static string GetDropReasonMessage(DropReason reason) DropReason.QueueFull => "queue is full", DropReason.Closing => "connection is closing", DropReason.WritingSuspended => "writing is suspended", + DropReason.ExceededMaxFrameSize => "exceeded max frame size", _ => throw new ArgumentOutOfRangeException(nameof(reason), reason, null) }; } @@ -545,13 +555,16 @@ private static IOException GetDropMessageException(DropReason reason) DropReason.QueueFull => DroppingWriteBecauseQueueIsFullException, DropReason.Closing => DroppingWriteBecauseClosingException, DropReason.WritingSuspended => DroppingWriteBecauseWritingIsSuspendedException, + DropReason.ExceededMaxFrameSize => DroppingWriteBecauseExceededMaxFrameSizeException, _ => throw new ArgumentOutOfRangeException(nameof(reason), reason, null) }; } private void DropWrite(WriteCommand write, DropReason reason = DropReason.QueueFull) { - if (_traceLogging) Log.Debug("Dropping write because {0}", GetDropReasonMessage(reason)); + // Don't log during closing + if (_traceLogging && reason != DropReason.Closing) Log.Warning("Dropping write [{0}] because {1} - (maxQueueLength={2}, maxFrameSize={3}b)", write.Bytes, GetDropReasonMessage(reason), + Settings.WriteCommandsQueueMaxSize, Settings.MaxFrameSizeBytes); Sender.Tell(write.FailureMessage.WithCause(GetDropMessageException(reason))); } From 334a2e4a7fd3c7d31e3019b51f7774bd892320ee Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Thu, 15 May 2025 16:23:26 -0500 Subject: [PATCH 16/60] fixup --- src/core/Akka/IO/TcpConnection.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index fb2e587753b..aedd2800eb5 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -391,7 +391,8 @@ private Receive WaitingForRegistration(IActorRef commander) case WriteCommand write: if (write.Bytes > Settings.MaxFrameSizeBytes) { - DropWrite(write, ); + DropWrite(write, DropReason.ExceededMaxFrameSize); + return true; } // Have to buffer writes until registration @@ -402,7 +403,7 @@ private Receive WaitingForRegistration(IActorRef commander) } else { - Log.Warning("Received Write command before Register command. " + + Log.Debug("Received Write command before Register command. " + "It will be buffered until Register will be received (buffered write size is {0} bytes)", write.Bytes); } @@ -441,6 +442,12 @@ private Receive Connected(ConnectionInfo info) HandleRead(info.Handler, r); return true; case WriteCommand write: + if (write.Bytes > Settings.MaxFrameSizeBytes) + { + DropWrite(write, DropReason.ExceededMaxFrameSize); + return true; + } + var buffered = TryBuffer(write, Sender); if (!buffered) { From d74cbd79bdd45ca9b78b3b3877f742a214def3fd Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Thu, 15 May 2025 16:25:25 -0500 Subject: [PATCH 17/60] added ability to override `TcpSettings` using new configuration types --- src/core/Akka.Tests/IO/TcpIntegrationSpec.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs index 297abbe338c..69f0cc12637 100644 --- a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs +++ b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs @@ -557,9 +557,11 @@ class TestSetup private readonly bool _shouldBindServer; private readonly TestProbe _bindHandler; private IPEndPoint _endpoint; + private readonly TcpSettings _settings; - public TestSetup(AkkaSpec spec, bool shouldBindServer = true) + public TestSetup(AkkaSpec spec, bool shouldBindServer = true, TcpSettings? settings = null) { + _settings = settings ?? TcpSettings.Create(spec.Sys); BindOptions = []; ConnectOptions = []; _spec = spec; @@ -570,14 +572,15 @@ public TestSetup(AkkaSpec spec, bool shouldBindServer = true) public async Task BindServer() { var bindCommander = _spec.CreateTestProbe(); - bindCommander.Send(_spec.Sys.Tcp(), new Tcp.Bind(_bindHandler.Ref, new IPEndPoint(IPAddress.Loopback, 0), options: BindOptions)); + bindCommander.Send(_spec.Sys.Tcp(), + new Tcp.Bind(_bindHandler.Ref, new IPEndPoint(IPAddress.Loopback, 0), options: BindOptions){ TcpSettings = _settings}); await bindCommander.ExpectMsgAsync(bound => _endpoint = (IPEndPoint) bound.LocalAddress); } - public async Task EstablishNewClientConnectionAsync(bool registerClientHandler = true) + public async Task EstablishNewClientConnectionAsync(bool registerClientHandler = true, TcpSettings? settings = null) { var connectCommander = _spec.CreateTestProbe("connect-commander-probe"); - connectCommander.Send(_spec.Sys.Tcp(), new Tcp.Connect(_endpoint, options: ConnectOptions)); + connectCommander.Send(_spec.Sys.Tcp(), new Tcp.Connect(_endpoint, options: ConnectOptions){ TcpSettings = settings ?? _settings}); await connectCommander.ExpectMsgAsync(); var clientHandler = _spec.CreateTestProbe("client-handler-probe"); From 7ce01a64eeda072cbcdc3351edb45d8c76d2f02a Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Thu, 15 May 2025 16:38:57 -0500 Subject: [PATCH 18/60] fixing tests --- src/core/Akka.Tests/IO/TcpIntegrationSpec.cs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs index 69f0cc12637..254c3940f16 100644 --- a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs +++ b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs @@ -51,7 +51,7 @@ public TcpIntegrationSpec(ITestOutputHelper output) private async Task VerifyActorTermination(IActorRef actor) { - Watch(actor); + await WatchAsync(actor); await ExpectTerminatedAsync(actor); } @@ -282,7 +282,7 @@ public async Task Write_before_Register_should_not_be_silently_dropped() var msg = ByteString.FromString("msg"); // 3 bytes - await EventFilter.Warning(new Regex("Received Write command before Register[^3]+3 bytes")).ExpectOneAsync(() => { + await EventFilter.Debug(new Regex("Received Write command before Register[^3]+3 bytes")).ExpectOneAsync(() => { actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(msg)); actors.ClientConnection.Tell(new Tcp.Register(actors.ClientHandler)); return Task.CompletedTask; @@ -298,27 +298,30 @@ await EventFilter.Warning(new Regex("Received Write command before Register[^3]+ } [Fact] - public async Task Write_before_Register_should_Be_dropped_if_buffer_is_full() + public async Task Write_before_Register_should_Be_dropped_if_WriteQueue_is_full() { - await new TestSetup(this).RunAsync(async x => + var smallBufferSettings = TcpSettings.Create(Sys) with { WriteCommandsQueueMaxSize = 1 }; + + await new TestSetup(this, settings:smallBufferSettings).RunAsync(async x => { var actors = await x.EstablishNewClientConnectionAsync(registerClientHandler: false); - + + var happyMessage = ByteString.FromString("msg"); // 3 bytes var overflowData = ByteString.FromBytes(new byte[InternalConnectionActorMaxQueueSize + 1]); // We do not want message about receiving Write to be logged, if the write was actually discarded await EventFilter.Warning(new Regex("Received Write command before Register[^3]+3 bytes")).ExpectAsync(0, () => { + actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(happyMessage)); actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(overflowData)); return Task.CompletedTask; }); - await actors.ClientHandler.ExpectMsgAsync(TimeSpan.FromSeconds(10)); + await actors.ClientHandler.ExpectMsgAsync(); // After failed receive, next "good" writes should be handled with no issues - actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(ByteString.FromBytes(new byte[1]))); actors.ClientHandler.Send(actors.ClientConnection, new Tcp.Register(actors.ClientHandler)); var serverMsgs = await actors.ServerHandler.ReceiveWhileAsync(o => o as Tcp.Received, RemainingOrDefault, TimeSpan.FromSeconds(2)).ToListAsync(); - serverMsgs.Should().HaveCount(1).And.Subject.Should().Contain(m => m.Data.Count == 1); + serverMsgs.Should().HaveCount(1).And.Subject.Should().Contain(m => m.Data.Count == 3); }); } From e205480527980e1114b77b35867c1429c68a77f3 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Thu, 15 May 2025 16:44:37 -0500 Subject: [PATCH 19/60] remove --- src/core/Akka/IO/TcpConnection.cs | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index aedd2800eb5..d25bf7948f2 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -112,9 +112,6 @@ private sealed class SocketSendCompleted(int bytes, SocketError error) : INoSeri private static readonly IOException DroppingWriteBecauseQueueIsFullException = new("Dropping write because queue is full"); - private static readonly IOException DroppingWriteBecauseExceededMaxFrameSizeException = - new("Dropping write because it exceeds the max frame size"); - protected TcpConnection(TcpSettings settings, Socket socket) { Settings = settings; @@ -235,7 +232,7 @@ private void TrySendNext() switch (cmd) { case Write w when !w.Data.IsEmpty: - int wouldBe = accumulated + w.Data.Count; + var wouldBe = accumulated + w.Data.Count; if (wouldBe > maxBytes && batch.Count > 0) goto done; _pendingWrites.Dequeue(); batch.Add(w.Data); @@ -389,12 +386,6 @@ private Receive WaitingForRegistration(IActorRef commander) Context.Stop(Self); return true; case WriteCommand write: - if (write.Bytes > Settings.MaxFrameSizeBytes) - { - DropWrite(write, DropReason.ExceededMaxFrameSize); - return true; - } - // Have to buffer writes until registration var buffered = TryBuffer(write, Sender); if (!buffered) @@ -442,12 +433,6 @@ private Receive Connected(ConnectionInfo info) HandleRead(info.Handler, r); return true; case WriteCommand write: - if (write.Bytes > Settings.MaxFrameSizeBytes) - { - DropWrite(write, DropReason.ExceededMaxFrameSize); - return true; - } - var buffered = TryBuffer(write, Sender); if (!buffered) { @@ -540,7 +525,6 @@ private enum DropReason QueueFull = 1, Closing = 2, WritingSuspended = 3, - ExceededMaxFrameSize = 4 } private static string GetDropReasonMessage(DropReason reason) @@ -550,7 +534,6 @@ private static string GetDropReasonMessage(DropReason reason) DropReason.QueueFull => "queue is full", DropReason.Closing => "connection is closing", DropReason.WritingSuspended => "writing is suspended", - DropReason.ExceededMaxFrameSize => "exceeded max frame size", _ => throw new ArgumentOutOfRangeException(nameof(reason), reason, null) }; } @@ -562,7 +545,6 @@ private static IOException GetDropMessageException(DropReason reason) DropReason.QueueFull => DroppingWriteBecauseQueueIsFullException, DropReason.Closing => DroppingWriteBecauseClosingException, DropReason.WritingSuspended => DroppingWriteBecauseWritingIsSuspendedException, - DropReason.ExceededMaxFrameSize => DroppingWriteBecauseExceededMaxFrameSizeException, _ => throw new ArgumentOutOfRangeException(nameof(reason), reason, null) }; } From e3611f8794fd14b56d5b1e739f00debe32af0cdb Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Thu, 15 May 2025 20:01:40 -0500 Subject: [PATCH 20/60] improved error handling around socket reads - removed `_peerClosed` check that blocked shutdown --- src/core/Akka/IO/TcpConnection.cs | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index d25bf7948f2..a3a3171eba1 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // // Copyright (C) 2009-2022 Lightbend Inc. // Copyright (C) 2013-2025 .NET Foundation @@ -175,8 +175,21 @@ protected void UnsignDeathPact() private void IssueReceive() { if(_peerClosed) return; // can't read if peer is closed - if (!Socket.ReceiveAsync(_receiveArgs)) - Self.Tell(new SocketReceiveCompleted(_receiveArgs.BytesTransferred, _receiveArgs.SocketError)); + + try + { + if (!Socket.ReceiveAsync(_receiveArgs)) + Self.Tell(new SocketReceiveCompleted(_receiveArgs.BytesTransferred, _receiveArgs.SocketError)); + } + catch (ObjectDisposedException) + { + // Socket was closed, signal peer closed + Self.Tell(PeerClosed.Instance); + } + catch (SocketException ex) + { + Self.Tell(new SocketReceiveCompleted(0, ex.SocketErrorCode)); + } } private void HandleRead(IActorRef handler, SocketReceiveCompleted rc) @@ -312,8 +325,14 @@ private void DeliverCloseMessages() private void TryCloseIfDone() { if (!_closingRequested) return; + + if (_traceLogging) Log.Debug("TryCloseIfDone called, sending={0}, pendingWrites={1}, peerClosed={2}", + _sending, _pendingWrites.Count, _peerClosed); + + // When all pending writes are completed, we can close the connection + // even if the peer hasn't closed its side yet if (_sending || _pendingWrites.Count > 0) return; - if (!_peerClosed) return; + Context.Stop(Self); } From 231c521dc06cdaa3a7143915555c4152d0ee22c5 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Thu, 15 May 2025 20:29:58 -0500 Subject: [PATCH 21/60] fixed all `TcpIntegrationSpecs` --- src/core/Akka.Tests/IO/TcpIntegrationSpec.cs | 44 +++++++++++--------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs index 254c3940f16..2a2e195ccc5 100644 --- a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs +++ b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // // Copyright (C) 2009-2022 Lightbend Inc. // Copyright (C) 2013-2025 .NET Foundation @@ -84,7 +84,7 @@ public async Task The_TCP_transport_implementation_should_properly_handle_connec var actors = await x.EstablishNewClientConnectionAsync(); actors.ClientHandler.Send(actors.ClientConnection, Tcp.Abort.Instance); await actors.ClientHandler.ExpectMsgAsync(); - await actors.ServerHandler.ExpectMsgAsync(); + await actors.ServerHandler.ExpectMsgAsync(); await VerifyActorTermination(actors.ClientConnection); await VerifyActorTermination(actors.ServerConnection); }); @@ -100,7 +100,7 @@ public async Task The_TCP_transport_implementation_should_properly_handle_connec actors.ClientHandler.Send(actors.ClientConnection, Tcp.Abort.Instance); await actors.ClientHandler.ExpectMsgAsync(); - await actors.ServerHandler.ExpectMsgAsync(); + await actors.ServerHandler.ExpectMsgAsync(); await VerifyActorTermination(actors.ClientConnection); await VerifyActorTermination(actors.ServerConnection); }); @@ -408,33 +408,39 @@ public async Task When_multiple_writing_clients_Should_receive_messages_in_order [Fact] public async Task Should_fail_writing_when_buffer_is_filled() { - await new TestSetup(this).RunAsync(async x => + // Set the write queue max size to 1 message + var smallBufferSettings = TcpSettings.Create(Sys) with { WriteCommandsQueueMaxSize = 1 }; + + await new TestSetup(this, settings: smallBufferSettings).RunAsync(async x => { var actors = await x.EstablishNewClientConnectionAsync(); - // create a buffer-overflow message - var overflowData = ByteString.FromBytes(new byte[InternalConnectionActorMaxQueueSize + 1]); - var goodData = ByteString.FromBytes(new byte[InternalConnectionActorMaxQueueSize]); + // Small test messages + var testData = ByteString.FromString("test message"); // If test runner is too loaded, let it try ~3 times with 5 pause interval await AwaitAssertAsync(async () => { - // try sending overflow - actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(overflowData)); // this is sent immediately - actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(overflowData)); // this will try to buffer + // Send first message - should be sent immediately + actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(testData)); + + // Send second message - should be buffered and fail since queue size is 1 + actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(testData)); + + // Third message - should fail immediately since queue is full + actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(testData)); await actors.ClientHandler.ExpectMsgAsync(TimeSpan.FromSeconds(20)); - // First overflow data will be received anyway - (await actors.ServerHandler.ReceiveWhileAsync(TimeSpan.FromSeconds(1), m => m as Tcp.Received).ToListAsync()) - .Sum(m => m.Data.Count) - .Should().Be(InternalConnectionActorMaxQueueSize + 1); + // First message should be received + var received = await actors.ServerHandler.ReceiveWhileAsync(TimeSpan.FromSeconds(1), m => m as Tcp.Received).ToListAsync(); + received.Count.Should().BeGreaterOrEqualTo(1); + received.Sum(m => m.Data.Count).Should().BeGreaterOrEqualTo(testData.Count); - // Check that almost-overflow size does not cause any problems + // Check we can resume writing after clearing the failure actors.ClientHandler.Send(actors.ClientConnection, Tcp.ResumeWriting.Instance); // Recover after send failure - actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(goodData)); - (await actors.ServerHandler.ReceiveWhileAsync(TimeSpan.FromSeconds(1), m => m as Tcp.Received).ToListAsync()) - .Sum(m => m.Data.Count) - .Should().Be(InternalConnectionActorMaxQueueSize); + actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(testData)); + received = await actors.ServerHandler.ReceiveWhileAsync(TimeSpan.FromSeconds(1), m => m as Tcp.Received).ToListAsync(); + received.Count.Should().BeGreaterOrEqualTo(1); }, TimeSpan.FromSeconds(30 * 3), TimeSpan.FromSeconds(5)); // 3 attempts by ~25 seconds + 5 sec pause }); } From ea4294f6c650710818f6282af3f84b2c8a285916 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Thu, 15 May 2025 21:20:32 -0500 Subject: [PATCH 22/60] fixed most Akka.Streams.IO.Tcp specs disabled waiting on peers to close connection --- src/core/Akka.Streams.Tests/IO/TcpSpec.cs | 3 ++- src/core/Akka/IO/TcpConnection.cs | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/core/Akka.Streams.Tests/IO/TcpSpec.cs b/src/core/Akka.Streams.Tests/IO/TcpSpec.cs index b70e75f04f8..0c5c945494c 100644 --- a/src/core/Akka.Streams.Tests/IO/TcpSpec.cs +++ b/src/core/Akka.Streams.Tests/IO/TcpSpec.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // // Copyright (C) 2009-2022 Lightbend Inc. // Copyright (C) 2013-2025 .NET Foundation @@ -103,6 +103,7 @@ public async Task Outgoing_TCP_stream_must_be_able_to_read_a_sequence_of_ByteStr serverConnection.Write(input); serverConnection.ConfirmedClose(); + // Reduced timeout - otherwise we're just waiting longer for the failure var result = await resultFuture.ShouldCompleteWithin(3.Seconds()); result.ShouldBe(expectedOutput); } diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index a3a3171eba1..0553556d58a 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -329,10 +329,13 @@ private void TryCloseIfDone() if (_traceLogging) Log.Debug("TryCloseIfDone called, sending={0}, pendingWrites={1}, peerClosed={2}", _sending, _pendingWrites.Count, _peerClosed); - // When all pending writes are completed, we can close the connection - // even if the peer hasn't closed its side yet + // When outstanding writes exist, we must finish them before closing if (_sending || _pendingWrites.Count > 0) return; + // No pending writes, so we can safely close + // Note: We no longer wait for _peerClosed in any scenario to avoid deadlocks in Akka.Streams + // The previous implementation could hang when Streams TCP stages sent ConfirmedClose but were + // waiting for connection completion which never happened because we were waiting for peer close Context.Stop(Self); } @@ -377,6 +380,7 @@ private Receive WaitingForRegistration(IActorRef commander) if (_traceLogging) Log.Debug("[{0}] registered as connection handler", register.Handler); + var registerInfo = new ConnectionInfo(register.Handler, register.KeepOpenOnPeerClosed, register.UseResumeWriting); @@ -394,6 +398,7 @@ private Receive WaitingForRegistration(IActorRef commander) IssueReceive(); return true; case CloseCommand cmd: + // Default connection info for unregistered connections - always uses keepOpenOnPeerClosed: false var info = new ConnectionInfo(commander, keepOpenOnPeerClosed: false, useResumeWriting: false); HandleCloseCommand(info, Sender, cmd); return true; From e06f81e4e667a7dc62ff69f81e473f5cb9aeb9f6 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 16 May 2025 12:51:03 -0500 Subject: [PATCH 23/60] added `ConnectionState` to better organize termination conditions --- src/core/Akka.Streams.Tests/IO/TcpSpec.cs | 6 +- src/core/Akka/IO/TcpConnection.cs | 400 ++++++++++++++++------ 2 files changed, 292 insertions(+), 114 deletions(-) diff --git a/src/core/Akka.Streams.Tests/IO/TcpSpec.cs b/src/core/Akka.Streams.Tests/IO/TcpSpec.cs index 0c5c945494c..a33c0d8aed4 100644 --- a/src/core/Akka.Streams.Tests/IO/TcpSpec.cs +++ b/src/core/Akka.Streams.Tests/IO/TcpSpec.cs @@ -40,7 +40,7 @@ public async Task Outgoing_TCP_stream_must_work_in_the_happy_case() { await this.AssertAllStagesStoppedAsync(async () => { - var testData = ByteString.FromBytes(new byte[] {1, 2, 3, 4, 5}); + var testData = ByteString.FromBytes([1, 2, 3, 4, 5]); var server = await new Server(this).InitializeAsync(); var tcpReadProbe = new TcpReadProbe(this); @@ -65,7 +65,7 @@ await ValidateServerClientCommunicationAsync(testData, serverConnection, tcpRead public async Task Outgoing_TCP_stream_must_be_able_to_write_a_sequence_of_ByteStrings() { var server = await new Server(this).InitializeAsync(); - var testInput = Enumerable.Range(0, 256).Select(i => ByteString.FromBytes(new byte[] {Convert.ToByte(i)})); + var testInput = Enumerable.Range(0, 256).Select(i => ByteString.FromBytes([Convert.ToByte(i)])); var expectedOutput = ByteString.FromBytes(Enumerable.Range(0, 256).Select(Convert.ToByte).ToArray()); Source.From(testInput) @@ -86,7 +86,7 @@ public async Task Outgoing_TCP_stream_must_be_able_to_read_a_sequence_of_ByteStr var testOutput = new byte[255]; for (byte i = 0; i < 255; i++) { - testInput[i] = ByteString.FromBytes(new [] {i}); + testInput[i] = ByteString.FromBytes([i]); testOutput[i] = i; } diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 0553556d58a..bbedfc292a0 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -8,17 +8,13 @@ using System; using System.Buffers; using System.Collections.Generic; -using System.Collections.Immutable; using System.IO; using System.Linq; using System.Net.Sockets; -using System.Runtime.CompilerServices; using Akka.Actor; using Akka.Dispatch; using Akka.Event; using Akka.Pattern; -using Akka.Util; -using Akka.Util.Internal; #nullable enable @@ -27,6 +23,157 @@ namespace Akka.IO using static Akka.IO.Tcp; using ByteBuffer = ArraySegment; + internal static class TcpStateTransitions + { + public static ConnectionState Update(this in ConnectionState state, Event e) + { + switch (e) + { + case PeerClosed: + return state with { PeerClosed = true }; + case ErrorClosed: // have to close right now + return state with + { + PeerClosed = true, CloseRequested = true, WritingSuspended = true, ReadingSuspended = true + }; + case WritingResumed: + return state with { WritingSuspended = false }; + default: + return state; + } + } + + public static ConnectionState Update(this in ConnectionState state, Command cmd) + { + switch (cmd) + { + case Register r: + return state with { HasConnected = true, KeepOpenOnPeerClosed = r.KeepOpenOnPeerClosed}; + case ResumeWriting: + return state with { WritingSuspended = false }; + case ResumeReading: + return state with { ReadingSuspended = false }; + case SuspendReading: + return state with { ReadingSuspended = true }; + case ConfirmedClose: + return state with { KeepOpenOnPeerClosed = true, CloseRequested = true }; + case Close: + return state with { CloseRequested = true }; + case Abort: + return state with + { + WritingSuspended = true, + ReadingSuspended = true, + KeepOpenOnPeerClosed = false, + CloseRequested = true + }; + default: + return state; + } + } + + public static ConnectionState Sending(this in ConnectionState state) + { + return state with { IsSending = true }; + } + + public static ConnectionState DoneSending(this in ConnectionState state) + { + return state with { IsSending = false }; + } + } + + /// + /// Maintains the state of the connection. + /// + /// Externally managed set of pending writes. + /// + /// This data structure is largely needed around dealing with disconnections - because there's several different + /// pre-existing methods we need to support in order to maintain backwards compatibility. + /// + internal readonly record struct ConnectionState(Queue<(WriteCommand Cmd, IActorRef Sender)> PendingWrites) + { + /// + /// A setting that can either be set upon connecting or as a result of the . + /// + public bool KeepOpenOnPeerClosed { get; init; } + + /// + /// We've completed the connection handshake and are now connected. + /// + public bool HasConnected { get; init; } + + /// + /// A closure request has been received from our own process. _We_ are doing the closing. + /// + public bool CloseRequested { get; init; } + + /// + /// Peer has closed for writes - but they might still be open for reading. + /// + /// This happens after we get a 0-byte read from the socket. + /// + public bool PeerClosed { get; init; } + + /// + /// Writing has been suspended - this can be done by the user or by the system. + /// + public bool WritingSuspended { get; init; } + + /// + /// Reading has been suspended - this can be done by the user or by the system. + /// + /// Happens, for instance, when we have processed a and + /// are waiting on our peer to close their end of the connection. + /// + public bool ReadingSuspended { get; init; } + + /// + /// We've fully closed the socket for reading and writing. The socket itself is no longer accessible. + /// + public bool SocketDisposed { get; init; } + + /// + /// Are we sending packets over the network right now? + /// + public bool IsSending { get; init; } + + /// + /// We have half-closed our socket for writing, but we are still open for reading. + /// + public bool ClosedForWrites { get; init; } + + /// + /// Can't receive unless: + /// + /// 1. We are connected + /// 2. We are not closed for reading + /// 3. Peer is not closed for writing + /// + public bool CanReceive => (!ReadingSuspended && !PeerClosed) && HasConnected; + + /// + /// Can send as long as we are not closed for writing, and we haven't suspended writing. + /// + public bool CanSend => !WritingSuspended && !ClosedForWrites; + + /// + /// True if we have live writes in the queue or if we are currently sending over network. + /// + public bool IsWritePending => IsSending || PendingWrites.Count > 0; + + /// + /// If we are trying to do a fully graceful close - we can only close in two situations: + /// + /// 1. We have no pending writes / we can still send writes over the network + /// 2. The peer has closed the socket for writing (we're getting no more data from them) and + /// we have not been told to keep the socket open upon peer closure. + /// + /// If either of these conditions are true, we can close the socket SO LONG AS: closing has been requested. + /// + public bool IsCloseable => CloseRequested && (!(IsWritePending && CanSend) || (PeerClosed && !KeepOpenOnPeerClosed)); + } + /// /// INTERNAL API: Base class for TcpIncomingConnection and TcpOutgoingConnection. /// @@ -61,13 +208,15 @@ private sealed class AckSocketAsyncEventArgs : SocketAsyncEventArgs #region completion msgs - private sealed class SocketReceiveCompleted(int bytes, SocketError error) : INoSerializationVerificationNeeded, IDeadLetterSuppression + private sealed class SocketReceiveCompleted(int bytes, SocketError error) + : INoSerializationVerificationNeeded, IDeadLetterSuppression { public int Bytes { get; } = bytes; public SocketError Error { get; } = error; } - private sealed class SocketSendCompleted(int bytes, SocketError error) : INoSerializationVerificationNeeded, IDeadLetterSuppression + private sealed class SocketSendCompleted(int bytes, SocketError error) + : INoSerializationVerificationNeeded, IDeadLetterSuppression { public int Bytes { get; } = bytes; public SocketError Error { get; } = error; @@ -86,23 +235,21 @@ private sealed class SocketSendCompleted(int bytes, SocketError error) : INoSeri private readonly byte[] _receiveBuffer; private SocketAsyncEventArgs _receiveArgs; private AckSocketAsyncEventArgs _sendArgs; - + private IActorRef _watchedActor = Context.System.DeadLetters; private readonly int _maxWriteCapacity; - private volatile bool _sending; - private volatile bool _closingRequested; - private volatile bool _peerClosed; + private ConnectionState _state; private readonly bool _traceLogging; - private long _pendingOutboundBytes; - + private long _pendingOutboundBytes; + // so we don't try to close the socket a second time during PostStop private bool _socketAlreadyClosed; private CloseInformation? _closedMessage; // for ConnectionClosed message in postStop - + private static readonly IOException DroppingWriteBecauseClosingException = new("Dropping write because the connection is closing"); @@ -111,7 +258,7 @@ private sealed class SocketSendCompleted(int bytes, SocketError error) : INoSeri private static readonly IOException DroppingWriteBecauseQueueIsFullException = new("Dropping write because queue is full"); - + protected TcpConnection(TcpSettings settings, Socket socket) { Settings = settings; @@ -119,8 +266,9 @@ protected TcpConnection(TcpSettings settings, Socket socket) _pendingWrites = _maxWriteCapacity > 0 ? new Queue<(WriteCommand Cmd, IActorRef Sender)>(_maxWriteCapacity) : new Queue<(WriteCommand Cmd, IActorRef Sender)>(); // unbounded -; _traceLogging = Settings.TraceLogging; - + ; + _traceLogging = Settings.TraceLogging; + _state = new ConnectionState(_pendingWrites); Socket = socket ?? throw new ArgumentNullException(nameof(socket)); _receiveBuffer = _bufferPool.Rent(settings.MaxFrameSizeBytes); InitSocketEventArgs(); @@ -154,11 +302,11 @@ private static void OnCompleted(object? sender, SocketAsyncEventArgs e) break; } } - + /// /// Returns true if write is in-progress over the wire or if we have writes pending in the queue. /// - public bool IsWritePending => _sending || _pendingWrites.Count > 0; + public bool IsWritePending => _state.IsWritePending; protected void SignDeathPact(IActorRef actor) { @@ -174,9 +322,9 @@ protected void UnsignDeathPact() private void IssueReceive() { - if(_peerClosed) return; // can't read if peer is closed - - try + if (!_state.CanReceive) return; + + try { if (!Socket.ReceiveAsync(_receiveArgs)) Self.Tell(new SocketReceiveCompleted(_receiveArgs.BytesTransferred, _receiveArgs.SocketError)); @@ -191,12 +339,12 @@ private void IssueReceive() Self.Tell(new SocketReceiveCompleted(0, ex.SocketErrorCode)); } } - + private void HandleRead(IActorRef handler, SocketReceiveCompleted rc) { - if(_traceLogging) + if (_traceLogging) Log.Debug("Received {0} bytes from {1}", rc.Bytes, Socket.RemoteEndPoint); - + // todo: need to harden our SocketError handling if (rc.Error != SocketError.Success) { @@ -205,13 +353,8 @@ private void HandleRead(IActorRef handler, SocketReceiveCompleted rc) return; } - // TODO: what about a closed for writing event handler? - // When the peer says "I'm not sending you any more data, but you should send me some?" - if (rc.Bytes == 0) // CLOSED FOR READING { - _peerClosed = true; - // signal to the handler that the peer has closed the connection Self.Tell(PeerClosed.Instance); return; @@ -232,7 +375,7 @@ private void IssueSend(IList> buffers) private void TrySendNext() { // already sending or no writes to send - if (_sending || _pendingWrites.Count == 0) return; + if (_state.IsSending || _pendingWrites.Count == 0) return; var maxBytes = _receiveBuffer.Length; var accumulated = 0; @@ -271,11 +414,11 @@ private void TrySendNext() return; } - _sending = true; + _state = _state.Sending(); var payload = FlattenByteStrings(batch); IssueSend(payload); } - + /// /// Called when the socket closes before we have processed all pending writes. /// @@ -286,16 +429,17 @@ private void FailUnprocessedPendingWrites(Exception cause) var failure = cmd.FailureMessage.WithCause(cause); ack.Tell(failure); } + _pendingWrites.Clear(); } private void HandleSendCompleted(SocketSendCompleted socketSendCompleted) { - _sending = false; - - if(_traceLogging) + _state = _state.DoneSending(); + + if (_traceLogging) Log.Debug("Sent {0} bytes to {1}", socketSendCompleted.Bytes, Socket.RemoteEndPoint); - + // check for errors if (socketSendCompleted.Error != SocketError.Success) { @@ -303,16 +447,16 @@ private void HandleSendCompleted(SocketSendCompleted socketSendCompleted) Self.Tell(new ErrorClosed(socketSendCompleted.Error.ToString())); return; } - + foreach (var (c, ack) in _sendArgs.PendingAcks) c.Tell(ack); _sendArgs.ClearAcks(); _sendArgs.BufferList = null; - + TrySendNext(); TryCloseIfDone(); } - + private void DeliverCloseMessages() { if (_closedMessage == null) return; @@ -324,14 +468,15 @@ private void DeliverCloseMessages() private void TryCloseIfDone() { - if (!_closingRequested) return; - - if (_traceLogging) Log.Debug("TryCloseIfDone called, sending={0}, pendingWrites={1}, peerClosed={2}", - _sending, _pendingWrites.Count, _peerClosed); - - // When outstanding writes exist, we must finish them before closing - if (_sending || _pendingWrites.Count > 0) return; - + if (!_state.CloseRequested) return; + + if (_traceLogging) + Log.Debug("TryCloseIfDone called, sending={0}, pendingWrites={1}, peerClosed={2}", + _state.IsSending, _pendingWrites.Count, _state.PeerClosed); + + // Factors in several different configuration options to determine if we can close ourselves or not + if (!_state.IsCloseable) return; + // No pending writes, so we can safely close // Note: We no longer wait for _peerClosed in any scenario to avoid deadlocks in Akka.Streams // The previous implementation could hang when Streams TCP stages sent ConfirmedClose but were @@ -358,7 +503,7 @@ private bool TryBuffer(WriteCommand cmd, IActorRef sender) _pendingOutboundBytes += cmd.Bytes; return true; } - + // buffer is full return false; } @@ -379,11 +524,11 @@ private Receive WaitingForRegistration(IActorRef commander) SignDeathPact(register.Handler); // will unsign death pact with commander automatically if (_traceLogging) Log.Debug("[{0}] registered as connection handler", register.Handler); + _state = _state.Update(register); - var registerInfo = new ConnectionInfo(register.Handler, register.KeepOpenOnPeerClosed, register.UseResumeWriting); - + // we set a default close message here in case the actor dies before we get a close message // this will prevent close messages from going missing // part of the fix for https://github.com/akkadotnet/akka.net/issues/7634 @@ -398,6 +543,7 @@ private Receive WaitingForRegistration(IActorRef commander) IssueReceive(); return true; case CloseCommand cmd: + _state = _state.Update(cmd); // Default connection info for unregistered connections - always uses keepOpenOnPeerClosed: false var info = new ConnectionInfo(commander, keepOpenOnPeerClosed: false, useResumeWriting: false); HandleCloseCommand(info, Sender, cmd); @@ -419,9 +565,10 @@ private Receive WaitingForRegistration(IActorRef commander) else { Log.Debug("Received Write command before Register command. " + - "It will be buffered until Register will be received (buffered write size is {0} bytes)", + "It will be buffered until Register will be received (buffered write size is {0} bytes)", write.Bytes); } + return true; case Terminated t: { @@ -462,10 +609,7 @@ private Receive Connected(ConnectionInfo info) { DropWrite(write); } - else - { - TrySendNext(); - } + TrySendNext(); return true; case SocketSendCompleted sendCompleted: HandleSendCompleted(sendCompleted); @@ -476,9 +620,13 @@ private Receive Connected(ConnectionInfo info) case ConnectionClosed closed: // peer is closing the socket HandleCloseEvent(info, Sender, closed); return true; - case SuspendReading: - case ResumeReading: - // no-ops + case SuspendReading sr: + _state = _state.Update(sr); + return true; + case ResumeReading rr: + // try to drive read-loop forward again + _state = _state.Update(rr); + IssueReceive(); return true; case Terminated t: // handler died { @@ -505,30 +653,35 @@ private Receive Closing(ConnectionInfo info, bool confirmClose) return true; case SocketSendCompleted s: HandleSendCompleted(s); - if (confirmClose && !IsWritePending) + if (_state.IsWritePending) { // done writing, so we can now half-close the socket - if (_traceLogging) Log.Debug("Running in close-confirm mode, half-closing socket for writes"); - + if (_traceLogging) + Log.Debug("Running in close-confirm mode, half-closing socket for writes"); + // We will need to get an EOF Socket.Shutdown(SocketShutdown.Send); } return true; - case WriteCommand write: - DropWrite(write); + case WriteCommand write: // no more writes once we start closing + DropWrite(write, DropReason.Closing); return true; - case SuspendReading: - case ResumeReading: - // no-ops + case SuspendReading sr: + _state = _state.Update(sr); return true; - case Abort a: + case ResumeReading rr: + // try to drive read-loop forward again + _state = _state.Update(rr); + IssueReceive(); + return true; + case CloseCommand a: HandleCloseCommand(info, Sender, a); return true; - case ConnectionClosed: - _peerClosed = true; - if (!IsWritePending) + case PeerClosed peerClosed: + _state = _state.Update(peerClosed); + if (_state.IsCloseable) { - Log.Debug("Peer closed connection, stopping");; + Log.Debug("Peer closed connection, stopping"); Context.Stop(Self); } return true; @@ -538,7 +691,7 @@ private Receive Closing(ConnectionInfo info, bool confirmClose) Context.Stop(Self); return true; } - default: + default: return false; } }; @@ -550,7 +703,7 @@ private enum DropReason Closing = 2, WritingSuspended = 3, } - + private static string GetDropReasonMessage(DropReason reason) { return reason switch @@ -561,7 +714,7 @@ private static string GetDropReasonMessage(DropReason reason) _ => throw new ArgumentOutOfRangeException(nameof(reason), reason, null) }; } - + private static IOException GetDropMessageException(DropReason reason) { return reason switch @@ -576,8 +729,10 @@ private static IOException GetDropMessageException(DropReason reason) private void DropWrite(WriteCommand write, DropReason reason = DropReason.QueueFull) { // Don't log during closing - if (_traceLogging && reason != DropReason.Closing) Log.Warning("Dropping write [{0}] because {1} - (maxQueueLength={2}, maxFrameSize={3}b)", write.Bytes, GetDropReasonMessage(reason), - Settings.WriteCommandsQueueMaxSize, Settings.MaxFrameSizeBytes); + if (_traceLogging && reason != DropReason.Closing) + Log.Warning("Dropping write [{0}] because {1} - (maxQueueLength={2}, maxFrameSize={3}b)", write.Bytes, + GetDropReasonMessage(reason), + Settings.WriteCommandsQueueMaxSize, Settings.MaxFrameSizeBytes); Sender.Tell(write.FailureMessage.WithCause(GetDropMessageException(reason))); } @@ -597,8 +752,8 @@ protected void CompleteConnect(IActorRef commander, IEnumerable /// We are in the driver's seat and want to close the connection. /// private void HandleCloseCommand(ConnectionInfo info, IActorRef sender, CloseCommand cmd) { // we are closing the connection, so set the hook now. - _closedMessage = new CloseInformation(new HashSet { info.Handler, sender }, cmd.Event); + _closedMessage = new CloseInformation(new HashSet { info.Handler, sender }, cmd.Event); + _state = _state.Update(cmd); switch (cmd) { @@ -632,10 +788,9 @@ private void HandleCloseCommand(ConnectionInfo info, IActorRef sender, CloseComm } case Close: { - _closingRequested = true; - if (IsWritePending) // if we have writes pending, we need to send them + if (IsWritePending) // if we have writes pending { - if(_traceLogging) Log.Debug("Got Close command but writes are still pending."); + if (_traceLogging) Log.Debug("Got Close command but writes are still pending."); Become(Closing(info, false)); } else @@ -645,12 +800,12 @@ private void HandleCloseCommand(ConnectionInfo info, IActorRef sender, CloseComm CloseSocket(); Context.Stop(Self); } + break; } case ConfirmedClose: { - _closingRequested = true; - if(_traceLogging) Log.Debug("Got ConfirmedClose command - waiting for peer to terminate."); + if (_traceLogging) Log.Debug("Got ConfirmedClose command - waiting for peer to terminate."); Become(Closing(info, true)); break; } @@ -665,8 +820,8 @@ private void HandleCloseCommand(ConnectionInfo info, IActorRef sender, CloseComm private void HandleCloseEvent(ConnectionInfo info, IActorRef closeCommander, ConnectionClosed closedEvent) { _closedMessage = new CloseInformation(new HashSet { info.Handler, closeCommander }, closedEvent); - _closingRequested = true; - + _state = _state.Update(closedEvent); + switch (closedEvent) { case Aborted: @@ -677,33 +832,51 @@ private void HandleCloseEvent(ConnectionInfo info, IActorRef closeCommander, Con } case PeerClosed: { - if (_traceLogging) Log.Debug("Got PeerClosed event. Closing connection."); - _peerClosed = true; - if (info.KeepOpenOnPeerClosed) + // we have probably not requested a close yet, but the peer has closed + if (_state is { IsCloseable: false, KeepOpenOnPeerClosed: true }) + { + if (_traceLogging) Log.Debug("Got PeerClosed event but keepOpenOnPeerClosed is set."); + + // set the closure to true - only way we can terminate now is by draining writes + _state = _state with { CloseRequested = true }; + } + + // we are basically checking + if (!_state.IsCloseable) { // we are not closing the socket, but we need to stop reading Context.Become(Closing(info, false)); } else { + if (_traceLogging) Log.Debug("Got PeerClosed event. Closing connection."); // we are closing the socket CloseSocket(); Context.Stop(Self); } + + break; + } + case ErrorClosed: + { + if (_traceLogging) Log.Debug("Got ErrorClosed event. Closing connection."); + // we are closing the socket + CloseSocket(); + Context.Stop(Self); break; } default: { // log a warning - someone sent us the wrong message type Log.Warning("Received unexpected ConnectionClosed event type [{0}]", closedEvent.GetType()); - + // closing connection anyway, I guess Become(Closing(info, false)); break; } } } - + /* Mostly called from outside */ protected void StopWith(CloseInformation closeInfo) { @@ -711,7 +884,7 @@ protected void StopWith(CloseInformation closeInfo) UnsignDeathPact(); Context.Stop(Self); } - + private void Abort() { try @@ -722,6 +895,7 @@ private void Abort() { if (_traceLogging) Log.Debug("setSoLinger(true, 0) failed with [{0}]", e); } + Context.Stop(Self); } @@ -729,8 +903,23 @@ private void CloseSocket() { if (_socketAlreadyClosed) return; _socketAlreadyClosed = true; - try { Socket.Shutdown(SocketShutdown.Both); } catch { /* ignore */ } - try{ Socket.Dispose(); } catch { /* ignore */ } + try + { + Socket.Shutdown(SocketShutdown.Both); + } + catch + { + /* ignore */ + } + + try + { + Socket.Dispose(); + } + catch + { + /* ignore */ + } } protected override void PostStop() @@ -740,14 +929,14 @@ protected override void PostStop() _receiveArgs.Dispose(); _sendArgs.Dispose(); _bufferPool.Return(_receiveBuffer); - + FailUnprocessedPendingWrites(DroppingWriteBecauseClosingException); - if(_closedMessage != null) + if (_closedMessage != null) { // if we have a close message, we need to deliver it DeliverCloseMessages(); } - + base.PostStop(); } @@ -755,7 +944,7 @@ protected override void PostRestart(Exception reason) { throw new IllegalStateException("Restarting not supported for connection actors."); } - + /// /// Groups required connection-related data that are only available once the connection has been fully established. /// @@ -777,17 +966,6 @@ public ConnectionInfo(IActorRef handler, bool keepOpenOnPeerClosed, bool useResu /// Used to transport information to the postStop method to notify /// interested party about a connection close. /// - protected sealed class CloseInformation - { - public ISet NotificationsTo { get; } - - public Event ClosedEvent { get; } - - public CloseInformation(ISet notificationsTo, Tcp.Event closedEvent) - { - NotificationsTo = notificationsTo; - ClosedEvent = closedEvent; - } - } + protected sealed record CloseInformation(ISet NotificationsTo, Event ClosedEvent); } } \ No newline at end of file From 284bbb43217c8eb1c19e5e8f5242bc41489bd27d Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 16 May 2025 12:56:43 -0500 Subject: [PATCH 24/60] fix --- src/core/Akka/IO/TcpConnection.cs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index bbedfc292a0..84b86262bd5 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -832,18 +832,14 @@ private void HandleCloseEvent(ConnectionInfo info, IActorRef closeCommander, Con } case PeerClosed: { - // we have probably not requested a close yet, but the peer has closed - if (_state is { IsCloseable: false, KeepOpenOnPeerClosed: true }) - { - if (_traceLogging) Log.Debug("Got PeerClosed event but keepOpenOnPeerClosed is set."); - - // set the closure to true - only way we can terminate now is by draining writes - _state = _state with { CloseRequested = true }; - } + // set the closure to true - only way we can terminate now is by draining writes + _state = _state with { CloseRequested = true }; // we are basically checking if (!_state.IsCloseable) { + if(_traceLogging) Log.Debug("Got PeerClosed event but we are not closing the socket."); + // we are not closing the socket, but we need to stop reading Context.Become(Closing(info, false)); } From df52514430879b12026d7404f86ca097e4a3b376 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 16 May 2025 13:28:26 -0500 Subject: [PATCH 25/60] hardened receive / stop receiving --- src/core/Akka.Streams.Tests/IO/TcpSpec.cs | 6 +++--- src/core/Akka.Streams/Dsl/Tcp.cs | 3 ++- src/core/Akka/IO/TcpConnection.cs | 20 +++++++++++++++++++- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/core/Akka.Streams.Tests/IO/TcpSpec.cs b/src/core/Akka.Streams.Tests/IO/TcpSpec.cs index a33c0d8aed4..7f09d19ea39 100644 --- a/src/core/Akka.Streams.Tests/IO/TcpSpec.cs +++ b/src/core/Akka.Streams.Tests/IO/TcpSpec.cs @@ -133,7 +133,7 @@ public async Task Outgoing_TCP_stream_must_work_when_client_closes_write_then_re { await this.AssertAllStagesStoppedAsync(async () => { - var testData = ByteString.FromBytes(new byte[] { 1, 2, 3, 4, 5 }); + var testData = ByteString.FromBytes([1, 2, 3, 4, 5]); var server = await new Server(this).InitializeAsync(); var tcpWriteProbe = new TcpWriteProbe(this); @@ -574,7 +574,7 @@ public async Task Tcp_listen_stream_must_be_able_to_implement_echo() var binding = await bindTask.ShouldCompleteWithin(3.Seconds()); var testInput = Enumerable.Range(0, 255) - .Select(i => ByteString.FromBytes(new byte[] { Convert.ToByte(i) })) + .Select(i => ByteString.FromBytes([Convert.ToByte(i)])) .ToList(); var expectedOutput = testInput.Aggregate(ByteString.Empty, (agg, b) => agg.Concat(b)); @@ -604,7 +604,7 @@ public async Task Tcp_listen_stream_must_work_with_a_chain_of_echoes() var echoConnection = Sys.TcpStream().OutgoingConnection(serverAddress); var testInput = Enumerable.Range(0, 255) - .Select(i => ByteString.FromBytes(new byte[] { Convert.ToByte(i) })) + .Select(i => ByteString.FromBytes([Convert.ToByte(i)])) .ToList(); var expectedOutput = testInput.Aggregate(ByteString.Empty, (agg, b) => agg.Concat(b)); diff --git a/src/core/Akka.Streams/Dsl/Tcp.cs b/src/core/Akka.Streams/Dsl/Tcp.cs index c60e239a226..7b4df572688 100644 --- a/src/core/Akka.Streams/Dsl/Tcp.cs +++ b/src/core/Akka.Streams/Dsl/Tcp.cs @@ -180,13 +180,14 @@ public TcpExt(ExtendedActorSystem system) /// TBD /// TBD /// TBD + // TODO: this really needs to be an async method public Source> Bind(string host, int port, int backlog = 100, IImmutableList options = null, bool halfClose = false, TimeSpan? idleTimeout = null) { // DnsEndpoint isn't allowed var ipAddresses = System.Net.Dns.GetHostAddressesAsync(host).Result; if (ipAddresses.Length == 0) - throw new ArgumentException($"Couldn't resolve IpAdress for host {host}", nameof(host)); + throw new ArgumentException($"Couldn't resolve IpAddress for host {host}", nameof(host)); return Source.FromGraph(new ConnectionSourceStage(_system.Tcp(), new IPEndPoint(ipAddresses[0], port), backlog, options, halfClose, idleTimeout, BindShutdownTimeout)); diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 84b86262bd5..fb2e12ed459 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -81,6 +81,16 @@ public static ConnectionState DoneSending(this in ConnectionState state) { return state with { IsSending = false }; } + + public static ConnectionState Receiving(this in ConnectionState state) + { + return state with { IsReceiving = true }; + } + + public static ConnectionState DoneReceiving(this in ConnectionState state) + { + return state with { IsReceiving = false }; + } } /// @@ -132,6 +142,11 @@ internal readonly record struct ConnectionState(Queue<(WriteCommand Cmd, IActorR /// We've fully closed the socket for reading and writing. The socket itself is no longer accessible. /// public bool SocketDisposed { get; init; } + + /// + /// Set when we start an async receive operation, finished when we get a receive complete event. + /// + public bool IsReceiving { get; init; } /// /// Are we sending packets over the network right now? @@ -322,10 +337,11 @@ protected void UnsignDeathPact() private void IssueReceive() { - if (!_state.CanReceive) return; + if (_state.IsReceiving || !_state.CanReceive) return; try { + _state = _state.Receiving(); if (!Socket.ReceiveAsync(_receiveArgs)) Self.Tell(new SocketReceiveCompleted(_receiveArgs.BytesTransferred, _receiveArgs.SocketError)); } @@ -342,6 +358,8 @@ private void IssueReceive() private void HandleRead(IActorRef handler, SocketReceiveCompleted rc) { + _state = _state.DoneReceiving(); + if (_traceLogging) Log.Debug("Received {0} bytes from {1}", rc.Bytes, Socket.RemoteEndPoint); From e96c5392ec713409481582cfc1e896b3d3fd26f4 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 16 May 2025 13:59:40 -0500 Subject: [PATCH 26/60] cleanup --- src/core/Akka.Streams.Tests/IO/TcpSpec.cs | 8 +- .../Implementation/IO/TcpStages.cs | 50 ++++++++---- src/core/Akka/IO/TcpConnection.cs | 77 +++++++++++++++---- src/core/Akka/IO/TcpOutgoingConnection.cs | 5 +- 4 files changed, 103 insertions(+), 37 deletions(-) diff --git a/src/core/Akka.Streams.Tests/IO/TcpSpec.cs b/src/core/Akka.Streams.Tests/IO/TcpSpec.cs index 7f09d19ea39..ca7489c46fb 100644 --- a/src/core/Akka.Streams.Tests/IO/TcpSpec.cs +++ b/src/core/Akka.Streams.Tests/IO/TcpSpec.cs @@ -29,9 +29,11 @@ namespace Akka.Streams.Tests.IO { public class TcpSpec : TcpHelper { - public TcpSpec(ITestOutputHelper helper) : base(@" -akka.loglevel = DEBUG -akka.stream.materializer.subscription-timeout.timeout = 2s", helper) + public TcpSpec(ITestOutputHelper helper) : base(""" + akka.io.tcp.trace-logging = on + akka.loglevel = DEBUG + akka.stream.materializer.subscription-timeout.timeout = 2s + """, helper) { } diff --git a/src/core/Akka.Streams/Implementation/IO/TcpStages.cs b/src/core/Akka.Streams/Implementation/IO/TcpStages.cs index 2b41f1f99a3..db09d374806 100644 --- a/src/core/Akka.Streams/Implementation/IO/TcpStages.cs +++ b/src/core/Akka.Streams/Implementation/IO/TcpStages.cs @@ -424,7 +424,10 @@ public TcpStreamLogic(FlowShape shape, ITcpRole role, En _bytesOut = shape.Outlet; _readHandler = new LambdaOutHandler( - onPull: () => _connection.Tell(Tcp.ResumeReading.Instance, StageActor.Ref), + onPull: () => + { + _connection.Tell(Tcp.ResumeReading.Instance, StageActor.Ref); + }, onDownstreamFinish: cause => { if (cause is SubscriptionWithCancelException.NonFailureCancellation) @@ -549,22 +552,39 @@ private void Connected((IActorRef, object) args) { var msg = args.Item2; - if (msg is Terminated) FailStage(new StreamTcpException("The connection actor has terminated. Stopping now.")); - else if (msg is Tcp.CommandFailed failed) FailStage(new StreamTcpException($"Tcp command {failed.Cmd} failed")); - else if (msg is Tcp.ErrorClosed closed) FailStage(new StreamTcpException($"The connection closed with error: {closed.Cause}")); - else if (msg is Tcp.Aborted) FailStage(new StreamTcpException("The connection has been aborted")); - else if (msg is Tcp.Closed) CompleteStage(); - else if (msg is Tcp.ConfirmedClosed) CompleteStage(); - else if (msg is Tcp.PeerClosed) Complete(_bytesOut); - else if (msg is Tcp.Received received) + switch (msg) { // Keep on reading even when closed. There is no "close-read-side" in TCP - if (IsClosed(_bytesOut)) _connection.Tell(Tcp.ResumeReading.Instance, StageActor.Ref); - else Push(_bytesOut, received.Data); - } - else if (msg is WriteAck) - { - if (!IsClosed(_bytesIn)) Pull(_bytesIn); + case Tcp.Received received when IsClosed(_bytesOut): + _connection.Tell(Tcp.ResumeReading.Instance, StageActor.Ref); + break; + case Tcp.Received received: + Push(_bytesOut, received.Data); + break; + case WriteAck: + { + if (!IsClosed(_bytesIn)) Pull(_bytesIn); + break; + } + case Terminated: + FailStage(new StreamTcpException("The connection actor has terminated. Stopping now.")); + break; + case Tcp.CommandFailed failed: + FailStage(new StreamTcpException($"Tcp command {failed.Cmd} failed")); + break; + case Tcp.ErrorClosed closed: + FailStage(new StreamTcpException($"The connection closed with error: {closed.Cause}")); + break; + case Tcp.Aborted: + FailStage(new StreamTcpException("The connection has been aborted")); + break; + case Tcp.Closed: + case Tcp.ConfirmedClosed: + CompleteStage(); + break; + case Tcp.PeerClosed: + Complete(_bytesOut); + break; } } } diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index fb2e12ed459..8a8dc453d55 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -8,6 +8,7 @@ using System; using System.Buffers; using System.Collections.Generic; +using System.Collections.Immutable; using System.IO; using System.Linq; using System.Net.Sockets; @@ -186,7 +187,7 @@ internal readonly record struct ConnectionState(Queue<(WriteCommand Cmd, IActorR /// /// If either of these conditions are true, we can close the socket SO LONG AS: closing has been requested. /// - public bool IsCloseable => CloseRequested && (!(IsWritePending && CanSend) || (PeerClosed && !KeepOpenOnPeerClosed)); + public bool IsCloseable => CloseRequested && !IsSending && (!(IsWritePending && CanSend) || (PeerClosed && !KeepOpenOnPeerClosed)); } /// @@ -362,22 +363,41 @@ private void HandleRead(IActorRef handler, SocketReceiveCompleted rc) if (_traceLogging) Log.Debug("Received {0} bytes from {1}", rc.Bytes, Socket.RemoteEndPoint); - - // todo: need to harden our SocketError handling - if (rc.Error != SocketError.Success) - { - Log.Error("Closing connection due to IO error {0}", rc.Error); - Self.Tell(new ErrorClosed(rc.Error.ToString())); - return; - } if (rc.Bytes == 0) // CLOSED FOR READING { // signal to the handler that the peer has closed the connection Self.Tell(PeerClosed.Instance); + + // We need to ensure we complete the current receive operation before proceeding, + // because we may still have writes to process before fully closing + IssueReceive(); return; } + // todo: need to harden our SocketError handling + if (rc.Error != SocketError.Success) + { + if (_state.PeerClosed || _state.CloseRequested) + { + Log.Warning("Received IO error {0} while trying to close " + + "- this might be totally normal during closures.", rc.Error); + + // don't bother with the close message if we are already closing + Context.Stop(Self); + } + else + { + Log.Error("Closing connection due to IO error {0} received during read", rc.Error); + var errorClosed = new ErrorClosed(rc.Error.ToString()); + _state = _state.Update(errorClosed); // block further read attempts + Self.Tell(new ErrorClosed(rc.Error.ToString())); + } + return; + } + + + var bs = ByteString.CopyFrom(_receiveBuffer, 0, rc.Bytes); handler.Tell(new Received(bs)); IssueReceive(); @@ -550,9 +570,8 @@ private Receive WaitingForRegistration(IActorRef commander) // we set a default close message here in case the actor dies before we get a close message // this will prevent close messages from going missing // part of the fix for https://github.com/akkadotnet/akka.net/issues/7634 - _closedMessage = - new CloseInformation(new HashSet([register.Handler]), Aborted.Instance); - + UpdateCloseMessage(register.Handler, Aborted.Instance); + Context.SetReceiveTimeout(null); Context.Become(Connected(registerInfo)); // If there is something buffered before we got Register message - put it all to the socket @@ -697,7 +716,14 @@ private Receive Closing(ConnectionInfo info, bool confirmClose) return true; case PeerClosed peerClosed: _state = _state.Update(peerClosed); - if (_state.IsCloseable) + // Even if the peer closed the connection, we might still have data to send + if (_state.IsWritePending) + { + // Continue sending pending data before closing + Log.Debug("Peer closed connection but we still have pending writes"); + TrySendNext(); + } + else if (_state.IsCloseable) { Log.Debug("Peer closed connection, stopping"); Context.Stop(Self); @@ -787,13 +813,25 @@ protected void CompleteConnect(IActorRef commander, IEnumerable.Empty.Add(newCloser), newCloseEvent); + } + else + { + _closedMessage = new CloseInformation(NotificationsTo: _closedMessage.NotificationsTo.Add(newCloser), ClosedEvent: newCloseEvent); + } + } + /// /// We are in the driver's seat and want to close the connection. /// private void HandleCloseCommand(ConnectionInfo info, IActorRef sender, CloseCommand cmd) { // we are closing the connection, so set the hook now. - _closedMessage = new CloseInformation(new HashSet { info.Handler, sender }, cmd.Event); + UpdateCloseMessage(sender, cmd.Event); _state = _state.Update(cmd); switch (cmd) @@ -837,7 +875,8 @@ private void HandleCloseCommand(ConnectionInfo info, IActorRef sender, CloseComm /// private void HandleCloseEvent(ConnectionInfo info, IActorRef closeCommander, ConnectionClosed closedEvent) { - _closedMessage = new CloseInformation(new HashSet { info.Handler, closeCommander }, closedEvent); + UpdateCloseMessage(closeCommander, closedEvent); + UpdateCloseMessage(info.Handler, closedEvent); _state = _state.Update(closedEvent); switch (closedEvent) @@ -894,7 +933,11 @@ private void HandleCloseEvent(ConnectionInfo info, IActorRef closeCommander, Con /* Mostly called from outside */ protected void StopWith(CloseInformation closeInfo) { - _closedMessage = closeInfo; + // need to merge with existing close message + foreach(var n in closeInfo.NotificationsTo) + { + UpdateCloseMessage(n, closeInfo.ClosedEvent); + } UnsignDeathPact(); Context.Stop(Self); } @@ -980,6 +1023,6 @@ public ConnectionInfo(IActorRef handler, bool keepOpenOnPeerClosed, bool useResu /// Used to transport information to the postStop method to notify /// interested party about a connection close. /// - protected sealed record CloseInformation(ISet NotificationsTo, Event ClosedEvent); + protected sealed record CloseInformation(ImmutableHashSet NotificationsTo, Event ClosedEvent); } } \ No newline at end of file diff --git a/src/core/Akka/IO/TcpOutgoingConnection.cs b/src/core/Akka/IO/TcpOutgoingConnection.cs index d9eb7f8e760..88051449179 100644 --- a/src/core/Akka/IO/TcpOutgoingConnection.cs +++ b/src/core/Akka/IO/TcpOutgoingConnection.cs @@ -7,6 +7,7 @@ using System; using System.Collections.Generic; +using System.Collections.Immutable; using System.Linq; using System.Net; using System.Net.Sockets; @@ -81,8 +82,8 @@ private void ReleaseConnectionSocketArgs() private void Stop(Exception cause) { ReleaseConnectionSocketArgs(); - - StopWith(new CloseInformation(new HashSet([_commander]), _connect.FailureMessage.WithCause(cause))); + + StopWith(new CloseInformation(ImmutableHashSet.Empty.Add(_commander), _connect.FailureMessage.WithCause(cause))); } [MethodImpl(MethodImplOptions.AggressiveInlining)] From f8df40698d74ee8d014da69a66fdaeeedc30c2d6 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 16 May 2025 16:03:36 -0500 Subject: [PATCH 27/60] v2 implementation --- src/core/Akka/IO/TcpConnection.cs | 1081 +++++---------------- src/core/Akka/IO/TcpIncomingConnection.cs | 5 - src/core/Akka/IO/TcpOutgoingConnection.cs | 124 ++- 3 files changed, 317 insertions(+), 893 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 8a8dc453d55..5744701f3e1 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -24,171 +24,31 @@ namespace Akka.IO using static Akka.IO.Tcp; using ByteBuffer = ArraySegment; - internal static class TcpStateTransitions - { - public static ConnectionState Update(this in ConnectionState state, Event e) - { - switch (e) - { - case PeerClosed: - return state with { PeerClosed = true }; - case ErrorClosed: // have to close right now - return state with - { - PeerClosed = true, CloseRequested = true, WritingSuspended = true, ReadingSuspended = true - }; - case WritingResumed: - return state with { WritingSuspended = false }; - default: - return state; - } - } + // A **green‑field** rewrite of the connection actor, distilled to + // • 4 stable phases (Connecting ▸ AwaitRegistration ▸ Open ▸ HalfOpen) + // • 8 booleans that fully describe the transient aspects of the socket. + // • single immutable record `ConnState` passed by value. + // • all close logic in one method (TryStop). + // + // ┌───────────────────────── ASCII *phase* diagram ─────────────────────────┐ + // │ │ + // │ (socket.ConnectAsync) │ + // │ +-----------+ Connected +---------------+ │ + // │ |Connecting |──────────────►|AwaitReg |──Register────────────+│ + // │ +-----------+ +-------┬-------+ │ + // │ │ │ + // │ writes/reads ▼ │ + // │ +-----------+ Close +------+ │ + // │ | Open |────────►|Closed| │ + // │ +----┬------+ +------+ │ + // │ │ ConfirmedClose │ + // │ ▼ │ + // │ +-----------+ FIN↑ +------+ │ + // │ | HalfOpen |────────►|Closed| │ + // │ +-----------+ +------+ │ + // │ │ + // └─────────────────────────────────────────────────────────────────────────┘ - public static ConnectionState Update(this in ConnectionState state, Command cmd) - { - switch (cmd) - { - case Register r: - return state with { HasConnected = true, KeepOpenOnPeerClosed = r.KeepOpenOnPeerClosed}; - case ResumeWriting: - return state with { WritingSuspended = false }; - case ResumeReading: - return state with { ReadingSuspended = false }; - case SuspendReading: - return state with { ReadingSuspended = true }; - case ConfirmedClose: - return state with { KeepOpenOnPeerClosed = true, CloseRequested = true }; - case Close: - return state with { CloseRequested = true }; - case Abort: - return state with - { - WritingSuspended = true, - ReadingSuspended = true, - KeepOpenOnPeerClosed = false, - CloseRequested = true - }; - default: - return state; - } - } - - public static ConnectionState Sending(this in ConnectionState state) - { - return state with { IsSending = true }; - } - - public static ConnectionState DoneSending(this in ConnectionState state) - { - return state with { IsSending = false }; - } - - public static ConnectionState Receiving(this in ConnectionState state) - { - return state with { IsReceiving = true }; - } - - public static ConnectionState DoneReceiving(this in ConnectionState state) - { - return state with { IsReceiving = false }; - } - } - - /// - /// Maintains the state of the connection. - /// - /// Externally managed set of pending writes. - /// - /// This data structure is largely needed around dealing with disconnections - because there's several different - /// pre-existing methods we need to support in order to maintain backwards compatibility. - /// - internal readonly record struct ConnectionState(Queue<(WriteCommand Cmd, IActorRef Sender)> PendingWrites) - { - /// - /// A setting that can either be set upon connecting or as a result of the . - /// - public bool KeepOpenOnPeerClosed { get; init; } - - /// - /// We've completed the connection handshake and are now connected. - /// - public bool HasConnected { get; init; } - - /// - /// A closure request has been received from our own process. _We_ are doing the closing. - /// - public bool CloseRequested { get; init; } - - /// - /// Peer has closed for writes - but they might still be open for reading. - /// - /// This happens after we get a 0-byte read from the socket. - /// - public bool PeerClosed { get; init; } - - /// - /// Writing has been suspended - this can be done by the user or by the system. - /// - public bool WritingSuspended { get; init; } - - /// - /// Reading has been suspended - this can be done by the user or by the system. - /// - /// Happens, for instance, when we have processed a and - /// are waiting on our peer to close their end of the connection. - /// - public bool ReadingSuspended { get; init; } - - /// - /// We've fully closed the socket for reading and writing. The socket itself is no longer accessible. - /// - public bool SocketDisposed { get; init; } - - /// - /// Set when we start an async receive operation, finished when we get a receive complete event. - /// - public bool IsReceiving { get; init; } - - /// - /// Are we sending packets over the network right now? - /// - public bool IsSending { get; init; } - - /// - /// We have half-closed our socket for writing, but we are still open for reading. - /// - public bool ClosedForWrites { get; init; } - - /// - /// Can't receive unless: - /// - /// 1. We are connected - /// 2. We are not closed for reading - /// 3. Peer is not closed for writing - /// - public bool CanReceive => (!ReadingSuspended && !PeerClosed) && HasConnected; - - /// - /// Can send as long as we are not closed for writing, and we haven't suspended writing. - /// - public bool CanSend => !WritingSuspended && !ClosedForWrites; - - /// - /// True if we have live writes in the queue or if we are currently sending over network. - /// - public bool IsWritePending => IsSending || PendingWrites.Count > 0; - - /// - /// If we are trying to do a fully graceful close - we can only close in two situations: - /// - /// 1. We have no pending writes / we can still send writes over the network - /// 2. The peer has closed the socket for writing (we're getting no more data from them) and - /// we have not been told to keep the socket open upon peer closure. - /// - /// If either of these conditions are true, we can close the socket SO LONG AS: closing has been requested. - /// - public bool IsCloseable => CloseRequested && !IsSending && (!(IsWritePending && CanSend) || (PeerClosed && !KeepOpenOnPeerClosed)); - } /// /// INTERNAL API: Base class for TcpIncomingConnection and TcpOutgoingConnection. @@ -210,32 +70,75 @@ internal readonly record struct ConnectionState(Queue<(WriteCommand Cmd, IActorR /// Similar approach can be found on other networking libraries (i.e. System.IO.Pipelines and EventStore). /// Both buffers and are pooled to reduce GC pressure. /// - internal abstract class TcpConnection : ActorBase, IRequiresMessageQueue + internal abstract class TcpConnection : ReceiveActor, IRequiresMessageQueue { + /// + /// Immutable connection state – the *only* mutable field in the actor is this record. + /// + private enum Phase + { + Connecting, + AwaitReg, + Open, + HalfOpen, + Closed + } + + private readonly record struct ConnState( + Phase Phase, + bool IsReceiving, + bool IsSending, + bool PeerClosed, + bool ClosedForWrites, + bool ReadingSuspended, + bool WritingSuspended, + bool KeepOpenOnPeerClosed, + Queue<(WriteCommand Cmd, IActorRef Sender)> Queue, + int QueuedBytes) + { + public bool HasPending => IsSending || Queue.Count > 0; + public bool CanSend => !ClosedForWrites && !WritingSuspended; + public bool CanReceive => !PeerClosed && !ReadingSuspended; + + // may stop only when *we* requested it OR peer closed and we are NOT asked to stay open + public bool Closeable(bool closeRequested) => + closeRequested && + !IsReceiving && + !HasPending && + (!PeerClosed || !KeepOpenOnPeerClosed); + + public static ConnState Initial(Queue<(WriteCommand Cmd, IActorRef Sender)> queue) => + new(Phase.Connecting, false, false, false, false, false, false, false, queue, 0); + } + #region Ack‑aware SAEA - private sealed class AckSocketAsyncEventArgs : SocketAsyncEventArgs + private sealed class AckSocketAsyncEventArgs : SocketAsyncEventArgs, INoSerializationVerificationNeeded, + IDeadLetterSuppression { public readonly List<(IActorRef Commander, object Ack)> PendingAcks = new(8); public void ClearAcks() => PendingAcks.Clear(); } - #endregion + private sealed class ReadSocketAsyncEventArgs : SocketAsyncEventArgs, INoSerializationVerificationNeeded, + IDeadLetterSuppression; - #region completion msgs - - private sealed class SocketReceiveCompleted(int bytes, SocketError error) - : INoSerializationVerificationNeeded, IDeadLetterSuppression + private class CommanderDied : IDeadLetterSuppression { - public int Bytes { get; } = bytes; - public SocketError Error { get; } = error; + public static readonly CommanderDied Instance = new(); + + private CommanderDied() + { + } } - private sealed class SocketSendCompleted(int bytes, SocketError error) - : INoSerializationVerificationNeeded, IDeadLetterSuppression + private class HandlerDied : IDeadLetterSuppression { - public int Bytes { get; } = bytes; - public SocketError Error { get; } = error; + public static readonly HandlerDied Instance = new(); + + private HandlerDied() + { + } } #endregion @@ -247,24 +150,20 @@ private sealed class SocketSendCompleted(int bytes, SocketError error) private readonly ArrayPool _bufferPool = ArrayPool.Shared; private readonly Queue<(WriteCommand Cmd, IActorRef Sender)> _pendingWrites; - private readonly byte[] _receiveBuffer; - private SocketAsyncEventArgs _receiveArgs; + private ReadSocketAsyncEventArgs _receiveArgs; private AckSocketAsyncEventArgs _sendArgs; - private IActorRef _watchedActor = Context.System.DeadLetters; - private readonly int _maxWriteCapacity; - private ConnectionState _state; + private bool _closeRequested; + private readonly int _maxQueuedBytes; - private readonly bool _traceLogging; + private ConnState _state; - private long _pendingOutboundBytes; - - // so we don't try to close the socket a second time during PostStop - private bool _socketAlreadyClosed; + private readonly bool _traceLogging; - private CloseInformation? _closedMessage; // for ConnectionClosed message in postStop + private IActorRef? _commander; + private IActorRef? _handler; private static readonly IOException DroppingWriteBecauseClosingException = new("Dropping write because the connection is closing"); @@ -278,26 +177,26 @@ private sealed class SocketSendCompleted(int bytes, SocketError error) protected TcpConnection(TcpSettings settings, Socket socket) { Settings = settings; - _maxWriteCapacity = settings.WriteCommandsQueueMaxSize; - _pendingWrites = _maxWriteCapacity > 0 - ? new Queue<(WriteCommand Cmd, IActorRef Sender)>(_maxWriteCapacity) - : new Queue<(WriteCommand Cmd, IActorRef Sender)>(); // unbounded - ; + _maxQueuedBytes = settings.WriteCommandsQueueMaxSize; // –1 ⇒ unlimited; + _pendingWrites = new Queue<(WriteCommand Cmd, IActorRef Sender)>(16); + _traceLogging = Settings.TraceLogging; - _state = new ConnectionState(_pendingWrites); + _state = ConnState.Initial(_pendingWrites); Socket = socket ?? throw new ArgumentNullException(nameof(socket)); _receiveBuffer = _bufferPool.Rent(settings.MaxFrameSizeBytes); + _receiveArgs = new ReadSocketAsyncEventArgs(); + _sendArgs = new AckSocketAsyncEventArgs(); InitSocketEventArgs(); } private void InitSocketEventArgs() { - _receiveArgs = new SocketAsyncEventArgs(); + _receiveArgs.SetBuffer(_receiveBuffer, 0, _receiveBuffer.Length); _receiveArgs.UserToken = Self; _receiveArgs.Completed += OnCompleted; - _sendArgs = new AckSocketAsyncEventArgs(); + _sendArgs.UserToken = Self; _sendArgs.Completed += OnCompleted; } @@ -305,724 +204,268 @@ private void InitSocketEventArgs() private static void OnCompleted(object? sender, SocketAsyncEventArgs e) { if (e.UserToken is not IActorRef self) return; - switch (e.LastOperation) - { - case SocketAsyncOperation.Receive: - self.Tell(new SocketReceiveCompleted(e.BytesTransferred, e.SocketError)); - break; - case SocketAsyncOperation.Send: - self.Tell(new SocketSendCompleted(e.BytesTransferred, e.SocketError)); - break; - default: - self.Tell(new ErrorClosed($"Unexpected socket op {e.LastOperation}")); - break; - } + self.Tell(e); } - /// - /// Returns true if write is in-progress over the wire or if we have writes pending in the queue. - /// - public bool IsWritePending => _state.IsWritePending; + /* ================================================================= */ + /* Base‑class public API */ + /* ================================================================= */ - protected void SignDeathPact(IActorRef actor) - { - UnsignDeathPact(); - _watchedActor = actor; - Context.Watch(actor); - } - - protected void UnsignDeathPact() - { - if (!ReferenceEquals(_watchedActor, Context.System.DeadLetters)) Context.Unwatch(_watchedActor); - } - - private void IssueReceive() + protected override void PostStop() { - if (_state.IsReceiving || !_state.CanReceive) return; - try { - _state = _state.Receiving(); - if (!Socket.ReceiveAsync(_receiveArgs)) - Self.Tell(new SocketReceiveCompleted(_receiveArgs.BytesTransferred, _receiveArgs.SocketError)); - } - catch (ObjectDisposedException) - { - // Socket was closed, signal peer closed - Self.Tell(PeerClosed.Instance); - } - catch (SocketException ex) - { - Self.Tell(new SocketReceiveCompleted(0, ex.SocketErrorCode)); - } - } - - private void HandleRead(IActorRef handler, SocketReceiveCompleted rc) - { - _state = _state.DoneReceiving(); - - if (_traceLogging) - Log.Debug("Received {0} bytes from {1}", rc.Bytes, Socket.RemoteEndPoint); - - if (rc.Bytes == 0) // CLOSED FOR READING - { - // signal to the handler that the peer has closed the connection - Self.Tell(PeerClosed.Instance); - - // We need to ensure we complete the current receive operation before proceeding, - // because we may still have writes to process before fully closing - IssueReceive(); - return; + Socket.Dispose(); } - - // todo: need to harden our SocketError handling - if (rc.Error != SocketError.Success) + catch { - if (_state.PeerClosed || _state.CloseRequested) - { - Log.Warning("Received IO error {0} while trying to close " + - "- this might be totally normal during closures.", rc.Error); - - // don't bother with the close message if we are already closing - Context.Stop(Self); - } - else - { - Log.Error("Closing connection due to IO error {0} received during read", rc.Error); - var errorClosed = new ErrorClosed(rc.Error.ToString()); - _state = _state.Update(errorClosed); // block further read attempts - Self.Tell(new ErrorClosed(rc.Error.ToString())); - } - return; + /* ignore */ } - - - - var bs = ByteString.CopyFrom(_receiveBuffer, 0, rc.Bytes); - handler.Tell(new Received(bs)); - IssueReceive(); - } - - private void IssueSend(IList> buffers) - { - _sendArgs.BufferList = buffers; - if (!Socket.SendAsync(_sendArgs)) - Self.Tell(new SocketSendCompleted(_sendArgs.BytesTransferred, _sendArgs.SocketError)); - } - - private void TrySendNext() - { - // already sending or no writes to send - if (_state.IsSending || _pendingWrites.Count == 0) return; - var maxBytes = _receiveBuffer.Length; - var accumulated = 0; - var batch = new List(); - _sendArgs.ClearAcks(); + _receiveArgs.Dispose(); + _sendArgs.Dispose(); + _bufferPool.Return(_receiveBuffer); - while (_pendingWrites.Count > 0 && accumulated < maxBytes) + // fail everything still queued + while (_pendingWrites.Count > 0) { - var (cmd, snd) = _pendingWrites.Peek(); - switch (cmd) - { - case Write w when !w.Data.IsEmpty: - var wouldBe = accumulated + w.Data.Count; - if (wouldBe > maxBytes && batch.Count > 0) goto done; - _pendingWrites.Dequeue(); - batch.Add(w.Data); - accumulated = wouldBe; - if (!Equals(w.Ack, NoAck.Instance)) - _sendArgs.PendingAcks.Add((snd, w.Ack)); - break; - case Write w: - // empty write, discard and ACK if needed - can't send a 0-length message - _pendingWrites.Dequeue(); - if (w.WantsAck) snd.Tell(w.Ack); - break; - default: - _pendingWrites.Dequeue(); - snd.Tell(new CommandFailed(cmd)); - break; - } + var (cmd, snd) = _pendingWrites.Dequeue(); + snd.Tell(cmd.FailureMessage.WithCause(DroppingWriteBecauseClosingException)); } - done: - if (batch.Count == 0) + if (_closeEvent != null) { - return; + foreach (var sub in _closeNotify) + sub.Tell(_closeEvent); } - - _state = _state.Sending(); - var payload = FlattenByteStrings(batch); - IssueSend(payload); } /// - /// Called when the socket closes before we have processed all pending writes. + /// Used in subclasses to start the common machinery above once a channel is connected /// - private void FailUnprocessedPendingWrites(Exception cause) + protected void CompleteConnect(IActorRef commander, IEnumerable options) { - foreach (var (cmd, ack) in _pendingWrites) + // Turn off Nagle's algorithm by default + try { - var failure = cmd.FailureMessage.WithCause(cause); - ack.Tell(failure); + Socket.NoDelay = true; } - - _pendingWrites.Clear(); - } - - private void HandleSendCompleted(SocketSendCompleted socketSendCompleted) - { - _state = _state.DoneSending(); - - if (_traceLogging) - Log.Debug("Sent {0} bytes to {1}", socketSendCompleted.Bytes, Socket.RemoteEndPoint); - - // check for errors - if (socketSendCompleted.Error != SocketError.Success) + catch (SocketException e) { - Log.Error("Closing connection due to IO error {0} received during send", socketSendCompleted.Error); - Self.Tell(new ErrorClosed(socketSendCompleted.Error.ToString())); - return; + Log.Debug("Could not enable TcpNoDelay: {0}", e.Message); } - foreach (var (c, ack) in _sendArgs.PendingAcks) - c.Tell(ack); - _sendArgs.ClearAcks(); - _sendArgs.BufferList = null; - - TrySendNext(); - TryCloseIfDone(); - } - - private void DeliverCloseMessages() - { - if (_closedMessage == null) return; - foreach (var handler in _closedMessage.NotificationsTo) + foreach (var option in options) { - handler.Tell(_closedMessage.ClosedEvent); + option.AfterConnect(Socket); } - } - private void TryCloseIfDone() - { - if (!_state.CloseRequested) return; - - if (_traceLogging) - Log.Debug("TryCloseIfDone called, sending={0}, pendingWrites={1}, peerClosed={2}", - _state.IsSending, _pendingWrites.Count, _state.PeerClosed); - - // Factors in several different configuration options to determine if we can close ourselves or not - if (!_state.IsCloseable) return; - - // No pending writes, so we can safely close - // Note: We no longer wait for _peerClosed in any scenario to avoid deadlocks in Akka.Streams - // The previous implementation could hang when Streams TCP stages sent ConfirmedClose but were - // waiting for connection completion which never happened because we were waiting for peer close - Context.Stop(Self); - } - - private static IList> FlattenByteStrings(List parts) - { - if (parts.Count == 1) - return parts[0].Buffers; + _commander = commander; + Context.WatchWith(_commander, CommanderDied.Instance); + commander.Tell(new Connected(Socket.RemoteEndPoint, Socket.LocalEndPoint)); - return parts.SelectMany(c => c.Buffers).ToArray(); + Context.SetReceiveTimeout(Settings.RegisterTimeout); + _state = _state with { Phase = Phase.AwaitReg }; + _commander = commander; + Become(AwaitRegBehaviour); } - // STATES + /* ================================================================= */ + /* Close‑notification tracking */ + /* ---------------------------------------------------------------- */ + private readonly HashSet _closeNotify = []; + private Event? _closeEvent; - private bool TryBuffer(WriteCommand cmd, IActorRef sender) + protected void MarkClose(IActorRef src, Event evt) { - // buffer is unlimited OR we're below the max write capacity - if (_maxWriteCapacity < 0 || _pendingWrites.Count < _maxWriteCapacity) - { - _pendingWrites.Enqueue((cmd, sender)); - _pendingOutboundBytes += cmd.Bytes; - return true; - } - - // buffer is full - return false; + _closeEvent = evt; + _closeNotify.Add(src); + if (_handler != null) _closeNotify.Add(_handler); } - /// - /// Connection established, waiting for registration from user handler. - /// - private Receive WaitingForRegistration(IActorRef commander) - { - return message => - { - switch (message) - { - case Register register: - // up to this point we've been watching the commander, - // but since registration is now complete we only need to watch the handler from here on - if (!Equals(register.Handler, commander)) - SignDeathPact(register.Handler); // will unsign death pact with commander automatically - - if (_traceLogging) Log.Debug("[{0}] registered as connection handler", register.Handler); - _state = _state.Update(register); - - var registerInfo = new ConnectionInfo(register.Handler, register.KeepOpenOnPeerClosed, - register.UseResumeWriting); - - // we set a default close message here in case the actor dies before we get a close message - // this will prevent close messages from going missing - // part of the fix for https://github.com/akkadotnet/akka.net/issues/7634 - UpdateCloseMessage(register.Handler, Aborted.Instance); - - Context.SetReceiveTimeout(null); - Context.Become(Connected(registerInfo)); - // If there is something buffered before we got Register message - put it all to the socket - TrySendNext(); - // start reading - IssueReceive(); - return true; - case CloseCommand cmd: - _state = _state.Update(cmd); - // Default connection info for unregistered connections - always uses keepOpenOnPeerClosed: false - var info = new ConnectionInfo(commander, keepOpenOnPeerClosed: false, useResumeWriting: false); - HandleCloseCommand(info, Sender, cmd); - return true; - case ReceiveTimeout: - // after sending `Register` user should watch this actor to make sure - // it didn't die because of the timeout - Log.Debug("Configured registration timeout of [{0}] expired, stopping", - Settings.RegisterTimeout); - Context.Stop(Self); - return true; - case WriteCommand write: - // Have to buffer writes until registration - var buffered = TryBuffer(write, Sender); - if (!buffered) - { - DropWrite(write); - } - else - { - Log.Debug("Received Write command before Register command. " + - "It will be buffered until Register will be received (buffered write size is {0} bytes)", - write.Bytes); - } - - return true; - case Terminated t: - { - // if the handler dies before registration, we need to stop - if (t.ActorRef.Equals(commander)) - { - Log.Debug("Handler [{0}] died before registration, stopping", t.ActorRef); - Context.Stop(Self); - } - else - { - // ignore - Log.Debug("Handler [{0}] died before registration, ignoring", t.ActorRef); - } - - return true; - } - default: return false; - } - }; - } - /// - /// Normal connected state. - /// - private Receive Connected(ConnectionInfo info) + private void AwaitRegBehaviour() { - return message => + Receive(reg => { - switch (message) - { - case SocketReceiveCompleted r: - HandleRead(info.Handler, r); - return true; - case WriteCommand write: - var buffered = TryBuffer(write, Sender); - if (!buffered) - { - DropWrite(write); - } - TrySendNext(); - return true; - case SocketSendCompleted sendCompleted: - HandleSendCompleted(sendCompleted); - return true; - case CloseCommand cmd: // we are trying to close the socket first - HandleCloseCommand(info, Sender, cmd); - return true; - case ConnectionClosed closed: // peer is closing the socket - HandleCloseEvent(info, Sender, closed); - return true; - case SuspendReading sr: - _state = _state.Update(sr); - return true; - case ResumeReading rr: - // try to drive read-loop forward again - _state = _state.Update(rr); - IssueReceive(); - return true; - case Terminated t: // handler died - { - Log.Debug("Handler [{0}] died, stopping", t.ActorRef); - Context.Stop(Self); - return true; - } - default: return false; - } - }; - } + _handler = reg.Handler; + Context.WatchWith(_handler, HandlerDied.Instance); + Context.Unwatch(_commander); + _state = _state with { Phase = Phase.Open, KeepOpenOnPeerClosed = reg.KeepOpenOnPeerClosed }; + Context.SetReceiveTimeout(null); + Become(OpenBehaviour); + IssueReceive(); + TrySend(); + }); - /// - /// Connection is closing but a write has to be finished first - /// - private Receive Closing(ConnectionInfo info, bool confirmClose) - { - return message => + Receive(Enqueue); + Receive(c => { - switch (message) - { - case SocketReceiveCompleted r: - HandleRead(info.Handler, r); - return true; - case SocketSendCompleted s: - HandleSendCompleted(s); - if (_state.IsWritePending) - { - // done writing, so we can now half-close the socket - if (_traceLogging) - Log.Debug("Running in close-confirm mode, half-closing socket for writes"); - - // We will need to get an EOF - Socket.Shutdown(SocketShutdown.Send); - } - return true; - case WriteCommand write: // no more writes once we start closing - DropWrite(write, DropReason.Closing); - return true; - case SuspendReading sr: - _state = _state.Update(sr); - return true; - case ResumeReading rr: - // try to drive read-loop forward again - _state = _state.Update(rr); - IssueReceive(); - return true; - case CloseCommand a: - HandleCloseCommand(info, Sender, a); - return true; - case PeerClosed peerClosed: - _state = _state.Update(peerClosed); - // Even if the peer closed the connection, we might still have data to send - if (_state.IsWritePending) - { - // Continue sending pending data before closing - Log.Debug("Peer closed connection but we still have pending writes"); - TrySendNext(); - } - else if (_state.IsCloseable) - { - Log.Debug("Peer closed connection, stopping"); - Context.Stop(Self); - } - return true; - case Terminated t: // handler died - { - Log.Debug("Handler [{0}] died, stopping", t.ActorRef); - Context.Stop(Self); - return true; - } - default: - return false; - } - }; + _closeRequested = true; + TryStop(); + }); + Receive(_ => Context.Stop(Self)); } - private enum DropReason + private void OpenBehaviour() { - QueueFull = 1, - Closing = 2, - WritingSuspended = 3, - } + Receive(HandleReceiveCompleted); + Receive(HandleSendCompleted); - private static string GetDropReasonMessage(DropReason reason) - { - return reason switch - { - DropReason.QueueFull => "queue is full", - DropReason.Closing => "connection is closing", - DropReason.WritingSuspended => "writing is suspended", - _ => throw new ArgumentOutOfRangeException(nameof(reason), reason, null) - }; - } + Receive(Enqueue); - private static IOException GetDropMessageException(DropReason reason) - { - return reason switch + Receive(_ => { - DropReason.QueueFull => DroppingWriteBecauseQueueIsFullException, - DropReason.Closing => DroppingWriteBecauseClosingException, - DropReason.WritingSuspended => DroppingWriteBecauseWritingIsSuspendedException, - _ => throw new ArgumentOutOfRangeException(nameof(reason), reason, null) - }; - } + _closeRequested = true; + TryStop(); + }); + Receive(_ => + { + HalfCloseWriteSide(); + _closeRequested = true; + TryStop(); + }); + Receive(_ => { Context.Stop(Self); }); - private void DropWrite(WriteCommand write, DropReason reason = DropReason.QueueFull) - { - // Don't log during closing - if (_traceLogging && reason != DropReason.Closing) - Log.Warning("Dropping write [{0}] because {1} - (maxQueueLength={2}, maxFrameSize={3}b)", write.Bytes, - GetDropReasonMessage(reason), - Settings.WriteCommandsQueueMaxSize, Settings.MaxFrameSizeBytes); - Sender.Tell(write.FailureMessage.WithCause(GetDropMessageException(reason))); + Receive(_ => + { + _state = _state with { ReadingSuspended = false }; + IssueReceive(); + }); + Receive(_ => { _state = _state with { ReadingSuspended = true }; }); + Receive(_ => + { + _state = _state with { WritingSuspended = false }; + TrySend(); + }); + + Receive(h => + { + Log.Debug("Handler died, stopping connection actor"); + Context.Stop(Self); + }); + //Receive(_=> { _st = _st with { WritingSuspended=true }; }); } - // AUXILIARIES and IMPLEMENTATION + /* ----------------------------------------------------------------- */ + /* Socket‑event handlers */ + /* ----------------------------------------------------------------- */ - /// - /// Used in subclasses to start the common machinery above once a channel is connected - /// - protected void CompleteConnect(IActorRef commander, IEnumerable options) + private void HandleReceiveCompleted(SocketAsyncEventArgs ea) { - // Turn off Nagle's algorithm by default - try + _state = _state with { IsReceiving = false }; + if (ea is { SocketError: SocketError.Success, BytesTransferred: > 0 }) { - Socket.NoDelay = true; - } - catch (SocketException e) - { - Log.Debug(e, "Could not enable TcpNoDelay: {0}", e.Message); + _handler!.Tell(new Received(ByteString.CopyFrom(_receiveBuffer, 0, ea.BytesTransferred))); + IssueReceive(); + return; } + // FIN or error + MarkClose(Self, PeerClosed.Instance); + _handler!.Tell(PeerClosed.Instance); + _state = _state with { PeerClosed = true }; + TryStop(); + } - // set the system buffer sizes - Socket.SendBufferSize = Settings.SendBufferSize; - Socket.ReceiveBufferSize = Settings.ReceiveBufferSize; - - foreach (var option in options) - { - option.AfterConnect(Socket); - } + private void HandleSendCompleted(AckSocketAsyncEventArgs ea) + { + _state = _state with { IsSending = false }; - commander.Tell(new Connected(Socket.RemoteEndPoint, Socket.LocalEndPoint)); + foreach (var (c, ack) in ea.PendingAcks) + c.Tell(ack); + ea.ClearAcks(); + ea.BufferList = null; // release refs - Context.SetReceiveTimeout(Settings.RegisterTimeout); - Context.Become(WaitingForRegistration(commander)); + TrySend(); + TryStop(); } - private void UpdateCloseMessage(IActorRef newCloser, Event newCloseEvent) + /* ----------------------------------------------------------------- */ + /* Read / Write helpers */ + /* ----------------------------------------------------------------- */ + + private void IssueReceive() { - if(_closedMessage == null) - { - _closedMessage = new CloseInformation(ImmutableHashSet.Empty.Add(newCloser), newCloseEvent); - } - else - { - _closedMessage = new CloseInformation(NotificationsTo: _closedMessage.NotificationsTo.Add(newCloser), ClosedEvent: newCloseEvent); - } + if (!_state.CanReceive || _state.IsReceiving) return; + _receiveArgs.SetBuffer(_receiveBuffer, 0, _receiveBuffer.Length); + _state = _state with { IsReceiving = true }; + if (!Socket.ReceiveAsync(_receiveArgs)) Self.Tell(_receiveArgs, Self); } - /// - /// We are in the driver's seat and want to close the connection. - /// - private void HandleCloseCommand(ConnectionInfo info, IActorRef sender, CloseCommand cmd) + private void Enqueue(WriteCommand cmd) { - // we are closing the connection, so set the hook now. - UpdateCloseMessage(sender, cmd.Event); - _state = _state.Update(cmd); - - switch (cmd) + var b = (int)cmd.Bytes; + if (_maxQueuedBytes >= 0 && _state.QueuedBytes + b > _maxQueuedBytes) { - case Abort _: - { - if (_traceLogging) Log.Debug("Got Abort command. RESETing connection."); - Abort(); - break; - } - case Close: - { - if (IsWritePending) // if we have writes pending - { - if (_traceLogging) Log.Debug("Got Close command but writes are still pending."); - Become(Closing(info, false)); - } - else - { - // if we are not writing, we can close the socket right away - if (_traceLogging) Log.Debug("Got Close command, closing connection."); - CloseSocket(); - Context.Stop(Self); - } - - break; - } - case ConfirmedClose: - { - if (_traceLogging) Log.Debug("Got ConfirmedClose command - waiting for peer to terminate."); - Become(Closing(info, true)); - break; - } - default: - throw new ArgumentOutOfRangeException(nameof(cmd), cmd, "Unknown close command"); + Sender.Tell(cmd.FailureMessage.WithCause(new IOException("write‑queue full"))); + return; } + + _pendingWrites.Enqueue((cmd, Sender)); + _state = _state with { QueuedBytes = _state.QueuedBytes + b }; + TrySend(); } - /// - /// Someone else is closing the connection, so we need to handle it. - /// - private void HandleCloseEvent(ConnectionInfo info, IActorRef closeCommander, ConnectionClosed closedEvent) + private void TrySend() { - UpdateCloseMessage(closeCommander, closedEvent); - UpdateCloseMessage(info.Handler, closedEvent); - _state = _state.Update(closedEvent); + if (_state.IsSending || _pendingWrites.Count == 0 || !_state.CanSend) return; + var segs = new List>(8); + var batchBytes = 0; - switch (closedEvent) + while(_pendingWrites.Count>0) { - case Aborted: - { - if (_traceLogging) Log.Debug("Got Aborted event. RESETing connection."); - Context.Stop(Self); - break; - } - case PeerClosed: - { - // set the closure to true - only way we can terminate now is by draining writes - _state = _state with { CloseRequested = true }; - - // we are basically checking - if (!_state.IsCloseable) - { - if(_traceLogging) Log.Debug("Got PeerClosed event but we are not closing the socket."); - - // we are not closing the socket, but we need to stop reading - Context.Become(Closing(info, false)); - } - else - { - if (_traceLogging) Log.Debug("Got PeerClosed event. Closing connection."); - // we are closing the socket - CloseSocket(); - Context.Stop(Self); - } - - break; - } - case ErrorClosed: + var (cmd,snd) = _pendingWrites.Peek(); + if(cmd is not Write w) { - if (_traceLogging) Log.Debug("Got ErrorClosed event. Closing connection."); - // we are closing the socket - CloseSocket(); - Context.Stop(Self); - break; + // unsupported command, fail fast + _pendingWrites.Dequeue(); + snd.Tell(cmd.FailureMessage); + continue; } - default: - { - // log a warning - someone sent us the wrong message type - Log.Warning("Received unexpected ConnectionClosed event type [{0}]", closedEvent.GetType()); - // closing connection anyway, I guess - Become(Closing(info, false)); + // do not break MTU / send‑buffer – simple heuristic + if(batchBytes !=0 && batchBytes + w.Data.Count > Settings.MaxFrameSizeBytes) break; - } - } - } - /* Mostly called from outside */ - protected void StopWith(CloseInformation closeInfo) - { - // need to merge with existing close message - foreach(var n in closeInfo.NotificationsTo) - { - UpdateCloseMessage(n, closeInfo.ClosedEvent); - } - UnsignDeathPact(); - Context.Stop(Self); - } + // dequeue & account + _pendingWrites.Dequeue(); + _state = _state with { QueuedBytes = _state.QueuedBytes - w.Data.Count }; + batchBytes += w.Data.Count; + segs.AddRange(w.Data.Buffers); - private void Abort() - { - try - { - Socket.LingerState = new LingerOption(true, 0); // causes the following close() to send TCP RST - } - catch (Exception e) - { - if (_traceLogging) Log.Debug("setSoLinger(true, 0) failed with [{0}]", e); + if(w.WantsAck) _sendArgs.PendingAcks.Add((snd, w.Ack)); } - Context.Stop(Self); + if(segs.Count == 0) return; // only empty writes encountered + + _sendArgs.BufferList = segs; + _state = _state with { IsSending = true }; + if(!Socket.SendAsync(_sendArgs)) Self.Tell(_sendArgs, Self); } - private void CloseSocket() + private void HalfCloseWriteSide() { - if (_socketAlreadyClosed) return; - _socketAlreadyClosed = true; + if (_state.ClosedForWrites) return; try { - Socket.Shutdown(SocketShutdown.Both); + Socket.Shutdown(SocketShutdown.Send); } catch { /* ignore */ } - try - { - Socket.Dispose(); - } - catch - { - /* ignore */ - } + _state = _state with { ClosedForWrites = true, Phase = Phase.HalfOpen }; } - protected override void PostStop() + private void TryStop() { - if (_traceLogging) Log.Debug("Stopping connection actor [{0}]", Self); - CloseSocket(); // just in case we didn't shut ourselves down gracefully first - _receiveArgs.Dispose(); - _sendArgs.Dispose(); - _bufferPool.Return(_receiveBuffer); - - FailUnprocessedPendingWrites(DroppingWriteBecauseClosingException); - if (_closedMessage != null) - { - // if we have a close message, we need to deliver it - DeliverCloseMessages(); - } - - base.PostStop(); - } - - protected override void PostRestart(Exception reason) - { - throw new IllegalStateException("Restarting not supported for connection actors."); - } - - /// - /// Groups required connection-related data that are only available once the connection has been fully established. - /// - private sealed class ConnectionInfo - { - public readonly IActorRef Handler; - public readonly bool KeepOpenOnPeerClosed; - public readonly bool UseResumeWriting; - - public ConnectionInfo(IActorRef handler, bool keepOpenOnPeerClosed, bool useResumeWriting) + if (_state.Closeable(_closeRequested)) { - Handler = handler; - KeepOpenOnPeerClosed = keepOpenOnPeerClosed; - UseResumeWriting = useResumeWriting; + // graceful stop + Self.Tell(PoisonPill.Instance); } } - - /// - /// Used to transport information to the postStop method to notify - /// interested party about a connection close. - /// - protected sealed record CloseInformation(ImmutableHashSet NotificationsTo, Event ClosedEvent); } } \ No newline at end of file diff --git a/src/core/Akka/IO/TcpIncomingConnection.cs b/src/core/Akka/IO/TcpIncomingConnection.cs index a7ac8549262..26a618b1371 100644 --- a/src/core/Akka/IO/TcpIncomingConnection.cs +++ b/src/core/Akka/IO/TcpIncomingConnection.cs @@ -38,10 +38,5 @@ protected override void PreStart() { CompleteConnect(_bindHandler, _options); } - - protected override bool Receive(object message) - { - throw new NotSupportedException(); - } } } diff --git a/src/core/Akka/IO/TcpOutgoingConnection.cs b/src/core/Akka/IO/TcpOutgoingConnection.cs index 88051449179..9e36c2238c8 100644 --- a/src/core/Akka/IO/TcpOutgoingConnection.cs +++ b/src/core/Akka/IO/TcpOutgoingConnection.cs @@ -6,8 +6,6 @@ //----------------------------------------------------------------------- using System; -using System.Collections.Generic; -using System.Collections.Immutable; using System.Linq; using System.Net; using System.Net.Sockets; @@ -15,7 +13,6 @@ using Akka.Actor; using Akka.Annotations; using Akka.Event; -using Akka.Util; namespace Akka.IO { @@ -35,26 +32,24 @@ internal sealed class TcpOutgoingConnection : TcpConnection public TcpOutgoingConnection(TcpExt tcp, IActorRef commander, Tcp.Connect connect) : base( - (connect.TcpSettings ?? tcp.Settings), - (connect.TcpSettings ?? tcp.Settings).OutgoingSocketForceIpv4 - ? new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp) { Blocking = false } - : new Socket(SocketType.Stream, ProtocolType.Tcp) { Blocking = false }) + (connect.TcpSettings ?? tcp.Settings), + (connect.TcpSettings ?? tcp.Settings).OutgoingSocketForceIpv4 + ? new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp) { Blocking = false } + : new Socket(SocketType.Stream, ProtocolType.Tcp) { Blocking = false }) { _commander = commander; _connect = connect; - SignDeathPact(commander); - foreach (var option in connect.Options) { option.BeforeConnect(Socket); } - + if (connect.LocalAddress != null) Socket.Bind(connect.LocalAddress); if (connect.Timeout.HasValue) - Context.SetReceiveTimeout(connect.Timeout.Value); //Initiate connection timeout if supplied + Context.SetReceiveTimeout(connect.Timeout.Value); //Initiate connection timeout if supplied } private void ReleaseConnectionSocketArgs() @@ -82,8 +77,8 @@ private void ReleaseConnectionSocketArgs() private void Stop(Exception cause) { ReleaseConnectionSocketArgs(); - - StopWith(new CloseInformation(ImmutableHashSet.Empty.Add(_commander), _connect.FailureMessage.WithCause(cause))); + + MarkClose(_commander, _connect.FailureMessage.WithCause(cause)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -99,7 +94,7 @@ private void ReportConnectFailure(Action thunk) Stop(e); } } - + protected override void PreStart() { ReportConnectFailure(() => @@ -109,9 +104,10 @@ protected override void PreStart() Log.Debug("Resolving {0} before connecting", remoteAddress.Host); var resolved = Dns.ResolveName(remoteAddress.Host, Context.System, Self); if (resolved == null) - Become(Resolving(remoteAddress)); + Become(() => Resolving(remoteAddress)); else if (resolved.Ipv4.Any() && resolved.Ipv6.Any()) // one of both families - Register(new IPEndPoint(resolved.Ipv4.First(), remoteAddress.Port), new IPEndPoint(resolved.Ipv6.First(), remoteAddress.Port)); + Register(new IPEndPoint(resolved.Ipv4.First(), remoteAddress.Port), + new IPEndPoint(resolved.Ipv6.First(), remoteAddress.Port)); else // one or the other Register(new IPEndPoint(resolved.Addr, remoteAddress.Port), null); } @@ -119,7 +115,9 @@ protected override void PreStart() { Register(point, null); } - else throw new NotSupportedException($"Couldn't connect to [{_connect.RemoteAddress}]: only IP and DNS-based endpoints are supported"); + else + throw new NotSupportedException( + $"Couldn't connect to [{_connect.RemoteAddress}]: only IP and DNS-based endpoints are supported"); }); } @@ -131,33 +129,23 @@ protected override void PostStop() base.PostStop(); } - protected override bool Receive(object message) - { - throw new NotSupportedException(); - } - - private Receive Resolving(DnsEndPoint remoteAddress) + private void Resolving(DnsEndPoint remoteAddress) { - return message => + Receive(resolved => { - if (message is Dns.Resolved resolved) + if (resolved.Ipv4.Any() && resolved.Ipv6.Any()) // multiple addresses { - if (resolved.Ipv4.Any() && resolved.Ipv6.Any()) // multiple addresses - { - ReportConnectFailure(() => Register( - new IPEndPoint(resolved.Ipv4.First(), remoteAddress.Port), - new IPEndPoint(resolved.Ipv6.First(), remoteAddress.Port))); - } - else // only one address family. No fallbacks. - { - ReportConnectFailure(() => Register( - new IPEndPoint(resolved.Addr, remoteAddress.Port), - null)); - } - return true; + ReportConnectFailure(() => Register( + new IPEndPoint(resolved.Ipv4.First(), remoteAddress.Port), + new IPEndPoint(resolved.Ipv6.First(), remoteAddress.Port))); } - return false; - }; + else // only one address family. No fallbacks. + { + ReportConnectFailure(() => Register( + new IPEndPoint(resolved.Addr, remoteAddress.Port), + null)); + } + }); } private static SocketAsyncEventArgs CreateSocketEventArgs(IActorRef onCompleteNotificationsReceiver) @@ -195,26 +183,26 @@ private void Register(IPEndPoint address, IPEndPoint fallbackAddress) if (!Socket.ConnectAsync(_connectArgs)) Self.Tell(IO.Tcp.SocketConnected.Instance); - Become(Connecting(Settings.FinishConnectRetries, _connectArgs, fallbackAddress)); + Become(() => Connecting(Settings.FinishConnectRetries, _connectArgs, fallbackAddress)); }); } - private Receive Connecting(int remainingFinishConnectRetries, SocketAsyncEventArgs args, IPEndPoint fallbackAddress) + private void Connecting(int remainingFinishConnectRetries, SocketAsyncEventArgs args, + IPEndPoint fallbackAddress) { - return message => + Receive(_ => { - if (message is Tcp.SocketConnected) + if (args.SocketError == SocketError.Success) { - if (args.SocketError == SocketError.Success) - { - if (_connect.Timeout.HasValue) Context.SetReceiveTimeout(null); - Log.Debug("Connection established to [{0}]", _connect.RemoteAddress); + if (_connect.Timeout.HasValue) Context.SetReceiveTimeout(null); + Log.Debug("Connection established to [{0}]", _connect.RemoteAddress); - ReleaseConnectionSocketArgs(); + ReleaseConnectionSocketArgs(); - CompleteConnect(_commander, _connect.Options); - } - else switch (remainingFinishConnectRetries) + CompleteConnect(_commander, _connect.Options); + } + else + switch (remainingFinishConnectRetries) { // used only when we've resolved a DNS endpoint. case > 0 when fallbackAddress != null: @@ -227,7 +215,7 @@ private Receive Connecting(int remainingFinishConnectRetries, SocketAsyncEventAr if (!Socket.ConnectAsync(args)) self.Tell(IO.Tcp.SocketConnected.Instance); }); - Context.Become(Connecting(remainingFinishConnectRetries - 1, args, previousAddress)); + Become(() => Connecting(remainingFinishConnectRetries - 1, args, previousAddress)); break; } case > 0: @@ -238,33 +226,31 @@ private Receive Connecting(int remainingFinishConnectRetries, SocketAsyncEventAr if (!Socket.ConnectAsync(args)) self.Tell(IO.Tcp.SocketConnected.Instance); }); - Context.Become(Connecting(remainingFinishConnectRetries - 1, args, null)); + Become(() => Connecting(remainingFinishConnectRetries - 1, args, null)); break; } default: - Log.Debug("Could not establish connection because finishConnect never returned true (consider increasing akka.io.tcp.finish-connect-retries)"); + Log.Debug( + "Could not establish connection because finishConnect never returned true (consider increasing akka.io.tcp.finish-connect-retries)"); Stop(_finishConnectNeverReturnedTrueException); break; } - return true; - } - if (message is ReceiveTimeout) - { - if (_connect.Timeout.HasValue) Context.SetReceiveTimeout(null); // Clear the timeout - Log.Debug("Connect timeout expired, could not establish connection to [{0}]", _connect.RemoteAddress); - Stop(new ConnectException($"Connect timeout of {_connect.Timeout} expired")); - return true; - } - return false; - }; + }); + Receive(_ => + { + if (_connect.Timeout.HasValue) Context.SetReceiveTimeout(null); // Clear the timeout + Log.Debug("Connect timeout expired, could not establish connection to [{0}]", _connect.RemoteAddress); + Stop(new ConnectException($"Connect timeout of {_connect.Timeout} expired")); + }); } } [InternalApi] public class ConnectException : Exception { - public ConnectException(string message) + public ConnectException(string message) : base(message) - { } + { + } } -} +} \ No newline at end of file From d8dd04a85b2010862681a4a6262d1d10162d39b5 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 16 May 2025 16:14:43 -0500 Subject: [PATCH 28/60] fixed issues with `PeerClosed` --- src/core/Akka/IO/TcpConnection.cs | 39 +++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 5744701f3e1..5e496896672 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -235,6 +235,9 @@ protected override void PostStop() if (_closeEvent != null) { + if(Settings.TraceLogging) + Log.Debug("[TcpConnection] sending close event to {0}", _closeNotify); + foreach (var sub in _closeNotify) sub.Tell(_closeEvent); } @@ -278,6 +281,11 @@ protected void CompleteConnect(IActorRef commander, IEnumerable(reg => { _handler = reg.Handler; + if (_traceLogging) Log.Debug("[{0}] registered as connection handler", reg.Handler); Context.WatchWith(_handler, HandlerDied.Instance); Context.Unwatch(_commander); _state = _state with { Phase = Phase.Open, KeepOpenOnPeerClosed = reg.KeepOpenOnPeerClosed }; @@ -297,7 +306,6 @@ private void AwaitRegBehaviour() IssueReceive(); TrySend(); }); - Receive(Enqueue); Receive(c => { @@ -305,6 +313,13 @@ private void AwaitRegBehaviour() TryStop(); }); Receive(_ => Context.Stop(Self)); + Receive(_ => + { + // after sending `Register` user should watch this actor to make sure + // it didn't die because of the timeout + Log.Debug("Configured registration timeout of [{0}] expired, stopping", Settings.RegisterTimeout); + Context.Stop(Self); + }); } private void OpenBehaviour() @@ -356,10 +371,19 @@ private void HandleReceiveCompleted(SocketAsyncEventArgs ea) _state = _state with { IsReceiving = false }; if (ea is { SocketError: SocketError.Success, BytesTransferred: > 0 }) { + if (Settings.TraceLogging) + { + Log.Debug("[TcpConnection] received {0} bytes", ea.BytesTransferred); + } + _handler!.Tell(new Received(ByteString.CopyFrom(_receiveBuffer, 0, ea.BytesTransferred))); IssueReceive(); return; } + + // unless we've been told otherwise, we want to close down the connection + if (!_state.KeepOpenOnPeerClosed) + _closeRequested = true; // FIN or error MarkClose(Self, PeerClosed.Instance); @@ -374,9 +398,15 @@ private void HandleSendCompleted(AckSocketAsyncEventArgs ea) foreach (var (c, ack) in ea.PendingAcks) c.Tell(ack); + + if (Settings.TraceLogging) + { + Log.Debug("[TcpConnection] completed write of {0} bytes (queued={1}/{2})", ea.BufferList.Sum(c => c.Count), _state.QueuedBytes, _maxQueuedBytes); + } + ea.ClearAcks(); ea.BufferList = null; // release refs - + TrySend(); TryStop(); } @@ -449,6 +479,9 @@ private void HalfCloseWriteSide() if (_state.ClosedForWrites) return; try { + if(Settings.TraceLogging) + Log.Debug("[TcpConnection] half‑closing write side"); + Socket.Shutdown(SocketShutdown.Send); } catch @@ -463,6 +496,8 @@ private void TryStop() { if (_state.Closeable(_closeRequested)) { + Log.Debug("[TcpConnection] stopping connection actor"); + // graceful stop Self.Tell(PoisonPill.Instance); } From eb16a658af5816bdc3df411b2b7e2e6b81a260b8 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 16 May 2025 16:26:30 -0500 Subject: [PATCH 29/60] cleanup outgoing connection error handling --- src/core/Akka/IO/TcpConnection.cs | 25 +++++++++++++++++++++-- src/core/Akka/IO/TcpOutgoingConnection.cs | 4 +++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 5e496896672..910f1cab24e 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -236,7 +236,7 @@ protected override void PostStop() if (_closeEvent != null) { if(Settings.TraceLogging) - Log.Debug("[TcpConnection] sending close event to {0}", _closeNotify); + Log.Debug("[TcpConnection] sending close event [{0}] to {1}", _closeEvent, string.Join(",", _closeNotify)); foreach (var sub in _closeNotify) sub.Tell(_closeEvent); @@ -290,6 +290,21 @@ protected void MarkClose(IActorRef src, Event evt) _closeNotify.Add(src); if (_handler != null) _closeNotify.Add(_handler); } + + private void Abort() + { + try + { + Socket.LingerState = new LingerOption(true, 0); // causes the following close() to send TCP RST + } + catch (Exception e) + { + if (_traceLogging) Log.Debug("setSoLinger(true, 0) failed with [{0}]", e); + } + + Context.Stop(Self); + } + private void AwaitRegBehaviour() @@ -332,6 +347,8 @@ private void OpenBehaviour() Receive(_ => { _closeRequested = true; + _state = _state with { ReadingSuspended = true }; + Socket.Shutdown(SocketShutdown.Receive); TryStop(); }); Receive(_ => @@ -340,7 +357,11 @@ private void OpenBehaviour() _closeRequested = true; TryStop(); }); - Receive(_ => { Context.Stop(Self); }); + Receive(s => + { + MarkClose(Sender, s.Event); + Abort(); + }); Receive(_ => { diff --git a/src/core/Akka/IO/TcpOutgoingConnection.cs b/src/core/Akka/IO/TcpOutgoingConnection.cs index 9e36c2238c8..4c9e78d7b77 100644 --- a/src/core/Akka/IO/TcpOutgoingConnection.cs +++ b/src/core/Akka/IO/TcpOutgoingConnection.cs @@ -78,7 +78,9 @@ private void Stop(Exception cause) { ReleaseConnectionSocketArgs(); - MarkClose(_commander, _connect.FailureMessage.WithCause(cause)); + var failureEvent = _connect.FailureMessage.WithCause(cause); + MarkClose(_commander, failureEvent); + Context.Stop(Self); } [MethodImpl(MethodImplOptions.AggressiveInlining)] From ab65ef740f883c5381f7d9bd136d6e505d124a01 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 16 May 2025 16:47:36 -0500 Subject: [PATCH 30/60] stash --- src/core/Akka/IO/TcpConnection.cs | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 910f1cab24e..923aee4121e 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -99,13 +99,16 @@ private readonly record struct ConnState( public bool HasPending => IsSending || Queue.Count > 0; public bool CanSend => !ClosedForWrites && !WritingSuspended; public bool CanReceive => !PeerClosed && !ReadingSuspended; + + private bool PeerIsReadyForUsToShutdown => (KeepOpenOnPeerClosed && PeerClosed) || + !KeepOpenOnPeerClosed; // may stop only when *we* requested it OR peer closed and we are NOT asked to stay open public bool Closeable(bool closeRequested) => closeRequested && !IsReceiving && !HasPending && - (!PeerClosed || !KeepOpenOnPeerClosed); + PeerIsReadyForUsToShutdown; public static ConnState Initial(Queue<(WriteCommand Cmd, IActorRef Sender)> queue) => new(Phase.Connecting, false, false, false, false, false, false, false, queue, 0); @@ -346,19 +349,34 @@ private void OpenBehaviour() Receive(_ => { + if (Settings.TraceLogging) + Log.Debug("[TcpConnection] Close requested"); _closeRequested = true; _state = _state with { ReadingSuspended = true }; - Socket.Shutdown(SocketShutdown.Receive); + try + { + Socket.Shutdown(SocketShutdown.Receive); + } + catch(Exception e) + { + if (Settings.TraceLogging) + Log.Debug("[TcpConnection] Socket receive shutdown failed: {0}", e.Message); + } TryStop(); + TrySend(); }); Receive(_ => { + if (Settings.TraceLogging) + Log.Debug("[TcpConnection] ConfirmedClose requested"); HalfCloseWriteSide(); _closeRequested = true; TryStop(); }); Receive(s => { + if (Settings.TraceLogging) + Log.Debug("[TcpConnection] Abort requested"); MarkClose(Sender, s.Event); Abort(); }); From d8e580de81f9697baaafbd7c3cf0a53264b607ce Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 16 May 2025 17:22:55 -0500 Subject: [PATCH 31/60] fixed issue with `Close` commands --- src/core/Akka/IO/TcpConnection.cs | 118 ++++++++++++++++-------------- 1 file changed, 65 insertions(+), 53 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 923aee4121e..b1830f882e1 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -84,34 +84,38 @@ private enum Phase Closed } + /// + /// Immutable flags – reference to the live Queue + byte counter **and any deferred half‑close**. + /// Moving every transient flag in here lets us reason over shutdown with a single value. + /// private readonly record struct ConnState( Phase Phase, - bool IsReceiving, - bool IsSending, - bool PeerClosed, - bool ClosedForWrites, - bool ReadingSuspended, - bool WritingSuspended, - bool KeepOpenOnPeerClosed, - Queue<(WriteCommand Cmd, IActorRef Sender)> Queue, - int QueuedBytes) + bool IsReceiving, + bool IsSending, + bool PeerClosed, + bool ClosedForWrites, + bool ReadingSuspended, + bool WritingSuspended, + bool KeepOpenOnPeerClosed, + bool PendingHalfClose, // ≙ we have seen ConfirmedClose, waiting for queue to drain + Queue<(WriteCommand Cmd, IActorRef Snd)> Queue, + int QueuedBytes) { - public bool HasPending => IsSending || Queue.Count > 0; - public bool CanSend => !ClosedForWrites && !WritingSuspended; - public bool CanReceive => !PeerClosed && !ReadingSuspended; - - private bool PeerIsReadyForUsToShutdown => (KeepOpenOnPeerClosed && PeerClosed) || - !KeepOpenOnPeerClosed; + public bool HasPending => IsSending || Queue.Count != 0; + public bool CanSend => !ClosedForWrites && !WritingSuspended; + public bool CanReceive => !PeerClosed && !ReadingSuspended; + + private bool PeerIsReadyForUsToShutdown => !PeerClosed || !KeepOpenOnPeerClosed; - // may stop only when *we* requested it OR peer closed and we are NOT asked to stay open public bool Closeable(bool closeRequested) => + (closeRequested && Phase < Phase.Open) || // IMMEDIATE close if requested during connect or reg closeRequested && - !IsReceiving && - !HasPending && + !IsReceiving && + !HasPending && PeerIsReadyForUsToShutdown; - public static ConnState Initial(Queue<(WriteCommand Cmd, IActorRef Sender)> queue) => - new(Phase.Connecting, false, false, false, false, false, false, false, queue, 0); + public static ConnState Initial(Queue<(WriteCommand Cmd, IActorRef Snd)> q) => + new(Phase.Connecting, false, false, false, false, false, false, false, false, q, 0); } #region Ack‑aware SAEA @@ -293,22 +297,6 @@ protected void MarkClose(IActorRef src, Event evt) _closeNotify.Add(src); if (_handler != null) _closeNotify.Add(_handler); } - - private void Abort() - { - try - { - Socket.LingerState = new LingerOption(true, 0); // causes the following close() to send TCP RST - } - catch (Exception e) - { - if (_traceLogging) Log.Debug("setSoLinger(true, 0) failed with [{0}]", e); - } - - Context.Stop(Self); - } - - private void AwaitRegBehaviour() { @@ -328,7 +316,7 @@ private void AwaitRegBehaviour() Receive(c => { _closeRequested = true; - TryStop(); + EvaluateShutdown(); }); Receive(_ => Context.Stop(Self)); Receive(_ => @@ -347,12 +335,13 @@ private void OpenBehaviour() Receive(Enqueue); - Receive(_ => + Receive(c => { if (Settings.TraceLogging) Log.Debug("[TcpConnection] Close requested"); _closeRequested = true; - _state = _state with { ReadingSuspended = true }; + _state = _state with { ReadingSuspended = true, IsReceiving = false }; + MarkClose(Sender, c.Event); try { Socket.Shutdown(SocketShutdown.Receive); @@ -362,23 +351,24 @@ private void OpenBehaviour() if (Settings.TraceLogging) Log.Debug("[TcpConnection] Socket receive shutdown failed: {0}", e.Message); } - TryStop(); + EvaluateShutdown(); TrySend(); }); - Receive(_ => + Receive(cc => { if (Settings.TraceLogging) Log.Debug("[TcpConnection] ConfirmedClose requested"); + MarkClose(Sender, cc.Event); HalfCloseWriteSide(); _closeRequested = true; - TryStop(); + EvaluateShutdown(); }); Receive(s => { if (Settings.TraceLogging) - Log.Debug("[TcpConnection] Abort requested"); + Log.Debug("[TcpConnection] AbortSocket requested"); MarkClose(Sender, s.Event); - Abort(); + AbortSocket(); }); Receive(_ => @@ -428,7 +418,7 @@ private void HandleReceiveCompleted(SocketAsyncEventArgs ea) MarkClose(Self, PeerClosed.Instance); _handler!.Tell(PeerClosed.Instance); _state = _state with { PeerClosed = true }; - TryStop(); + EvaluateShutdown(); } private void HandleSendCompleted(AckSocketAsyncEventArgs ea) @@ -446,8 +436,14 @@ private void HandleSendCompleted(AckSocketAsyncEventArgs ea) ea.ClearAcks(); ea.BufferList = null; // release refs + /* check deferred FIN */ + if(_state.PendingHalfClose && _pendingWrites.Count==0) + { + HalfCloseWriteSide(); + } + TrySend(); - TryStop(); + EvaluateShutdown(); } /* ----------------------------------------------------------------- */ @@ -528,18 +524,34 @@ private void HalfCloseWriteSide() /* ignore */ } - _state = _state with { ClosedForWrites = true, Phase = Phase.HalfOpen }; + _state = _state with { ClosedForWrites = true, Phase = Phase.HalfOpen, PendingHalfClose = false}; } - private void TryStop() + /* ====================================================================*/ + /* Shutdown decision */ + /* ====================================================================*/ + private void EvaluateShutdown() { - if (_state.Closeable(_closeRequested)) + if(!_state.Closeable(_closeRequested)) return; + if(Settings.TraceLogging) + Log.Debug("[TcpConnection] shutting down connection"); + + Self.Tell(PoisonPill.Instance); + } + + private void AbortSocket() + { + try { - Log.Debug("[TcpConnection] stopping connection actor"); - - // graceful stop - Self.Tell(PoisonPill.Instance); + Socket.LingerState = new LingerOption(true, 0); // causes the following close() to send TCP RST } + catch (Exception e) + { + if (_traceLogging) Log.Debug("setSoLinger(true, 0) failed with [{0}]", e); + } + + Context.Stop(Self); } + } } \ No newline at end of file From 87a017571c26f8f55094668de748a1afc70a99e2 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 16 May 2025 17:36:21 -0500 Subject: [PATCH 32/60] stash --- src/core/Akka/IO/TcpConnection.cs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index b1830f882e1..a548021d85e 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -425,13 +425,22 @@ private void HandleSendCompleted(AckSocketAsyncEventArgs ea) { _state = _state with { IsSending = false }; - foreach (var (c, ack) in ea.PendingAcks) - c.Tell(ack); + if (ea.SocketError != SocketError.Success) + { + Log.Warning("[TcpConnection] send failed with error [{0}]", ea.SocketError); + MarkClose(_handler!, new ErrorClosed(ea.SocketError.ToString())); + Context.Stop(Self); + } if (Settings.TraceLogging) { - Log.Debug("[TcpConnection] completed write of {0} bytes (queued={1}/{2})", ea.BufferList.Sum(c => c.Count), _state.QueuedBytes, _maxQueuedBytes); + Log.Debug("[TcpConnection] completed write of {0}/{1} bytes (queued={2}/{3})", ea.BytesTransferred, ea.BufferList.Sum(c => c.Count), _state.QueuedBytes, _maxQueuedBytes); } + + foreach (var (c, ack) in ea.PendingAcks) + c.Tell(ack); + + ea.ClearAcks(); ea.BufferList = null; // release refs From 79cf97e932a2be073579f4ad5fb6412dbc746b3f Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 16 May 2025 19:15:35 -0500 Subject: [PATCH 33/60] give Akka.IO.Tcp actors clearer role-based names in logs --- src/core/Akka.Streams.Tests/IO/TcpSpec.cs | 2 +- src/core/Akka/IO/TcpConnection.cs | 2 +- src/core/Akka/IO/TcpManager.cs | 20 ++++++++++++++++++-- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/core/Akka.Streams.Tests/IO/TcpSpec.cs b/src/core/Akka.Streams.Tests/IO/TcpSpec.cs index ca7489c46fb..9b7a3654b9e 100644 --- a/src/core/Akka.Streams.Tests/IO/TcpSpec.cs +++ b/src/core/Akka.Streams.Tests/IO/TcpSpec.cs @@ -469,7 +469,7 @@ public async Task Outgoing_TCP_stream_must_Echo_should_work_even_if_server_is_in .Run(Materializer); var result = await Source.From(Enumerable.Repeat(0, 1000) - .Select(i => ByteString.FromBytes(new byte[] { Convert.ToByte(i) }))) + .Select(i => ByteString.FromBytes([Convert.ToByte(i)]))) .Via(Sys.TcpStream().OutgoingConnection(serverAddress, halfClose: true)) .RunAggregate(0, (i, s) => i + s.Count, Materializer).ShouldCompleteWithin(10.Seconds()); diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index a548021d85e..0b555bea58a 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -385,7 +385,7 @@ private void OpenBehaviour() Receive(h => { - Log.Debug("Handler died, stopping connection actor"); + Log.Debug("Handler [{0}] died, stopping connection actor", _handler); Context.Stop(Self); }); //Receive(_=> { _st = _st with { WritingSuspended=true }; }); diff --git a/src/core/Akka/IO/TcpManager.cs b/src/core/Akka/IO/TcpManager.cs index 2c1189008ff..cfd369c34a6 100644 --- a/src/core/Akka/IO/TcpManager.cs +++ b/src/core/Akka/IO/TcpManager.cs @@ -54,6 +54,22 @@ internal sealed class TcpManager : ActorBase { private readonly TcpExt _tcp; + const string TcpListenerNamePrefix = "tcp-listener"; + const string TcpOutgoingConnectionNamePrefix = "tcp-client-connection"; + + private long _tcpListenerCounter; + private long _tcpOutgoingConnectionCounter; + + private string NextTcpListenerName() + { + return $"{TcpListenerNamePrefix}-{_tcpListenerCounter++}"; + } + + private string NextTcpOutgoingConnectionName() + { + return $"{TcpOutgoingConnectionNamePrefix}-{_tcpOutgoingConnectionCounter++}"; + } + public TcpManager(TcpExt tcp) { _tcp = tcp; @@ -66,13 +82,13 @@ protected override bool Receive(object message) case Connect c: { var commander = Sender; - Context.ActorOf(Props.Create(_tcp, commander, c).WithDeploy(Deploy.Local)); + Context.ActorOf(Props.Create(_tcp, commander, c).WithDeploy(Deploy.Local), NextTcpOutgoingConnectionName()); return true; } case Bind b: { var commander = Sender; - Context.ActorOf(Props.Create(_tcp, commander, b).WithDeploy(Deploy.Local)); + Context.ActorOf(Props.Create(_tcp, commander, b).WithDeploy(Deploy.Local), NextTcpListenerName()); return true; } default: From db0eaf00d2a5ae75b395cd710f38c2f7206bb895 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 16 May 2025 21:07:48 -0500 Subject: [PATCH 34/60] reintroduced pull mode --- src/core/Akka/IO/TcpConnection.cs | 23 +++++++++++++++++++++-- src/core/Akka/IO/TcpIncomingConnection.cs | 2 +- src/core/Akka/IO/TcpOutgoingConnection.cs | 2 +- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 0b555bea58a..7b798c195a9 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -168,6 +168,9 @@ private HandlerDied() private ConnState _state; private readonly bool _traceLogging; + + // used by Akka.Streams + private readonly bool _pullMode; private IActorRef? _commander; private IActorRef? _handler; @@ -181,11 +184,12 @@ private HandlerDied() private static readonly IOException DroppingWriteBecauseQueueIsFullException = new("Dropping write because queue is full"); - protected TcpConnection(TcpSettings settings, Socket socket) + protected TcpConnection(TcpSettings settings, Socket socket, bool pullMode) { Settings = settings; _maxQueuedBytes = settings.WriteCommandsQueueMaxSize; // –1 ⇒ unlimited; _pendingWrites = new Queue<(WriteCommand Cmd, IActorRef Sender)>(16); + _pullMode = pullMode; _traceLogging = Settings.TraceLogging; _state = ConnState.Initial(_pendingWrites); @@ -194,6 +198,12 @@ protected TcpConnection(TcpSettings settings, Socket socket) _receiveArgs = new ReadSocketAsyncEventArgs(); _sendArgs = new AckSocketAsyncEventArgs(); InitSocketEventArgs(); + + if (_pullMode) + { + // have to wait for the first pull request to start reading + _state = _state with { ReadingSuspended = true }; + } } private void InitSocketEventArgs() @@ -406,7 +416,16 @@ private void HandleReceiveCompleted(SocketAsyncEventArgs ea) } _handler!.Tell(new Received(ByteString.CopyFrom(_receiveBuffer, 0, ea.BytesTransferred))); - IssueReceive(); + + if (_pullMode) + { + // in pull mode we need to wait for the next pull request + _state = _state with { ReadingSuspended = true }; + } + else + { + IssueReceive(); + } return; } diff --git a/src/core/Akka/IO/TcpIncomingConnection.cs b/src/core/Akka/IO/TcpIncomingConnection.cs index 26a618b1371..46efdc4f011 100644 --- a/src/core/Akka/IO/TcpIncomingConnection.cs +++ b/src/core/Akka/IO/TcpIncomingConnection.cs @@ -26,7 +26,7 @@ public TcpIncomingConnection(TcpSettings settings, IActorRef bindHandler, IEnumerable options, bool readThrottling) - : base(settings, socket) + : base(settings, socket, readThrottling) { _bindHandler = bindHandler; _options = options; diff --git a/src/core/Akka/IO/TcpOutgoingConnection.cs b/src/core/Akka/IO/TcpOutgoingConnection.cs index 4c9e78d7b77..d8d5680f5cf 100644 --- a/src/core/Akka/IO/TcpOutgoingConnection.cs +++ b/src/core/Akka/IO/TcpOutgoingConnection.cs @@ -35,7 +35,7 @@ public TcpOutgoingConnection(TcpExt tcp, IActorRef commander, Tcp.Connect connec (connect.TcpSettings ?? tcp.Settings), (connect.TcpSettings ?? tcp.Settings).OutgoingSocketForceIpv4 ? new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp) { Blocking = false } - : new Socket(SocketType.Stream, ProtocolType.Tcp) { Blocking = false }) + : new Socket(SocketType.Stream, ProtocolType.Tcp) { Blocking = false }, connect.PullMode) { _commander = commander; _connect = connect; From 6589ae2fec55486d39c739578157211317fc8882 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 16 May 2025 21:22:33 -0500 Subject: [PATCH 35/60] fixed issue with missing messages during `Tcp.Close` with writes pending --- src/core/Akka/IO/TcpConnection.cs | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 7b798c195a9..fccd0811e20 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -352,17 +352,8 @@ private void OpenBehaviour() _closeRequested = true; _state = _state with { ReadingSuspended = true, IsReceiving = false }; MarkClose(Sender, c.Event); - try - { - Socket.Shutdown(SocketShutdown.Receive); - } - catch(Exception e) - { - if (Settings.TraceLogging) - Log.Debug("[TcpConnection] Socket receive shutdown failed: {0}", e.Message); - } - EvaluateShutdown(); TrySend(); + EvaluateShutdown(); }); Receive(cc => { @@ -405,6 +396,9 @@ private void OpenBehaviour() /* Socket‑event handlers */ /* ----------------------------------------------------------------- */ + private long _totalSentBytes; + private long _totalReceivedBytes; + private void HandleReceiveCompleted(SocketAsyncEventArgs ea) { _state = _state with { IsReceiving = false }; @@ -412,7 +406,8 @@ private void HandleReceiveCompleted(SocketAsyncEventArgs ea) { if (Settings.TraceLogging) { - Log.Debug("[TcpConnection] received {0} bytes", ea.BytesTransferred); + _totalReceivedBytes += ea.BytesTransferred; + Log.Debug("[TcpConnection] received {0} bytes [{1} total]", ea.BytesTransferred, _totalReceivedBytes); } _handler!.Tell(new Received(ByteString.CopyFrom(_receiveBuffer, 0, ea.BytesTransferred))); @@ -453,7 +448,8 @@ private void HandleSendCompleted(AckSocketAsyncEventArgs ea) if (Settings.TraceLogging) { - Log.Debug("[TcpConnection] completed write of {0}/{1} bytes (queued={2}/{3})", ea.BytesTransferred, ea.BufferList.Sum(c => c.Count), _state.QueuedBytes, _maxQueuedBytes); + _totalSentBytes += ea.BytesTransferred; + Log.Debug("[TcpConnection] completed write of {0}/{1} bytes (queued={2}/{3}) [{4} total sent]", ea.BytesTransferred, ea.BufferList.Sum(c => c.Count), _state.QueuedBytes, _maxQueuedBytes, _totalSentBytes); } foreach (var (c, ack) in ea.PendingAcks) From 5ad22b48836fdbdf04d11ac2542621b4cae27366 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 16 May 2025 21:32:51 -0500 Subject: [PATCH 36/60] fixed logic around flushing all writes --- src/core/Akka/IO/TcpConnection.cs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index fccd0811e20..049c6268a37 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -105,7 +105,8 @@ private readonly record struct ConnState( public bool CanSend => !ClosedForWrites && !WritingSuspended; public bool CanReceive => !PeerClosed && !ReadingSuspended; - private bool PeerIsReadyForUsToShutdown => !PeerClosed || !KeepOpenOnPeerClosed; + private bool PeerIsReadyForUsToShutdown => (KeepOpenOnPeerClosed && !HasPending && PeerClosed && CanSend) || + (!KeepOpenOnPeerClosed && PeerClosed); public bool Closeable(bool closeRequested) => (closeRequested && Phase < Phase.Open) || // IMMEDIATE close if requested during connect or reg @@ -556,9 +557,19 @@ private void HalfCloseWriteSide() /* ====================================================================*/ private void EvaluateShutdown() { - if(!_state.Closeable(_closeRequested)) return; - if(Settings.TraceLogging) - Log.Debug("[TcpConnection] shutting down connection"); + if (_closeRequested) + { + var canClose = _state.Closeable(_closeRequested); + if (!canClose) + { + if (Settings.TraceLogging) + Log.Debug("[TcpConnection] can't close yet - state is [{0}]", _state); + } + } + + if (!_state.Closeable(_closeRequested)) return; + if (Settings.TraceLogging) + Log.Debug("[TcpConnection] shutting down connection [{0}]", _state); Self.Tell(PoisonPill.Instance); } From 6f1668aa7db0b77779e229ee9cd8d1d50d2cdddd Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 16 May 2025 21:44:23 -0500 Subject: [PATCH 37/60] fixed issues with detecting half-closed connections --- src/core/Akka/IO/TcpConnection.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 049c6268a37..d2ebc736c80 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // // Copyright (C) 2009-2022 Lightbend Inc. // Copyright (C) 2013-2025 .NET Foundation @@ -113,7 +113,13 @@ public bool Closeable(bool closeRequested) => closeRequested && !IsReceiving && !HasPending && - PeerIsReadyForUsToShutdown; + ( + // If we're in HalfOpen, both sides have closed their write sides, and nothing is left to do + (Phase == Phase.HalfOpen && ClosedForWrites && PeerClosed) + || + // Fallback to previous logic for other phases + PeerIsReadyForUsToShutdown + ); public static ConnState Initial(Queue<(WriteCommand Cmd, IActorRef Snd)> q) => new(Phase.Connecting, false, false, false, false, false, false, false, false, q, 0); From edbca26c01bfa8c56a858651a89403de8cb40a88 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 16 May 2025 21:58:19 -0500 Subject: [PATCH 38/60] fixed `Outgoing_TCP_stream_must_handle_when_connection_actor_terminates_unexpectedly` broke due to us trying to rename outbound TCP connection actors --- src/core/Akka.Streams.Tests/IO/TcpSpec.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/Akka.Streams.Tests/IO/TcpSpec.cs b/src/core/Akka.Streams.Tests/IO/TcpSpec.cs index 9b7a3654b9e..1ec954d63a0 100644 --- a/src/core/Akka.Streams.Tests/IO/TcpSpec.cs +++ b/src/core/Akka.Streams.Tests/IO/TcpSpec.cs @@ -506,7 +506,7 @@ await WithinAsync(TimeSpan.FromSeconds(15), async () => await AwaitAssertAsync(async () => { // Getting rid of existing connection actors by using a blunt instrument - system2.ActorSelection(system2.Tcp().Path / "$a" / "*").Tell(Kill.Instance); + system2.ActorSelection(system2.Tcp().Path / "tcp-client-connection-*").Tell(Kill.Instance); await result.ShouldCompleteWithin(3.Seconds()); }, interval:TimeSpan.FromSeconds(4)); From b7aa8dd6834033f088894b2e550951da80767942 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 16 May 2025 22:13:01 -0500 Subject: [PATCH 39/60] fixed issues with "pending half closures" --- src/core/Akka/IO/TcpConnection.cs | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index d2ebc736c80..fecf5d577e9 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -97,7 +97,12 @@ private readonly record struct ConnState( bool ReadingSuspended, bool WritingSuspended, bool KeepOpenOnPeerClosed, - bool PendingHalfClose, // ≙ we have seen ConfirmedClose, waiting for queue to drain + /// + /// Indicates that a half-close (shutdown of the write side) has been requested (via ConfirmedClose or Close), + /// but there are still pending writes in the queue. When all writes have been delivered, the write side will + /// be closed (Socket.Shutdown(SocketShutdown.Send)), and this flag will be reset to false. + /// + bool PendingHalfClose, Queue<(WriteCommand Cmd, IActorRef Snd)> Queue, int QueuedBytes) { @@ -360,6 +365,14 @@ private void OpenBehaviour() _state = _state with { ReadingSuspended = true, IsReceiving = false }; MarkClose(Sender, c.Event); TrySend(); + if (_state.HasPending) + { + _state = _state with { PendingHalfClose = true }; + } + else + { + HalfCloseWriteSide(); + } EvaluateShutdown(); }); Receive(cc => @@ -367,8 +380,15 @@ private void OpenBehaviour() if (Settings.TraceLogging) Log.Debug("[TcpConnection] ConfirmedClose requested"); MarkClose(Sender, cc.Event); - HalfCloseWriteSide(); _closeRequested = true; + if (_state.HasPending) + { + _state = _state with { PendingHalfClose = true }; + } + else + { + HalfCloseWriteSide(); + } EvaluateShutdown(); }); Receive(s => @@ -471,6 +491,7 @@ private void HandleSendCompleted(AckSocketAsyncEventArgs ea) if(_state.PendingHalfClose && _pendingWrites.Count==0) { HalfCloseWriteSide(); + _state = _state with { PendingHalfClose = false }; } TrySend(); From cd7814b80a155fc822f8befb7c064dd2bb52b9b1 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 16 May 2025 22:18:58 -0500 Subject: [PATCH 40/60] fixed issue with `_closeEvent` being overwritten --- src/core/Akka/IO/TcpConnection.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index fecf5d577e9..ddfbf0e4b49 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -315,7 +315,8 @@ protected void MarkClose(IActorRef src, Event evt) Log.Debug("[TcpConnection] working on connection closure: {0}", evt); } - _closeEvent = evt; + if (_closeEvent == null) + _closeEvent = evt; _closeNotify.Add(src); if (_handler != null) _closeNotify.Add(_handler); } From b601351757a1c94067eeef358131b6819e148df0 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 16 May 2025 22:30:51 -0500 Subject: [PATCH 41/60] cleaned up XML-DOC --- src/core/Akka/IO/TcpConnection.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index ddfbf0e4b49..d9ba205504c 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -88,6 +88,11 @@ private enum Phase /// Immutable flags – reference to the live Queue + byte counter **and any deferred half‑close**. /// Moving every transient flag in here lets us reason over shutdown with a single value. /// + /// + /// Indicates that a half-close (shutdown of the write side) has been requested (via ConfirmedClose or Close), + /// but there are still pending writes in the queue. When all writes have been delivered, the write side will + /// be closed (Socket.Shutdown(SocketShutdown.Send)), and this flag will be reset to false. + /// private readonly record struct ConnState( Phase Phase, bool IsReceiving, @@ -97,11 +102,6 @@ private readonly record struct ConnState( bool ReadingSuspended, bool WritingSuspended, bool KeepOpenOnPeerClosed, - /// - /// Indicates that a half-close (shutdown of the write side) has been requested (via ConfirmedClose or Close), - /// but there are still pending writes in the queue. When all writes have been delivered, the write side will - /// be closed (Socket.Shutdown(SocketShutdown.Send)), and this flag will be reset to false. - /// bool PendingHalfClose, Queue<(WriteCommand Cmd, IActorRef Snd)> Queue, int QueuedBytes) From af4973ed0a3e4f4c5c4fb7a7409bdeeff749c503 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 17 May 2025 10:21:05 -0500 Subject: [PATCH 42/60] simplified `TcpConnection` state and behavior --- src/core/Akka.Tests/IO/TcpIntegrationSpec.cs | 62 ++- src/core/Akka/IO/TcpConnection.cs | 433 +++++++++++-------- src/core/Akka/IO/TcpOutgoingConnection.cs | 3 +- 3 files changed, 289 insertions(+), 209 deletions(-) diff --git a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs index 2a2e195ccc5..fdc1e624a0e 100644 --- a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs +++ b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs @@ -114,9 +114,8 @@ public async Task The_TCP_transport_implementation_should_properly_handle_connec var actors = await x.EstablishNewClientConnectionAsync(); actors.ClientHandler.Send(actors.ClientConnection, PoisonPill.Instance); await VerifyActorTermination(actors.ClientConnection); - - // PeerClosed because the PostStop of the client connection actor will have it send a graceful termination - await actors.ServerHandler.ExpectMsgAsync(); + + await actors.ServerHandler.ExpectMsgAsync(); await VerifyActorTermination(actors.ServerConnection); }); } @@ -132,7 +131,7 @@ public async Task The_TCP_transport_implementation_should_properly_handle_connec actors.ClientHandler.Send(actors.ClientConnection, PoisonPill.Instance); await VerifyActorTermination(actors.ClientConnection); - await actors.ServerHandler.ExpectMsgAsync(); + await actors.ServerHandler.ExpectMsgAsync(); await VerifyActorTermination(actors.ServerConnection); }); } @@ -298,30 +297,27 @@ await EventFilter.Debug(new Regex("Received Write command before Register[^3]+3 } [Fact] - public async Task Write_before_Register_should_Be_dropped_if_WriteQueue_is_full() + public async Task Write_before_Register_should_Be_dropped_if_buffer_is_full() { - var smallBufferSettings = TcpSettings.Create(Sys) with { WriteCommandsQueueMaxSize = 1 }; - - await new TestSetup(this, settings:smallBufferSettings).RunAsync(async x => + await new TestSetup(this).RunAsync(async x => { var actors = await x.EstablishNewClientConnectionAsync(registerClientHandler: false); - - var happyMessage = ByteString.FromString("msg"); // 3 bytes + var overflowData = ByteString.FromBytes(new byte[InternalConnectionActorMaxQueueSize + 1]); // We do not want message about receiving Write to be logged, if the write was actually discarded await EventFilter.Warning(new Regex("Received Write command before Register[^3]+3 bytes")).ExpectAsync(0, () => { - actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(happyMessage)); actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(overflowData)); return Task.CompletedTask; }); - await actors.ClientHandler.ExpectMsgAsync(); + await actors.ClientHandler.ExpectMsgAsync(TimeSpan.FromSeconds(10)); // After failed receive, next "good" writes should be handled with no issues + actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(ByteString.FromBytes(new byte[1]))); actors.ClientHandler.Send(actors.ClientConnection, new Tcp.Register(actors.ClientHandler)); var serverMsgs = await actors.ServerHandler.ReceiveWhileAsync(o => o as Tcp.Received, RemainingOrDefault, TimeSpan.FromSeconds(2)).ToListAsync(); - serverMsgs.Should().HaveCount(1).And.Subject.Should().Contain(m => m.Data.Count == 3); + serverMsgs.Should().HaveCount(1).And.Subject.Should().Contain(m => m.Data.Count == 1); }); } @@ -405,42 +401,36 @@ public async Task When_multiple_writing_clients_Should_receive_messages_in_order }); } - [Fact] + [Fact(Skip = "Have to re-implement pagination")] public async Task Should_fail_writing_when_buffer_is_filled() { - // Set the write queue max size to 1 message - var smallBufferSettings = TcpSettings.Create(Sys) with { WriteCommandsQueueMaxSize = 1 }; - - await new TestSetup(this, settings: smallBufferSettings).RunAsync(async x => + await new TestSetup(this).RunAsync(async x => { var actors = await x.EstablishNewClientConnectionAsync(); - // Small test messages - var testData = ByteString.FromString("test message"); + // create a buffer-overflow message + var overflowData = ByteString.FromBytes(new byte[InternalConnectionActorMaxQueueSize + 1]); + var goodData = ByteString.FromBytes(new byte[InternalConnectionActorMaxQueueSize]); // If test runner is too loaded, let it try ~3 times with 5 pause interval await AwaitAssertAsync(async () => { - // Send first message - should be sent immediately - actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(testData)); - - // Send second message - should be buffered and fail since queue size is 1 - actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(testData)); - - // Third message - should fail immediately since queue is full - actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(testData)); + // try sending overflow + actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(overflowData)); // this is sent immidiately + actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(overflowData)); // this will try to buffer await actors.ClientHandler.ExpectMsgAsync(TimeSpan.FromSeconds(20)); - // First message should be received - var received = await actors.ServerHandler.ReceiveWhileAsync(TimeSpan.FromSeconds(1), m => m as Tcp.Received).ToListAsync(); - received.Count.Should().BeGreaterOrEqualTo(1); - received.Sum(m => m.Data.Count).Should().BeGreaterOrEqualTo(testData.Count); + // First overflow data will be received anyway + (await actors.ServerHandler.ReceiveWhileAsync(TimeSpan.FromSeconds(1), m => m as Tcp.Received).ToListAsync()) + .Sum(m => m.Data.Count) + .Should().Be(InternalConnectionActorMaxQueueSize + 1); - // Check we can resume writing after clearing the failure + // Check that almost-overflow size does not cause any problems actors.ClientHandler.Send(actors.ClientConnection, Tcp.ResumeWriting.Instance); // Recover after send failure - actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(testData)); - received = await actors.ServerHandler.ReceiveWhileAsync(TimeSpan.FromSeconds(1), m => m as Tcp.Received).ToListAsync(); - received.Count.Should().BeGreaterOrEqualTo(1); + actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(goodData)); + (await actors.ServerHandler.ReceiveWhileAsync(TimeSpan.FromSeconds(1), m => m as Tcp.Received).ToListAsync()) + .Sum(m => m.Data.Count) + .Should().Be(InternalConnectionActorMaxQueueSize); }, TimeSpan.FromSeconds(30 * 3), TimeSpan.FromSeconds(5)); // 3 attempts by ~25 seconds + 5 sec pause }); } diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index d9ba205504c..88b056b0567 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -95,39 +95,22 @@ private enum Phase /// private readonly record struct ConnState( Phase Phase, - bool IsReceiving, - bool IsSending, - bool PeerClosed, - bool ClosedForWrites, - bool ReadingSuspended, - bool WritingSuspended, - bool KeepOpenOnPeerClosed, - bool PendingHalfClose, + bool IsReceiving, + bool IsSending, + bool PeerClosed, + bool OutputShutdown, + bool ReadingSuspended, + bool WritingSuspended, + bool KeepOpenOnPeerClosed, Queue<(WriteCommand Cmd, IActorRef Snd)> Queue, - int QueuedBytes) + int QueuedBytes) { public bool HasPending => IsSending || Queue.Count != 0; - public bool CanSend => !ClosedForWrites && !WritingSuspended; - public bool CanReceive => !PeerClosed && !ReadingSuspended; - - private bool PeerIsReadyForUsToShutdown => (KeepOpenOnPeerClosed && !HasPending && PeerClosed && CanSend) || - (!KeepOpenOnPeerClosed && PeerClosed); - - public bool Closeable(bool closeRequested) => - (closeRequested && Phase < Phase.Open) || // IMMEDIATE close if requested during connect or reg - closeRequested && - !IsReceiving && - !HasPending && - ( - // If we're in HalfOpen, both sides have closed their write sides, and nothing is left to do - (Phase == Phase.HalfOpen && ClosedForWrites && PeerClosed) - || - // Fallback to previous logic for other phases - PeerIsReadyForUsToShutdown - ); + public bool CanSend => !OutputShutdown && !WritingSuspended; + public bool CanReceive => !PeerClosed && !ReadingSuspended; public static ConnState Initial(Queue<(WriteCommand Cmd, IActorRef Snd)> q) => - new(Phase.Connecting, false, false, false, false, false, false, false, false, q, 0); + new(Phase.Connecting, false, false, false, false, false, false, false, q, 0); } #region Ack‑aware SAEA @@ -170,22 +153,21 @@ private HandlerDied() private readonly Queue<(WriteCommand Cmd, IActorRef Sender)> _pendingWrites; private readonly byte[] _receiveBuffer; - private ReadSocketAsyncEventArgs _receiveArgs; - private AckSocketAsyncEventArgs _sendArgs; - - - private bool _closeRequested; + private readonly ReadSocketAsyncEventArgs _receiveArgs; + private readonly AckSocketAsyncEventArgs _sendArgs; + private readonly int _maxQueuedBytes; private ConnState _state; private readonly bool _traceLogging; - + // used by Akka.Streams private readonly bool _pullMode; private IActorRef? _commander; private IActorRef? _handler; + private CloseInformation? _closeInformation; private static readonly IOException DroppingWriteBecauseClosingException = new("Dropping write because the connection is closing"); @@ -220,12 +202,11 @@ protected TcpConnection(TcpSettings settings, Socket socket, bool pullMode) private void InitSocketEventArgs() { - _receiveArgs.SetBuffer(_receiveBuffer, 0, _receiveBuffer.Length); _receiveArgs.UserToken = Self; _receiveArgs.Completed += OnCompleted; - + _sendArgs.UserToken = Self; _sendArgs.Completed += OnCompleted; } @@ -242,14 +223,8 @@ private static void OnCompleted(object? sender, SocketAsyncEventArgs e) protected override void PostStop() { - try - { - Socket.Dispose(); - } - catch - { - /* ignore */ - } + if (Socket.Connected) AbortSocket(); + else CloseSocket(); _receiveArgs.Dispose(); _sendArgs.Dispose(); @@ -262,15 +237,22 @@ protected override void PostStop() snd.Tell(cmd.FailureMessage.WithCause(DroppingWriteBecauseClosingException)); } - if (_closeEvent != null) + if (_closeInformation != null) { - if(Settings.TraceLogging) - Log.Debug("[TcpConnection] sending close event [{0}] to {1}", _closeEvent, string.Join(",", _closeNotify)); - - foreach (var sub in _closeNotify) - sub.Tell(_closeEvent); + if (Settings.TraceLogging) + Log.Debug("[TcpConnection] sending close event [{0}] to {1}", _closeInformation.ClosedEvent, + string.Join(",", _closeInformation.NotificationsTo)); + + foreach (var sub in _closeInformation.NotificationsTo) + sub.Tell(_closeInformation.ClosedEvent); } } + + protected override void PostRestart(Exception reason) + { + // have to assert that we are not restarting + throw new IllegalStateException("Restarting not supported for connection actors."); + } /// /// Used in subclasses to start the common machinery above once a channel is connected @@ -305,22 +287,17 @@ protected void CompleteConnect(IActorRef commander, IEnumerable _closeNotify = []; - private Event? _closeEvent; - - protected void MarkClose(IActorRef src, Event evt) + protected void StopWith(CloseInformation closeInformation) { - if (Settings.TraceLogging) + if(_handler != null) { - Log.Debug("[TcpConnection] working on connection closure: {0}", evt); + closeInformation = closeInformation with { NotificationsTo = closeInformation.NotificationsTo.Add(_handler!) }; } - if (_closeEvent == null) - _closeEvent = evt; - _closeNotify.Add(src); - if (_handler != null) _closeNotify.Add(_handler); + _closeInformation = closeInformation; + Context.Stop(Self); } - + private void AwaitRegBehaviour() { Receive(reg => @@ -336,10 +313,11 @@ private void AwaitRegBehaviour() TrySend(); }); Receive(Enqueue); - Receive(c => + Receive(c => HandleClose(Sender, c.Event)); + Receive(_ => { _state = _state with { ReadingSuspended = true }; }); + Receive(_ => { - _closeRequested = true; - EvaluateShutdown(); + _state = _state with { ReadingSuspended = false }; }); Receive(_ => Context.Stop(Self)); Receive(_ => @@ -355,51 +333,19 @@ private void OpenBehaviour() { Receive(HandleReceiveCompleted); Receive(HandleSendCompleted); - Receive(Enqueue); - - Receive(c => - { - if (Settings.TraceLogging) - Log.Debug("[TcpConnection] Close requested"); - _closeRequested = true; - _state = _state with { ReadingSuspended = true, IsReceiving = false }; - MarkClose(Sender, c.Event); - TrySend(); - if (_state.HasPending) - { - _state = _state with { PendingHalfClose = true }; - } - else - { - HalfCloseWriteSide(); - } - EvaluateShutdown(); - }); - Receive(cc => - { - if (Settings.TraceLogging) - Log.Debug("[TcpConnection] ConfirmedClose requested"); - MarkClose(Sender, cc.Event); - _closeRequested = true; - if (_state.HasPending) - { - _state = _state with { PendingHalfClose = true }; - } - else - { - HalfCloseWriteSide(); - } - EvaluateShutdown(); - }); - Receive(s => + Receive(c => HandleClose(Sender, c.Event)); + SuspendResumeHandlers(); + Receive(h => { - if (Settings.TraceLogging) - Log.Debug("[TcpConnection] AbortSocket requested"); - MarkClose(Sender, s.Event); - AbortSocket(); + Log.Debug("Handler [{0}] died, stopping connection actor", _handler); + Context.Stop(Self); }); + //Receive(_=> { _st = _st with { WritingSuspended=true }; }); + } + private void SuspendResumeHandlers() + { Receive(_ => { _state = _state with { ReadingSuspended = false }; @@ -411,13 +357,57 @@ private void OpenBehaviour() _state = _state with { WritingSuspended = false }; TrySend(); }); - + } + + private void PeerSentEOF() + { + Receive(HandleSendCompleted); + Receive(Enqueue); + Receive(c => HandleClose(Sender, c.Event)); + SuspendResumeHandlers(); + } + + private void ClosingWithPendingWrite(IActorRef closeSender, ConnectionClosed e) + { + Receive(HandleReceiveCompleted); + Receive(s => + { + HandleSendCompleted(s); + if (!_state.HasPending) + { + // we are finished sending + HandleClose(closeSender, e); + } + }); + Receive(Enqueue); + Receive(c => HandleClose(Sender, c.Event)); + SuspendResumeHandlers(); + Receive(h => + { + Log.Debug("Handler [{0}] died, stopping connection actor", _handler); + Context.Stop(Self); + }); + } + + /// + /// Connection is closed on our side, and we're waiting from confirmation from the other side. + /// + private void Closing(IActorRef closeSender) + { + Receive(HandleReceiveCompleted); + Receive(HandleSendCompleted); + Receive(w => + { + // fail all writes + Sender.Tell(w.FailureMessage.WithCause(DroppingWriteBecauseClosingException)); + }); + Receive(c => HandleClose(Sender, c.Event)); + SuspendResumeHandlers(); Receive(h => { Log.Debug("Handler [{0}] died, stopping connection actor", _handler); Context.Stop(Self); }); - //Receive(_=> { _st = _st with { WritingSuspended=true }; }); } /* ----------------------------------------------------------------- */ @@ -435,11 +425,12 @@ private void HandleReceiveCompleted(SocketAsyncEventArgs ea) if (Settings.TraceLogging) { _totalReceivedBytes += ea.BytesTransferred; - Log.Debug("[TcpConnection] received {0} bytes [{1} total]", ea.BytesTransferred, _totalReceivedBytes); + Log.Debug("[TcpConnection] received {0} bytes [{1} total]", ea.BytesTransferred, + _totalReceivedBytes); } - + _handler!.Tell(new Received(ByteString.CopyFrom(_receiveBuffer, 0, ea.BytesTransferred))); - + if (_pullMode) { // in pull mode we need to wait for the next pull request @@ -449,18 +440,27 @@ private void HandleReceiveCompleted(SocketAsyncEventArgs ea) { IssueReceive(); } + return; } - // unless we've been told otherwise, we want to close down the connection - if (!_state.KeepOpenOnPeerClosed) - _closeRequested = true; - - // FIN or error - MarkClose(Self, PeerClosed.Instance); - _handler!.Tell(PeerClosed.Instance); - _state = _state with { PeerClosed = true }; - EvaluateShutdown(); + // check for an error code + if (ea.SocketError != SocketError.Success) + { + if(_traceLogging) + Log.Debug("[TcpConnection] read failed with error [{0}]", ea.SocketError); + HandleError(new SocketException((int)ea.SocketError)); + return; + } + + // check for EOF + if (ea.BytesTransferred == 0) + { + if (_traceLogging) + Log.Debug("[TcpConnection] EOF received"); + _state = _state with { PeerClosed = true }; + HandleClose(_handler!, PeerClosed.Instance); + } } private void HandleSendCompleted(AckSocketAsyncEventArgs ea) @@ -469,34 +469,26 @@ private void HandleSendCompleted(AckSocketAsyncEventArgs ea) if (ea.SocketError != SocketError.Success) { - Log.Warning("[TcpConnection] send failed with error [{0}]", ea.SocketError); - MarkClose(_handler!, new ErrorClosed(ea.SocketError.ToString())); - Context.Stop(Self); + if(_traceLogging) + Log.Debug("[TcpConnection] write failed with error [{0}]", ea.SocketError); + HandleError(new SocketException((int)ea.SocketError)); + return; } - + if (Settings.TraceLogging) { _totalSentBytes += ea.BytesTransferred; - Log.Debug("[TcpConnection] completed write of {0}/{1} bytes (queued={2}/{3}) [{4} total sent]", ea.BytesTransferred, ea.BufferList.Sum(c => c.Count), _state.QueuedBytes, _maxQueuedBytes, _totalSentBytes); + Log.Debug("[TcpConnection] completed write of {0}/{1} bytes (queued={2}/{3}) [{4} total sent]", + ea.BytesTransferred, ea.BufferList.Sum(c => c.Count), _state.QueuedBytes, _maxQueuedBytes, + _totalSentBytes); } foreach (var (c, ack) in ea.PendingAcks) c.Tell(ack); - - - + ea.ClearAcks(); ea.BufferList = null; // release refs - - /* check deferred FIN */ - if(_state.PendingHalfClose && _pendingWrites.Count==0) - { - HalfCloseWriteSide(); - _state = _state with { PendingHalfClose = false }; - } - TrySend(); - EvaluateShutdown(); } /* ----------------------------------------------------------------- */ @@ -516,7 +508,7 @@ private void Enqueue(WriteCommand cmd) var b = (int)cmd.Bytes; if (_maxQueuedBytes >= 0 && _state.QueuedBytes + b > _maxQueuedBytes) { - Sender.Tell(cmd.FailureMessage.WithCause(new IOException("write‑queue full"))); + Sender.Tell(cmd.FailureMessage.WithCause(DroppingWriteBecauseQueueIsFullException)); return; } @@ -531,10 +523,10 @@ private void TrySend() var segs = new List>(8); var batchBytes = 0; - while(_pendingWrites.Count>0) + while (_pendingWrites.Count > 0) { - var (cmd,snd) = _pendingWrites.Peek(); - if(cmd is not Write w) + var (cmd, snd) = _pendingWrites.Peek(); + if (cmd is not Write w) { // unsupported command, fail fast _pendingWrites.Dequeue(); @@ -543,7 +535,7 @@ private void TrySend() } // do not break MTU / send‑buffer – simple heuristic - if(batchBytes !=0 && batchBytes + w.Data.Count > Settings.MaxFrameSizeBytes) + if (batchBytes != 0 && batchBytes + w.Data.Count > Settings.MaxFrameSizeBytes) break; // dequeue & account @@ -552,69 +544,166 @@ private void TrySend() batchBytes += w.Data.Count; segs.AddRange(w.Data.Buffers); - if(w.WantsAck) _sendArgs.PendingAcks.Add((snd, w.Ack)); + if (w.WantsAck) _sendArgs.PendingAcks.Add((snd, w.Ack)); } - if(segs.Count == 0) return; // only empty writes encountered - + if (segs.Count == 0) return; // only empty writes encountered + _sendArgs.BufferList = segs; _state = _state with { IsSending = true }; - if(!Socket.SendAsync(_sendArgs)) Self.Tell(_sendArgs, Self); + if (!Socket.SendAsync(_sendArgs)) Self.Tell(_sendArgs, Self); + } + + /* ====================================================================*/ + /* Shutdown decision */ + /* ====================================================================*/ + private void HandleClose(IActorRef closeSender, ConnectionClosed closeEvent) + { + switch (closeEvent) + { + case Aborted: + if(_traceLogging) + Log.Debug("Got Abort command. RESETing connection."); + DoCloseConnection(closeSender, closeEvent); + break; + // this shouldn't happen really - ErrorClosed is mostly just a message we send to handler. + // but in case we get it, we should close the connection immediately. + case ErrorClosed: + DoCloseConnection(closeSender, closeEvent); + break; + case PeerClosed when _state.KeepOpenOnPeerClosed: + _handler.Tell(PeerClosed.Instance); + _state = _state with { PeerClosed = true }; + Become(PeerSentEOF); + break; + case not null when _state.HasPending: + Context.Unwatch(_handler); // stop watching the handler + if(_traceLogging) + Log.Debug("Got Close command but write is still pending."); + Become(() => ClosingWithPendingWrite(closeSender, closeEvent)); + break; + case ConfirmedClosed: //shutdown output and wait for confirmation + if(_traceLogging) + Log.Debug("Got ConfirmedClose command, sending FIN."); + /* + * If peer closed first, the socket is now fully closed. + * Also, if ShutdownOutput threw an exception we expect this to be an indication + * that the peer closed first or concurrently with this code running. + */ + if(_state.PeerClosed || !ShutdownOutput()) + { + DoCloseConnection(closeSender, closeEvent); + } + else + { + if(_traceLogging) + Log.Debug("Got ConfirmedClose command, but write is still pending."); + Become(() => Closing(closeSender)); + } + break; + default: // no pending writes, not required to stay open when peer is closed + if(_traceLogging) + Log.Debug("Got Close command, closing connection."); + try + { + Socket.Shutdown(SocketShutdown.Both); + } + catch (SocketException e) + { + Log.Error(e, "Graceful socket shutdown failed"); + } + DoCloseConnection(closeSender, closeEvent!); + break; + } + } + + private void HandleError(SocketException e) + { + Log.Debug(e, "Closing connection due to I/O error: {0}", e.SocketErrorCode); + var errorClosed = new ErrorClosed(e.Message); + if(_closeInformation != null) + { + _closeInformation = _closeInformation with { ClosedEvent = errorClosed }; + } + else + { + _closeInformation = CloseInformation.Single(_handler ?? _commander!, errorClosed); + } + Context.Stop(Self); } - private void HalfCloseWriteSide() + private bool ShutdownOutput() { - if (_state.ClosedForWrites) return; try { - if(Settings.TraceLogging) - Log.Debug("[TcpConnection] half‑closing write side"); - Socket.Shutdown(SocketShutdown.Send); + _state = _state with { OutputShutdown = true }; + return true; } - catch + catch (SocketException) { - /* ignore */ + return false; + } + } + + private void DoCloseConnection(IActorRef closeSender, ConnectionClosed closedEvent) + { + switch (closedEvent) + { + case Aborted: + AbortSocket(); + break; + default: + CloseSocket(); + break; } - _state = _state with { ClosedForWrites = true, Phase = Phase.HalfOpen, PendingHalfClose = false}; + StopWith(new CloseInformation(ImmutableHashSet.Empty.Add(closeSender), closedEvent)); } - /* ====================================================================*/ - /* Shutdown decision */ - /* ====================================================================*/ - private void EvaluateShutdown() + private void CloseSocket() { - if (_closeRequested) + try { - var canClose = _state.Closeable(_closeRequested); - if (!canClose) - { - if (Settings.TraceLogging) - Log.Debug("[TcpConnection] can't close yet - state is [{0}]", _state); - } + Socket.Close(); + } + catch + { + /* ignore */ } - if (!_state.Closeable(_closeRequested)) return; - if (Settings.TraceLogging) - Log.Debug("[TcpConnection] shutting down connection [{0}]", _state); - - Self.Tell(PoisonPill.Instance); + try + { + Socket.Dispose(); + } + catch + { + /* ignore */ + } + + _state = _state with { Phase = Phase.Closed, OutputShutdown = true }; } - + private void AbortSocket() { try { - Socket.LingerState = new LingerOption(true, 0); // causes the following close() to send TCP RST + Socket.LingerState = new LingerOption(true, 0); // causes the following close() to send TCP RST } catch (Exception e) { if (_traceLogging) Log.Debug("setSoLinger(true, 0) failed with [{0}]", e); } - Context.Stop(Self); + CloseSocket(); } + protected sealed record CloseInformation(ImmutableHashSet NotificationsTo, Tcp.Event ClosedEvent) + { + public static CloseInformation Single(IActorRef closeSender, Tcp.Event closedEvent) + { + return new CloseInformation(ImmutableHashSet.Empty.Add(closeSender), closedEvent); + } + } } } \ No newline at end of file diff --git a/src/core/Akka/IO/TcpOutgoingConnection.cs b/src/core/Akka/IO/TcpOutgoingConnection.cs index d8d5680f5cf..6f0f7764ca9 100644 --- a/src/core/Akka/IO/TcpOutgoingConnection.cs +++ b/src/core/Akka/IO/TcpOutgoingConnection.cs @@ -79,7 +79,8 @@ private void Stop(Exception cause) ReleaseConnectionSocketArgs(); var failureEvent = _connect.FailureMessage.WithCause(cause); - MarkClose(_commander, failureEvent); + var closeInfo = CloseInformation.Single(_commander, failureEvent); + StopWith(closeInfo); Context.Stop(Self); } From 3c40fa1a97ffa391528572456571e28a3ba9bc40 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 17 May 2025 10:22:42 -0500 Subject: [PATCH 43/60] fixed a number of specs --- src/core/Akka.Tests/IO/TcpIntegrationSpec.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs index fdc1e624a0e..bcc7d6359bc 100644 --- a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs +++ b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs @@ -84,7 +84,7 @@ public async Task The_TCP_transport_implementation_should_properly_handle_connec var actors = await x.EstablishNewClientConnectionAsync(); actors.ClientHandler.Send(actors.ClientConnection, Tcp.Abort.Instance); await actors.ClientHandler.ExpectMsgAsync(); - await actors.ServerHandler.ExpectMsgAsync(); + await actors.ServerHandler.ExpectMsgAsync(); await VerifyActorTermination(actors.ClientConnection); await VerifyActorTermination(actors.ServerConnection); }); @@ -100,7 +100,7 @@ public async Task The_TCP_transport_implementation_should_properly_handle_connec actors.ClientHandler.Send(actors.ClientConnection, Tcp.Abort.Instance); await actors.ClientHandler.ExpectMsgAsync(); - await actors.ServerHandler.ExpectMsgAsync(); + await actors.ServerHandler.ExpectMsgAsync(); await VerifyActorTermination(actors.ClientConnection); await VerifyActorTermination(actors.ServerConnection); }); @@ -145,7 +145,7 @@ public async Task The_TCP_transport_implementation_should_properly_handle_connec actors.ServerHandler.Send(actors.ServerConnection, PoisonPill.Instance); await VerifyActorTermination(actors.ServerConnection); - await actors.ClientHandler.ExpectMsgAsync(); + await actors.ClientHandler.ExpectMsgAsync(); await VerifyActorTermination(actors.ClientConnection); }); } @@ -161,7 +161,7 @@ public async Task The_TCP_transport_implementation_should_properly_handle_connec actors.ServerHandler.Send(actors.ServerConnection, PoisonPill.Instance); await VerifyActorTermination(actors.ServerConnection); - await actors.ClientHandler.ExpectMsgAsync(); + await actors.ClientHandler.ExpectMsgAsync(); await VerifyActorTermination(actors.ClientConnection); }); } From 4c7be29173e5bdfbd173c227ee2447b53a9b0845 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 17 May 2025 10:32:13 -0500 Subject: [PATCH 44/60] all Akka.Streams.IO.Tcp specs now pass --- src/core/Akka/IO/TcpConnection.cs | 25 +++++++++++++++++-------- src/core/Akka/IO/TcpListener.cs | 4 ++-- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 88b056b0567..2d465ddbda3 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -331,7 +331,7 @@ private void AwaitRegBehaviour() private void OpenBehaviour() { - Receive(HandleReceiveCompleted); + Receive(s => HandleReceiveCompleted(s, null)); Receive(HandleSendCompleted); Receive(Enqueue); Receive(c => HandleClose(Sender, c.Event)); @@ -369,7 +369,7 @@ private void PeerSentEOF() private void ClosingWithPendingWrite(IActorRef closeSender, ConnectionClosed e) { - Receive(HandleReceiveCompleted); + Receive(s => HandleReceiveCompleted(s, closeSender)); Receive(s => { HandleSendCompleted(s); @@ -394,7 +394,7 @@ private void ClosingWithPendingWrite(IActorRef closeSender, ConnectionClosed e) /// private void Closing(IActorRef closeSender) { - Receive(HandleReceiveCompleted); + Receive(s => HandleReceiveCompleted(s, closeSender)); Receive(HandleSendCompleted); Receive(w => { @@ -417,7 +417,7 @@ private void Closing(IActorRef closeSender) private long _totalSentBytes; private long _totalReceivedBytes; - private void HandleReceiveCompleted(SocketAsyncEventArgs ea) + private void HandleReceiveCompleted(SocketAsyncEventArgs ea, IActorRef? closeCommander) { _state = _state with { IsReceiving = false }; if (ea is { SocketError: SocketError.Success, BytesTransferred: > 0 }) @@ -456,10 +456,19 @@ private void HandleReceiveCompleted(SocketAsyncEventArgs ea) // check for EOF if (ea.BytesTransferred == 0) { - if (_traceLogging) - Log.Debug("[TcpConnection] EOF received"); - _state = _state with { PeerClosed = true }; - HandleClose(_handler!, PeerClosed.Instance); + if (_state.OutputShutdown) + { + if(_traceLogging) + Log.Debug("[TcpConnection] EOF received; our side is already closed. Closing connection."); + DoCloseConnection(closeCommander ?? _handler!, ConfirmedClosed.Instance); + } + else + { + if (_traceLogging) + Log.Debug("[TcpConnection] EOF received"); + _state = _state with { PeerClosed = true }; + HandleClose(closeCommander ?? _handler!, PeerClosed.Instance); + } } } diff --git a/src/core/Akka/IO/TcpListener.cs b/src/core/Akka/IO/TcpListener.cs index 29a677d67d1..01dd20f3486 100644 --- a/src/core/Akka/IO/TcpListener.cs +++ b/src/core/Akka/IO/TcpListener.cs @@ -88,9 +88,9 @@ private ConnectionTerminated() public static ConnectionTerminated Instance { get; } = new(); } - private sealed record AcceptCompleted(SocketAsyncEventArgs EventArgs) : INoSerializationVerificationNeeded; + private sealed record AcceptCompleted(SocketAsyncEventArgs EventArgs) : INoSerializationVerificationNeeded, IDeadLetterSuppression; - private sealed record RetryAccept(SocketAsyncEventArgs EventArgs) : INoSerializationVerificationNeeded; + private sealed record RetryAccept(SocketAsyncEventArgs EventArgs) : INoSerializationVerificationNeeded, IDeadLetterSuppression; public TcpListener(TcpExt tcp, IActorRef bindCommander, Tcp.Bind bind) From f7e7d9889a92fec54380ddea768553a455886eb4 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 17 May 2025 10:34:46 -0500 Subject: [PATCH 45/60] removed unused `ConnectionState` fields --- src/core/Akka/IO/TcpConnection.cs | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 2d465ddbda3..829cafde6db 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -72,29 +72,11 @@ namespace Akka.IO /// internal abstract class TcpConnection : ReceiveActor, IRequiresMessageQueue { - /// - /// Immutable connection state – the *only* mutable field in the actor is this record. - /// - private enum Phase - { - Connecting, - AwaitReg, - Open, - HalfOpen, - Closed - } - /// /// Immutable flags – reference to the live Queue + byte counter **and any deferred half‑close**. /// Moving every transient flag in here lets us reason over shutdown with a single value. /// - /// - /// Indicates that a half-close (shutdown of the write side) has been requested (via ConfirmedClose or Close), - /// but there are still pending writes in the queue. When all writes have been delivered, the write side will - /// be closed (Socket.Shutdown(SocketShutdown.Send)), and this flag will be reset to false. - /// private readonly record struct ConnState( - Phase Phase, bool IsReceiving, bool IsSending, bool PeerClosed, @@ -110,7 +92,7 @@ private readonly record struct ConnState( public bool CanReceive => !PeerClosed && !ReadingSuspended; public static ConnState Initial(Queue<(WriteCommand Cmd, IActorRef Snd)> q) => - new(Phase.Connecting, false, false, false, false, false, false, false, q, 0); + new(false, false, false, false, false, false, false, q, 0); } #region Ack‑aware SAEA @@ -279,7 +261,6 @@ protected void CompleteConnect(IActorRef commander, IEnumerable Date: Sat, 17 May 2025 10:37:25 -0500 Subject: [PATCH 46/60] added default `_closeInformation` so socket closures can never no-op --- src/core/Akka/IO/TcpConnection.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 829cafde6db..116dcf80960 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -288,6 +288,9 @@ private void AwaitRegBehaviour() Context.WatchWith(_handler, HandlerDied.Instance); Context.Unwatch(_commander); _state = _state with { KeepOpenOnPeerClosed = reg.KeepOpenOnPeerClosed }; + + // set a default close event - if someone hard-kills us we log an aborted + _closeInformation = CloseInformation.Single(_handler, Aborted.Instance); Context.SetReceiveTimeout(null); Become(OpenBehaviour); IssueReceive(); From 5fa23bf0e40daa3f5c8c55d815666d6dfe754908 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 17 May 2025 14:54:58 -0500 Subject: [PATCH 47/60] fixed issue with `ExpectReceivedDataAsync` --- src/core/Akka.Tests/IO/TcpIntegrationSpec.cs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs index bcc7d6359bc..0e99552b895 100644 --- a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs +++ b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs @@ -464,6 +464,7 @@ public async Task The_TCP_transport_implementation_should_properly_complete_one_ [Fact] public async Task The_TCP_transport_implementation_should_support_waiting_for_writes_with_backpressure() { + var transmittedBytes = InternalConnectionActorMaxQueueSize; await new TestSetup(this).RunAsync(async x => { x.BindOptions = [new Inet.SO.SendBufferSize(1024)]; @@ -471,10 +472,10 @@ public async Task The_TCP_transport_implementation_should_support_waiting_for_wr var actors = await x.EstablishNewClientConnectionAsync(); - actors.ServerHandler.Send(actors.ServerConnection, Tcp.Write.Create(ByteString.FromBytes(new byte[100000]), Ack.Instance)); + actors.ServerHandler.Send(actors.ServerConnection, Tcp.Write.Create(ByteString.FromBytes(new byte[transmittedBytes]), Ack.Instance)); await actors.ServerHandler.ExpectMsgAsync(Ack.Instance); - await x.ExpectReceivedDataAsync(actors.ClientHandler, 100000); + await x.ExpectReceivedDataAsync(actors.ClientHandler, transmittedBytes); }); } @@ -609,10 +610,16 @@ public class ConnectionDetail public async Task ExpectReceivedDataAsync(TestProbe handler, int remaining) { - if (remaining > 0) + while (true) { - var recv = await handler.ExpectMsgAsync(); - await ExpectReceivedDataAsync(handler, remaining - recv.Data.Count); + if (remaining > 0) + { + var recv = await handler.ExpectMsgAsync(); + remaining = remaining - recv.Data.Count; + continue; + } + + break; } } From f15802a64e443bbb8261e6c1fdffbf74e3728f21 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 17 May 2025 15:07:30 -0500 Subject: [PATCH 48/60] fixed partial write offset handling --- src/core/Akka.Tests/IO/TcpIntegrationSpec.cs | 4 +- src/core/Akka/IO/TcpConnection.cs | 72 ++++++++++++++++---- 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs index 0e99552b895..e6b4b669cc6 100644 --- a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs +++ b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs @@ -281,7 +281,7 @@ public async Task Write_before_Register_should_not_be_silently_dropped() var msg = ByteString.FromString("msg"); // 3 bytes - await EventFilter.Debug(new Regex("Received Write command before Register[^3]+3 bytes")).ExpectOneAsync(() => { + await EventFilter.Warning(new Regex("Received Write command before Register[^3]+3 bytes")).ExpectOneAsync(() => { actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(msg)); actors.ClientConnection.Tell(new Tcp.Register(actors.ClientHandler)); return Task.CompletedTask; @@ -401,7 +401,7 @@ public async Task When_multiple_writing_clients_Should_receive_messages_in_order }); } - [Fact(Skip = "Have to re-implement pagination")] + [Fact] public async Task Should_fail_writing_when_buffer_is_filled() { await new TestSetup(this).RunAsync(async x => diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 116dcf80960..4bc447f9077 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -160,6 +160,8 @@ private HandlerDied() private static readonly IOException DroppingWriteBecauseQueueIsFullException = new("Dropping write because queue is full"); + private int? _partialWriteOffset = null; + protected TcpConnection(TcpSettings settings, Socket socket, bool pullMode) { Settings = settings; @@ -296,7 +298,14 @@ private void AwaitRegBehaviour() IssueReceive(); TrySend(); }); - Receive(Enqueue); + Receive(c => + { + if (Enqueue(c)) + { + Log.Warning("Received Write command before Register command. " + + "It will be buffered until Register will be received (buffered write size is {0} bytes)", c.Bytes); + } + }); Receive(c => HandleClose(Sender, c.Event)); Receive(_ => { _state = _state with { ReadingSuspended = true }; }); Receive(_ => @@ -460,6 +469,9 @@ private void HandleSendCompleted(AckSocketAsyncEventArgs ea) { _state = _state with { IsSending = false }; + if (_traceLogging) + Log.Debug($"[TcpConnection] HandleSendCompleted: BytesTransferred={ea.BytesTransferred}, PendingAcks={ea.PendingAcks.Count}, PartialWriteOffset={_partialWriteOffset}"); + if (ea.SocketError != SocketError.Success) { if(_traceLogging) @@ -496,54 +508,84 @@ private void IssueReceive() if (!Socket.ReceiveAsync(_receiveArgs)) Self.Tell(_receiveArgs, Self); } - private void Enqueue(WriteCommand cmd) + private bool Enqueue(WriteCommand cmd) { var b = (int)cmd.Bytes; if (_maxQueuedBytes >= 0 && _state.QueuedBytes + b > _maxQueuedBytes) { Sender.Tell(cmd.FailureMessage.WithCause(DroppingWriteBecauseQueueIsFullException)); - return; + return false; } _pendingWrites.Enqueue((cmd, Sender)); _state = _state with { QueuedBytes = _state.QueuedBytes + b }; TrySend(); + return true; } private void TrySend() { + if (_traceLogging) + Log.Debug($"[TcpConnection] TrySend called. IsSending={_state.IsSending}, PendingWrites={_pendingWrites.Count}, CanSend={_state.CanSend}, PartialWriteOffset={_partialWriteOffset}"); if (_state.IsSending || _pendingWrites.Count == 0 || !_state.CanSend) return; var segs = new List>(8); var batchBytes = 0; - while (_pendingWrites.Count > 0) + while (_pendingWrites.Count > 0 && batchBytes < Settings.MaxFrameSizeBytes) { var (cmd, snd) = _pendingWrites.Peek(); if (cmd is not Write w) { - // unsupported command, fail fast _pendingWrites.Dequeue(); snd.Tell(cmd.FailureMessage); + _partialWriteOffset = null; continue; } - // do not break MTU / send‑buffer – simple heuristic - if (batchBytes != 0 && batchBytes + w.Data.Count > Settings.MaxFrameSizeBytes) - break; + var data = w.Data; + var offset = _partialWriteOffset ?? 0; + var remaining = data.Count - offset; + var toSend = Math.Min(remaining, Settings.MaxFrameSizeBytes - batchBytes); + + if (_traceLogging) + Log.Debug($"[TcpConnection] TrySend batching: offset={offset}, remaining={remaining}, toSend={toSend}, batchBytes={batchBytes}"); - // dequeue & account - _pendingWrites.Dequeue(); - _state = _state with { QueuedBytes = _state.QueuedBytes - w.Data.Count }; - batchBytes += w.Data.Count; - segs.AddRange(w.Data.Buffers); + if (toSend <= 0) + break; // batch is full - if (w.WantsAck) _sendArgs.PendingAcks.Add((snd, w.Ack)); + var chunk = data.Slice(offset, toSend); + segs.AddRange(chunk.Buffers); + batchBytes += toSend; + + if (toSend == remaining) + { + _pendingWrites.Dequeue(); + _state = _state with { QueuedBytes = _state.QueuedBytes - w.Data.Count }; + _partialWriteOffset = null; + if (w.WantsAck) _sendArgs.PendingAcks.Add((snd, w.Ack)); + if (_traceLogging) + Log.Debug($"[TcpConnection] TrySend: completed full write, dequeued. Remaining queue: {_pendingWrites.Count}"); + } + else + { + _partialWriteOffset = offset + toSend; + if (_traceLogging) + Log.Debug($"[TcpConnection] TrySend: partial write, will resume at offset {_partialWriteOffset}"); + break; // batch is full + } } - if (segs.Count == 0) return; // only empty writes encountered + if (segs.Count == 0) + { + if (_traceLogging) + Log.Debug("[TcpConnection] TrySend: no segments to send (only empty writes encountered)"); + return; // only empty writes encountered + } _sendArgs.BufferList = segs; _state = _state with { IsSending = true }; + if (_traceLogging) + Log.Debug($"[TcpConnection] TrySend: sending {segs.Count} segments, total bytes={batchBytes}"); if (!Socket.SendAsync(_sendArgs)) Self.Tell(_sendArgs, Self); } From 0c00783e9f2ea8eb5304905283dad856e43af5cf Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 17 May 2025 15:37:26 -0500 Subject: [PATCH 49/60] fixed a bug with writing being enabled right away --- src/core/Akka.Tests/IO/TcpIntegrationSpec.cs | 6 +++--- src/core/Akka/IO/TcpConnection.cs | 18 +++++++++++------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs index e6b4b669cc6..6902926cf35 100644 --- a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs +++ b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs @@ -416,9 +416,9 @@ public async Task Should_fail_writing_when_buffer_is_filled() await AwaitAssertAsync(async () => { // try sending overflow - actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(overflowData)); // this is sent immidiately + actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(overflowData)); // this is sent immediately actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(overflowData)); // this will try to buffer - await actors.ClientHandler.ExpectMsgAsync(TimeSpan.FromSeconds(20)); + await actors.ClientHandler.ExpectMsgAsync(); // First overflow data will be received anyway (await actors.ServerHandler.ReceiveWhileAsync(TimeSpan.FromSeconds(1), m => m as Tcp.Received).ToListAsync()) @@ -431,7 +431,7 @@ await AwaitAssertAsync(async () => (await actors.ServerHandler.ReceiveWhileAsync(TimeSpan.FromSeconds(1), m => m as Tcp.Received).ToListAsync()) .Sum(m => m.Data.Count) .Should().Be(InternalConnectionActorMaxQueueSize); - }, TimeSpan.FromSeconds(30 * 3), TimeSpan.FromSeconds(5)); // 3 attempts by ~25 seconds + 5 sec pause + }, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(3)); // 3 attempts by ~25 seconds + 5 sec pause }); } diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 4bc447f9077..4466f88b24c 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -92,7 +92,7 @@ private readonly record struct ConnState( public bool CanReceive => !PeerClosed && !ReadingSuspended; public static ConnState Initial(Queue<(WriteCommand Cmd, IActorRef Snd)> q) => - new(false, false, false, false, false, false, false, q, 0); + new(false, false, false, false, true, true, false, q, 0); } #region Ack‑aware SAEA @@ -289,7 +289,7 @@ private void AwaitRegBehaviour() if (_traceLogging) Log.Debug("[{0}] registered as connection handler", reg.Handler); Context.WatchWith(_handler, HandlerDied.Instance); Context.Unwatch(_commander); - _state = _state with { KeepOpenOnPeerClosed = reg.KeepOpenOnPeerClosed }; + _state = _state with { KeepOpenOnPeerClosed = reg.KeepOpenOnPeerClosed, ReadingSuspended = false, WritingSuspended = false }; // set a default close event - if someone hard-kills us we log an aborted _closeInformation = CloseInformation.Single(_handler, Aborted.Instance); @@ -300,7 +300,11 @@ private void AwaitRegBehaviour() }); Receive(c => { - if (Enqueue(c)) + var queueSize = _pendingWrites.Count; + Enqueue(c); + + // check if we buffered and log a warning + if(_pendingWrites.Count > queueSize) { Log.Warning("Received Write command before Register command. " + "It will be buffered until Register will be received (buffered write size is {0} bytes)", c.Bytes); @@ -508,26 +512,26 @@ private void IssueReceive() if (!Socket.ReceiveAsync(_receiveArgs)) Self.Tell(_receiveArgs, Self); } - private bool Enqueue(WriteCommand cmd) + private void Enqueue(WriteCommand cmd) { var b = (int)cmd.Bytes; if (_maxQueuedBytes >= 0 && _state.QueuedBytes + b > _maxQueuedBytes) { Sender.Tell(cmd.FailureMessage.WithCause(DroppingWriteBecauseQueueIsFullException)); - return false; + return; } _pendingWrites.Enqueue((cmd, Sender)); _state = _state with { QueuedBytes = _state.QueuedBytes + b }; TrySend(); - return true; } private void TrySend() { if (_traceLogging) Log.Debug($"[TcpConnection] TrySend called. IsSending={_state.IsSending}, PendingWrites={_pendingWrites.Count}, CanSend={_state.CanSend}, PartialWriteOffset={_partialWriteOffset}"); - if (_state.IsSending || _pendingWrites.Count == 0 || !_state.CanSend) return; + if (!_state.CanSend) return; + if (_state.IsSending || _pendingWrites.Count == 0) return; var segs = new List>(8); var batchBytes = 0; From 841302044814b4f6b7418fa39a7da97578036992 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 17 May 2025 16:02:50 -0500 Subject: [PATCH 50/60] cleaned up `Should_fail_writing_when_buffer_is_filled` spec --- src/core/Akka.Tests/IO/TcpIntegrationSpec.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs index 6902926cf35..c36e447aa86 100644 --- a/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs +++ b/src/core/Akka.Tests/IO/TcpIntegrationSpec.cs @@ -416,17 +416,17 @@ public async Task Should_fail_writing_when_buffer_is_filled() await AwaitAssertAsync(async () => { // try sending overflow - actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(overflowData)); // this is sent immediately - actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(overflowData)); // this will try to buffer + actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(goodData)); // this is sent immediately + actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(overflowData)); // this will fail await actors.ClientHandler.ExpectMsgAsync(); - // First overflow data will be received anyway + // First message will go through, second one will not (await actors.ServerHandler.ReceiveWhileAsync(TimeSpan.FromSeconds(1), m => m as Tcp.Received).ToListAsync()) .Sum(m => m.Data.Count) - .Should().Be(InternalConnectionActorMaxQueueSize + 1); + .Should().Be(InternalConnectionActorMaxQueueSize); // Check that almost-overflow size does not cause any problems - actors.ClientHandler.Send(actors.ClientConnection, Tcp.ResumeWriting.Instance); // Recover after send failure + //actors.ClientHandler.Send(actors.ClientConnection, Tcp.ResumeWriting.Instance); // Recover after send failure actors.ClientHandler.Send(actors.ClientConnection, Tcp.Write.Create(goodData)); (await actors.ServerHandler.ReceiveWhileAsync(TimeSpan.FromSeconds(1), m => m as Tcp.Received).ToListAsync()) .Sum(m => m.Data.Count) From 9e9cd2cd7ab7764611b40efe38eb93f206ddb64d Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 17 May 2025 16:24:56 -0500 Subject: [PATCH 51/60] fixed issue with reads starting too early in pull mode --- src/core/Akka.Streams.Tests/IO/TcpHelper.cs | 4 ++-- src/core/Akka/IO/TcpConnection.cs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/Akka.Streams.Tests/IO/TcpHelper.cs b/src/core/Akka.Streams.Tests/IO/TcpHelper.cs index 3467ecbd144..714d86be9e9 100644 --- a/src/core/Akka.Streams.Tests/IO/TcpHelper.cs +++ b/src/core/Akka.Streams.Tests/IO/TcpHelper.cs @@ -330,9 +330,9 @@ await _connectionProbe.FishForMessageAsync( public async Task ExpectTerminatedAsync(CancellationToken cancellationToken = default) { - _connectionProbe.Watch(_connectionActor); + await _connectionProbe.WatchAsync(_connectionActor); await _connectionProbe.ExpectTerminatedAsync(_connectionActor, cancellationToken: cancellationToken); - _connectionProbe.Unwatch(_connectionActor); + await _connectionProbe.UnwatchAsync(_connectionActor); } } diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 4466f88b24c..d03d18f4d78 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -289,7 +289,7 @@ private void AwaitRegBehaviour() if (_traceLogging) Log.Debug("[{0}] registered as connection handler", reg.Handler); Context.WatchWith(_handler, HandlerDied.Instance); Context.Unwatch(_commander); - _state = _state with { KeepOpenOnPeerClosed = reg.KeepOpenOnPeerClosed, ReadingSuspended = false, WritingSuspended = false }; + _state = _state with { KeepOpenOnPeerClosed = reg.KeepOpenOnPeerClosed, ReadingSuspended = _pullMode, WritingSuspended = false }; // set a default close event - if someone hard-kills us we log an aborted _closeInformation = CloseInformation.Single(_handler, Aborted.Instance); @@ -532,7 +532,7 @@ private void TrySend() Log.Debug($"[TcpConnection] TrySend called. IsSending={_state.IsSending}, PendingWrites={_pendingWrites.Count}, CanSend={_state.CanSend}, PartialWriteOffset={_partialWriteOffset}"); if (!_state.CanSend) return; if (_state.IsSending || _pendingWrites.Count == 0) return; - var segs = new List>(8); + var segs = new List>(1); var batchBytes = 0; while (_pendingWrites.Count > 0 && batchBytes < Settings.MaxFrameSizeBytes) From 4237808bc096a959cce11bff63ff6ec8f251480c Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 17 May 2025 16:32:40 -0500 Subject: [PATCH 52/60] added logging support for `sys2` in `Outgoing_TCP_stream_must_not_thrown_on_unbind_after_system_has_been_shut_down` --- src/core/Akka.Streams.Tests/IO/TcpSpec.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/Akka.Streams.Tests/IO/TcpSpec.cs b/src/core/Akka.Streams.Tests/IO/TcpSpec.cs index 1ec954d63a0..2c3b66b8626 100644 --- a/src/core/Akka.Streams.Tests/IO/TcpSpec.cs +++ b/src/core/Akka.Streams.Tests/IO/TcpSpec.cs @@ -528,6 +528,7 @@ await AwaitAssertAsync(async () => public async Task Outgoing_TCP_stream_must_not_thrown_on_unbind_after_system_has_been_shut_down() { var sys2 = ActorSystem.Create("shutdown-test-system", Sys.Settings.Config); + InitializeLogger(sys2); try { From 3ee94cb63eeb0726a874b427bd8c6322832e99ea Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 17 May 2025 17:00:53 -0500 Subject: [PATCH 53/60] fixed batching code --- src/core/Akka/IO/TcpConnection.cs | 62 ++++++++++++++----------------- 1 file changed, 28 insertions(+), 34 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index d03d18f4d78..084fa316a46 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -84,14 +84,14 @@ private readonly record struct ConnState( bool ReadingSuspended, bool WritingSuspended, bool KeepOpenOnPeerClosed, - Queue<(WriteCommand Cmd, IActorRef Snd)> Queue, + Queue<(Write Cmd, IActorRef Snd)> Queue, int QueuedBytes) { public bool HasPending => IsSending || Queue.Count != 0; public bool CanSend => !OutputShutdown && !WritingSuspended; public bool CanReceive => !PeerClosed && !ReadingSuspended; - public static ConnState Initial(Queue<(WriteCommand Cmd, IActorRef Snd)> q) => + public static ConnState Initial(Queue<(Write Cmd, IActorRef Snd)> q) => new(false, false, false, false, true, true, false, q, 0); } @@ -133,7 +133,7 @@ private HandlerDied() private readonly ArrayPool _bufferPool = ArrayPool.Shared; - private readonly Queue<(WriteCommand Cmd, IActorRef Sender)> _pendingWrites; + private readonly Queue<(Write Cmd, IActorRef Sender)> _pendingWrites; private readonly byte[] _receiveBuffer; private readonly ReadSocketAsyncEventArgs _receiveArgs; private readonly AckSocketAsyncEventArgs _sendArgs; @@ -166,7 +166,7 @@ protected TcpConnection(TcpSettings settings, Socket socket, bool pullMode) { Settings = settings; _maxQueuedBytes = settings.WriteCommandsQueueMaxSize; // –1 ⇒ unlimited; - _pendingWrites = new Queue<(WriteCommand Cmd, IActorRef Sender)>(16); + _pendingWrites = new Queue<(Write Cmd, IActorRef Sender)>(16); _pullMode = pullMode; _traceLogging = Settings.TraceLogging; @@ -290,7 +290,6 @@ private void AwaitRegBehaviour() Context.WatchWith(_handler, HandlerDied.Instance); Context.Unwatch(_commander); _state = _state with { KeepOpenOnPeerClosed = reg.KeepOpenOnPeerClosed, ReadingSuspended = _pullMode, WritingSuspended = false }; - // set a default close event - if someone hard-kills us we log an aborted _closeInformation = CloseInformation.Single(_handler, Aborted.Instance); Context.SetReceiveTimeout(null); @@ -298,18 +297,7 @@ private void AwaitRegBehaviour() IssueReceive(); TrySend(); }); - Receive(c => - { - var queueSize = _pendingWrites.Count; - Enqueue(c); - - // check if we buffered and log a warning - if(_pendingWrites.Count > queueSize) - { - Log.Warning("Received Write command before Register command. " + - "It will be buffered until Register will be received (buffered write size is {0} bytes)", c.Bytes); - } - }); + Receive(Enqueue); Receive(c => HandleClose(Sender, c.Event)); Receive(_ => { _state = _state with { ReadingSuspended = true }; }); Receive(_ => @@ -330,7 +318,7 @@ private void OpenBehaviour() { Receive(s => HandleReceiveCompleted(s, null)); Receive(HandleSendCompleted); - Receive(Enqueue); + Receive(Enqueue); Receive(c => HandleClose(Sender, c.Event)); SuspendResumeHandlers(); Receive(h => @@ -359,7 +347,7 @@ private void SuspendResumeHandlers() private void PeerSentEOF() { Receive(HandleSendCompleted); - Receive(Enqueue); + Receive(Enqueue); Receive(c => HandleClose(Sender, c.Event)); SuspendResumeHandlers(); } @@ -376,7 +364,7 @@ private void ClosingWithPendingWrite(IActorRef closeSender, ConnectionClosed e) HandleClose(closeSender, e); } }); - Receive(Enqueue); + Receive(Enqueue); Receive(c => HandleClose(Sender, c.Event)); SuspendResumeHandlers(); Receive(h => @@ -393,7 +381,7 @@ private void Closing(IActorRef closeSender) { Receive(s => HandleReceiveCompleted(s, closeSender)); Receive(HandleSendCompleted); - Receive(w => + Receive(w => { // fail all writes Sender.Tell(w.FailureMessage.WithCause(DroppingWriteBecauseClosingException)); @@ -512,7 +500,7 @@ private void IssueReceive() if (!Socket.ReceiveAsync(_receiveArgs)) Self.Tell(_receiveArgs, Self); } - private void Enqueue(WriteCommand cmd) + private void Enqueue(Write cmd) { var b = (int)cmd.Bytes; if (_maxQueuedBytes >= 0 && _state.QueuedBytes + b > _maxQueuedBytes) @@ -532,37 +520,42 @@ private void TrySend() Log.Debug($"[TcpConnection] TrySend called. IsSending={_state.IsSending}, PendingWrites={_pendingWrites.Count}, CanSend={_state.CanSend}, PartialWriteOffset={_partialWriteOffset}"); if (!_state.CanSend) return; if (_state.IsSending || _pendingWrites.Count == 0) return; - var segs = new List>(1); + + var segs = new List>(8); var batchBytes = 0; while (_pendingWrites.Count > 0 && batchBytes < Settings.MaxFrameSizeBytes) { - var (cmd, snd) = _pendingWrites.Peek(); - if (cmd is not Write w) + var (w, snd) = _pendingWrites.Peek(); + + var data = w.Data; + var offset = _partialWriteOffset ?? 0; + var remaining = data.Count - offset; + + // Handle empty writes immediately + if (remaining == 0) { _pendingWrites.Dequeue(); - snd.Tell(cmd.FailureMessage); + _state = _state with { QueuedBytes = _state.QueuedBytes - w.Data.Count }; _partialWriteOffset = null; + if (w.WantsAck) _sendArgs.PendingAcks.Add((snd, w.Ack)); + if (_traceLogging) + Log.Debug($"[TcpConnection] TrySend: encountered empty write, dequeued. Remaining queue: {_pendingWrites.Count}"); continue; } - var data = w.Data; - var offset = _partialWriteOffset ?? 0; - var remaining = data.Count - offset; var toSend = Math.Min(remaining, Settings.MaxFrameSizeBytes - batchBytes); if (_traceLogging) Log.Debug($"[TcpConnection] TrySend batching: offset={offset}, remaining={remaining}, toSend={toSend}, batchBytes={batchBytes}"); - if (toSend <= 0) - break; // batch is full - var chunk = data.Slice(offset, toSend); segs.AddRange(chunk.Buffers); batchBytes += toSend; if (toSend == remaining) { + // Full write completed _pendingWrites.Dequeue(); _state = _state with { QueuedBytes = _state.QueuedBytes - w.Data.Count }; _partialWriteOffset = null; @@ -572,10 +565,11 @@ private void TrySend() } else { + // Partial write, update offset and break _partialWriteOffset = offset + toSend; if (_traceLogging) Log.Debug($"[TcpConnection] TrySend: partial write, will resume at offset {_partialWriteOffset}"); - break; // batch is full + break; } } @@ -583,7 +577,7 @@ private void TrySend() { if (_traceLogging) Log.Debug("[TcpConnection] TrySend: no segments to send (only empty writes encountered)"); - return; // only empty writes encountered + return; } _sendArgs.BufferList = segs; From 547474a9e4b92c38e9fd09239970159aa6f429fe Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 17 May 2025 17:15:25 -0500 Subject: [PATCH 54/60] added support for all `WriteCommand` implementations --- src/core/Akka/IO/TcpConnection.cs | 35 ++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 084fa316a46..9bf66ca3c3d 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -297,7 +297,7 @@ private void AwaitRegBehaviour() IssueReceive(); TrySend(); }); - Receive(Enqueue); + Receive(Enqueue); Receive(c => HandleClose(Sender, c.Event)); Receive(_ => { _state = _state with { ReadingSuspended = true }; }); Receive(_ => @@ -318,7 +318,7 @@ private void OpenBehaviour() { Receive(s => HandleReceiveCompleted(s, null)); Receive(HandleSendCompleted); - Receive(Enqueue); + Receive(Enqueue); Receive(c => HandleClose(Sender, c.Event)); SuspendResumeHandlers(); Receive(h => @@ -347,7 +347,7 @@ private void SuspendResumeHandlers() private void PeerSentEOF() { Receive(HandleSendCompleted); - Receive(Enqueue); + Receive(Enqueue); Receive(c => HandleClose(Sender, c.Event)); SuspendResumeHandlers(); } @@ -364,7 +364,7 @@ private void ClosingWithPendingWrite(IActorRef closeSender, ConnectionClosed e) HandleClose(closeSender, e); } }); - Receive(Enqueue); + Receive(Enqueue); Receive(c => HandleClose(Sender, c.Event)); SuspendResumeHandlers(); Receive(h => @@ -381,7 +381,7 @@ private void Closing(IActorRef closeSender) { Receive(s => HandleReceiveCompleted(s, closeSender)); Receive(HandleSendCompleted); - Receive(w => + Receive(w => { // fail all writes Sender.Tell(w.FailureMessage.WithCause(DroppingWriteBecauseClosingException)); @@ -500,7 +500,7 @@ private void IssueReceive() if (!Socket.ReceiveAsync(_receiveArgs)) Self.Tell(_receiveArgs, Self); } - private void Enqueue(Write cmd) + private void Enqueue(WriteCommand cmd) { var b = (int)cmd.Bytes; if (_maxQueuedBytes >= 0 && _state.QueuedBytes + b > _maxQueuedBytes) @@ -508,10 +508,31 @@ private void Enqueue(Write cmd) Sender.Tell(cmd.FailureMessage.WithCause(DroppingWriteBecauseQueueIsFullException)); return; } + + EnqueueInner(cmd); - _pendingWrites.Enqueue((cmd, Sender)); _state = _state with { QueuedBytes = _state.QueuedBytes + b }; TrySend(); + return; + + void EnqueueInner(WriteCommand wCmd) + { + switch (cmd) + { + case Write realWrite: + _pendingWrites.Enqueue((realWrite, Sender)); + break; + case CompoundWrite compounds: //TODO: poorly designed API we should remove + foreach (var c in compounds) + { + EnqueueInner(c); + } + break; + default: + Sender.Tell(cmd.FailureMessage.WithCause(new InvalidOperationException($"Cannot enqueue {cmd} - only valid classes are Write and CompoundWrite"))); + break; + } + } } private void TrySend() From fbae5d6828e8528fa081f45258cb30466c67b7cb Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 17 May 2025 17:16:03 -0500 Subject: [PATCH 55/60] removed recursion --- src/core/Akka/IO/TcpConnection.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 9bf66ca3c3d..3e1342af404 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -525,7 +525,14 @@ void EnqueueInner(WriteCommand wCmd) case CompoundWrite compounds: //TODO: poorly designed API we should remove foreach (var c in compounds) { - EnqueueInner(c); + if(c is Write w) + { + _pendingWrites.Enqueue((w, Sender)); + } + else + { + Sender.Tell(c.FailureMessage.WithCause(new InvalidOperationException($"Cannot enqueue {c} - only valid classes are Write and CompoundWrite"))); + } } break; default: From c4efa937b192ae7a1065f046712b18ffad513c0d Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 17 May 2025 17:16:25 -0500 Subject: [PATCH 56/60] cleanup --- src/core/Akka/IO/TcpConnection.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 3e1342af404..c04e3c50d6f 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -509,13 +509,13 @@ private void Enqueue(WriteCommand cmd) return; } - EnqueueInner(cmd); + EnqueueInner(); _state = _state with { QueuedBytes = _state.QueuedBytes + b }; TrySend(); return; - void EnqueueInner(WriteCommand wCmd) + void EnqueueInner() { switch (cmd) { From 5974533d45eeb868aea79f97566ff88ee4aa4205 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Sat, 17 May 2025 17:35:30 -0500 Subject: [PATCH 57/60] api approvals --- .../verify/CoreAPISpec.ApproveCore.DotNet.verified.txt | 6 +++--- .../verify/CoreAPISpec.ApproveCore.Net.verified.txt | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCore.DotNet.verified.txt b/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCore.DotNet.verified.txt index dd6804edcb4..36bf7bcfab8 100644 --- a/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCore.DotNet.verified.txt +++ b/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCore.DotNet.verified.txt @@ -4252,13 +4252,13 @@ namespace Akka.IO [System.ObsoleteAttribute("This property is unused")] public int InitialSocketAsyncEventArgs { get; } public string ManagementDispatcher { get; } - public long MaxFrameSizeBytes { get; set; } + public int MaxFrameSizeBytes { get; set; } public bool OutgoingSocketForceIpv4 { get; set; } - public long ReceiveBufferSize { get; set; } + public int ReceiveBufferSize { get; set; } [System.ObsoleteAttribute("This property is now MaxFrameSizeBytes")] public long ReceivedMessageSizeLimit { get; } public System.Nullable RegisterTimeout { get; set; } - public long SendBufferSize { get; set; } + public int SendBufferSize { get; set; } public bool TraceLogging { get; set; } [System.ObsoleteAttribute("This property is unused")] public int TransferToLimit { get; set; } diff --git a/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCore.Net.verified.txt b/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCore.Net.verified.txt index 03a679d93e4..7c2892fc6ce 100644 --- a/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCore.Net.verified.txt +++ b/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCore.Net.verified.txt @@ -4241,13 +4241,13 @@ namespace Akka.IO [System.ObsoleteAttribute("This property is unused")] public int InitialSocketAsyncEventArgs { get; } public string ManagementDispatcher { get; } - public long MaxFrameSizeBytes { get; set; } + public int MaxFrameSizeBytes { get; set; } public bool OutgoingSocketForceIpv4 { get; set; } - public long ReceiveBufferSize { get; set; } + public int ReceiveBufferSize { get; set; } [System.ObsoleteAttribute("This property is now MaxFrameSizeBytes")] public long ReceivedMessageSizeLimit { get; } public System.Nullable RegisterTimeout { get; set; } - public long SendBufferSize { get; set; } + public int SendBufferSize { get; set; } public bool TraceLogging { get; set; } [System.ObsoleteAttribute("This property is unused")] public int TransferToLimit { get; set; } From ad63320fc83fd17c196cef0b45b68052609126c9 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Mon, 19 May 2025 10:20:32 -0500 Subject: [PATCH 58/60] fixed `Write_before_Register_should_not_be_silently_dropped` --- src/core/Akka/IO/TcpConnection.cs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index c04e3c50d6f..1d1cbdee5bf 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -297,7 +297,17 @@ private void AwaitRegBehaviour() IssueReceive(); TrySend(); }); - Receive(Enqueue); + Receive(w => + { + var queueSizeBefore = _pendingWrites.Count; + Enqueue(w); + if(_pendingWrites.Count > queueSizeBefore) + { + // need to log a warning here about writing before registration + Log.Warning("Received Write command before Register command. " + + "It will be buffered until Register will be received (buffered write size is {0} bytes)", w.Bytes); + } + }); Receive(c => HandleClose(Sender, c.Event)); Receive(_ => { _state = _state with { ReadingSuspended = true }; }); Receive(_ => From 30e5a28bdf6b89ac8ddfaf1109a338c10b7e7a68 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Mon, 19 May 2025 11:56:04 -0500 Subject: [PATCH 59/60] Update `ConnectionSourceStageLogic` to buffer pending connections --- .../Implementation/IO/TcpStages.cs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/core/Akka.Streams/Implementation/IO/TcpStages.cs b/src/core/Akka.Streams/Implementation/IO/TcpStages.cs index db09d374806..dd2eef48325 100644 --- a/src/core/Akka.Streams/Implementation/IO/TcpStages.cs +++ b/src/core/Akka.Streams/Implementation/IO/TcpStages.cs @@ -6,7 +6,9 @@ //----------------------------------------------------------------------- using System; +using System.Collections.Generic; using System.Collections.Immutable; +using System.Linq; using System.Net; using System.Threading.Tasks; using Akka.Actor; @@ -41,6 +43,7 @@ private sealed class ConnectionSourceStageLogic : TimerGraphStageLogic, IOutHand private readonly TaskCompletionSource _bindingPromise; private readonly TaskCompletionSource _unbindPromise = new(); private bool _unbindStarted = false; + private readonly Queue _pendingConnections = new(); public ConnectionSourceStageLogic(Shape shape, ConnectionSourceStage stage, TaskCompletionSource bindingPromise) : base(shape) @@ -53,8 +56,16 @@ public ConnectionSourceStageLogic(Shape shape, ConnectionSourceStage stage, Task public void OnPull() { - // Ignore if still binding - _listener?.Tell(new Tcp.ResumeAccepting(1), StageActor.Ref); + TryPush(); + } + + private void TryPush() + { + if (!IsAvailable(_stage._out)) return; // we have demand and can push + if (_pendingConnections.Count <= 0) return; + + var toPush = _pendingConnections.Dequeue(); + Push(_stage._out, toPush); } public void OnDownstreamFinish(Exception cause) @@ -160,7 +171,8 @@ private void Receive((IActorRef, object) args) break; case Tcp.Connected connected: - Push(_stage._out, ConnectionFor(connected, sender)); + _pendingConnections.Enqueue(ConnectionFor(connected, sender)); + TryPush(); break; case Tcp.Unbind _: From e3331eeedd2c5db37a43a0d0f930222003b5e785 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Mon, 19 May 2025 14:12:43 -0500 Subject: [PATCH 60/60] fixes found during review --- src/core/Akka/IO/TcpConnection.cs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/core/Akka/IO/TcpConnection.cs b/src/core/Akka/IO/TcpConnection.cs index 1d1cbdee5bf..280c5f274d5 100644 --- a/src/core/Akka/IO/TcpConnection.cs +++ b/src/core/Akka/IO/TcpConnection.cs @@ -331,7 +331,7 @@ private void OpenBehaviour() Receive(Enqueue); Receive(c => HandleClose(Sender, c.Event)); SuspendResumeHandlers(); - Receive(h => + Receive(_ => { Log.Debug("Handler [{0}] died, stopping connection actor", _handler); Context.Stop(Self); @@ -360,6 +360,11 @@ private void PeerSentEOF() Receive(Enqueue); Receive(c => HandleClose(Sender, c.Event)); SuspendResumeHandlers(); + Receive(_ => + { + Log.Debug("Handler [{0}] died, stopping connection actor", _handler); + Context.Stop(Self); + }); } private void ClosingWithPendingWrite(IActorRef closeSender, ConnectionClosed e) @@ -377,11 +382,6 @@ private void ClosingWithPendingWrite(IActorRef closeSender, ConnectionClosed e) Receive(Enqueue); Receive(c => HandleClose(Sender, c.Event)); SuspendResumeHandlers(); - Receive(h => - { - Log.Debug("Handler [{0}] died, stopping connection actor", _handler); - Context.Stop(Self); - }); } /// @@ -576,7 +576,7 @@ private void TrySend() _pendingWrites.Dequeue(); _state = _state with { QueuedBytes = _state.QueuedBytes - w.Data.Count }; _partialWriteOffset = null; - if (w.WantsAck) _sendArgs.PendingAcks.Add((snd, w.Ack)); + if (w.WantsAck) snd.Tell(w.Ack); // message was already sent - ACK right away if (_traceLogging) Log.Debug($"[TcpConnection] TrySend: encountered empty write, dequeued. Remaining queue: {_pendingWrites.Count}"); continue; @@ -587,6 +587,7 @@ private void TrySend() if (_traceLogging) Log.Debug($"[TcpConnection] TrySend batching: offset={offset}, remaining={remaining}, toSend={toSend}, batchBytes={batchBytes}"); + // non-copying operation - just creates a new ArraySegment without copying any bytes var chunk = data.Slice(offset, toSend); segs.AddRange(chunk.Buffers); batchBytes += toSend; @@ -752,7 +753,7 @@ private void CloseSocket() /* ignore */ } - _state = _state with { OutputShutdown = true }; + _state = _state with { OutputShutdown = true, ReadingSuspended = true }; } private void AbortSocket()