Add OpenTelemetry plugin for comprehensive observability

src/Neo/IEventHandlers/IMemPoolMetricsHandler.cs

@@ -0,0 +1,42 @@
+// Copyright (C) 2015-2025 The Neo Project.
+//
+// IMemPoolMetricsHandler.cs file belongs to the neo project and is free
+// software distributed under the MIT software license, see the
+// accompanying file LICENSE in the main directory of the
+// repository or http://www.opensource.org/licenses/mit-license.php
+// for more details.
+//
+// Redistribution and use in source and binary forms with or without
+// modifications are permitted.
+
+using Neo.Ledger;
+
+namespace Neo.IEventHandlers
+{
+    /// <summary>
+    /// Interface for plugins that need to collect memory pool metrics
+    /// </summary>
+    public interface IMemPoolMetricsHandler
+    {
+        /// <summary>
+        /// Called periodically with memory pool statistics
+        /// </summary>
+        /// <param name="memPool">The memory pool instance</param>
+        /// <param name="stats">Current memory pool statistics</param>
+        void MemPool_StatsSnapshot_Handler(MemoryPool memPool, MemPoolStats stats);
+    }
+
+    /// <summary>
+    /// Memory pool statistics snapshot
+    /// </summary>
+    public class MemPoolStats
+    {
+        public int Count { get; set; }
+        public int VerifiedCount { get; set; }
+        public int UnverifiedCount { get; set; }
+        public int Capacity { get; set; }
+        public long TotalMemoryBytes { get; set; }
+        public int ConflictsCount { get; set; }
+        public int LastBatchRemovedCount { get; set; }
+    }
+}

src/Neo/IEventHandlers/INetworkMetricsHandler.cs

@@ -0,0 +1,60 @@
+// Copyright (C) 2015-2025 The Neo Project.
+//
+// INetworkMetricsHandler.cs file belongs to the neo project and is free
+// software distributed under the MIT software license, see the
+// accompanying file LICENSE in the main directory of the
+// repository or http://www.opensource.org/licenses/mit-license.php
+// for more details.
+//
+// Redistribution and use in source and binary forms with or without
+// modifications are permitted.
+
+using Akka.Actor;
+using Neo.Network.P2P;
+using System.Collections.Generic;
+
+namespace Neo.IEventHandlers
+{
+    /// <summary>
+    /// Interface for plugins that need to collect network metrics
+    /// </summary>
+    public interface INetworkMetricsHandler
+    {
+        /// <summary>
+        /// Called when a peer connects to the network
+        /// </summary>
+        /// <param name="node">The local node instance</param>
+        /// <param name="peer">The connected peer</param>
+        void Network_PeerConnected_Handler(LocalNode node, IActorRef peer);
+
+        /// <summary>
+        /// Called when a peer disconnects from the network
+        /// </summary>
+        /// <param name="node">The local node instance</param>
+        /// <param name="peer">The disconnected peer</param>
+        void Network_PeerDisconnected_Handler(LocalNode node, IActorRef peer);
+
+        /// <summary>
+        /// Called periodically with network statistics
+        /// </summary>
+        /// <param name="node">The local node instance</param>
+        /// <param name="stats">Current network statistics</param>
+        void Network_StatsSnapshot_Handler(LocalNode node, NetworkStats stats);
+    }
+
+    /// <summary>
+    /// Network statistics snapshot
+    /// </summary>
+    public class NetworkStats
+    {
+        public int ConnectedPeers { get; set; }
+        public int UnconnectedPeers { get; set; }
+        public long BytesSent { get; set; }
+        public long BytesReceived { get; set; }
+        public int PendingTasks { get; set; }
+        public int HighPriorityQueueSize { get; set; }
+        public int LowPriorityQueueSize { get; set; }
+        public Dictionary<string, long> MessagesSentByType { get; set; } = new Dictionary<string, long>();
+        public Dictionary<string, long> MessagesReceivedByType { get; set; } = new Dictionary<string, long>();
+    }
+}

src/Neo/IEventHandlers/IStorageMetricsHandler.cs

@@ -0,0 +1,61 @@
+// Copyright (C) 2015-2025 The Neo Project.
+//
+// IStorageMetricsHandler.cs file belongs to the neo project and is free
+// software distributed under the MIT software license, see the
+// accompanying file LICENSE in the main directory of the
+// repository or http://www.opensource.org/licenses/mit-license.php
+// for more details.
+//
+// Redistribution and use in source and binary forms with or without
+// modifications are permitted.
+
+using Neo.Persistence;
+using System;
+
+namespace Neo.IEventHandlers
+{
+    /// <summary>
+    /// Interface for plugins that need to collect storage metrics
+    /// </summary>
+    public interface IStorageMetricsHandler
+    {
+        /// <summary>
+        /// Called when a storage read operation occurs
+        /// </summary>
+        /// <param name="store">The store instance</param>
+        /// <param name="key">The key being read</param>
+        /// <param name="found">Whether the key was found</param>
+        /// <param name="duration">Time taken for the operation</param>
+        void Storage_Read_Handler(IStore store, byte[] key, bool found, TimeSpan duration);
+
+        /// <summary>
+        /// Called when a storage write operation occurs
+        /// </summary>
+        /// <param name="store">The store instance</param>
+        /// <param name="key">The key being written</param>
+        /// <param name="valueSize">Size of the value in bytes</param>
+        /// <param name="duration">Time taken for the operation</param>
+        void Storage_Write_Handler(IStore store, byte[] key, int valueSize, TimeSpan duration);
+
+        /// <summary>
+        /// Called periodically with storage statistics
+        /// </summary>
+        /// <param name="store">The store instance</param>
+        /// <param name="stats">Current storage statistics</param>
+        void Storage_StatsSnapshot_Handler(IStore store, StorageStats stats);
+    }
+
+    /// <summary>
+    /// Storage statistics snapshot
+    /// </summary>
+    public class StorageStats
+    {
+        public long TotalReads { get; set; }
+        public long TotalWrites { get; set; }
+        public long CacheHits { get; set; }
+        public long CacheMisses { get; set; }
+        public double CacheHitRate => TotalReads > 0 ? (double)CacheHits / TotalReads : 0;
+        public long StorageSizeBytes { get; set; }
+        public int SnapshotCount { get; set; }
+    }
+}

src/Neo/Ledger/MemoryPool.cs

@@ -11,6 +11,8 @@
 
 #nullable enable
 
+using Neo.Extensions;
+using Neo.IEventHandlers;
 using Neo.Network.P2P;
 using Neo.Network.P2P.Payloads;
 using Neo.Persistence;
@@ -32,6 +34,18 @@ public class MemoryPool : IReadOnlyCollection<Transaction>
         public event EventHandler<Transaction>? TransactionAdded;
         public event EventHandler<TransactionRemovedEventArgs>? TransactionRemoved;
 
+        // MemPool metrics event delegates
+        public delegate void MemPoolStatsSnapshotHandler(MemoryPool memPool, MemPoolStats stats);
+
+        // MemPool metrics events
+        public static event MemPoolStatsSnapshotHandler? MemPoolStatsSnapshot;
+
+        // Metrics tracking fields
+        private readonly Timer? _metricsTimer;
+        private int _lastBatchRemovedCount;
+        private int _conflictsCount;
+        private readonly object _metricsLock = new object();
+
         // Allow a reverified transaction to be rebroadcast if it has been this many block times since last broadcast.
         private const int BlocksTillRebroadcast = 10;
         private int RebroadcastMultiplierThreshold => Capacity / 10;
@@ -130,6 +144,12 @@ public MemoryPool(NeoSystem system)
             var timePerBlock = system.GetTimePerBlock().TotalMilliseconds;
             MaxMillisecondsToReverifyTx = timePerBlock / 3;
             MaxMillisecondsToReverifyTxPerIdle = timePerBlock / 15;
+
+            // Start metrics collection timer if there are handlers
+            if (MemPoolStatsSnapshot != null)
+            {
+                _metricsTimer = new Timer(CollectMemPoolStats, null, TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(5));
+            }
         }
 
         /// <summary>
@@ -400,6 +420,10 @@ private bool CheckConflicts(Transaction tx, out List<PoolItem> conflictsList)
             // Step 3: take into account sender's conflicting transactions while balance check,
             // this will be done in VerifyStateDependant.
 
+            // Track conflicts if any were found
+            if (conflictsList.Count > 0)
+                IncrementConflictsCount();
+
             return true;
         }
 
@@ -422,6 +446,10 @@ private List<Transaction> RemoveOverCapacity()
                 }
             } while (Count > Capacity);
 
+            // Track the number of transactions removed
+            if (removedTransactions.Count > 0)
+                SetLastBatchRemovedCount(removedTransactions.Count);
+
             return removedTransactions;
         }
 
@@ -699,6 +727,105 @@ internal void Clear()
                 _txRwLock.ExitReadLock();
             }
         }
+
+        // Event invocation methods
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static void InvokeMemPoolStatsSnapshot(MemoryPool memPool, MemPoolStats stats)
+        {
+            InvokeHandlers(MemPoolStatsSnapshot?.GetInvocationList(), h => ((MemPoolStatsSnapshotHandler)h)(memPool, stats));
+        }
+
+        private static void InvokeHandlers(Delegate[]? handlers, Action<Delegate> handlerAction)
+        {
+            if (handlers == null) return;
+
+            foreach (var handler in handlers)
+            {
+                try
+                {
+                    // skip stopped plugin.
+                    if (handler.Target is Neo.Plugins.Plugin { IsStopped: true })
+                    {
+                        continue;
+                    }
+
+                    handlerAction(handler);
+                }
+                catch (Exception ex) when (handler.Target is Neo.Plugins.Plugin plugin)
+                {
+                    Utility.Log(nameof(plugin), LogLevel.Error, ex);
+                    switch (plugin.ExceptionPolicy)
+                    {
+                        case Neo.Plugins.UnhandledExceptionPolicy.StopNode:
+                            throw;
+                        case Neo.Plugins.UnhandledExceptionPolicy.StopPlugin:
+                            //Stop plugin on exception
+                            plugin.IsStopped = true;
+                            break;
+                        case Neo.Plugins.UnhandledExceptionPolicy.Ignore:
+                            // Log the exception and continue with the next handler
+                            break;
+                    }
+                }
+            }
+        }
+
+        private void CollectMemPoolStats(object? state)
+        {
+            try
+            {
+                long totalMemoryBytes = 0;
+                _txRwLock.EnterReadLock();
+                try
+                {
+                    // Calculate total memory usage
+                    foreach (var tx in _unsortedTransactions.Values)
+                    {
+                        totalMemoryBytes += tx.Tx.Size;
+                    }
+                    foreach (var tx in _unverifiedTransactions.Values)
+                    {
+                        totalMemoryBytes += tx.Tx.Size;
+                    }
+                }
+                finally
+                {
+                    _txRwLock.ExitReadLock();
+                }
+
+                var stats = new MemPoolStats
+                {
+                    Count = Count,
+                    VerifiedCount = VerifiedCount,
+                    UnverifiedCount = UnVerifiedCount,
+                    Capacity = Capacity,
+                    TotalMemoryBytes = totalMemoryBytes,
+                    ConflictsCount = _conflictsCount,
+                    LastBatchRemovedCount = _lastBatchRemovedCount
+                };
+
+                InvokeMemPoolStatsSnapshot(this, stats);
+            }
+            catch (Exception ex)
+            {
+                Utility.Log(nameof(MemoryPool), LogLevel.Error, ex);
+            }
+        }
+
+        public void IncrementConflictsCount()
+        {
+            Interlocked.Increment(ref _conflictsCount);
+        }
+
+        public void SetLastBatchRemovedCount(int count)
+        {
+            Interlocked.Exchange(ref _lastBatchRemovedCount, count);
+        }
+
+        public void Dispose()
+        {
+            _metricsTimer?.Dispose();
+        }
     }
 }