Add OpenTelemetry plugin for comprehensive observability

src/Plugins/OTelPlugin/AdditionalMetrics.cs

@@ -0,0 +1,67 @@
+// Copyright (C) 2015-2025 The Neo Project.
+//
+// AdditionalMetrics.cs file belongs to the neo project and is free
+// software distributed under the MIT software license, see the
+// accompanying file LICENSE in the main directory of the
+// repository or http://www.opensource.org/licenses/mit-license.php
+// for more details.
+//
+// Redistribution and use in source and binary forms with or without
+// modifications are permitted.
+
+namespace Neo.Plugins.OpenTelemetry
+{
+    /// <summary>
+    /// Additional critical metrics for comprehensive observability
+    /// </summary>
+    public static class AdditionalMetricNames
+    {
+        // Consensus metrics
+        public const string ConsensusRound = "neo.consensus.round";
+        public const string ConsensusView = "neo.consensus.view";
+        public const string ConsensusState = "neo.consensus.state";
+        public const string ConsensusMessagesSent = "neo.consensus.messages_sent_total";
+        public const string ConsensusMessagesReceived = "neo.consensus.messages_received_total";
+        public const string ConsensusTimeToFinality = "neo.consensus.time_to_finality";
+
+        // Storage metrics
+        public const string StorageReadLatency = "neo.storage.read_latency";
+        public const string StorageWriteLatency = "neo.storage.write_latency";
+        public const string StorageSize = "neo.storage.size_bytes";
+        public const string StorageReadTotal = "neo.storage.reads_total";
+        public const string StorageWriteTotal = "neo.storage.writes_total";
+
+        // Contract execution metrics
+        public const string ContractExecutionTime = "neo.contract.execution_time";
+        public const string ContractGasConsumed = "neo.contract.gas_consumed";
+        public const string ContractExecutionErrors = "neo.contract.execution_errors_total";
+        public const string ContractDeployments = "neo.contract.deployments_total";
+
+        // Transaction pool metrics
+        public const string TransactionPoolAddLatency = "neo.transaction_pool.add_latency";
+        public const string TransactionPoolRemoveLatency = "neo.transaction_pool.remove_latency";
+        public const string TransactionPoolRejections = "neo.transaction_pool.rejections_total";
+        public const string TransactionPoolEvictions = "neo.transaction_pool.evictions_total";
+
+        // State metrics
+        public const string StateRootHeight = "neo.state.root_height";
+        public const string StateValidations = "neo.state.validations_total";
+        public const string StateValidationErrors = "neo.state.validation_errors_total";
+
+        // P2P detailed metrics
+        public const string P2PMessageLatency = "neo.p2p.message_latency";
+        public const string P2PPeerLatency = "neo.p2p.peer_latency";
+        public const string P2PPeerQuality = "neo.p2p.peer_quality";
+        public const string P2PBannedPeers = "neo.p2p.banned_peers";
+
+        // Health and readiness
+        public const string NodeHealthScore = "neo.node.health_score";
+        public const string NodeReadiness = "neo.node.readiness";
+        public const string NodeLastActivity = "neo.node.last_activity";
+
+        // Resource utilization
+        public const string FileDescriptors = "process.file_descriptors";
+        public const string OpenConnections = "process.open_connections";
+        public const string GoroutineCount = "process.goroutines"; // For compatibility with other systems
+    }
+}

src/Plugins/OTelPlugin/BlockchainMetrics.cs

@@ -0,0 +1,26 @@
+// Copyright (C) 2015-2025 The Neo Project.
+//
+// BlockchainMetrics.cs file belongs to the neo project and is free
+// software distributed under the MIT software license, see the
+// accompanying file LICENSE in the main directory of the
+// repository or http://www.opensource.org/licenses/mit-license.php
+// for more details.
+//
+// Redistribution and use in source and binary forms with or without
+// modifications are permitted.
+
+using System;
+
+namespace Neo.Plugins.OpenTelemetry
+{
+    /// <summary>
+    /// Blockchain metrics snapshot
+    /// </summary>
+    public class BlockchainMetrics
+    {
+        public DateTime Timestamp { get; set; }
+        public uint CurrentHeight { get; set; }
+        public bool IsSyncing { get; set; }
+        public int NetworkId { get; set; }
+    }
+}

src/Plugins/OTelPlugin/MemPoolMetrics.cs

@@ -0,0 +1,29 @@
+// Copyright (C) 2015-2025 The Neo Project.
+//
+// MemPoolMetrics.cs file belongs to the neo project and is free
+// software distributed under the MIT software license, see the
+// accompanying file LICENSE in the main directory of the
+// repository or http://www.opensource.org/licenses/mit-license.php
+// for more details.
+//
+// Redistribution and use in source and binary forms with or without
+// modifications are permitted.
+
+using System;
+
+namespace Neo.Plugins.OpenTelemetry
+{
+    /// <summary>
+    /// Memory pool metrics snapshot
+    /// </summary>
+    public class MemPoolMetrics
+    {
+        public DateTime Timestamp { get; set; }
+        public int Count { get; set; }
+        public int VerifiedCount { get; set; }
+        public int UnverifiedCount { get; set; }
+        public int Capacity { get; set; }
+        public double CapacityRatio { get; set; }
+        public long EstimatedMemoryBytes { get; set; }
+    }
+}

src/Plugins/OTelPlugin/MetricNames.cs

@@ -0,0 +1,66 @@
+// Copyright (C) 2015-2025 The Neo Project.
+//
+// MetricNames.cs file belongs to the neo project and is free
+// software distributed under the MIT software license, see the
+// accompanying file LICENSE in the main directory of the
+// repository or http://www.opensource.org/licenses/mit-license.php
+// for more details.
+//
+// Redistribution and use in source and binary forms with or without
+// modifications are permitted.
+
+namespace Neo.Plugins.OpenTelemetry
+{
+    /// <summary>
+    /// Constants for metric names used throughout the plugin
+    /// </summary>
+    public static class MetricNames
+    {
+        // Blockchain metrics
+        public const string BlocksProcessedTotal = "neo.blocks.processed_total";
+        public const string TransactionsProcessedTotal = "neo.transactions.processed_total";
+        public const string ContractInvocationsTotal = "neo.contracts.invocations_total";
+        public const string BlockProcessingTime = "neo.block.processing_time";
+        public const string BlockchainHeight = "neo.blockchain.height";
+        public const string BlockProcessingRate = "neo.block.processing_rate";
+        public const string IsSyncing = "neo.blockchain.is_syncing";
+
+        // MemPool metrics
+        public const string MempoolSize = "neo.mempool.size";
+        public const string MempoolVerifiedCount = "neo.mempool.verified_count";
+        public const string MempoolUnverifiedCount = "neo.mempool.unverified_count";
+        public const string MempoolMemoryBytes = "neo.mempool.memory_bytes";
+        public const string MempoolConflictsTotal = "neo.mempool.conflicts_total";
+        public const string MempoolBatchRemovedSize = "neo.mempool.batch_removed_size";
+        public const string MempoolCapacityRatio = "neo.mempool.capacity_ratio";
+
+        // Network/P2P metrics
+        public const string P2PConnectedPeers = "neo.p2p.connected_peers";
+        public const string P2PUnconnectedPeers = "neo.p2p.unconnected_peers";
+        public const string P2PPeerConnectedTotal = "neo.p2p.peer_connected_total";
+        public const string P2PPeerDisconnectedTotal = "neo.p2p.peer_disconnected_total";
+        public const string P2PBytesSentTotal = "neo.p2p.bytes_sent_total";
+        public const string P2PBytesReceivedTotal = "neo.p2p.bytes_received_total";
+        public const string P2PMessagesReceivedTotal = "neo.p2p.messages_received_total";
+        public const string P2PMessagesSentTotal = "neo.p2p.messages_sent_total";
+
+        // Transaction metrics
+        public const string TransactionVerificationTime = "neo.transaction.verification_time";
+        public const string TransactionVerificationFailuresTotal = "neo.transaction.verification_failures_total";
+
+        // Error metrics
+        public const string ProtocolErrorsTotal = "neo.protocol.errors_total";
+        public const string NetworkErrorsTotal = "neo.network.errors_total";
+        public const string StorageErrorsTotal = "neo.storage.errors_total";
+
+        // System metrics
+        public const string ProcessCpuUsage = "process.cpu.usage";
+        public const string SystemCpuUsage = "system.cpu.usage";
+        public const string ProcessMemoryWorkingSet = "process.memory.working_set";
+        public const string ProcessMemoryVirtual = "process.memory.virtual";
+        public const string DotnetGcHeapSize = "dotnet.gc.heap_size";
+        public const string ProcessThreadCount = "process.thread_count";
+        public const string NodeStartTime = "neo.node.start_time";
+        public const string NetworkId = "neo.network.id";
+    }
+}

src/Plugins/OTelPlugin/MetricsCollector.cs

@@ -0,0 +1,166 @@
+// Copyright (C) 2015-2025 The Neo Project.
+//
+// MetricsCollector.cs file belongs to the neo project and is free
+// software distributed under the MIT software license, see the
+// accompanying file LICENSE in the main directory of the
+// repository or http://www.opensource.org/licenses/mit-license.php
+// for more details.
+//
+// Redistribution and use in source and binary forms with or without
+// modifications are permitted.
+
+using Neo.ConsoleService;
+using Neo.Ledger;
+using Neo.Network.P2P;
+using Neo.SmartContract.Native;
+using System;
+using System.Threading;
+
+namespace Neo.Plugins.OpenTelemetry
+{
+    /// <summary>
+    /// Collects metrics from Neo core classes without modifying them.
+    /// This class reads publicly exposed information only.
+    /// </summary>
+    public class MetricsCollector : IDisposable
+    {
+        private readonly NeoSystem _neoSystem;
+        private readonly Timer _collectionTimer;
+        private readonly object _lock = new object();
+
+        // Metrics snapshots
+        public NetworkMetrics LastNetworkMetrics { get; private set; }
+        public MemPoolMetrics LastMemPoolMetrics { get; private set; }
+        public BlockchainMetrics LastBlockchainMetrics { get; private set; }
+
+        // Events for metrics updates
+        public event Action<NetworkMetrics>? NetworkMetricsUpdated;
+        public event Action<MemPoolMetrics>? MemPoolMetricsUpdated;
+        public event Action<BlockchainMetrics>? BlockchainMetricsUpdated;
+
+        public MetricsCollector(NeoSystem neoSystem, TimeSpan collectionInterval)
+        {
+            _neoSystem = neoSystem ?? throw new ArgumentNullException(nameof(neoSystem));
+
+            // Initialize with empty metrics
+            LastNetworkMetrics = new NetworkMetrics();
+            LastMemPoolMetrics = new MemPoolMetrics();
+            LastBlockchainMetrics = new BlockchainMetrics();
+
+            // Start periodic collection
+            _collectionTimer = new Timer(
+                CollectMetrics,
+                null,
+                TimeSpan.Zero,
+                collectionInterval);
+        }
+
+        private void CollectMetrics(object? state)
+        {
+            try
+            {
+                CollectNetworkMetrics();
+                CollectMemPoolMetrics();
+                CollectBlockchainMetrics();
+            }
+            catch (Exception ex)
+            {
+                // Log error but don't crash
+                ConsoleHelper.Error($"Error collecting metrics: {ex.Message}");
+            }
+        }
+
+        private void CollectNetworkMetrics()
+        {
+            if (_neoSystem?.LocalNode is LocalNode localNode)
+            {
+                var metrics = new NetworkMetrics
+                {
+                    Timestamp = DateTime.UtcNow,
+                    ConnectedPeers = localNode.ConnectedCount,
+                    UnconnectedPeers = localNode.UnconnectedCount
+                };
+
+                lock (_lock)
+                {
+                    LastNetworkMetrics = metrics;
+                }
+
+                NetworkMetricsUpdated?.Invoke(metrics);
+            }
+        }
+
+        private void CollectMemPoolMetrics()
+        {
+            var memPool = _neoSystem?.MemPool;
+            if (memPool != null)
+            {
+                var metrics = new MemPoolMetrics
+                {
+                    Timestamp = DateTime.UtcNow,
+                    Count = memPool.Count,
+                    VerifiedCount = memPool.VerifiedCount,
+                    UnverifiedCount = memPool.UnVerifiedCount,
+                    Capacity = memPool.Capacity
+                };
+
+                // Calculate derived metrics
+                metrics.CapacityRatio = metrics.Capacity > 0
+                    ? (double)metrics.Count / metrics.Capacity
+                    : 0;
+
+                // Estimate memory usage (approximate based on average tx size)
+                // This is an estimation since we can't access internal transaction data
+                const int AverageTxSize = 250; // bytes, approximate
+                metrics.EstimatedMemoryBytes = metrics.Count * AverageTxSize;
+
+                lock (_lock)
+                {
+                    LastMemPoolMetrics = metrics;
+                }
+
+                MemPoolMetricsUpdated?.Invoke(metrics);
+            }
+        }
+
+        private void CollectBlockchainMetrics()
+        {
+            try
+            {
+                using var snapshot = _neoSystem?.GetSnapshotCache();
+                if (snapshot != null)
+                {
+                    var metrics = new BlockchainMetrics
+                    {
+                        Timestamp = DateTime.UtcNow,
+                        CurrentHeight = NativeContract.Ledger.CurrentIndex(snapshot)
+                    };
+
+                    // Check if syncing (simplified check)
+                    var headerHeight = _neoSystem?.HeaderCache?.Count > 0
+                        ? _neoSystem.HeaderCache.Last?.Index ?? metrics.CurrentHeight
+                        : metrics.CurrentHeight;
+
+                    metrics.IsSyncing = headerHeight - metrics.CurrentHeight > 10;
+                    metrics.NetworkId = (int)(_neoSystem?.Settings.Network ?? 0);
+
+                    lock (_lock)
+                    {
+                        LastBlockchainMetrics = metrics;
+                    }
+
+                    BlockchainMetricsUpdated?.Invoke(metrics);
+                }
+            }
+            catch (Exception ex)
+            {
+                ConsoleHelper.Warning($"Error collecting blockchain metrics: {ex.Message}");
+            }
+        }
+
+        public void Dispose()
+        {
+            _collectionTimer?.Dispose();
+        }
+    }
+}

src/Plugins/OTelPlugin/MetricsSettings.cs

@@ -0,0 +1,31 @@
+// Copyright (C) 2015-2025 The Neo Project.
+//
+// MetricsSettings.cs file belongs to the neo project and is free
+// software distributed under the MIT software license, see the
+// accompanying file LICENSE in the main directory of the
+// repository or http://www.opensource.org/licenses/mit-license.php
+// for more details.
+//
+// Redistribution and use in source and binary forms with or without
+// modifications are permitted.
+
+using Microsoft.Extensions.Configuration;
+
+namespace Neo.Plugins.OpenTelemetry
+{
+    public class MetricsSettings
+    {
+        public bool Enabled { get; init; }
+        public int Interval { get; init; }
+        public PrometheusExporterSettings PrometheusExporter { get; init; }
+        public ConsoleExporterSettings ConsoleExporter { get; init; }
+
+        public MetricsSettings(IConfigurationSection section)
+        {
+            Enabled = section.GetValue("Enabled", true);
+            Interval = section.GetValue("Interval", OTelConstants.DefaultMetricsInterval);
+            PrometheusExporter = new PrometheusExporterSettings(section.GetSection("PrometheusExporter"));
+            ConsoleExporter = new ConsoleExporterSettings(section.GetSection("ConsoleExporter"));
+        }
+    }
+}