2554 Enhance REST OTEL instrumentation with custom metrics and traces (#2617)

## Why make this change?

- Closes #2554
- Enhances OTEL instrumentation with custom traces and metrics for the
REST APIs

## What is this change?

This PR enhances the OTEL instrumentation for the REST APIs by adding
custom traces and metrics.

I have removed ASP NET Core standard instrumentation since it does not
provide great value given the custom nature of the webservice. I have
written two main Helper classes: `TelemetryMetricsHelper` and
`TelemetryTracesHelper` to provide a single point of management for
custom traces and metrics.

Metrics can be filtered for `status_code`, `api_type`, `endpoint` and
`method`.

I have also fixed the loggings which are now sent to the configured OTEL
endpoint.

### Logs

![image](https://github.com/user-attachments/assets/2b2b21ab-b16e-4678-8a3f-2c6bc3ab7168)

### Metrics
![Screenshot 2025-03-14
190730](https://github.com/user-attachments/assets/322f6111-8580-49c2-b6ff-5cddc191178c)

### Traces
![image
(2)](https://github.com/user-attachments/assets/1e9f320a-7445-44d6-a1c6-b0d5eb21fa8f)
![image
(3)](https://github.com/user-attachments/assets/efc754c1-a646-4500-9ebf-4f5cfba34b85)
![image
(4)](https://github.com/user-attachments/assets/cadc9665-934f-48bd-a1bc-afedbf5fe395)




## How was this tested?

- [ ] Integration Tests
- [ ] Unit Tests

## Sample Request(s)

To test everything locally I recommend using [this
repo](https://github.com/tommasodotNET/dab-workbench) that allows to run
the local build of the dab cli and send metrics to the .NET Aspire OTEL
endoint.

---------

Co-authored-by: Aaron Powell <[email protected]>
Co-authored-by: RubenCerna2079 <[email protected]>
Co-authored-by: aaronburtle <[email protected]>
Co-authored-by: Ruben Cerna <[email protected]>
Co-authored-by: Aniruddh Munde <[email protected]>

Author: 5 people

src/Core/Resolvers/SqlQueryEngine.cs

@@ -332,6 +332,7 @@ public object ResolveList(JsonElement array, IObjectField fieldSchema, ref IMeta
                     JsonElement result = await _cache.GetOrSetAsync<JsonElement>(queryExecutor, queryMetadata, cacheEntryTtl: runtimeConfig.GetEntityCacheEntryTtl(entityName: structure.EntityName));
                     byte[] jsonBytes = JsonSerializer.SerializeToUtf8Bytes(result);
                     JsonDocument cacheServiceResponse = JsonDocument.Parse(jsonBytes);
+
                     return cacheServiceResponse;
                 }
             }
@@ -348,6 +349,7 @@ public object ResolveList(JsonElement array, IObjectField fieldSchema, ref IMeta
                 httpContext: _httpContextAccessor.HttpContext!,
                 args: null,
                 dataSourceName: dataSourceName);
+
             return response;
         }
 

src/Service.Tests/Configuration/OpenTelemetryTests.cs

@@ -61,7 +61,6 @@ public void CleanUpTelemetryConfig()
             File.Delete(CONFIG_WITHOUT_TELEMETRY);
         }
 
-        Startup.OpenTelemetryOptions = new();
     }
 
     /// <summary>

src/Service.Tests/dab-config.CosmosDb_NoSql.json

@@ -31,6 +31,12 @@
         "provider": "StaticWebApps"
       },
       "mode": "development"
+    },
+    "telemetry": {
+      "app-insights": {
+        "enabled": true,
+        "connection-string": "InstrumentationKey=00000000-0000-0000-0000-000000000000;IngestionEndpoint=https://dc.services.visualstudio.com/v2/track"
+      }
     }
   },
   "entities": {
@@ -794,4 +800,4 @@
       ]
     }
   }
-}
+}

src/Service/Azure.DataApiBuilder.Service.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk.Web">
+<Project Sdk="Microsoft.NET.Sdk.Web">
 
     <PropertyGroup>
         <TargetFrameworks>net8.0</TargetFrameworks>
@@ -77,7 +77,7 @@
         <PackageReference Include="System.CommandLine" />
         <PackageReference Include="System.IO.Abstractions" />
         <PackageReference Include="ZiggyCreatures.FusionCache" />
-        <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol"  />
+        <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" />
         <PackageReference Include="OpenTelemetry.Extensions.Hosting" />
         <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" />
         <PackageReference Include="OpenTelemetry.Instrumentation.Http" />

src/Service/Controllers/RestController.cs

@@ -2,15 +2,20 @@
 // Licensed under the MIT License.
 
 using System;
+using System.Diagnostics;
+using System.Linq;
 using System.Net;
 using System.Net.Mime;
 using System.Threading.Tasks;
 using Azure.DataApiBuilder.Config.ObjectModel;
+using Azure.DataApiBuilder.Core.Configurations;
 using Azure.DataApiBuilder.Core.Models;
 using Azure.DataApiBuilder.Core.Services;
 using Azure.DataApiBuilder.Service.Exceptions;
+using Azure.DataApiBuilder.Service.Telemetry;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;
 using Microsoft.Extensions.Logging;
 
 namespace Azure.DataApiBuilder.Service.Controllers
@@ -47,11 +52,14 @@ public class RestController : ControllerBase
 
         private readonly ILogger<RestController> _logger;
 
+        private readonly RuntimeConfigProvider _runtimeConfigProvider;
+
         /// <summary>
         /// Constructor.
         /// </summary>
-        public RestController(RestService restService, IOpenApiDocumentor openApiDocumentor, ILogger<RestController> logger)
+        public RestController(RuntimeConfigProvider runtimeConfigProvider, RestService restService, IOpenApiDocumentor openApiDocumentor, ILogger<RestController> logger)
         {
+            _runtimeConfigProvider = runtimeConfigProvider;
             _restService = restService;
             _openApiDocumentor = openApiDocumentor;
             _logger = logger;
@@ -185,11 +193,29 @@ private async Task<IActionResult> HandleOperation(
             string route,
             EntityActionOperation operationType)
         {
+            if (route.Equals(REDIRECTED_ROUTE))
+            {
+                return NotFound();
+            }
+
+            Stopwatch stopwatch = Stopwatch.StartNew();
+            // This activity tracks the entire REST request.
+            using Activity? activity = TelemetryTracesHelper.DABActivitySource.StartActivity($"{HttpContext.Request.Method} {(route.Split('/').Length > 1 ? route.Split('/')[1] : string.Empty)}");
+
             try
             {
-                if (route.Equals(REDIRECTED_ROUTE))
+                TelemetryMetricsHelper.IncrementActiveRequests(ApiType.REST);
+
+                if (activity is not null)
                 {
-                    return NotFound();
+                    activity.TrackRestControllerActivityStarted(
+                        Enum.Parse<HttpMethod>(HttpContext.Request.Method, ignoreCase: true),
+                        HttpContext.Request.Headers["User-Agent"].ToString(),
+                        operationType.ToString(),
+                        route,
+                        HttpContext.Request.QueryString.ToString(),
+                        HttpContext.Request.Headers["X-MS-API-ROLE"].FirstOrDefault() ?? HttpContext.User.FindFirst("role")?.Value,
+                        ApiType.REST);
                 }
 
                 // Validate the PathBase matches the configured REST path.
@@ -208,8 +234,21 @@ private async Task<IActionResult> HandleOperation(
 
                 (string entityName, string primaryKeyRoute) = _restService.GetEntityNameAndPrimaryKeyRouteFromRoute(routeAfterPathBase);
 
+                // This activity tracks the query execution. This will create a new activity nested under the REST request activity.
+                using Activity? queryActivity = TelemetryTracesHelper.DABActivitySource.StartActivity($"QUERY {entityName}");
                 IActionResult? result = await _restService.ExecuteAsync(entityName, operationType, primaryKeyRoute);
 
+                RuntimeConfig runtimeConfig = _runtimeConfigProvider.GetConfig();
+                string dataSourceName = runtimeConfig.GetDataSourceNameFromEntityName(entityName);
+                DatabaseType databaseType = runtimeConfig.GetDataSourceFromDataSourceName(dataSourceName).DatabaseType;
+
+                if (queryActivity is not null)
+                {
+                    queryActivity.TrackQueryActivityStarted(
+                        databaseType,
+                        dataSourceName);
+                }
+
                 if (result is null)
                 {
                     throw new DataApiBuilderException(
@@ -218,6 +257,13 @@ private async Task<IActionResult> HandleOperation(
                         subStatusCode: DataApiBuilderException.SubStatusCodes.EntityNotFound);
                 }
 
+                int statusCode = (result as ObjectResult)?.StatusCode ?? (result as StatusCodeResult)?.StatusCode ?? (result as JsonResult)?.StatusCode ?? 200;
+                if (activity is not null && activity.IsAllDataRequested)
+                {
+                    HttpStatusCode httpStatusCode = Enum.Parse<HttpStatusCode>(statusCode.ToString(), ignoreCase: true);
+                    activity.TrackRestControllerActivityFinished(httpStatusCode);
+                }
+
                 return result;
             }
             catch (DataApiBuilderException ex)
@@ -228,6 +274,10 @@ private async Task<IActionResult> HandleOperation(
                     HttpContextExtensions.GetLoggerCorrelationId(HttpContext));
 
                 Response.StatusCode = (int)ex.StatusCode;
+                activity?.TrackRestControllerActivityFinishedWithException(ex, ex.StatusCode);
+
+                HttpMethod method = Enum.Parse<HttpMethod>(HttpContext.Request.Method, ignoreCase: true);
+                TelemetryMetricsHelper.TrackError(method, ex.StatusCode, route, ApiType.REST, ex);
                 return ErrorResponse(ex.SubStatusCode.ToString(), ex.Message, ex.StatusCode);
             }
             catch (Exception ex)
@@ -238,11 +288,26 @@ private async Task<IActionResult> HandleOperation(
                     HttpContextExtensions.GetLoggerCorrelationId(HttpContext));
 
                 Response.StatusCode = (int)HttpStatusCode.InternalServerError;
+
+                HttpMethod method = Enum.Parse<HttpMethod>(HttpContext.Request.Method, ignoreCase: true);
+                activity?.TrackRestControllerActivityFinishedWithException(ex, HttpStatusCode.InternalServerError);
+
+                TelemetryMetricsHelper.TrackError(method, HttpStatusCode.InternalServerError, route, ApiType.REST, ex);
                 return ErrorResponse(
                     DataApiBuilderException.SubStatusCodes.UnexpectedError.ToString(),
                     SERVER_ERROR,
                     HttpStatusCode.InternalServerError);
             }
+            finally
+            {
+                stopwatch.Stop();
+                HttpMethod method = Enum.Parse<HttpMethod>(HttpContext.Request.Method, ignoreCase: true);
+                HttpStatusCode httpStatusCode = Enum.Parse<HttpStatusCode>(Response.StatusCode.ToString(), ignoreCase: true);
+                TelemetryMetricsHelper.TrackRequest(method, httpStatusCode, route, ApiType.REST);
+                TelemetryMetricsHelper.TrackRequestDuration(method, httpStatusCode, route, ApiType.REST, stopwatch.Elapsed);
+
+                TelemetryMetricsHelper.DecrementActiveRequests(ApiType.REST);
+            }
         }
 
         /// <summary>

src/Service/Startup.cs

@@ -61,9 +61,9 @@ public class Startup
         public static LogLevel MinimumLogLevel = LogLevel.Error;
 
         public static bool IsLogLevelOverriddenByCli;
-        public static OpenTelemetryOptions OpenTelemetryOptions = new();
 
         public static ApplicationInsightsOptions AppInsightsOptions = new();
+        public static OpenTelemetryOptions OpenTelemetryOptions = new();
         public const string NO_HTTPS_REDIRECT_FLAG = "--no-https-redirect";
         private HotReloadEventHandler<HotReloadEventArgs> _hotReloadEventHandler = new();
         private RuntimeConfigProvider? _configProvider;
@@ -119,31 +119,46 @@ public void ConfigureServices(IServiceCollection services)
                 && runtimeConfig?.Runtime?.Telemetry?.OpenTelemetry is not null
                 && runtimeConfig.Runtime.Telemetry.OpenTelemetry.Enabled)
             {
+                services.Configure<OpenTelemetryLoggerOptions>(options =>
+                {
+                    options.IncludeScopes = true;
+                    options.ParseStateValues = true;
+                    options.IncludeFormattedMessage = true;
+                });
                 services.AddOpenTelemetry()
+                .WithLogging(logging =>
+                {
+                    logging.SetResourceBuilder(ResourceBuilder.CreateDefault().AddService(runtimeConfig.Runtime.Telemetry.OpenTelemetry.ServiceName!))
+                    .AddOtlpExporter(configure =>
+                    {
+                        configure.Endpoint = new Uri(runtimeConfig.Runtime.Telemetry.OpenTelemetry.Endpoint!);
+                        configure.Headers = runtimeConfig.Runtime.Telemetry.OpenTelemetry.Headers;
+                        configure.Protocol = OtlpExportProtocol.Grpc;
+                    });
+
+                })
                 .WithMetrics(metrics =>
                 {
                     metrics.SetResourceBuilder(ResourceBuilder.CreateDefault().AddService(runtimeConfig.Runtime.Telemetry.OpenTelemetry.ServiceName!))
-                        .AddAspNetCoreInstrumentation()
-                        .AddHttpClientInstrumentation()
                         .AddOtlpExporter(configure =>
                         {
                             configure.Endpoint = new Uri(runtimeConfig.Runtime.Telemetry.OpenTelemetry.Endpoint!);
                             configure.Headers = runtimeConfig.Runtime.Telemetry.OpenTelemetry.Headers;
                             configure.Protocol = OtlpExportProtocol.Grpc;
                         })
-                        .AddRuntimeInstrumentation();
+                        .AddMeter(TelemetryMetricsHelper.MeterName);
                 })
                 .WithTracing(tracing =>
                 {
                     tracing.SetResourceBuilder(ResourceBuilder.CreateDefault().AddService(runtimeConfig.Runtime.Telemetry.OpenTelemetry.ServiceName!))
-                        .AddAspNetCoreInstrumentation()
                         .AddHttpClientInstrumentation()
                         .AddOtlpExporter(configure =>
                         {
                             configure.Endpoint = new Uri(runtimeConfig.Runtime.Telemetry.OpenTelemetry.Endpoint!);
                             configure.Headers = runtimeConfig.Runtime.Telemetry.OpenTelemetry.Headers;
                             configure.Protocol = OtlpExportProtocol.Grpc;
-                        });
+                        })
+                        .AddSource(TelemetryTracesHelper.DABActivitySource.Name);
                 });
             }
 
@@ -395,6 +410,7 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env, RuntimeC
             {
                 // Configure Application Insights Telemetry
                 ConfigureApplicationInsightsTelemetry(app, runtimeConfig);
+                ConfigureOpenTelemetry(runtimeConfig);
 
                 // Config provided before starting the engine.
                 isRuntimeReady = PerformOnConfigChangeAsync(app).Result;
@@ -712,6 +728,37 @@ private void ConfigureApplicationInsightsTelemetry(IApplicationBuilder app, Runt
             }
         }
 
+        /// <summary>
+        /// Configure Open Telemetry based on the loaded runtime configuration. If Open Telemetry
+        /// is enabled, we can track different events and metrics.
+        /// </summary>
+        /// <param name="runtimeConfigurationProvider">The provider used to load runtime configuration.</param>
+        /// <seealso cref="https://docs.microsoft.com/en-us/azure/azure-monitor/app/asp-net-core#enable-application-insights-telemetry-collection"/>
+        private void ConfigureOpenTelemetry(RuntimeConfig runtimeConfig)
+        {
+            if (runtimeConfig?.Runtime?.Telemetry is not null
+                && runtimeConfig.Runtime.Telemetry.OpenTelemetry is not null)
+            {
+                OpenTelemetryOptions = runtimeConfig.Runtime.Telemetry.OpenTelemetry;
+
+                if (!OpenTelemetryOptions.Enabled)
+                {
+                    _logger.LogInformation("Open Telemetry are disabled.");
+                    return;
+                }
+
+                if (string.IsNullOrWhiteSpace(OpenTelemetryOptions?.Endpoint))
+                {
+                    _logger.LogWarning("Logs won't be sent to Open Telemetry because an Open Telemetry connection string is not available in the runtime config.");
+                    return;
+                }
+
+                // Updating Startup Logger to Log from Startup Class.
+                ILoggerFactory? loggerFactory = Program.GetLoggerFactoryForLogLevel(MinimumLogLevel);
+                _logger = loggerFactory.CreateLogger<Startup>();
+            }
+        }
+
         /// <summary>
         /// Sets Static Web Apps EasyAuth as the authentication scheme for the engine.
         /// </summary>

src/Service/Telemetry/TelemetryMetricsHelper.cs

@@ -0,0 +1,81 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.Metrics;
+using System.Net;
+using Azure.DataApiBuilder.Config.ObjectModel;
+using Kestral = Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http.HttpMethod;
+
+namespace Azure.DataApiBuilder.Service.Telemetry
+{
+    /// <summary>
+    /// Helper class for tracking telemetry metrics such as active requests, errors, total requests,
+    /// and request durations using the .NET Meter and Counter APIs.
+    /// </summary>
+    public static class TelemetryMetricsHelper
+    {
+        public static readonly string MeterName = "DataApiBuilder.Metrics";
+        private static readonly Meter _meter = new(MeterName);
+        private static readonly UpDownCounter<long> _activeRequests = _meter.CreateUpDownCounter<long>("active_requests");
+        private static readonly Counter<long> _errorCounter = _meter.CreateCounter<long>("total_errors");
+        private static readonly Counter<long> _totalRequests = _meter.CreateCounter<long>("total_requests");
+        private static readonly Histogram<double> _requestDuration = _meter.CreateHistogram<double>("request_duration", "ms");
+
+        public static void IncrementActiveRequests(ApiType kind) => _activeRequests.Add(1, new KeyValuePair<string, object?>("api_type", kind));
+
+        public static void DecrementActiveRequests(ApiType kind) => _activeRequests.Add(-1, new KeyValuePair<string, object?>("api_type", kind));
+
+        /// <summary>
+        /// Tracks a request by incrementing the total requests counter and associating it with metadata.
+        /// </summary>
+        /// <param name="method">The HTTP method of the request (e.g., GET, POST).</param>
+        /// <param name="statusCode">The HTTP status code of the response.</param>
+        /// <param name="endpoint">The endpoint being accessed.</param>
+        /// <param name="apiType">The type of API being used (e.g., REST, GraphQL).</param>
+        public static void TrackRequest(Kestral method, HttpStatusCode statusCode, string endpoint, ApiType apiType)
+        {
+            _totalRequests.Add(1,
+                new("method", method),
+                new("status_code", statusCode),
+                new("endpoint", endpoint),
+                new("api_type", apiType));
+        }
+
+        /// <summary>
+        /// Tracks an error by incrementing the error counter and associating it with metadata.
+        /// </summary>
+        /// <param name="method">The HTTP method of the request (e.g., GET, POST).</param>
+        /// <param name="statusCode">The HTTP status code of the response.</param>
+        /// <param name="endpoint">The endpoint being accessed.</param>
+        /// <param name="apiType">The type of API being used (e.g., REST, GraphQL).</param>
+        /// <param name="ex">The exception that occurred.</param>
+        public static void TrackError(Kestral method, HttpStatusCode statusCode, string endpoint, ApiType apiType, Exception ex)
+        {
+            _errorCounter.Add(1,
+                new("method", method),
+                new("status_code", statusCode),
+                new("endpoint", endpoint),
+                new("api_type", apiType),
+                new("error_type", ex.GetType().Name));
+        }
+
+        /// <summary>
+        /// Tracks the duration of a request by recording it in a histogram and associating it with metadata.
+        /// </summary>
+        /// <param name="method">The HTTP method of the request (e.g., GET, POST).</param>
+        /// <param name="statusCode">The HTTP status code of the response.</param>
+        /// <param name="endpoint">The endpoint being accessed.</param>
+        /// <param name="apiType">The type of API being used (e.g., REST, GraphQL).</param>
+        /// <param name="duration">The duration of the request in milliseconds.</param>
+        public static void TrackRequestDuration(Kestral method, HttpStatusCode statusCode, string endpoint, ApiType apiType, TimeSpan duration)
+        {
+            _requestDuration.Record(duration.TotalMilliseconds,
+                new("method", method),
+                new("status_code", statusCode),
+                new("endpoint", endpoint),
+                new("api_type", apiType));
+        }
+    }
+}