[otlp] Add experimental feature flag for enabling retries

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExperimentalOptions.cs

@@ -16,6 +16,8 @@ internal sealed class ExperimentalOptions
 
     public const string EmitLogEventEnvVar = "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EVENT_LOG_ATTRIBUTES";
 
+    public const string EnableInMemoryRetryEnvVar = "OTEL_DOTNET_EXPERIMENTAL_OTLP_ENABLE_RETRIES";
+
     public ExperimentalOptions()
         : this(new ConfigurationBuilder().AddEnvironmentVariables().Build())
     {
@@ -27,10 +29,20 @@ public ExperimentalOptions(IConfiguration configuration)
         {
             this.EmitLogEventAttributes = emitLogEventAttributes;
         }
+
+        if (configuration.TryGetBoolValue(EnableInMemoryRetryEnvVar, out var enableInMemoryRetry))
+        {
+            this.EnableInMemoryRetry = enableInMemoryRetry;
+        }
     }
 
     /// <summary>
     /// Gets or sets a value indicating whether log event attributes should be exported.
     /// </summary>
     public bool EmitLogEventAttributes { get; set; } = false;
+
+    /// <summary>
+    /// Gets or sets a value indicating whether retries should be enabled in case of transient errors.
+    /// </summary>
+    public bool EnableInMemoryRetry { get; set; } = false;
 }

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpExporterOptionsExtensions.cs

@@ -88,7 +88,7 @@ public static THeaders GetHeaders<THeaders>(this OtlpExporterOptions options, Ac
         return headers;
     }
 
-    public static OtlpExporterTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest> GetTraceExportTransmissionHandler(this OtlpExporterOptions options)
+    public static OtlpExporterTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest> GetTraceExportTransmissionHandler(this OtlpExporterOptions options, bool enableRetry = false)
     {
         var exportClient = GetTraceExportClient(options);
 
@@ -99,10 +99,12 @@ public static THeaders GetHeaders<THeaders>(this OtlpExporterOptions options, Ac
             ? httpTraceExportClient.HttpClient.Timeout.TotalMilliseconds
             : options.TimeoutMilliseconds;
 
-        return new OtlpExporterTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest>(exportClient, timeoutMilliseconds);
+        return enableRetry
+            ? new OtlpExporterRetryTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest>(exportClient, timeoutMilliseconds)
+            : new OtlpExporterTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest>(exportClient, timeoutMilliseconds);
     }
 
-    public static OtlpExporterTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest> GetMetricsExportTransmissionHandler(this OtlpExporterOptions options)
+    public static OtlpExporterTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest> GetMetricsExportTransmissionHandler(this OtlpExporterOptions options, bool enableRetry = false)
     {
         var exportClient = GetMetricsExportClient(options);
 
@@ -113,21 +115,21 @@ public static THeaders GetHeaders<THeaders>(this OtlpExporterOptions options, Ac
             ? httpMetricsExportClient.HttpClient.Timeout.TotalMilliseconds
             : options.TimeoutMilliseconds;
 
-        return new OtlpExporterTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest>(exportClient, timeoutMilliseconds);
+        return enableRetry
+            ? new OtlpExporterRetryTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest>(exportClient, timeoutMilliseconds)
+            : new OtlpExporterTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest>(exportClient, timeoutMilliseconds);
     }
 
-    public static OtlpExporterTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest> GetLogsExportTransmissionHandler(this OtlpExporterOptions options)
+    public static OtlpExporterTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest> GetLogsExportTransmissionHandler(this OtlpExporterOptions options, bool enableRetry = false)
     {
         var exportClient = GetLogExportClient(options);
-
-        // `HttpClient.Timeout.TotalMilliseconds` would be populated with the correct timeout value for both the exporter configuration cases:
-        // 1. User provides their own HttpClient. This case is straightforward as the user wants to use their `HttpClient` and thereby the same client's timeout value.
-        // 2. If the user configures timeout via the exporter options, then the timeout set for the `HttpClient` initialized by the exporter will be set to user provided value.
         double timeoutMilliseconds = exportClient is OtlpHttpLogExportClient httpLogExportClient
             ? httpLogExportClient.HttpClient.Timeout.TotalMilliseconds
             : options.TimeoutMilliseconds;
 
-        return new OtlpExporterTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest>(exportClient, timeoutMilliseconds);
+        return enableRetry
+            ? new OtlpExporterRetryTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest>(exportClient, timeoutMilliseconds)
+            : new OtlpExporterTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest>(exportClient, timeoutMilliseconds);
     }
 
     public static IExportClient<TraceOtlpCollector.ExportTraceServiceRequest> GetTraceExportClient(this OtlpExporterOptions options) =>

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpLogExporter.cs

@@ -62,7 +62,7 @@ internal OtlpLogExporter(
             OpenTelemetryProtocolExporterEventSource.Log.InvalidEnvironmentVariable(key, value);
         };
 
-        this.transmissionHandler = transmissionHandler ?? exporterOptions.GetLogsExportTransmissionHandler();
+        this.transmissionHandler = transmissionHandler ?? exporterOptions.GetLogsExportTransmissionHandler(experimentalOptions!.EnableInMemoryRetry);
 
         this.otlpLogRecordTransformer = new OtlpLogRecordTransformer(sdkLimitOptions!, experimentalOptions!);
     }

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpMetricExporter.cs

@@ -25,17 +25,19 @@ public class OtlpMetricExporter : BaseExporter<Metric>
     /// </summary>
     /// <param name="options">Configuration options for the exporter.</param>
     public OtlpMetricExporter(OtlpExporterOptions options)
-        : this(options, transmissionHandler: null)
+        : this(options, experimentalOptions: new(), transmissionHandler: null)
     {
     }
 
     /// <summary>
     /// Initializes a new instance of the <see cref="OtlpMetricExporter"/> class.
     /// </summary>
     /// <param name="options">Configuration options for the export.</param>
+    /// <param name="experimentalOptions"><see cref="ExperimentalOptions"/>.</param>
     /// <param name="transmissionHandler"><see cref="OtlpExporterTransmissionHandler{T}"/>.</param>
     internal OtlpMetricExporter(
         OtlpExporterOptions options,
+        ExperimentalOptions experimentalOptions,
         OtlpExporterTransmissionHandler<OtlpCollector.ExportMetricsServiceRequest> transmissionHandler = null)
     {
         // Each of the Otlp exporters: Traces, Metrics, and Logs set the same value for `OtlpKeyValueTransformer.LogUnsupportedAttributeType`
@@ -50,7 +52,7 @@ internal OtlpMetricExporter(
             OpenTelemetryProtocolExporterEventSource.Log.InvalidEnvironmentVariable(key, value);
         };
 
-        this.transmissionHandler = transmissionHandler ?? options.GetMetricsExportTransmissionHandler();
+        this.transmissionHandler = transmissionHandler ?? options.GetMetricsExportTransmissionHandler(experimentalOptions.EnableInMemoryRetry);
     }
 
     internal OtlpResource.Resource ProcessResource => this.processResource ??= this.ParentProvider.GetResource().ToOtlpResource();

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpMetricExporterExtensions.cs

@@ -7,6 +7,7 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Options;
 using OpenTelemetry.Exporter;
+using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation;
 using OpenTelemetry.Internal;
 
 namespace OpenTelemetry.Metrics;
@@ -60,6 +61,8 @@ public static MeterProviderBuilder AddOtlpExporter(
 
             OtlpExporterOptions.RegisterOtlpExporterOptionsFactory(services);
 
+            services.RegisterOptionsFactory(configuration => new ExperimentalOptions(configuration));
+
             services.AddOptions<MetricReaderOptions>(finalOptionsName).Configure<IConfiguration>(
                 (readerOptions, config) =>
                 {
@@ -98,6 +101,7 @@ public static MeterProviderBuilder AddOtlpExporter(
             return BuildOtlpExporterMetricReader(
                 exporterOptions,
                 sp.GetRequiredService<IOptionsMonitor<MetricReaderOptions>>().Get(finalOptionsName),
+                sp.GetRequiredService<IOptionsMonitor<ExperimentalOptions>>().Get(finalOptionsName),
                 sp);
         });
     }
@@ -137,6 +141,8 @@ public static MeterProviderBuilder AddOtlpExporter(
         {
             OtlpExporterOptions.RegisterOtlpExporterOptionsFactory(services);
 
+            services.RegisterOptionsFactory(configuration => new ExperimentalOptions(configuration));
+
             services.AddOptions<MetricReaderOptions>(finalOptionsName).Configure<IConfiguration>(
                 (readerOptions, config) =>
                 {
@@ -169,19 +175,24 @@ public static MeterProviderBuilder AddOtlpExporter(
 
             configureExporterAndMetricReader?.Invoke(exporterOptions, metricReaderOptions);
 
-            return BuildOtlpExporterMetricReader(exporterOptions, metricReaderOptions, sp);
+            return BuildOtlpExporterMetricReader(
+                exporterOptions,
+                metricReaderOptions,
+                sp.GetRequiredService<IOptionsMonitor<ExperimentalOptions>>().Get(finalOptionsName),
+                sp);
         });
     }
 
     internal static MetricReader BuildOtlpExporterMetricReader(
         OtlpExporterOptions exporterOptions,
         MetricReaderOptions metricReaderOptions,
+        ExperimentalOptions experimentalOptions,
         IServiceProvider serviceProvider,
         Func<BaseExporter<Metric>, BaseExporter<Metric>>? configureExporterInstance = null)
     {
         exporterOptions.TryEnableIHttpClientFactoryIntegration(serviceProvider, "OtlpMetricExporter");
 
-        BaseExporter<Metric> metricExporter = new OtlpMetricExporter(exporterOptions);
+        BaseExporter<Metric> metricExporter = new OtlpMetricExporter(exporterOptions, experimentalOptions);
 
         if (configureExporterInstance != null)
         {

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpTraceExporter.cs

@@ -26,7 +26,7 @@ public class OtlpTraceExporter : BaseExporter<Activity>
     /// </summary>
     /// <param name="options">Configuration options for the export.</param>
     public OtlpTraceExporter(OtlpExporterOptions options)
-        : this(options, sdkLimitOptions: new(), transmissionHandler: null)
+        : this(options, sdkLimitOptions: new(), experimentalOptions: new(), transmissionHandler: null)
     {
     }
 
@@ -35,10 +35,12 @@ public OtlpTraceExporter(OtlpExporterOptions options)
     /// </summary>
     /// <param name="exporterOptions"><see cref="OtlpExporterOptions"/>.</param>
     /// <param name="sdkLimitOptions"><see cref="SdkLimitOptions"/>.</param>
+    /// <param name="experimentalOptions"><see cref="ExperimentalOptions"/>.</param>
     /// <param name="transmissionHandler"><see cref="OtlpExporterTransmissionHandler{T}"/>.</param>
     internal OtlpTraceExporter(
     OtlpExporterOptions exporterOptions,
     SdkLimitOptions sdkLimitOptions,
+    ExperimentalOptions experimentalOptions,
     OtlpExporterTransmissionHandler<OtlpCollector.ExportTraceServiceRequest> transmissionHandler = null)
     {
         Debug.Assert(exporterOptions != null, "exporterOptions was null");
@@ -50,7 +52,7 @@ internal OtlpTraceExporter(
 
         ConfigurationExtensions.LogInvalidEnvironmentVariable = OpenTelemetryProtocolExporterEventSource.Log.InvalidEnvironmentVariable;
 
-        this.transmissionHandler = transmissionHandler ?? exporterOptions.GetTraceExportTransmissionHandler();
+        this.transmissionHandler = transmissionHandler ?? exporterOptions.GetTraceExportTransmissionHandler(experimentalOptions.EnableInMemoryRetry);
     }
 
     internal OtlpResource.Resource ProcessResource => this.processResource ??= this.ParentProvider.GetResource().ToOtlpResource();

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpTraceExporterHelperExtensions.cs

@@ -61,6 +61,7 @@ public static TracerProviderBuilder AddOtlpExporter(
 
             OtlpExporterOptions.RegisterOtlpExporterOptionsFactory(services);
             services.RegisterOptionsFactory(configuration => new SdkLimitOptions(configuration));
+            services.RegisterOptionsFactory(configuration => new ExperimentalOptions(configuration));
         });
 
         return builder.AddProcessor(sp =>
@@ -92,19 +93,24 @@ public static TracerProviderBuilder AddOtlpExporter(
             // instance.
             var sdkOptionsManager = sp.GetRequiredService<IOptionsMonitor<SdkLimitOptions>>().CurrentValue;
 
-            return BuildOtlpExporterProcessor(exporterOptions, sdkOptionsManager, sp);
+            return BuildOtlpExporterProcessor(
+                exporterOptions,
+                sdkOptionsManager,
+                sp.GetRequiredService<IOptionsMonitor<ExperimentalOptions>>().Get(finalOptionsName),
+                sp);
         });
     }
 
     internal static BaseProcessor<Activity> BuildOtlpExporterProcessor(
         OtlpExporterOptions exporterOptions,
         SdkLimitOptions sdkLimitOptions,
+        ExperimentalOptions experimentalOptions,
         IServiceProvider serviceProvider,
         Func<BaseExporter<Activity>, BaseExporter<Activity>>? configureExporterInstance = null)
     {
         exporterOptions.TryEnableIHttpClientFactoryIntegration(serviceProvider, "OtlpTraceExporter");
 
-        BaseExporter<Activity> otlpExporter = new OtlpTraceExporter(exporterOptions, sdkLimitOptions);
+        BaseExporter<Activity> otlpExporter = new OtlpTraceExporter(exporterOptions, sdkLimitOptions, experimentalOptions);
 
         if (configureExporterInstance != null)
         {

test/Benchmarks/Exporter/OtlpGrpcExporterBenchmarks.cs

@@ -35,6 +35,7 @@ public void GlobalSetup()
         this.exporter = new OtlpTraceExporter(
             options,
             new SdkLimitOptions(),
+            new ExperimentalOptions(),
             new OtlpExporterTransmissionHandler<ExportTraceServiceRequest>(new OtlpGrpcTraceExportClient(options, new TestTraceServiceClient()), options.TimeoutMilliseconds));
 
         this.activity = ActivityHelper.CreateTestActivity();

test/Benchmarks/Exporter/OtlpHttpExporterBenchmarks.cs

@@ -63,6 +63,7 @@ public void GlobalSetup()
         this.exporter = new OtlpTraceExporter(
             options,
             new SdkLimitOptions(),
+            new ExperimentalOptions(),
             new OtlpExporterTransmissionHandler<ExportTraceServiceRequest>(new OtlpHttpTraceExportClient(options, options.HttpClientFactory()), options.TimeoutMilliseconds));
 
         this.activity = ActivityHelper.CreateTestActivity();

test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/IntegrationTest/IntegrationTests.cs

@@ -72,6 +72,7 @@ public void TraceExportResultIsSuccess(OtlpExportProtocol protocol, string endpo
         builder.AddProcessor(OtlpTraceExporterHelperExtensions.BuildOtlpExporterProcessor(
             exporterOptions,
             DefaultSdkLimitOptions,
+            experimentalOptions: new(),
             serviceProvider: null,
             configureExporterInstance: otlpExporter =>
             {
@@ -153,6 +154,7 @@ public void MetricExportResultIsSuccess(OtlpExportProtocol protocol, string endp
         builder.AddReader(OtlpMetricExporterExtensions.BuildOtlpExporterMetricReader(
             exporterOptions,
             readerOptions,
+            experimentalOptions: new(),
             serviceProvider: null,
             configureExporterInstance: otlpExporter =>
             {

test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/OtlpTraceExporterTests.cs

@@ -23,6 +23,8 @@ public class OtlpTraceExporterTests : Http2UnencryptedSupportTests
 {
     private static readonly SdkLimitOptions DefaultSdkLimitOptions = new();
 
+    private static readonly ExperimentalOptions DefaultExperimentalOptions = new();
+
     static OtlpTraceExporterTests()
     {
         Activity.DefaultIdFormat = ActivityIdFormat.W3C;
@@ -627,8 +629,7 @@ public void Shutdown_ClientShutdownIsCalled()
         var exporterOptions = new OtlpExporterOptions();
         var transmissionHandler = new OtlpExporterTransmissionHandler<OtlpCollector.ExportTraceServiceRequest>(exportClientMock, exporterOptions.TimeoutMilliseconds);
 
-        var exporter = new OtlpTraceExporter(exporterOptions, DefaultSdkLimitOptions, transmissionHandler);
-
+        var exporter = new OtlpTraceExporter(new OtlpExporterOptions(), DefaultSdkLimitOptions, DefaultExperimentalOptions, transmissionHandler);
         exporter.Shutdown();
 
         Assert.True(exportClientMock.ShutdownCalled);