From f019852748edb820ffb5b1c61fbfc7c146d638be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20H=C3=A9mery?= Date: Tue, 2 Jul 2019 11:27:38 +0200 Subject: [PATCH] Added: GZip compression feature (#621) [changelog] --- src/Algolia.Search.Test/BaseTest.cs | 14 ++++--- src/Algolia.Search/Clients/AlgoliaConfig.cs | 6 +++ src/Algolia.Search/Clients/AnalyticsConfig.cs | 4 +- src/Algolia.Search/Clients/InsightsConfig.cs | 4 +- src/Algolia.Search/Clients/SearchConfig.cs | 5 +++ .../Http/AlgoliaHttpRequester.cs | 5 +++ .../Http/HttpRequestHeadersExtensions.cs | 23 ++++++++++- src/Algolia.Search/Models/Common/Request.cs | 27 +++++++++++- .../Models/Enums/CompressionType.cs | 41 +++++++++++++++++++ .../Serializer/SerializerHelper.cs | 33 ++++++++++++--- src/Algolia.Search/Transport/HttpTransport.cs | 13 ++++-- src/Algolia.Search/Utils/Defaults.cs | 3 ++ 12 files changed, 159 insertions(+), 19 deletions(-) create mode 100644 src/Algolia.Search/Models/Enums/CompressionType.cs diff --git a/src/Algolia.Search.Test/BaseTest.cs b/src/Algolia.Search.Test/BaseTest.cs index c46aecddc..645691981 100644 --- a/src/Algolia.Search.Test/BaseTest.cs +++ b/src/Algolia.Search.Test/BaseTest.cs @@ -1,17 +1,17 @@ /* * Copyright (c) 2018 Algolia * http://www.algolia.com/ -* +* * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: -* +* * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. -* +* * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE @@ -45,7 +45,11 @@ public void Setup() { TestHelper.CheckEnvironmentVariable(); SearchClient = new SearchClient(TestHelper.ApplicationId1, TestHelper.AdminKey1); - SearchClient2 = new SearchClient(TestHelper.ApplicationId2, TestHelper.AdminKey2); + SearchConfig configClient2 = new SearchConfig(TestHelper.ApplicationId2, TestHelper.AdminKey2) + { + Compression = CompressionType.NONE + }; + SearchClient2 = new SearchClient(configClient2); McmClient = new SearchClient(TestHelper.McmApplicationId, TestHelper.McmAdminKey); AnalyticsClient = new AnalyticsClient(TestHelper.ApplicationId1, TestHelper.AdminKey1); } @@ -83,4 +87,4 @@ protected void PreviousTestCleanUp() SearchClient.MultipleBatch(operations); } -} \ No newline at end of file +} diff --git a/src/Algolia.Search/Clients/AlgoliaConfig.cs b/src/Algolia.Search/Clients/AlgoliaConfig.cs index 6a1ae1474..acbdde120 100644 --- a/src/Algolia.Search/Clients/AlgoliaConfig.cs +++ b/src/Algolia.Search/Clients/AlgoliaConfig.cs @@ -21,6 +21,7 @@ * THE SOFTWARE. */ +using Algolia.Search.Models.Enums; using Algolia.Search.Serializer; using Algolia.Search.Transport; using System.Collections.Generic; @@ -94,6 +95,11 @@ public AlgoliaConfig(string applicationId, string apiKey) /// public int? WriteTimeout { get; set; } + /// + /// Compression for outgoing http requests + /// + public virtual CompressionType Compression { get; protected set; } + /// /// Configurations hosts /// diff --git a/src/Algolia.Search/Clients/AnalyticsConfig.cs b/src/Algolia.Search/Clients/AnalyticsConfig.cs index 81094fcc7..4bf297b73 100644 --- a/src/Algolia.Search/Clients/AnalyticsConfig.cs +++ b/src/Algolia.Search/Clients/AnalyticsConfig.cs @@ -48,6 +48,8 @@ public AnalyticsConfig(string applicationId, string apiKey) : base(applicationId Accept = CallType.Read | CallType.Write } }; + + Compression = CompressionType.NONE; } } -} \ No newline at end of file +} diff --git a/src/Algolia.Search/Clients/InsightsConfig.cs b/src/Algolia.Search/Clients/InsightsConfig.cs index 660e7dd95..84c3c298f 100644 --- a/src/Algolia.Search/Clients/InsightsConfig.cs +++ b/src/Algolia.Search/Clients/InsightsConfig.cs @@ -49,6 +49,8 @@ public InsightsConfig(string applicationId, string apiKey, string region = "us") Accept = CallType.Read | CallType.Write } }; + + Compression = CompressionType.NONE; } } -} \ No newline at end of file +} diff --git a/src/Algolia.Search/Clients/SearchConfig.cs b/src/Algolia.Search/Clients/SearchConfig.cs index 1de1891bb..4aa7d9a0f 100644 --- a/src/Algolia.Search/Clients/SearchConfig.cs +++ b/src/Algolia.Search/Clients/SearchConfig.cs @@ -88,6 +88,11 @@ public SearchConfig(string applicationId, string apiKey) : base(applicationId, a hosts.AddRange(commonHosts); DefaultHosts = hosts; + + Compression = CompressionType.NONE; } + + /// + public new CompressionType Compression { get; set; } } } diff --git a/src/Algolia.Search/Http/AlgoliaHttpRequester.cs b/src/Algolia.Search/Http/AlgoliaHttpRequester.cs index 23ca5b82a..945a726c4 100644 --- a/src/Algolia.Search/Http/AlgoliaHttpRequester.cs +++ b/src/Algolia.Search/Http/AlgoliaHttpRequester.cs @@ -80,6 +80,11 @@ public async Task SendRequestAsync(Request request, int tot Content = request.Body != null ? new StreamContent(request.Body) : null }; + if (request.Body != null) + { + httpRequestMessage.Content.Headers.Fill(request); + } + httpRequestMessage.Headers.Fill(request.Headers); httpRequestMessage.SetTimeout(TimeSpan.FromSeconds(totalTimeout)); diff --git a/src/Algolia.Search/Http/HttpRequestHeadersExtensions.cs b/src/Algolia.Search/Http/HttpRequestHeadersExtensions.cs index 3cadbd8cc..7e99a93d1 100644 --- a/src/Algolia.Search/Http/HttpRequestHeadersExtensions.cs +++ b/src/Algolia.Search/Http/HttpRequestHeadersExtensions.cs @@ -21,6 +21,7 @@ * THE SOFTWARE. */ +using Algolia.Search.Models.Common; using System.Collections.Generic; using System.Net.Http.Headers; @@ -43,5 +44,25 @@ internal static HttpRequestHeaders Fill(this HttpRequestHeaders headers, Diction return headers; } + + /// + /// Extension method to easily fill HttpContentHeaders with the Request object + /// + /// + /// + internal static HttpContentHeaders Fill(this HttpContentHeaders headers, Request request) + { + if (request.Body != null) + { + headers.Add(Defaults.ContentType, Defaults.ApplicationJson); + + if (request.CanCompress) + { + headers.ContentEncoding.Add(Defaults.GzipEncoding); + } + } + + return headers; + } } -} \ No newline at end of file +} diff --git a/src/Algolia.Search/Models/Common/Request.cs b/src/Algolia.Search/Models/Common/Request.cs index 2d52fbc3c..dd22769c6 100644 --- a/src/Algolia.Search/Models/Common/Request.cs +++ b/src/Algolia.Search/Models/Common/Request.cs @@ -21,6 +21,7 @@ * THE SOFTWARE. */ +using Algolia.Search.Models.Enums; using System; using System.Collections.Generic; using System.IO; @@ -52,5 +53,29 @@ public class Request /// Body of the request /// public Stream Body { get; set; } + + /// + /// Compression type of the request + /// + public CompressionType Compression { get; set; } + + /// + /// Tells if the request can be compressed or not + /// + public bool CanCompress + { + get + { + if (Method == null) + { + return false; + } + + bool isMethodValid = Method.Equals(HttpMethod.Post) || Method.Equals(HttpMethod.Put); + bool isCompressionEnabled = Compression.Equals(CompressionType.GZIP); + + return isMethodValid && isCompressionEnabled; + } + } } -} \ No newline at end of file +} diff --git a/src/Algolia.Search/Models/Enums/CompressionType.cs b/src/Algolia.Search/Models/Enums/CompressionType.cs new file mode 100644 index 000000000..b2d2d0a35 --- /dev/null +++ b/src/Algolia.Search/Models/Enums/CompressionType.cs @@ -0,0 +1,41 @@ +/* +* Copyright (c) 2018 Algolia +* http://www.algolia.com/ +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to deal +* in the Software without restriction, including without limitation the rights +* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +* copies of the Software, and to permit persons to whom the Software is +* furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +* THE SOFTWARE. +*/ + +namespace Algolia.Search.Models.Enums +{ + /// + /// Compression type for outgoing HTTP requests + /// + public enum CompressionType + { + /// + /// No compression + /// + NONE, + + /// + /// GZip Compression. Only supported by Search API. + /// + GZIP + } +} diff --git a/src/Algolia.Search/Serializer/SerializerHelper.cs b/src/Algolia.Search/Serializer/SerializerHelper.cs index 4371da176..c0d05d47f 100644 --- a/src/Algolia.Search/Serializer/SerializerHelper.cs +++ b/src/Algolia.Search/Serializer/SerializerHelper.cs @@ -23,6 +23,7 @@ using Newtonsoft.Json; using System.IO; +using System.IO.Compression; using System.Text; using System.Threading.Tasks; @@ -34,16 +35,36 @@ namespace Algolia.Search.Serializer /// internal static class SerializerHelper { - private static readonly UTF8Encoding _encoding = new UTF8Encoding(false); + private static readonly UTF8Encoding DefaultEncoding = new UTF8Encoding(false); + private static readonly int DefaultBufferSize = 1024; + // Buffer sized as recommended by Bradley Grainger, http://faithlife.codes/blog/2012/06/always-wrap-gzipstream-with-bufferedstream/ + private static readonly int GZipBufferSize = 8192; - public static void Serialize(T data, Stream stream, JsonSerializerSettings settings) + public static void Serialize(T data, Stream stream, JsonSerializerSettings settings, bool gzipCompress) { - using (var sw = new StreamWriter(stream, _encoding, 1024, true)) - using (var jtw = new JsonTextWriter(sw) { Formatting = Formatting.None }) + if (gzipCompress) + { + using (var gzipStream = new GZipStream(stream, CompressionMode.Compress, true)) + using (var sw = new StreamWriter(gzipStream, DefaultEncoding, GZipBufferSize)) + using (var jtw = new JsonTextWriter(sw) { Formatting = Formatting.None }) + { + JsonSerialize(jtw); + } + } + else + { + using (var sw = new StreamWriter(stream, DefaultEncoding, DefaultBufferSize, true)) + using (var jtw = new JsonTextWriter(sw) { Formatting = Formatting.None }) + { + JsonSerialize(jtw); + } + } + + void JsonSerialize(JsonTextWriter writer) { JsonSerializer serializer = JsonSerializer.Create(settings); - serializer.Serialize(jtw, data); - jtw.Flush(); + serializer.Serialize(writer, data); + writer.Flush(); } } diff --git a/src/Algolia.Search/Transport/HttpTransport.cs b/src/Algolia.Search/Transport/HttpTransport.cs index a40260a85..90753fc8d 100644 --- a/src/Algolia.Search/Transport/HttpTransport.cs +++ b/src/Algolia.Search/Transport/HttpTransport.cs @@ -106,10 +106,12 @@ public async Task ExecuteRequestAsync(HttpMethod method var request = new Request { Method = method, - Body = CreateRequestContent(data), - Headers = GenerateHeaders(requestOptions?.Headers) + Headers = GenerateHeaders(requestOptions?.Headers), + Compression = _algoliaConfig.Compression }; + request.Body = CreateRequestContent(data, request.CanCompress); + foreach (var host in _retryStrategy.GetTryableHost(callType)) { request.Uri = BuildUri(host.Url, uri, requestOptions?.QueryParameters); @@ -138,14 +140,17 @@ public async Task ExecuteRequestAsync(HttpMethod method /// Generate stream for serializing objects /// /// Data to send + /// Whether the stream should be compressed or not /// Type of the data to send/retrieve /// - private MemoryStream CreateRequestContent(T data) + private MemoryStream CreateRequestContent(T data, bool compress) { if (data != null) { MemoryStream ms = new MemoryStream(); - SerializerHelper.Serialize(data, ms, JsonConfig.AlgoliaJsonSerializerSettings); + + SerializerHelper.Serialize(data, ms, JsonConfig.AlgoliaJsonSerializerSettings, compress); + ms.Seek(0, SeekOrigin.Begin); return ms; } diff --git a/src/Algolia.Search/Utils/Defaults.cs b/src/Algolia.Search/Utils/Defaults.cs index 9a13f96af..dc4c3dad0 100644 --- a/src/Algolia.Search/Utils/Defaults.cs +++ b/src/Algolia.Search/Utils/Defaults.cs @@ -47,4 +47,7 @@ internal class Defaults public const string UserAgentHeader = "User-Agent"; public const string Connection = "Connection"; public const string KeepAlive = "keep-alive"; + public const string ContentType = "Content-Type"; + public const string ApplicationJson = "application/json; charset=utf-8"; + public const string GzipEncoding = "gzip"; }