Summarise Subreddits

.gitignore

@@ -396,3 +396,4 @@ FodyWeavers.xsd
 
 # JetBrains Rider
 *.sln.iml
+/tests/LocalPlaylists

Elzik.Breef.sln

@@ -26,6 +26,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "TestData", "TestData", "{7F
 	ProjectSection(SolutionItems) = preProject
 		tests\TestData\BbcNewsPage-ExpectedContent.txt = tests\TestData\BbcNewsPage-ExpectedContent.txt
 		tests\TestData\BbcNewsPage.html = tests\TestData\BbcNewsPage.html
+		tests\TestData\SampleRedditPost-1kqiwzc.json = tests\TestData\SampleRedditPost-1kqiwzc.json
 		tests\TestData\StaticTestPage.html = tests\TestData\StaticTestPage.html
 		tests\TestData\TestHtmlPage-ExpectedContent.txt = tests\TestData\TestHtmlPage-ExpectedContent.txt
 		tests\TestData\TestHtmlPage.html = tests\TestData\TestHtmlPage.html

README.md

@@ -80,6 +80,34 @@ Example
 
 ### Optional
 
+#### Reddit
+
+These config items relate to the Reddit integration using the Options pattern with support for multiple Reddit instances.
+
+- **DefaultBaseAddress** - The primary base address for Reddit API requests. Default: `"https://www.reddit.com"`. Must be a valid URL. Used for Refit HTTP client configuration, fallback subreddit image extraction, and primary Reddit instance for content extraction.
+- **AdditionalBaseAddresses** - Additional Reddit instances that the content extractors can handle. Default: `["https://reddit.com"]` (includes non-www variant by default). Domain matching is **exact** - if you want to support both `reddit.com` and `www.reddit.com`, you must explicitly configure both.
+- **FallbackImageUrl** - The fallback image URL used when subreddit-specific images cannot be retrieved. Default: `"https://redditinc.com/hubfs/Reddit%20Inc/Brand/Reddit_Lockup_Logo.svg"`. This URL is used as the default Reddit logo when no subreddit banner, icon, or community image is available.
+
+The Reddit integration allows extraction of content from:
+- Custom Reddit instances
+- Alternative Reddit domains  
+- Corporate or self-hosted Reddit installations
+- Specific subdomains (e.g., `old.reddit.com`, `api.reddit.com`)
+
+**Domain Validation**: The content extractors validate URLs using **exact domain matching**. `reddit.com` does NOT automatically allow `www.reddit.com` - each domain variant must be explicitly configured.
+
+Example:
+
+```jsonc
+"Reddit": {
+    "DefaultBaseAddress": "https://www.reddit.com",         // breef_Reddit__DefaultBaseAddress
+    "AdditionalBaseAddresses": [                            // breef_Reddit__AdditionalBaseAddresses__0
+        "https://reddit.com",                               // breef_Reddit__AdditionalBaseAddresses__0
+    ],
+    "FallbackImageUrl": "https://redditinc.com/hubfs/Reddit%20Inc/Brand/Reddit_Lockup_Logo.svg"  // breef_Reddit__FallbackImageUrl
+}
+```
+
 #### AiService
 
 - **TimeOut** - Sets the number of seconds before the AiService used will time out. The default used if not set is 100 seconds. This may need to be increased where Ollama is used with limiting hardware.
@@ -113,12 +141,14 @@ Example:
 These settings affect how pages are downloaded prior to being summarised.
 
   - **UserAgent** - The user agent used when downloading pages. By default this is set to `Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36` but can be overridden here.
+  - **TimeoutSeconds** - The timeout in seconds for HTTP requests when downloading pages. By default this is set to `30` seconds but can be overridden here. Must be at least 1 second.
 
 Example:
 
 ```jsonc
-"WebPageDownLoader" : {
-    "UserAgent": "<custom-agent>"   // breef_WebPageDownLoader__UserAgent
+"HttpClient" : {
+    "UserAgent": "<custom-agent>",          // breef_HttpClient__UserAgent
+    "TimeoutSeconds": 30                    // breef_HttpClient__TimeoutSeconds
 }
 ```
 
@@ -131,5 +161,4 @@ Logging is handled by Serilog and configuration is documented [here](https://git
   "MinimumLevel": {
     "Default": "Debug"   // breef_Serilog__MinimumLevel__Default
   }
-}
-```
+}

src/Elzik.Breef.Api/Elzik.Breef.Api.http

@@ -4,5 +4,5 @@ Post {{Elzik.Breef.Api_HostAddress}}/breefs
 Content-Type: application/json
 BREEF-API-KEY: test-key
 {
-  "url":"https://www.bbc.co.uk/news/articles/cdedkr9439wo"
+  "url":"https://www.reddit.com/r/dotnet/comments/1o0j6or/im_giving_up_on_copilot_i_spend_more_time/"
 }

src/Elzik.Breef.Api/Program.cs

@@ -4,15 +4,15 @@
 using Elzik.Breef.Domain;
 using Elzik.Breef.Infrastructure;
 using Elzik.Breef.Infrastructure.AI;
+using Elzik.Breef.Infrastructure.ContentExtractors;
+using Elzik.Breef.Infrastructure.ContentExtractors.Reddit;
+using Elzik.Breef.Infrastructure.ContentExtractors.Reddit.Client;
+using Elzik.Breef.Infrastructure.ContentExtractors.Reddit.Client.Raw;
 using Elzik.Breef.Infrastructure.Wallabag;
 using Microsoft.Extensions.Options;
-using Microsoft.SemanticKernel;
-using Microsoft.SemanticKernel.ChatCompletion;
 using Refit;
 using Serilog;
 using System.Reflection;
-using System.Text.Json;
-using System.Text.Json.Serialization;
 
 namespace Elzik.Breef.Api;
 
@@ -49,7 +49,7 @@
         {
             options.AddDefaultPolicy(builder =>
             {
                 builder.AllowAnyOrigin()
                        .AllowAnyMethod()
                        .AllowAnyHeader();
             });
@@ -61,13 +61,57 @@
             .ValidateOnStart();
         builder.Services.AddAuth();
 
-        builder.Services.AddOptions<WebPageDownLoaderOptions>()
-            .Bind(configuration.GetSection("WebPageDownLoader"))
+        builder.Services.AddOptions<HttpClientOptions>()
+            .Bind(configuration.GetSection("HttpClient"))
             .ValidateDataAnnotations()
             .ValidateOnStart();
-        builder.Services.AddTransient<IWebPageDownloader, WebPageDownloader>();
 
-        builder.Services.AddTransient<IContentExtractor, ContentExtractor>();
+        builder.Services.AddHttpClient("BreefDownloader")
+            .ConfigureHttpClient((provider, client) =>
+            {
+                var httpClientOptions = provider.GetRequiredService<IOptions<HttpClientOptions>>().Value;
+                client.Timeout = TimeSpan.FromSeconds(httpClientOptions.TimeoutSeconds);
+                client.DefaultRequestHeaders.Add("User-Agent", httpClientOptions.UserAgent);
+            });
+
+        builder.Services.AddOptions<RedditOptions>()
+            .Bind(configuration.GetSection("Reddit"))
+            .ValidateDataAnnotations()
+            .ValidateOnStart();
+
+        builder.Services.AddRefitClient<IRawRedditPostClient>()
+            .ConfigureHttpClient((provider, client) =>
+            {
+                var redditOptions = provider.GetRequiredService<IOptions<RedditOptions>>().Value;
+                client.BaseAddress = new Uri(redditOptions.DefaultBaseAddress);
+            });
+
+        builder.Services.AddRefitClient<IRawSubredditClient>()
+            .ConfigureHttpClient((provider, client) =>
+            {
+                var redditOptions = provider.GetRequiredService<IOptions<RedditOptions>>().Value;
+                client.BaseAddress = new Uri(redditOptions.DefaultBaseAddress);
+            });
+
+        builder.Services.AddTransient<IRawRedditPostTransformer, RawRedditPostTransformer>();
+        builder.Services.AddTransient<IRedditPostClient, RedditPostClient>();
+        builder.Services.AddTransient<IRawNewInSubredditTransformer, RawNewInSubredditTransformer>();
+        builder.Services.AddTransient<ISubredditClient, SubredditClient>();
+
+        builder.Services.AddTransient<HtmlContentExtractor>();
+        builder.Services.AddTransient<SubredditContentExtractor>();
+        builder.Services.AddTransient<RedditPostContentExtractor>();
+        builder.Services.AddTransient<ISubredditImageExtractor, SubredditContentExtractor>();
+        builder.Services.AddTransient<IContentExtractor>(provider =>
+        {
+            var logger = provider.GetRequiredService<ILogger<ContentExtractorStrategy>>();
+            var defaultContentExtractor = provider.GetRequiredService<HtmlContentExtractor>();
+            var subredditExtractor = provider.GetRequiredService<SubredditContentExtractor>();
+            var redditPostExtractor = provider.GetRequiredService<RedditPostContentExtractor>();
+            return new ContentExtractorStrategy(logger, 
+                [subredditExtractor, redditPostExtractor], 
+                defaultContentExtractor);
+        });
 
         builder.Services.AddOptions<AiServiceOptions>()
             .Bind(configuration.GetSection("AiService"))

src/Elzik.Breef.Application/BreefGenerator.cs

@@ -1,5 +1,4 @@
 using Elzik.Breef.Domain;
-using System.Diagnostics;
 
 namespace Elzik.Breef.Application
 {

src/Elzik.Breef.Application/Elzik.Breef.Application.csproj

@@ -6,6 +6,13 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="SonarAnalyzer.CSharp" Version="10.15.0.120848">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\Elzik.Breef.Domain\Elzik.Breef.Domain.csproj" />
   </ItemGroup>

src/Elzik.Breef.Domain/Elzik.Breef.Domain.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>net8.0</TargetFramework>
@@ -9,6 +9,10 @@
   <ItemGroup>
     <PackageReference Include="HtmlAgilityPack" Version="1.12.4" />
     <PackageReference Include="Microsoft.SemanticKernel.Abstractions" Version="1.65.0" />
+    <PackageReference Include="SonarAnalyzer.CSharp" Version="10.15.0.120848">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
   </ItemGroup>
 
 </Project>

src/Elzik.Breef.Domain/IContentExtractor.cs

@@ -2,6 +2,8 @@
 {
     public interface IContentExtractor
     {
+        bool CanHandle(string webPageUrl);
+
         Task<Extract> ExtractAsync(string webPageUrl);
     }
 }

src/Elzik.Breef.Infrastructure/ContentExtractors/ContentExtractorStrategy.cs

@@ -0,0 +1,38 @@
+using Elzik.Breef.Domain;
+using Microsoft.Extensions.Logging;
+
+namespace Elzik.Breef.Infrastructure.ContentExtractors
+{
+    public class ContentExtractorStrategy : IContentExtractor
+    {
+        private readonly ILogger<ContentExtractorStrategy> _logger;
+        private readonly List<IContentExtractor> _extractors;
+
+        public ContentExtractorStrategy(ILogger<ContentExtractorStrategy> logger, 
+            IEnumerable<IContentExtractor> specificExtractors, IContentExtractor defaultExtractor)
+        {
+            ArgumentNullException.ThrowIfNull(logger);
+            ArgumentNullException.ThrowIfNull(specificExtractors);
+            ArgumentNullException.ThrowIfNull(defaultExtractor);
+
+            _logger = logger;
+
+            if (specificExtractors.Contains(defaultExtractor))
+                throw new ArgumentException("Default extractor should not be in the specific extractors list.");
+
+            _extractors = [.. specificExtractors, defaultExtractor];
+        }
+
+        public bool CanHandle(string webPageUrl) => true;
+
+        public async Task<Extract> ExtractAsync(string webPageUrl)
+        {
+            var extractor = _extractors.First(e => e.CanHandle(webPageUrl));
+
+            _logger.LogInformation("Extraction will be provided for by {ExtractorName}", extractor.GetType().Name);
+
+            return await extractor.ExtractAsync(webPageUrl);
+        }
+    }
+
+}

src/Elzik.Breef.Infrastructure/ContentExtractor.cs → src/Elzik.Breef.Infrastructure/ContentExtractors/HtmlContentExtractor.cs

@@ -1,13 +1,14 @@
 using Elzik.Breef.Domain;
 using HtmlAgilityPack;
 
-namespace Elzik.Breef.Infrastructure;
+namespace Elzik.Breef.Infrastructure.ContentExtractors;
 
-public class ContentExtractor(IWebPageDownloader httpClient) : IContentExtractor
+public class HtmlContentExtractor(IHttpClientFactory httpClientFactory) : IContentExtractor
 {
     public async Task<Extract> ExtractAsync(string webPageUrl)
     {
-        var html = await httpClient.DownloadAsync(webPageUrl);
+        var httpClient = httpClientFactory.CreateClient("BreefDownloader");
+        var html = await httpClient.GetStringAsync(webPageUrl);
         var htmlDocument = new HtmlDocument();
         htmlDocument.LoadHtml(html);
 
@@ -78,4 +79,6 @@ private static string GetTitle(HtmlDocument htmlDocument, string defaultWhenMiss
 
         return imageNodesSortedBySize.FirstOrDefault()?.ImageUrl;
     }
+
+    public bool CanHandle(string webPageUrl) => true;
 }

src/Elzik.Breef.Infrastructure/ContentExtractors/Reddit/Client/AboutSubreddit.cs

@@ -0,0 +1,30 @@
+using System.Text.Json.Serialization;
+
+namespace Elzik.Breef.Infrastructure.ContentExtractors.Reddit.Client;
+
+public class AboutSubreddit
+{
+    [JsonPropertyName("data")]
+    public AboutSubredditData? Data { get; set; }
+}
+
+public class AboutSubredditData
+{
+    [JsonPropertyName("public_description")]
+    public string? PublicDescription { get; set; }
+
+    [JsonPropertyName("icon_img")]
+    public string? IconImg { get; set; }
+
+    [JsonPropertyName("banner_img")]
+    public string? BannerImg { get; set; }
+
+    [JsonPropertyName("banner_background_image")]
+    public string? BannerBackgroundImage { get; set; }
+
+    [JsonPropertyName("mobile_banner_image")]
+    public string? MobileBannerImage { get; set; }
+
+    [JsonPropertyName("community_icon")]
+    public string? CommunityIcon { get; set; }
+}

src/Elzik.Breef.Infrastructure/ContentExtractors/Reddit/Client/IRawNewInSubredditTransformer.cs

@@ -0,0 +1,6 @@
+namespace Elzik.Breef.Infrastructure.ContentExtractors.Reddit.Client;
+
+public interface IRawNewInSubredditTransformer
+{
+    Task<NewInSubreddit> Transform(RawNewInSubreddit rawNewInSubreddit);
+}

src/Elzik.Breef.Infrastructure/ContentExtractors/Reddit/Client/IRedditPostClient.cs

@@ -0,0 +1,7 @@
+namespace Elzik.Breef.Infrastructure.ContentExtractors.Reddit.Client
+{
+    public interface IRedditPostClient
+    {
+        Task<RedditPost> GetPost(string postId);
+    }
+}

src/Elzik.Breef.Infrastructure/ContentExtractors/Reddit/Client/ISubredditClient.cs

@@ -0,0 +1,6 @@
+namespace Elzik.Breef.Infrastructure.ContentExtractors.Reddit.Client;
+
+public interface ISubredditClient
+{
+    Task<NewInSubreddit> GetNewInSubreddit(string subRedditName);
+}

src/Elzik.Breef.Infrastructure/ContentExtractors/Reddit/Client/NewInSubreddit.cs

@@ -0,0 +1,6 @@
+namespace Elzik.Breef.Infrastructure.ContentExtractors.Reddit.Client;
+
+public class NewInSubreddit
+{
+    public List<RedditPost> Posts { get; set; } = [];
+}

src/Elzik.Breef.Infrastructure/ContentExtractors/Reddit/Client/Raw/FlexibleStringConverter.cs

@@ -0,0 +1,30 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Elzik.Breef.Infrastructure.ContentExtractors.Reddit.Client.Raw;
+
+public class FlexibleStringConverter : JsonConverter<string?>
+{
+    public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+    {
+        return reader.TokenType switch
+        {
+            JsonTokenType.String => reader.GetString(),
+            JsonTokenType.Number => reader.GetInt64().ToString(),
+            JsonTokenType.Null => null,
+            _ => throw new JsonException($"Cannot convert {reader.TokenType} to string")
+        };
+    }
+
+    public override void Write(Utf8JsonWriter writer, string? value, JsonSerializerOptions options)
+    {
+        if (value == null)
+        {
+            writer.WriteNullValue();
+        }
+        else
+        {
+            writer.WriteStringValue(value);
+        }
+    }
+}

src/Elzik.Breef.Infrastructure/ContentExtractors/Reddit/Client/Raw/IRawRedditPostClient.cs

@@ -0,0 +1,11 @@
+using Refit;
+
+namespace Elzik.Breef.Infrastructure.ContentExtractors.Reddit.Client.Raw
+{
+    public interface IRawRedditPostClient
+    {
+        [Get("/comments/{postId}.json")]
+        [Headers("User-Agent: breef/1.0.0 (https://github.com/elzik/breef)")]
+        Task<RawRedditPost> GetPost(string postId);
+    }
+}

src/Elzik.Breef.Infrastructure/ContentExtractors/Reddit/Client/Raw/IRawRedditPostTransformer.cs

@@ -0,0 +1,6 @@
+namespace Elzik.Breef.Infrastructure.ContentExtractors.Reddit.Client.Raw;
+
+public interface IRawRedditPostTransformer
+{
+    RedditPost Transform(RawRedditPost rawRedditPost);
+}