Add support for forward compatible binlog format

src/BinlogTool/BinlogToolCommandBase.cs

@@ -0,0 +1,30 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.Build.Logging.StructuredLogger;
+
+namespace BinlogTool
+{
+    public abstract class BinlogToolCommandBase
+    {
+        public ForwardCompatibilityReadingHandler CompatibilityHandler { protected get; init; }
+
+        protected Build ReadBuild(string binLogFilePath, bool throwOnPathNotFound = true)
+        {
+            if (string.IsNullOrEmpty(binLogFilePath) || !File.Exists(binLogFilePath))
+            {
+                if(throwOnPathNotFound)
+                {
+                    throw new FileNotFoundException("Specified binlog was not found.", binLogFilePath);
+                }
+
+                return null;
+            }
+
+            return CompatibilityHandler?.ReadBuild(binLogFilePath) ?? BinaryLog.ReadBuild(binLogFilePath);
+        }
+    }
+}

src/BinlogTool/ForwardCompatibilityReadingHandler.cs

@@ -0,0 +1,179 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using Microsoft.Build.Logging.StructuredLogger;
+
+namespace BinlogTool
+{
+    public class ForwardCompatibilityReadingHandler
+    {
+        public enum Mode
+        {
+            Disallow,
+            FailOnError,
+            LogErrorsSummary,
+            LogErrorsDetailed,
+            IgnoreErrors,
+            Invalid
+        }
+
+        private bool _isInitialized;
+        private Mode _mode;
+
+        private void CheckInitialized()
+        {
+            if (!_isInitialized)
+            {
+                throw new InvalidOperationException("ForwardCompatibilityReadingHandler is not initialized");
+            }
+        }
+
+        public void SetMode(Mode mode)
+        {
+            _isInitialized = true;
+            _mode = mode;
+        }
+
+        public bool ForwardCompatibilityExplicitlyConfigured { get; private set; }
+
+        public bool ProcessCommandLine(ref string[] args)
+        {
+            _isInitialized = true;
+
+            var compatArgs = args
+                .Where(arg =>
+                    arg.StartsWith("--forwardCompatibility", StringComparison.CurrentCultureIgnoreCase) ||
+                    arg.StartsWith("-fwd", StringComparison.CurrentCultureIgnoreCase))
+                .ToList();
+
+            if (compatArgs.Count == 0)
+            {
+                return true;
+            }
+
+            ForwardCompatibilityExplicitlyConfigured = true;
+
+            if (compatArgs.Count > 1)
+            {
+                Console.Error.WriteLine("Only one --forwardCompatibility/-fwd argument is allowed");
+                return false;
+            }
+
+            var compatArg = compatArgs[0];
+            args = args.Where(arg => arg != compatArg).ToArray();
+
+            int colonIndex = compatArg.IndexOf(':');
+
+            if (colonIndex == -1)
+            {
+                _mode = Mode.LogErrorsSummary;
+                return true;
+            }
+
+            _mode = compatArg.Substring(colonIndex + 1).ToLowerInvariant() switch
+            {
+                "d" or "disallow" => Mode.Disallow,
+                "f" or "failonerror" => Mode.FailOnError,
+                "l" or "logerrorssummary" => Mode.LogErrorsSummary,
+                "lv" or "logerrorsdetailed" => Mode.LogErrorsDetailed,
+                "i" or "ignoreerrors" => Mode.IgnoreErrors,
+                _ => Mode.Invalid
+            };
+
+            if (_mode == Mode.Invalid)
+            {
+                Console.Error.WriteLine("Invalid forward compatibility mode");
+                return false;
+            }
+
+            return true;
+        }
+
+        public Build ReadBuild(string binLogFilePath)
+        {
+            if (string.IsNullOrEmpty(binLogFilePath) || !File.Exists(binLogFilePath))
+            {
+                return null;
+            }
+
+            binLogFilePath = Path.GetFullPath(binLogFilePath);
+
+            var build = BinaryLog.ReadBuild(binLogFilePath, this.AllowForwardCompatibilityDelegate);
+            if (_compatibilityException != null)
+            {
+                throw _compatibilityException;
+            }
+            return build;
+        }
+
+        private Exception _compatibilityException = null;
+        public IForwardCompatibilityReadSettings AllowForwardCompatibilityDelegate
+        {
+            get
+            {
+                CheckInitialized();
+
+                IForwardCompatibilityReadSettings allowCompatSettings =
+                    ((AllowForwardCompatibilityDelegate)(_ => true)).WithDefaultHandler();
+
+                return _mode switch
+                {
+                    Mode.Disallow => null,
+                    Mode.FailOnError => allowCompatSettings.WithCustomErrorHandler(err =>
+                        throw (_compatibilityException = new Exception(err.GetFormattedMessage()))),
+                    _ => allowCompatSettings
+                };
+            }
+        }
+
+        public void HandleBuildResults(Build build, TextWriter? errorWriter = null)
+        {
+            CheckInitialized();
+
+            errorWriter ??= Console.Error;
+
+            if (_mode != Mode.LogErrorsSummary && _mode != Mode.LogErrorsDetailed)
+            {
+                return;
+            }
+
+            var errors = build.RecoverableReadingErrors;
+            if (errors == null || errors.Count == 0)
+            {
+                return;
+            }
+
+            IEnumerable<string> summaryLines;
+
+            if (_mode == Mode.LogErrorsSummary)
+            {
+                summaryLines = errors
+                    .GroupBy(e => e.errorType)
+                    .Select(g =>
+                        $"{SpacedReaderErrorType(g.Key)}: {g.Sum(e => e.count)} Total errors (in {g.GroupBy(f => f.recordKind).Count()} distinct types of records)");
+            }
+            else
+            {
+                summaryLines = errors
+                    .Select(e => $"| {SpacedReaderErrorType(e.errorType)} | {e.recordKind} | {e.count} |")
+                    .Prepend($"| :{new string('-', _maxErrorTypeLength-1)} | :---------- | -----------: |")
+                    .Prepend($"| {SpacedReaderErrorType("Error Type")} | Record Kind | Errors Count |");
+            }
+
+            var summary = string.Join(Environment.NewLine, summaryLines);
+
+            errorWriter.WriteLine();
+            errorWriter.WriteLine($"Forward compatibility recoverable errors summary:");
+            errorWriter.WriteLine();
+            errorWriter.WriteLine(summary);
+        }
+
+        private static readonly int _maxErrorTypeLength = Enum.GetNames(typeof(ReaderErrorType)).Max(s => s.Length);
+        private string SpacedReaderErrorType(ReaderErrorType errorType)
+            => SpacedReaderErrorType(errorType.ToString());
+
+        private string SpacedReaderErrorType(string text)
+            => string.Format("{0,-" + _maxErrorTypeLength + "}", text);
+    }
+}

src/BinlogTool/ListTools.cs

@@ -1,16 +1,16 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using Microsoft.Build.Logging.StructuredLogger;
 
 namespace BinlogTool
 {
-    public class ListTools
+    public class ListTools : BinlogToolCommandBase
     {
         public void Run(string binLogFilePath)
         {
-            var build = BinaryLog.ReadBuild(binLogFilePath);
+            var build = this.ReadBuild(binLogFilePath);
             BuildAnalyzer.AnalyzeBuild(build);
             var strings = build.StringTable.Instances.OrderBy(s => s).ToArray();
 
@@ -38,6 +38,8 @@ public void Run(string binLogFilePath)
             {
                 Console.WriteLine(tool);
             }
+
+            CompatibilityHandler?.HandleBuildResults(build);
         }
 
         private string GetSourceCommitId(Build build)

src/BinlogTool/Program.cs

@@ -19,7 +19,32 @@ binlogtool savefiles input.binlog output_path
     binlogtool reconstruct input.binlog output_path
     binlogtool savestrings input.binlog output.txt
     binlogtool search *.binlog search string
-    binlogtool redact --input:path --recurse --in-place -p:list -p:of -p:secrets -p:to -p:redact");
+    binlogtool redact --input:path --recurse --in-place -p:list -p:of -p:secrets -p:to -p:redact
+
+    Global Options:
+    [--forwardCompatibility|-fwd][:specifier] - Optional. Controls forward compatibility mode. Optional mode specifier:
+        d|disallow - Not allowed to read logs with higher then current version.
+        f|failOnError - Fail if the log contains any unsupported records.
+        l|logErrorsSummary - Default, if mode unspecified.
+                             Log errors summary if the log contains any unsupported records.
+        lv|logErrorsDetailed - Log detailed summary if the log contains any unsupported records.
+        i|ignoreErrors - Silently ignore any unsupported records.
+
+    Sampe usage:
+     // List all tools used in the build. Allow reading newer versions of the log, but fail if any unsupported records are encountered.
+     binlogtool --forwardCompatibility:failOnError listtools input.binlog
+     // List all tools used in the build. Allow reading only known versions of the log.
+     binlogtool listtools input.binlog
+     // List all tools used in the build. Allow reading newer versions of the log, log summary of errors if any unsupported records are encountered.
+     binlogtool listtools input.binlog -fwd
+
+");
+                return;
+            }
+
+            ForwardCompatibilityReadingHandler forwardCompatibilityReadingHandler = new();
+            if (!forwardCompatibilityReadingHandler.ProcessCommandLine(ref args))
+            {
                 return;
             }
 
@@ -30,7 +55,8 @@ binlogtool search *.binlog search string
                 var binlog = args[1];
                 var outputRoot = args[2];
 
-                new SaveFiles(args).Run(binlog, outputRoot);
+                new SaveFiles(args) { CompatibilityHandler = forwardCompatibilityReadingHandler }
+                    .Run(binlog, outputRoot);
                 return;
             }
 
@@ -39,7 +65,8 @@ binlogtool search *.binlog search string
                 var binlog = args[1];
                 var outputRoot = args[2];
 
-                new SaveFiles(args).Run(binlog, outputRoot, reconstruct: true);
+                new SaveFiles(args) { CompatibilityHandler = forwardCompatibilityReadingHandler }
+                    .Run(binlog, outputRoot, reconstruct: true);
                 return;
             }
 
@@ -48,15 +75,17 @@ binlogtool search *.binlog search string
                 var binlog = args[1];
                 var outputFile = args[2];
 
-                new SaveStrings().Run(binlog, outputFile);
+                new SaveStrings() { CompatibilityHandler = forwardCompatibilityReadingHandler }
+                    .Run(binlog, outputFile);
                 return;
             }
 
             if (args.Length == 2 && string.Equals(firstArg, "listtools", StringComparison.OrdinalIgnoreCase))
             {
                 var binlog = args[1];
 
-                new ListTools().Run(binlog);
+                new ListTools() { CompatibilityHandler = forwardCompatibilityReadingHandler }
+                    .Run(binlog);
                 return;
             }
 
@@ -70,12 +99,19 @@ binlogtool search *.binlog search string
 
                 var binlogs = args[1];
                 var search = string.Join(" ", args.Skip(2));
-                Searcher.Search(binlogs, search);
+                new Searcher() { CompatibilityHandler = forwardCompatibilityReadingHandler }
+                    .Search2(binlogs, search);
                 return;
             }
 
             if (firstArg == "redact")
             {
+                if (forwardCompatibilityReadingHandler.ForwardCompatibilityExplicitlyConfigured)
+                {
+                    Console.Error.WriteLine(
+                        "Forward compatibility mode will be ignored. Redact command doesn't interpret structured events - so it doesn't need to know the binlog format.");
+                }
+
                 List<string> redactTokens = new List<string>();
                 List<string> inputPaths = new List<string>();
                 bool recurse = false;

src/BinlogTool/SaveFiles.cs

@@ -7,7 +7,7 @@
 
 namespace BinlogTool
 {
-    public class SaveFiles
+    public class SaveFiles : BinlogToolCommandBase
     {
         private string[] args;
 
@@ -18,7 +18,8 @@ public SaveFiles(string[] args)
 
         public void Run(string binlog, string outputDirectory, bool reconstruct = false)
         {
-            if (string.IsNullOrEmpty(binlog) || !File.Exists(binlog))
+            Build build = this.ReadBuild(binlog, false);
+            if (build == null)
             {
                 return;
             }
@@ -29,15 +30,14 @@ public void Run(string binlog, string outputDirectory, bool reconstruct = false)
                 Directory.CreateDirectory(outputDirectory);
             }
 
-            binlog = Path.GetFullPath(binlog);
-
-            var build = BinaryLog.ReadBuild(binlog);
             SaveFilesFrom(build, outputDirectory);
 
             if (reconstruct)
             {
                 GenerateSources(build, outputDirectory);
             }
+
+            CompatibilityHandler?.HandleBuildResults(build);
         }
 
         private void GenerateSources(Build build, string outputDirectory)

src/BinlogTool/SaveStrings.cs

@@ -1,16 +1,18 @@
-using System.Linq;
+using System.Linq;
 using Microsoft.Build.Logging.StructuredLogger;
 
 namespace BinlogTool
 {
-    public class SaveStrings
+    public class SaveStrings : BinlogToolCommandBase
     {
         public void Run(string binLogFilePath, string outputFilePath)
         {
-            var build = BinaryLog.ReadBuild(binLogFilePath);
+            var build = this.ReadBuild(binLogFilePath);
             var strings = build.StringTable.Instances.OrderBy(s => s).ToArray();
 
             Serialization.WriteStringsToFile(outputFilePath, strings);
+
+            this.CompatibilityHandler?.HandleBuildResults(build);
         }
     }
 }