[Binlogs Redacting] API and support for external redaction functionality

src/Build/Logging/BinaryLogger/BinaryLogRecordKind.cs

@@ -24,7 +24,7 @@ internal enum BinaryLogRecordKind
         ProjectEvaluationStarted,
         ProjectEvaluationFinished,
         ProjectImported,
-        ProjectImportArchive,
+        ProjectImportArchive = 17,
         TargetSkipped,
         PropertyReassignment,
         UninitializedPropertyRead,

src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs

@@ -16,7 +16,7 @@ namespace Microsoft.Build.Logging
     /// by implementing IEventSource and raising corresponding events.
     /// </summary>
     /// <remarks>The class is public so that we can call it from MSBuild.exe when replaying a log file.</remarks>
-    public sealed class BinaryLogReplayEventSource : EventArgsDispatcher
+    public sealed class BinaryLogReplayEventSource : EventArgsDispatcher, IEmbeddedContentSource
     {
         /// Touches the <see cref="ItemGroupLoggingHelper"/> static constructor
         /// to ensure it initializes <see cref="TaskParameterEventArgs.MessageGetter"/>
@@ -26,11 +26,6 @@ static BinaryLogReplayEventSource()
             _ = ItemGroupLoggingHelper.ItemGroupIncludeLogMessagePrefix;
         }
 
-        /// <summary>
-        /// Raised once <see cref="BuildEventArgsReader"/> is created during replaying
-        /// </summary>
-        public event Action<IBuildEventArgsReaderNotifications>? NotificationsSourceCreated;
-
         /// <summary>
         /// Read the provided binary log file and raise corresponding events for each BuildEventArgs
         /// </summary>
@@ -68,41 +63,91 @@ public static BinaryReader OpenReader(string sourceFilePath)
             }
         }
 
+        /// <summary>
+        /// Creates a <see cref="BuildEventArgsReader"/> for the provided binary reader over binary log file.
+        /// Caller is responsible for disposing the returned reader.
+        /// </summary>
+        /// <param name="binaryReader"></param>
+        /// <param name="closeInput">Indicates whether the passed BinaryReader should be closed on disposing.</param>
+        /// <returns>BuildEventArgsReader over the given binlog file binary reader.</returns>
+        public static BuildEventArgsReader OpenBuildEventsReader(BinaryReader binaryReader, bool closeInput)
+        {
+            int fileFormatVersion = binaryReader.ReadInt32();
+
+            // the log file is written using a newer version of file format
+            // that we don't know how to read
+            if (fileFormatVersion > BinaryLogger.FileFormatVersion)
+            {
+                var text = ResourceUtilities.FormatResourceStringStripCodeAndKeyword("UnsupportedLogFileFormat", fileFormatVersion, BinaryLogger.FileFormatVersion);
+                throw new NotSupportedException(text);
+            }
+
+            return new BuildEventArgsReader(binaryReader, fileFormatVersion) { CloseInput = closeInput };
+        }
+
+        /// <summary>
+        /// Creates a <see cref="BinaryReader"/> for the provided binary log file.
+        /// Performs decompression and buffering in the optimal way.
+        /// Caller is responsible for disposing the returned reader.
+        /// </summary>
+        /// <param name="sourceFilePath"></param>
+        /// <returns>BinaryReader of the given binlog file.</returns>
+        public static BuildEventArgsReader OpenBuildEventsReader(string sourceFilePath)
+            => OpenBuildEventsReader(OpenReader(sourceFilePath), true);
+
         /// <summary>
         /// Read the provided binary log file and raise corresponding events for each BuildEventArgs
         /// </summary>
         /// <param name="sourceFilePath">The full file path of the binary log file</param>
         /// <param name="cancellationToken">A <see cref="CancellationToken"/> indicating the replay should stop as soon as possible.</param>
         public void Replay(string sourceFilePath, CancellationToken cancellationToken)
         {
-            using var binaryReader = OpenReader(sourceFilePath);
-            Replay(binaryReader, cancellationToken);
+            using var eventsReader = OpenBuildEventsReader(sourceFilePath);
+            Replay(eventsReader, cancellationToken);
         }
 
         /// <summary>
         /// Read the provided binary log file and raise corresponding events for each BuildEventArgs
         /// </summary>
         /// <param name="binaryReader">The binary log content binary reader - caller is responsible for disposing.</param>
+        /// <param name="closeInput">Indicates whether the passed BinaryReader should be closed on disposing.</param>
         /// <param name="cancellationToken">A <see cref="CancellationToken"/> indicating the replay should stop as soon as possible.</param>
-        public void Replay(BinaryReader binaryReader, CancellationToken cancellationToken)
+        public void Replay(BinaryReader binaryReader, bool closeInput, CancellationToken cancellationToken)
         {
-            int fileFormatVersion = binaryReader.ReadInt32();
-
-            // the log file is written using a newer version of file format
-            // that we don't know how to read
-            if (fileFormatVersion > BinaryLogger.FileFormatVersion)
-            {
-                var text = ResourceUtilities.FormatResourceStringStripCodeAndKeyword("UnsupportedLogFileFormat", fileFormatVersion, BinaryLogger.FileFormatVersion);
-                throw new NotSupportedException(text);
-            }
+            using var reader = OpenBuildEventsReader(binaryReader, closeInput);
+            Replay(reader, cancellationToken);
+        }
 
-            using var reader = new BuildEventArgsReader(binaryReader, fileFormatVersion);
-            NotificationsSourceCreated?.Invoke(reader);
+        /// <summary>
+        /// Read the provided binary log file and raise corresponding events for each BuildEventArgs
+        /// </summary>
+        /// <param name="reader">The build events reader - caller is responsible for disposing.</param>
+        /// <param name="cancellationToken">A <see cref="CancellationToken"/> indicating the replay should stop as soon as possible.</param>
+        public void Replay(BuildEventArgsReader reader, CancellationToken cancellationToken)
+        {
+            _fileFormatVersionRead?.Invoke(reader.FileFormatVersion);
+            reader.EmbeddedContentRead += _embeddedContentRead;
 
             while (!cancellationToken.IsCancellationRequested && reader.Read() is { } instance)
             {
                 Dispatch(instance);
             }
         }
+
+        private Action<int>? _fileFormatVersionRead;
+        event Action<int> ILogVersionInfo.FileFormatVersionRead
+        {
+            add => _fileFormatVersionRead += value;
+            remove => _fileFormatVersionRead -= value;
+        }
+        private Action<EmbeddedContentEventArgs>? _embeddedContentRead;
+        /// <inheritdoc cref="IEmbeddedContentSource.EmbeddedContentRead"/>
+        event Action<EmbeddedContentEventArgs>? IEmbeddedContentSource.EmbeddedContentRead
+        {
+            // Explicitly implemented event has to declare explicit add/remove accessors
+            //  https://stackoverflow.com/a/2268472/2308106
+            add => _embeddedContentRead += value;
+            remove => _embeddedContentRead -= value;
+        }
     }
 }

src/Build/Logging/BinaryLogger/BinaryLogger.cs

@@ -61,7 +61,10 @@ public sealed class BinaryLogger : ILogger
         //   - new record kind: ResponseFileUsedEventArgs
         // version 16:
         //   - AssemblyLoadBuildEventArgs
-        internal const int FileFormatVersion = 16;
+        // version 17:
+        //   - Making ProjectStartedEventArgs, ProjectEvaluationFinishedEventArgs, AssemblyLoadBuildEventArgs equal
+        //     between de/serialization roundtrips.
+        internal const int FileFormatVersion = 17;
 
         private Stream stream;
         private BinaryWriter binaryWriter;
@@ -90,7 +93,12 @@ public enum ProjectImportsCollectionMode
             /// <summary>
             /// Create an external .ProjectImports.zip archive for the project files.
             /// </summary>
-            ZipFile
+            ZipFile,
+
+            /// <summary>
+            /// Don't collect any files from build events, but instead replay them from the given event source (if that one supports it).
+            /// </summary>
+            Replay,
         }
 
         /// <summary>
@@ -113,7 +121,7 @@ public enum ProjectImportsCollectionMode
         public string Parameters { get; set; }
 
         /// <summary>
-        /// Initializes the logger by subscribing to events of the specified event source.
+        /// Initializes the logger by subscribing to events of the specified event source and embedded content source.
         /// </summary>
         public void Initialize(IEventSource eventSource)
         {
@@ -128,7 +136,9 @@ public void Initialize(IEventSource eventSource)
             Traits.Instance.EscapeHatches.LogProjectImports = true;
             bool logPropertiesAndItemsAfterEvaluation = Traits.Instance.EscapeHatches.LogPropertiesAndItemsAfterEvaluation ?? true;
 
-            ProcessParameters();
+            bool replayInitialInfo;
+            ILogVersionInfo versionInfo = null;
+            ProcessParameters(out replayInitialInfo);
 
             try
             {
@@ -150,7 +160,7 @@ public void Initialize(IEventSource eventSource)
 
                 stream = new FileStream(FilePath, FileMode.Create);
 
-                if (CollectProjectImports != ProjectImportsCollectionMode.None)
+                if (CollectProjectImports != ProjectImportsCollectionMode.None && CollectProjectImports != ProjectImportsCollectionMode.Replay)
                 {
                     projectImportsCollector = new ProjectImportsCollector(FilePath, CollectProjectImports == ProjectImportsCollectionMode.ZipFile);
                 }
@@ -164,6 +174,20 @@ public void Initialize(IEventSource eventSource)
                 {
                     eventSource4.IncludeEvaluationPropertiesAndItems();
                 }
+
+                if (eventSource is IEmbeddedContentSource embeddedFilesSource)
+                {
+                    if (CollectProjectImports == ProjectImportsCollectionMode.Replay)
+                    {
+                        embeddedFilesSource.EmbeddedContentRead += args =>
+                            eventArgsWriter.WriteBlob(args.ContentKind.ToBinaryLogRecordKind(), args.ContentStream, args.Length);
+                    }
+
+                    if (replayInitialInfo)
+                    {
+                        versionInfo = embeddedFilesSource;
+                    }
+                }
             }
             catch (Exception e)
             {
@@ -178,7 +202,9 @@ public void Initialize(IEventSource eventSource)
             // wrapping the GZipStream in a buffered stream significantly improves performance
             // and the max throughput is reached with a 32K buffer. See details here:
             // https://github.com/dotnet/runtime/issues/39233#issuecomment-745598847
-            stream = new BufferedStream(stream, bufferSize: 32768);
+            stream = Traits.Instance.DeterministicBinlogStreamBuffering ?
+                new GreedyBufferedStream(stream, bufferSize: 32768) :
+                new BufferedStream(stream, bufferSize: 32768);
             binaryWriter = new BinaryWriter(stream);
             eventArgsWriter = new BuildEventArgsWriter(binaryWriter);
 
@@ -187,9 +213,15 @@ public void Initialize(IEventSource eventSource)
                 eventArgsWriter.EmbedFile += EventArgsWriter_EmbedFile;
             }
 
-            binaryWriter.Write(FileFormatVersion);
-
-            LogInitialInfo();
+            if (versionInfo == null)
+            {
+                binaryWriter.Write(FileFormatVersion);
+                LogInitialInfo();
+            }
+            else
+            {
+                versionInfo.FileFormatVersionRead += version => binaryWriter.Write(version);
+            }
 
             eventSource.AnyEventRaised += EventSource_AnyEventRaised;
 
@@ -228,32 +260,18 @@ public void Shutdown()
 
             Traits.Instance.EscapeHatches.LogProjectImports = _initialLogImports;
 
+
             if (projectImportsCollector != null)
             {
                 projectImportsCollector.Close();
 
                 if (CollectProjectImports == ProjectImportsCollectionMode.Embed)
                 {
-                    var archiveFilePath = projectImportsCollector.ArchiveFilePath;
+                    projectImportsCollector.ProcessResult(
+                        streamToEmbed => eventArgsWriter.WriteBlob(BinaryLogRecordKind.ProjectImportArchive, streamToEmbed),
+                        LogMessage);
 
-                    // It is possible that the archive couldn't be created for some reason.
-                    // Only embed it if it actually exists.
-                    if (FileSystems.Default.FileExists(archiveFilePath))
-                    {
-                        using (FileStream fileStream = File.OpenRead(archiveFilePath))
-                        {
-                            if (fileStream.Length > int.MaxValue)
-                            {
-                                LogMessage("Imported files archive exceeded 2GB limit and it's not embedded.");
-                            }
-                            else
-                            {
-                                eventArgsWriter.WriteBlob(BinaryLogRecordKind.ProjectImportArchive, fileStream);
-                            }
-                        }
-
-                        File.Delete(archiveFilePath);
-                    }
+                    projectImportsCollector.DeleteArchive();
                 }
 
                 projectImportsCollector = null;
@@ -302,7 +320,7 @@ private void CollectImports(BuildEventArgs e)
             {
                 projectImportsCollector.AddFile(projectArgs.ProjectFile);
             }
-            else if (e is MetaprojectGeneratedEventArgs metaprojectArgs)
+            else if (e is MetaprojectGeneratedEventArgs { metaprojectXml: { } } metaprojectArgs)
             {
                 projectImportsCollector.AddFileFromMemory(metaprojectArgs.ProjectFile, metaprojectArgs.metaprojectXml);
             }
@@ -317,13 +335,14 @@ private void CollectImports(BuildEventArgs e)
         /// </summary>
         /// <exception cref="LoggerException">
         /// </exception>
-        private void ProcessParameters()
+        private void ProcessParameters(out bool replayInitialInfo)
         {
             if (Parameters == null)
             {
                 throw new LoggerException(ResourceUtilities.FormatResourceStringStripCodeAndKeyword("InvalidBinaryLoggerParameters", ""));
             }
 
+            replayInitialInfo = false;
             var parameters = Parameters.Split(MSBuildConstants.SemicolonChar, StringSplitOptions.RemoveEmptyEntries);
             foreach (var parameter in parameters)
             {
@@ -339,6 +358,14 @@ private void ProcessParameters()
                 {
                     CollectProjectImports = ProjectImportsCollectionMode.ZipFile;
                 }
+                else if (string.Equals(parameter, "ProjectImports=Replay", StringComparison.OrdinalIgnoreCase))
+                {
+                    CollectProjectImports = ProjectImportsCollectionMode.Replay;
+                }
+                else if (string.Equals(parameter, "ReplayInitialInfo", StringComparison.OrdinalIgnoreCase))
+                {
+                    replayInitialInfo = true;
+                }
                 else if (parameter.EndsWith(".binlog", StringComparison.OrdinalIgnoreCase))
                 {
                     FilePath = parameter;