diff --git a/src/Build/CompatibilitySuppressions.xml b/src/Build/CompatibilitySuppressions.xml index 05317adadab..23427182e98 100644 --- a/src/Build/CompatibilitySuppressions.xml +++ b/src/Build/CompatibilitySuppressions.xml @@ -1,4 +1,174 @@  - + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.add_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.remove_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.add_StringEncountered(System.Action) + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.remove_StringEncountered(System.Action) + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.IBuildEventStringsReader.add_StringEncountered(System.Action) + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.IBuildEventStringsReader.remove_StringEncountered(System.Action) + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.add_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.remove_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.add_StringEncountered(System.Action) + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.remove_StringEncountered(System.Action) + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.IBuildEventStringsReader.add_StringEncountered(System.Action) + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.IBuildEventStringsReader.remove_StringEncountered(System.Action) + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.add_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.remove_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.add_StringEncountered(System.Action) + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.remove_StringEncountered(System.Action) + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.IBuildEventStringsReader.add_StringEncountered(System.Action) + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.IBuildEventStringsReader.remove_StringEncountered(System.Action) + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.add_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.remove_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.add_StringEncountered(System.Action) + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.remove_StringEncountered(System.Action) + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.IBuildEventStringsReader.add_StringEncountered(System.Action) + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.IBuildEventStringsReader.remove_StringEncountered(System.Action) + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + \ No newline at end of file diff --git a/src/Build/Logging/BinaryLogger/BinaryLogRecordKind.cs b/src/Build/Logging/BinaryLogger/BinaryLogRecordKind.cs index 9fe1638fd3a..28333110721 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogRecordKind.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogRecordKind.cs @@ -24,7 +24,7 @@ internal enum BinaryLogRecordKind ProjectEvaluationStarted, ProjectEvaluationFinished, ProjectImported, - ProjectImportArchive, + ProjectImportArchive = 17, TargetSkipped, PropertyReassignment, UninitializedPropertyRead, diff --git a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs index a19a06c2d37..b6128c390d2 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs @@ -16,7 +16,7 @@ namespace Microsoft.Build.Logging /// by implementing IEventSource and raising corresponding events. /// /// The class is public so that we can call it from MSBuild.exe when replaying a log file. - public sealed class BinaryLogReplayEventSource : EventArgsDispatcher + public sealed class BinaryLogReplayEventSource : EventArgsDispatcher, IEmbeddedContentSource { /// Touches the static constructor /// to ensure it initializes @@ -26,11 +26,6 @@ static BinaryLogReplayEventSource() _ = ItemGroupLoggingHelper.ItemGroupIncludeLogMessagePrefix; } - /// - /// Raised once is created during replaying - /// - public event Action? NotificationsSourceCreated; - /// /// Read the provided binary log file and raise corresponding events for each BuildEventArgs /// @@ -68,6 +63,38 @@ public static BinaryReader OpenReader(string sourceFilePath) } } + /// + /// Creates a for the provided binary reader over binary log file. + /// Caller is responsible for disposing the returned reader. + /// + /// + /// Indicates whether the passed BinaryReader should be closed on disposing. + /// BuildEventArgsReader over the given binlog file binary reader. + public static BuildEventArgsReader OpenBuildEventsReader(BinaryReader binaryReader, bool closeInput) + { + int fileFormatVersion = binaryReader.ReadInt32(); + + // the log file is written using a newer version of file format + // that we don't know how to read + if (fileFormatVersion > BinaryLogger.FileFormatVersion) + { + var text = ResourceUtilities.FormatResourceStringStripCodeAndKeyword("UnsupportedLogFileFormat", fileFormatVersion, BinaryLogger.FileFormatVersion); + throw new NotSupportedException(text); + } + + return new BuildEventArgsReader(binaryReader, fileFormatVersion) { CloseInput = closeInput }; + } + + /// + /// Creates a for the provided binary log file. + /// Performs decompression and buffering in the optimal way. + /// Caller is responsible for disposing the returned reader. + /// + /// + /// BinaryReader of the given binlog file. + public static BuildEventArgsReader OpenBuildEventsReader(string sourceFilePath) + => OpenBuildEventsReader(OpenReader(sourceFilePath), true); + /// /// Read the provided binary log file and raise corresponding events for each BuildEventArgs /// @@ -75,34 +102,52 @@ public static BinaryReader OpenReader(string sourceFilePath) /// A indicating the replay should stop as soon as possible. public void Replay(string sourceFilePath, CancellationToken cancellationToken) { - using var binaryReader = OpenReader(sourceFilePath); - Replay(binaryReader, cancellationToken); + using var eventsReader = OpenBuildEventsReader(sourceFilePath); + Replay(eventsReader, cancellationToken); } /// /// Read the provided binary log file and raise corresponding events for each BuildEventArgs /// /// The binary log content binary reader - caller is responsible for disposing. + /// Indicates whether the passed BinaryReader should be closed on disposing. /// A indicating the replay should stop as soon as possible. - public void Replay(BinaryReader binaryReader, CancellationToken cancellationToken) + public void Replay(BinaryReader binaryReader, bool closeInput, CancellationToken cancellationToken) { - int fileFormatVersion = binaryReader.ReadInt32(); - - // the log file is written using a newer version of file format - // that we don't know how to read - if (fileFormatVersion > BinaryLogger.FileFormatVersion) - { - var text = ResourceUtilities.FormatResourceStringStripCodeAndKeyword("UnsupportedLogFileFormat", fileFormatVersion, BinaryLogger.FileFormatVersion); - throw new NotSupportedException(text); - } + using var reader = OpenBuildEventsReader(binaryReader, closeInput); + Replay(reader, cancellationToken); + } - using var reader = new BuildEventArgsReader(binaryReader, fileFormatVersion); - NotificationsSourceCreated?.Invoke(reader); + /// + /// Read the provided binary log file and raise corresponding events for each BuildEventArgs + /// + /// The build events reader - caller is responsible for disposing. + /// A indicating the replay should stop as soon as possible. + public void Replay(BuildEventArgsReader reader, CancellationToken cancellationToken) + { + _fileFormatVersionRead?.Invoke(reader.FileFormatVersion); + reader.EmbeddedContentRead += _embeddedContentRead; while (!cancellationToken.IsCancellationRequested && reader.Read() is { } instance) { Dispatch(instance); } } + + private Action? _fileFormatVersionRead; + event Action ILogVersionInfo.FileFormatVersionRead + { + add => _fileFormatVersionRead += value; + remove => _fileFormatVersionRead -= value; + } + private Action? _embeddedContentRead; + /// + event Action? IEmbeddedContentSource.EmbeddedContentRead + { + // Explicitly implemented event has to declare explicit add/remove accessors + // https://stackoverflow.com/a/2268472/2308106 + add => _embeddedContentRead += value; + remove => _embeddedContentRead -= value; + } } } diff --git a/src/Build/Logging/BinaryLogger/BinaryLogger.cs b/src/Build/Logging/BinaryLogger/BinaryLogger.cs index 6c4e32345fb..94b28d10eb2 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogger.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogger.cs @@ -63,7 +63,10 @@ public sealed class BinaryLogger : ILogger // - AssemblyLoadBuildEventArgs // version 17: // - Added extended data for types implementing IExtendedBuildEventArgs - internal const int FileFormatVersion = 17; + // version 18: + // - Making ProjectStartedEventArgs, ProjectEvaluationFinishedEventArgs, AssemblyLoadBuildEventArgs equal + // between de/serialization roundtrips. + internal const int FileFormatVersion = 18; private Stream stream; private BinaryWriter binaryWriter; @@ -92,7 +95,12 @@ public enum ProjectImportsCollectionMode /// /// Create an external .ProjectImports.zip archive for the project files. /// - ZipFile + ZipFile, + + /// + /// Don't collect any files from build events, but instead replay them from the given event source (if that one supports it). + /// + Replay, } /// @@ -115,7 +123,7 @@ public enum ProjectImportsCollectionMode public string Parameters { get; set; } /// - /// Initializes the logger by subscribing to events of the specified event source. + /// Initializes the logger by subscribing to events of the specified event source and embedded content source. /// public void Initialize(IEventSource eventSource) { @@ -130,7 +138,9 @@ public void Initialize(IEventSource eventSource) Traits.Instance.EscapeHatches.LogProjectImports = true; bool logPropertiesAndItemsAfterEvaluation = Traits.Instance.EscapeHatches.LogPropertiesAndItemsAfterEvaluation ?? true; - ProcessParameters(); + bool replayInitialInfo; + ILogVersionInfo versionInfo = null; + ProcessParameters(out replayInitialInfo); try { @@ -152,7 +162,7 @@ public void Initialize(IEventSource eventSource) stream = new FileStream(FilePath, FileMode.Create); - if (CollectProjectImports != ProjectImportsCollectionMode.None) + if (CollectProjectImports != ProjectImportsCollectionMode.None && CollectProjectImports != ProjectImportsCollectionMode.Replay) { projectImportsCollector = new ProjectImportsCollector(FilePath, CollectProjectImports == ProjectImportsCollectionMode.ZipFile); } @@ -166,6 +176,20 @@ public void Initialize(IEventSource eventSource) { eventSource4.IncludeEvaluationPropertiesAndItems(); } + + if (eventSource is IEmbeddedContentSource embeddedFilesSource) + { + if (CollectProjectImports == ProjectImportsCollectionMode.Replay) + { + embeddedFilesSource.EmbeddedContentRead += args => + eventArgsWriter.WriteBlob(args.ContentKind.ToBinaryLogRecordKind(), args.ContentStream, args.Length); + } + + if (replayInitialInfo) + { + versionInfo = embeddedFilesSource; + } + } } catch (Exception e) { @@ -180,7 +204,9 @@ public void Initialize(IEventSource eventSource) // wrapping the GZipStream in a buffered stream significantly improves performance // and the max throughput is reached with a 32K buffer. See details here: // https://github.com/dotnet/runtime/issues/39233#issuecomment-745598847 - stream = new BufferedStream(stream, bufferSize: 32768); + stream = Traits.Instance.DeterministicBinlogStreamBuffering ? + new GreedyBufferedStream(stream, bufferSize: 32768) : + new BufferedStream(stream, bufferSize: 32768); binaryWriter = new BinaryWriter(stream); eventArgsWriter = new BuildEventArgsWriter(binaryWriter); @@ -189,9 +215,15 @@ public void Initialize(IEventSource eventSource) eventArgsWriter.EmbedFile += EventArgsWriter_EmbedFile; } - binaryWriter.Write(FileFormatVersion); - - LogInitialInfo(); + if (versionInfo == null) + { + binaryWriter.Write(FileFormatVersion); + LogInitialInfo(); + } + else + { + versionInfo.FileFormatVersionRead += version => binaryWriter.Write(version); + } eventSource.AnyEventRaised += EventSource_AnyEventRaised; @@ -230,32 +262,18 @@ public void Shutdown() Traits.Instance.EscapeHatches.LogProjectImports = _initialLogImports; + if (projectImportsCollector != null) { projectImportsCollector.Close(); if (CollectProjectImports == ProjectImportsCollectionMode.Embed) { - var archiveFilePath = projectImportsCollector.ArchiveFilePath; + projectImportsCollector.ProcessResult( + streamToEmbed => eventArgsWriter.WriteBlob(BinaryLogRecordKind.ProjectImportArchive, streamToEmbed), + LogMessage); - // It is possible that the archive couldn't be created for some reason. - // Only embed it if it actually exists. - if (FileSystems.Default.FileExists(archiveFilePath)) - { - using (FileStream fileStream = File.OpenRead(archiveFilePath)) - { - if (fileStream.Length > int.MaxValue) - { - LogMessage("Imported files archive exceeded 2GB limit and it's not embedded."); - } - else - { - eventArgsWriter.WriteBlob(BinaryLogRecordKind.ProjectImportArchive, fileStream); - } - } - - File.Delete(archiveFilePath); - } + projectImportsCollector.DeleteArchive(); } projectImportsCollector = null; @@ -304,7 +322,7 @@ private void CollectImports(BuildEventArgs e) { projectImportsCollector.AddFile(projectArgs.ProjectFile); } - else if (e is MetaprojectGeneratedEventArgs metaprojectArgs) + else if (e is MetaprojectGeneratedEventArgs { metaprojectXml: { } } metaprojectArgs) { projectImportsCollector.AddFileFromMemory(metaprojectArgs.ProjectFile, metaprojectArgs.metaprojectXml); } @@ -319,13 +337,14 @@ private void CollectImports(BuildEventArgs e) /// /// /// - private void ProcessParameters() + private void ProcessParameters(out bool replayInitialInfo) { if (Parameters == null) { throw new LoggerException(ResourceUtilities.FormatResourceStringStripCodeAndKeyword("InvalidBinaryLoggerParameters", "")); } + replayInitialInfo = false; var parameters = Parameters.Split(MSBuildConstants.SemicolonChar, StringSplitOptions.RemoveEmptyEntries); foreach (var parameter in parameters) { @@ -341,6 +360,14 @@ private void ProcessParameters() { CollectProjectImports = ProjectImportsCollectionMode.ZipFile; } + else if (string.Equals(parameter, "ProjectImports=Replay", StringComparison.OrdinalIgnoreCase)) + { + CollectProjectImports = ProjectImportsCollectionMode.Replay; + } + else if (string.Equals(parameter, "ReplayInitialInfo", StringComparison.OrdinalIgnoreCase)) + { + replayInitialInfo = true; + } else if (parameter.EndsWith(".binlog", StringComparison.OrdinalIgnoreCase)) { FilePath = parameter; diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs index 55c330ccd27..dedc7c8a1ab 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs @@ -5,6 +5,8 @@ using System.Collections; using System.Collections.Generic; using System.IO; +using System.IO.Compression; +using System.Linq; using System.Reflection; using System.Text; using Microsoft.Build.BackEnd; @@ -63,9 +65,19 @@ public BuildEventArgsReader(BinaryReader binaryReader, int fileFormatVersion) this.fileFormatVersion = fileFormatVersion; } + /// + /// Directs whether the passed should be closed when this instance is disposed. + /// Defaults to "false". + /// + public bool CloseInput { private get; set; } = false; + public void Dispose() { stringStorage.Dispose(); + if (CloseInput) + { + binaryReader.Dispose(); + } } /// @@ -75,17 +87,17 @@ public void Dispose() /// public event Action? StringReadDone; - /// - /// An event that allows the caller to be notified when a string is encountered in the binary log. - /// BinaryReader passed in ctor is at the beginning of the string at this point. - /// - public event Action? StringEncountered; + public int FileFormatVersion => fileFormatVersion; /// - /// Raised when the log reader encounters a binary blob embedded in the stream. - /// The arguments include the blob kind and the byte buffer with the contents. + /// Raised when the log reader encounters a project import archive (embedded content) in the stream. + /// The subscriber must read the exactly given length of binary data from the stream - otherwise exception is raised. + /// If no subscriber is attached, the data is skipped. /// - internal event Action? OnBlobRead; + internal event Action? EmbeddedContentRead; + + /// + public event Action? ArchiveFileEncountered; /// /// Reads the next log record from the . @@ -113,7 +125,7 @@ public void Dispose() } else if (recordKind == BinaryLogRecordKind.ProjectImportArchive) { - ReadBlob(recordKind); + ReadEmbeddedContent(recordKind); } recordNumber += 1; @@ -212,11 +224,86 @@ private static bool IsAuxiliaryRecord(BinaryLogRecordKind recordKind) || recordKind == BinaryLogRecordKind.ProjectImportArchive; } - private void ReadBlob(BinaryLogRecordKind kind) + private void ReadEmbeddedContent(BinaryLogRecordKind recordKind) { int length = ReadInt32(); - byte[] bytes = binaryReader.ReadBytes(length); - OnBlobRead?.Invoke(kind, bytes); + + if (ArchiveFileEncountered != null) + { + // We could create ZipArchive over the target stream, and write to that directly, + // however, binlog format needs to know stream size upfront - which is unknown, + // so we would need to write the size after - and that would require the target stream to be seekable (which it's not) + ProjectImportsCollector? projectImportsCollector = null; + + if (EmbeddedContentRead != null) + { + projectImportsCollector = + new ProjectImportsCollector(Path.GetRandomFileName(), false, runOnBackground: false); + } + + Stream embeddedStream = new SubStream(binaryReader.BaseStream, length); + + // We are intentionally not grace handling corrupt embedded stream + + using var zipArchive = new ZipArchive(embeddedStream, ZipArchiveMode.Read); + + foreach (var entry in zipArchive.Entries/*.OrderBy(e => e.LastWriteTime)*/) + { + var file = ArchiveFile.From(entry); + ArchiveFileEventArgs archiveFileEventArgs = new(file); + // ArchiveFileEventArgs is not IDisposable as we do not want to clutter exposed API + using var cleanupScope = new CleanupScope(archiveFileEventArgs.Dispose); + ArchiveFileEncountered(archiveFileEventArgs); + + if (projectImportsCollector != null) + { + var resultFile = archiveFileEventArgs.ObtainArchiveFile(); + + if (resultFile.CanUseReader) + { + projectImportsCollector.AddFileFromMemory( + resultFile.FullPath, + resultFile.GetContentReader().BaseStream, + makePathAbsolute: false, + entryCreationStamp: entry.LastWriteTime); + } + else + { + projectImportsCollector.AddFileFromMemory( + resultFile.FullPath, + resultFile.GetContent(), + encoding: resultFile.Encoding, + makePathAbsolute: false, + entryCreationStamp: entry.LastWriteTime); + } + } + } + + if (EmbeddedContentRead != null) + { + projectImportsCollector!.ProcessResult( + streamToEmbed => EmbeddedContentRead(new EmbeddedContentEventArgs(EmbeddedContentKind.ProjectImportArchive, streamToEmbed)), + error => throw new InvalidDataException(error)); + projectImportsCollector.DeleteArchive(); + } + } + else if (EmbeddedContentRead != null) + { + EmbeddedContentRead(new EmbeddedContentEventArgs( + recordKind.ToEmbeddedContentKind(), + new SubStream(binaryReader.BaseStream, length))); + } + else + { + if (binaryReader.BaseStream.CanSeek) + { + binaryReader.BaseStream.Seek(length, SeekOrigin.Current); + } + else + { + binaryReader.ReadBytes(length); + } + } } private void ReadNameValueList() @@ -419,11 +506,12 @@ private BuildEventArgs ReadProjectEvaluationFinishedEventArgs() if (fileFormatVersion >= 12) { - IEnumerable? globalProperties = null; - if (ReadBoolean()) + if (fileFormatVersion < 18) { - globalProperties = ReadStringDictionary(); + // Throw away, but need to advance past it + ReadBoolean(); } + IEnumerable? globalProperties = ReadStringDictionary(); var propertyList = ReadPropertyList(); var itemList = ReadProjectItems(); @@ -474,10 +562,12 @@ private BuildEventArgs ReadProjectStartedEventArgs() if (fileFormatVersion > 6) { - if (ReadBoolean()) + if (fileFormatVersion < 18) { - globalProperties = ReadStringDictionary(); + // Throw away, but need to advance past it + ReadBoolean(); } + globalProperties = ReadStringDictionary(); } var propertyList = ReadPropertyList(); @@ -922,6 +1012,7 @@ private AssemblyLoadBuildEventArgs ReadAssemblyLoadEventArgs() mvid, appDomainName); SetCommonFields(e, fields); + e.ProjectFile = fields.ProjectFile; return e; } @@ -1274,7 +1365,6 @@ private ITaskItem ReadTaskItem() private string ReadString() { - this.StringEncountered?.Invoke(); string text = binaryReader.ReadString(); if (this.StringReadDone != null) { diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs index 0a21182e83c..99ffe22d96d 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Buffers; using System.Collections; using System.Collections.Generic; using System.Globalization; @@ -229,20 +230,9 @@ private void WriteCore(BuildEventArgs e) } } - public void WriteBlob(BinaryLogRecordKind kind, byte[] bytes) + public void WriteBlob(BinaryLogRecordKind kind, Stream stream, int? length = null) { - // write the blob directly to the underlying writer, - // bypassing the memory stream - using var redirection = RedirectWritesToOriginalWriter(); - - Write(kind); - Write(bytes.Length); - Write(bytes); - } - - public void WriteBlob(BinaryLogRecordKind kind, Stream stream) - { - if (stream.Length > int.MaxValue) + if (stream.CanSeek && stream.Length > int.MaxValue) { throw new ArgumentOutOfRangeException(nameof(stream)); } @@ -252,8 +242,8 @@ public void WriteBlob(BinaryLogRecordKind kind, Stream stream) using var redirection = RedirectWritesToOriginalWriter(); Write(kind); - Write((int)stream.Length); - Write(stream); + Write(length ?? (int)stream.Length); + Write(stream, length); } /// @@ -317,15 +307,7 @@ private void Write(ProjectEvaluationFinishedEventArgs e) WriteBuildEventArgsFields(e, writeMessage: false); WriteDeduplicatedString(e.ProjectFile); - if (e.GlobalProperties == null) - { - Write(false); - } - else - { - Write(true); - WriteProperties(e.GlobalProperties); - } + WriteProperties(e.GlobalProperties); WriteProperties(e.Properties); @@ -366,15 +348,7 @@ private void Write(ProjectStartedEventArgs e) WriteDeduplicatedString(e.TargetNames); WriteDeduplicatedString(e.ToolsVersion); - if (e.GlobalProperties == null) - { - Write(false); - } - else - { - Write(true); - Write(e.GlobalProperties); - } + Write(e.GlobalProperties); WriteProperties(e.Properties); @@ -1124,7 +1098,7 @@ private void Write(BinaryLogRecordKind kind) Write((int)kind); } - private void Write(int value) + internal void Write(int value) { BinaryWriterExtensions.Write7BitEncodedInt(binaryWriter, value); } @@ -1139,9 +1113,35 @@ private void Write(byte[] bytes) binaryWriter.Write(bytes); } - private void Write(Stream stream) + private void Write(Stream stream, int? length) { - stream.CopyTo(binaryWriter.BaseStream); + Stream destinationStream = binaryWriter.BaseStream; + if (length == null) + { + stream.CopyTo(destinationStream); + return; + } + + // borrowed from runtime from Stream.cs + const int defaultCopyBufferSize = 81920; + int bufferSize = Math.Min(defaultCopyBufferSize, length.Value); + + byte[] buffer = ArrayPool.Shared.Rent(bufferSize); + try + { + int bytesRead; + while ( + length > 0 && + (bytesRead = stream.Read(buffer, 0, Math.Min(buffer.Length, length.Value))) != 0) + { + destinationStream.Write(buffer, 0, bytesRead); + length -= bytesRead; + } + } + finally + { + ArrayPool.Shared.Return(buffer); + } } private void Write(byte b) diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs new file mode 100644 index 00000000000..88e80cd9d59 --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs @@ -0,0 +1,94 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.IO.Compression; +using System.Text; + +namespace Microsoft.Build.Logging +{ + public sealed class ArchiveFile + { + // We need to specify encoding without preamble - as then StreamReader will + // automatically adjust the encoding to match the preamble (if present). + // It will as well change to other encoding if detected. + private static readonly Encoding s_utf8WithoutPreamble = new UTF8Encoding(false); + + public ArchiveFile(string fullPath, Stream contentStream) + { + FullPath = fullPath; + _contentReader = new StreamReader(contentStream, s_utf8WithoutPreamble); + } + + public ArchiveFile(string fullPath, string content, Encoding? contentEncoding = null) + { + FullPath = fullPath; + _content = content; + _stringAcquired = true; + _contentReader = StreamReader.Null; + _stringEncoding = contentEncoding ?? Encoding.UTF8; + } + + internal static ArchiveFile From(ZipArchiveEntry entry) + { + return new ArchiveFile(entry.FullName, entry.Open()); + } + + public string FullPath { get; } + + public Encoding Encoding => _stringEncoding ?? _contentReader.CurrentEncoding; + + public bool CanUseReader => !_stringAcquired; + public bool CanUseString => !_streamAcquired; + + /// + /// Fetches the file content as a stream reader (forward only). + /// This prevents the content to be read as string. + /// + /// + /// + public StreamReader GetContentReader() + { + if (_stringAcquired) + { + throw new InvalidOperationException("Content already acquired as string via GetContent or initialized as string only."); + } + + _streamAcquired = true; + return _contentReader; + } + + /// + /// Fetches the file content as a string. + /// This prevents the content to be fetched via StreamReader. + /// + /// + /// + public string GetContent() + { + if (_streamAcquired) + { + throw new InvalidOperationException("Content already acquired as StreamReader via GetContnetReader."); + } + + if (!_stringAcquired) + { + _stringAcquired = true; + _content = _contentReader.ReadToEnd(); + } + + return _content!; + } + + private bool _streamAcquired; + private bool _stringAcquired; + private readonly StreamReader _contentReader; + private string? _content; + private readonly Encoding? _stringEncoding; + + // Intentionally not exposing this publicly (e.g. as IDisposable implementation) + // as we don't want to user to be bothered with ownership and disposing concerns. + internal void Dispose() => _contentReader.Dispose(); + } +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs new file mode 100644 index 00000000000..120362bcf55 --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs @@ -0,0 +1,54 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; + +namespace Microsoft.Build.Logging; + +public sealed class ArchiveFileEventArgs : EventArgs +{ + private ArchiveFile _archiveFile; + private bool _resultSet; + private Action _disposeAction; + + public ArchiveFileEventArgs(ArchiveFile archiveFile) => + (_archiveFile, _resultSet, _disposeAction) = (archiveFile, true, archiveFile.Dispose); + + /// + /// Acquires the instance. This method can only be called once and + /// or must be called afterwards + /// (this is because the embedded files are stored as forward only stream - reading them prevents re-reads). + /// + /// + /// + public ArchiveFile ObtainArchiveFile() + { + if (!_resultSet) + { + throw new InvalidOperationException( + "ArchiveFile was obtained, but the final edited version was not set."); + } + + _resultSet = false; + return _archiveFile; + } + + public void SetResult(string resultPath, Stream resultStream) + { + _archiveFile = new ArchiveFile(resultPath, resultStream); + _disposeAction += _archiveFile.Dispose; + _resultSet = true; + } + + public void SetResult(string resultPath, string resultContent) + { + _archiveFile = new ArchiveFile(resultPath, resultContent, _archiveFile.Encoding); + _disposeAction += _archiveFile.Dispose; + _resultSet = true; + } + + // Intentionally not exposing this publicly (e.g. as IDisposable implementation) + // as we don't want to user to be bothered with ownership and disposing concerns. + internal void Dispose() => _disposeAction(); +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgsExtensions.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgsExtensions.cs new file mode 100644 index 00000000000..818cffaa91a --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgsExtensions.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging; + +public static class ArchiveFileEventArgsExtensions +{ + public static Action ToArchiveFileHandler(this Action stringHandler) + { + return args => + { + var archiveFile = args.ObtainArchiveFile(); + var pathArgs = new StringReadEventArgs(archiveFile.FullPath); + stringHandler(pathArgs); + var contentArgs = new StringReadEventArgs(archiveFile.GetContent()); + stringHandler(contentArgs); + + args.SetResult(pathArgs.StringToBeUsed, contentArgs.StringToBeUsed); + }; + } +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/CleanupScope.cs b/src/Build/Logging/BinaryLogger/Postprocessing/CleanupScope.cs new file mode 100644 index 00000000000..bef26ff4d13 --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/CleanupScope.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging; + +internal class CleanupScope : IDisposable +{ + private readonly Action _disposeAction; + + public CleanupScope(Action disposeAction) => _disposeAction = disposeAction; + + public void Dispose() => _disposeAction(); +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs new file mode 100644 index 00000000000..68969f2af4a --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; + +namespace Microsoft.Build.Logging +{ + internal sealed class EmbeddedContentEventArgs : EventArgs + { + public EmbeddedContentEventArgs(EmbeddedContentKind contentKind, Stream contentStream, int length) + { + ContentKind = contentKind; + ContentStream = contentStream; + Length = length; + } + + public EmbeddedContentEventArgs(EmbeddedContentKind contentKind, Stream contentStream) + { + ContentKind = contentKind; + ContentStream = contentStream; + } + + public EmbeddedContentKind ContentKind { get; } + public Stream ContentStream { get; } + public int? Length { get; } + } +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs new file mode 100644 index 00000000000..2fca5d7eaa3 --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.Build.Logging +{ + internal enum EmbeddedContentKind + { + Unknown = -1, + ProjectImportArchive = 17, + } +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKindExtensions.cs b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKindExtensions.cs new file mode 100644 index 00000000000..73e9251cd77 --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKindExtensions.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Build.Logging +{ + internal static class EmbeddedContentKindExtensions + { + internal static EmbeddedContentKind ToEmbeddedContentKind(this BinaryLogRecordKind kind) + { + return kind == BinaryLogRecordKind.ProjectImportArchive + ? EmbeddedContentKind.ProjectImportArchive + : EmbeddedContentKind.Unknown; + } + + internal static BinaryLogRecordKind ToBinaryLogRecordKind(this EmbeddedContentKind kind) + { + return kind == EmbeddedContentKind.ProjectImportArchive + ? BinaryLogRecordKind.ProjectImportArchive + : (BinaryLogRecordKind)EmbeddedContentKind.Unknown; + } + } +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/GreedyBufferedStream.cs b/src/Build/Logging/BinaryLogger/Postprocessing/GreedyBufferedStream.cs new file mode 100644 index 00000000000..e334eac4b8f --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/GreedyBufferedStream.cs @@ -0,0 +1,83 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; + +namespace Microsoft.Build.Logging +{ + /// + /// This is write-only, append-only stream that always buffers the wrapped stream + /// into the chunks of the same size (except the possible shorter last chunk). + /// So unlike the it never writes to the wrapped stream + /// until it has full chunk or is closing. + /// + /// This is not supposed to bring performance benefits, but it allows to avoid nondeterministic + /// GZipStream output for the identical input. + /// + internal class GreedyBufferedStream : Stream + { + private readonly Stream _stream; + private readonly byte[] _buffer; + private int _position; + + public GreedyBufferedStream(Stream stream, int bufferSize) + { + _stream = stream; + _buffer = new byte[bufferSize]; + } + + public override void Flush() + { + _stream.Write(_buffer, 0, _position); + _position = 0; + } + + public override int Read(byte[] buffer, int offset, int count) => throw UnsupportedException; + + public override long Seek(long offset, SeekOrigin origin) => throw UnsupportedException; + + public override void SetLength(long value) => throw UnsupportedException; + + public override void Write(byte[] buffer, int offset, int count) + { + // Appends input to the buffer until it is full - then flushes it to the wrapped stream. + // Repeat above until all input is processed. + + int srcOffset = offset; + do + { + int currentCount = Math.Min(count, _buffer.Length - _position); + Buffer.BlockCopy(buffer, srcOffset, _buffer, _position, currentCount); + _position += currentCount; + count -= currentCount; + srcOffset += currentCount; + + if (_position == _buffer.Length) + { + Flush(); + } + } while (count > 0); + } + + public override bool CanRead => false; + public override bool CanSeek => false; + public override bool CanWrite => _stream.CanWrite; + public override long Length => _stream.Length + _position; + + public override long Position + { + get => _stream.Position + _position; + set => throw UnsupportedException; + } + + public override void Close() + { + Flush(); + _stream.Close(); + base.Close(); + } + + private Exception UnsupportedException => new NotSupportedException("GreedyBufferedStream is write-only, append-only"); + } +} diff --git a/src/Build/Logging/BinaryLogger/IBuildEventArgsReaderNotifications.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs similarity index 91% rename from src/Build/Logging/BinaryLogger/IBuildEventArgsReaderNotifications.cs rename to src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs index 415bd7c71fd..13bc343362a 100644 --- a/src/Build/Logging/BinaryLogger/IBuildEventArgsReaderNotifications.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs @@ -6,7 +6,7 @@ namespace Microsoft.Build.Logging /// /// An interface for notifications from BuildEventArgsReader /// - public interface IBuildEventArgsReaderNotifications : IBuildEventStringsReader + public interface IBuildEventArgsReaderNotifications : IBuildEventStringsReader, IBuildFileReader { /* For future use */ } diff --git a/src/Build/Logging/BinaryLogger/IBuildEventStringsReader.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs similarity index 72% rename from src/Build/Logging/BinaryLogger/IBuildEventStringsReader.cs rename to src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs index e9e7651ee78..bf3d54f8ff8 100644 --- a/src/Build/Logging/BinaryLogger/IBuildEventStringsReader.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs @@ -16,11 +16,5 @@ public interface IBuildEventStringsReader /// The passed event arg can be reused and should not be stored. /// public event Action? StringReadDone; - - /// - /// An event that allows the caller to be notified when a string is encountered in the binary log. - /// BinaryReader passed in ctor is at the beginning of the string at this point. - /// - public event Action? StringEncountered; } } diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/IBuildFileReader.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildFileReader.cs new file mode 100644 index 00000000000..9910ee1a06a --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildFileReader.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging; + +public interface IBuildFileReader +{ + /// + /// An event that allows the caller to be notified when an embedded file is encountered in the binary log. + /// Subscribing to this event obligates the subscriber to read the file content and set the resulting content + /// via or . + /// When subscriber is OK with greedy reading entire content of the file, it can simplify subscribing to this event, + /// by using handler with same signature as handler for and wrapping it via + /// extension. + /// + /// + /// + /// private void OnStringReadDone(StringReadEventArgs e) + /// { + /// e.StringToBeUsed = e.StringToBeUsed.Replace("foo", "bar"); + /// } + /// + /// private void SubscribeToEvents() + /// { + /// reader.StringReadDone += OnStringReadDone; + /// reader.ArchiveFileEncountered += ((Action<StringReadEventArgs>)OnStringReadDone).ToArchiveFileHandler(); + /// } + /// + /// + /// + public event Action? ArchiveFileEncountered; +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs new file mode 100644 index 00000000000..eb9262939fa --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging +{ + + internal interface ILogVersionInfo + { + event Action FileFormatVersionRead; + } + + internal interface IEmbeddedContentSource : ILogVersionInfo + { + + /// + /// Raised when the log reader encounters a project import archive (embedded content) in the stream. + /// The subscriber must read the exactly given length of binary data from the stream - otherwise exception is raised. + /// If no subscriber is attached, the data is skipped. + /// + event Action EmbeddedContentRead; + } +} diff --git a/src/Build/Logging/BinaryLogger/StringReadEventArgs.cs b/src/Build/Logging/BinaryLogger/Postprocessing/StringReadEventArgs.cs similarity index 100% rename from src/Build/Logging/BinaryLogger/StringReadEventArgs.cs rename to src/Build/Logging/BinaryLogger/Postprocessing/StringReadEventArgs.cs diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/SubStream.cs b/src/Build/Logging/BinaryLogger/Postprocessing/SubStream.cs new file mode 100644 index 00000000000..277c9ac66c0 --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/SubStream.cs @@ -0,0 +1,56 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.Build.Logging +{ + /// + /// Bounded read-only, forward-only view over an underlying stream. + /// + internal class SubStream : Stream + { + // Do not Dispose/Close on Dispose/Close !! + private readonly Stream _stream; + private readonly long _length; + private long _position; + + public SubStream(Stream stream, long length) + { + _stream = stream; + _length = length; + + if (!stream.CanRead) + { + throw new InvalidOperationException("Stream must be readable."); + } + } + + public override bool CanRead => true; + + public override bool CanSeek => false; + + public override bool CanWrite => false; + + public override long Length => _length; + + public override long Position { get => _position; set => throw new NotImplementedException(); } + + public override void Flush() { } + public override int Read(byte[] buffer, int offset, int count) + { + count = Math.Min((int)Math.Max(Length - _position, 0), count); + int read = _stream.Read(buffer, offset, count); + _position += read; + return read; + } + public override long Seek(long offset, SeekOrigin origin) => throw new NotImplementedException(); + public override void SetLength(long value) => throw new NotImplementedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotImplementedException(); + } +} diff --git a/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs b/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs index 27ededae8cc..24e86af2991 100644 --- a/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs +++ b/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs @@ -8,8 +8,7 @@ using System.Text; using System.Threading.Tasks; using Microsoft.Build.Shared; - -#nullable disable +using Microsoft.Build.Shared.FileSystem; namespace Microsoft.Build.Logging { @@ -21,10 +20,10 @@ namespace Microsoft.Build.Logging /// internal class ProjectImportsCollector { - private Stream _fileStream; - private ZipArchive _zipArchive; - - public string ArchiveFilePath { get; } + private Stream? _fileStream; + private ZipArchive? _zipArchive; + private readonly string _archiveFilePath; + private readonly bool _runOnBackground; /// /// Avoid visiting each file more than once. @@ -34,12 +33,16 @@ internal class ProjectImportsCollector // this will form a chain of file write tasks, running sequentially on a background thread private Task _currentTask = Task.CompletedTask; - public ProjectImportsCollector(string logFilePath, bool createFile, string sourcesArchiveExtension = ".ProjectImports.zip") + public ProjectImportsCollector( + string logFilePath, + bool createFile, + string sourcesArchiveExtension = ".ProjectImports.zip", + bool runOnBackground = true) { if (createFile) { // Archive file will be stored alongside the binlog - ArchiveFilePath = Path.ChangeExtension(logFilePath, sourcesArchiveExtension); + _archiveFilePath = Path.ChangeExtension(logFilePath, sourcesArchiveExtension); } else { @@ -50,7 +53,7 @@ public ProjectImportsCollector(string logFilePath, bool createFile, string sourc } // Archive file will be temporarily stored in MSBuild cache folder and deleted when no longer needed - ArchiveFilePath = Path.Combine( + _archiveFilePath = Path.Combine( cacheDirectory, Path.ChangeExtension( Path.GetFileName(logFilePath), @@ -59,7 +62,7 @@ public ProjectImportsCollector(string logFilePath, bool createFile, string sourc try { - _fileStream = new FileStream(ArchiveFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.Delete); + _fileStream = new FileStream(_archiveFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.Delete); _zipArchive = new ZipArchive(_fileStream, ZipArchiveMode.Create); } catch @@ -69,50 +72,72 @@ public ProjectImportsCollector(string logFilePath, bool createFile, string sourc _fileStream = null; _zipArchive = null; } + _runOnBackground = runOnBackground; + } + + public void AddFile(string? filePath) + { + AddFileHelper(filePath, AddFileCore); } - public void AddFile(string filePath) + public void AddFileFromMemory( + string? filePath, + string data, + Encoding? encoding = null, + DateTimeOffset? entryCreationStamp = null, + bool makePathAbsolute = true) + { + AddFileHelper(filePath, path => + AddFileFromMemoryCore(path, data, encoding ?? Encoding.UTF8, makePathAbsolute, entryCreationStamp)); + } + + public void AddFileFromMemory( + string? filePath, + Stream data, + DateTimeOffset? entryCreationStamp = null, + bool makePathAbsolute = true) + { + AddFileHelper(filePath, path => AddFileFromMemoryCore(path, data, makePathAbsolute, entryCreationStamp)); + } + + private void AddFileHelper( + string? filePath, + Action addFileWorker) { if (filePath != null && _fileStream != null) { + Action addFileAction = WrapWithExceptionSwallowing(() => addFileWorker(filePath)); + lock (_fileStream) { - // enqueue the task to add a file and return quickly - // to avoid holding up the current thread - _currentTask = _currentTask.ContinueWith(t => + if (_runOnBackground) + { + // enqueue the task to add a file and return quickly + // to avoid holding up the current thread + _currentTask = _currentTask.ContinueWith( + t => { addFileAction(); }, + TaskScheduler.Default); + } + else { - try - { - AddFileCore(filePath); - } - catch - { - } - }, TaskScheduler.Default); + addFileAction(); + } } } } - public void AddFileFromMemory(string filePath, string data) + private Action WrapWithExceptionSwallowing(Action action) { - if (filePath != null && data != null && _fileStream != null) + return () => { - lock (_fileStream) + try { - // enqueue the task to add a file and return quickly - // to avoid holding up the current thread - _currentTask = _currentTask.ContinueWith(t => - { - try - { - AddFileFromMemoryCore(filePath, data); - } - catch - { - } - }, TaskScheduler.Default); + action(); } - } + catch + { + } + }; } /// @@ -122,61 +147,94 @@ public void AddFileFromMemory(string filePath, string data) private void AddFileCore(string filePath) { // quick check to avoid repeated disk access for Exists etc. - if (_processedFiles.Contains(filePath)) + if (!ShouldAddFile(ref filePath, true, true)) { return; } - if (!File.Exists(filePath)) + using FileStream content = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete); + AddFileData(filePath, content, null); + } + + /// + /// This method doesn't need locking/synchronization because it's only called + /// from a task that is chained linearly + /// + private void AddFileFromMemoryCore(string filePath, string data, Encoding encoding, bool makePathAbsolute, DateTimeOffset? entryCreationStamp) + { + // quick check to avoid repeated disk access for Exists etc. + if (!ShouldAddFile(ref filePath, false, makePathAbsolute)) { - _processedFiles.Add(filePath); return; } - filePath = Path.GetFullPath(filePath); + AddFileData(filePath, data, encoding, entryCreationStamp); + } - // if the file is already included, don't include it again - if (!_processedFiles.Add(filePath)) + private void AddFileFromMemoryCore(string filePath, Stream data, bool makePathAbsolute, DateTimeOffset? entryCreationStamp) + { + // quick check to avoid repeated disk access for Exists etc. + if (!ShouldAddFile(ref filePath, false, makePathAbsolute)) { return; } - using FileStream content = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete); - using Stream entryStream = OpenArchiveEntry(filePath); - content.CopyTo(entryStream); + AddFileData(filePath, data, entryCreationStamp); } - /// - /// This method doesn't need locking/synchronization because it's only called - /// from a task that is chained linearly - /// - private void AddFileFromMemoryCore(string filePath, string data) + private void AddFileData(string filePath, Stream data, DateTimeOffset? entryCreationStamp) + { + using Stream entryStream = OpenArchiveEntry(filePath, entryCreationStamp); + data.CopyTo(entryStream); + } + + private void AddFileData(string filePath, string data, Encoding encoding, DateTimeOffset? entryCreationStamp) + { + using Stream entryStream = OpenArchiveEntry(filePath, entryCreationStamp); + using MemoryStream memoryStream = new MemoryStream(); + // We need writer as encoding.GetBytes() isn't obliged to output preamble + // We cannot write directly to entryStream (preamble is written separately) as it's compressed differnetly, then writing the whole stream at once + using StreamWriter writer = new StreamWriter(memoryStream, encoding); + writer.Write(data); + writer.Flush(); + memoryStream.Position = 0; + memoryStream.CopyTo(entryStream); + } + + private bool ShouldAddFile(ref string filePath, bool checkFileExistence, bool makeAbsolute) { // quick check to avoid repeated disk access for Exists etc. if (_processedFiles.Contains(filePath)) { - return; + return false; } - filePath = Path.GetFullPath(filePath); - - // if the file is already included, don't include it again - if (!_processedFiles.Add(filePath)) + if (checkFileExistence && !File.Exists(filePath)) { - return; + _processedFiles.Add(filePath); + return false; } - using (Stream entryStream = OpenArchiveEntry(filePath)) - using (var content = new MemoryStream(Encoding.UTF8.GetBytes(data))) + // Only make the path absolute if it's request. In the replay scenario, the file entries + // are read from zip archive - where ':' is stripped and path can then seem relative. + if (makeAbsolute) { - content.CopyTo(entryStream); + filePath = Path.GetFullPath(filePath); } + + // if the file is already included, don't include it again + return _processedFiles.Add(filePath); } - private Stream OpenArchiveEntry(string filePath) + private Stream OpenArchiveEntry(string filePath, DateTimeOffset? entryCreationStamp) { string archivePath = CalculateArchivePath(filePath); - var archiveEntry = _zipArchive.CreateEntry(archivePath); + var archiveEntry = _zipArchive!.CreateEntry(archivePath); + if (entryCreationStamp.HasValue) + { + archiveEntry.LastWriteTime = entryCreationStamp.Value; + } + return archiveEntry.Open(); } @@ -191,6 +249,27 @@ private static string CalculateArchivePath(string filePath) return archivePath; } + public void ProcessResult(Action consumeStream, Action onError) + { + Close(); + + // It is possible that the archive couldn't be created for some reason. + // Only embed it if it actually exists. + if (FileSystems.Default.FileExists(_archiveFilePath)) + { + using FileStream fileStream = File.OpenRead(_archiveFilePath); + + if (fileStream.Length > int.MaxValue) + { + onError("Imported files archive exceeded 2GB limit and it's not embedded."); + } + else + { + consumeStream(fileStream); + } + } + } + public void Close() { // wait for all pending file writes to complete @@ -208,5 +287,11 @@ public void Close() _fileStream = null; } } + + public void DeleteArchive() + { + Close(); + File.Delete(_archiveFilePath); + } } } diff --git a/src/Build/Microsoft.Build.csproj b/src/Build/Microsoft.Build.csproj index 6669e7c3e40..43cfb1b3f80 100644 --- a/src/Build/Microsoft.Build.csproj +++ b/src/Build/Microsoft.Build.csproj @@ -164,10 +164,21 @@ + + + + + + + + + + + + + + - - - diff --git a/src/Framework/Traits.cs b/src/Framework/Traits.cs index 04b2fc90237..3c81043f963 100644 --- a/src/Framework/Traits.cs +++ b/src/Framework/Traits.cs @@ -120,6 +120,12 @@ public Traits() /// public readonly int LogPropertyTracking = ParseIntFromEnvironmentVariableOrDefault("MsBuildLogPropertyTracking", 0); // Default to logging nothing via the property tracker. + /// + /// Turn on greedy buffering stream decorator for binlog writer. + /// This will ensure that 2 identical binlog contents will result into identical binlog files (as writing different chunks to GZipStream can lead to different result). + /// + public readonly bool DeterministicBinlogStreamBuffering = !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("MSBUILDDETERMNISTICBINLOG")); + /// /// When evaluating items, this is the minimum number of items on the running list to use a dictionary-based remove optimization. ///