Skip to content

[Binlogs Redacting] API and support for external redaction functionality #9219

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 9 commits into from
Closed

[Binlogs Redacting] API and support for external redaction functionality #9219

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
b8e3048
Add embedded content transfering and build events binary equality
JanKrivanek Sep 12, 2023
282de14
Added support for embedded files; fixing, cleaning
JanKrivanek Sep 15, 2023
19fd913
Decrease scope where accidentally exposed
JanKrivanek Sep 18, 2023
6e03efe
Sealing types
JanKrivanek Sep 19, 2023
8a3bb35
Adjust visibility, add comments
JanKrivanek Sep 20, 2023
09f32ed
Cache encoding
JanKrivanek Sep 21, 2023
248043f
Merge remote-tracking branch 'upstream/main' into proto/binlog-replay
JanKrivanek Oct 2, 2023
7cdd0e0
Fix code after merge
JanKrivanek Oct 2, 2023
e970a99
Fix after merge
JanKrivanek Oct 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
172 changes: 171 additions & 1 deletion src/Build/CompatibilitySuppressions.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,174 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- https://learn.microsoft.com/en-us/dotnet/fundamentals/package-validation/diagnostic-ids -->
<Suppressions xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
</Suppressions>
<!-- Start of suppressions needed for adjustments of added binlog reader API -->
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Build.Logging.BinaryLogReplayEventSource.add_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications})</Target>
<Left>lib/net472/Microsoft.Build.dll</Left>
<Right>lib/net472/Microsoft.Build.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Build.Logging.BinaryLogReplayEventSource.remove_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications})</Target>
<Left>lib/net472/Microsoft.Build.dll</Left>
<Right>lib/net472/Microsoft.Build.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Build.Logging.BuildEventArgsReader.add_StringEncountered(System.Action)</Target>
<Left>lib/net472/Microsoft.Build.dll</Left>
<Right>lib/net472/Microsoft.Build.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Build.Logging.BuildEventArgsReader.remove_StringEncountered(System.Action)</Target>
<Left>lib/net472/Microsoft.Build.dll</Left>
<Right>lib/net472/Microsoft.Build.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Build.Logging.IBuildEventStringsReader.add_StringEncountered(System.Action)</Target>
<Left>lib/net472/Microsoft.Build.dll</Left>
<Right>lib/net472/Microsoft.Build.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Build.Logging.IBuildEventStringsReader.remove_StringEncountered(System.Action)</Target>
<Left>lib/net472/Microsoft.Build.dll</Left>
<Right>lib/net472/Microsoft.Build.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Build.Logging.BinaryLogReplayEventSource.add_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications})</Target>
<Left>lib/net8.0/Microsoft.Build.dll</Left>
<Right>lib/net8.0/Microsoft.Build.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Build.Logging.BinaryLogReplayEventSource.remove_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications})</Target>
<Left>lib/net8.0/Microsoft.Build.dll</Left>
<Right>lib/net8.0/Microsoft.Build.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Build.Logging.BuildEventArgsReader.add_StringEncountered(System.Action)</Target>
<Left>lib/net8.0/Microsoft.Build.dll</Left>
<Right>lib/net8.0/Microsoft.Build.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Build.Logging.BuildEventArgsReader.remove_StringEncountered(System.Action)</Target>
<Left>lib/net8.0/Microsoft.Build.dll</Left>
<Right>lib/net8.0/Microsoft.Build.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Build.Logging.IBuildEventStringsReader.add_StringEncountered(System.Action)</Target>
<Left>lib/net8.0/Microsoft.Build.dll</Left>
<Right>lib/net8.0/Microsoft.Build.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Build.Logging.IBuildEventStringsReader.remove_StringEncountered(System.Action)</Target>
<Left>lib/net8.0/Microsoft.Build.dll</Left>
<Right>lib/net8.0/Microsoft.Build.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Build.Logging.BinaryLogReplayEventSource.add_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications})</Target>
<Left>ref/net472/Microsoft.Build.dll</Left>
<Right>ref/net472/Microsoft.Build.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Build.Logging.BinaryLogReplayEventSource.remove_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications})</Target>
<Left>ref/net472/Microsoft.Build.dll</Left>
<Right>ref/net472/Microsoft.Build.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Build.Logging.BuildEventArgsReader.add_StringEncountered(System.Action)</Target>
<Left>ref/net472/Microsoft.Build.dll</Left>
<Right>ref/net472/Microsoft.Build.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Build.Logging.BuildEventArgsReader.remove_StringEncountered(System.Action)</Target>
<Left>ref/net472/Microsoft.Build.dll</Left>
<Right>ref/net472/Microsoft.Build.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Build.Logging.IBuildEventStringsReader.add_StringEncountered(System.Action)</Target>
<Left>ref/net472/Microsoft.Build.dll</Left>
<Right>ref/net472/Microsoft.Build.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Build.Logging.IBuildEventStringsReader.remove_StringEncountered(System.Action)</Target>
<Left>ref/net472/Microsoft.Build.dll</Left>
<Right>ref/net472/Microsoft.Build.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Build.Logging.BinaryLogReplayEventSource.add_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications})</Target>
<Left>ref/net8.0/Microsoft.Build.dll</Left>
<Right>ref/net8.0/Microsoft.Build.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Build.Logging.BinaryLogReplayEventSource.remove_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications})</Target>
<Left>ref/net8.0/Microsoft.Build.dll</Left>
<Right>ref/net8.0/Microsoft.Build.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Build.Logging.BuildEventArgsReader.add_StringEncountered(System.Action)</Target>
<Left>ref/net8.0/Microsoft.Build.dll</Left>
<Right>ref/net8.0/Microsoft.Build.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Build.Logging.BuildEventArgsReader.remove_StringEncountered(System.Action)</Target>
<Left>ref/net8.0/Microsoft.Build.dll</Left>
<Right>ref/net8.0/Microsoft.Build.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Build.Logging.IBuildEventStringsReader.add_StringEncountered(System.Action)</Target>
<Left>ref/net8.0/Microsoft.Build.dll</Left>
<Right>ref/net8.0/Microsoft.Build.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Build.Logging.IBuildEventStringsReader.remove_StringEncountered(System.Action)</Target>
<Left>ref/net8.0/Microsoft.Build.dll</Left>
<Right>ref/net8.0/Microsoft.Build.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<!-- End of suppressions needed for adjustments of added binlog reader API -->
</Suppressions>
2 changes: 1 addition & 1 deletion src/Build/Logging/BinaryLogger/BinaryLogRecordKind.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ internal enum BinaryLogRecordKind
ProjectEvaluationStarted,
ProjectEvaluationFinished,
ProjectImported,
ProjectImportArchive,
ProjectImportArchive = 17,
TargetSkipped,
PropertyReassignment,
UninitializedPropertyRead,
Expand Down
85 changes: 65 additions & 20 deletions src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ namespace Microsoft.Build.Logging
/// by implementing IEventSource and raising corresponding events.
/// </summary>
/// <remarks>The class is public so that we can call it from MSBuild.exe when replaying a log file.</remarks>
public sealed class BinaryLogReplayEventSource : EventArgsDispatcher
public sealed class BinaryLogReplayEventSource : EventArgsDispatcher, IEmbeddedContentSource
{
/// Touches the <see cref="ItemGroupLoggingHelper"/> static constructor
/// to ensure it initializes <see cref="TaskParameterEventArgs.MessageGetter"/>
Expand All @@ -26,11 +26,6 @@ static BinaryLogReplayEventSource()
_ = ItemGroupLoggingHelper.ItemGroupIncludeLogMessagePrefix;
}

/// <summary>
/// Raised once <see cref="BuildEventArgsReader"/> is created during replaying
/// </summary>
public event Action<IBuildEventArgsReaderNotifications>? NotificationsSourceCreated;

/// <summary>
/// Read the provided binary log file and raise corresponding events for each BuildEventArgs
/// </summary>
Expand Down Expand Up @@ -68,41 +63,91 @@ public static BinaryReader OpenReader(string sourceFilePath)
}
}

/// <summary>
/// Creates a <see cref="BuildEventArgsReader"/> for the provided binary reader over binary log file.
/// Caller is responsible for disposing the returned reader.
/// </summary>
/// <param name="binaryReader"></param>
/// <param name="closeInput">Indicates whether the passed BinaryReader should be closed on disposing.</param>
/// <returns>BuildEventArgsReader over the given binlog file binary reader.</returns>
public static BuildEventArgsReader OpenBuildEventsReader(BinaryReader binaryReader, bool closeInput)
{
int fileFormatVersion = binaryReader.ReadInt32();

// the log file is written using a newer version of file format
// that we don't know how to read
if (fileFormatVersion > BinaryLogger.FileFormatVersion)
{
var text = ResourceUtilities.FormatResourceStringStripCodeAndKeyword("UnsupportedLogFileFormat", fileFormatVersion, BinaryLogger.FileFormatVersion);
throw new NotSupportedException(text);
}

return new BuildEventArgsReader(binaryReader, fileFormatVersion) { CloseInput = closeInput };
}

/// <summary>
/// Creates a <see cref="BinaryReader"/> for the provided binary log file.
/// Performs decompression and buffering in the optimal way.
/// Caller is responsible for disposing the returned reader.
/// </summary>
/// <param name="sourceFilePath"></param>
/// <returns>BinaryReader of the given binlog file.</returns>
public static BuildEventArgsReader OpenBuildEventsReader(string sourceFilePath)
=> OpenBuildEventsReader(OpenReader(sourceFilePath), true);

/// <summary>
/// Read the provided binary log file and raise corresponding events for each BuildEventArgs
/// </summary>
/// <param name="sourceFilePath">The full file path of the binary log file</param>
/// <param name="cancellationToken">A <see cref="CancellationToken"/> indicating the replay should stop as soon as possible.</param>
public void Replay(string sourceFilePath, CancellationToken cancellationToken)
{
using var binaryReader = OpenReader(sourceFilePath);
Replay(binaryReader, cancellationToken);
using var eventsReader = OpenBuildEventsReader(sourceFilePath);
Replay(eventsReader, cancellationToken);
}

/// <summary>
/// Read the provided binary log file and raise corresponding events for each BuildEventArgs
/// </summary>
/// <param name="binaryReader">The binary log content binary reader - caller is responsible for disposing.</param>
/// <param name="closeInput">Indicates whether the passed BinaryReader should be closed on disposing.</param>
/// <param name="cancellationToken">A <see cref="CancellationToken"/> indicating the replay should stop as soon as possible.</param>
public void Replay(BinaryReader binaryReader, CancellationToken cancellationToken)
public void Replay(BinaryReader binaryReader, bool closeInput, CancellationToken cancellationToken)
{
int fileFormatVersion = binaryReader.ReadInt32();

// the log file is written using a newer version of file format
// that we don't know how to read
if (fileFormatVersion > BinaryLogger.FileFormatVersion)
{
var text = ResourceUtilities.FormatResourceStringStripCodeAndKeyword("UnsupportedLogFileFormat", fileFormatVersion, BinaryLogger.FileFormatVersion);
throw new NotSupportedException(text);
}
using var reader = OpenBuildEventsReader(binaryReader, closeInput);
Replay(reader, cancellationToken);
}

using var reader = new BuildEventArgsReader(binaryReader, fileFormatVersion);
NotificationsSourceCreated?.Invoke(reader);
/// <summary>
/// Read the provided binary log file and raise corresponding events for each BuildEventArgs
/// </summary>
/// <param name="reader">The build events reader - caller is responsible for disposing.</param>
/// <param name="cancellationToken">A <see cref="CancellationToken"/> indicating the replay should stop as soon as possible.</param>
public void Replay(BuildEventArgsReader reader, CancellationToken cancellationToken)
{
_fileFormatVersionRead?.Invoke(reader.FileFormatVersion);
reader.EmbeddedContentRead += _embeddedContentRead;

while (!cancellationToken.IsCancellationRequested && reader.Read() is { } instance)
{
Dispatch(instance);
}
}

private Action<int>? _fileFormatVersionRead;
event Action<int> ILogVersionInfo.FileFormatVersionRead
{
add => _fileFormatVersionRead += value;
remove => _fileFormatVersionRead -= value;
}
private Action<EmbeddedContentEventArgs>? _embeddedContentRead;
/// <inheritdoc cref="IEmbeddedContentSource.EmbeddedContentRead"/>
event Action<EmbeddedContentEventArgs>? IEmbeddedContentSource.EmbeddedContentRead
{
// Explicitly implemented event has to declare explicit add/remove accessors
// https://stackoverflow.com/a/2268472/2308106
add => _embeddedContentRead += value;
remove => _embeddedContentRead -= value;
}
}
}
Loading