/// <summary> /// Read the provided binary log file and raise corresponding events for each BuildEventArgs /// </summary> /// <param name="sourceFilePath">The full file path of the binary log file</param> public void Replay(string sourceFilePath) { using (var stream = new FileStream(sourceFilePath, FileMode.Open)) { var gzipStream = new GZipStream(stream, CompressionMode.Decompress, leaveOpen: true); var binaryReader = new BinaryReader(gzipStream); int fileFormatVersion = binaryReader.ReadInt32(); // the log file is written using a newer version of file format // that we don't know how to read if (fileFormatVersion > BinaryLogger.FileFormatVersion) { var text = $"Unsupported log file format. Latest supported version is {BinaryLogger.FileFormatVersion}, the log file has version {fileFormatVersion}."; throw new NotSupportedException(text); } var reader = new BuildEventArgsReader(binaryReader); while (true) { BuildEventArgs instance = null; instance = reader.Read(); if (instance == null) { break; } Dispatch(instance); } } }
/// <summary> /// Read the provided binary log file and raise corresponding events for each BuildEventArgs /// </summary> /// <param name="sourceFilePath">The full file path of the binary log file</param> /// <param name="cancellationToken">A <see cref="CancellationToken"/> indicating the replay should stop as soon as possible.</param> public void Replay(string sourceFilePath, CancellationToken cancellationToken) { using (var stream = new FileStream(sourceFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { var gzipStream = new GZipStream(stream, CompressionMode.Decompress, leaveOpen: true); var binaryReader = new BinaryReader(gzipStream); int fileFormatVersion = binaryReader.ReadInt32(); // the log file is written using a newer version of file format // that we don't know how to read if (fileFormatVersion > BinaryLogger.FileFormatVersion) { var text = ResourceUtilities.FormatResourceStringStripCodeAndKeyword("UnsupportedLogFileFormat", fileFormatVersion, BinaryLogger.FileFormatVersion); throw new NotSupportedException(text); } var reader = new BuildEventArgsReader(binaryReader, fileFormatVersion); while (true) { if (cancellationToken.IsCancellationRequested) { return; } BuildEventArgs instance = reader.Read(); if (instance == null) { break; } Dispatch(instance); } } }
/// <summary> /// Read the provided binary log file and raise corresponding events for each BuildEventArgs /// </summary> /// <param name="sourceFilePath">The full file path of the binary log file</param> public void Replay(string sourceFilePath) { using (var stream = new FileStream(sourceFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { var gzipStream = new GZipStream(stream, CompressionMode.Decompress, leaveOpen: true); var binaryReader = new BinaryReader(gzipStream); int fileFormatVersion = binaryReader.ReadInt32(); // the log file is written using a newer version of file format // that we don't know how to read if (fileFormatVersion > BinaryLogger.FileFormatVersion) { var text = $"Unsupported log file format. Latest supported version is {BinaryLogger.FileFormatVersion}, the log file has version {fileFormatVersion}."; throw new NotSupportedException(text); } // Use a producer-consumer queue so that IO can happen on one thread // while processing can happen on another thread decoupled. The speed // up is from 4.65 to 4.15 seconds. var queue = new BlockingCollection <BuildEventArgs>(boundedCapacity: 5000); var processingTask = System.Threading.Tasks.Task.Run(() => { foreach (var args in queue.GetConsumingEnumerable()) { Dispatch(args); } }); int recordsRead = 0; var reader = new BuildEventArgsReader(binaryReader, fileFormatVersion); reader.OnBlobRead += OnBlobRead; while (true) { BuildEventArgs instance = null; try { instance = reader.Read(); } catch (Exception ex) { OnException?.Invoke(ex); } recordsRead++; if (instance == null) { queue.CompleteAdding(); break; } queue.Add(instance); } processingTask.Wait(); } }
/// <summary> /// Enumerate over all records in the file. For each record store the bytes, /// the start position in the stream, length in bytes and the deserialized object. /// </summary> /// <remarks>Useful for debugging and analyzing binary logs</remarks> public IEnumerable <Record> ReadRecords(string sourceFilePath) { using (var stream = new FileStream(sourceFilePath, FileMode.Open)) { var gzipStream = new GZipStream(stream, CompressionMode.Decompress, leaveOpen: true); var memoryStream = new MemoryStream(); gzipStream.CopyTo(memoryStream); memoryStream.Position = 0; var binaryReader = new BinaryReader(memoryStream); var bytes = memoryStream.ToArray(); int fileFormatVersion = binaryReader.ReadInt32(); // the log file is written using a newer version of file format // that we don't know how to read if (fileFormatVersion > BinaryLogger.FileFormatVersion) { var text = $"Unsupported log file format. Latest supported version is {BinaryLogger.FileFormatVersion}, the log file has version {fileFormatVersion}."; throw new NotSupportedException(text); } var reader = new BuildEventArgsReader(binaryReader); long index = memoryStream.Position; while (true) { BuildEventArgs instance = null; instance = reader.Read(); if (instance == null) { break; } var position = memoryStream.Position; var length = position - index; var chunk = new byte[length]; Array.Copy(bytes, (int)index, chunk, 0, (int)length); var record = new Record { Bytes = chunk, Args = instance, Start = index, Length = length }; yield return(record); index = position; } } }
/// <summary> /// Read the provided binary log file and raise corresponding events for each BuildEventArgs /// </summary> /// <param name="sourceFilePath">The full file path of the binary log file</param> /// <param name="cancellationToken">A <see cref="CancellationToken"/> indicating the replay should stop as soon as possible.</param> public void Replay(string sourceFilePath, CancellationToken cancellationToken) { using (var stream = new FileStream(sourceFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { var gzipStream = new GZipStream(stream, CompressionMode.Decompress, leaveOpen: true); // wrapping the GZipStream in a buffered stream significantly improves performance // and the max throughput is reached with a 32K buffer. See details here: // https://github.com/dotnet/runtime/issues/39233#issuecomment-745598847 var bufferedStream = new BufferedStream(gzipStream, 32768); var binaryReader = new BinaryReader(bufferedStream); int fileFormatVersion = binaryReader.ReadInt32(); // the log file is written using a newer version of file format // that we don't know how to read if (fileFormatVersion > BinaryLogger.FileFormatVersion) { var text = ResourceUtilities.FormatResourceStringStripCodeAndKeyword("UnsupportedLogFileFormat", fileFormatVersion, BinaryLogger.FileFormatVersion); throw new NotSupportedException(text); } using var reader = new BuildEventArgsReader(binaryReader, fileFormatVersion); while (true) { if (cancellationToken.IsCancellationRequested) { return; } BuildEventArgs instance = reader.Read(); if (instance == null) { break; } Dispatch(instance); } } }
/// <summary> /// Enumerate over all records in the file. For each record store the bytes, /// the start position in the stream, length in bytes and the deserialized object. /// </summary> /// <remarks>Useful for debugging and analyzing binary logs</remarks> public IEnumerable <Record> ReadRecords(string sourceFilePath) { using (var stream = new FileStream(sourceFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { var gzipStream = new GZipStream(stream, CompressionMode.Decompress, leaveOpen: true); var memoryStream = new MemoryStream(); gzipStream.CopyTo(memoryStream); memoryStream.Position = 0; var binaryReader = new BinaryReader(memoryStream); var bytes = memoryStream.ToArray(); int fileFormatVersion = binaryReader.ReadInt32(); // the log file is written using a newer version of file format // that we don't know how to read if (fileFormatVersion > BinaryLogger.FileFormatVersion) { var text = $"Unsupported log file format. Latest supported version is {BinaryLogger.FileFormatVersion}, the log file has version {fileFormatVersion}."; throw new NotSupportedException(text); } long index = memoryStream.Position; long lengthOfBlobsAddedLastTime = 0; List <Record> blobs = new List <Record>(); var reader = new BuildEventArgsReader(binaryReader, fileFormatVersion); reader.OnBlobRead += (kind, blob) => { var record = new Record { Bytes = blob, Args = null, Start = index, Length = blob.Length }; blobs.Add(record); lengthOfBlobsAddedLastTime += blob.Length; }; while (true) { BuildEventArgs instance = null; instance = reader.Read(); if (instance == null) { break; } var position = memoryStream.Position; var length = position - index - lengthOfBlobsAddedLastTime; var chunk = new byte[length]; Array.Copy(bytes, (int)(index + lengthOfBlobsAddedLastTime), chunk, 0, (int)length); var record = new Record { Bytes = chunk, Args = instance, Start = index, Length = length }; yield return(record); index = position; lengthOfBlobsAddedLastTime = 0; } foreach (var blob in blobs) { yield return(blob); } } }
public IEnumerable <Record> ReadRecordsFromDecompressedStream(Stream decompressedStream) { var binaryReader = new BinaryReader(decompressedStream); int fileFormatVersion = binaryReader.ReadInt32(); // the log file is written using a newer version of file format // that we don't know how to read if (fileFormatVersion > BinaryLogger.FileFormatVersion) { var text = $"Unsupported log file format. Latest supported version is {BinaryLogger.FileFormatVersion}, the log file has version {fileFormatVersion}."; throw new NotSupportedException(text); } long lengthOfBlobsAddedLastTime = 0; List <Record> blobs = new List <Record>(); var reader = new BuildEventArgsReader(binaryReader, fileFormatVersion); reader.OnBlobRead += (kind, blob) => { var record = new Record { Bytes = blob, Args = null, Start = 0, // TODO: see if we can re-add that Length = blob.Length }; blobs.Add(record); lengthOfBlobsAddedLastTime += blob.Length; }; while (true) { BuildEventArgs instance = null; instance = reader.Read(); if (instance == null) { break; } var record = new Record { Bytes = null, // probably can reconstruct this from the Args if necessary Args = instance, Start = 0, Length = 0 }; yield return(record); lengthOfBlobsAddedLastTime = 0; } foreach (var blob in blobs) { yield return(blob); } }