/// <summary> /// Serialize result to writer /// </summary> public static void Serialize(BuildXLWriter writer, IReadOnlyCollection <ProcessDetouringStatusData> processDetouringStatuses) { if (processDetouringStatuses == null) { writer.WriteCompact(0); } else { writer.WriteCompact(processDetouringStatuses.Count); foreach (var processDetouringData in processDetouringStatuses) { WriteReportedProcessDetouringStatus(writer, processDetouringData); } } }
/// <nodoc /> public void Serialize(BuildXLWriter writer) { writer.Write(EmptyQualifierId); writer.Write(EmptyQualifierSpaceId); m_qualifiers.Serialize(writer); m_qualifierSpaces.Serialize(writer); writer.WriteCompact(m_qualifierToFriendlyQualifierName.Count); foreach (var kv in m_qualifierToFriendlyQualifierName) { writer.Write(kv.Key); writer.Write(kv.Value); } }
private static void WriteReportedProcessDetouringStatus(BuildXLWriter writer, ProcessDetouringStatusData detouringData) { writer.Write(detouringData.ProcessId); writer.Write(detouringData.ReportStatus); writer.Write(detouringData.ProcessName); writer.Write(detouringData.StartApplicationName); writer.Write(detouringData.StartCommandLine); writer.Write(detouringData.NeedsInjection); writer.Write(detouringData.Job); writer.Write(detouringData.DisableDetours); writer.Write(detouringData.CreationFlags); writer.Write(detouringData.Detoured); writer.Write(detouringData.Error); writer.Write(detouringData.CreateProcessStatusReturn); }
/// <nodoc /> public void Serialize(BuildXLWriter writer) { if (Range.IsEmpty) { writer.Write(false); } else { writer.Write(true); writer.Write(Range.FromInclusive.Value); writer.Write(Range.ToInclusive.Value); } m_members.Serialize(writer); }
/// <nodoc /> public void Serialize(BuildXLWriter writer) { writer.Write(ProcessId); writer.Write(ReportStatus); writer.Write(ProcessName); writer.Write(StartApplicationName); writer.Write(StartCommandLine); writer.Write(NeedsInjection); writer.Write(Job); writer.Write(DisableDetours); writer.Write(CreationFlags); writer.Write(Detoured); writer.Write(Error); writer.Write(CreateProcessStatusReturn); }
/// <nodoc /> public void Serialize(Stream stream) { using (var writer = new BuildXLWriter(false, stream, true, true)) { writer.WriteNullableString(m_arguments); writer.WriteNullableString(m_commandLine); writer.Write(DisableConHostSharing); writer.WriteNullableString(FileName); writer.Write(StandardInputEncoding, (w, v) => w.Write(v)); writer.Write(StandardOutputEncoding, (w, v) => w.Write(v)); writer.Write(StandardErrorEncoding, (w, v) => w.Write(v)); writer.WriteNullableString(WorkingDirectory); writer.Write( EnvironmentVariables, (w, v) => w.WriteReadOnlyList( v.ToDictionary().ToList(), (w2, kvp) => { w2.Write(kvp.Key); w2.Write(kvp.Value); })); writer.Write( AllowedSurvivingChildProcessNames, (w, v) => w.WriteReadOnlyList(v, (w2, v2) => w2.Write(v2))); writer.Write(MaxLengthInMemory); writer.Write(Timeout, (w, v) => w.Write(v)); writer.Write(NestedProcessTerminationTimeout); writer.Write(PipSemiStableHash); writer.WriteNullableString(TimeoutDumpDirectory); writer.Write((byte)SandboxKind); writer.WriteNullableString(PipDescription); if (SandboxedProcessStandardFiles == null) { SandboxedProcessStandardFiles.From(FileStorage).Serialize(writer); } else { SandboxedProcessStandardFiles.Serialize(writer); } writer.Write(StandardInputSourceInfo, (w, v) => v.Serialize(w)); writer.Write(StandardObserverDescriptor, (w, v) => v.Serialize(w)); // File access manifest should be serialize the last. writer.Write(FileAccessManifest, (w, v) => FileAccessManifest.Serialize(stream)); } }
private void Serialize(StreamingContext context) { { using var stream = new MemoryStream(); using var writer = new BuildXLWriter(debug: false, stream, leaveOpen: false, logStats: false); StrongFingerprint.Serialize(writer); _strongFingerprint = stream.ToArray(); } { using var stream = new MemoryStream(); using var writer = new BuildXLWriter(debug: false, stream, leaveOpen: false, logStats: false); ContentHashListWithDeterminism.Serialize(writer); _contentHashListWithDeterminism = stream.ToArray(); } }
private void SerializeCore(BuildXLWriter writer) { ValuePathFileAccessWhitelistEntry[] valuePathEntries = m_valuePathEntries.Values.SelectMany((e) => { return(e); }).ToArray(); writer.WriteCompact(valuePathEntries.Length); foreach (ValuePathFileAccessWhitelistEntry entry in valuePathEntries) { entry.Serialize(writer); } ExecutablePathWhitelistEntry[] executablePathEntries = m_executablePathEntries.Values.SelectMany((e) => { return(e); }).ToArray(); writer.WriteCompact(executablePathEntries.Length); foreach (ExecutablePathWhitelistEntry entry in executablePathEntries) { entry.Serialize(writer); } }
private static SidebandWriter CloneViaSerialization(SidebandWriter original) { using (var stream = new MemoryStream()) { using (var writer = new BuildXLWriter(debug: true, stream, leaveOpen: true, logStats: true)) { original.Serialize(writer); } stream.Seek(0, SeekOrigin.Begin); using (var reader = new BuildXLReader(debug: true, stream, leaveOpen: true)) { return(SidebandWriter.Deserialize(reader)); } } }
public static void TestSerializationRoundtrip <T>(T expected, Action <BuildXLWriter> serializer, Func <BuildXLReader, T> deserializer) { using (var ms = new MemoryStream()) { using (var bw = new BuildXLWriter(false, ms, false, false)) { serializer(bw); ms.Position = 0; using (var reader = new BuildXLReader(false, ms, false)) { var deserialized = deserializer(reader); Assert.Equal(expected, deserialized); } } } }
/// <summary> /// Serializes an instance into a binary stream. /// </summary> public void Serialize(BuildXLWriter writer) { Contract.Requires(writer != null); var writeContentHashList = ContentHashList != null; writer.Write(writeContentHashList); if (writeContentHashList) { ContentHashList.Serialize(writer); } var determinism = Determinism.Serialize(); writer.Write(determinism.Length); writer.Write(determinism); }
private static MachineIdSet Copy(MachineIdSet source) { using (var memoryStream = new MemoryStream()) { using (var writer = BuildXLWriter.Create(memoryStream, leaveOpen: true)) { source.Serialize(writer); } memoryStream.Position = 0; using (var reader = BuildXLReader.Create(memoryStream)) { return(MachineIdSet.Deserialize(reader)); } } }
internal static void InternalSerialize(BuildXLWriter writer, ServiceInfo info) { writer.Write((byte)info.Kind); if (info.Kind == ServicePipKind.None) { return; } writer.Write(info.ServicePipDependencies, (w, v) => WritePipId(w, v)); writer.Write(info.ShutdownPipId.IsValid); if (info.ShutdownPipId.IsValid) { WritePipId(writer, info.ShutdownPipId); } writer.Write(info.FinalizationPipIds, (w, v) => WritePipId(w, v)); }
protected override void SpecializedSerialize(BuildXLWriter writer) { writer.Write((byte)SealDirectoryKind); writer.Write(Patterns, (w, v) => w.Write(v)); writer.Write(IsComposite); writer.Write(Scrub); if (ContentFilter != null) { writer.Write(true); writer.Write((byte)ContentFilter.Value.Kind); writer.Write(ContentFilter.Value.Regex); } else { writer.Write(false); } }
/// <summary> /// Creates a new binary logger to write to the given stream /// </summary> /// <param name="logStream">the stream to write events to</param> /// <param name="context">the context containing the path table</param> /// <param name="logId">the log id to place in the header of execution used to verify with other data structures on load.</param> /// <param name="lastStaticAbsolutePathIndex">the last absolute path guaranteed to be in the serialized form of the corresponding path table</param> /// <param name="closeStreamOnDispose">specifies whether the stream is closed on disposal of the logger</param> /// <param name="onEventWritten">optional callback after each event is written to underlying stream</param> public BinaryLogger(Stream logStream, PipExecutionContext context, Guid logId, int lastStaticAbsolutePathIndex = int.MinValue, bool closeStreamOnDispose = true, Action onEventWritten = null) { m_context = context; LogId = logId; m_lastStaticAbsolutePathIndex = lastStaticAbsolutePathIndex; m_logStreamWriter = new BuildXLWriter(debug: false, stream: logStream, leaveOpen: !closeStreamOnDispose, logStats: false); m_watch = Stopwatch.StartNew(); m_capturedPaths = new ConcurrentBigMap <AbsolutePath, bool>(); m_capturedStrings = new ConcurrentBigMap <StringId, bool>(); m_capturedStrings.Add(StringId.Invalid, true); m_writerPool = new ObjectPool <EventWriter>( () => new EventWriter(this), writer => { writer.Seek(0, SeekOrigin.Begin); return(writer); }); var logIdBytes = logId.ToByteArray(); Contract.Assert(logIdBytes.Length == LogIdByteLength); logStream.Write(logIdBytes, 0, logIdBytes.Length); LogStartTime(DateTime.UtcNow); m_pendingEventsDrainingThread = new Thread( () => { try { foreach (PooledObjectWrapper <EventWriter> wrapper in m_pendingEvents.GetConsumingEnumerable()) { var eventWriter = wrapper.Instance; WriteEventData(eventWriter); m_writerPool.PutInstance(wrapper); onEventWritten?.Invoke(); } } catch (InvalidOperationException) { // InvalidOperationException is thrown when calling Take() for a marked-as-completed blocking collection. // However, GetConsumingEnumerable throws an InvalidOperationException here, which is unusual. // In further investigations, we discovered that it might throw one if the collection in BlockingCollection // is passed in the constructor and we externally modify that collection outside of BlockingCollection. // Even we do not do that, we rarely have InvalidOperationException here, which is a NetCore bug. // We reported the bug; but for now, we swallow that exception and we treat it as a signal for completion. return; } }); m_pendingEventsDrainingThread.Start(); }
/// <summary> /// Serializes a path set to the given buffer. /// </summary> protected async Task <ContentHash> SerializePathSet(ObservedPathSet pathSet, MemoryStream pathSetBuffer, ContentHash?pathSetHash = null) { using (var writer = new BuildXLWriter(stream: pathSetBuffer, debug: false, leaveOpen: true, logStats: false)) { pathSet.Serialize(PathTable, writer, m_pathExpander); if (pathSetHash == null) { pathSetBuffer.Position = 0; pathSetHash = await ContentHashingUtilities.HashContentStreamAsync(pathSetBuffer); } pathSetBuffer.Position = 0; return(pathSetHash.Value); } }
/// <summary> /// Serializes this instance to a writer. /// </summary> public void Serialize(BuildXLWriter writer, Action <BinaryWriter, T> writeValue) { Contract.Requires(writer != null); Contract.Requires(writeValue != null); m_pathToValue.Serialize( writer, kv => { writer.Write(kv.Key); // path writer.WriteCompact(kv.Value.Count); // nodes foreach (var value in kv.Value) { writeValue(writer, value); } }); }
/// <nodoc /> public void SerializeReconcileData(OperationContext context, BuildXLWriter writer, MachineId machine, IReadOnlyList <ShortHashWithSize> addedContent, IReadOnlyList <ShortHash> removedContent) { var entries = new ContentLocationEventData[] { new AddContentLocationEventData(machine, addedContent), new RemoveContentLocationEventData(machine, removedContent) }; var finalEntries = SplitLargeInstancesIfNeeded(entries); context.TraceDebug($"{nameof(ContentLocationEventDataSerializer)}: EntriesCount={finalEntries.Count}"); writer.WriteCompact(finalEntries.Count); foreach (var eventData in finalEntries) { eventData.Serialize(writer); } }
/// <nodoc/> public static void Serialize(this IUnsafeSandboxConfiguration @this, BuildXLWriter writer) { writer.Write((byte)@this.SandboxKind); writer.Write(@this.ExistingDirectoryProbesAsEnumerations); writer.Write(@this.IgnoreGetFinalPathNameByHandle); writer.Write(@this.IgnoreNonCreateFileReparsePoints); writer.Write(@this.IgnoreReparsePoints); writer.Write(@this.IgnoreSetFileInformationByHandle); writer.Write(@this.IgnoreZwOtherFileInformation); writer.Write(@this.IgnoreZwRenameFileInformation); writer.Write(@this.MonitorFileAccesses); writer.Write(@this.MonitorNtCreateFile); writer.Write(@this.MonitorZwCreateOpenQueryFile); writer.Write((byte)@this.PreserveOutputs); writer.Write(@this.UnexpectedFileAccessesAreErrors); writer.Write(@this.IgnorePreloadedDlls); writer.Write(@this.IgnoreDynamicWritesOnAbsentProbes); }
/// <summary> /// Serializes /// </summary> public void Serialize(BuildXLWriter writer) { Contract.Requires(writer != null); FileAccessAllowlist.Serialize(writer); writer.Write(DefaultPipFilter); writer.Write(CacheSalt); DirectoryMembershipFingerprinterRules.Serialize(writer); writer.WriteCompact(ModuleConfigurations.Count); foreach (var moduleConfiguration in ModuleConfigurations) { writer.Write(moduleConfiguration.ModuleId); writer.Write(moduleConfiguration.Name); // TODO: Write everything else instead of doing it in many different places? } }
/// <nodoc /> public void Serialize(BuildXLWriter writer) { Contract.Requires(writer != null); WriteState(writer); object executableValue = Executable.GetValue(); if (executableValue is AbsolutePath absolutePath) { writer.Write(true); writer.Write(absolutePath); } else if (executableValue is PathAtom pathAtom) { writer.Write(false); writer.Write(pathAtom); } }
private IncrementalSchedulingPathToNodeMapping SaveAndReload(IncrementalSchedulingPathToNodeMapping current) { using (var stream = new MemoryStream()) { using (var writer = new BuildXLWriter(debug: false, stream: stream, leaveOpen: true, logStats: false)) { current.Serialize(writer, (w, n) => w.Write(n.Value)); } // rewind stream.Position = 0; using (var reader = new BuildXLReader(debug: false, stream: stream, leaveOpen: true)) { return(IncrementalSchedulingPathToNodeMapping.Deserialize(reader, r => new NodeId(r.ReadUInt32()))); } } }
/// <nodoc/> public static void Serialize(this IUnsafeSandboxConfiguration @this, BuildXLWriter writer) { writer.Write((byte)@this.SandboxKind); writer.Write(@this.ExistingDirectoryProbesAsEnumerations); writer.Write(@this.IgnoreGetFinalPathNameByHandle); writer.Write(@this.IgnoreNonCreateFileReparsePoints); writer.Write(@this.IgnoreReparsePoints); writer.Write(@this.IgnoreSetFileInformationByHandle); writer.Write(@this.IgnoreZwOtherFileInformation); writer.Write(@this.IgnoreZwRenameFileInformation); writer.Write(@this.MonitorFileAccesses); writer.Write(@this.MonitorNtCreateFile); writer.Write(@this.MonitorZwCreateOpenQueryFile); writer.Write((byte)@this.PreserveOutputs); writer.Write(@this.PreserveOutputsTrustLevel); writer.Write(@this.UnexpectedFileAccessesAreErrors); writer.Write(@this.IgnorePreloadedDlls); writer.WriteCompact((int)@this.IgnoreDynamicWritesOnAbsentProbes); writer.Write(@this.DoubleWritePolicy.HasValue); if (@this.DoubleWritePolicy.HasValue) { writer.Write((byte)@this.DoubleWritePolicy.Value); } writer.Write(@this.IgnoreUndeclaredAccessesUnderSharedOpaques); writer.Write(@this.IgnoreCreateProcessReport); writer.Write(@this.ProbeDirectorySymlinkAsDirectory); writer.Write(@this.ProcessSymlinkedAccesses.HasValue); if (@this.ProcessSymlinkedAccesses.HasValue) { writer.Write(@this.ProcessSymlinkedAccesses.Value); } writer.Write(@this.IgnoreFullReparsePointResolving); writer.Write(@this.SkipFlaggingSharedOpaqueOutputs.HasValue); if (@this.SkipFlaggingSharedOpaqueOutputs.HasValue) { writer.Write(@this.SkipFlaggingSharedOpaqueOutputs.Value); } writer.Write(@this.EnableFullReparsePointResolving.HasValue); if (@this.EnableFullReparsePointResolving.HasValue) { writer.Write(@this.EnableFullReparsePointResolving.Value); } }
private void SerializeCore(BuildXLWriter writer) { ValuePathFileAccessAllowlistEntry[] valuePathEntries = m_valuePathEntries.Values.SelectMany((e) => { return(e); }).ToArray(); writer.WriteCompact(valuePathEntries.Length); foreach (ValuePathFileAccessAllowlistEntry entry in valuePathEntries) { entry.Serialize(writer); } var pathEntries = m_executablePathEntries.Values.SelectMany((e) => { return(e); }); var toolEntries = m_executableToolExeEntries.Values.SelectMany((e) => { return(e); }); ExecutablePathAllowlistEntry[] entries = pathEntries.Concat(toolEntries).ToArray(); writer.WriteCompact(entries.Length); foreach (ExecutablePathAllowlistEntry entry in entries) { entry.Serialize(writer); } }
public void Serialize(BuildXLWriter writer) { var count = Count; writer.WriteCompact(count); for (int i = 0; i < count; i++) { var entry = this[i]; entry.Serialize(writer); if (entry.EntryType == PipDataEntryType.DirectoryIdHeaderSealId) { // Get next entry and write out the full directory artifact rather than serializing an entry path i++; var nextEntry = this[i]; Contract.Assert(nextEntry.EntryType == PipDataEntryType.AbsolutePath); writer.Write(new DirectoryArtifact(nextEntry.GetPathValue(), entry.GetUInt32Value())); } } }
public async Task RunCommonTestsAsync() { XAssert.AreNotEqual(StringTable.Count, AddedStrings.Count, "Ensure strings are added through harness"); StringTable deserializedStringTable = null; using (MemoryStream ms = new MemoryStream()) { using (BuildXLWriter writer = new BuildXLWriter(true, ms, true, logStats: true)) { StringTable.Serialize(writer); } if (MaxSize < StringTable.BytesPerBuffer) { XAssert.IsTrue(ms.Position < StringTable.BytesPerBuffer, "Small StringTable should not serialize a full empty byte buffer."); } ms.Position = 0; using (BuildXLReader reader = new BuildXLReader(true, ms, true)) { deserializedStringTable = await StringTable.DeserializeAsync(reader); } } foreach (var entry in AddedStrings) { // Test deserialization behaviors XAssert.AreEqual(entry.Value.Length, deserializedStringTable.GetLength(entry.Key)); XAssert.AreEqual(entry.Value.Length, deserializedStringTable.GetBinaryString(entry.Key).Length); XAssert.AreEqual(entry.Value, deserializedStringTable.GetString(entry.Key)); XAssert.AreEqual(entry.Key, deserializedStringTable.AddString(entry.Value)); // Test original string table behaviors XAssert.AreEqual(entry.Key, StringTable.AddString(entry.Value)); XAssert.AreEqual(entry.Value.Length, StringTable.GetLength(entry.Key)); XAssert.AreEqual(entry.Value.Length, StringTable.GetBinaryString(entry.Key).Length); XAssert.AreEqual(entry.Value, StringTable.GetString(entry.Key)); } }
public void Serialize(BuildXLWriter writer) { writer.Write((byte)Type); switch (Type) { case CommandType.Get: writer.WriteNullableByteArray(((GetResult)this).Value); break; case CommandType.Set: { var typedResult = (SetResult)this; writer.Write(typedResult.Set); break; } case CommandType.CompareExchange: { var typedResult = (CompareExchangeResult)this; writer.WriteNullableByteArray(typedResult.Previous); writer.Write(typedResult.Exchanged); break; } case CommandType.CompareRemove: writer.WriteNullableByteArray(((CompareRemoveResult)this).Value); break; case CommandType.Remove: break; case CommandType.PrefixEnumerate: writer.WriteReadOnlyList(((PrefixEnumerateResult)this).Pairs, (w, kvp) => { w.WriteNullableByteArray(kvp.Key); w.WriteNullableByteArray(kvp.Value); }); break; } }
public void CreateAndReloadCombinedFileAlignmentBug() { FakeFile file = default(FakeFile); file.Path = R("foo", "bar1.txt"); using (var contentStream = new MemoryStream()) { using (var binaryWriter = new BuildXLWriter(debug: false, stream: contentStream, leaveOpen: true, logStats: false)) { binaryWriter.WriteCompact(-1); } file.Content = contentStream.ToArray(); file.Hash = ContentHashingUtilities.HashBytes(file.Content); } int maxBackingBufferBytes = 4 + 6 + 8 + file.Content.Length; // magic size that used to trigger a bug Logger.FileCombinerStats stats = new Logger.FileCombinerStats(); // Create a combiner and add some files using (FileCombiner combiner = CreateFileCombiner(m_loggingContext, m_path, 1)) { combiner.GetStatsRefForTest(ref stats); AddFile(combiner, file); } XAssert.AreEqual(1, stats.EndCount); XAssert.AreEqual(0, stats.CompactingTimeMs, "FileCombiner should not have been compacted"); // reload the combiner using (FileCombiner combiner = CreateFileCombiner(m_loggingContext, m_path, 1, maxBackingBufferBytes)) { // fetch a file that exists and verify the correct data is returned using (MemoryStream ms = combiner.RequestFile(file.Path, file.Hash)) { XAssert.IsNotNull(ms); AssertContentMatches(ms, file); } } }
/// <inheritdoc /> public void WriteFingerprintInputs(IFingerprinter writer) { using (var stream = new MemoryStream()) { using (var buildXLWriter = new BuildXLWriter( debug: false, stream: stream, leaveOpen: false, logStats: false)) { UnsafeOptions.Serialize(buildXLWriter); writer.Add("SerializedUnsafeOptions", System.BitConverter.ToString(stream.ToArray())); } } var thisRef = this; writer.AddNested(ObservedPathEntryConstants.PathSet, w => { foreach (var p in thisRef.PathEntries) { w.Add(ObservedPathEntryConstants.Path, p.Path); if (p.Flags != ObservedPathEntryFlags.None) { w.Add(ObservedPathEntryConstants.Flags, p.Flags.ToString()); } if (p.EnumeratePatternRegex != null) { w.Add(ObservedPathEntryConstants.EnumeratePatternRegex, p.EnumeratePatternRegex); } } }); writer.AddCollection <StringId, ReadOnlyArray <StringId> >( "ObservedAccessedFileNames", ObservedAccessedFileNames, (w, v) => w.Add(v)); // Observed inputs are included directly into the strong fingerprint hash computation, // so they do not need to be serialized here }
/// <inheritdoc/> public override void Serialize(BuildXLWriter writer) { // The kind for the base class (FileModuleLiteral) is sealed and thera are consumers // that rely on it for any subclass of FileModuleLiteral. So serializing the most specific kind // explicitly so deserialization knows what's coming writer.WriteCompact((int)SyntaxKind.FileModuleLiteral); LineMap.Write(writer); writer.Write(Path); WritePackage(Package, writer); // Don't need to save qualifier, because it only valid for instantiated modules, and this module should be uninstantiated. writer.WriteCompact(m_resolvedEntries.Count); foreach (var kvp in m_resolvedEntries) { WriteFilePosition(kvp.Key, writer); kvp.Value.Serialize(writer); } }