private void Save() { FileEnvelopeId fileEnvelopeId = m_fileChangeTracker.GetFileEnvelopeToSaveWith(); m_fileChangeTracker.SaveTrackingStateIfChanged(m_fileChangeTrackerPath.ToString(m_pathTable), fileEnvelopeId); m_fileContentTable.SaveAsync(m_fileContentTablePath.ToString(m_pathTable)).Wait(); }
private IIncrementalSchedulingState LoadOrReuseInternal( FileEnvelopeId atomicSaveToken, PipGraph pipGraph, IConfiguration configuration, ContentHash preserveOutputSalt, string incrementalSchedulingStatePath, SchedulerState schedulerState) { Contract.Requires(pipGraph != null); Contract.Requires(!string.IsNullOrWhiteSpace(incrementalSchedulingStatePath)); Contract.Assert(m_analysisMode || configuration != null); if (!m_analysisMode && schedulerState != null && schedulerState.IncrementalSchedulingState != null) { IIncrementalSchedulingState reusedState = schedulerState.IncrementalSchedulingState.Reuse(m_loggingContext, pipGraph, configuration, preserveOutputSalt, m_tempDirectoryCleaner); if (reusedState != null) { return(reusedState); } } return(GraphAgnosticIncrementalSchedulingState.Load( m_loggingContext, atomicSaveToken, pipGraph, configuration, preserveOutputSalt, incrementalSchedulingStatePath, analysisModeOnly: m_analysisMode, tempDirectoryCleaner: m_tempDirectoryCleaner)); }
private FileChangeTracker( LoggingContext loggingContext, FileEnvelopeId fileEnvelopeId, FileChangeTrackingState initialState, VolumeMap volumeMap, IChangeJournalAccessor journal, FileChangeTrackingSet currentChangeTrackingSet, string buildEngineFingerprint) { Contract.Requires(loggingContext != null); Contract.Requires(fileEnvelopeId.IsValid); Contract.Requires( initialState == FileChangeTrackingState.BuildingInitialChangeTrackingSet || initialState == FileChangeTrackingState.TrackingChanges); Contract.Requires(volumeMap != null); Contract.Requires(journal != null); Contract.Requires(currentChangeTrackingSet != null); FileEnvelopeId = fileEnvelopeId; m_loggingContext = loggingContext; m_volumeMap = volumeMap; m_journal = journal; m_changeTrackingSet = currentChangeTrackingSet; m_trackingStateValue = (int)initialState; Counters = m_changeTrackingSet.Counters; m_buildEngineFingerprint = buildEngineFingerprint; }
internal void Save(Stream stream) { ExceptionUtilities.HandleRecoverableIOException( () => { // We don't have anything in particular to correlate this file to, // so we are simply creating a unique correlation id that is used as part // of the header consistency check. FileEnvelopeId correlationId = FileEnvelopeId.Create(); FileEnvelope.WriteHeader(stream, correlationId); using (BuildXLWriter writer = new BuildXLWriter(debug: false, stream: stream, leaveOpen: true, logStats: false)) { writer.Write(m_runtimeData.Count); foreach (KeyValuePair<long, PipHistoricPerfData> kvp in m_runtimeData) { writer.Write(kvp.Key); kvp.Value.Serialize(writer); } } FileEnvelope.FixUpHeader(stream, correlationId); return (object)null; }, ex => { throw new BuildXLException("Writing of file failed", ex); }); }
/// <summary> /// Creates a new change tracker in the <see cref="FileChangeTrackingState.BuildingInitialChangeTrackingSet"/> state. /// The caller may then add tracking for full set of files of interest, for later re-use by /// <see cref="ResumeTrackingChanges(LoggingContext,BuildXL.Utilities.FileEnvelopeId,VolumeMap,IChangeJournalAccessor,FileChangeTrackingSet,string)"/>. /// </summary> public static FileChangeTracker StartTrackingChanges( LoggingContext loggingContext, VolumeMap volumeMap, IChangeJournalAccessor journal, string buildEngineFingerprint, FileEnvelopeId?correlatedId = default) { Contract.Requires(loggingContext != null); Contract.Requires(volumeMap != null); Contract.Requires(journal != null); Contract.Ensures(Contract.Result <FileChangeTracker>().TrackingState == FileChangeTrackingState.BuildingInitialChangeTrackingSet); var tracker = new FileChangeTracker( loggingContext, correlatedId ?? FileEnvelopeId.Create(), FileChangeTrackingState.BuildingInitialChangeTrackingSet, volumeMap, journal, FileChangeTrackingSet.CreateForAllCapableVolumes(loggingContext, volumeMap, journal), buildEngineFingerprint); foreach (var gvfsProjectionFile in volumeMap.GvfsProjections) { var maybeTracking = tracker.TryProbeAndTrackPath(gvfsProjectionFile); if (!maybeTracking.Succeeded) { Logger.Log.TrackChangesToGvfsProjectionFailed(loggingContext, gvfsProjectionFile, maybeTracking.Failure.DescribeIncludingInnerFailures()); } } return(tracker); }
public void WrongCorrelationId() { FileEnvelopeId fe0 = FileEnvelopeId.Create(); FileEnvelopeId fe1 = FileEnvelopeId.Create(); Assert.Throws <BuildXLException>( () => { FileEnvelope.CheckCorrelationIds(fe0, fe1); }); }
/// <summary> /// Loads an existing instance of <see cref="IIncrementalSchedulingState"/> from a given file or reuse it from a given <see cref="SchedulerState"/>. /// </summary> public IIncrementalSchedulingState LoadOrReuse( FileEnvelopeId atomicSaveToken, PipGraph pipGraph, IConfiguration configuration, ContentHash preserveOutputSalt, string incrementalSchedulingStatePath, SchedulerState schedulerState) { Contract.Requires(atomicSaveToken.IsValid); return(LoadOrReuseInternal(atomicSaveToken, pipGraph, configuration, preserveOutputSalt, incrementalSchedulingStatePath, schedulerState)); }
private void Save(FileEnvelopeId atomicSaveToken, string path) { Contract.Requires(!string.IsNullOrWhiteSpace(path)); FileUtilities.DeleteFile(path); FileUtilities.CreateDirectory(Path.GetDirectoryName(path)); using (var stream = FileUtilities.CreateFileStream( path, FileMode.Create, FileAccess.Write, FileShare.Delete, // Do not write the file with SequentialScan since it will be reread in the subsequent build FileOptions.None)) { ExceptionUtilities.HandleRecoverableIOException( () => { using (var pm = BuildXL.Tracing.PerformanceMeasurement.StartWithoutStatistic( m_loggingContext, loggingContext => Logger.Log.StartSavingChangeTracker(loggingContext, path), loggingContext => Logger.Log.EndSavingChangeTracker(loggingContext))) { Stopwatch sw = Stopwatch.StartNew(); FileEnvelope.WriteHeader(stream, atomicSaveToken); using (var writer = new BuildXLWriter(debug: false, stream: stream, leaveOpen: true, logStats: false)) { if (IsDisabledOrNullTrackingSet) { writer.Write(true); } else { writer.Write(false); writer.Write(m_buildEngineFingerprint, (w, s) => w.Write(s)); m_changeTrackingSet.Save(writer); } } FileEnvelope.FixUpHeader(stream, atomicSaveToken); Logger.Log.SavingChangeTracker( pm.LoggingContext, path, atomicSaveToken.ToString(), m_changeTrackingSet == null ? "Null" : TrackingState.ToString(), m_changeTrackingSet == null ? 0 : m_changeTrackingSet.TrackedVolumes.Count(), sw.ElapsedMilliseconds); } }, ex => { throw new BuildXLException("Failed to save file change tracker", ex); }); } }
/// <summary> /// Saves state to later resume change tracking where there is new file or checkpoint data. /// </summary> public bool SaveTrackingStateIfChanged(string path, FileEnvelopeId fileEnvelopeIdToSaveWith) { if (TrackingState == FileChangeTrackingState.TrackingChanges && !HasNewFileOrCheckpointData) { return(false); } // Save tracking set if it is disabled or has new checkpoint data. Save(fileEnvelopeIdToSaveWith, path); return(true); }
public void Success() { var fe = new FileEnvelope("Dummy", 0); using (var stream = new MemoryStream()) { FileEnvelopeId id = FileEnvelopeId.Create(); fe.WriteHeader(stream, id); fe.FixUpHeader(stream, id); stream.Position = 0; fe.ReadHeader(stream); } }
public void MissingFixup() { var fe = new FileEnvelope("Dummy", 0); using (var stream = new MemoryStream()) { FileEnvelopeId id = FileEnvelopeId.Create(); fe.WriteHeader(stream, id); // fe.FixUpHeader(stream, id); stream.Position = 0; Assert.Throws <BuildXLException>( () => { fe.ReadHeader(stream); }); } }
public void WrongEnvelopeVersion() { var fe0 = new FileEnvelope("Dummy0", 0); var fe1 = new FileEnvelope("Dummy0", 1); using (var stream = new MemoryStream()) { FileEnvelopeId id = FileEnvelopeId.Create(); fe0.WriteHeader(stream, id); fe0.FixUpHeader(stream, id); stream.Position = 0; Assert.Throws <BuildXLException>( () => { fe1.ReadHeader(stream); }); } }
public void DetectFileLengthCorruption() { var fe = new FileEnvelope("Dummy", 0); using (var stream = new MemoryStream()) { FileEnvelopeId id = FileEnvelopeId.Create(); fe.WriteHeader(stream, id); fe.FixUpHeader(stream, id); stream.WriteByte(0); // not taken into account in fixed up header magic stream.Position = 0; Assert.Throws <BuildXLException>( () => { fe.ReadHeader(stream); }); } }
/// <summary> /// Creates a new instance of <see cref="IIncrementalSchedulingState"/>. /// </summary> public IIncrementalSchedulingState CreateNew( FileEnvelopeId atomicSaveToken, PipGraph pipGraph, IConfiguration configuration, ContentHash preserveOutputSalt) { Contract.Requires(atomicSaveToken.IsValid); Contract.Requires(pipGraph != null); Contract.Requires(configuration != null); return(GraphAgnosticIncrementalSchedulingState.CreateNew( m_loggingContext, atomicSaveToken, pipGraph, configuration, preserveOutputSalt, m_tempDirectoryCleaner)); }
public override int Analyze() { using (var fileStreamWrapper = FileSystemStreamProvider.Default.OpenReadStream(m_inputFile)) { var fileStream = fileStreamWrapper.Value; FileEnvelopeId persistedCorrelationId = InputTracker.FileEnvelope.ReadHeader(fileStream); var isCompressed = fileStream.ReadByte() == 1; using (Stream readStream = isCompressed ? new DeflateStream(fileStream, CompressionMode.Decompress) : fileStream) using (BinaryReader reader = new BinaryReader(readStream)) using (var writer = File.CreateText(Path.GetFullPath(m_outputFile))) { InputTracker.ReadAndWriteText(reader, writer); } } return(0); }
/// <summary> /// Creates a new change tracker in the <see cref="FileChangeTrackingState.BuildingInitialChangeTrackingSet"/> state. /// The caller may then add tracking for full set of files of interest, for later re-use by /// <see cref="ResumeTrackingChanges(LoggingContext,BuildXL.Utilities.FileEnvelopeId,VolumeMap,IChangeJournalAccessor,FileChangeTrackingSet,string)"/>. /// </summary> public static FileChangeTracker StartTrackingChanges( LoggingContext loggingContext, VolumeMap volumeMap, IChangeJournalAccessor journal, string buildEngineFingerprint, FileEnvelopeId?correlatedId = default) { Contract.Requires(loggingContext != null); Contract.Requires(volumeMap != null); Contract.Requires(journal != null); Contract.Ensures(Contract.Result <FileChangeTracker>().TrackingState == FileChangeTrackingState.BuildingInitialChangeTrackingSet); return(new FileChangeTracker( loggingContext, correlatedId ?? FileEnvelopeId.Create(), FileChangeTrackingState.BuildingInitialChangeTrackingSet, volumeMap, journal, FileChangeTrackingSet.CreateForAllCapableVolumes(loggingContext, volumeMap, journal), buildEngineFingerprint)); }
/// <summary> /// Creates a new change tracker in the <see cref="FileChangeTrackingState.TrackingChanges"/> state. /// The caller may query for new changes since the tracking set was last checkpointed and persisted, and may track additional files. /// </summary> private static FileChangeTracker ResumeTrackingChanges( LoggingContext loggingContext, FileEnvelopeId fileEnvelopeId, VolumeMap volumeMap, IChangeJournalAccessor journal, FileChangeTrackingSet previousChangeTrackingSet, string buildEngineFingerprint) { Contract.Requires(loggingContext != null); Contract.Requires(volumeMap != null); Contract.Requires(journal != null); Contract.Requires(previousChangeTrackingSet != null); Contract.Requires(fileEnvelopeId.IsValid); return(new FileChangeTracker( loggingContext, fileEnvelopeId, FileChangeTrackingState.TrackingChanges, volumeMap, journal, previousChangeTrackingSet, buildEngineFingerprint)); }
/// <summary> /// Creates a new output logger. /// /// The underlying file is created only upon first write. /// </summary> /// <param name="metadata">Metadata</param> /// <param name="sidebandLogFile">File to which to save the log.</param> /// <param name="rootDirectories">Only paths under one of these root directories will be recorded.</param> public SidebandWriter(SidebandMetadata metadata, string sidebandLogFile, [CanBeNull] IReadOnlyList <string> rootDirectories) { Metadata = metadata; SidebandLogFile = sidebandLogFile; RootDirectories = rootDirectories; m_recordedPathsCache = new HashSet <AbsolutePath>(); m_envelopeId = FileEnvelopeId.Create(); m_lazyBxlWriter = Lazy.Create(() => { Directory.CreateDirectory(Path.GetDirectoryName(SidebandLogFile)); var writer = new BuildXLWriter( stream: new FileStream(SidebandLogFile, FileMode.Create, FileAccess.Write, FileShare.Read | FileShare.Delete), debug: false, logStats: false, leaveOpen: false); // write header and metadata before anything else FileEnvelope.WriteHeader(writer.BaseStream, m_envelopeId); Metadata.Serialize(writer); return(writer); }); }
/// <summary> /// Serializes and saves state needed for execution analyzers. /// </summary> private Task <bool> SaveExecutionStateToDiskAsync(ScheduleRunResult result) { // Every scheduler run has a unique log directory based off timestamp of run string logDirectory = Path.Combine( result.Config.Logging.LogsDirectory.ToString(Context.PathTable), result.Config.Logging.LogPrefix); var serializer = new EngineSerializer( LoggingContext, logDirectory, correlationId: FileEnvelopeId.Create()); var dummyHistoricData = (IReadOnlyList <HistoricDataPoint>)CollectionUtilities.EmptyArray <HistoricDataPoint>(); var dummyHistoricTableSizes = new HistoricTableSizes(dummyHistoricData); return(EngineSchedule.SaveExecutionStateToDiskAsync( serializer, Context, PipTable, result.Graph, Expander, dummyHistoricTableSizes)); }
/// <summary> /// Creates a new output logger. /// /// The underlying file is created only upon first write. /// </summary> /// <param name="sidebandLogFile">File to which to save the log.</param> /// <param name="rootDirectories">Only paths under one of the root directories are recorded in <see cref="RecordFileWrite(PathTable, AbsolutePath)"/>.</param> public SharedOpaqueOutputLogger(string sidebandLogFile, [CanBeNull] IReadOnlyList <string> rootDirectories) { SidebandLogFile = sidebandLogFile; RootDirectories = rootDirectories; m_recordedPathsCache = new HashSet <AbsolutePath>(); m_envelopeId = FileEnvelopeId.Create(); m_lazyBxlWriter = Lazy.Create(() => { Directory.CreateDirectory(Path.GetDirectoryName(SidebandLogFile)); return(new BuildXLWriter( stream: new FileStream(SidebandLogFile, FileMode.Create, FileAccess.Write, FileShare.Read | FileShare.Delete), debug: false, logStats: false, leaveOpen: false)); }); m_lazyWriteHeader = Lazy.Create(() => { FileEnvelope.WriteHeader(m_lazyBxlWriter.Value.BaseStream, m_envelopeId); return(Unit.Void); }); }
/// <summary> /// Writes a file containing versioning information to the provided store directory. /// </summary> private static void WriteVersionFile(string storeDirectory, int storeVersion) { var versionFile = GetVersionFile(storeDirectory); using (var stream = FileUtilities.CreateFileStream( versionFile, FileMode.Create, FileAccess.ReadWrite, FileShare.Delete)) { // We don't have anything in particular to correlate this file to, // so we are simply creating a unique correlation id that is used as part // of the header consistency check. var correlationId = FileEnvelopeId.Create(); s_fileEnvelope.WriteHeader(stream, correlationId); using (var writer = new BuildXLWriter(debug: false, stream: stream, leaveOpen: true, logStats: false)) { writer.Write(storeVersion); } s_fileEnvelope.FixUpHeader(stream, correlationId); } }
public void DetectHeaderCorruption() { var r = new Random(0); var fe = new FileEnvelope("Dummy", 0); for (int i = 0; i < 10000; i++) { using (var stream = new MemoryStream()) { FileEnvelopeId id = FileEnvelopeId.Create(); fe.WriteHeader(stream, id); fe.FixUpHeader(stream, id); stream.Position = r.Next((int)stream.Length - 1); int b = stream.ReadByte(); stream.Position = stream.Position - 1; stream.WriteByte((byte)(b ^ (1 << r.Next(8)))); stream.Position = 0; Assert.Throws <BuildXLException>( () => { fe.ReadHeader(stream); }); } } }
private void SaveInternal(string fileContentTablePath) { Contract.Requires(!string.IsNullOrWhiteSpace(fileContentTablePath)); Contract.EnsuresOnThrow <BuildXLException>(true); ExceptionUtilities.HandleRecoverableIOException( () => { int numEvicted = 0; Directory.CreateDirectory(Path.GetDirectoryName(fileContentTablePath)); // Note that we are using a non-async file stream here. That's because we're doing lots of tiny writes for simplicity, // but tiny writes on an async stream end up blocking anyway while adding silly overhead. using (FileStream stream = FileUtilities.CreateFileStream( fileContentTablePath, FileMode.Create, FileAccess.Write, FileShare.Delete, // Do not write the file with SequentialScan since it will be reread in the subsequent build FileOptions.None)) { // We don't have anything in particular to correlate this file to, // so we are simply creating a unique correlation id that is used as part // of the header consistency check. FileEnvelopeId correlationId = FileEnvelopeId.Create(); s_fileEnvelope.WriteHeader(stream, correlationId); using (var writer = new BuildXLWriter(debug: false, stream: stream, leaveOpen: true, logStats: false)) { long numberOfEntriesPosition = writer.BaseStream.Position; writer.Write(0U); uint entriesWritten = 0; var hashBuffer = new byte[ContentHashingUtilities.HashInfo.ByteLength]; foreach (var fileAndEntryPair in m_entries) { // Skip saving anything with a TTL of zero. These entries were loaded // with a TTL of one (immediately decremented) and were not used since load. // See class remarks. if (fileAndEntryPair.Value.TimeToLive == 0) { numEvicted++; continue; } // Key: Volume and File ID fileAndEntryPair.Key.Serialize(writer); // Entry: USN, hash, time to live. writer.Write(fileAndEntryPair.Value.Usn.Value); fileAndEntryPair.Value.Hash.SerializeHashBytes(hashBuffer, 0); writer.Write(hashBuffer); writer.Write(fileAndEntryPair.Value.Length); writer.Write(fileAndEntryPair.Value.TimeToLive); entriesWritten++; } var endPosition = writer.BaseStream.Position; writer.BaseStream.Position = numberOfEntriesPosition; writer.Write(entriesWritten); writer.BaseStream.Position = endPosition; } s_fileEnvelope.FixUpHeader(stream, correlationId); } Counters.AddToCounter(FileContentTableCounters.NumEvicted, numEvicted); return(Unit.Void); }, ex => { throw new BuildXLException("Failure writing file content table", ex); }); }
/// <summary> /// Creates and starts a task to deserialize an object /// </summary> /// <param name="file">This will become the filename</param> /// <param name="deserializer">Deserialization function; its get a reader for the file stream, and a function that allows obtaining additional streams if needed</param> /// <param name="skipHeader">If enabled, the correlation id is not checked for consistency</param> /// <returns>task for deserialized value</returns> internal Task <TObject> DeserializeFromFileAsync <TObject>( GraphCacheFile file, Func <BuildXLReader, Task <TObject> > deserializer, bool skipHeader = false) { var task = Task.Run( async() => { var objectLabel = GetFileName(file); string path = GetFullPath(objectLabel); FileEnvelope fileEnvelope = GetFileEnvelope(file); var result = default(TObject); try { Stopwatch sw = Stopwatch.StartNew(); using (var fileStreamWrapper = m_readStreamProvider.OpenReadStream(path)) { var fileStream = fileStreamWrapper.Value; FileEnvelopeId persistedCorrelationId = fileEnvelope.ReadHeader(fileStream); if (!skipHeader) { // We are going to check if all files that are going to be (concurrently) deserialized have matching correlation ids. // The first discovered correlation id is going to be used to check all others. if (m_correlationId == null) { Interlocked.CompareExchange(ref m_correlationId, persistedCorrelationId, null); } FileEnvelope.CheckCorrelationIds(persistedCorrelationId, (FileEnvelopeId)m_correlationId); } var isCompressed = fileStream.ReadByte() == 1; using (Stream readStream = isCompressed ? new TrackedStream(new BufferedStream(new DeflateStream(fileStream, CompressionMode.Decompress), 64 << 10)) : fileStream) using (BuildXLReader reader = new BuildXLReader(m_debug, readStream, leaveOpen: false)) { result = await deserializer(reader); } } Tracing.Logger.Log.DeserializedFile(LoggingContext, path, sw.ElapsedMilliseconds); return(result); } catch (BuildXLException ex) { if (ex.InnerException is FileNotFoundException) { // Files might be deleted manually in the EngineCache directory. Log it as verbose. Tracing.Logger.Log.FailedToDeserializeDueToFileNotFound(LoggingContext, path); return(result); } Tracing.Logger.Log.FailedToDeserializePipGraph(LoggingContext, path, ex.LogEventMessage); return(result); } catch (IOException ex) { Tracing.Logger.Log.FailedToDeserializePipGraph(LoggingContext, path, ex.Message); return(result); } catch (TaskCanceledException) { throw; } catch (Exception ex) { // There are 2 reasons to be here. // 1. A malformed file can cause ContractException, IndexOutOfRangeException, MemoryException or something else. // 2. We may have a bug. // Since the malformed file will always cause a crash until someone removes the file from the cache, allow BuildXL to recover // by eating the exception. However remember to log it in order to keep track of bugs. ExceptionRootCause rootCause = ExceptionUtilities.AnalyzeExceptionRootCause(ex); BuildXL.Tracing.UnexpectedCondition.Log(LoggingContext, ex.ToStringDemystified() + Environment.NewLine + rootCause); Tracing.Logger.Log.FailedToDeserializePipGraph(LoggingContext, path, ex.Message); return(result); } }); lock (m_deserializationSyncObject) { m_deserializationTasks.Add(task); } return(task); }
/// <summary> /// Get the file envelope id to save with. If no change has been made, reuse existing file envelope id. Otherwise, use overrideFileEnvelopeId or a new id if overrideFileEnvelopeId is not specified. /// </summary> public FileEnvelopeId GetFileEnvelopeToSaveWith(FileEnvelopeId?overrideFileEnvelopeId = default) { if (TrackingState == FileChangeTrackingState.TrackingChanges && !HasNewFileOrCheckpointData) { return(FileEnvelopeId); } // Use override when provided, otherwise use the existing file id if we are building the initial change tracking set, // otherwise recreate a new file id. return(overrideFileEnvelopeId ?? (TrackingState == FileChangeTrackingState.BuildingInitialChangeTrackingSet ? FileEnvelopeId : FileEnvelopeId.Create())); }