/// <summary> /// Reads a file containing versioning information from the provided store directory. /// </summary> private static Possible <int> ReadStoreVersion(string storeDirectory) { var versionFile = GetVersionFile(storeDirectory); if (!FileUtilities.FileExistsNoFollow(versionFile)) { return(VersionConstants.UnversionedStore); } using (var stream = FileUtilities.CreateFileStream( versionFile, FileMode.Open, FileAccess.Read, FileShare.Delete)) { try { // Check that the accessor version is valid. s_fileEnvelope.ReadHeader(stream); // Check that the current store version must match the persisted store's version. using (var reader = new BuildXLReader(debug: false, stream: stream, leaveOpen: false)) { // Represents persisted store version return(reader.ReadInt32()); } } catch (Exception e) { return(new Failure <string>($"Error reading existing version file: {e.ToStringDemystified()}")); } } }
internal static PipRuntimeTimeTable Load(Stream stream) { return ExceptionUtilities.HandleRecoverableIOException( () => { Analysis.IgnoreResult(FileEnvelope.ReadHeader(stream)); using (BuildXLReader reader = new BuildXLReader(debug: false, stream: stream, leaveOpen: true)) { int size = reader.ReadInt32(); var table = new PipRuntimeTimeTable(initialCapacity: size); for (int i = 0; i < size; ++i) { long semiStableHash = reader.ReadInt64(); PipHistoricPerfData historicData; if (PipHistoricPerfData.Deserialize(reader, out historicData)) { if (!table.m_runtimeData.TryAdd(semiStableHash, historicData)) { throw new BuildXLException("Corrupted file has duplicate records"); } } } return table; } }, ex => { throw new BuildXLException("Reading of file failed", ex); }); }
public void Success() { var fe = new FileEnvelope("Dummy", 0); using (var stream = new MemoryStream()) { FileEnvelopeId id = FileEnvelopeId.Create(); fe.WriteHeader(stream, id); fe.FixUpHeader(stream, id); stream.Position = 0; fe.ReadHeader(stream); } }
public void MissingFixup() { var fe = new FileEnvelope("Dummy", 0); using (var stream = new MemoryStream()) { FileEnvelopeId id = FileEnvelopeId.Create(); fe.WriteHeader(stream, id); // fe.FixUpHeader(stream, id); stream.Position = 0; Assert.Throws <BuildXLException>( () => { fe.ReadHeader(stream); }); } }
public void WrongEnvelopeVersion() { var fe0 = new FileEnvelope("Dummy0", 0); var fe1 = new FileEnvelope("Dummy0", 1); using (var stream = new MemoryStream()) { FileEnvelopeId id = FileEnvelopeId.Create(); fe0.WriteHeader(stream, id); fe0.FixUpHeader(stream, id); stream.Position = 0; Assert.Throws <BuildXLException>( () => { fe1.ReadHeader(stream); }); } }
public void DetectFileLengthCorruption() { var fe = new FileEnvelope("Dummy", 0); using (var stream = new MemoryStream()) { FileEnvelopeId id = FileEnvelopeId.Create(); fe.WriteHeader(stream, id); fe.FixUpHeader(stream, id); stream.WriteByte(0); // not taken into account in fixed up header magic stream.Position = 0; Assert.Throws <BuildXLException>( () => { fe.ReadHeader(stream); }); } }
public void DetectHeaderCorruption() { var r = new Random(0); var fe = new FileEnvelope("Dummy", 0); for (int i = 0; i < 10000; i++) { using (var stream = new MemoryStream()) { FileEnvelopeId id = FileEnvelopeId.Create(); fe.WriteHeader(stream, id); fe.FixUpHeader(stream, id); stream.Position = r.Next((int)stream.Length - 1); int b = stream.ReadByte(); stream.Position = stream.Position - 1; stream.WriteByte((byte)(b ^ (1 << r.Next(8)))); stream.Position = 0; Assert.Throws <BuildXLException>( () => { fe.ReadHeader(stream); }); } } }
public static LoadingTrackerResult TryLoad( LoggingContext loggingContext, string path, VolumeMap volumeMap, IChangeJournalAccessor journal, string buildEngineFingerprint, bool loadForAllCapableVolumes = true) { Contract.Requires(loggingContext != null); Contract.Requires(!loadForAllCapableVolumes || volumeMap != null); Contract.Requires(!loadForAllCapableVolumes || journal != null); Stopwatch stopwatch = Stopwatch.StartNew(); SafeFileHandle handle; OpenFileResult result = FileUtilities.TryCreateOrOpenFile( path, FileDesiredAccess.GenericRead, FileShare.Read | FileShare.Delete, FileMode.Open, // Ok to evict the file from standby since the file will be overwritten and never reread from disk after this point. FileFlagsAndAttributes.FileFlagSequentialScan, out handle); if (result.Succeeded) { Contract.Assume(handle != null); Contract.Assume(!handle.IsInvalid); using (handle) { using (var stream = new FileStream(handle, FileAccess.Read)) { FileEnvelopeId fileEnvelopeId; try { fileEnvelopeId = FileEnvelope.ReadHeader(stream); } catch (BuildXLException e) { return(LoadingTrackerResult.FailBadFormatMarker(e.Message)); } try { return(ExceptionUtilities.HandleRecoverableIOException( () => { using (var reader = new BuildXLReader(debug: false, stream: stream, leaveOpen: true)) { bool wasTrackerDisabled = reader.ReadBoolean(); if (wasTrackerDisabled) { return LoadingTrackerResult.FailPriorTrackerDisabled(); } var previousFingerprint = reader.ReadNullable(r => r.ReadString()); // only check for fingerprints match if the supplied fingerprint is valid // this is to support special cases where we might want to load ChangeTracker // regardless of the previously stored fingerprint value if (buildEngineFingerprint != null && !string.Equals(previousFingerprint, buildEngineFingerprint, StringComparison.Ordinal)) { return LoadingTrackerResult.FailBuildEngineFingerprintMismatch(); } return FileChangeTrackingSet.TryLoad( loggingContext, fileEnvelopeId, reader, volumeMap, journal, stopwatch, loadForAllCapableVolumes); } }, ex => { throw new BuildXLException(ex.Message); })); } catch (Exception e) { // Catch any exception. Failure in loading FileChangeTracker should not // cause build break, or worse, make people stuck on erroneous state. // In such a case, BuildXL simply has to start tracking from scratch. return(LoadingTrackerResult.FailLoadException(e.GetLogEventMessage())); } } } } Contract.Assume(handle == null); return(LoadingTrackerResult.FailTrackingSetCannotBeOpened(result.CreateFailureForError().DescribeIncludingInnerFailures())); }
/// <summary> /// Creates and starts a task to deserialize an object /// </summary> /// <param name="file">This will become the filename</param> /// <param name="deserializer">Deserialization function; its get a reader for the file stream, and a function that allows obtaining additional streams if needed</param> /// <param name="skipHeader">If enabled, the correlation id is not checked for consistency</param> /// <returns>task for deserialized value</returns> internal Task <TObject> DeserializeFromFileAsync <TObject>( GraphCacheFile file, Func <BuildXLReader, Task <TObject> > deserializer, bool skipHeader = false) { var task = Task.Run( async() => { var objectLabel = GetFileName(file); string path = GetFullPath(objectLabel); FileEnvelope fileEnvelope = GetFileEnvelope(file); var result = default(TObject); try { Stopwatch sw = Stopwatch.StartNew(); using (var fileStreamWrapper = m_readStreamProvider.OpenReadStream(path)) { var fileStream = fileStreamWrapper.Value; FileEnvelopeId persistedCorrelationId = fileEnvelope.ReadHeader(fileStream); if (!skipHeader) { // We are going to check if all files that are going to be (concurrently) deserialized have matching correlation ids. // The first discovered correlation id is going to be used to check all others. if (m_correlationId == null) { Interlocked.CompareExchange(ref m_correlationId, persistedCorrelationId, null); } FileEnvelope.CheckCorrelationIds(persistedCorrelationId, (FileEnvelopeId)m_correlationId); } var isCompressed = fileStream.ReadByte() == 1; using (Stream readStream = isCompressed ? new TrackedStream(new BufferedStream(new DeflateStream(fileStream, CompressionMode.Decompress), 64 << 10)) : fileStream) using (BuildXLReader reader = new BuildXLReader(m_debug, readStream, leaveOpen: false)) { result = await deserializer(reader); } } Tracing.Logger.Log.DeserializedFile(LoggingContext, path, sw.ElapsedMilliseconds); return(result); } catch (BuildXLException ex) { if (ex.InnerException is FileNotFoundException) { // Files might be deleted manually in the EngineCache directory. Log it as verbose. Tracing.Logger.Log.FailedToDeserializeDueToFileNotFound(LoggingContext, path); return(result); } Tracing.Logger.Log.FailedToDeserializePipGraph(LoggingContext, path, ex.LogEventMessage); return(result); } catch (IOException ex) { Tracing.Logger.Log.FailedToDeserializePipGraph(LoggingContext, path, ex.Message); return(result); } catch (TaskCanceledException) { throw; } catch (Exception ex) { // There are 2 reasons to be here. // 1. A malformed file can cause ContractException, IndexOutOfRangeException, MemoryException or something else. // 2. We may have a bug. // Since the malformed file will always cause a crash until someone removes the file from the cache, allow BuildXL to recover // by eating the exception. However remember to log it in order to keep track of bugs. ExceptionRootCause rootCause = ExceptionUtilities.AnalyzeExceptionRootCause(ex); BuildXL.Tracing.UnexpectedCondition.Log(LoggingContext, ex.ToStringDemystified() + Environment.NewLine + rootCause); Tracing.Logger.Log.FailedToDeserializePipGraph(LoggingContext, path, ex.Message); return(result); } }); lock (m_deserializationSyncObject) { m_deserializationTasks.Add(task); } return(task); }