/// <summary>
/// Reads a file containing versioning information from the provided store directory.
/// </summary>
private static Possible <int> ReadStoreVersion(string storeDirectory)
{
var versionFile = GetVersionFile(storeDirectory);
if (!FileUtilities.FileExistsNoFollow(versionFile))
{
return(VersionConstants.UnversionedStore);
}
using (var stream = FileUtilities.CreateFileStream(
versionFile,
FileMode.Open,
FileAccess.Read,
FileShare.Delete))
{
try
{
// Check that the accessor version is valid.
s_fileEnvelope.ReadHeader(stream);
// Check that the current store version must match the persisted store's version.
using (var reader = new BuildXLReader(debug: false, stream: stream, leaveOpen: false))
{
// Represents persisted store version
return(reader.ReadInt32());
}
}
catch (Exception e)
{
return(new Failure <string>($"Error reading existing version file: {e.ToStringDemystified()}"));
}
}
}
internal static PipRuntimeTimeTable Load(Stream stream)
{
return ExceptionUtilities.HandleRecoverableIOException(
() =>
{
Analysis.IgnoreResult(FileEnvelope.ReadHeader(stream));
using (BuildXLReader reader = new BuildXLReader(debug: false, stream: stream, leaveOpen: true))
{
int size = reader.ReadInt32();
var table = new PipRuntimeTimeTable(initialCapacity: size);
for (int i = 0; i < size; ++i)
{
long semiStableHash = reader.ReadInt64();
PipHistoricPerfData historicData;
if (PipHistoricPerfData.Deserialize(reader, out historicData))
{
if (!table.m_runtimeData.TryAdd(semiStableHash, historicData))
{
throw new BuildXLException("Corrupted file has duplicate records");
}
}
}
return table;
}
},
ex => { throw new BuildXLException("Reading of file failed", ex); });
}
public void Success()
{
var fe = new FileEnvelope("Dummy", 0);
using (var stream = new MemoryStream())
{
FileEnvelopeId id = FileEnvelopeId.Create();
fe.WriteHeader(stream, id);
fe.FixUpHeader(stream, id);
stream.Position = 0;
fe.ReadHeader(stream);
}
}
public void MissingFixup()
{
var fe = new FileEnvelope("Dummy", 0);
using (var stream = new MemoryStream())
{
FileEnvelopeId id = FileEnvelopeId.Create();
fe.WriteHeader(stream, id);
// fe.FixUpHeader(stream, id);
stream.Position = 0;
Assert.Throws <BuildXLException>(
() => { fe.ReadHeader(stream); });
}
}
public void WrongEnvelopeVersion()
{
var fe0 = new FileEnvelope("Dummy0", 0);
var fe1 = new FileEnvelope("Dummy0", 1);
using (var stream = new MemoryStream())
{
FileEnvelopeId id = FileEnvelopeId.Create();
fe0.WriteHeader(stream, id);
fe0.FixUpHeader(stream, id);
stream.Position = 0;
Assert.Throws <BuildXLException>(
() => { fe1.ReadHeader(stream); });
}
}
public void DetectFileLengthCorruption()
{
var fe = new FileEnvelope("Dummy", 0);
using (var stream = new MemoryStream())
{
FileEnvelopeId id = FileEnvelopeId.Create();
fe.WriteHeader(stream, id);
fe.FixUpHeader(stream, id);
stream.WriteByte(0); // not taken into account in fixed up header magic
stream.Position = 0;
Assert.Throws <BuildXLException>(
() => { fe.ReadHeader(stream); });
}
}
public void DetectHeaderCorruption()
{
var r = new Random(0);
var fe = new FileEnvelope("Dummy", 0);
for (int i = 0; i < 10000; i++)
{
using (var stream = new MemoryStream())
{
FileEnvelopeId id = FileEnvelopeId.Create();
fe.WriteHeader(stream, id);
fe.FixUpHeader(stream, id);
stream.Position = r.Next((int)stream.Length - 1);
int b = stream.ReadByte();
stream.Position = stream.Position - 1;
stream.WriteByte((byte)(b ^ (1 << r.Next(8))));
stream.Position = 0;
Assert.Throws <BuildXLException>(
() => { fe.ReadHeader(stream); });
}
}
}
public static LoadingTrackerResult TryLoad(
LoggingContext loggingContext,
string path,
VolumeMap volumeMap,
IChangeJournalAccessor journal,
string buildEngineFingerprint,
bool loadForAllCapableVolumes = true)
{
Contract.Requires(loggingContext != null);
Contract.Requires(!loadForAllCapableVolumes || volumeMap != null);
Contract.Requires(!loadForAllCapableVolumes || journal != null);
Stopwatch stopwatch = Stopwatch.StartNew();
SafeFileHandle handle;
OpenFileResult result = FileUtilities.TryCreateOrOpenFile(
path,
FileDesiredAccess.GenericRead,
FileShare.Read | FileShare.Delete,
FileMode.Open,
// Ok to evict the file from standby since the file will be overwritten and never reread from disk after this point.
FileFlagsAndAttributes.FileFlagSequentialScan,
out handle);
if (result.Succeeded)
{
Contract.Assume(handle != null);
Contract.Assume(!handle.IsInvalid);
using (handle)
{
using (var stream = new FileStream(handle, FileAccess.Read))
{
FileEnvelopeId fileEnvelopeId;
try
{
fileEnvelopeId = FileEnvelope.ReadHeader(stream);
}
catch (BuildXLException e)
{
return(LoadingTrackerResult.FailBadFormatMarker(e.Message));
}
try
{
return(ExceptionUtilities.HandleRecoverableIOException(
() =>
{
using (var reader = new BuildXLReader(debug: false, stream: stream, leaveOpen: true))
{
bool wasTrackerDisabled = reader.ReadBoolean();
if (wasTrackerDisabled)
{
return LoadingTrackerResult.FailPriorTrackerDisabled();
}
var previousFingerprint = reader.ReadNullable(r => r.ReadString());
// only check for fingerprints match if the supplied fingerprint is valid
// this is to support special cases where we might want to load ChangeTracker
// regardless of the previously stored fingerprint value
if (buildEngineFingerprint != null && !string.Equals(previousFingerprint, buildEngineFingerprint, StringComparison.Ordinal))
{
return LoadingTrackerResult.FailBuildEngineFingerprintMismatch();
}
return FileChangeTrackingSet.TryLoad(
loggingContext,
fileEnvelopeId,
reader,
volumeMap,
journal,
stopwatch,
loadForAllCapableVolumes);
}
},
ex =>
{
throw new BuildXLException(ex.Message);
}));
}
catch (Exception e)
{
// Catch any exception. Failure in loading FileChangeTracker should not
// cause build break, or worse, make people stuck on erroneous state.
// In such a case, BuildXL simply has to start tracking from scratch.
return(LoadingTrackerResult.FailLoadException(e.GetLogEventMessage()));
}
}
}
}
Contract.Assume(handle == null);
return(LoadingTrackerResult.FailTrackingSetCannotBeOpened(result.CreateFailureForError().DescribeIncludingInnerFailures()));
}
/// <summary>
/// Creates and starts a task to deserialize an object
/// </summary>
/// <param name="file">This will become the filename</param>
/// <param name="deserializer">Deserialization function; its get a reader for the file stream, and a function that allows obtaining additional streams if needed</param>
/// <param name="skipHeader">If enabled, the correlation id is not checked for consistency</param>
/// <returns>task for deserialized value</returns>
internal Task <TObject> DeserializeFromFileAsync <TObject>(
GraphCacheFile file,
Func <BuildXLReader, Task <TObject> > deserializer,
bool skipHeader = false)
{
var task = Task.Run(
async() =>
{
var objectLabel = GetFileName(file);
string path = GetFullPath(objectLabel);
FileEnvelope fileEnvelope = GetFileEnvelope(file);
var result = default(TObject);
try
{
Stopwatch sw = Stopwatch.StartNew();
using (var fileStreamWrapper = m_readStreamProvider.OpenReadStream(path))
{
var fileStream = fileStreamWrapper.Value;
FileEnvelopeId persistedCorrelationId = fileEnvelope.ReadHeader(fileStream);
if (!skipHeader)
{
// We are going to check if all files that are going to be (concurrently) deserialized have matching correlation ids.
// The first discovered correlation id is going to be used to check all others.
if (m_correlationId == null)
{
Interlocked.CompareExchange(ref m_correlationId, persistedCorrelationId, null);
}
FileEnvelope.CheckCorrelationIds(persistedCorrelationId, (FileEnvelopeId)m_correlationId);
}
var isCompressed = fileStream.ReadByte() == 1;
using (Stream readStream = isCompressed ? new TrackedStream(new BufferedStream(new DeflateStream(fileStream, CompressionMode.Decompress), 64 << 10)) : fileStream)
using (BuildXLReader reader = new BuildXLReader(m_debug, readStream, leaveOpen: false))
{
result = await deserializer(reader);
}
}
Tracing.Logger.Log.DeserializedFile(LoggingContext, path, sw.ElapsedMilliseconds);
return(result);
}
catch (BuildXLException ex)
{
if (ex.InnerException is FileNotFoundException)
{
// Files might be deleted manually in the EngineCache directory. Log it as verbose.
Tracing.Logger.Log.FailedToDeserializeDueToFileNotFound(LoggingContext, path);
return(result);
}
Tracing.Logger.Log.FailedToDeserializePipGraph(LoggingContext, path, ex.LogEventMessage);
return(result);
}
catch (IOException ex)
{
Tracing.Logger.Log.FailedToDeserializePipGraph(LoggingContext, path, ex.Message);
return(result);
}
catch (TaskCanceledException)
{
throw;
}
catch (Exception ex)
{
// There are 2 reasons to be here.
// 1. A malformed file can cause ContractException, IndexOutOfRangeException, MemoryException or something else.
// 2. We may have a bug.
// Since the malformed file will always cause a crash until someone removes the file from the cache, allow BuildXL to recover
// by eating the exception. However remember to log it in order to keep track of bugs.
ExceptionRootCause rootCause = ExceptionUtilities.AnalyzeExceptionRootCause(ex);
BuildXL.Tracing.UnexpectedCondition.Log(LoggingContext, ex.ToStringDemystified() + Environment.NewLine + rootCause);
Tracing.Logger.Log.FailedToDeserializePipGraph(LoggingContext, path, ex.Message);
return(result);
}
});
lock (m_deserializationSyncObject)
{
m_deserializationTasks.Add(task);
}
return(task);
}
