internal void Save(Stream stream)
{
ExceptionUtilities.HandleRecoverableIOException(
() =>
{
// We don't have anything in particular to correlate this file to,
// so we are simply creating a unique correlation id that is used as part
// of the header consistency check.
FileEnvelopeId correlationId = FileEnvelopeId.Create();
FileEnvelope.WriteHeader(stream, correlationId);
using (BuildXLWriter writer = new BuildXLWriter(debug: false, stream: stream, leaveOpen: true, logStats: false))
{
writer.Write(m_runtimeData.Count);
foreach (KeyValuePair<long, PipHistoricPerfData> kvp in m_runtimeData)
{
writer.Write(kvp.Key);
kvp.Value.Serialize(writer);
}
}
FileEnvelope.FixUpHeader(stream, correlationId);
return (object)null;
},
ex => { throw new BuildXLException("Writing of file failed", ex); });
}
/// <summary>
/// Creates a new change tracker in the <see cref="FileChangeTrackingState.BuildingInitialChangeTrackingSet"/> state.
/// The caller may then add tracking for full set of files of interest, for later re-use by
/// <see cref="ResumeTrackingChanges(LoggingContext,BuildXL.Utilities.FileEnvelopeId,VolumeMap,IChangeJournalAccessor,FileChangeTrackingSet,string)"/>.
/// </summary>
public static FileChangeTracker StartTrackingChanges(
LoggingContext loggingContext,
VolumeMap volumeMap,
IChangeJournalAccessor journal,
string buildEngineFingerprint,
FileEnvelopeId?correlatedId = default)
{
Contract.Requires(loggingContext != null);
Contract.Requires(volumeMap != null);
Contract.Requires(journal != null);
Contract.Ensures(Contract.Result <FileChangeTracker>().TrackingState == FileChangeTrackingState.BuildingInitialChangeTrackingSet);
var tracker = new FileChangeTracker(
loggingContext,
correlatedId ?? FileEnvelopeId.Create(),
FileChangeTrackingState.BuildingInitialChangeTrackingSet,
volumeMap,
journal,
FileChangeTrackingSet.CreateForAllCapableVolumes(loggingContext, volumeMap, journal),
buildEngineFingerprint);
foreach (var gvfsProjectionFile in volumeMap.GvfsProjections)
{
var maybeTracking = tracker.TryProbeAndTrackPath(gvfsProjectionFile);
if (!maybeTracking.Succeeded)
{
Logger.Log.TrackChangesToGvfsProjectionFailed(loggingContext, gvfsProjectionFile, maybeTracking.Failure.DescribeIncludingInnerFailures());
}
}
return(tracker);
}
public void WrongCorrelationId()
{
FileEnvelopeId fe0 = FileEnvelopeId.Create();
FileEnvelopeId fe1 = FileEnvelopeId.Create();
Assert.Throws <BuildXLException>(
() => { FileEnvelope.CheckCorrelationIds(fe0, fe1); });
}
public void Success()
{
var fe = new FileEnvelope("Dummy", 0);
using (var stream = new MemoryStream())
{
FileEnvelopeId id = FileEnvelopeId.Create();
fe.WriteHeader(stream, id);
fe.FixUpHeader(stream, id);
stream.Position = 0;
fe.ReadHeader(stream);
}
}
public void MissingFixup()
{
var fe = new FileEnvelope("Dummy", 0);
using (var stream = new MemoryStream())
{
FileEnvelopeId id = FileEnvelopeId.Create();
fe.WriteHeader(stream, id);
// fe.FixUpHeader(stream, id);
stream.Position = 0;
Assert.Throws <BuildXLException>(
() => { fe.ReadHeader(stream); });
}
}
public void WrongEnvelopeVersion()
{
var fe0 = new FileEnvelope("Dummy0", 0);
var fe1 = new FileEnvelope("Dummy0", 1);
using (var stream = new MemoryStream())
{
FileEnvelopeId id = FileEnvelopeId.Create();
fe0.WriteHeader(stream, id);
fe0.FixUpHeader(stream, id);
stream.Position = 0;
Assert.Throws <BuildXLException>(
() => { fe1.ReadHeader(stream); });
}
}
public void DetectFileLengthCorruption()
{
var fe = new FileEnvelope("Dummy", 0);
using (var stream = new MemoryStream())
{
FileEnvelopeId id = FileEnvelopeId.Create();
fe.WriteHeader(stream, id);
fe.FixUpHeader(stream, id);
stream.WriteByte(0); // not taken into account in fixed up header magic
stream.Position = 0;
Assert.Throws <BuildXLException>(
() => { fe.ReadHeader(stream); });
}
}
/// <summary>
/// Creates a new change tracker in the <see cref="FileChangeTrackingState.BuildingInitialChangeTrackingSet"/> state.
/// The caller may then add tracking for full set of files of interest, for later re-use by
/// <see cref="ResumeTrackingChanges(LoggingContext,BuildXL.Utilities.FileEnvelopeId,VolumeMap,IChangeJournalAccessor,FileChangeTrackingSet,string)"/>.
/// </summary>
public static FileChangeTracker StartTrackingChanges(
LoggingContext loggingContext,
VolumeMap volumeMap,
IChangeJournalAccessor journal,
string buildEngineFingerprint,
FileEnvelopeId?correlatedId = default)
{
Contract.Requires(loggingContext != null);
Contract.Requires(volumeMap != null);
Contract.Requires(journal != null);
Contract.Ensures(Contract.Result <FileChangeTracker>().TrackingState == FileChangeTrackingState.BuildingInitialChangeTrackingSet);
return(new FileChangeTracker(
loggingContext,
correlatedId ?? FileEnvelopeId.Create(),
FileChangeTrackingState.BuildingInitialChangeTrackingSet,
volumeMap,
journal,
FileChangeTrackingSet.CreateForAllCapableVolumes(loggingContext, volumeMap, journal),
buildEngineFingerprint));
}
/// <summary>
/// Creates a new output logger.
///
/// The underlying file is created only upon first write.
/// </summary>
/// <param name="metadata">Metadata</param>
/// <param name="sidebandLogFile">File to which to save the log.</param>
/// <param name="rootDirectories">Only paths under one of these root directories will be recorded.</param>
public SidebandWriter(SidebandMetadata metadata, string sidebandLogFile, [CanBeNull] IReadOnlyList <string> rootDirectories)
{
Metadata = metadata;
SidebandLogFile = sidebandLogFile;
RootDirectories = rootDirectories;
m_recordedPathsCache = new HashSet <AbsolutePath>();
m_envelopeId = FileEnvelopeId.Create();
m_lazyBxlWriter = Lazy.Create(() =>
{
Directory.CreateDirectory(Path.GetDirectoryName(SidebandLogFile));
var writer = new BuildXLWriter(
stream: new FileStream(SidebandLogFile, FileMode.Create, FileAccess.Write, FileShare.Read | FileShare.Delete),
debug: false,
logStats: false,
leaveOpen: false);
// write header and metadata before anything else
FileEnvelope.WriteHeader(writer.BaseStream, m_envelopeId);
Metadata.Serialize(writer);
return(writer);
});
}
/// <summary>
/// Serializes and saves state needed for execution analyzers.
/// </summary>
private Task <bool> SaveExecutionStateToDiskAsync(ScheduleRunResult result)
{
// Every scheduler run has a unique log directory based off timestamp of run
string logDirectory = Path.Combine(
result.Config.Logging.LogsDirectory.ToString(Context.PathTable),
result.Config.Logging.LogPrefix);
var serializer = new EngineSerializer(
LoggingContext,
logDirectory,
correlationId: FileEnvelopeId.Create());
var dummyHistoricData = (IReadOnlyList <HistoricDataPoint>)CollectionUtilities.EmptyArray <HistoricDataPoint>();
var dummyHistoricTableSizes = new HistoricTableSizes(dummyHistoricData);
return(EngineSchedule.SaveExecutionStateToDiskAsync(
serializer,
Context,
PipTable,
result.Graph,
Expander,
dummyHistoricTableSizes));
}
/// <summary>
/// Creates a new output logger.
///
/// The underlying file is created only upon first write.
/// </summary>
/// <param name="sidebandLogFile">File to which to save the log.</param>
/// <param name="rootDirectories">Only paths under one of the root directories are recorded in <see cref="RecordFileWrite(PathTable, AbsolutePath)"/>.</param>
public SharedOpaqueOutputLogger(string sidebandLogFile, [CanBeNull] IReadOnlyList <string> rootDirectories)
{
SidebandLogFile = sidebandLogFile;
RootDirectories = rootDirectories;
m_recordedPathsCache = new HashSet <AbsolutePath>();
m_envelopeId = FileEnvelopeId.Create();
m_lazyBxlWriter = Lazy.Create(() =>
{
Directory.CreateDirectory(Path.GetDirectoryName(SidebandLogFile));
return(new BuildXLWriter(
stream: new FileStream(SidebandLogFile, FileMode.Create, FileAccess.Write, FileShare.Read | FileShare.Delete),
debug: false,
logStats: false,
leaveOpen: false));
});
m_lazyWriteHeader = Lazy.Create(() =>
{
FileEnvelope.WriteHeader(m_lazyBxlWriter.Value.BaseStream, m_envelopeId);
return(Unit.Void);
});
}
public void DetectHeaderCorruption()
{
var r = new Random(0);
var fe = new FileEnvelope("Dummy", 0);
for (int i = 0; i < 10000; i++)
{
using (var stream = new MemoryStream())
{
FileEnvelopeId id = FileEnvelopeId.Create();
fe.WriteHeader(stream, id);
fe.FixUpHeader(stream, id);
stream.Position = r.Next((int)stream.Length - 1);
int b = stream.ReadByte();
stream.Position = stream.Position - 1;
stream.WriteByte((byte)(b ^ (1 << r.Next(8))));
stream.Position = 0;
Assert.Throws <BuildXLException>(
() => { fe.ReadHeader(stream); });
}
}
}
/// <summary>
/// Writes a file containing versioning information to the provided store directory.
/// </summary>
private static void WriteVersionFile(string storeDirectory, int storeVersion)
{
var versionFile = GetVersionFile(storeDirectory);
using (var stream = FileUtilities.CreateFileStream(
versionFile,
FileMode.Create,
FileAccess.ReadWrite,
FileShare.Delete))
{
// We don't have anything in particular to correlate this file to,
// so we are simply creating a unique correlation id that is used as part
// of the header consistency check.
var correlationId = FileEnvelopeId.Create();
s_fileEnvelope.WriteHeader(stream, correlationId);
using (var writer = new BuildXLWriter(debug: false, stream: stream, leaveOpen: true, logStats: false))
{
writer.Write(storeVersion);
}
s_fileEnvelope.FixUpHeader(stream, correlationId);
}
}
private void SaveInternal(string fileContentTablePath)
{
Contract.Requires(!string.IsNullOrWhiteSpace(fileContentTablePath));
Contract.EnsuresOnThrow <BuildXLException>(true);
ExceptionUtilities.HandleRecoverableIOException(
() =>
{
int numEvicted = 0;
Directory.CreateDirectory(Path.GetDirectoryName(fileContentTablePath));
// Note that we are using a non-async file stream here. That's because we're doing lots of tiny writes for simplicity,
// but tiny writes on an async stream end up blocking anyway while adding silly overhead.
using (FileStream stream = FileUtilities.CreateFileStream(
fileContentTablePath,
FileMode.Create,
FileAccess.Write,
FileShare.Delete,
// Do not write the file with SequentialScan since it will be reread in the subsequent build
FileOptions.None))
{
// We don't have anything in particular to correlate this file to,
// so we are simply creating a unique correlation id that is used as part
// of the header consistency check.
FileEnvelopeId correlationId = FileEnvelopeId.Create();
s_fileEnvelope.WriteHeader(stream, correlationId);
using (var writer = new BuildXLWriter(debug: false, stream: stream, leaveOpen: true, logStats: false))
{
long numberOfEntriesPosition = writer.BaseStream.Position;
writer.Write(0U);
uint entriesWritten = 0;
var hashBuffer = new byte[ContentHashingUtilities.HashInfo.ByteLength];
foreach (var fileAndEntryPair in m_entries)
{
// Skip saving anything with a TTL of zero. These entries were loaded
// with a TTL of one (immediately decremented) and were not used since load.
// See class remarks.
if (fileAndEntryPair.Value.TimeToLive == 0)
{
numEvicted++;
continue;
}
// Key: Volume and File ID
fileAndEntryPair.Key.Serialize(writer);
// Entry: USN, hash, time to live.
writer.Write(fileAndEntryPair.Value.Usn.Value);
fileAndEntryPair.Value.Hash.SerializeHashBytes(hashBuffer, 0);
writer.Write(hashBuffer);
writer.Write(fileAndEntryPair.Value.Length);
writer.Write(fileAndEntryPair.Value.TimeToLive);
entriesWritten++;
}
var endPosition = writer.BaseStream.Position;
writer.BaseStream.Position = numberOfEntriesPosition;
writer.Write(entriesWritten);
writer.BaseStream.Position = endPosition;
}
s_fileEnvelope.FixUpHeader(stream, correlationId);
}
Counters.AddToCounter(FileContentTableCounters.NumEvicted, numEvicted);
return(Unit.Void);
},
ex => { throw new BuildXLException("Failure writing file content table", ex); });
}
/// <summary>
/// Get the file envelope id to save with. If no change has been made, reuse existing file envelope id. Otherwise, use overrideFileEnvelopeId or a new id if overrideFileEnvelopeId is not specified.
/// </summary>
public FileEnvelopeId GetFileEnvelopeToSaveWith(FileEnvelopeId?overrideFileEnvelopeId = default)
{
if (TrackingState == FileChangeTrackingState.TrackingChanges && !HasNewFileOrCheckpointData)
{
return(FileEnvelopeId);
}
// Use override when provided, otherwise use the existing file id if we are building the initial change tracking set,
// otherwise recreate a new file id.
return(overrideFileEnvelopeId ?? (TrackingState == FileChangeTrackingState.BuildingInitialChangeTrackingSet ? FileEnvelopeId : FileEnvelopeId.Create()));
}