/// <summary>
/// Creates and starts a task to serialize an object.
/// </summary>
/// <param name="fileType">Type for the object to serialize. This will become the filename</param>
/// <param name="serializer">Serialization action to perform</param>
/// <param name="overrideName">Overrides the default file name for the file type. This is used to atomically write some files (via renames)</param>
/// <returns>whether serialization was successful</returns>
public Task <SerializationResult> SerializeToFileAsync(
GraphCacheFile fileType,
Action <BuildXLWriter> serializer,
string overrideName = null)
{
var task = SerializeToFileInternalAsync(fileType, serializer, overrideName);
SerializationTasks.Add(task);
return(task);
}
private static string GetFileName(GraphCacheFile fileType)
{
switch (fileType)
{
case GraphCacheFile.PreviousInputs:
return(PreviousInputsFile);
case GraphCacheFile.PipTable:
return(PipTableFile);
case GraphCacheFile.PathTable:
return(PathTableFile);
case GraphCacheFile.StringTable:
return(StringTableFile);
case GraphCacheFile.SymbolTable:
return(SymbolTableFile);
case GraphCacheFile.QualifierTable:
return(QualifierTableFile);
case GraphCacheFile.MountPathExpander:
return(MountPathExpanderFile);
case GraphCacheFile.ConfigState:
return(ConfigFileStateFile);
case GraphCacheFile.DirectedGraph:
return(DirectedGraphFile);
case GraphCacheFile.PipGraph:
return(PipGraphFile);
case GraphCacheFile.PipGraphId:
return(PipGraphIdFile);
case GraphCacheFile.HistoricTableSizes:
return(HistoricTableSizes);
default:
throw Contract.AssertFailure("Unhandled GraphCacheFile");
}
}
private static FileEnvelope GetFileEnvelope(GraphCacheFile fileType)
{
switch (fileType)
{
case GraphCacheFile.PreviousInputs:
return(InputTracker.FileEnvelope);
case GraphCacheFile.PipTable:
return(PipTable.FileEnvelope);
case GraphCacheFile.PathTable:
return(PathTable.FileEnvelope);
case GraphCacheFile.StringTable:
return(StringTable.FileEnvelope);
case GraphCacheFile.SymbolTable:
return(SymbolTable.FileEnvelope);
case GraphCacheFile.QualifierTable:
return(QualifierTable.FileEnvelope);
case GraphCacheFile.MountPathExpander:
return(MountPathExpander.FileEnvelope);
case GraphCacheFile.ConfigState:
return(ConfigFileState.FileEnvelope);
case GraphCacheFile.DirectedGraph:
return(DirectedGraph.FileEnvelope);
case GraphCacheFile.PipGraph:
return(PipGraph.FileEnvelopeGraph);
case GraphCacheFile.PipGraphId:
return(PipGraph.FileEnvelopeGraphId);
case GraphCacheFile.HistoricTableSizes:
return(EngineContext.HistoricTableSizesFileEnvelope);
default:
throw Contract.AssertFailure("Unhandled GraphCacheFile");
}
}
private async Task <SerializationResult> SerializeToFileInternalAsync(GraphCacheFile fileType, Action <BuildXLWriter> serializer, string overrideName)
{
// Unblock the caller
await Task.Yield();
FileUtilities.CreateDirectory(m_engineCacheLocation);
string fileName = overrideName ?? GetFileName(fileType);
string path = Path.Combine(m_engineCacheLocation, fileName);
SerializationResult serializationResult = new SerializationResult()
{
Success = false,
FileType = fileType,
FullPath = path,
};
var fileEnvelope = GetFileEnvelope(fileType);
Contract.Assume(m_correlationId is FileEnvelopeId, "EngineSerializer must be initialized with a valid correlation id");
var correlationId = (FileEnvelopeId)m_correlationId;
try
{
// We must delete the existing file in case it was hardlinked from the cache. Opening a filestream
// that truncates the existing file will fail if it is a hardlink.
FileUtilities.DeleteFile(path, tempDirectoryCleaner: m_tempDirectoryCleaner);
using (
FileStream fileStream = FileUtilities.CreateFileStream(
path,
FileMode.Create,
FileAccess.Write,
FileShare.Delete,
// Do not write the file with SequentialScan since it will be reread in the subsequent build
FileOptions.None))
{
Stopwatch sw = Stopwatch.StartNew();
fileEnvelope.WriteHeader(fileStream, correlationId);
// Write whether the file is compressed or not.
fileStream.WriteByte(m_useCompression ? (byte)1 : (byte)0);
long uncompressedLength = 0;
if (m_useCompression)
{
using (var writer = new BuildXLWriter(m_debug, new TrackedStream(new BufferedStream(new DeflateStream(fileStream, CompressionLevel.Fastest, leaveOpen: true), bufferSize: 64 << 10)), false, false))
{
// TODO: We can improve performance significantly by parallelizing the compression.
// There's no setting to do that, but given you have the entire file content upfront in a memory stream,
// it shouldn't be particularly complicated to split the memory stream into reasonably sized chunks (say 100MB)
// and compress each of them into separate MemoryStream backed DeflateStreams in separate threads.
// Then just write those out to a file. Of course you'll need to write out the position of those streams
// into the header when you write out the actual file.
serializer(writer);
uncompressedLength = writer.BaseStream.Length;
}
}
else
{
using (var writer = new BuildXLWriter(m_debug, fileStream, leaveOpen: true, logStats: false))
{
serializer(writer);
uncompressedLength = writer.BaseStream.Length;
}
}
Interlocked.Add(ref m_bytesSavedDueToCompression, uncompressedLength - fileStream.Length);
fileEnvelope.FixUpHeader(fileStream, correlationId);
Tracing.Logger.Log.SerializedFile(LoggingContext, fileName, sw.ElapsedMilliseconds);
serializationResult.Success = true;
Interlocked.Add(ref m_bytesSerialized, fileStream.Position);
}
}
catch (BuildXLException ex)
{
Tracing.Logger.Log.FailedToSerializePipGraph(LoggingContext, ex.LogEventMessage);
}
catch (IOException ex)
{
Tracing.Logger.Log.FailedToSerializePipGraph(LoggingContext, ex.Message);
}
return(serializationResult);
}
/// <summary>
/// Creates and starts a task to deserialize an object
/// </summary>
/// <param name="file">This will become the filename</param>
/// <param name="deserializer">Deserialization function; its get a reader for the file stream, and a function that allows obtaining additional streams if needed</param>
/// <param name="skipHeader">If enabled, the correlation id is not checked for consistency</param>
/// <returns>task for deserialized value</returns>
internal Task <TObject> DeserializeFromFileAsync <TObject>(
GraphCacheFile file,
Func <BuildXLReader, Task <TObject> > deserializer,
bool skipHeader = false)
{
var task = Task.Run(
async() =>
{
var objectLabel = GetFileName(file);
string path = GetFullPath(objectLabel);
FileEnvelope fileEnvelope = GetFileEnvelope(file);
var result = default(TObject);
try
{
Stopwatch sw = Stopwatch.StartNew();
using (var fileStreamWrapper = m_readStreamProvider.OpenReadStream(path))
{
var fileStream = fileStreamWrapper.Value;
FileEnvelopeId persistedCorrelationId = fileEnvelope.ReadHeader(fileStream);
if (!skipHeader)
{
// We are going to check if all files that are going to be (concurrently) deserialized have matching correlation ids.
// The first discovered correlation id is going to be used to check all others.
if (m_correlationId == null)
{
Interlocked.CompareExchange(ref m_correlationId, persistedCorrelationId, null);
}
FileEnvelope.CheckCorrelationIds(persistedCorrelationId, (FileEnvelopeId)m_correlationId);
}
var isCompressed = fileStream.ReadByte() == 1;
using (Stream readStream = isCompressed ? new TrackedStream(new BufferedStream(new DeflateStream(fileStream, CompressionMode.Decompress), 64 << 10)) : fileStream)
using (BuildXLReader reader = new BuildXLReader(m_debug, readStream, leaveOpen: false))
{
result = await deserializer(reader);
}
}
Tracing.Logger.Log.DeserializedFile(LoggingContext, path, sw.ElapsedMilliseconds);
return(result);
}
catch (BuildXLException ex)
{
if (ex.InnerException is FileNotFoundException)
{
// Files might be deleted manually in the EngineCache directory. Log it as verbose.
Tracing.Logger.Log.FailedToDeserializeDueToFileNotFound(LoggingContext, path);
return(result);
}
Tracing.Logger.Log.FailedToDeserializePipGraph(LoggingContext, path, ex.LogEventMessage);
return(result);
}
catch (IOException ex)
{
Tracing.Logger.Log.FailedToDeserializePipGraph(LoggingContext, path, ex.Message);
return(result);
}
catch (TaskCanceledException)
{
throw;
}
catch (Exception ex)
{
// There are 2 reasons to be here.
// 1. A malformed file can cause ContractException, IndexOutOfRangeException, MemoryException or something else.
// 2. We may have a bug.
// Since the malformed file will always cause a crash until someone removes the file from the cache, allow BuildXL to recover
// by eating the exception. However remember to log it in order to keep track of bugs.
ExceptionRootCause rootCause = ExceptionUtilities.AnalyzeExceptionRootCause(ex);
BuildXL.Tracing.UnexpectedCondition.Log(LoggingContext, ex.ToStringDemystified() + Environment.NewLine + rootCause);
Tracing.Logger.Log.FailedToDeserializePipGraph(LoggingContext, path, ex.Message);
return(result);
}
});
lock (m_deserializationSyncObject)
{
m_deserializationTasks.Add(task);
}
return(task);
}
/// <summary>
/// Gets the full path to a serializable file given its file type
/// </summary>
internal string GetFullPath(GraphCacheFile fileType)
{
return(Path.Combine(m_engineCacheLocation, GetFileName(fileType)));
}
private static Lazy <Task <T> > CreateLazyFileDeserialization <T>(EngineSerializer serializer, GraphCacheFile file, Func <BuildXLReader, Task <T> > deserializer)
{
return(CreateAsyncLazy(() => serializer.DeserializeFromFileAsync <T>(file, deserializer)));
}
