private async Task <MemoryContentDirectoryHeader> DeserializeHeaderAsync(Context context, AbsolutePath path)
{
try
{
var directoryHeader = new MemoryContentDirectoryHeader();
using (var stream = await _fileSystem.OpenSafeAsync(path, FileAccess.Read, FileMode.Open, FileShare.Read))
{
var header = new byte[BinaryHeaderSizeV2];
stream.Read(header, 0, header.Length);
var headerContext = new BufferSerializeContext(header);
directoryHeader.MagicFlag = headerContext.DeserializeByte();
if (directoryHeader.MagicFlag != BinaryFormatMagicFlag)
{
throw new CacheException("{0} binary format missing magic flag", Name);
}
directoryHeader.Version = headerContext.DeserializeByte();
directoryHeader.EntryCount = headerContext.DeserializeInt32();
if (directoryHeader.Version == BinaryFormatVersion)
{
header = new byte[BinaryHeaderExtraSizeV3];
stream.Read(header, 0, header.Length);
headerContext = new BufferSerializeContext(header);
directoryHeader.ContentSize = headerContext.DeserializeInt64();
directoryHeader.ReplicaCount = headerContext.DeserializeInt64();
directoryHeader.HeaderSize = BinaryHeaderSizeV2 + BinaryHeaderExtraSizeV3;
}
else
{
throw new CacheException("{0} expected {1} but read binary format version {2}", Name, BinaryFormatVersion, directoryHeader.Version);
}
return(directoryHeader);
}
}
catch (Exception exception)
{
context.Warning($"{Name} failed to deserialize header of {FilePath} - starting with empty directory: {exception}");
return(new MemoryContentDirectoryHeader());
}
}
/// <inheritdoc />
protected override async Task <BoolResult> StartupCoreAsync(OperationContext context)
{
Tracer.Info(context, $"{Name} startup");
if (_fileSystem.FileExists(_filePath))
{
_header = await DeserializeHeaderAsync(context, _filePath);
_initializeContentDirectory = InitializeContentDirectoryAsync(context, _header, _filePath, isLoadingBackup: false);
Tracer.Info(context, $"{Name} starting with {_header.EntryCount} entries.");
}
else if (_fileSystem.FileExists(_backupFilePath))
{
var backupHeader = await DeserializeHeaderAsync(context, _backupFilePath);
_header = new MemoryContentDirectoryHeader();
_initializeContentDirectory = InitializeContentDirectoryAsync(context, backupHeader, _backupFilePath, isLoadingBackup: true);
Tracer.Info(context, $"{Name} starting with {backupHeader.EntryCount} entries from backup file.");
}
else
{
_header = new MemoryContentDirectoryHeader();
_initializeContentDirectory = InitializeContentDirectoryAsync(context, _header, path: null, isLoadingBackup: false);
Tracer.Info(context, $"{Name} starting with {ContentDirectory.Count} entries from no file.");
}
// Tracing initialization asynchronously.
_initializeContentDirectory.ContinueWith(
t =>
{
Tracer.Info(context, $"{Name} started with {ContentDirectory.Count} entries.");
}).FireAndForget(context);
return(BoolResult.Success);
}
private async Task <ContentMap> DeserializeBodyAsync(Context context, MemoryContentDirectoryHeader header, AbsolutePath path, bool isLoadingBackup)
{
var contentDirectory = new ContentMap();
try
{
var sw = Stopwatch.StartNew();
using (var stream = await _fileSystem.OpenSafeAsync(path, FileAccess.Read, FileMode.Open, FileShare.Read))
{
byte[] headerBuffer = new byte[header.HeaderSize];
stream.Read(headerBuffer, 0, header.HeaderSize);
var streamSync = new object();
var entriesSync = new object();
var entries = new List <KeyValuePair <ContentHash, ContentFileInfo> >(header.EntryCount);
var entriesRemaining = header.EntryCount;
var tasks = new List <Task>();
var nowFileTimeUtc = DateTime.UtcNow.ToFileTimeUtc();
long totalSize = 0;
long totalUniqueSize = 0;
long oldestContentAccessTimeUtc = nowFileTimeUtc;
long totalReplicaCount = 0;
var statsLock = new object();
Action <int> readChunk = count =>
{
var bufferLength = count * BinaryEntrySize;
var buffer = new byte[bufferLength];
int bytesRead;
lock (streamSync)
{
bytesRead = stream.Read(buffer, 0, bufferLength);
}
if (bytesRead != buffer.Length)
{
return;
}
var serializeContext = new BufferSerializeContext(buffer);
var partitionEntries = new List <KeyValuePair <ContentHash, ContentFileInfo> >(count);
for (var i = 0; i < count; i++)
{
var contentHash = serializeContext.DeserializeFullContentHash();
var fileSize = serializeContext.DeserializeInt64();
var lastAccessedFileTimeUtc = serializeContext.DeserializeInt64();
// Guard against corruption of serialized timestamps which affect LRU. If we get something out of range,
// force it to now.
if (lastAccessedFileTimeUtc < 0 || lastAccessedFileTimeUtc > nowFileTimeUtc)
{
lastAccessedFileTimeUtc = nowFileTimeUtc;
}
// ReSharper disable once UnusedVariable
var accessCount = serializeContext.DeserializeInt32();
var replicaCount = serializeContext.DeserializeInt32();
var contentFileInfo = new ContentFileInfo(fileSize, lastAccessedFileTimeUtc, replicaCount);
Interlocked.Add(ref totalSize, fileSize * replicaCount);
Interlocked.Add(ref totalUniqueSize, fileSize);
Interlocked.Add(ref totalReplicaCount, replicaCount);
if (oldestContentAccessTimeUtc > lastAccessedFileTimeUtc)
{
lock (statsLock)
{
if (oldestContentAccessTimeUtc > lastAccessedFileTimeUtc)
{
oldestContentAccessTimeUtc = lastAccessedFileTimeUtc;
}
}
}
partitionEntries.Add(new KeyValuePair <ContentHash, ContentFileInfo>(contentHash, contentFileInfo));
}
lock (entriesSync)
{
entries.AddRange(partitionEntries);
}
};
while (entriesRemaining > 0)
{
var chunkCount = Math.Min(entriesRemaining, BinaryMaxEntriesPerChunk);
tasks.Add(Task.Run(() => readChunk(chunkCount)));
entriesRemaining -= chunkCount;
}
await TaskSafetyHelpers.WhenAll(tasks);
context.Debug($"{Name}: Loaded content directory with {entries.Count} entries by {sw.ElapsedMilliseconds}ms: TotalContentSize={totalSize}, TotalUniqueSize={totalUniqueSize}, TotalReplicaCount={totalReplicaCount}, OldestContentTime={DateTime.FromFileTimeUtc(oldestContentAccessTimeUtc)}.");
if (entries.Count == header.EntryCount)
{
contentDirectory = new ContentMap(entries);
}
else
{
throw new CacheException($"Failed to read expected number of entries. Entries.Count={entries.Count}, Header.EntryCount={header.EntryCount}.");
}
}
if (!isLoadingBackup)
{
// At this point, we've either successfully read the file or tried and failed. Either way, the existing file should now be
// deleted. On a clean shutdown, it will be regenerated. On a dirty shutdown, we want it to already be gone otherwise
// we'll read in a file that is out-of-date.
_fileSystem.MoveFile(path, _backupFilePath, true);
}
}
catch (Exception exception)
{
context.Warning($"{Name} failed to deserialize {FilePath} - starting with empty directory: {exception}");
contentDirectory.Clear();
}
return(contentDirectory);
}
private async Task <ContentMap> InitializeContentDirectoryAsync(Context context, MemoryContentDirectoryHeader header, AbsolutePath path, bool isLoadingBackup)
{
var operationContext = new OperationContext(context);
var result = await operationContext.PerformOperationAsync <Result <ContentMap> >(
Tracer,
async() =>
{
// Making this method asynchronous to initialize _initializeContentDirectory field as fast as possible
// even if the initialization is synchronous.
// This will enforce the invariant that ContentDirectory property is not null.
await Task.Yield();
bool canLoadContentDirectory = path != null && header.Version == BinaryFormatVersion;
var loadedContentDirectory = canLoadContentDirectory
? await DeserializeBodyAsync(context, header, path, isLoadingBackup)
: new ContentMap();
if (!isLoadingBackup && loadedContentDirectory.Count != 0)
{
// Successfully loaded primary content directory
return(loadedContentDirectory);
}
// Primary content directory is empty, missing, or failed to load
// Reconstruct to ensure content is populated.
ContentMap contentDirectory = new ContentMap();
var backupContentDirectory = loadedContentDirectory;
if (_host != null)
{
await AddBulkAsync(
contentDirectory: contentDirectory,
backupContentDirectory: backupContentDirectory,
hashInfoPairs: _host.Reconstruct(context));
}
else
{
// Host may be null in tests. Warn?
}
return(contentDirectory);
},
_counters[MemoryContentDirectoryCounters.InitializeContentDirectory]).ThrowIfFailure();
return(result.Value);
}
