/// <inheritdoc />
public async Task UpdateAsync(ContentHash contentHash, bool touch, IClock clock, UpdateFileInfo updateFileInfo)
{
ContentDirectory.TryGetValue(contentHash, out var existingInfo);
ContentFileInfo cloneInfo = null;
if (existingInfo != null)
{
cloneInfo = new ContentFileInfo(existingInfo.FileSize, existingInfo.LastAccessedFileTimeUtc, existingInfo.ReplicaCount);
if (touch)
{
existingInfo.UpdateLastAccessed(clock);
}
}
var updateTask = updateFileInfo(cloneInfo);
if (updateTask != null)
{
var updateInfo = await updateTask;
if (updateInfo != null)
{
ContentDirectory.TryGetValue(contentHash, out existingInfo);
if (existingInfo == null)
{
ContentDirectory.TryAdd(contentHash, updateInfo);
}
else if (existingInfo.ReplicaCount != updateInfo.ReplicaCount)
{
ContentDirectory[contentHash].ReplicaCount = updateInfo.ReplicaCount;
}
}
}
}
/// <inheritdoc />
public bool TryGetFileInfo(ContentHash contentHash, out ContentFileInfo fileInfo)
{
return(ContentDirectory.TryGetValue(contentHash, out fileInfo));
}
private async Task SerializeAsync()
{
if (ContentDirectory == null || ContentDirectory.Count == 0)
{
return;
}
var openTask = _fileSystem.OpenSafeAsync(FilePath, FileAccess.Write, FileMode.Create, FileShare.Delete);
var sync = new object();
var writeHeader = true;
var entries = ContentDirectory.ToArray();
var entriesRemaining = entries.Length;
var startIndex = 0;
var tasks = new List <Task>();
GetSizeAndReplicaCount(out var contentSize, out var replicaCount);
using (var stream = await openTask)
{
Action <int, int> writeChunk = (index, count) =>
{
var endIndexExclusive = index + count;
var entryCount = endIndexExclusive - index;
var bufferLength = entryCount * BinaryEntrySize;
var partitionBuffer = new byte[bufferLength];
var partitionContext = new BufferSerializeContext(partitionBuffer);
for (var i = index; i < endIndexExclusive; i++)
{
ContentFileInfo entry = entries[i].Value;
partitionContext.SerializeFull(entries[i].Key);
partitionContext.Serialize(entry.FileSize);
partitionContext.Serialize(entry.LastAccessedFileTimeUtc);
partitionContext.Serialize(UnusedAccessCount);
partitionContext.Serialize(entry.ReplicaCount);
}
lock (sync)
{
if (writeHeader)
{
writeHeader = false;
var headerBuffer = new byte[22];
var headerContext = new BufferSerializeContext(headerBuffer);
headerContext.Serialize(BinaryFormatMagicFlag);
headerContext.Serialize(BinaryFormatVersion);
headerContext.Serialize(entries.Length);
headerContext.Serialize(contentSize);
headerContext.Serialize(replicaCount);
stream.Write(headerBuffer, 0, headerBuffer.Length);
}
stream.Write(partitionBuffer, 0, partitionContext.Offset);
}
};
while (startIndex < entries.Length)
{
var i = startIndex;
var chunkCount = Math.Min(entriesRemaining, BinaryMaxEntriesPerChunk);
tasks.Add(Task.Run(() => writeChunk(i, chunkCount)));
startIndex += chunkCount;
entriesRemaining -= chunkCount;
}
await TaskSafetyHelpers.WhenAll(tasks);
Contract.Assert(startIndex == entries.Length);
}
}
private async Task <ContentMap> DeserializeBodyAsync(Context context, MemoryContentDirectoryHeader header, AbsolutePath path, bool isLoadingBackup)
{
var contentDirectory = new ContentMap();
try
{
var sw = Stopwatch.StartNew();
using (var stream = await _fileSystem.OpenSafeAsync(path, FileAccess.Read, FileMode.Open, FileShare.Read))
{
byte[] headerBuffer = new byte[header.HeaderSize];
stream.Read(headerBuffer, 0, header.HeaderSize);
var streamSync = new object();
var entriesSync = new object();
var entries = new List <KeyValuePair <ContentHash, ContentFileInfo> >(header.EntryCount);
var entriesRemaining = header.EntryCount;
var tasks = new List <Task>();
var nowFileTimeUtc = DateTime.UtcNow.ToFileTimeUtc();
long totalSize = 0;
long totalUniqueSize = 0;
long oldestContentAccessTimeUtc = nowFileTimeUtc;
long totalReplicaCount = 0;
var statsLock = new object();
Action <int> readChunk = count =>
{
var bufferLength = count * BinaryEntrySize;
var buffer = new byte[bufferLength];
int bytesRead;
lock (streamSync)
{
bytesRead = stream.Read(buffer, 0, bufferLength);
}
if (bytesRead != buffer.Length)
{
return;
}
var serializeContext = new BufferSerializeContext(buffer);
var partitionEntries = new List <KeyValuePair <ContentHash, ContentFileInfo> >(count);
for (var i = 0; i < count; i++)
{
var contentHash = serializeContext.DeserializeFullContentHash();
var fileSize = serializeContext.DeserializeInt64();
var lastAccessedFileTimeUtc = serializeContext.DeserializeInt64();
// Guard against corruption of serialized timestamps which affect LRU. If we get something out of range,
// force it to now.
if (lastAccessedFileTimeUtc < 0 || lastAccessedFileTimeUtc > nowFileTimeUtc)
{
lastAccessedFileTimeUtc = nowFileTimeUtc;
}
// ReSharper disable once UnusedVariable
var accessCount = serializeContext.DeserializeInt32();
var replicaCount = serializeContext.DeserializeInt32();
var contentFileInfo = new ContentFileInfo(fileSize, lastAccessedFileTimeUtc, replicaCount);
Interlocked.Add(ref totalSize, fileSize * replicaCount);
Interlocked.Add(ref totalUniqueSize, fileSize);
Interlocked.Add(ref totalReplicaCount, replicaCount);
if (oldestContentAccessTimeUtc > lastAccessedFileTimeUtc)
{
lock (statsLock)
{
if (oldestContentAccessTimeUtc > lastAccessedFileTimeUtc)
{
oldestContentAccessTimeUtc = lastAccessedFileTimeUtc;
}
}
}
partitionEntries.Add(new KeyValuePair <ContentHash, ContentFileInfo>(contentHash, contentFileInfo));
}
lock (entriesSync)
{
entries.AddRange(partitionEntries);
}
};
while (entriesRemaining > 0)
{
var chunkCount = Math.Min(entriesRemaining, BinaryMaxEntriesPerChunk);
tasks.Add(Task.Run(() => readChunk(chunkCount)));
entriesRemaining -= chunkCount;
}
await TaskSafetyHelpers.WhenAll(tasks);
context.Debug($"{Name}: Loaded content directory with {entries.Count} entries by {sw.ElapsedMilliseconds}ms: TotalContentSize={totalSize}, TotalUniqueSize={totalUniqueSize}, TotalReplicaCount={totalReplicaCount}, OldestContentTime={DateTime.FromFileTimeUtc(oldestContentAccessTimeUtc)}.");
if (entries.Count == header.EntryCount)
{
contentDirectory = new ContentMap(entries);
}
else
{
throw new CacheException($"Failed to read expected number of entries. Entries.Count={entries.Count}, Header.EntryCount={header.EntryCount}.");
}
}
if (!isLoadingBackup)
{
// At this point, we've either successfully read the file or tried and failed. Either way, the existing file should now be
// deleted. On a clean shutdown, it will be regenerated. On a dirty shutdown, we want it to already be gone otherwise
// we'll read in a file that is out-of-date.
_fileSystem.MoveFile(path, _backupFilePath, true);
}
}
catch (Exception exception)
{
context.Warning($"{Name} failed to deserialize {FilePath} - starting with empty directory: {exception}");
contentDirectory.Clear();
}
return(contentDirectory);
}
/// <summary>
/// For testing purposes only.
/// Attempts to add the given file info for the hash.
/// </summary>
internal bool TryAdd(ContentHash hash, ContentFileInfo fileInfo)
{
return(ContentDirectory.TryAdd(hash, fileInfo));
}
