public void Int32Roundtrip()
{
var buffer = new byte[4];
var context1 = new BufferSerializeContext(buffer);
context1.Serialize(7);
var context2 = new BufferSerializeContext(buffer);
var i = context2.DeserializeInt32();
i.Should().Be(7);
}
public void GuidRoundtrip()
{
var buffer = new byte[16];
var id1 = new Guid("0647396E-C99F-47DB-B567-C022E953CB69");
var context1 = new BufferSerializeContext(buffer);
context1.Serialize(id1);
var context2 = new BufferSerializeContext(buffer);
var id2 = context2.DeserializeGuid();
id2.Should().Be(id1);
}
public void Int64Roundtrip()
{
var buffer = new byte[16];
var ticks = DateTime.UtcNow.Ticks;
var context1 = new BufferSerializeContext(buffer);
context1.Serialize(ticks);
context1.Serialize(ticks + 1);
var context2 = new BufferSerializeContext(buffer);
var t1 = context2.DeserializeInt64();
var t2 = context2.DeserializeInt64();
t1.Should().Be(ticks);
t2.Should().Be(ticks + 1);
}
private async Task <MemoryContentDirectoryHeader> DeserializeHeaderAsync(Context context, AbsolutePath path)
{
try
{
var directoryHeader = new MemoryContentDirectoryHeader();
using (var stream = await _fileSystem.OpenSafeAsync(path, FileAccess.Read, FileMode.Open, FileShare.Read))
{
var header = new byte[BinaryHeaderSizeV2];
stream.Read(header, 0, header.Length);
var headerContext = new BufferSerializeContext(header);
directoryHeader.MagicFlag = headerContext.DeserializeByte();
if (directoryHeader.MagicFlag != BinaryFormatMagicFlag)
{
throw new CacheException("{0} binary format missing magic flag", Name);
}
directoryHeader.Version = headerContext.DeserializeByte();
directoryHeader.EntryCount = headerContext.DeserializeInt32();
if (directoryHeader.Version == BinaryFormatVersion)
{
header = new byte[BinaryHeaderExtraSizeV3];
stream.Read(header, 0, header.Length);
headerContext = new BufferSerializeContext(header);
directoryHeader.ContentSize = headerContext.DeserializeInt64();
directoryHeader.ReplicaCount = headerContext.DeserializeInt64();
directoryHeader.HeaderSize = BinaryHeaderSizeV2 + BinaryHeaderExtraSizeV3;
}
else
{
throw new CacheException("{0} expected {1} but read binary format version {2}", Name, BinaryFormatVersion, directoryHeader.Version);
}
return(directoryHeader);
}
}
catch (Exception exception)
{
context.Warning($"{Name} failed to deserialize header of {FilePath} - starting with empty directory: {exception}");
return(new MemoryContentDirectoryHeader());
}
}
public void OffsetPropertyInitialized()
{
var context = new BufferSerializeContext(new byte[0]);
context.Offset.Should().Be(0);
}
public void LengthProperty()
{
var context = new BufferSerializeContext(new byte[0]);
context.Length.Should().Be(0);
}
private async Task <ContentMap> DeserializeBodyAsync(Context context, MemoryContentDirectoryHeader header, AbsolutePath path, bool isLoadingBackup)
{
var contentDirectory = new ContentMap();
try
{
var sw = Stopwatch.StartNew();
using (var stream = await _fileSystem.OpenSafeAsync(path, FileAccess.Read, FileMode.Open, FileShare.Read))
{
byte[] headerBuffer = new byte[header.HeaderSize];
stream.Read(headerBuffer, 0, header.HeaderSize);
var streamSync = new object();
var entriesSync = new object();
var entries = new List <KeyValuePair <ContentHash, ContentFileInfo> >(header.EntryCount);
var entriesRemaining = header.EntryCount;
var tasks = new List <Task>();
var nowFileTimeUtc = DateTime.UtcNow.ToFileTimeUtc();
long totalSize = 0;
long totalUniqueSize = 0;
long oldestContentAccessTimeUtc = nowFileTimeUtc;
long totalReplicaCount = 0;
var statsLock = new object();
Action <int> readChunk = count =>
{
var bufferLength = count * BinaryEntrySize;
var buffer = new byte[bufferLength];
int bytesRead;
lock (streamSync)
{
bytesRead = stream.Read(buffer, 0, bufferLength);
}
if (bytesRead != buffer.Length)
{
return;
}
var serializeContext = new BufferSerializeContext(buffer);
var partitionEntries = new List <KeyValuePair <ContentHash, ContentFileInfo> >(count);
for (var i = 0; i < count; i++)
{
var contentHash = serializeContext.DeserializeFullContentHash();
var fileSize = serializeContext.DeserializeInt64();
var lastAccessedFileTimeUtc = serializeContext.DeserializeInt64();
// Guard against corruption of serialized timestamps which affect LRU. If we get something out of range,
// force it to now.
if (lastAccessedFileTimeUtc < 0 || lastAccessedFileTimeUtc > nowFileTimeUtc)
{
lastAccessedFileTimeUtc = nowFileTimeUtc;
}
// ReSharper disable once UnusedVariable
var accessCount = serializeContext.DeserializeInt32();
var replicaCount = serializeContext.DeserializeInt32();
var contentFileInfo = new ContentFileInfo(fileSize, lastAccessedFileTimeUtc, replicaCount);
Interlocked.Add(ref totalSize, fileSize * replicaCount);
Interlocked.Add(ref totalUniqueSize, fileSize);
Interlocked.Add(ref totalReplicaCount, replicaCount);
if (oldestContentAccessTimeUtc > lastAccessedFileTimeUtc)
{
lock (statsLock)
{
if (oldestContentAccessTimeUtc > lastAccessedFileTimeUtc)
{
oldestContentAccessTimeUtc = lastAccessedFileTimeUtc;
}
}
}
partitionEntries.Add(new KeyValuePair <ContentHash, ContentFileInfo>(contentHash, contentFileInfo));
}
lock (entriesSync)
{
entries.AddRange(partitionEntries);
}
};
while (entriesRemaining > 0)
{
var chunkCount = Math.Min(entriesRemaining, BinaryMaxEntriesPerChunk);
tasks.Add(Task.Run(() => readChunk(chunkCount)));
entriesRemaining -= chunkCount;
}
await TaskSafetyHelpers.WhenAll(tasks);
context.Debug($"{Name}: Loaded content directory with {entries.Count} entries by {sw.ElapsedMilliseconds}ms: TotalContentSize={totalSize}, TotalUniqueSize={totalUniqueSize}, TotalReplicaCount={totalReplicaCount}, OldestContentTime={DateTime.FromFileTimeUtc(oldestContentAccessTimeUtc)}.");
if (entries.Count == header.EntryCount)
{
contentDirectory = new ContentMap(entries);
}
else
{
throw new CacheException($"Failed to read expected number of entries. Entries.Count={entries.Count}, Header.EntryCount={header.EntryCount}.");
}
}
if (!isLoadingBackup)
{
// At this point, we've either successfully read the file or tried and failed. Either way, the existing file should now be
// deleted. On a clean shutdown, it will be regenerated. On a dirty shutdown, we want it to already be gone otherwise
// we'll read in a file that is out-of-date.
_fileSystem.MoveFile(path, _backupFilePath, true);
}
}
catch (Exception exception)
{
context.Warning($"{Name} failed to deserialize {FilePath} - starting with empty directory: {exception}");
contentDirectory.Clear();
}
return(contentDirectory);
}
private async Task SerializeAsync()
{
if (ContentDirectory == null || ContentDirectory.Count == 0)
{
return;
}
var openTask = _fileSystem.OpenSafeAsync(FilePath, FileAccess.Write, FileMode.Create, FileShare.Delete);
var sync = new object();
var writeHeader = true;
var entries = ContentDirectory.ToArray();
var entriesRemaining = entries.Length;
var startIndex = 0;
var tasks = new List <Task>();
GetSizeAndReplicaCount(out var contentSize, out var replicaCount);
using (var stream = await openTask)
{
Action <int, int> writeChunk = (index, count) =>
{
var endIndexExclusive = index + count;
var entryCount = endIndexExclusive - index;
var bufferLength = entryCount * BinaryEntrySize;
var partitionBuffer = new byte[bufferLength];
var partitionContext = new BufferSerializeContext(partitionBuffer);
for (var i = index; i < endIndexExclusive; i++)
{
ContentFileInfo entry = entries[i].Value;
partitionContext.SerializeFull(entries[i].Key);
partitionContext.Serialize(entry.FileSize);
partitionContext.Serialize(entry.LastAccessedFileTimeUtc);
partitionContext.Serialize(UnusedAccessCount);
partitionContext.Serialize(entry.ReplicaCount);
}
lock (sync)
{
if (writeHeader)
{
writeHeader = false;
var headerBuffer = new byte[22];
var headerContext = new BufferSerializeContext(headerBuffer);
headerContext.Serialize(BinaryFormatMagicFlag);
headerContext.Serialize(BinaryFormatVersion);
headerContext.Serialize(entries.Length);
headerContext.Serialize(contentSize);
headerContext.Serialize(replicaCount);
stream.Write(headerBuffer, 0, headerBuffer.Length);
}
stream.Write(partitionBuffer, 0, partitionContext.Offset);
}
};
while (startIndex < entries.Length)
{
var i = startIndex;
var chunkCount = Math.Min(entriesRemaining, BinaryMaxEntriesPerChunk);
tasks.Add(Task.Run(() => writeChunk(i, chunkCount)));
startIndex += chunkCount;
entriesRemaining -= chunkCount;
}
await TaskSafetyHelpers.WhenAll(tasks);
Contract.Assert(startIndex == entries.Length);
}
}
