protected async Task RunTestAsync(
Context context,
int storeCount,
Func <TestContext, Task> testFunc,
ImplicitPin implicitPin = ImplicitPin.PutAndGet,
bool enableDistributedEviction = false,
int?replicaCreditInMinutes = null,
bool enableRepairHandling = false,
int iterations = 1,
TestContext outerContext = null)
{
var additionalArgs = PrepareAdditionalCreateStoreArgs(storeCount);
var startIndex = outerContext?.Stores.Count ?? 0;
var indexedDirectories = Enumerable.Range(0, storeCount)
.Select(i => new { Index = i, Directory = new DisposableDirectory(FileSystem, TestRootDirectoryPath / (i + startIndex).ToString()) })
.ToList();
for (int iteration = 0; iteration < iterations; iteration++)
{
var testFileCopier = outerContext?.FileCopier ?? new TestFileCopier()
{
WorkingDirectory = indexedDirectories[0].Directory.Path
};
context.Always($"Starting test iteration {iteration}");
var stores = indexedDirectories.Select(
directory =>
CreateStore(
context,
testFileCopier,
directory.Directory,
directory.Index,
enableDistributedEviction,
replicaCreditInMinutes,
enableRepairHandling,
additionalArgs)).ToList();
var startupResults = await TaskSafetyHelpers.WhenAll(stores.Select(async store => await store.StartupAsync(context)));
Assert.True(startupResults.All(x => x.Succeeded), $"Failed to startup: {string.Join(Environment.NewLine, startupResults.Where(s => !s))}");
var id = 0;
var sessions = stores.Select(store => store.CreateSession(context, "store" + id++, implicitPin).Session).ToList();
await TaskSafetyHelpers.WhenAll(sessions.Select(async session => await session.StartupAsync(context)));
var testContext = new TestContext(context, testFileCopier, indexedDirectories.Select(p => p.Directory).ToList(), sessions, stores, iteration);
await testFunc(testContext);
await TaskSafetyHelpers.WhenAll(
sessions.Select(async session =>
{
if (!session.ShutdownCompleted)
{
await session.ShutdownAsync(context).ThrowIfFailure();
}
}));
sessions.ForEach(session => session.Dispose());
await TaskSafetyHelpers.WhenAll(Enumerable.Range(0, storeCount).Select(storeId => LogStats(testContext, storeId)));
await TaskSafetyHelpers.WhenAll(stores.Select(async store => await store.ShutdownAsync(context)));
stores.ForEach(store => store.Dispose());
}
indexedDirectories.ForEach(directory => directory.Directory.Dispose());
}
private async Task SerializeAsync()
{
if (ContentDirectory == null || ContentDirectory.Count == 0)
{
return;
}
var openTask = _fileSystem.OpenSafeAsync(FilePath, FileAccess.Write, FileMode.Create, FileShare.Delete);
var sync = new object();
var writeHeader = true;
var entries = ContentDirectory.ToArray();
var entriesRemaining = entries.Length;
var startIndex = 0;
var tasks = new List <Task>();
GetSizeAndReplicaCount(out var contentSize, out var replicaCount);
using (var stream = await openTask)
{
Action <int, int> writeChunk = (index, count) =>
{
var endIndexExclusive = index + count;
var entryCount = endIndexExclusive - index;
var bufferLength = entryCount * BinaryEntrySize;
var partitionBuffer = new byte[bufferLength];
var partitionContext = new BufferSerializeContext(partitionBuffer);
for (var i = index; i < endIndexExclusive; i++)
{
ContentFileInfo entry = entries[i].Value;
partitionContext.SerializeFull(entries[i].Key);
partitionContext.Serialize(entry.FileSize);
partitionContext.Serialize(entry.LastAccessedFileTimeUtc);
partitionContext.Serialize(UnusedAccessCount);
partitionContext.Serialize(entry.ReplicaCount);
}
lock (sync)
{
if (writeHeader)
{
writeHeader = false;
var headerBuffer = new byte[22];
var headerContext = new BufferSerializeContext(headerBuffer);
headerContext.Serialize(BinaryFormatMagicFlag);
headerContext.Serialize(BinaryFormatVersion);
headerContext.Serialize(entries.Length);
headerContext.Serialize(contentSize);
headerContext.Serialize(replicaCount);
stream.Write(headerBuffer, 0, headerBuffer.Length);
}
stream.Write(partitionBuffer, 0, partitionContext.Offset);
}
};
while (startIndex < entries.Length)
{
var i = startIndex;
var chunkCount = Math.Min(entriesRemaining, BinaryMaxEntriesPerChunk);
tasks.Add(Task.Run(() => writeChunk(i, chunkCount)));
startIndex += chunkCount;
entriesRemaining -= chunkCount;
}
await TaskSafetyHelpers.WhenAll(tasks);
Contract.Assert(startIndex == entries.Length);
}
}
private async Task <ContentMap> DeserializeBodyAsync(Context context, MemoryContentDirectoryHeader header, AbsolutePath path, bool isLoadingBackup)
{
var contentDirectory = new ContentMap();
try
{
var sw = Stopwatch.StartNew();
using (var stream = await _fileSystem.OpenSafeAsync(path, FileAccess.Read, FileMode.Open, FileShare.Read))
{
byte[] headerBuffer = new byte[header.HeaderSize];
stream.Stream.Read(headerBuffer, 0, header.HeaderSize);
var streamSync = new object();
var entriesSync = new object();
var entries = new List <KeyValuePair <ContentHash, ContentFileInfo> >(header.EntryCount);
var entriesRemaining = header.EntryCount;
var tasks = new List <Task>();
var nowFileTimeUtc = DateTime.UtcNow.ToFileTimeUtc();
long totalSize = 0;
long totalUniqueSize = 0;
long oldestContentAccessTimeUtc = nowFileTimeUtc;
long totalReplicaCount = 0;
var statsLock = new object();
Action <int> readChunk = count =>
{
var bufferLength = count * BinaryEntrySize;
var buffer = new byte[bufferLength];
int bytesRead;
lock (streamSync)
{
bytesRead = stream.Stream.Read(buffer, 0, bufferLength);
}
if (bytesRead != buffer.Length)
{
return;
}
var serializeContext = new BufferSerializeContext(buffer);
var partitionEntries = new List <KeyValuePair <ContentHash, ContentFileInfo> >(count);
for (var i = 0; i < count; i++)
{
var contentHash = serializeContext.DeserializeFullContentHash();
var fileSize = serializeContext.DeserializeInt64();
var lastAccessedFileTimeUtc = serializeContext.DeserializeInt64();
// Guard against corruption of serialized timestamps which affect LRU. If we get something out of range,
// force it to now.
if (lastAccessedFileTimeUtc < 0 || lastAccessedFileTimeUtc > nowFileTimeUtc)
{
lastAccessedFileTimeUtc = nowFileTimeUtc;
}
// ReSharper disable once UnusedVariable
var accessCount = serializeContext.DeserializeInt32();
var replicaCount = serializeContext.DeserializeInt32();
var contentFileInfo = new ContentFileInfo(fileSize, lastAccessedFileTimeUtc, replicaCount);
Interlocked.Add(ref totalSize, fileSize * replicaCount);
Interlocked.Add(ref totalUniqueSize, fileSize);
Interlocked.Add(ref totalReplicaCount, replicaCount);
if (oldestContentAccessTimeUtc > lastAccessedFileTimeUtc)
{
lock (statsLock)
{
if (oldestContentAccessTimeUtc > lastAccessedFileTimeUtc)
{
oldestContentAccessTimeUtc = lastAccessedFileTimeUtc;
}
}
}
partitionEntries.Add(new KeyValuePair <ContentHash, ContentFileInfo>(contentHash, contentFileInfo));
}
lock (entriesSync)
{
entries.AddRange(partitionEntries);
}
};
while (entriesRemaining > 0)
{
var chunkCount = Math.Min(entriesRemaining, BinaryMaxEntriesPerChunk);
tasks.Add(Task.Run(() => readChunk(chunkCount)));
entriesRemaining -= chunkCount;
}
await TaskSafetyHelpers.WhenAll(tasks);
context.Debug($"{Name}: Loaded content directory with {entries.Count} entries by {sw.ElapsedMilliseconds}ms: TotalContentSize={totalSize}, TotalUniqueSize={totalUniqueSize}, TotalReplicaCount={totalReplicaCount}, OldestContentTime={DateTime.FromFileTimeUtc(oldestContentAccessTimeUtc)}.");
if (entries.Count == header.EntryCount)
{
contentDirectory = new ContentMap(entries);
}
else
{
throw new CacheException($"Failed to read expected number of entries. Entries.Count={entries.Count}, Header.EntryCount={header.EntryCount}.");
}
}
if (!isLoadingBackup)
{
// At this point, we've either successfully read the file or tried and failed. Either way, the existing file should now be
// deleted. On a clean shutdown, it will be regenerated. On a dirty shutdown, we want it to already be gone otherwise
// we'll read in a file that is out-of-date.
_fileSystem.MoveFile(path, _backupFilePath, true);
}
}
catch (Exception exception)
{
context.Warning($"{Name} failed to deserialize {FilePath} - starting with empty directory: {exception}");
contentDirectory.Clear();
}
return(contentDirectory);
}
