private async Task <BoolResult> ShutdownStoresAsync(Context context)
{
var tasks = new List <Task <BoolResult> >(StoresByName.Count);
tasks.AddRange(StoresByName.Select(kvp => kvp.Value.ShutdownAsync(context)));
await TaskSafetyHelpers.WhenAll(tasks);
var result = BoolResult.Success;
foreach (var task in tasks)
{
var r = await task;
if (!result.Succeeded && r.Succeeded)
{
result = r;
}
}
return(result);
}
public Task OpenStreamParallelPinsToMultiplePinContexts()
{
var context = new Context(Logger);
return(TestStore(context, _clock, async store =>
{
ContentHash contentHash;
using (var pinContext = store.CreatePinContext())
{
// Pin some new content.
const int size = MaxSizeHard / 3;
var r = await store.PutRandomAsync(context, size, ContentHashType, new PinRequest(pinContext));
contentHash = r.ContentHash;
store.PinMapForTest.ContainsKey(contentHash).Should().BeTrue();
store.PinMapForTest[contentHash].Count.Should().Be(1);
// Open multiple streams to same content, each pinning to a separate pin context.
var streams = Enumerable.Repeat <Stream>(null, NumParallelTasks).ToList();
var contexts = Enumerable.Range(0, NumParallelTasks).Select(i => store.CreatePinContext()).ToList();
var tasks = Enumerable.Range(0, NumParallelTasks).Select(i => Task.Run(async() => streams[i] =
(await store.OpenStreamAsync(context, contentHash, new PinRequest(contexts[i]))).Stream));
await TaskSafetyHelpers.WhenAll(tasks);
store.PinMapForTest[contentHash].Count.Should().Be(NumParallelTasks + 1);
// Disposing the streams to not unpin the content.
streams.ForEach(stream => stream.Dispose());
store.PinMapForTest[contentHash].Count.Should().Be(NumParallelTasks + 1);
// Disposing the separate pin contexts still does not unpin the content.
contexts.ForEach(c => c.Dispose());
store.PinMapForTest[contentHash].Count.Should().Be(1);
await pinContext.DisposeAsync();
}
// After all pin contexts disposed, content is no longer pinned.
Pin pin;
store.PinMapForTest.TryGetValue(contentHash, out pin).Should().Be(false);
}));
}
private Task RunManyForStoreAcrossServerRestartAsync(Func <Context, IContentStore, Task> requestFunc)
{
return(RunStoreTestAsync(async(context, store) =>
{
// Launch a bunch of duplicate requests in the background, with server restart mixed in.
var tasks = new List <Task>(101);
tasks.AddRange(Enumerable.Range(0, 50).Select(i => Task.Run(() =>
requestFunc(new Context(Logger), store))));
tasks.Add(((ITestServiceClientContentStore)store).RestartServerAsync(context));
tasks.AddRange(Enumerable.Range(0, 50).Select(i => Task.Run(() =>
requestFunc(new Context(Logger), store))));
try
{
await TaskSafetyHelpers.WhenAll(tasks);
}
catch (AggregateException ex)
{
AggregateException singleException = ex.Flatten();
string failureMessage = string.Join(",", singleException.InnerExceptions.Select(x => x.Message));
Assert.True(false, failureMessage);
}
}));
}
public void DedupHashFile
(
[Required] string[] path,
[DefaultValue(false)] bool chunks,
[DefaultValue(false)] bool childNodes,
[DefaultValue(false)] bool rollingHash,
[DefaultValue(FileSystemConstants.FileIOBufferSize)] int bufferSize,
[DefaultValue((long)0)] long startOffset
)
{
Initialize();
_displayChunks = chunks;
_displayChildNodes = childNodes;
var paths = new List <AbsolutePath>();
foreach (AbsolutePath root in path.Select(p => new AbsolutePath(p)))
{
if (_fileSystem.DirectoryExists(root))
{
paths.AddRange(_fileSystem.EnumerateFiles(root, EnumerateOptions.Recurse).Select(fileInfo => fileInfo.FullPath));
}
else if (_fileSystem.FileExists(root))
{
paths.Add(root);
}
else
{
throw new ArgumentException("given path is not an existing file or directory");
}
}
var buffer = new byte[bufferSize];
using (var hasher = new DedupNodeHashAlgorithm(rollingHash ? DedupNodeTree.Algorithm.RollingHash : DedupNodeTree.Algorithm.MaximallyPacked))
{
foreach (var p in paths)
{
hasher.Initialize();
TaskSafetyHelpers.SyncResultOnThreadPool(async() =>
{
using (var fs = await _fileSystem.OpenReadOnlySafeAsync(p, FileShare.Read | FileShare.Delete))
{
fs.Position = startOffset;
int bytesRead;
while ((bytesRead = await fs.ReadAsync(buffer, 0, buffer.Length)) > 0)
{
hasher.TransformBlock(buffer, 0, bytesRead, null, 0);
}
hasher.TransformFinalBlock(new byte[0], 0, 0);
DedupNode root = hasher.GetNode();
ulong offset = 0;
LogNode(true, string.Empty, root, p, ref offset);
}
return(0);
});
}
}
_logger.Always("Totals:");
_logger.Always($"Bytes: Unique={_uniqueBytes:N0} Total={_totalBytes:N0}");
_logger.Always($"Chunks: Unique={_allChunks.Count:N0} Total={_totalChunks:N0}");
_logger.Always($"Nodes: Unique={_allNodes.Count:N0} Total={_totalNodes:N0}");
}
protected async Task RunTestAsync(
Context context,
int storeCount,
Func <TestContext, Task> testFunc,
ImplicitPin implicitPin = ImplicitPin.PutAndGet,
bool enableDistributedEviction = false,
int?replicaCreditInMinutes = null,
bool enableRepairHandling = false,
bool emptyFileHashShortcutEnabled = false,
int iterations = 1)
{
var indexedDirectories = Enumerable.Range(0, storeCount)
.Select(i => new { Index = i, Directory = new DisposableDirectory(FileSystem, TestRootDirectoryPath / i.ToString()) })
.ToList();
var testFileCopier = new TestFileCopier();
for (int iteration = 0; iteration < iterations; iteration++)
{
var stores = indexedDirectories.Select(
directory =>
CreateStore(
context,
testFileCopier,
directory.Directory,
directory.Index,
enableDistributedEviction,
replicaCreditInMinutes,
enableRepairHandling,
emptyFileHashShortcutEnabled)).ToList();
var startupResults = await TaskSafetyHelpers.WhenAll(stores.Select(async store => await store.StartupAsync(context)));
Assert.True(startupResults.All(x => x.Succeeded), $"Failed to startup: {string.Join(Environment.NewLine, startupResults.Where(s => !s))}");
var id = 0;
var sessions = stores.Select(store => store.CreateSession(context, "store" + id++, implicitPin).Session).ToList();
await TaskSafetyHelpers.WhenAll(sessions.Select(async session => await session.StartupAsync(context)));
var testContext = new TestContext(context, testFileCopier, indexedDirectories.Select(p => p.Directory).ToList(), sessions, stores, iteration);
await testFunc(testContext);
await TaskSafetyHelpers.WhenAll(
sessions.Select(async session =>
{
if (!session.ShutdownCompleted)
{
await session.ShutdownAsync(context).ThrowIfFailure();
}
}));
sessions.ForEach(session => session.Dispose());
await TaskSafetyHelpers.WhenAll(Enumerable.Range(0, storeCount).Select(storeId => LogStats(testContext, storeId)));
await TaskSafetyHelpers.WhenAll(stores.Select(async store => await store.ShutdownAsync(context)));
stores.ForEach(store => store.Dispose());
}
indexedDirectories.ForEach(directory => directory.Directory.Dispose());
}
private async Task <ContentMap> DeserializeBodyAsync(Context context, MemoryContentDirectoryHeader header, AbsolutePath path, bool isLoadingBackup)
{
var contentDirectory = new ContentMap();
try
{
var sw = Stopwatch.StartNew();
using (var stream = await _fileSystem.OpenSafeAsync(path, FileAccess.Read, FileMode.Open, FileShare.Read))
{
byte[] headerBuffer = new byte[header.HeaderSize];
stream.Read(headerBuffer, 0, header.HeaderSize);
var streamSync = new object();
var entriesSync = new object();
var entries = new List <KeyValuePair <ContentHash, ContentFileInfo> >(header.EntryCount);
var entriesRemaining = header.EntryCount;
var tasks = new List <Task>();
var nowFileTimeUtc = DateTime.UtcNow.ToFileTimeUtc();
long totalSize = 0;
long totalUniqueSize = 0;
long oldestContentAccessTimeUtc = nowFileTimeUtc;
long totalReplicaCount = 0;
var statsLock = new object();
Action <int> readChunk = count =>
{
var bufferLength = count * BinaryEntrySize;
var buffer = new byte[bufferLength];
int bytesRead;
lock (streamSync)
{
bytesRead = stream.Read(buffer, 0, bufferLength);
}
if (bytesRead != buffer.Length)
{
return;
}
var serializeContext = new BufferSerializeContext(buffer);
var partitionEntries = new List <KeyValuePair <ContentHash, ContentFileInfo> >(count);
for (var i = 0; i < count; i++)
{
var contentHash = serializeContext.DeserializeFullContentHash();
var fileSize = serializeContext.DeserializeInt64();
var lastAccessedFileTimeUtc = serializeContext.DeserializeInt64();
// Guard against corruption of serialized timestamps which affect LRU. If we get something out of range,
// force it to now.
if (lastAccessedFileTimeUtc < 0 || lastAccessedFileTimeUtc > nowFileTimeUtc)
{
lastAccessedFileTimeUtc = nowFileTimeUtc;
}
// ReSharper disable once UnusedVariable
var accessCount = serializeContext.DeserializeInt32();
var replicaCount = serializeContext.DeserializeInt32();
var contentFileInfo = new ContentFileInfo(fileSize, lastAccessedFileTimeUtc, replicaCount);
Interlocked.Add(ref totalSize, fileSize * replicaCount);
Interlocked.Add(ref totalUniqueSize, fileSize);
Interlocked.Add(ref totalReplicaCount, replicaCount);
if (oldestContentAccessTimeUtc > lastAccessedFileTimeUtc)
{
lock (statsLock)
{
if (oldestContentAccessTimeUtc > lastAccessedFileTimeUtc)
{
oldestContentAccessTimeUtc = lastAccessedFileTimeUtc;
}
}
}
partitionEntries.Add(new KeyValuePair <ContentHash, ContentFileInfo>(contentHash, contentFileInfo));
}
lock (entriesSync)
{
entries.AddRange(partitionEntries);
}
};
while (entriesRemaining > 0)
{
var chunkCount = Math.Min(entriesRemaining, BinaryMaxEntriesPerChunk);
tasks.Add(Task.Run(() => readChunk(chunkCount)));
entriesRemaining -= chunkCount;
}
await TaskSafetyHelpers.WhenAll(tasks);
context.Debug($"{Name}: Loaded content directory with {entries.Count} entries by {sw.ElapsedMilliseconds}ms: TotalContentSize={totalSize}, TotalUniqueSize={totalUniqueSize}, TotalReplicaCount={totalReplicaCount}, OldestContentTime={DateTime.FromFileTimeUtc(oldestContentAccessTimeUtc)}.");
if (entries.Count == header.EntryCount)
{
contentDirectory = new ContentMap(entries);
}
else
{
throw new CacheException($"Failed to read expected number of entries. Entries.Count={entries.Count}, Header.EntryCount={header.EntryCount}.");
}
}
if (!isLoadingBackup)
{
// At this point, we've either successfully read the file or tried and failed. Either way, the existing file should now be
// deleted. On a clean shutdown, it will be regenerated. On a dirty shutdown, we want it to already be gone otherwise
// we'll read in a file that is out-of-date.
_fileSystem.MoveFile(path, _backupFilePath, true);
}
}
catch (Exception exception)
{
context.Warning($"{Name} failed to deserialize {FilePath} - starting with empty directory: {exception}");
contentDirectory.Clear();
}
return(contentDirectory);
}
private async Task SerializeAsync()
{
if (ContentDirectory == null || ContentDirectory.Count == 0)
{
return;
}
var openTask = _fileSystem.OpenSafeAsync(FilePath, FileAccess.Write, FileMode.Create, FileShare.Delete);
var sync = new object();
var writeHeader = true;
var entries = ContentDirectory.ToArray();
var entriesRemaining = entries.Length;
var startIndex = 0;
var tasks = new List <Task>();
GetSizeAndReplicaCount(out var contentSize, out var replicaCount);
using (var stream = await openTask)
{
Action <int, int> writeChunk = (index, count) =>
{
var endIndexExclusive = index + count;
var entryCount = endIndexExclusive - index;
var bufferLength = entryCount * BinaryEntrySize;
var partitionBuffer = new byte[bufferLength];
var partitionContext = new BufferSerializeContext(partitionBuffer);
for (var i = index; i < endIndexExclusive; i++)
{
ContentFileInfo entry = entries[i].Value;
partitionContext.SerializeFull(entries[i].Key);
partitionContext.Serialize(entry.FileSize);
partitionContext.Serialize(entry.LastAccessedFileTimeUtc);
partitionContext.Serialize(UnusedAccessCount);
partitionContext.Serialize(entry.ReplicaCount);
}
lock (sync)
{
if (writeHeader)
{
writeHeader = false;
var headerBuffer = new byte[22];
var headerContext = new BufferSerializeContext(headerBuffer);
headerContext.Serialize(BinaryFormatMagicFlag);
headerContext.Serialize(BinaryFormatVersion);
headerContext.Serialize(entries.Length);
headerContext.Serialize(contentSize);
headerContext.Serialize(replicaCount);
stream.Write(headerBuffer, 0, headerBuffer.Length);
}
stream.Write(partitionBuffer, 0, partitionContext.Offset);
}
};
while (startIndex < entries.Length)
{
var i = startIndex;
var chunkCount = Math.Min(entriesRemaining, BinaryMaxEntriesPerChunk);
tasks.Add(Task.Run(() => writeChunk(i, chunkCount)));
startIndex += chunkCount;
entriesRemaining -= chunkCount;
}
await TaskSafetyHelpers.WhenAll(tasks);
Contract.Assert(startIndex == entries.Length);
}
}
public void DedupHashFile
(
[Required] string[] path,
[Required] string hashType,
[DefaultValue(false)] bool chunks,
[DefaultValue(false)] bool childNodes,
[DefaultValue(FileSystemConstants.FileIOBufferSize)] int bufferSize,
[DefaultValue((long)0)] long startOffset
)
{
Initialize();
_displayChunks = chunks;
_displayChildNodes = childNodes;
if (!Enum.TryParse(hashType, out HashType dedupHashType))
{
throw new ArgumentException($"HashType couldn't be inferred - {hashType}. Valid HashType is required.");
}
var paths = new List <AbsolutePath>();
foreach (AbsolutePath root in path.Select(p => new AbsolutePath(Path.GetFullPath(p))))
{
if (_fileSystem.DirectoryExists(root))
{
paths.AddRange(_fileSystem.EnumerateFiles(root, EnumerateOptions.Recurse).Select(fileInfo => fileInfo.FullPath));
}
else if (_fileSystem.FileExists(root))
{
paths.Add(root);
}
else
{
throw new ArgumentException("given path is not an existing file or directory");
}
}
var buffer = new byte[bufferSize];
using (var contentHasher = new DedupNodeOrChunkHashAlgorithm(new ManagedChunker(dedupHashType.GetChunkerConfiguration())))
{
foreach (var p in paths)
{
contentHasher.Initialize();
TaskSafetyHelpers.SyncResultOnThreadPool(async() =>
{
using (Stream fs = _fileSystem.OpenReadOnly(p, FileShare.Read | FileShare.Delete))
{
fs.Position = startOffset;
int bytesRead;
while ((bytesRead = await fs.ReadAsync(buffer, 0, buffer.Length)) > 0)
{
contentHasher.TransformBlock(buffer, 0, bytesRead, null, 0);
}
contentHasher.TransformFinalBlock(new byte[0], 0, 0);
DedupNode root = contentHasher.GetNode();
ulong offset = 0;
LogNode(true, string.Empty, root, p, ref offset);
}
return(0);
});
}
}
_logger.Always("Totals:");
_logger.Always($"Bytes: Unique={_uniqueBytes:N0} Total={_totalBytes:N0}");
_logger.Always($"Chunks: Unique={_allChunks.Count:N0} Total={_totalChunks:N0}");
_logger.Always($"Nodes: Unique={_allNodes.Count:N0} Total={_totalNodes:N0}");
}
