protected async Task RunTestAsync( Context context, int storeCount, Func <TestContext, Task> testFunc, ImplicitPin implicitPin = ImplicitPin.PutAndGet, bool enableDistributedEviction = false, int?replicaCreditInMinutes = null, bool enableRepairHandling = false, int iterations = 1, TestContext outerContext = null) { var additionalArgs = PrepareAdditionalCreateStoreArgs(storeCount); var startIndex = outerContext?.Stores.Count ?? 0; var indexedDirectories = Enumerable.Range(0, storeCount) .Select(i => new { Index = i, Directory = new DisposableDirectory(FileSystem, TestRootDirectoryPath / (i + startIndex).ToString()) }) .ToList(); for (int iteration = 0; iteration < iterations; iteration++) { var testFileCopier = outerContext?.FileCopier ?? new TestFileCopier() { WorkingDirectory = indexedDirectories[0].Directory.Path }; context.Always($"Starting test iteration {iteration}"); var stores = indexedDirectories.Select( directory => CreateStore( context, testFileCopier, directory.Directory, directory.Index, enableDistributedEviction, replicaCreditInMinutes, enableRepairHandling, additionalArgs)).ToList(); var startupResults = await TaskSafetyHelpers.WhenAll(stores.Select(async store => await store.StartupAsync(context))); Assert.True(startupResults.All(x => x.Succeeded), $"Failed to startup: {string.Join(Environment.NewLine, startupResults.Where(s => !s))}"); var id = 0; var sessions = stores.Select(store => store.CreateSession(context, "store" + id++, implicitPin).Session).ToList(); await TaskSafetyHelpers.WhenAll(sessions.Select(async session => await session.StartupAsync(context))); var testContext = new TestContext(context, testFileCopier, indexedDirectories.Select(p => p.Directory).ToList(), sessions, stores, iteration); await testFunc(testContext); await TaskSafetyHelpers.WhenAll( sessions.Select(async session => { if (!session.ShutdownCompleted) { await session.ShutdownAsync(context).ThrowIfFailure(); } })); sessions.ForEach(session => session.Dispose()); await TaskSafetyHelpers.WhenAll(Enumerable.Range(0, storeCount).Select(storeId => LogStats(testContext, storeId))); await TaskSafetyHelpers.WhenAll(stores.Select(async store => await store.ShutdownAsync(context))); stores.ForEach(store => store.Dispose()); } indexedDirectories.ForEach(directory => directory.Directory.Dispose()); }
private async Task SerializeAsync() { if (ContentDirectory == null || ContentDirectory.Count == 0) { return; } var openTask = _fileSystem.OpenSafeAsync(FilePath, FileAccess.Write, FileMode.Create, FileShare.Delete); var sync = new object(); var writeHeader = true; var entries = ContentDirectory.ToArray(); var entriesRemaining = entries.Length; var startIndex = 0; var tasks = new List <Task>(); GetSizeAndReplicaCount(out var contentSize, out var replicaCount); using (var stream = await openTask) { Action <int, int> writeChunk = (index, count) => { var endIndexExclusive = index + count; var entryCount = endIndexExclusive - index; var bufferLength = entryCount * BinaryEntrySize; var partitionBuffer = new byte[bufferLength]; var partitionContext = new BufferSerializeContext(partitionBuffer); for (var i = index; i < endIndexExclusive; i++) { ContentFileInfo entry = entries[i].Value; partitionContext.SerializeFull(entries[i].Key); partitionContext.Serialize(entry.FileSize); partitionContext.Serialize(entry.LastAccessedFileTimeUtc); partitionContext.Serialize(UnusedAccessCount); partitionContext.Serialize(entry.ReplicaCount); } lock (sync) { if (writeHeader) { writeHeader = false; var headerBuffer = new byte[22]; var headerContext = new BufferSerializeContext(headerBuffer); headerContext.Serialize(BinaryFormatMagicFlag); headerContext.Serialize(BinaryFormatVersion); headerContext.Serialize(entries.Length); headerContext.Serialize(contentSize); headerContext.Serialize(replicaCount); stream.Write(headerBuffer, 0, headerBuffer.Length); } stream.Write(partitionBuffer, 0, partitionContext.Offset); } }; while (startIndex < entries.Length) { var i = startIndex; var chunkCount = Math.Min(entriesRemaining, BinaryMaxEntriesPerChunk); tasks.Add(Task.Run(() => writeChunk(i, chunkCount))); startIndex += chunkCount; entriesRemaining -= chunkCount; } await TaskSafetyHelpers.WhenAll(tasks); Contract.Assert(startIndex == entries.Length); } }
private async Task <ContentMap> DeserializeBodyAsync(Context context, MemoryContentDirectoryHeader header, AbsolutePath path, bool isLoadingBackup) { var contentDirectory = new ContentMap(); try { var sw = Stopwatch.StartNew(); using (var stream = await _fileSystem.OpenSafeAsync(path, FileAccess.Read, FileMode.Open, FileShare.Read)) { byte[] headerBuffer = new byte[header.HeaderSize]; stream.Stream.Read(headerBuffer, 0, header.HeaderSize); var streamSync = new object(); var entriesSync = new object(); var entries = new List <KeyValuePair <ContentHash, ContentFileInfo> >(header.EntryCount); var entriesRemaining = header.EntryCount; var tasks = new List <Task>(); var nowFileTimeUtc = DateTime.UtcNow.ToFileTimeUtc(); long totalSize = 0; long totalUniqueSize = 0; long oldestContentAccessTimeUtc = nowFileTimeUtc; long totalReplicaCount = 0; var statsLock = new object(); Action <int> readChunk = count => { var bufferLength = count * BinaryEntrySize; var buffer = new byte[bufferLength]; int bytesRead; lock (streamSync) { bytesRead = stream.Stream.Read(buffer, 0, bufferLength); } if (bytesRead != buffer.Length) { return; } var serializeContext = new BufferSerializeContext(buffer); var partitionEntries = new List <KeyValuePair <ContentHash, ContentFileInfo> >(count); for (var i = 0; i < count; i++) { var contentHash = serializeContext.DeserializeFullContentHash(); var fileSize = serializeContext.DeserializeInt64(); var lastAccessedFileTimeUtc = serializeContext.DeserializeInt64(); // Guard against corruption of serialized timestamps which affect LRU. If we get something out of range, // force it to now. if (lastAccessedFileTimeUtc < 0 || lastAccessedFileTimeUtc > nowFileTimeUtc) { lastAccessedFileTimeUtc = nowFileTimeUtc; } // ReSharper disable once UnusedVariable var accessCount = serializeContext.DeserializeInt32(); var replicaCount = serializeContext.DeserializeInt32(); var contentFileInfo = new ContentFileInfo(fileSize, lastAccessedFileTimeUtc, replicaCount); Interlocked.Add(ref totalSize, fileSize * replicaCount); Interlocked.Add(ref totalUniqueSize, fileSize); Interlocked.Add(ref totalReplicaCount, replicaCount); if (oldestContentAccessTimeUtc > lastAccessedFileTimeUtc) { lock (statsLock) { if (oldestContentAccessTimeUtc > lastAccessedFileTimeUtc) { oldestContentAccessTimeUtc = lastAccessedFileTimeUtc; } } } partitionEntries.Add(new KeyValuePair <ContentHash, ContentFileInfo>(contentHash, contentFileInfo)); } lock (entriesSync) { entries.AddRange(partitionEntries); } }; while (entriesRemaining > 0) { var chunkCount = Math.Min(entriesRemaining, BinaryMaxEntriesPerChunk); tasks.Add(Task.Run(() => readChunk(chunkCount))); entriesRemaining -= chunkCount; } await TaskSafetyHelpers.WhenAll(tasks); context.Debug($"{Name}: Loaded content directory with {entries.Count} entries by {sw.ElapsedMilliseconds}ms: TotalContentSize={totalSize}, TotalUniqueSize={totalUniqueSize}, TotalReplicaCount={totalReplicaCount}, OldestContentTime={DateTime.FromFileTimeUtc(oldestContentAccessTimeUtc)}."); if (entries.Count == header.EntryCount) { contentDirectory = new ContentMap(entries); } else { throw new CacheException($"Failed to read expected number of entries. Entries.Count={entries.Count}, Header.EntryCount={header.EntryCount}."); } } if (!isLoadingBackup) { // At this point, we've either successfully read the file or tried and failed. Either way, the existing file should now be // deleted. On a clean shutdown, it will be regenerated. On a dirty shutdown, we want it to already be gone otherwise // we'll read in a file that is out-of-date. _fileSystem.MoveFile(path, _backupFilePath, true); } } catch (Exception exception) { context.Warning($"{Name} failed to deserialize {FilePath} - starting with empty directory: {exception}"); contentDirectory.Clear(); } return(contentDirectory); }