async Task <IReadOnlyDictionary <string, FileFingerprint> > LoadBlobsImplAsync(CancellationToken cancellationToken) { var blobs = new Dictionary <string, FileFingerprint>(); var needRebuild = false; var blobCount = 0; var fileCount = 0; _fileSequence.Rescan(); var totalSize = 0L; var compressedSize = 0L; foreach (var fileInfo in _fileSequence.Files) { ++fileCount; try { fileInfo.Refresh(); if (fileInfo.Length < 5) { continue; } using (var fileStream = OpenCacheFileForRead(fileInfo)) using (var decodeStream = new DeflateStream(fileStream, CompressionMode.Decompress)) using (var bs = new SequentialReadStream(decodeStream)) { while (await br.ReadAsync(cancellationToken).ConfigureAwait(false)) { cancellationToken.ThrowIfCancellationRequested(); try { var fileFingerprint = await ReadFileFingerprintAsync(br, cancellationToken).ConfigureAwait(false); if (null == fileFingerprint) { needRebuild = true; break; } if (blobs.ContainsKey(fileFingerprint.FullFilePath)) { Debug.WriteLine($"Collision for {fileFingerprint.FullFilePath}"); } blobs[fileFingerprint.FullFilePath] = fileFingerprint; ++blobCount; } catch (IOException ex) { needRebuild = true; // The entry might or might not be valid. Debug.WriteLine("BsonFileFingerprintStore.LoadBlobsImplAsync() read failed: " + ex.Message); } } totalSize += bs.Position; compressedSize += fileStream.Length; } } catch (IOException) { needRebuild = true; } catch (InvalidDataException) { needRebuild = true; } catch (JsonException) { needRebuild = true; } } Debug.WriteLine($"Read {totalSize.BytesToMiB():F2}MiB bytes from {compressedSize.BytesToMiB():F2}MiB file"); var count = (double)blobs.Count; Debug.WriteLine($"Average size {totalSize / count:F1} bytes or {compressedSize / count:F1} compressed"); if (blobCount > blobs.Count + 100 + blobs.Count / 8) { needRebuild = true; } if (fileCount > 16) { needRebuild = true; } if (needRebuild) { Console.WriteLine("Rebuilding cache files"); await RebuildCacheAsync(blobs, cancellationToken).ConfigureAwait(false); } return(blobs); }
async Task <IReadOnlyDictionary <string, FileFingerprint> > LoadBlobsImplAsync(CancellationToken cancellationToken) { var blobs = new Dictionary <string, FileFingerprint>(); var needRebuild = false; var blobCount = 0; var fileCount = 0; _fileSequence.Rescan(); var totalSize = 0L; var compressedSize = 0L; foreach (var fileInfo in _fileSequence.Files) { ++fileCount; try { fileInfo.Refresh(); if (fileInfo.Length < 5) { needRebuild = true; continue; } cancellationToken.ThrowIfCancellationRequested(); var pl = new Pipe(); var writer = pl.Writer; void CopyToWriter(int length, byte[] buffer) { var input = buffer.AsSpan(0, length); while (input.Length > 0) { var output = writer.GetSpan(length); if (output.Length <= 0) { throw new InvalidOperationException("Unexpected non-positive span length: " + output.Length); } var copySize = Math.Min(input.Length, output.Length); input.Slice(0, copySize).CopyTo(output); input = input.Slice(copySize); writer.Advance(copySize); } } var decompressTask = Task.Run(async() => { var buffer = ArrayPool <byte> .Shared.Rent(64 * 1024); using (var fileStream = OpenMsgPackFileForRead(fileInfo)) using (var decodeStream = new DeflateStream(fileStream, CompressionMode.Decompress)) { try { for (; ;) { var read = await decodeStream.ReadAsync(buffer, 0, buffer.Length, cancellationToken).ConfigureAwait(false); if (read < 1) { break; } totalSize += read; CopyToWriter(read, buffer); await writer.FlushAsync(cancellationToken).ConfigureAwait(false); } compressedSize += fileStream.Length; } catch (IOException ex) { needRebuild = true; // The entry might or might not be valid. Debug.WriteLine("MessagePackFileFingerprintStore.LoadBlobsImplAsync() read failed: " + ex.Message); } finally { ArrayPool <byte> .Shared.Return(buffer); writer.Complete(); } } }, cancellationToken); var reader = pl.Reader; var deserializeTask = Task.Run(async() => { var workBuffer = ArrayPool <byte> .Shared.Rent(64 * 1024); try { for (; ;) { if (!reader.TryRead(out var input)) { input = await reader.ReadAsync(cancellationToken).ConfigureAwait(false); } var buffer = input.Buffer; while (buffer.Length > 32) { // We now have enough data to start reading. buffer.Slice(0, 5).CopyTo(workBuffer.AsSpan()); var length = IntFormatter.Deserialize(workBuffer, 0, MessagePackSerializer.DefaultResolver, out var actualSize); if (length == 0) { return; // EOF } if (length < 0 || length > workBuffer.Length) { throw new FileFormatException($"Invalid block length {length}"); } buffer = buffer.Slice(actualSize); while (buffer.Length < length) { if (input.IsCompleted) { return; } reader.AdvanceTo(buffer.Start, buffer.End); if (!reader.TryRead(out input)) { input = await reader.ReadAsync(cancellationToken).ConfigureAwait(false); } buffer = input.Buffer; } // We now have enough data to decode the block. buffer.Slice(0, length).CopyTo(workBuffer.AsSpan()); buffer = buffer.Slice(length); var fileFingerprint = MessagePackSerializer.Deserialize <FileFingerprint>(workBuffer, 0, MessagePackSerializer.DefaultResolver, out actualSize); if (length != actualSize) { throw new FileFormatException($"Block length mismatch {length} != {actualSize}"); } if (blobs.ContainsKey(fileFingerprint.FullFilePath)) { Debug.WriteLine($"Collision for {fileFingerprint.FullFilePath}"); } blobs[fileFingerprint.FullFilePath] = fileFingerprint; ++blobCount; } reader.AdvanceTo(buffer.Start, buffer.End); if (input.IsCompleted) { break; } } } finally { ArrayPool <byte> .Shared.Return(workBuffer); reader.Complete(); } }); await Task.WhenAll(decompressTask, deserializeTask).ConfigureAwait(false); } catch (IOException) { needRebuild = true; } catch (InvalidDataException) { needRebuild = true; } catch (FileFormatException) { needRebuild = true; } } Debug.WriteLine($"Read {totalSize.BytesToMiB():F2}MiB bytes from {compressedSize.BytesToMiB():F2}MiB file"); var count = (double)blobs.Count; Debug.WriteLine($"Average size {totalSize / count:F1} bytes or {compressedSize / count:F1} compressed"); if (blobCount > blobs.Count + 100 + blobs.Count / 8) { needRebuild = true; } if (fileCount > 16) { needRebuild = true; } if (needRebuild) { Console.WriteLine("Rebuilding cache files"); await RebuildCacheAsync(blobs, cancellationToken).ConfigureAwait(false); } return(blobs); }