async Task <IReadOnlyDictionary <string, FileFingerprint> > LoadBlobsImplAsync(CancellationToken cancellationToken)
{
var blobs = new Dictionary <string, FileFingerprint>();
var needRebuild = false;
var blobCount = 0;
var fileCount = 0;
_fileSequence.Rescan();
var totalSize = 0L;
var compressedSize = 0L;
foreach (var fileInfo in _fileSequence.Files)
{
++fileCount;
try
{
fileInfo.Refresh();
if (fileInfo.Length < 5)
{
continue;
}
using (var fileStream = OpenCacheFileForRead(fileInfo))
using (var decodeStream = new DeflateStream(fileStream, CompressionMode.Decompress))
using (var bs = new SequentialReadStream(decodeStream))
{
while (await br.ReadAsync(cancellationToken).ConfigureAwait(false))
{
cancellationToken.ThrowIfCancellationRequested();
try
{
var fileFingerprint = await ReadFileFingerprintAsync(br, cancellationToken).ConfigureAwait(false);
if (null == fileFingerprint)
{
needRebuild = true;
break;
}
if (blobs.ContainsKey(fileFingerprint.FullFilePath))
{
Debug.WriteLine($"Collision for {fileFingerprint.FullFilePath}");
}
blobs[fileFingerprint.FullFilePath] = fileFingerprint;
++blobCount;
}
catch (IOException ex)
{
needRebuild = true;
// The entry might or might not be valid.
Debug.WriteLine("BsonFileFingerprintStore.LoadBlobsImplAsync() read failed: " + ex.Message);
}
}
totalSize += bs.Position;
compressedSize += fileStream.Length;
}
}
catch (IOException)
{
needRebuild = true;
}
catch (InvalidDataException)
{
needRebuild = true;
}
catch (JsonException)
{
needRebuild = true;
}
}
Debug.WriteLine($"Read {totalSize.BytesToMiB():F2}MiB bytes from {compressedSize.BytesToMiB():F2}MiB file");
var count = (double)blobs.Count;
Debug.WriteLine($"Average size {totalSize / count:F1} bytes or {compressedSize / count:F1} compressed");
if (blobCount > blobs.Count + 100 + blobs.Count / 8)
{
needRebuild = true;
}
if (fileCount > 16)
{
needRebuild = true;
}
if (needRebuild)
{
Console.WriteLine("Rebuilding cache files");
await RebuildCacheAsync(blobs, cancellationToken).ConfigureAwait(false);
}
return(blobs);
}
async Task <IReadOnlyDictionary <string, FileFingerprint> > LoadBlobsImplAsync(CancellationToken cancellationToken)
{
var blobs = new Dictionary <string, FileFingerprint>();
var needRebuild = false;
var blobCount = 0;
var fileCount = 0;
_fileSequence.Rescan();
var totalSize = 0L;
var compressedSize = 0L;
foreach (var fileInfo in _fileSequence.Files)
{
++fileCount;
try
{
fileInfo.Refresh();
if (fileInfo.Length < 5)
{
needRebuild = true;
continue;
}
cancellationToken.ThrowIfCancellationRequested();
var pl = new Pipe();
var writer = pl.Writer;
void CopyToWriter(int length, byte[] buffer)
{
var input = buffer.AsSpan(0, length);
while (input.Length > 0)
{
var output = writer.GetSpan(length);
if (output.Length <= 0)
{
throw new InvalidOperationException("Unexpected non-positive span length: " + output.Length);
}
var copySize = Math.Min(input.Length, output.Length);
input.Slice(0, copySize).CopyTo(output);
input = input.Slice(copySize);
writer.Advance(copySize);
}
}
var decompressTask = Task.Run(async() =>
{
var buffer = ArrayPool <byte> .Shared.Rent(64 * 1024);
using (var fileStream = OpenMsgPackFileForRead(fileInfo))
using (var decodeStream = new DeflateStream(fileStream, CompressionMode.Decompress))
{
try
{
for (; ;)
{
var read = await decodeStream.ReadAsync(buffer, 0, buffer.Length, cancellationToken).ConfigureAwait(false);
if (read < 1)
{
break;
}
totalSize += read;
CopyToWriter(read, buffer);
await writer.FlushAsync(cancellationToken).ConfigureAwait(false);
}
compressedSize += fileStream.Length;
}
catch (IOException ex)
{
needRebuild = true;
// The entry might or might not be valid.
Debug.WriteLine("MessagePackFileFingerprintStore.LoadBlobsImplAsync() read failed: " + ex.Message);
}
finally
{
ArrayPool <byte> .Shared.Return(buffer);
writer.Complete();
}
}
}, cancellationToken);
var reader = pl.Reader;
var deserializeTask = Task.Run(async() =>
{
var workBuffer = ArrayPool <byte> .Shared.Rent(64 * 1024);
try
{
for (; ;)
{
if (!reader.TryRead(out var input))
{
input = await reader.ReadAsync(cancellationToken).ConfigureAwait(false);
}
var buffer = input.Buffer;
while (buffer.Length > 32)
{
// We now have enough data to start reading.
buffer.Slice(0, 5).CopyTo(workBuffer.AsSpan());
var length = IntFormatter.Deserialize(workBuffer, 0, MessagePackSerializer.DefaultResolver, out var actualSize);
if (length == 0)
{
return; // EOF
}
if (length < 0 || length > workBuffer.Length)
{
throw new FileFormatException($"Invalid block length {length}");
}
buffer = buffer.Slice(actualSize);
while (buffer.Length < length)
{
if (input.IsCompleted)
{
return;
}
reader.AdvanceTo(buffer.Start, buffer.End);
if (!reader.TryRead(out input))
{
input = await reader.ReadAsync(cancellationToken).ConfigureAwait(false);
}
buffer = input.Buffer;
}
// We now have enough data to decode the block.
buffer.Slice(0, length).CopyTo(workBuffer.AsSpan());
buffer = buffer.Slice(length);
var fileFingerprint = MessagePackSerializer.Deserialize <FileFingerprint>(workBuffer, 0, MessagePackSerializer.DefaultResolver, out actualSize);
if (length != actualSize)
{
throw new FileFormatException($"Block length mismatch {length} != {actualSize}");
}
if (blobs.ContainsKey(fileFingerprint.FullFilePath))
{
Debug.WriteLine($"Collision for {fileFingerprint.FullFilePath}");
}
blobs[fileFingerprint.FullFilePath] = fileFingerprint;
++blobCount;
}
reader.AdvanceTo(buffer.Start, buffer.End);
if (input.IsCompleted)
{
break;
}
}
}
finally
{
ArrayPool <byte> .Shared.Return(workBuffer);
reader.Complete();
}
});
await Task.WhenAll(decompressTask, deserializeTask).ConfigureAwait(false);
}
catch (IOException)
{
needRebuild = true;
}
catch (InvalidDataException)
{
needRebuild = true;
}
catch (FileFormatException)
{
needRebuild = true;
}
}
Debug.WriteLine($"Read {totalSize.BytesToMiB():F2}MiB bytes from {compressedSize.BytesToMiB():F2}MiB file");
var count = (double)blobs.Count;
Debug.WriteLine($"Average size {totalSize / count:F1} bytes or {compressedSize / count:F1} compressed");
if (blobCount > blobs.Count + 100 + blobs.Count / 8)
{
needRebuild = true;
}
if (fileCount > 16)
{
needRebuild = true;
}
if (needRebuild)
{
Console.WriteLine("Rebuilding cache files");
await RebuildCacheAsync(blobs, cancellationToken).ConfigureAwait(false);
}
return(blobs);
}
