/// <summary>
/// Reads a sub-range from the specified <paramref name="blob"/> into <see cref="_buffer"/>.
/// </summary>
/// <returns>
/// Total bytes read, may be smaller than requested, but only if there were
/// not enough bytes in the blob. <paramref name="likelyLong"/> will be true
/// if the caller expects the maxBytes to be reached by the read.
/// </returns>
private static async Task <int> ReadSubRangeAsync(
CloudBlob blob,
byte[] buffer,
long start,
int bufferStart,
long maxBytes,
bool likelyLong,
CancellationToken cancel)
{
while (true)
{
try
{
return(await AzureHelpers.RetryAsync(cancel, likelyLong, c =>
blob.DownloadRangeToByteArrayAsync(
buffer,
bufferStart,
start,
maxBytes,
new AccessCondition(),
new BlobRequestOptions(),
new OperationContext(),
c)));
}
catch (StorageException e) when(e.Message.StartsWith("Incorrect number of bytes received."))
{
// Due to a miscommunication between the Azure servers for Append Blob and the
// .NET client library, some responses are garbled due to a race condition out
// of our control. We retry straight away, as the problem is rarely present
// too many times in a row.
//
// See also: https://github.com/Azure/azure-storage-net/issues/366
}
}
}
/// <summary>
/// Refreshes the local cache of positions, keys and blobs from Azure.
/// Returns the write position.
/// </summary>
internal async Task <long> RefreshCache(CancellationToken cancel)
{
// Download the complete list of blobs from the server. It's easier to
// perform processing in-memory rather than streaming through the listing.
var newBlobs = await AzureHelpers.RetryAsync(cancel, false, _container.ListEventBlobsAsync);
if (newBlobs.Count == 0)
{
// This is an empty stream.
return(_lastKnownPosition = 0);
}
if (newBlobs.Count >= 3)
{
// Can benefit from compaction.
TriggerCompaction(newBlobs);
}
if (_blobs.Count > newBlobs.Count)
{
// A compaction has occurred, so all cached information is wrong.
_blobs.Clear();
_firstKey.Clear();
_firstPosition.Clear();
}
// STEP 1: _firstPosition and _lastKnownPosition
// =============================================
// Take into account any blobs not already present in the cache.
// This code computes _firstPosition exactly once for each blob (over multiple
// calls).
for (var i = _blobs.Count; i < newBlobs.Count; ++i)
{
_blobs.Add(newBlobs[i]);
_firstPosition.Add(i == 0 ? 0L : _firstPosition[i - 1] + newBlobs[i - 1].Properties.Length);
}
// The last known position always increases based on the length of the last blob.
_lastKnownPosition = newBlobs[newBlobs.Count - 1].Properties.Length
+ _firstPosition[_firstPosition.Count - 1];
// STEP 2: _firstKey
// =================
// Is the last blob empty or not ? If it is, we don't want to compute
// its _firstKey yet.
var nonEmptyBlobCount = _blobs[_blobs.Count - 1].Properties.Length < 6
? _blobs.Count - 1
: _blobs.Count;
var oldNonEmptyBlobCount = _firstKey.Count;
if (nonEmptyBlobCount == oldNonEmptyBlobCount)
{
// No new _firstKey entries to compute.
return(_lastKnownPosition);
}
// We have new '_firstKey' entries to compute, so first extend the list to
// the appropriate size with dummy data...
while (_firstKey.Count < nonEmptyBlobCount)
{
_firstKey.Add(0);
}
// ...then, in parallel, query the first key of each new blob.
await Task.WhenAll(
Enumerable.Range(oldNonEmptyBlobCount, nonEmptyBlobCount - oldNonEmptyBlobCount)
.Select(async pos =>
{
var buffer = new byte[6];
await ReadSubRangeAsync(_blobs[pos], buffer, 0, 0, 6, false, cancel);
_firstKey[pos] = buffer[2]
+ ((uint)buffer[3] << 8)
+ ((uint)buffer[4] << 16)
+ ((uint)buffer[5] << 24);
}));
return(_lastKnownPosition);
}
