/// <summary> /// Reads a sub-range from the specified <paramref name="blob"/> into <see cref="_buffer"/>. /// </summary> /// <returns> /// Total bytes read, may be smaller than requested, but only if there were /// not enough bytes in the blob. <paramref name="likelyLong"/> will be true /// if the caller expects the maxBytes to be reached by the read. /// </returns> private static async Task <int> ReadSubRangeAsync( CloudBlob blob, byte[] buffer, long start, int bufferStart, long maxBytes, bool likelyLong, CancellationToken cancel) { while (true) { try { return(await AzureHelpers.RetryAsync(cancel, likelyLong, c => blob.DownloadRangeToByteArrayAsync( buffer, bufferStart, start, maxBytes, new AccessCondition(), new BlobRequestOptions(), new OperationContext(), c))); } catch (StorageException e) when(e.Message.StartsWith("Incorrect number of bytes received.")) { // Due to a miscommunication between the Azure servers for Append Blob and the // .NET client library, some responses are garbled due to a race condition out // of our control. We retry straight away, as the problem is rarely present // too many times in a row. // // See also: https://github.com/Azure/azure-storage-net/issues/366 } } }
/// <summary> /// Refreshes the local cache of positions, keys and blobs from Azure. /// Returns the write position. /// </summary> internal async Task <long> RefreshCache(CancellationToken cancel) { // Download the complete list of blobs from the server. It's easier to // perform processing in-memory rather than streaming through the listing. var newBlobs = await AzureHelpers.RetryAsync(cancel, false, _container.ListEventBlobsAsync); if (newBlobs.Count == 0) { // This is an empty stream. return(_lastKnownPosition = 0); } if (newBlobs.Count >= 3) { // Can benefit from compaction. TriggerCompaction(newBlobs); } if (_blobs.Count > newBlobs.Count) { // A compaction has occurred, so all cached information is wrong. _blobs.Clear(); _firstKey.Clear(); _firstPosition.Clear(); } // STEP 1: _firstPosition and _lastKnownPosition // ============================================= // Take into account any blobs not already present in the cache. // This code computes _firstPosition exactly once for each blob (over multiple // calls). for (var i = _blobs.Count; i < newBlobs.Count; ++i) { _blobs.Add(newBlobs[i]); _firstPosition.Add(i == 0 ? 0L : _firstPosition[i - 1] + newBlobs[i - 1].Properties.Length); } // The last known position always increases based on the length of the last blob. _lastKnownPosition = newBlobs[newBlobs.Count - 1].Properties.Length + _firstPosition[_firstPosition.Count - 1]; // STEP 2: _firstKey // ================= // Is the last blob empty or not ? If it is, we don't want to compute // its _firstKey yet. var nonEmptyBlobCount = _blobs[_blobs.Count - 1].Properties.Length < 6 ? _blobs.Count - 1 : _blobs.Count; var oldNonEmptyBlobCount = _firstKey.Count; if (nonEmptyBlobCount == oldNonEmptyBlobCount) { // No new _firstKey entries to compute. return(_lastKnownPosition); } // We have new '_firstKey' entries to compute, so first extend the list to // the appropriate size with dummy data... while (_firstKey.Count < nonEmptyBlobCount) { _firstKey.Add(0); } // ...then, in parallel, query the first key of each new blob. await Task.WhenAll( Enumerable.Range(oldNonEmptyBlobCount, nonEmptyBlobCount - oldNonEmptyBlobCount) .Select(async pos => { var buffer = new byte[6]; await ReadSubRangeAsync(_blobs[pos], buffer, 0, 0, 6, false, cancel); _firstKey[pos] = buffer[2] + ((uint)buffer[3] << 8) + ((uint)buffer[4] << 16) + ((uint)buffer[5] << 24); })); return(_lastKnownPosition); }