/// <summary> /// Downloads the next block. /// </summary> private async Task DownloadBlock(bool async, CancellationToken cancellationToken) { Response <BlobDownloadStreamingResult> response; HttpRange range = new HttpRange(_offset, _blockSize); response = async ? await _blobClient.DownloadStreamingAsync(range, cancellationToken : cancellationToken).ConfigureAwait(false) : _blobClient.DownloadStreaming(range, cancellationToken: cancellationToken); _stream = response.Value.Content; _offset += response.Value.Details.ContentLength; _lastDownloadBytes = response.Value.Details.ContentLength; _blobLength = GetBlobLength(response); }
#pragma warning disable CA1822 // Does not acces instance data can be marked static. public virtual async Task <Segment> BuildSegment( #pragma warning restore CA1822 // Can't mock static methods in MOQ. bool async, string manifestPath, SegmentCursor cursor = default) { // Models we need for later List <Shard> shards = new List <Shard>(); DateTimeOffset dateTime = BlobChangeFeedExtensions.ToDateTimeOffset(manifestPath).Value; // Download segment manifest BlobClient blobClient = _containerClient.GetBlobClient(manifestPath); BlobDownloadStreamingResult blobDownloadStreamingResult; if (async) { blobDownloadStreamingResult = await blobClient.DownloadStreamingAsync().ConfigureAwait(false); } else { blobDownloadStreamingResult = blobClient.DownloadStreaming(); } // Parse segment manifest JsonDocument jsonManifest; if (async) { jsonManifest = await JsonDocument.ParseAsync(blobDownloadStreamingResult.Content).ConfigureAwait(false); } else { jsonManifest = JsonDocument.Parse(blobDownloadStreamingResult.Content); } foreach (JsonElement shardJsonElement in jsonManifest.RootElement.GetProperty("chunkFilePaths").EnumerateArray()) { string shardPath = shardJsonElement.ToString().Substring("$blobchangefeed/".Length); var shardCursor = cursor?.ShardCursors?.Find(x => x.CurrentChunkPath.StartsWith(shardPath, StringComparison.InvariantCulture)); Shard shard = await _shardFactory.BuildShard( async, shardPath, shardCursor) .ConfigureAwait(false); if (shard.HasNext()) { shards.Add(shard); } } int shardIndex = 0; string currentShardPath = cursor?.CurrentShardPath; if (!string.IsNullOrWhiteSpace(currentShardPath)) { shardIndex = shards.FindIndex(s => s.ShardPath == currentShardPath); if (shardIndex < 0) { // Either shard doesn't exist or cursor is pointing to end of shard. So start from beginning. shardIndex = 0; } } return(new Segment( shards, shardIndex, dateTime, manifestPath)); }
public async Task <ChangeFeed> BuildChangeFeed( DateTimeOffset?startTime, DateTimeOffset?endTime, string continuation, bool async, CancellationToken cancellationToken) { DateTimeOffset lastConsumable; Queue <string> years = new Queue <string>(); Queue <string> segments = new Queue <string>(); ChangeFeedCursor cursor = null; // Create cursor if (continuation != null) { cursor = JsonSerializer.Deserialize <ChangeFeedCursor>(continuation); ValidateCursor(_containerClient, cursor); startTime = BlobChangeFeedExtensions.ToDateTimeOffset(cursor.CurrentSegmentCursor.SegmentPath).Value; endTime = cursor.EndTime; } // Round start and end time if we are not using the cursor. else { startTime = startTime.RoundDownToNearestHour(); endTime = endTime.RoundUpToNearestHour(); } // Check if Change Feed has been abled for this account. bool changeFeedContainerExists; if (async) { changeFeedContainerExists = await _containerClient.ExistsAsync(cancellationToken : cancellationToken).ConfigureAwait(false); } else { changeFeedContainerExists = _containerClient.Exists(cancellationToken: cancellationToken); } if (!changeFeedContainerExists) { throw new ArgumentException("Change Feed hasn't been enabled on this account, or is currently being enabled."); } // Get last consumable BlobClient blobClient = _containerClient.GetBlobClient(Constants.ChangeFeed.MetaSegmentsPath); BlobDownloadStreamingResult blobDownloadInfo; try { if (async) { blobDownloadInfo = await blobClient.DownloadStreamingAsync(cancellationToken : cancellationToken).ConfigureAwait(false); } else { blobDownloadInfo = blobClient.DownloadStreaming(cancellationToken: cancellationToken); } } catch (RequestFailedException e) when(e.ErrorCode == BlobErrorCode.BlobNotFound) { return(ChangeFeed.Empty()); } JsonDocument jsonMetaSegment; if (async) { jsonMetaSegment = await JsonDocument.ParseAsync( blobDownloadInfo.Content, cancellationToken : cancellationToken ).ConfigureAwait(false); } else { jsonMetaSegment = JsonDocument.Parse(blobDownloadInfo.Content); } lastConsumable = jsonMetaSegment.RootElement.GetProperty("lastConsumable").GetDateTimeOffset(); // Get year paths years = await GetYearPathsInternal( async, cancellationToken).ConfigureAwait(false); // Dequeue any years that occur before start time if (startTime.HasValue) { while (years.Count > 0 && BlobChangeFeedExtensions.ToDateTimeOffset(years.Peek()) < startTime.RoundDownToNearestYear()) { years.Dequeue(); } } // There are no years. if (years.Count == 0) { return(ChangeFeed.Empty()); } while (segments.Count == 0 && years.Count > 0) { // Get Segments for year segments = await BlobChangeFeedExtensions.GetSegmentsInYearInternal( containerClient : _containerClient, yearPath : years.Dequeue(), startTime : startTime, endTime : BlobChangeFeedExtensions.MinDateTime(lastConsumable, endTime), async : async, cancellationToken : cancellationToken) .ConfigureAwait(false); } // We were on the last year, and there were no more segments. if (segments.Count == 0) { return(ChangeFeed.Empty()); } Segment currentSegment = await _segmentFactory.BuildSegment( async, segments.Dequeue(), cursor?.CurrentSegmentCursor) .ConfigureAwait(false); return(new ChangeFeed( _containerClient, _segmentFactory, years, segments, currentSegment, lastConsumable, startTime, endTime)); }