#pragma warning disable CA1822 // Does not acces instance data can be marked static.
public virtual async Task <Shard> BuildShard(
#pragma warning restore CA1822 // Can't mock static methods in MOQ.
bool async,
string shardPath,
ShardCursor shardCursor = default)
{
// Models we'll need later
Queue <BlobItem> chunks = new Queue <BlobItem>();
long blockOffset = shardCursor?.BlockOffset ?? 0;
long eventIndex = shardCursor?.EventIndex ?? 0;
// Get Chunks
if (async)
{
await foreach (BlobHierarchyItem blobHierarchyItem in _containerClient.GetBlobsByHierarchyAsync(
prefix: shardPath).ConfigureAwait(false))
{
if (blobHierarchyItem.IsPrefix)
{
continue;
}
//Chunk chunk = new Chunk(_containerClient, blobHierarchyItem.Blob.Name);
chunks.Enqueue(blobHierarchyItem.Blob);
}
}
else
{
foreach (BlobHierarchyItem blobHierarchyItem in _containerClient.GetBlobsByHierarchy(
prefix: shardPath))
{
if (blobHierarchyItem.IsPrefix)
{
continue;
}
chunks.Enqueue(blobHierarchyItem.Blob);
}
}
long chunkIndex = 0;
string currentChunkPath = shardCursor?.CurrentChunkPath;
Chunk currentChunk = null;
if (chunks.Count > 0) // Chunks can be empty right after hour flips.
{
// Fast forward to current Chunk
if (!string.IsNullOrWhiteSpace(currentChunkPath))
{
while (chunks.Count > 0)
{
if (chunks.Peek().Name == currentChunkPath)
{
break;
}
else
{
chunks.Dequeue();
chunkIndex++;
}
}
if (chunks.Count == 0)
{
throw new ArgumentException($"Chunk {currentChunkPath} not found.");
}
}
BlobItem currentChunkBlobItem = chunks.Dequeue();
if (currentChunkBlobItem.Properties.ContentLength > blockOffset)
{
// There are more events to read from current chunk.
currentChunk = await _chunkFactory.BuildChunk(
async,
currentChunkBlobItem.Name,
blockOffset,
eventIndex).ConfigureAwait(false);
}
else if (currentChunkBlobItem.Properties.ContentLength < blockOffset)
{
// This shouldn't happen under normal circumstances, i.e. we couldn't read past the end of chunk.
throw new ArgumentException($"Cursor contains a blockOffset that is invalid. BlockOffset={blockOffset}");
}
else
{
// Otherwise we ended at the end of the chunk and no events has been written since then. Check if new chunk was created in case of current chunk overflow.
if (chunks.Count > 0)
{
currentChunk = await _chunkFactory.BuildChunk(
async,
chunks.Dequeue().Name).ConfigureAwait(false);
}
}
}
return(new Shard(
_containerClient,
_chunkFactory,
chunks,
currentChunk,
chunkIndex,
shardPath));
}
#pragma warning disable CA1822 // Does not acces instance data can be marked static.
public virtual async Task <Shard> BuildShard(
#pragma warning restore CA1822 // Can't mock static methods in MOQ.
bool async,
string shardPath,
ShardCursor shardCursor = default)
{
// Models we'll need later
Queue <string> chunks = new Queue <string>();
long chunkIndex = shardCursor?.ChunkIndex ?? 0;
long blockOffset = shardCursor?.BlockOffset ?? 0;
long eventIndex = shardCursor?.EventIndex ?? 0;
// Get Chunks
if (async)
{
await foreach (BlobHierarchyItem blobHierarchyItem in _containerClient.GetBlobsByHierarchyAsync(
prefix: shardPath).ConfigureAwait(false))
{
if (blobHierarchyItem.IsPrefix)
{
continue;
}
//Chunk chunk = new Chunk(_containerClient, blobHierarchyItem.Blob.Name);
chunks.Enqueue(blobHierarchyItem.Blob.Name);
}
}
else
{
foreach (BlobHierarchyItem blobHierarchyItem in _containerClient.GetBlobsByHierarchy(
prefix: shardPath))
{
if (blobHierarchyItem.IsPrefix)
{
continue;
}
chunks.Enqueue(blobHierarchyItem.Blob.Name);
}
}
Chunk currentChunk = null;
if (chunks.Count > 0) // Chunks can be empty right after hour flips.
{
// Fast forward to current Chunk
if (chunkIndex > 0)
{
for (int i = 0; i < chunkIndex; i++)
{
chunks.Dequeue();
}
}
currentChunk = _chunkFactory.BuildChunk(
chunks.Dequeue(),
blockOffset,
eventIndex);
}
return(new Shard(
_containerClient,
_chunkFactory,
chunks,
currentChunk,
chunkIndex));
}
#pragma warning disable CA1822 // Does not acces instance data can be marked static.
public virtual async Task <Segment> BuildSegment(
#pragma warning restore CA1822 // Can't mock static methods in MOQ.
bool async,
string manifestPath,
SegmentCursor cursor = default)
{
// Models we need for later
List <Shard> shards = new List <Shard>();
DateTimeOffset dateTime = BlobChangeFeedExtensions.ToDateTimeOffset(manifestPath).Value;
// Download segment manifest
BlobClient blobClient = _containerClient.GetBlobClient(manifestPath);
BlobDownloadStreamingResult blobDownloadStreamingResult;
if (async)
{
blobDownloadStreamingResult = await blobClient.DownloadStreamingAsync().ConfigureAwait(false);
}
else
{
blobDownloadStreamingResult = blobClient.DownloadStreaming();
}
// Parse segment manifest
JsonDocument jsonManifest;
if (async)
{
jsonManifest = await JsonDocument.ParseAsync(blobDownloadStreamingResult.Content).ConfigureAwait(false);
}
else
{
jsonManifest = JsonDocument.Parse(blobDownloadStreamingResult.Content);
}
foreach (JsonElement shardJsonElement in jsonManifest.RootElement.GetProperty("chunkFilePaths").EnumerateArray())
{
string shardPath = shardJsonElement.ToString().Substring("$blobchangefeed/".Length);
ShardCursor shardCursor = cursor?.ShardCursors?.Find(x => x.CurrentChunkPath.StartsWith(shardPath, StringComparison.InvariantCulture));
Shard shard = await _shardFactory.BuildShard(
async,
shardPath,
shardCursor)
.ConfigureAwait(false);
if (shard.HasNext())
{
shards.Add(shard);
}
}
int shardIndex = 0;
string currentShardPath = cursor?.CurrentShardPath;
if (!string.IsNullOrWhiteSpace(currentShardPath))
{
shardIndex = shards.FindIndex(s => s.ShardPath == currentShardPath);
if (shardIndex < 0)
{
// Either shard doesn't exist or cursor is pointing to end of shard. So start from beginning.
shardIndex = 0;
}
}
return(new Segment(
shards,
shardIndex,
dateTime,
manifestPath));
}