public async IAsyncEnumerable <StreamPartition> GetPartitionsAsync(
int maxActivePartitions,
int maxLoadedPartitions,
int size = Constants.DefaultBufferSize,
bool async = true,
[EnumeratorCancellation] CancellationToken ct = default
)
{
var activePartitionDisposalTasks = new List <Task>();
var loadedPartitions = new Queue <StreamPartition>();
var sourceComplete = false;
while (!ct.IsCancellationRequested)
{
// try to keep as many partitions loaded as possible
while (!sourceComplete && loadedPartitions.Count < maxLoadedPartitions)
{
Task <StreamPartition> partitionTask = GetNextPartitionAsync(size, async, ct);
#pragma warning disable IDE0068 // Use recommended dispose pattern // disposal is performed by the uploader
StreamPartition partition =
async
? await partitionTask.ConfigureAwait(false)
: partitionTask.EnsureCompleted();
#pragma warning restore IDE0068 // Use recommended dispose pattern
if (partition.Length == 0)
{
// we've run off the end of the source
if (_contentLength.HasValue)
{
// if we have a content length, then we should be able to guarantee this
Debug.Assert(partition.ParentPosition == _contentLength);
}
// don't yield this partition, and instead just let the load exhaust itself
sourceComplete = true;
partition.Dispose();
//Console.WriteLine("Source exhausted");
break;
}
else
{
loadedPartitions.Enqueue(partition);
//Console.WriteLine($"Enqueued partition {partition.ParentPosition}");
}
}
// if we already have a full set of active partitions, wait until some have completed
activePartitionDisposalTasks.RemoveAll(t => t.IsCompleted);
if (activePartitionDisposalTasks.Any() && activePartitionDisposalTasks.Count == maxActivePartitions)
{
//Console.WriteLine("Waiting for partition to complete...");
if (async)
{
await Task.WhenAny(activePartitionDisposalTasks).ConfigureAwait(false);
}
else
{
Task.WaitAny(activePartitionDisposalTasks.ToArray(), ct);
}
}
// now we are assured to have room for partitions to be worked on
if (loadedPartitions.Any())
{
StreamPartition partition = loadedPartitions.Dequeue();
//Console.WriteLine($"Activation partition {partition.ParentPosition}");
activePartitionDisposalTasks.Add(partition.DisposalTask);
// yield it to the consumer
yield return(partition);
}
else
{
// we've run out of buffered partitions, which means we've also run out of source
//Console.WriteLine("Partitions exhausted");
//Console.WriteLine("Waiting for remaining partitions to complete...");
if (async)
{
await Task.WhenAll(activePartitionDisposalTasks).ConfigureAwait(false);
}
else
{
Task.WaitAll(activePartitionDisposalTasks.ToArray(), ct);
}
//Console.WriteLine("All partitions complete...");
break;
}
}
// we're out of partitions, or cancellation was requested and we need to dispose of remaining partitions
foreach (StreamPartition partition in loadedPartitions)
{
//Console.WriteLine($"Disposing partition {partition.ParentPosition}");
partition.Dispose();
}
}
/// <summary>
/// Generate a forward-only sequence of substreams based on bufferSize.
/// </summary>
/// <returns>StreamPartition</returns>
private async Task <StreamPartition> GetNextPartitionAsync(Func <long, Stream> streamSource, bool disposeStream, Func <long> getStartPosition, Action <int> incrementStartPosition, int size = Constants.DefaultBufferSize, bool async = true, CancellationToken ct = default)
{
if (async)
{
await _getNextPartitionAsync_Semaphore.WaitAsync(ct).ConfigureAwait(false);
}
else
{
_getNextPartitionAsync_Semaphore.Wait();
}
var startPosition = getStartPosition();
Stream stream = streamSource(startPosition);
IMemoryOwner <byte> buffer;
lock (_memoryPool)
{
// TODO these operations should be simplified with Memory- and Span-accepting APIs in future NET Standard
buffer = _memoryPool.Rent(size);
}
//Console.WriteLine($"Rented buffer of size {size} bytes");
//this.logger?.LogTrace($"Rented buffer of size {size} bytes");
if (MemoryMarshal.TryGetArray <byte>(buffer.Memory, out ArraySegment <byte> segment))
{
var count =
async
? await stream.ReadAsync(segment.Array, 0, segment.Count, ct).ConfigureAwait(false)
: stream.Read(segment.Array, 0, segment.Count);
if (disposeStream)
{
stream.Dispose();
}
incrementStartPosition(count);
_getNextPartitionAsync_Semaphore.Release();
//this.logger?.LogTrace($"Read {count} bytes");
var partition = new StreamPartition(
buffer.Memory,
startPosition,
count,
() =>
{
buffer.Dispose();
//Console.WriteLine($"Disposed buffer of size {size} bytes");
//this.logger?.LogTrace($"Disposed buffer of size {size} bytes");
},
ct
);
//Console.WriteLine($"Creating partition {partition.ParentPosition}");
return(partition);
}
else
{
if (disposeStream)
{
stream.Dispose();
}
_getNextPartitionAsync_Semaphore.Release();
throw Errors.UnableAccessArray();
}
}