public async Task GetNextPartitionAsync_1GB_WithReadOnlyMemory()
{
long memoryStart;
long memoryEnd;
var buffersRead = 0L;
var length = (1L * Constants.GB) - (1 * Constants.MB) - (1 * Constants.KB);
using (var expectedStream = new MockNonSeekableStream(length))
using (var reader = new StreamPartitioner(expectedStream))
{
memoryStart = GC.GetTotalMemory(true);
Assert.IsTrue(expectedStream.CanRead);
Assert.IsFalse(expectedStream.CanSeek);
do
{
var position = expectedStream.Position;
using (StreamPartition buffer = await reader.GetNextPartitionAsync())
{
if (buffer.Length == 0)
{
Assert.AreEqual(expectedStream.Length, expectedStream.Position);
break;
}
else
{
buffersRead++;
Assert.IsTrue(buffer.CanRead);
Assert.IsTrue(buffer.CanSeek);
buffer.Read(out ReadOnlyMemory <byte> memory, (int)buffer.Length);
Assert.AreEqual((int)buffer.Length, memory.Length);
}
}
Assert.IsTrue(GC.GetTotalMemory(true) - memoryStart < 8 * Storage.Constants.DefaultBufferSize); // TODO Assuming at most 8 buffers allocated
}while (true);
}
memoryEnd = GC.GetTotalMemory(true);
//logger.LogInformation($"{buffersRead} buffers read");
//logger.LogInformation($"{nameof(memoryStart)} = {memoryStart}; {nameof(memoryEnd)} = {memoryEnd}");
//logger.LogInformation($"delta = {memoryEnd - memoryStart}");
Assert.AreEqual(Math.Ceiling(1d * length / Storage.Constants.DefaultBufferSize), buffersRead);
Assert.IsTrue(memoryEnd - memoryStart < 8 * Storage.Constants.DefaultBufferSize); // TODO Assuming at most 8 buffers allocated
}
static void Main()
{
Application.EnableVisualStyles();
Application.SetCompatibleTextRenderingDefault(false);
//Application.Run(new MainForm());
System.Net.ServicePointManager.DefaultConnectionLimit = 100;
var request = new DownloadRequest(new Uri("http://ipv4.download.thinkbroadband.com/10MB.zip"));
File.Delete("D:\\Downloads\\TEST\\10MB.zip.tmp");
var stream = File.OpenWrite("D:\\Downloads\\TEST\\10MB.zip.tmp");
var partitionManager = new StreamPartition(stream);
var streamDownload = new StreamDownload(request, partitionManager);
var form = new DownloadForm(streamDownload);
Application.Run(form);
}
public void PartitionAllocation()
{
const int bufferSize = 100;
const int numberOfVolumes = 20;
using (var stream = new MemoryStream())
{
stream.SetLength(bufferSize * numberOfVolumes);
var partition = new StreamPartition(stream);
for (int i = 0; i < numberOfVolumes; i++)
{
var volume = partition.GetFreeVolume(0, bufferSize);
Assert.IsNotNull(volume);
Assert.AreEqual(volume.Range.Start, i * bufferSize);
Assert.AreEqual(volume.Range.Length, bufferSize);
}
var lastVolume = partition.GetFreeVolume(0, bufferSize);
Assert.IsNull(lastVolume);
}
}
public async Task Read_WithReadOnlyMemory()
{
var expected = TestHelper.GetRandomBuffer(10 * Constants.MB);
var actual = new byte[expected.Length];
Assert.AreNotSame(expected, actual);
using (var expectedStream = new NonSeekableStream(expected))
using (var reader = new StreamPartitioner(expectedStream))
{
Assert.IsTrue(expectedStream.CanRead);
Assert.IsFalse(expectedStream.CanSeek);
do
{
var position = expectedStream.Position;
using (StreamPartition buffer = await reader.GetNextPartitionAsync())
{
if (buffer.Length == 0)
{
Assert.AreEqual(expectedStream.Length, expectedStream.Position);
break;
}
else
{
Assert.IsTrue(buffer.CanRead);
Assert.IsTrue(buffer.CanSeek);
buffer.Read(out ReadOnlyMemory <byte> memory, (int)buffer.Length);
memory.CopyTo(new Memory <byte>(actual, (int)position, (int)buffer.Length));
}
}
}while (true);
TestHelper.AssertSequenceEqual(expected, actual);
}
}
private async Task StageBlockAsync(
StreamPartition block,
string blockId,
BlobRequestConditions conditions,
IProgress <long> progressHandler,
CancellationToken cancellationToken)
{
try
{
await _client.StageBlockAsync(
blockId,
new MemoryStream(block.Bytes, 0, block.Length, writable : false),
conditions : conditions,
progressHandler : progressHandler,
cancellationToken : cancellationToken)
.ConfigureAwait(false);
}
finally
{
// Return the memory used by the block to our ArrayPool as soon
// as we've staged it
block.Dispose();
}
}
private Response <BlobContentInfo> UploadInSequence(
Stream content,
int blockSize,
BlobHttpHeaders blobHttpHeaders,
IDictionary <string, string> metadata,
BlobRequestConditions conditions,
IProgress <long> progressHandler,
AccessTier?accessTier,
CancellationToken cancellationToken)
{
// Wrap the staging and commit calls in an Upload span for
// distributed tracing
DiagnosticScope scope = _client.ClientDiagnostics.CreateScope(
_operationName ?? $"{nameof(Azure)}.{nameof(Storage)}.{nameof(Blobs)}.{nameof(BlobClient)}.{nameof(BlobClient.Upload)}");
try
{
scope.Start();
// Wrap progressHandler in a AggregatingProgressIncrementer to prevent
// progress from being reset with each stage blob operation.
if (progressHandler != null)
{
progressHandler = new AggregatingProgressIncrementer(progressHandler);
}
// The list tracking blocks IDs we're going to commit
List <string> blockIds = new List <string>();
// Partition the stream into individual blocks and stage them
IAsyncEnumerator <StreamPartition> enumerator =
GetBlocksAsync(content, blockSize, async: false, cancellationToken)
.GetAsyncEnumerator(cancellationToken);
while (enumerator.MoveNextAsync().EnsureCompleted())
{
// Dispose the block after the loop iterates and return its
// memory to our ArrayPool
using StreamPartition block = enumerator.Current;
// Stage the next block
string blockId = GenerateBlockId(block.AbsolutePosition);
_client.StageBlock(
blockId,
new MemoryStream(block.Bytes, 0, block.Length, writable: false),
conditions: conditions,
progressHandler: progressHandler,
cancellationToken: cancellationToken);
blockIds.Add(blockId);
}
// Commit the block list after everything has been staged to
// complete the upload
return(_client.CommitBlockList(
blockIds,
blobHttpHeaders,
metadata,
conditions,
accessTier,
cancellationToken));
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
finally
{
scope.Dispose();
}
}
public async Task <Tuple <File, ContentRangeHeaderValue> > SendResumableFileAsync(Uri sessionUri, string contentType, StreamPartition fileStream, long from, long to, long length, IProgress <StreamProgress> progress, CancellationToken cancellationToken)
{
var client = GetClient();
var content = new StreamedContent(fileStream, progress, cancellationToken);
content.Headers.ContentType = new MediaTypeHeaderValue(contentType);
if (from > 0 || to < length - 1)
{
content.Headers.ContentRange = new ContentRangeHeaderValue(from, to, length);
}
var response = await client.PutAsync(sessionUri, content, cancellationToken);
return(await ProcessResumableUploadResponse(response));
}
