protected static CloudQueueMessage CreateJobQueueMessage(TestMsg message)
{
DataContractJsonSerializer serializer = new DataContractJsonSerializer(message.GetType());
MemoryStream memoryStream = new MemoryStream();
serializer.WriteObject(memoryStream, message);
return(new CloudQueueMessage(System.Text.Encoding.Default.GetString(memoryStream.GetBuffer())));
}
public override async Task Run(string[] args)
{
// Parse the arguments.
if (!ParseArguments(args, out uint chunkSizeBytes, out uint numInstances, out string blobName, out string containerName))
{
// If invalid arguments were provided, exit the test.
return;
}
// Determine the total size of the chosen blob.
long startingIndex = 0;
long blobSizeBytes = await GetBlobSize(blobName, containerName);
long totalChunks = blobSizeBytes / chunkSizeBytes;
long numChunksPerPass = totalChunks / numInstances;
uint remainingChunks = (uint)(numChunksPerPass % numInstances);
long remainingBytes = blobSizeBytes % chunkSizeBytes;
Console.WriteLine($"Downloading {containerName}/{blobName} ({blobSizeBytes} bytes).");
Task[] tasks = new Task[numInstances];
HashSet <Guid> operationIDs = new HashSet <Guid>();
for (uint i = 0; i < numInstances; ++i)
{
long numChunks = numChunksPerPass;
// Evenly distribute remaining chunks across instances such that no instance differs by more than 1 chunk.
if (remainingChunks > 0)
{
numChunks++;
remainingChunks--;
}
Guid opId = Guid.NewGuid();
operationIDs.Add(opId);
long length = numChunks * chunkSizeBytes;
if (i == numInstances - 1)
{
// Add any remaining bytes ( < 1 chunk) to the last worker (fair since remaining chunks are distributed to the 1st workers).
length += remainingBytes;
}
GetBlobOperation operation = new GetBlobOperation(opId, blobName, containerName, startingIndex, length, chunkSizeBytes);
TestMsg putBlobMsg = new TestMsg(Environment.MachineName, operation);
tasks[i] = JobQueue.AddMessageAsync(CreateJobQueueMessage(putBlobMsg));
// Update the starting position.
startingIndex += chunkSizeBytes * numChunks;
}
await Task.WhenAll(tasks);
await ReportStatus(operationIDs, blobSizeBytes);
}
public async Task Start(string[] args)
{
// Parse the arguments.
if (!ParseArguments(args, out uint levelOfConcurrency))
{
// If invalid arguments were provided, exit the role.
return;
}
// Wait for a message to appear on the job queue.
while (true)
{
Console.WriteLine("Awaiting a job to perform.");
CloudQueueMessage rawMessage = await jobQueue_.GetMessageAsync();
while (rawMessage == null)
{
await Task.Delay(TimeSpan.FromSeconds(1));
rawMessage = await jobQueue_.GetMessageAsync();
}
TestMsg message = null;
try
{
message = ConvertQueueMessage(rawMessage);
}
catch (Exception ex)
{
// Upon failure to deserialize, log and throw the invalid message away.
Console.WriteLine($"[ERROR] Failed to deserialize message. Details: {ex.Message}");
continue;
}
finally
{
await jobQueue_.DeleteMessageAsync(rawMessage);
}
Console.WriteLine("Message received.");
CloudBlobContainer container = blobClient_.GetContainerReference(message.Operation.ContainerName);
CloudBlockBlob blob = container.GetBlockBlobReference(message.Operation.BlobName);
TestOperation operation = message.Operation;
if (operation is PutBlobOperation || operation is GetBlobOperation)
{
// Define the task to be performed.
Func <Task <TimeSpan> > executeOperation = null;
if (operation is PutBlobOperation)
{
var detailedOperation = operation as PutBlobOperation;
executeOperation = () => { return(UploadBlocksAsync(blobClient_, container, detailedOperation.BlobName, detailedOperation.BlockSizeBytes, detailedOperation.NumBlocks, detailedOperation.StartingBlockId, levelOfConcurrency)); };
}
else if (operation is GetBlobOperation)
{
var detailedOperation = operation as GetBlobOperation;
executeOperation = () => { return(GetBlocksAsync(blobClient_, container, detailedOperation.BlobName, detailedOperation.StartIndex, detailedOperation.LengthBytes, detailedOperation.ChunkSizeBytes, levelOfConcurrency)); };
}
// Perform the task and report corresponding metrics.
DateTimeOffset startTime = DateTimeOffset.UtcNow;
StatusMsg statusMessage = null;
try
{
TimeSpan duration = await executeOperation();
statusMessage = new StatusMsg(operation.ID, true, "NA", startTime, duration);
}
catch (Exception ex)
{
statusMessage = new StatusMsg(operation.ID, false, ex.Message, startTime, DateTimeOffset.UtcNow - startTime);
}
finally
{
DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(StatusMsg));
MemoryStream memoryStream = new MemoryStream();
serializer.WriteObject(memoryStream, statusMessage);
string messageStr = System.Text.Encoding.Default.GetString(memoryStream.GetBuffer());
messageStr = messageStr.Substring(0, (int)memoryStream.Length);
await statusQueue_.AddMessageAsync(new CloudQueueMessage(messageStr));
}
}
else
{
Console.WriteLine($"Operation for {message.GetType().FullName} currently unsupported.");
}
}
}
public override async Task Run(string[] args)
{
// Parse the arguments.
if (!ParseArguments(args, out uint blockSizeBytes, out uint totalBlocks, out uint numInstances, out string blobName, out string containerName))
{
// If invalid arguments were provided, exit the test.
return;
}
CloudBlobClient blobClient = StorageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
await container.CreateIfNotExistsAsync();
uint startingBlockId = 0;
uint numBlocksPerPass = totalBlocks / numInstances;
uint remainder = totalBlocks % numInstances;
Task[] tasks = new Task[numInstances];
HashSet <Guid> operationIDs = new HashSet <Guid>();
// Assign the Work
for (uint i = 0; i < numInstances; ++i)
{
uint numBlocks = numBlocksPerPass;
// Evenly distribute remaining blocks across instances such that no instance differs by more than 1 block.
if (remainder > 0)
{
numBlocks++;
remainder--;
}
Guid opId = Guid.NewGuid();
operationIDs.Add(opId);
PutBlobOperation operation = new PutBlobOperation(opId, blobName, containerName, startingBlockId, blockSizeBytes, numBlocks);
TestMsg putBlobMsg = new TestMsg(Environment.MachineName, operation);
tasks[i] = JobQueue.AddMessageAsync(CreateJobQueueMessage(putBlobMsg));
// Update the starting position.
startingBlockId += numBlocks;
}
await Task.WhenAll(tasks);
// Report timing status (NOTE: omits time taken to execute PutBlockList).
await ReportStatus(operationIDs, ((long)totalBlocks) *blockSizeBytes);
// Commit the blob.
CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName);
try
{
await FinalizeBlob(container, blobName, totalBlocks, false);
Console.WriteLine("Successfully committed the block list.");
}
catch (Exception ex)
{
Console.WriteLine($"[ERROR] Could not commit the block list. Details: {ex.Message}");
}
}
