protected static CloudQueueMessage CreateJobQueueMessage(TestMsg message) { DataContractJsonSerializer serializer = new DataContractJsonSerializer(message.GetType()); MemoryStream memoryStream = new MemoryStream(); serializer.WriteObject(memoryStream, message); return(new CloudQueueMessage(System.Text.Encoding.Default.GetString(memoryStream.GetBuffer()))); }
public override async Task Run(string[] args) { // Parse the arguments. if (!ParseArguments(args, out uint chunkSizeBytes, out uint numInstances, out string blobName, out string containerName)) { // If invalid arguments were provided, exit the test. return; } // Determine the total size of the chosen blob. long startingIndex = 0; long blobSizeBytes = await GetBlobSize(blobName, containerName); long totalChunks = blobSizeBytes / chunkSizeBytes; long numChunksPerPass = totalChunks / numInstances; uint remainingChunks = (uint)(numChunksPerPass % numInstances); long remainingBytes = blobSizeBytes % chunkSizeBytes; Console.WriteLine($"Downloading {containerName}/{blobName} ({blobSizeBytes} bytes)."); Task[] tasks = new Task[numInstances]; HashSet <Guid> operationIDs = new HashSet <Guid>(); for (uint i = 0; i < numInstances; ++i) { long numChunks = numChunksPerPass; // Evenly distribute remaining chunks across instances such that no instance differs by more than 1 chunk. if (remainingChunks > 0) { numChunks++; remainingChunks--; } Guid opId = Guid.NewGuid(); operationIDs.Add(opId); long length = numChunks * chunkSizeBytes; if (i == numInstances - 1) { // Add any remaining bytes ( < 1 chunk) to the last worker (fair since remaining chunks are distributed to the 1st workers). length += remainingBytes; } GetBlobOperation operation = new GetBlobOperation(opId, blobName, containerName, startingIndex, length, chunkSizeBytes); TestMsg putBlobMsg = new TestMsg(Environment.MachineName, operation); tasks[i] = JobQueue.AddMessageAsync(CreateJobQueueMessage(putBlobMsg)); // Update the starting position. startingIndex += chunkSizeBytes * numChunks; } await Task.WhenAll(tasks); await ReportStatus(operationIDs, blobSizeBytes); }
public async Task Start(string[] args) { // Parse the arguments. if (!ParseArguments(args, out uint levelOfConcurrency)) { // If invalid arguments were provided, exit the role. return; } // Wait for a message to appear on the job queue. while (true) { Console.WriteLine("Awaiting a job to perform."); CloudQueueMessage rawMessage = await jobQueue_.GetMessageAsync(); while (rawMessage == null) { await Task.Delay(TimeSpan.FromSeconds(1)); rawMessage = await jobQueue_.GetMessageAsync(); } TestMsg message = null; try { message = ConvertQueueMessage(rawMessage); } catch (Exception ex) { // Upon failure to deserialize, log and throw the invalid message away. Console.WriteLine($"[ERROR] Failed to deserialize message. Details: {ex.Message}"); continue; } finally { await jobQueue_.DeleteMessageAsync(rawMessage); } Console.WriteLine("Message received."); CloudBlobContainer container = blobClient_.GetContainerReference(message.Operation.ContainerName); CloudBlockBlob blob = container.GetBlockBlobReference(message.Operation.BlobName); TestOperation operation = message.Operation; if (operation is PutBlobOperation || operation is GetBlobOperation) { // Define the task to be performed. Func <Task <TimeSpan> > executeOperation = null; if (operation is PutBlobOperation) { var detailedOperation = operation as PutBlobOperation; executeOperation = () => { return(UploadBlocksAsync(blobClient_, container, detailedOperation.BlobName, detailedOperation.BlockSizeBytes, detailedOperation.NumBlocks, detailedOperation.StartingBlockId, levelOfConcurrency)); }; } else if (operation is GetBlobOperation) { var detailedOperation = operation as GetBlobOperation; executeOperation = () => { return(GetBlocksAsync(blobClient_, container, detailedOperation.BlobName, detailedOperation.StartIndex, detailedOperation.LengthBytes, detailedOperation.ChunkSizeBytes, levelOfConcurrency)); }; } // Perform the task and report corresponding metrics. DateTimeOffset startTime = DateTimeOffset.UtcNow; StatusMsg statusMessage = null; try { TimeSpan duration = await executeOperation(); statusMessage = new StatusMsg(operation.ID, true, "NA", startTime, duration); } catch (Exception ex) { statusMessage = new StatusMsg(operation.ID, false, ex.Message, startTime, DateTimeOffset.UtcNow - startTime); } finally { DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(StatusMsg)); MemoryStream memoryStream = new MemoryStream(); serializer.WriteObject(memoryStream, statusMessage); string messageStr = System.Text.Encoding.Default.GetString(memoryStream.GetBuffer()); messageStr = messageStr.Substring(0, (int)memoryStream.Length); await statusQueue_.AddMessageAsync(new CloudQueueMessage(messageStr)); } } else { Console.WriteLine($"Operation for {message.GetType().FullName} currently unsupported."); } } }
public override async Task Run(string[] args) { // Parse the arguments. if (!ParseArguments(args, out uint blockSizeBytes, out uint totalBlocks, out uint numInstances, out string blobName, out string containerName)) { // If invalid arguments were provided, exit the test. return; } CloudBlobClient blobClient = StorageAccount.CreateCloudBlobClient(); CloudBlobContainer container = blobClient.GetContainerReference(containerName); await container.CreateIfNotExistsAsync(); uint startingBlockId = 0; uint numBlocksPerPass = totalBlocks / numInstances; uint remainder = totalBlocks % numInstances; Task[] tasks = new Task[numInstances]; HashSet <Guid> operationIDs = new HashSet <Guid>(); // Assign the Work for (uint i = 0; i < numInstances; ++i) { uint numBlocks = numBlocksPerPass; // Evenly distribute remaining blocks across instances such that no instance differs by more than 1 block. if (remainder > 0) { numBlocks++; remainder--; } Guid opId = Guid.NewGuid(); operationIDs.Add(opId); PutBlobOperation operation = new PutBlobOperation(opId, blobName, containerName, startingBlockId, blockSizeBytes, numBlocks); TestMsg putBlobMsg = new TestMsg(Environment.MachineName, operation); tasks[i] = JobQueue.AddMessageAsync(CreateJobQueueMessage(putBlobMsg)); // Update the starting position. startingBlockId += numBlocks; } await Task.WhenAll(tasks); // Report timing status (NOTE: omits time taken to execute PutBlockList). await ReportStatus(operationIDs, ((long)totalBlocks) *blockSizeBytes); // Commit the blob. CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName); try { await FinalizeBlob(container, blobName, totalBlocks, false); Console.WriteLine("Successfully committed the block list."); } catch (Exception ex) { Console.WriteLine($"[ERROR] Could not commit the block list. Details: {ex.Message}"); } }