/// <summary>
/// Prints task information to the console for each of the nodes in the specified pool.
/// </summary>
/// <param name="poolId">The ID of the <see cref="CloudPool"/> containing the nodes whose task information should be printed to the console.</param>
/// <returns>A <see cref="System.Threading.Tasks.Task"/> object that represents the asynchronous operation.</returns>
public static async Task PrintNodeTasksAsync(BatchClient batchClient, string poolId)
{
Console.WriteLine("Listing Node Tasks");
Console.WriteLine("==================");
ODATADetailLevel nodeDetail = new ODATADetailLevel(selectClause: "id,recentTasks");
IPagedEnumerable<ComputeNode> nodes = batchClient.PoolOperations.ListComputeNodes(poolId, nodeDetail);
await nodes.ForEachAsync(node =>
{
Console.WriteLine();
Console.WriteLine(node.Id + " tasks:");
if (node.RecentTasks != null && node.RecentTasks.Any())
{
foreach (TaskInformation task in node.RecentTasks)
{
Console.WriteLine("\t{0}: {1}", task.TaskId, task.TaskState);
}
}
else
{
// No tasks found for the node
Console.WriteLine("\tNone");
}
}).ConfigureAwait(continueOnCapturedContext: false);
Console.WriteLine("==================");
}
private static void DeleteJob(BatchClient client)
{
client.JobOperations.DeleteJob("testjob1");
Console.WriteLine("Job was deleted.");
Console.WriteLine("Press Enter to continue.");
Console.ReadLine();
}
/// <summary>
/// Waits for all tasks under the specified job to complete and then prints each task's output to the console.
/// </summary>
/// <param name="batchClient">The BatchClient to use when interacting with the Batch service.</param>
/// <param name="tasks">The tasks to wait for.</param>
/// <param name="timeout">The timeout. After this time has elapsed if the job is not complete and exception will be thrown.</param>
/// <returns>An asynchronous <see cref="Task"/> representing the operation.</returns>
public static async Task WaitForTasksAndPrintOutputAsync(BatchClient batchClient, IEnumerable<CloudTask> tasks, TimeSpan timeout)
{
// We use the task state monitor to monitor the state of our tasks -- in this case we will wait for them all to complete.
TaskStateMonitor taskStateMonitor = batchClient.Utilities.CreateTaskStateMonitor();
// Wait until the tasks are in completed state.
List<CloudTask> ourTasks = tasks.ToList();
bool timedOut = await taskStateMonitor.WaitAllAsync(ourTasks, TaskState.Completed, timeout);
if (timedOut)
{
throw new TimeoutException("Timed out waiting for tasks");
}
// dump task output
foreach (CloudTask t in ourTasks)
{
Console.WriteLine("Task {0}", t.Id);
//Read the standard out of the task
NodeFile standardOutFile = await t.GetNodeFileAsync(Constants.StandardOutFileName);
string standardOutText = await standardOutFile.ReadAsStringAsync();
Console.WriteLine("Standard out:");
Console.WriteLine(standardOutText);
//Read the standard error of the task
NodeFile standardErrorFile = await t.GetNodeFileAsync(Constants.StandardErrorFileName);
string standardErrorText = await standardErrorFile.ReadAsStringAsync();
Console.WriteLine("Standard error:");
Console.WriteLine(standardErrorText);
Console.WriteLine();
}
}
/// <summary>
/// Asynchronous method that delays execution until the specified pool reaches the specified state.
/// </summary>
/// <param name="client">A fully intitialized <see cref="BatchClient"/>.</param>
/// <param name="poolId">The ID of the pool to monitor for the specified <see cref="AllocationState"/>.</param>
/// <param name="targetAllocationState">The allocation state to monitor.</param>
/// <param name="timeout">The maximum time to wait for the pool to reach the specified state.</param>
/// <returns>A <see cref="System.Threading.Tasks.Task"/> object that represents the asynchronous operation.</returns>
public static async Task WaitForPoolToReachStateAsync(BatchClient client, string poolId, AllocationState targetAllocationState, TimeSpan timeout)
{
Console.WriteLine("Waiting for pool {0} to reach allocation state {1}", poolId, targetAllocationState);
DateTime startTime = DateTime.UtcNow;
DateTime timeoutAfterThisTimeUtc = startTime.Add(timeout);
ODATADetailLevel detail = new ODATADetailLevel(selectClause: "id,allocationState");
CloudPool pool = await client.PoolOperations.GetPoolAsync(poolId, detail);
while (pool.AllocationState != targetAllocationState)
{
Console.Write(".");
await Task.Delay(TimeSpan.FromSeconds(10));
await pool.RefreshAsync(detail);
if (DateTime.UtcNow > timeoutAfterThisTimeUtc)
{
Console.WriteLine();
Console.WriteLine("Timed out waiting for pool {0} to reach state {1}", poolId, targetAllocationState);
return;
}
}
Console.WriteLine();
}
static void DeletePool(BatchClient client)
{
client.PoolOperations.DeletePool("testpool1");
Console.WriteLine("Pool was deleted.");
Console.WriteLine("Press Enter to continue.");
Console.ReadLine();
}
public BatchService(BatchSharedKeyCredential credentials)
{
this.Client = BatchClient.Open(credentials);
this.Credentials = credentials;
this.retryPolicy = new LinearRetry(TimeSpan.FromSeconds(10), 5);
this.Client.CustomBehaviors.Add(new RetryPolicyProvider(this.retryPolicy));
}
private static void ListTasks(BatchClient client)
{
IPagedEnumerable<CloudTask> tasks = client.JobOperations.ListTasks("testjob1");
foreach (CloudTask task in tasks)
{
Console.WriteLine("Task id: " + task.Id);
Console.WriteLine(" Task status: " + task.State);
Console.WriteLine(" Task start: " + task.ExecutionInformation.StartTime);
}
Console.ReadLine();
}
private static void DeleteTasks(BatchClient client)
{
CloudJob job = client.JobOperations.GetJob("testjob1");
foreach (CloudTask task in job.ListTasks())
{
task.Delete();
}
Console.WriteLine("All tasks deleted.");
Console.WriteLine("Press Enter to continue.");
Console.ReadLine();
}
/// <summary>
/// Lists all the jobs in the Batch account.
/// </summary>
/// <param name="batchClient">The BatchClient to use when interacting with the Batch service.</param>
/// <returns>An asynchronous <see cref="Task"/> representing the operation.</returns>
public static async Task PrintJobsAsync(BatchClient batchClient)
{
Console.WriteLine("Listing Jobs");
Console.WriteLine("============");
IPagedEnumerable<CloudJob> jobs = batchClient.JobOperations.ListJobs(new ODATADetailLevel(selectClause: "id,state"));
await jobs.ForEachAsync(job =>
{
Console.WriteLine("State of job " + job.Id + " is " + job.State);
});
Console.WriteLine("============");
}
/// <summary>
/// Lists all the pools in the Batch account.
/// </summary>
/// <param name="batchClient">The BatchClient to use when interacting with the Batch service.</param>
/// <returns>An asynchronous <see cref="Task"/> representing the operation.</returns>
public static async Task PrintPoolsAsync(BatchClient batchClient)
{
Console.WriteLine("Listing Pools");
Console.WriteLine("=============");
// Using optional select clause to return only the properties of interest. Makes query faster and reduces HTTP packet size impact
IPagedEnumerable<CloudPool> pools = batchClient.PoolOperations.ListPools(new ODATADetailLevel(selectClause: "id,state,currentDedicated,vmSize"));
await pools.ForEachAsync(pool =>
{
Console.WriteLine("State of pool {0} is {1} and it has {2} nodes of size {3}", pool.Id, pool.State, pool.CurrentDedicated, pool.VirtualMachineSize);
});
Console.WriteLine("=============");
}
/// <summary>
/// Creates a CloudJob in the specified pool if a job with the specified ID is not found
/// in the pool, otherwise returns the existing job.
/// </summary>
/// <param name="batchClient">A fully initialized <see cref="BatchClient"/>.</param>
/// <param name="poolId">The ID of the CloudPool in which the job should be created.</param>
/// <param name="jobId">The ID of the CloudJob.</param>
/// <returns>A bound version of the newly created CloudJob.</returns>
public static async Task<CloudJob> CreateJobAsync(BatchClient batchClient, string poolId, string jobId)
{
CloudJob job = await SampleHelpers.GetJobIfExistAsync(batchClient, jobId);
if (job == null)
{
Console.WriteLine("Job {0} not found, creating...", jobId);
CloudJob unboundJob = batchClient.JobOperations.CreateJob(jobId, new PoolInformation() { PoolId = poolId });
await unboundJob.CommitAsync();
// Get the bound version of the job with all of its properties populated
job = await batchClient.JobOperations.GetJobAsync(jobId);
}
return job;
}
/// <summary>
/// Creates a <see cref="CloudPool"/> associated with the specified Batch account. If an existing pool with the
/// specified ID is found, the pool is resized to match the specified node count.
/// </summary>
/// <param name="batchClient">A fully initialized <see cref="BatchClient"/>.</param>
/// <param name="poolId">The ID of the <see cref="CloudPool"/>.</param>
/// <param name="nodeSize">The size of the nodes within the pool.</param>
/// <param name="nodeCount">The number of nodes to create within the pool.</param>
/// <param name="maxTasksPerNode">The maximum number of tasks to run concurrently on each node.</param>
/// <returns>A bound <see cref="CloudPool"/> with the specified properties.</returns>
public async static Task<CloudPool> CreatePoolIfNotExistAsync(BatchClient batchClient, string poolId, string nodeSize, int nodeCount, int maxTasksPerNode)
{
// Create and configure an unbound pool with the specified ID
CloudPool pool = batchClient.PoolOperations.CreatePool(poolId: poolId,
osFamily: "3",
virtualMachineSize: nodeSize,
targetDedicated: nodeCount);
pool.MaxTasksPerComputeNode = maxTasksPerNode;
// We want each node to be completely filled with tasks (i.e. up to maxTasksPerNode) before
// tasks are assigned to the next node in the pool
pool.TaskSchedulingPolicy = new TaskSchedulingPolicy(ComputeNodeFillType.Pack);
await GettingStartedCommon.CreatePoolIfNotExistAsync(batchClient, pool).ConfigureAwait(continueOnCapturedContext: false);
return await batchClient.PoolOperations.GetPoolAsync(poolId).ConfigureAwait(continueOnCapturedContext: false);
}
static void AddTasks(BatchClient client)
{
CloudJob job = client.JobOperations.GetJob("testjob1");
ResourceFile programFile = new ResourceFile(
"https://mystorage00.blob.core.windows.net/testcon1/ProcessTaskData.exe",
"ProcessTaskData.exe"
);
ResourceFile assemblyFile = new ResourceFile(
"https://mystorage00.blob.core.windows.net/testcon1/Microsoft.WindowsAzure.Storage.dll",
"Microsoft.WindowsAzure.Storage.dll"
);
for (int i = 1; i < 4; ++i)
{
string blobName = "taskdata" + i;
string taskName = "mytask" + i;
ResourceFile taskData = new ResourceFile("https://mystorage00.blob.core.windows.net/testcon1/" +
blobName, blobName);
CloudTask task = new CloudTask(
taskName,
"ProcessTaskData.exe https://mystorage00.blob.core.windows.net/testcon1/" +
blobName + " 3");
List<ResourceFile> taskFiles = new List<ResourceFile>();
taskFiles.Add(taskData);
taskFiles.Add(programFile);
taskFiles.Add(assemblyFile);
task.ResourceFiles = taskFiles;
job.AddTask(task);
job.Commit();
job.Refresh();
}
client.Utilities.CreateTaskStateMonitor().WaitAll(job.ListTasks(),
TaskState.Completed, new TimeSpan(0, 30, 0));
Console.WriteLine("The tasks completed successfully.");
foreach (CloudTask task in job.ListTasks())
{
Console.WriteLine("Task " + task.Id + " says:\n" + task.GetNodeFile(Constants.StandardOutFileName).ReadAsString());
}
Console.WriteLine("Press Enter to continue.");
Console.ReadLine();
}
private static void CreatePoolIfNeeded(BatchClient client, string poolId)
{
// go through all the pools and see if the specified pool already exists
bool found = false;
// use an OData based select clause to limit the amount of data returned. This will result in incomplete
// client objects but reduces the amount of data on the wire leading to faster completion when there are
// a lot of objects for a given query. No spaces are allowed in the string and property names are case-sensitive.
foreach (CloudPool p in client.PoolOperations.ListPools(new ODATADetailLevel(selectClause: "id,currentDedicated")))
{
// pools are uniquely identified by their ID
if (string.Equals(p.Id, poolId))
{
Console.WriteLine("Using existing pool {0}", poolId);
found = true;
if (p.CurrentDedicated == 0)
{
Console.WriteLine("There are no compute nodes in this pool. No tasks will be run until at least one node has been added via resizing.");
Console.WriteLine("Resizing pool to add 3 nodes. This might take a while...");
p.Resize(3);
}
break;
}
}
if (!found)
{
Console.WriteLine("Creating pool: {0}", poolId);
// if pool not found, call CreatePool. You can learn more about os families and versions at:
// https://azure.microsoft.com/en-us/documentation/articles/cloud-services-guestos-update-matrix/
CloudPool pool = client.PoolOperations.CreatePool(poolId, targetDedicated: 3, virtualMachineSize: "small", osFamily: "3");
pool.Commit();
}
}
private async Task<string> WaitForReducerTaskToCompleteAsync(BatchClient batchClient)
{
//Get the bound reducer task and monitor it for completion.
CloudTask boundReducerTask = await batchClient.JobOperations.GetTaskAsync(this.jobId, Constants.ReducerTaskId);
TaskStateMonitor taskStateMonitor = batchClient.Utilities.CreateTaskStateMonitor();
bool timedOut = await taskStateMonitor.WaitAllAsync(new List<CloudTask> { boundReducerTask }, TaskState.Completed, TimeSpan.FromMinutes(2));
//Refresh the reducer task to get the most recent information about it from the Batch Service.
await boundReducerTask.RefreshAsync();
//Dump the reducer tasks exit code and scheduling error for debugging purposes.
string stdOut = await Helpers.CheckForTaskSuccessAsync(boundReducerTask, dumpStandardOutOnTaskSuccess: true);
//Handle the possibilty that the reducer task did not complete in the expected timeout.
if (timedOut)
{
const string errorMessage = "Reducer task did not complete within expected timeout.";
Console.WriteLine("Task {0} is in state: {1}", boundReducerTask.Id, boundReducerTask.State);
Console.WriteLine(errorMessage);
throw new TimeoutException(errorMessage);
}
return stdOut;
}
public void LongRunning_Bug1965363Wat7OSVersionFeaturesQuickJobWithAutoPool()
{
Action test = () =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment()))
{
string jobId = "Bug1965363Job-" + TestUtilities.GetMyName();
try
{
PoolInformation poolInfo = new PoolInformation()
{
AutoPoolSpecification = new AutoPoolSpecification()
{
PoolLifetimeOption = PoolLifetimeOption.Job,
PoolSpecification = new PoolSpecification()
{
CloudServiceConfiguration = new CloudServiceConfiguration(PoolFixture.OSFamily),
VirtualMachineSize = PoolFixture.VMSize,
TargetDedicatedComputeNodes = 1
}
}
};
CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, poolInfo);
this.testOutputHelper.WriteLine("Commiting quickjob");
unboundJob.Commit();
CloudTask task = new CloudTask("Bug1965363Wat7OSVersionFeaturesQuickJobWithAutoPoolTask", "cmd /c echo Bug1965363");
CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);
boundJob.AddTask(task);
this.testOutputHelper.WriteLine("Getting pool name: {0}", boundJob.ExecutionInformation.PoolId);
CloudPool boundPool = batchCli.PoolOperations.GetPool(boundJob.ExecutionInformation.PoolId);
TaskStateMonitor tsm = batchCli.Utilities.CreateTaskStateMonitor();
ODATAMonitorControl odControl = new ODATAMonitorControl();
// we know that the autopool compute nodes will take a long time to become scheduleable so we slow down polling/spam
odControl.DelayBetweenDataFetch = TimeSpan.FromSeconds(5);
this.testOutputHelper.WriteLine("Invoking TaskStateMonitor");
tsm.WaitAll(
boundJob.ListTasks(),
TaskState.Completed,
TimeSpan.FromMinutes(15),
odControl,
new[] {
// spam/logging interceptor
new Microsoft.Azure.Batch.Protocol.RequestInterceptor((x) =>
{
this.testOutputHelper.WriteLine("Issuing request type: " + x.GetType().ToString());
// print out the compute node states... we are actually waiting on the compute nodes
List <ComputeNode> allComputeNodes = boundPool.ListComputeNodes().ToList();
this.testOutputHelper.WriteLine(" #comnpute nodes: " + allComputeNodes.Count);
allComputeNodes.ForEach((icn) => { this.testOutputHelper.WriteLine(" computeNode.id: " + icn.Id + ", state: " + icn.State); });
this.testOutputHelper.WriteLine("");
})
});
// confirm the task ran by inspecting the stdOut
string stdOut = boundJob.ListTasks().ToList()[0].GetNodeFile(Constants.StandardOutFileName).ReadAsString();
Assert.Contains("Bug1965363", stdOut);
}
finally
{
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
}
}
};
SynchronizationContextHelper.RunTest(test, LongTestTimeout);
}
private async Task WaitForMapperTasksToCompleteAsync(BatchClient batchClient)
{
Console.WriteLine("Waiting for the mapper tasks to complete...");
//List all the mapper tasks using an id filter.
DetailLevel mapperTaskIdFilter = new ODATADetailLevel()
{
FilterClause = string.Format("startswith(id, '{0}')", Constants.MapperTaskPrefix)
};
IEnumerable<CloudTask> tasksToMonitor = batchClient.JobOperations.ListTasks(
this.jobId,
detailLevel: mapperTaskIdFilter);
// Use the task state monitor to wait for the tasks to complete. Monitoring the tasks
// for completion is necessary if you are using KillJobOnCompletion = TRUE, as otherwise when the job manager
// exits it will kill all of the tasks that are still running under the job.
TaskStateMonitor taskStateMonitor = batchClient.Utilities.CreateTaskStateMonitor();
bool timedOut = await taskStateMonitor.WaitAllAsync(tasksToMonitor, TaskState.Completed, TimeSpan.FromMinutes(5));
//Get the list of mapper tasks in order to analyze their state and ensure they completed successfully.
IPagedEnumerable<CloudTask> asyncEnumerable = batchClient.JobOperations.ListTasks(
this.jobId,
detailLevel: mapperTaskIdFilter);
await asyncEnumerable.ForEachAsync(async cloudTask =>
{
Console.WriteLine("Task {0} is in state: {1}", cloudTask.Id, cloudTask.State);
await Helpers.CheckForTaskSuccessAsync(cloudTask, dumpStandardOutOnTaskSuccess: false);
Console.WriteLine();
});
//If not all the tasks reached the desired state within the timeout then the job manager
//cannot continue.
if (timedOut)
{
const string errorMessage = "Mapper tasks did not complete within expected timeout.";
Console.WriteLine(errorMessage);
throw new TimeoutException(errorMessage);
}
}
public void TestExitConditionsAreBeingRoundTrippedCorrectly()
{
Action test = () =>
{
using (BatchClient client = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment()))
{
//Create a job
string jobId = Constants.DefaultConveniencePrefix + TestUtilities.GetMyName() + "-TestExitConditionsAreBeingRoundTrippedCorrectly";
string taskId = "task-id-1";
try
{
CloudJob boundJob = null;
{
// need a bound job/task for the tests so set one up
boundJob = CreateBoundJob(client, jobId, j => { j.OnTaskFailure = OnTaskFailure.PerformExitOptionsJobAction; });
CloudTask cloudTask = new CloudTask(taskId, "cmd /c exit 2");
cloudTask.ExitConditions = new ExitConditions
{
ExitCodes = new List <ExitCodeMapping> {
new ExitCodeMapping(1, new ExitOptions {
JobAction = JobAction.None
})
},
ExitCodeRanges = new List <ExitCodeRangeMapping>
{
new ExitCodeRangeMapping(2, 4, new ExitOptions {
JobAction = JobAction.Disable
})
},
PreProcessingError = new ExitOptions {
JobAction = JobAction.Terminate
},
FileUploadError = new ExitOptions {
JobAction = JobAction.Terminate
},
Default = new ExitOptions {
JobAction = JobAction.Terminate
},
};
boundJob.AddTask(cloudTask);
boundJob.Refresh();
Assert.Equal(OnTaskFailure.PerformExitOptionsJobAction, boundJob.OnTaskFailure);
CloudTask boundTask = client.JobOperations.GetTask(jobId, taskId);
Assert.Equal(JobAction.None, boundTask.ExitConditions.ExitCodes.First().ExitOptions.JobAction);
Assert.Equal(1, boundTask.ExitConditions.ExitCodes.First().Code);
var exitCodeRangeMappings = boundTask.ExitConditions.ExitCodeRanges;
Assert.Equal(2, exitCodeRangeMappings.First().Start);
Assert.Equal(4, exitCodeRangeMappings.First().End);
Assert.Equal(JobAction.Disable, exitCodeRangeMappings.First().ExitOptions.JobAction);
Assert.Equal(JobAction.Terminate, boundTask.ExitConditions.PreProcessingError.JobAction);
Assert.Equal(JobAction.Terminate, boundTask.ExitConditions.FileUploadError.JobAction);
Assert.Equal(JobAction.Terminate, boundTask.ExitConditions.Default.JobAction);
}
}
finally
{
TestUtilities.DeleteJobIfExistsAsync(client, jobId).Wait();
}
}
};
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
public async Task AutoScaleEvaluationIntervalTest()
{
await SynchronizationContextHelper.RunTestAsync(async() =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment(), addDefaultRetryPolicy: false))
{
const string poolASFormulaOrig = "$TargetDedicated = 0;";
TimeSpan evalInterval = TimeSpan.FromMinutes(6);
string poolId0 = "AutoScaleEvalInterval0-" + TestUtilities.GetMyName();
try
{
// create an empty pool with autoscale and an eval interval
CloudServiceConfiguration cloudServiceConfiguration = new CloudServiceConfiguration(PoolFixture.OSFamily);
CloudPool ubPool = batchCli.PoolOperations.CreatePool(
poolId0,
cloudServiceConfiguration: cloudServiceConfiguration,
virtualMachineSize: PoolFixture.VMSize);
ubPool.AutoScaleEnabled = true;
ubPool.AutoScaleEvaluationInterval = evalInterval;
ubPool.AutoScaleFormula = poolASFormulaOrig;
ubPool.Commit();
// confirm values are returned
CloudPool bndPool = batchCli.PoolOperations.GetPool(poolId0);
Assert.True(bndPool.AutoScaleEnabled.HasValue && bndPool.AutoScaleEnabled.Value);
Assert.Equal(evalInterval, bndPool.AutoScaleEvaluationInterval);
// change eval interval
TimeSpan newEvalInterval = evalInterval + TimeSpan.FromMinutes(1);
bndPool.EnableAutoScale(autoscaleEvaluationInterval: newEvalInterval);
int enableCallCounter = 1; // count these to validate server throttle
const int expectedEnableCallToFail = 2;
bndPool.Refresh();
Assert.True(bndPool.AutoScaleEnabled.HasValue && bndPool.AutoScaleEnabled.Value);
Assert.True(bndPool.AutoScaleEvaluationInterval.HasValue);
Assert.Equal(newEvalInterval, bndPool.AutoScaleEvaluationInterval.Value);
// check the interval floor assert
var batchException = TestUtilities.AssertThrows <BatchException>(
() => bndPool.EnableAutoScale(autoscaleEvaluationInterval: TimeSpan.FromMinutes(1)));
Assert.Equal(Microsoft.Azure.Batch.Common.BatchErrorCodeStrings.InvalidPropertyValue, batchException.RequestInformation.BatchError.Code);
// check for AutoScaleTooManyRequestsToEnable
try
{
// spam the server
for (int i = 0; i < 99; i++) // remember there was already one (1) call made above
{
enableCallCounter++; // one more call
bndPool.EnableAutoScale(autoscaleEvaluationInterval: newEvalInterval + TimeSpan.FromSeconds(i));
}
// server never pushed back on the spam. this is a bug
throw new Exception("AutoScaleEvaluationIntervalTest: unable to force AutoScaleTooManyRequestsToEnable");
}
catch (Exception ex)
{
TestUtilities.AssertIsBatchExceptionAndHasCorrectAzureErrorCode(ex, Microsoft.Azure.Batch.Common.BatchErrorCodeStrings.AutoScaleTooManyRequestsToEnable, this.testOutputHelper);
// if we get here the exception passed.
// confirm that the expected call fails
Assert.Equal(expectedEnableCallToFail, enableCallCounter);
}
}
finally
{
// cleanup
TestUtilities.DeletePoolIfExistsAsync(batchCli, poolId0).Wait();
}
}
},
TestTimeout);
}
public static void HelloWorld(
BatchClient batchCli,
ITestOutputHelper testOutputHelper,
CloudPool sharedPool,
out string jobId,
out string taskId,
bool deleteJob = true,
bool isLinux = false)
{
jobId = "HelloWorldJob-" + GetMyName() + "-" + GetTimeStamp();
try
{
// here we show how to use an unbound Job + Commit() to run a simple "Hello World" task
// get an empty unbound Job
CloudJob quickJob = batchCli.JobOperations.CreateJob();
quickJob.Id = jobId;
quickJob.PoolInformation = new PoolInformation()
{
PoolId = sharedPool.Id
};
// Commit Job
quickJob.Commit();
// get an empty unbound Task
taskId = "dwsHelloWorldTask";
const string winPaasHWTaskCmdLine = "cmd /c echo Hello World";
const string linuxIaasHWTaskCmdLine = "echo Hello World";
string winnerTaskCmdLine = isLinux ? linuxIaasHWTaskCmdLine : winPaasHWTaskCmdLine;
CloudTask hwTask = new CloudTask(id: taskId, commandline: winnerTaskCmdLine);
// Open the new Job as bound.
CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);
// add Task to Job
boundJob.AddTask(hwTask);
// wait for the task to complete
Utilities utilities = batchCli.Utilities;
TaskStateMonitor taskStateMonitor = utilities.CreateTaskStateMonitor();
taskStateMonitor.WaitAll(
boundJob.ListTasks(),
TaskState.Completed,
TimeSpan.FromMinutes(3));
CloudTask myCompletedTask = new List <CloudTask>(boundJob.ListTasks(null))[0];
string stdOut = myCompletedTask.GetNodeFile(Constants.StandardOutFileName).ReadAsString();
string stdErr = myCompletedTask.GetNodeFile(Constants.StandardErrorFileName).ReadAsString();
// confirm that stdout includes correct value
Assert.Contains("Hello World", stdOut);
testOutputHelper.WriteLine("StdOut: ");
testOutputHelper.WriteLine(stdOut);
testOutputHelper.WriteLine("StdErr: ");
testOutputHelper.WriteLine(stdErr);
}
finally
{
// delete the job to free the Pool compute nodes.
if (deleteJob)
{
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
}
}
}
private static void SetDeserializationSettings(BatchClient batchClient)
{
GetServiceClient(batchClient).DeserializationSettings.MissingMemberHandling = MissingMemberHandling.Error;
}
/// <summary>
/// Creates tasks to process each of the specified input files, and submits them to the
/// specified job for execution.
/// </summary>
/// <param name="batchClient">A <see cref="BatchClient"/>.</param>
/// <param name="jobId">The id of the job to which the tasks should be added.</param>
/// <param name="inputFiles">A collection of <see cref="ResourceFile"/> objects representing the input files to be
/// processed by the tasks executed on the compute nodes.</param>
/// <param name="outputContainerSasUrl">The shared access signature URL for the container within Azure Storage that
/// will receive the output files created by the tasks.</param>
/// <returns>A collection of the submitted tasks.</returns>
private static async Task <List <CloudTask> > AddTasksAsync(BatchClient batchClient, string jobId, List <ResourceFile> inputFiles, string outputContainerSasUrl, CloudStorageAccount linkedStorageAccount)
{
Console.WriteLine("Adding {0} tasks to job [{1}]...", inputFiles.Count, jobId);
// Create a collection to hold the tasks that we'll be adding to the job
List <CloudTask> tasks = new List <CloudTask>();
// Create each of the tasks. Because we copied the task application to the
// node's shared directory with the pool's StartTask, we can access it via
// the shared directory on whichever node each task will run.
foreach (ResourceFile inputFile in inputFiles)
{
string taskId = $"task-mapper-{inputFiles.IndexOf(inputFile)}";
string mapperTaskCommandLine = $"{BatchJobTaskMapperCommandline} {inputFile.FilePath}";
CloudTask mapper = new CloudTask(taskId, mapperTaskCommandLine)
{
OutputFiles = new List <OutputFile>
{
new OutputFile(filePattern: TaskMapOutputFilePattern,
destination: new OutputFileDestination(new OutputFileBlobContainerDestination(containerUrl: outputContainerSasUrl, path: $"{taskId}.txt")),
uploadOptions: new OutputFileUploadOptions(
uploadCondition: OutputFileUploadCondition.TaskCompletion))
},
ResourceFiles = new List <ResourceFile> {
inputFile
}
};
tasks.Add(mapper);
}
string reducerTaskId = "task-reducer";
string reducerTaskCommandLine = BatchJobTaskReducerCommandline;
CloudTask reducer = new CloudTask(reducerTaskId, reducerTaskCommandLine)
{
DependsOn = TaskDependencies.OnTasks(tasks),
OutputFiles = new List <OutputFile>
{
new OutputFile(filePattern: TaskReduceOutputFilePattern,
destination: new OutputFileDestination(new OutputFileBlobContainerDestination(containerUrl: outputContainerSasUrl, path: reducerTaskId)),
uploadOptions: new OutputFileUploadOptions(
uploadCondition: OutputFileUploadCondition.TaskCompletion))
}
};
var reducerInputFiles = new List <ResourceFile>();
// define the input files for the reducer task
foreach (var maptask in tasks)
{
foreach (var outFile in maptask.OutputFiles)
{
// ensure that the container SAS has the following rights: write|read|list - otherwise we can't download the files
var sasContainerUrl = outFile.Destination.Container.ContainerUrl;
var blobName = outFile.Destination.Container.Path;
var sasBlob = sasContainerUrl.Insert(sasContainerUrl.IndexOf('?'), $"/{blobName}");
reducerInputFiles.Add(new ResourceFile(sasBlob, blobName));
}
}
reducer.ResourceFiles = reducerInputFiles;
tasks.Add(reducer);
// Add the tasks as a collection opposed to a separate AddTask call for each. Bulk task submission
// helps to ensure efficient underlying API calls to the Batch service.
await batchClient.JobOperations.AddTaskAsync(jobId, tasks);
return(tasks);
}
/// <summary>
/// Creates a pool if it doesn't already exist. If the pool already exists, this method resizes it to meet the expected
/// targets specified in settings.
/// </summary>
/// <param name="batchClient">The BatchClient to use when interacting with the Batch service.</param>
/// <param name="cloudStorageAccount">The CloudStorageAccount to upload start task required files to.</param>
/// <returns>An asynchronous <see cref="Task"/> representing the operation.</returns>
private async Task CreatePoolIfNotExistAsync(BatchClient batchClient, CloudStorageAccount cloudStorageAccount)
{
// You can learn more about os families and versions at:
// https://azure.microsoft.com/en-us/documentation/articles/cloud-services-guestos-update-matrix/
CloudPool pool = batchClient.PoolOperations.CreatePool(
poolId: this.configurationSettings.PoolId,
targetDedicated: this.configurationSettings.PoolTargetNodeCount,
virtualMachineSize: this.configurationSettings.PoolNodeVirtualMachineSize,
osFamily: this.configurationSettings.PoolOSFamily);
// Create a new start task to facilitate pool-wide file management or installation.
// In this case, we just add a single dummy data file to the StartTask.
string localSampleFilePath = GettingStartedCommon.GenerateTemporaryFile("StartTask.txt", "hello from Batch PoolsAndResourceFiles sample!");
List<string> files = new List<string> { localSampleFilePath };
List<ResourceFile> resourceFiles = await SampleHelpers.UploadResourcesAndCreateResourceFileReferencesAsync(
cloudStorageAccount,
this.configurationSettings.BlobContainer,
files);
pool.StartTask = new StartTask()
{
CommandLine = "cmd /c dir",
ResourceFiles = resourceFiles
};
await GettingStartedCommon.CreatePoolIfNotExistAsync(batchClient, pool);
}
/// <summary>
/// Initializes a new instance of the <see cref="MetricMonitor"/> class.
/// </summary>
/// <param name="batchClient">The <see cref="BatchClient"/> to use for accessing the Azure Batch service.</param>
/// <param name="monitorInterval">The interval at which to update the metrics.</param>
public MetricMonitor(BatchClient batchClient, TimeSpan monitorInterval)
: this(batchClient, false, monitorInterval)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="MetricMonitor"/> class.
/// </summary>
/// <param name="baseUrl">The Batch service endpoint.</param>
/// <param name="accountName">The name of the Batch account.</param>
/// <param name="accountKey">The Base64 encoded account access key.</param>
/// <param name="monitorInterval">The interval at which to update the metrics.</param>
public MetricMonitor(string baseUrl, string accountName, string accountKey, TimeSpan monitorInterval)
: this(BatchClient.Open(new BatchSharedKeyCredentials(baseUrl, accountName, accountKey)), true, monitorInterval)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="MetricMonitor"/> class.
/// </summary>
/// <param name="batchClient">The <see cref="BatchClient"/> to use for accessing the Azure Batch service.</param>
public MetricMonitor(BatchClient batchClient)
: this(batchClient, false, DefaultMonitorInterval)
{
}
/// <summary>
/// Populates Azure Storage with the required files, and
/// submits the job to the Azure Batch service.
/// </summary>
public async Task RunAsync()
{
Console.WriteLine("Running with the following settings: ");
Console.WriteLine("-------------------------------------");
Console.WriteLine(this.jobManagerSettings.ToString());
Console.WriteLine(this.accountSettings.ToString());
// Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchSharedKeyCredentials credentials = new BatchSharedKeyCredentials(
this.accountSettings.BatchServiceUrl,
this.accountSettings.BatchAccountName,
this.accountSettings.BatchAccountKey);
CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(
new StorageCredentials(this.accountSettings.StorageAccountName,
this.accountSettings.StorageAccountKey),
this.accountSettings.StorageServiceUrl,
useHttps: true);
// Get an instance of the BatchClient for a given Azure Batch account.
using (BatchClient batchClient = await BatchClient.OpenAsync(credentials))
{
// add a retry policy. The built-in policies are No Retry (default), Linear Retry, and Exponential Retry
batchClient.CustomBehaviors.Add(RetryPolicyProvider.ExponentialRetryProvider(TimeSpan.FromSeconds(5), 3));
string jobId = null;
try
{
// Allocate a pool
await this.CreatePoolIfNotExistAsync(batchClient, cloudStorageAccount);
// Submit the job
jobId = GettingStartedCommon.CreateJobId("SimpleJob");
await this.SubmitJobAsync(batchClient, cloudStorageAccount, jobId);
// Print out the status of the pools/jobs under this account
await GettingStartedCommon.PrintJobsAsync(batchClient);
await GettingStartedCommon.PrintPoolsAsync(batchClient);
// Wait for the job manager to complete
CloudTask jobManagerTask = await batchClient.JobOperations.GetTaskAsync(jobId, JobManagerTaskId);
await GettingStartedCommon.WaitForTasksAndPrintOutputAsync(batchClient,
new List <CloudTask> {
jobManagerTask
}, TimeSpan.FromMinutes(10));
}
finally
{
// Delete Azure Batch resources
List <string> jobIdsToDelete = new List <string>();
List <string> poolIdsToDelete = new List <string>();
if (this.jobManagerSettings.ShouldDeleteJob)
{
jobIdsToDelete.Add(jobId);
}
if (this.jobManagerSettings.ShouldDeletePool)
{
poolIdsToDelete.Add(this.jobManagerSettings.PoolId);
}
await SampleHelpers.DeleteBatchResourcesAsync(batchClient, jobIdsToDelete, poolIdsToDelete);
}
}
}
public ActionResult JobFinished(int id, string key)
{
// Validate the key
if (key != ConfigurationManager.AppSettings["ResultsCallbackKey"])
{
return(HttpNotFound());
}
// Connect to Azure Batch
var credentials = new BatchSharedKeyCredentials(
ConfigurationManager.AppSettings["BatchAccountUrl"],
ConfigurationManager.AppSettings["BatchAccountName"],
ConfigurationManager.AppSettings["BatchAccountKey"]);
using (var batchClient = BatchClient.Open(credentials))
using (var con = new SqlConnection(ConfigurationManager.ConnectionStrings["Sql"].ConnectionString))
{
con.Open();
// Load the job details from the database
var job = Job.Load(con, id);
if (job == null)
{
return(HttpNotFound());
}
var result = "";
var success = false;
var startTime = (DateTime?)null;
var endTime = (DateTime?)null;
var emailBody = (string)null;
var attachments = new CloudBlob[0];
try
{
var jobName = $"autotune-job-{id}";
// Check if the Autotune job finished successfully
var autotune = batchClient.JobOperations.GetTask(jobName, "Autotune");
success = autotune.ExecutionInformation.ExitCode == 0;
startTime = autotune.ExecutionInformation.StartTime;
endTime = autotune.ExecutionInformation.EndTime;
// Connect to Azure Storage
var connectionString = ConfigurationManager.ConnectionStrings["Storage"].ConnectionString;
var storageAccount = CloudStorageAccount.Parse(connectionString);
var cloudBlobClient = storageAccount.CreateCloudBlobClient();
// Find the log file produced by Autotune
var container = cloudBlobClient.GetContainerReference(jobName);
if (success)
{
var blob = container.GetBlobReference("autotune_recommendations.log");
using (var stream = new MemoryStream())
using (var reader = new StreamReader(stream))
{
// Download the log file
blob.DownloadToStream(stream);
stream.Position = 0;
result = reader.ReadToEnd();
// Parse the results
var parsedResults = AutotuneResults.ParseResult(result, job);
emailBody = RenderViewToString(ControllerContext, "Success", parsedResults);
}
}
// Get the log files to attach to the email
attachments = container.ListBlobs()
.Select(b => new CloudBlockBlob(b.Uri, cloudBlobClient))
.Where(b => b.Name != "autotune_recommendations.log" && b.Name != "profile.json")
.ToArray();
}
catch (Exception ex)
{
result = ex.ToString();
success = false;
}
if (emailBody == null)
{
emailBody = RenderViewToString(ControllerContext, "Failure");
}
EmailResults(job.EmailResultsTo, emailBody, attachments);
// Update the details in the SQL database
using (var cmd = con.CreateCommand())
{
cmd.CommandText = "UPDATE Jobs SET ProcessingStarted = @ProcessingStarted, ProcessingCompleted = @ProcessingCompleted, Result = @Result, Failed = @Failed WHERE JobID = @Id";
cmd.Parameters.AddWithValue("@Id", id);
cmd.Parameters.AddWithValue("@ProcessingStarted", (object)startTime ?? DBNull.Value);
cmd.Parameters.AddWithValue("@ProcessingCompleted", (object)endTime ?? DBNull.Value);
cmd.Parameters.AddWithValue("@Result", result);
cmd.Parameters.AddWithValue("@Failed", !success);
cmd.ExecuteNonQuery();
}
}
return(Content(""));
}
public void TestJobUpdateWithAndWithoutPoolInfo()
{
Action test = () =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment()))
{
const string testName = "TestJobUpdateWithAndWithoutPoolInfo";
// Create a job
string jobId = testName + "_" + TestUtilities.GetMyName();
CloudJob unboundJob = batchCli.JobOperations.CreateJob();
unboundJob.Id = jobId;
// Use an auto pool with the job, since PoolInformation can't be updated otherwise.
PoolSpecification poolSpec = new PoolSpecification();
poolSpec.CloudServiceConfiguration = new CloudServiceConfiguration(PoolFixture.OSFamily, "*");
poolSpec.TargetDedicatedComputeNodes = 0;
poolSpec.VirtualMachineSize = PoolFixture.VMSize;
AutoPoolSpecification autoPoolSpec = new AutoPoolSpecification();
string autoPoolPrefix = "UpdPIAuto_" + TestUtilities.GetMyName();
autoPoolSpec.AutoPoolIdPrefix = autoPoolPrefix;
const bool originalKeepAlive = false;
autoPoolSpec.KeepAlive = originalKeepAlive;
autoPoolSpec.PoolLifetimeOption = PoolLifetimeOption.Job;
autoPoolSpec.PoolSpecification = poolSpec;
PoolInformation poolInfo = new PoolInformation();
poolInfo.AutoPoolSpecification = autoPoolSpec;
unboundJob.PoolInformation = poolInfo;
const int originalPriority = 0;
unboundJob.Priority = originalPriority;
List <MetadataItem> originalMetadata = new List <MetadataItem>();
originalMetadata.Add(new MetadataItem("meta1", "value1"));
originalMetadata.Add(new MetadataItem("meta2", "value2"));
unboundJob.Metadata = originalMetadata;
this.testOutputHelper.WriteLine("Creating job {0}", jobId);
unboundJob.Commit();
try
{
// Get bound job
CloudJob createdJob = batchCli.JobOperations.GetJob(jobId);
// Verify that we can update something besides PoolInformation without getting an error for not being in the Disabled state.
Assert.NotEqual(JobState.Disabled, createdJob.State);
int updatedPriority = originalPriority + 1;
List <MetadataItem> updatedMetadata = new List <MetadataItem>();
updatedMetadata.Add(new MetadataItem("updatedMeta1", "value1"));
createdJob.Priority = updatedPriority;
createdJob.Metadata = updatedMetadata;
this.testOutputHelper.WriteLine("Updating job {0} without altering PoolInformation", jobId);
createdJob.Commit();
// Verify update occurred
CloudJob updatedJob = batchCli.JobOperations.GetJob(jobId);
Assert.Equal(updatedPriority, updatedJob.Priority);
Assert.Equal(updatedJob.Metadata.Count, updatedJob.Priority);
Assert.Equal(updatedJob.Metadata[0].Name, updatedMetadata[0].Name);
Assert.Equal(updatedJob.Metadata[0].Value, updatedMetadata[0].Value);
// Verify that updating the PoolInformation works.
// PoolInformation can only be changed in the Disabled state.
this.testOutputHelper.WriteLine("Disabling job {0}", jobId);
updatedJob.Disable(DisableJobOption.Terminate);
while (updatedJob.State != JobState.Disabled)
{
Thread.Sleep(500);
updatedJob.Refresh();
}
Assert.Equal(JobState.Disabled, updatedJob.State);
bool updatedKeepAlive = !originalKeepAlive;
updatedJob.PoolInformation.AutoPoolSpecification.KeepAlive = updatedKeepAlive;
int updatedAgainPriority = updatedPriority + 1;
updatedJob.Priority = updatedAgainPriority;
this.testOutputHelper.WriteLine("Updating job {0} properties, including PoolInformation", jobId);
updatedJob.Commit();
CloudJob updatedPoolInfoJob = batchCli.JobOperations.GetJob(jobId);
Assert.Equal(updatedKeepAlive, updatedPoolInfoJob.PoolInformation.AutoPoolSpecification.KeepAlive);
Assert.Equal(updatedAgainPriority, updatedPoolInfoJob.Priority);
}
finally
{
this.testOutputHelper.WriteLine("Deleting job {0}", jobId);
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
// Explicitly delete auto pool
foreach (CloudPool pool in batchCli.PoolOperations.ListPools(new ODATADetailLevel(filterClause: string.Format("startswith(id,'{0}')", autoPoolPrefix))))
{
this.testOutputHelper.WriteLine("Deleting pool {0}", pool.Id);
TestUtilities.DeletePoolIfExistsAsync(batchCli, pool.Id).Wait();
}
}
}
};
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
/// <summary>
/// Creates a job in the specified pool.
/// </summary>
/// <param name="batchClient">A <see cref="BatchClient"/>.</param>
/// <param name="jobId">The id of the job to be created.</param>
/// <param name="poolId">The id of the <see cref="CloudPool"/> in which to create the job.</param>
/// <returns>A <see cref="System.Threading.Tasks.Task"/> object that represents the asynchronous operation.</returns>
private static async Task CreateJobAsync(BatchClient batchClient, string jobId, string poolId)
{
Console.WriteLine("Creating job [{0}]...", jobId);
CloudJob job = batchClient.JobOperations.CreateJob();
job.Id = jobId;
job.PoolInformation = new PoolInformation { PoolId = poolId };
await job.CommitAsync();
}
public static async Task <BatchClient> OpenBatchClientFromEnvironmentAsync()
{
BatchClient client = await OpenBatchClientAsync(GetCredentialsFromEnvironment());
return(client);
}
public void Bug1665834TaskStateMonitor()
{
Action test = () =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment()))
{
string jobId = "Bug1665834Job-" + TestUtilities.GetMyName();
try
{
CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, new PoolInformation());
unboundJob.PoolInformation.PoolId = this.poolFixture.PoolId;
unboundJob.Commit();
CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);
// add some noise tasks
for (int j = 0; j < 5; j++)
{
CloudTask unboundTaskQuick = new CloudTask((10 + j).ToString(), "cmd /c hostname");
boundJob.AddTask(unboundTaskQuick);
}
System.Threading.Thread.Sleep(5000);
// wait for fast tasks to complete
{
bool repeat = true;
while (repeat)
{
CloudPool boundPool = batchCli.PoolOperations.GetPool(this.poolFixture.PoolId);
repeat = false;
foreach (CloudTask curTask in boundJob.ListTasks())
{
if (curTask.State != Microsoft.Azure.Batch.Common.TaskState.Completed)
{
repeat = true;
this.testOutputHelper.WriteLine("Manual Wait Task Id: " + curTask.Id + ", state = " + curTask.State);
this.testOutputHelper.WriteLine(" poolstate: " + boundPool.State + ", currentdedicated: " + boundPool.CurrentDedicatedComputeNodes);
this.testOutputHelper.WriteLine(" compute nodes:");
foreach (ComputeNode curComputeNode in boundPool.ListComputeNodes())
{
this.testOutputHelper.WriteLine(" computeNode.Id: " + curComputeNode.Id + ", state: " + curComputeNode.State);
}
}
}
}
}
// add some longer running tasks
this.testOutputHelper.WriteLine("Adding longer running tasks");
for (int i = 0; i < 15; i++)
{
CloudTask unboundTask = new CloudTask(i.ToString() + "_a234567890a234567890a234567890a234567890a234567890a234567890", "cmd /c ping 127.0.0.1 -n 4");
boundJob.AddTask(unboundTask);
}
Utilities utilities = batchCli.Utilities;
TaskStateMonitor tsm = utilities.CreateTaskStateMonitor();
IPagedEnumerable <CloudTask> taskList = boundJob.ListTasks();
ODATAMonitorControl odmc = new ODATAMonitorControl();
// try to set really low delay
odmc.DelayBetweenDataFetch = new TimeSpan(0);
// confirm the floor is enforced
Assert.Equal(500, odmc.DelayBetweenDataFetch.Milliseconds);
this.testOutputHelper.WriteLine("Calling TaskStateMonitor.WaitAll(). This will take a while.");
TimeSpan timeToWait = TimeSpan.FromMinutes(5);
Task whenAll = tsm.WhenAll(taskList, Microsoft.Azure.Batch.Common.TaskState.Completed, timeToWait, controlParams: odmc);
//This could throw, if it does the test will fail, which is what we want
whenAll.Wait();
foreach (CloudTask curTask in boundJob.ListTasks())
{
Assert.Equal(TaskState.Completed, curTask.State);
}
}
finally
{
// cleanup
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
}
}
};
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
public static Microsoft.Azure.Batch.Protocol.BatchServiceClient GetServiceClient(BatchClient batchClient)
{
object protocolLayer = batchClient.GetType().GetProperty("ProtocolLayer", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(batchClient);
Microsoft.Azure.Batch.Protocol.BatchServiceClient protoClient = protocolLayer
.GetType()
.GetField("_client", BindingFlags.Instance | BindingFlags.NonPublic)
.GetValue(protocolLayer) as Microsoft.Azure.Batch.Protocol.BatchServiceClient;
return(protoClient);
}
/// <summary>
/// Provides an asynchronous version of the Main method, allowing for the awaiting of async method calls within.
/// </summary>
/// <returns>A <see cref="System.Threading.Tasks.Task"/> object that represents the asynchronous operation.</returns>
private static async Task MainAsync()
{
Console.WriteLine("Sample start: {0}", DateTime.Now);
Console.WriteLine();
Stopwatch timer = new Stopwatch();
timer.Start();
// Construct the Storage account connection string
string storageConnectionString =
$"DefaultEndpointsProtocol=https;AccountName={StorageAccountName};AccountKey={StorageAccountKey}";
// Retrieve the storage account
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString);
// Create the blob client, for use in obtaining references to blob storage containers
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Use the blob client to create the containers in Azure Storage if they don't yet exist
const string appContainerName = "application";
const string inputContainerName = "input";
const string outputContainerName = "output";
await CreateContainerIfNotExistAsync(blobClient, appContainerName);
await CreateContainerIfNotExistAsync(blobClient, inputContainerName);
await CreateContainerIfNotExistAsync(blobClient, outputContainerName);
// Paths to the executable and its dependencies that will be executed by the tasks
List <string> applicationFilePaths = new List <string>
{
// The DotNetTutorial project includes a project reference to TaskApplication, allowing us to
// determine the path of the task application binary dynamically
typeof(TaskApplication.Program).Assembly.Location,
"Microsoft.WindowsAzure.Storage.dll"
};
// The collection of data files that are to be processed by the tasks
List <string> inputFilePaths = new List <string>
{
//@"taskdata1.txt",
//@"taskdata2.txt",
//@"taskdata3.txt"
@"model1.zip",
@"model2.zip",
@"model3.zip",
@"model4.zip",
@"model5.zip",
@"model6.zip",
@"model7.zip",
@"model8.zip",
@"model9.zip",
@"model10.zip"
};
// Upload the application and its dependencies to Azure Storage. This is the application that will
// process the data files, and will be executed by each of the tasks on the compute nodes.
List <ResourceFile> applicationFiles = await UploadFilesToContainerAsync(blobClient, appContainerName, applicationFilePaths);
// Upload the data files. This is the data that will be processed by each of the tasks that are
// executed on the compute nodes within the pool.
List <ResourceFile> inputFiles = await UploadFilesToContainerAsync(blobClient, inputContainerName, inputFilePaths);
// Obtain a shared access signature that provides write access to the output container to which
// the tasks will upload their output.
string outputContainerSasUrl = GetContainerSasUrl(blobClient, outputContainerName, SharedAccessBlobPermissions.Write);
// Create a BatchClient. We'll now be interacting with the Batch service in addition to Storage
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(cred))
{
// Create the pool that will contain the compute nodes that will execute the tasks.
// The ResourceFile collection that we pass in is used for configuring the pool's StartTask
// which is executed each time a node first joins the pool (or is rebooted or reimaged).
await CreatePoolIfNotExistAsync(batchClient, PoolId, applicationFiles);
// Create the job that will run the tasks.
await CreateJobAsync(batchClient, JobId, PoolId);
// Add the tasks to the job. We need to supply a container shared access signature for the
// tasks so that they can upload their output to Azure Storage.
await AddTasksAsync(batchClient, JobId, inputFiles, outputContainerSasUrl);
// Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete
await MonitorTasks(batchClient, JobId, TimeSpan.FromMinutes(30));
// Download the task output files from the output Storage container to a local directory
await DownloadBlobsFromContainerAsync(blobClient, outputContainerName, Environment.GetEnvironmentVariable("TEMP"));
// Clean up Storage resources
await DeleteContainerAsync(blobClient, appContainerName);
await DeleteContainerAsync(blobClient, inputContainerName);
await DeleteContainerAsync(blobClient, outputContainerName);
// Print out some timing info
timer.Stop();
Console.WriteLine();
Console.WriteLine("Sample end: {0}", DateTime.Now);
Console.WriteLine("Elapsed time: {0}", timer.Elapsed);
// Clean up Batch resources (if the user so chooses)
Console.WriteLine();
Console.Write("Delete job? [yes] no: ");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
await batchClient.JobOperations.DeleteJobAsync(JobId);
}
Console.Write("Delete pool? [yes] no: ");
response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
await batchClient.PoolOperations.DeletePoolAsync(PoolId);
}
//List<string> outFiles = new List<string>(System.IO.Directory.GetFiles( Environment.GetEnvironmentVariable("TEMP"), "*_out.zip",SearchOption.TopDirectoryOnly)) ;
//foreach (string outFilePath in outFiles)
//{
// string outFile = System.IO.Path.GetFileName(outFilePath);
// string desPath = System.IO.Directory.GetCurrentDirectory() + outFile;
// System.IO.File.Move(outFile, desPath);
//}
}
}
public void TestSampleWithFilesAndPool()
{
void test()
{
StagingStorageAccount storageCreds = TestUtilities.GetStorageCredentialsFromEnvironment();
using BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment());
string jobId = "SampleWithFilesJob-" + TestUtilities.GetMyName();
try
{
CloudJob quickJob = batchCli.JobOperations.CreateJob();
quickJob.Id = jobId;
quickJob.PoolInformation = new PoolInformation()
{
PoolId = poolFixture.PoolId
};
quickJob.Commit();
CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);
CloudTask myTask = new CloudTask(id: "CountWordsTask", commandline: @"cmd /c dir /s .. & dir & wc localwords.txt");
// first we have local files that we want pushed to the compute node before the commandline is invoked
FileToStage wordsDotText = new FileToStage(Resources.LocalWordsDotText, storageCreds); // use "default" mapping to base name of local file
myTask.FilesToStage = new List <IFileStagingProvider> {
wordsDotText
};
// add the task to the job
var artifacts = boundJob.AddTask(myTask);
var specificArtifact = artifacts[typeof(FileToStage)];
SequentialFileStagingArtifact sfsa = specificArtifact as SequentialFileStagingArtifact;
Assert.NotNull(sfsa);
// add a million more tasks...
// test to ensure the task is read only
TestUtilities.AssertThrows <InvalidOperationException>(() => myTask.FilesToStage = new List <IFileStagingProvider>());
// Open the new Job as bound.
CloudPool boundPool = batchCli.PoolOperations.GetPool(boundJob.ExecutionInformation.PoolId);
// wait for the task to complete
Utilities utilities = batchCli.Utilities;
TaskStateMonitor taskStateMonitor = utilities.CreateTaskStateMonitor();
taskStateMonitor.WaitAll(
boundJob.ListTasks(),
Microsoft.Azure.Batch.Common.TaskState.Completed,
TimeSpan.FromMinutes(10),
controlParams: null,
additionalBehaviors:
new[]
{
// spam/logging interceptor
new RequestInterceptor((x) =>
{
testOutputHelper.WriteLine("Issuing request type: " + x.GetType().ToString());
try
{
// print out the compute node states... we are actually waiting on the compute nodes
List <ComputeNode> allComputeNodes = boundPool.ListComputeNodes().ToList();
testOutputHelper.WriteLine(" #compute nodes: " + allComputeNodes.Count);
allComputeNodes.ForEach(
(icn) =>
{
testOutputHelper.WriteLine(" computeNode.id: " + icn.Id + ", state: " + icn.State);
});
}
catch (Exception ex)
{
// there is a race between the pool-life-job and the end of the job.. and the ListComputeNodes above
Assert.True(false, "SampleWithFilesAndPool probably can ignore this if its pool not found: " + ex.ToString());
}
})
});
List <CloudTask> tasks = boundJob.ListTasks(null).ToList();
CloudTask myCompletedTask = tasks[0];
foreach (CloudTask curTask in tasks)
{
testOutputHelper.WriteLine("Task Id: " + curTask.Id + ", state: " + curTask.State);
}
boundPool.Refresh();
testOutputHelper.WriteLine("Pool Id: " + boundPool.Id + ", state: " + boundPool.State);
string stdOut = myCompletedTask.GetNodeFile(Constants.StandardOutFileName).ReadAsString();
string stdErr = myCompletedTask.GetNodeFile(Constants.StandardErrorFileName).ReadAsString();
testOutputHelper.WriteLine("StdOut: ");
testOutputHelper.WriteLine(stdOut);
testOutputHelper.WriteLine("StdErr: ");
testOutputHelper.WriteLine(stdErr);
testOutputHelper.WriteLine("Task Files:");
foreach (NodeFile curFile in myCompletedTask.ListNodeFiles(recursive: true))
{
testOutputHelper.WriteLine(" FilePath: " + curFile.Path);
}
// confirm the files are there
Assert.True(FoundFile("localwords.txt", myCompletedTask.ListNodeFiles(recursive: true)), "mising file: localwords.txt");
// test validation of StagingStorageAccount
TestUtilities.AssertThrows <ArgumentOutOfRangeException>(() => { new StagingStorageAccount(storageAccount: " ", storageAccountKey: "key", blobEndpoint: "blob"); });
TestUtilities.AssertThrows <ArgumentOutOfRangeException>(() => { new StagingStorageAccount(storageAccount: "account", storageAccountKey: " ", blobEndpoint: "blob"); });
TestUtilities.AssertThrows <ArgumentOutOfRangeException>(() => { new StagingStorageAccount(storageAccount: "account", storageAccountKey: "key", blobEndpoint: ""); });
if (null != sfsa)
{
// TODO: delete the container!
}
}
finally
{
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
}
}
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
/// <summary>
/// Monitors the specified tasks for completion and returns a value indicating whether all tasks completed successfully
/// within the timeout period.
/// </summary>
/// <param name="batchClient">A <see cref="BatchClient"/>.</param>
/// <param name="jobId">The id of the job containing the tasks that should be monitored.</param>
/// <param name="timeout">The period of time to wait for the tasks to reach the completed state.</param>
/// <returns><c>true</c> if all tasks in the specified job completed with an exit code of 0 within the specified timeout period, otherwise <c>false</c>.</returns>
private static async Task <bool> MonitorTasks(BatchClient batchClient, string jobId, TimeSpan timeout)
{
bool allTasksSuccessful = true;
const string successMessage = "All tasks reached state Completed.";
const string failureMessage = "One or more tasks failed to reach the Completed state within the timeout period.";
// Obtain the collection of tasks currently managed by the job. Note that we use a detail level to
// specify that only the "id" property of each task should be populated. Using a detail level for
// all list operations helps to lower response time from the Batch service.
ODATADetailLevel detail = new ODATADetailLevel(selectClause: "id");
List <CloudTask> tasks = await batchClient.JobOperations.ListTasks(JobId, detail).ToListAsync();
Console.WriteLine("Awaiting task completion, timeout in {0}...", timeout.ToString());
// We use a TaskStateMonitor to monitor the state of our tasks. In this case, we will wait for all tasks to
// reach the Completed state.
TaskStateMonitor taskStateMonitor = batchClient.Utilities.CreateTaskStateMonitor();
try
{
await taskStateMonitor.WhenAll(tasks, TaskState.Completed, timeout);
}
catch (TimeoutException)
{
await batchClient.JobOperations.TerminateJobAsync(jobId, failureMessage);
Console.WriteLine(failureMessage);
return(false);
}
await batchClient.JobOperations.TerminateJobAsync(jobId, successMessage);
// All tasks have reached the "Completed" state, however, this does not guarantee all tasks completed successfully.
// Here we further check each task's ExecutionInfo property to ensure that it did not encounter a scheduling error
// or return a non-zero exit code.
// Update the detail level to populate only the task id and executionInfo properties.
// We refresh the tasks below, and need only this information for each task.
detail.SelectClause = "id, executionInfo";
foreach (CloudTask task in tasks)
{
// Populate the task's properties with the latest info from the Batch service
await task.RefreshAsync(detail);
if (task.ExecutionInformation.Result == TaskExecutionResult.Failure)
{
// A task with failure information set indicates there was a problem with the task. It is important to note that
// the task's state can be "Completed," yet still have encountered a failure.
allTasksSuccessful = false;
Console.WriteLine("WARNING: Task [{0}] encountered a failure: {1}", task.Id, task.ExecutionInformation.FailureInformation.Message);
if (task.ExecutionInformation.ExitCode != 0)
{
// A non-zero exit code may indicate that the application executed by the task encountered an error
// during execution. As not every application returns non-zero on failure by default (e.g. robocopy),
// your implementation of error checking may differ from this example.
Console.WriteLine("WARNING: Task [{0}] returned a non-zero exit code - this may indicate task execution or completion failure.", task.Id);
}
}
}
if (allTasksSuccessful)
{
Console.WriteLine("Success! All tasks completed successfully within the specified timeout period.");
}
return(allTasksSuccessful);
}
/// <summary>
/// Monitors the specified tasks for completion and returns a value indicating whether all tasks completed successfully
/// within the timeout period.
/// </summary>
/// <param name="batchClient">A <see cref="BatchClient"/>.</param>
/// <param name="jobId">The id of the job containing the tasks that should be monitored.</param>
/// <param name="timeout">The period of time to wait for the tasks to reach the completed state.</param>
/// <returns><c>true</c> if all tasks in the specified job completed with an exit code of 0 within the specified timeout period, otherwise <c>false</c>.</returns>
private static async Task<bool> MonitorTasks(BatchClient batchClient, string jobId, TimeSpan timeout)
{
bool allTasksSuccessful = true;
const string successMessage = "All tasks reached state Completed.";
const string failureMessage = "One or more tasks failed to reach the Completed state within the timeout period.";
// Obtain the collection of tasks currently managed by the job. Note that we use a detail level to
// specify that only the "id" property of each task should be populated. Using a detail level for
// all list operations helps to lower response time from the Batch service.
ODATADetailLevel detail = new ODATADetailLevel(selectClause: "id");
List<CloudTask> tasks = await batchClient.JobOperations.ListTasks(JobId, detail).ToListAsync();
Console.WriteLine("Awaiting task completion, timeout in {0}...", timeout.ToString());
// We use a TaskStateMonitor to monitor the state of our tasks. In this case, we will wait for all tasks to
// reach the Completed state.
TaskStateMonitor taskStateMonitor = batchClient.Utilities.CreateTaskStateMonitor();
bool timedOut = await taskStateMonitor.WhenAllAsync(tasks, TaskState.Completed, timeout);
if (timedOut)
{
allTasksSuccessful = false;
await batchClient.JobOperations.TerminateJobAsync(jobId, failureMessage);
Console.WriteLine(failureMessage);
}
else
{
await batchClient.JobOperations.TerminateJobAsync(jobId, successMessage);
// All tasks have reached the "Completed" state, however, this does not guarantee all tasks completed successfully.
// Here we further check each task's ExecutionInfo property to ensure that it did not encounter a scheduling error
// or return a non-zero exit code.
// Update the detail level to populate only the task id and executionInfo properties.
// We refresh the tasks below, and need only this information for each task.
detail.SelectClause = "id, executionInfo";
foreach (CloudTask task in tasks)
{
// Populate the task's properties with the latest info from the Batch service
await task.RefreshAsync(detail);
if (task.ExecutionInformation.SchedulingError != null)
{
// A scheduling error indicates a problem starting the task on the node. It is important to note that
// the task's state can be "Completed," yet still have encountered a scheduling error.
allTasksSuccessful = false;
Console.WriteLine("WARNING: Task [{0}] encountered a scheduling error: {1}", task.Id, task.ExecutionInformation.SchedulingError.Message);
}
else if (task.ExecutionInformation.ExitCode != 0)
{
// A non-zero exit code may indicate that the application executed by the task encountered an error
// during execution. As not every application returns non-zero on failure by default (e.g. robocopy),
// your implementation of error checking may differ from this example.
allTasksSuccessful = false;
Console.WriteLine("WARNING: Task [{0}] returned a non-zero exit code - this may indicate task execution or completion failure.", task.Id);
}
}
}
if (allTasksSuccessful)
{
Console.WriteLine("Success! All tasks completed successfully within the specified timeout period.");
}
return allTasksSuccessful;
}
/// <summary>
/// Creates a pool if it doesn't already exist. If the pool already exists, this method resizes it to meet the expected
/// targets specified in settings.
/// </summary>
/// <param name="batchClient">The BatchClient to create the pool with.</param>
/// <param name="pool">The pool to create.</param>
/// <returns>An asynchronous <see cref="Task"/> representing the operation.</returns>
public static async Task <CreatePoolResult> CreatePoolIfNotExistAsync(BatchClient batchClient, CloudPool pool)
{
bool successfullyCreatedPool = false;
int targetDedicatedNodeCount = pool.TargetDedicatedComputeNodes ?? 0;
int targetLowPriorityNodeCount = pool.TargetLowPriorityComputeNodes ?? 0;
string poolNodeVirtualMachineSize = pool.VirtualMachineSize;
string poolId = pool.Id;
// Attempt to create the pool
try
{
// Create an in-memory representation of the Batch pool which we would like to create. We are free to modify/update
// this pool object in memory until we commit it to the service via the CommitAsync method.
Console.WriteLine("Attempting to create pool: {0}", pool.Id);
// Create the pool on the Batch Service
await pool.CommitAsync().ConfigureAwait(continueOnCapturedContext: false);
successfullyCreatedPool = true;
Console.WriteLine("Created pool {0} with {1} dedicated and {2} low priority {3} nodes",
poolId,
targetDedicatedNodeCount,
targetLowPriorityNodeCount,
poolNodeVirtualMachineSize);
}
catch (BatchException e)
{
// Swallow the specific error code PoolExists since that is expected if the pool already exists
if (e.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists)
{
// The pool already existed when we tried to create it
successfullyCreatedPool = false;
Console.WriteLine("The pool already existed when we tried to create it");
}
else
{
throw; // Any other exception is unexpected
}
}
// If the pool already existed, make sure that its targets are correct
if (!successfullyCreatedPool)
{
CloudPool existingPool = await batchClient.PoolOperations.GetPoolAsync(poolId).ConfigureAwait(continueOnCapturedContext: false);
// If the pool doesn't have the right number of nodes, isn't resizing, and doesn't have
// automatic scaling enabled, then we need to ask it to resize
if ((existingPool.CurrentDedicatedComputeNodes != targetDedicatedNodeCount || existingPool.CurrentLowPriorityComputeNodes != targetLowPriorityNodeCount) &&
existingPool.AllocationState != AllocationState.Resizing &&
existingPool.AutoScaleEnabled == false)
{
// Resize the pool to the desired target. Note that provisioning the nodes in the pool may take some time
await existingPool.ResizeAsync(targetDedicatedNodeCount, targetLowPriorityNodeCount).ConfigureAwait(continueOnCapturedContext: false);
return(CreatePoolResult.ResizedExisting);
}
else
{
return(CreatePoolResult.PoolExisted);
}
}
return(CreatePoolResult.CreatedNew);
}
/// <summary>
/// Queries and prints task information for the specified job.
/// </summary>
/// <param name="batchClient">A fully initialized <see cref="BatchClient"/>.</param>
/// <param name="jobId">The ID of the job whose tasks should be queried.</param>
/// <param name="detail">An <see cref="ODATADetailLevel"/> configured with one or more of expand, filter, select clauses.</param>
private static void QueryTasks(BatchClient batchClient, string jobId, ODATADetailLevel detail)
{
List<CloudTask> taskList = new List<CloudTask>();
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
try
{
taskList.AddRange(batchClient.JobOperations.ListTasks(jobId, detail).ToList());
}
catch (AggregateException ex)
{
AggregateException ax = ex.Flatten();
Console.WriteLine(ax.Message);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
finally
{
stopwatch.Stop();
}
Console.WriteLine("{0} tasks retrieved in {1} (ExpandClause: {2} | FilterClause: {3} | SelectClause: {4})",
taskList.Count,
stopwatch.Elapsed,
detail.ExpandClause,
detail.FilterClause,
detail.SelectClause);
}
/// <summary>
/// Submits a set of tasks to the job
/// </summary>
/// <param name="batchClient">The batch client to use.</param>
/// <returns>The set of blob artifacts created by file staging.</returns>
private async Task<HashSet<string>> SubmitTasks(BatchClient batchClient)
{
List<CloudTask> tasksToRun = new List<CloudTask>();
// Create a task which requires some resource files
CloudTask taskWithFiles = new CloudTask("task_with_file1", SimpleTaskExe);
// Set up a collection of files to be staged -- these files will be uploaded to Azure Storage
// when the tasks are submitted to the Azure Batch service.
taskWithFiles.FilesToStage = new List<IFileStagingProvider>();
// generate a local file in temp directory
string localSampleFilePath = GettingStartedCommon.GenerateTemporaryFile("HelloWorld.txt", "hello from Batch JobManager sample!");
StagingStorageAccount fileStagingStorageAccount = new StagingStorageAccount(
storageAccount: this.configurationSettings.StorageAccountName,
storageAccountKey: this.configurationSettings.StorageAccountKey,
blobEndpoint: this.configurationSettings.StorageBlobEndpoint);
// add the files as a task dependency so they will be uploaded to storage before the task
// is submitted and downloaded to the node before the task starts execution.
FileToStage helloWorldFile = new FileToStage(localSampleFilePath, fileStagingStorageAccount);
FileToStage simpleTaskFile = new FileToStage(SimpleTaskExe, fileStagingStorageAccount);
// When this task is added via JobOperations.AddTaskAsync below, the FilesToStage are uploaded to storage once.
// The Batch service does not automatically delete content from your storage account, so files added in this
// way must be manually removed when they are no longer used.
taskWithFiles.FilesToStage.Add(helloWorldFile);
taskWithFiles.FilesToStage.Add(simpleTaskFile);
tasksToRun.Add(taskWithFiles);
var fileStagingArtifacts = new ConcurrentBag<ConcurrentDictionary<Type, IFileStagingArtifact>>();
// Use the AddTask method which takes an enumerable of tasks for best performance, as it submits up to 100
// tasks at once in a single request. If the list of tasks is N where N > 100, this will correctly parallelize
// the requests and return when all N tasks have been added.
await batchClient.JobOperations.AddTaskAsync(jobId, tasksToRun, fileStagingArtifacts: fileStagingArtifacts);
// Extract the names of the blob containers from the file staging artifacts
HashSet<string> blobContainerNames = GettingStartedCommon.ExtractBlobContainerNames(fileStagingArtifacts);
return blobContainerNames;
}
/// <summary>
/// Runs the job manager task.
/// </summary>
public async Task RunAsync()
{
Console.WriteLine("JobManager for account: {0}, job: {1} has started...",
this.accountName,
this.jobId);
Console.WriteLine();
Console.WriteLine("JobManager running with the following settings: ");
Console.WriteLine("----------------------------------------");
Console.WriteLine(this.textSearchSettings.ToString());
//Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchSharedKeyCredentials batchSharedKeyCredentials = new BatchSharedKeyCredentials(
this.accountSettings.BatchServiceUrl,
this.accountSettings.BatchAccountName,
this.accountSettings.BatchAccountKey);
CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(
new StorageCredentials(
this.accountSettings.StorageAccountName,
this.accountSettings.StorageAccountKey),
this.accountSettings.StorageServiceUrl,
useHttps: true);
using (BatchClient batchClient = await BatchClient.OpenAsync(batchSharedKeyCredentials))
{
//Construct a container SAS to provide the Batch Service access to the files required to
//run the mapper and reducer tasks.
string containerSas = SampleHelpers.ConstructContainerSas(
cloudStorageAccount,
this.textSearchSettings.BlobContainer);
//
// Submit mapper tasks.
//
await this.SubmitMapperTasksAsync(batchClient, containerSas);
//
// Wait for the mapper tasks to complete.
//
await this.WaitForMapperTasksToCompleteAsync(batchClient);
//
// Create the reducer task.
//
await this.SubmitReducerTaskAsync(batchClient, containerSas);
//
// Wait for the reducer task to complete.
//
string textToUpload = await this.WaitForReducerTaskToCompleteAsync(batchClient);
//
// Upload the results of the reducer task to Azure storage for consumption later
//
await SampleHelpers.UploadBlobTextAsync(cloudStorageAccount, this.textSearchSettings.BlobContainer, Constants.ReducerTaskResultBlobName, textToUpload);
//The job manager has completed.
Console.WriteLine("JobManager completed successfully.");
}
}
/// <summary>
/// Provides an asynchronous version of the Main method, allowing for the awaiting of async method calls within.
/// </summary>
/// <returns>A <see cref="System.Threading.Tasks.Task"/> object that represents the asynchronous operation.</returns>
private static async Task MainAsync()
{
Console.WriteLine("Sample start: {0}", DateTime.Now);
Console.WriteLine();
Stopwatch timer = new Stopwatch();
timer.Start();
// Retrieve the storage account
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(StorageConnectionString);
// Create the blob client, for use in obtaining references to blob storage containers
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Use the blob client to create the containers in Azure Storage if they don't yet exist
await CreateContainerIfNotExistAsync(blobClient, StorageInputContainerName);
await CreateContainerIfNotExistAsync(blobClient, StorageOutputContainerName);
// Upload the data files. This is the data that will be processed by each of the tasks that are
// executed on the compute nodes within the pool.
List <ResourceFile> inputFiles = await UploadFilesToContainerAsync(blobClient, StorageInputContainerName, LocalInputDirectory);
// Obtain a shared access signature that provides write access to the output container to which
// the tasks will upload their output.
string outputContainerSasUrl = GetContainerSasUrl(blobClient, StorageOutputContainerName, SharedAccessBlobPermissions.Write | SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.List);
// Create a BatchClient. We'll now be interacting with the Batch service in addition to Storage
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(cred))
{
// Create the job that will run the tasks.
await CreateJobAsync(batchClient, BatchJobId, BatchPoolId, storageAccount);
// Add the tasks to the job. We need to supply a container shared access signature for the
// tasks so that they can upload their output to Azure Storage.
await AddTasksAsync(batchClient, BatchJobId, inputFiles, outputContainerSasUrl, storageAccount);
// Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete
await MonitorTasks(batchClient, BatchJobId, TimeSpan.FromMinutes(BatchJobTimeoutInMinutes));
// Download the task output files from the output Storage container to a local directory
//await DownloadBlobsFromContainerAsync(blobClient, outputContainerName, Environment.GetEnvironmentVariable("TEMP"));
await DownloadBlobsFromContainerAsync(blobClient, StorageOutputContainerName, LocalOutputDirectory);
// Clean up Storage resources
//await DeleteContainerAsync(blobClient, appContainerName);
await DeleteContainerAsync(blobClient, StorageInputContainerName);
await DeleteContainerAsync(blobClient, StorageOutputContainerName);
// Print out some timing info
timer.Stop();
Console.WriteLine();
Console.WriteLine("Sample end: {0}", DateTime.Now);
Console.WriteLine("Elapsed time: {0}", timer.Elapsed);
// Clean up Batch resources (if the user so chooses)
Console.WriteLine();
Console.Write("Delete job? [yes] no: ");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
await batchClient.JobOperations.DeleteJobAsync(BatchJobId);
}
}
}
/// <summary>
/// Returns an existing <see cref="CloudJob"/> if found in the Batch account.
/// </summary>
/// <param name="batchClient">A fully initialized <see cref="BatchClient"/>.</param>
/// <param name="jobId">The <see cref="CloudJob.Id"/> of the desired pool.</param>
/// <returns>A bound <see cref="CloudJob"/>, or <c>null</c> if the specified <see cref="CloudJob"/> does not exist.</returns>
public static async Task<CloudJob> GetJobIfExistAsync(BatchClient batchClient, string jobId)
{
Console.WriteLine("Checking for existing job {0}...", jobId);
// Construct a detail level with a filter clause that specifies the job ID so that only
// a single CloudJob is returned by the Batch service (if that job exists)
ODATADetailLevel detail = new ODATADetailLevel(filterClause: string.Format("id eq '{0}'", jobId));
List<CloudJob> jobs = await batchClient.JobOperations.ListJobs(detailLevel: detail).ToListAsync().ConfigureAwait(continueOnCapturedContext: false);
return jobs.FirstOrDefault();
}
static void Main(string[] args)
{
if (String.IsNullOrEmpty(BatchAccountName) ||
String.IsNullOrEmpty(BatchAccountKey) ||
String.IsNullOrEmpty(BatchAccountUrl) ||
String.IsNullOrEmpty(StorageAccountName) ||
String.IsNullOrEmpty(StorageAccountKey))
{
throw new InvalidOperationException("One or more account credential strings have not been populated. Please ensure that your Batch and Storage account credentials have been specified.");
}
try
{
Console.WriteLine("Sample start: {0}", DateTime.Now);
Console.WriteLine();
Stopwatch timer = new Stopwatch();
timer.Start();
// Construct the Storage account connection string
string storageConnectionString = String.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}",
StorageAccountName, StorageAccountKey);
// Retrieve the storage account
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString);
// Create the blob client, for use in obtaining references to blob storage containers
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Use the blob client to create the input container in Azure Storage
const string inputContainerName = "input";
CloudBlobContainer container = blobClient.GetContainerReference(inputContainerName);
container.CreateIfNotExists();
// The collection of data files that are to be processed by the tasks
List <string> inputFilePaths = new List <string>
{
@"..\..\taskdata0.txt",
@"..\..\taskdata1.txt",
@"..\..\taskdata2.txt"
};
// Upload the data files to Azure Storage. This is the data that will be processed by each of the tasks that are
// executed on the compute nodes within the pool.
List <ResourceFile> inputFiles = new List <ResourceFile>();
foreach (string filePath in inputFilePaths)
{
inputFiles.Add(UploadFileToContainer(blobClient, inputContainerName, filePath));
}
// Get a Batch client using account creds
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(cred))
{
// Create a Batch pool, VM configuration, Windows Server image
Console.WriteLine("Creating pool [{0}]...", PoolId);
ImageReference imageReference = new ImageReference(
publisher: "MicrosoftWindowsServer",
offer: "WindowsServer",
sku: "2012-R2-datacenter-smalldisk",
version: "latest");
VirtualMachineConfiguration virtualMachineConfiguration =
new VirtualMachineConfiguration(
imageReference: imageReference,
nodeAgentSkuId: "batch.node.windows amd64");
try
{
CloudPool pool = batchClient.PoolOperations.CreatePool(
poolId: PoolId,
targetDedicatedComputeNodes: PoolNodeCount,
virtualMachineSize: PoolVMSize,
virtualMachineConfiguration: virtualMachineConfiguration);
pool.Commit();
}
catch (BatchException be)
{
// Accept the specific error code PoolExists as that is expected if the pool already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists)
{
Console.WriteLine("The pool {0} already existed when we tried to create it", PoolId);
}
else
{
throw; // Any other exception is unexpected
}
}
// Create a Batch job
Console.WriteLine("Creating job [{0}]...", JobId);
try
{
CloudJob job = batchClient.JobOperations.CreateJob();
job.Id = JobId;
job.PoolInformation = new PoolInformation {
PoolId = PoolId
};
job.Commit();
}
catch (BatchException be)
{
// Accept the specific error code JobExists as that is expected if the job already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists)
{
Console.WriteLine("The job {0} already existed when we tried to create it", JobId);
}
else
{
throw; // Any other exception is unexpected
}
}
// Create a collection to hold the tasks that we'll be adding to the job
Console.WriteLine("Adding {0} tasks to job [{1}]...", inputFiles.Count, JobId);
List <CloudTask> tasks = new List <CloudTask>();
// Create each of the tasks to process one of the input files.
for (int i = 0; i < inputFiles.Count; i++)
{
string taskId = String.Format("Task{0}", i);
string inputFilename = inputFiles[i].FilePath;
string taskCommandLine = String.Format("cmd /c type {0}", inputFilename);
CloudTask task = new CloudTask(taskId, taskCommandLine);
task.ResourceFiles = new List <ResourceFile> {
inputFiles[i]
};
tasks.Add(task);
}
// Add all tasks to the job.
batchClient.JobOperations.AddTask(JobId, tasks);
// Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete.
TimeSpan timeout = TimeSpan.FromMinutes(30);
Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout);
IEnumerable <CloudTask> addedTasks = batchClient.JobOperations.ListTasks(JobId);
batchClient.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout);
Console.WriteLine("All tasks reached state Completed.");
// Print task output
Console.WriteLine();
Console.WriteLine("Printing task output...");
IEnumerable <CloudTask> completedtasks = batchClient.JobOperations.ListTasks(JobId);
foreach (CloudTask task in completedtasks)
{
string nodeId = String.Format(task.ComputeNodeInformation.ComputeNodeId);
Console.WriteLine("Task: {0}", task.Id);
Console.WriteLine("Node: {0}", nodeId);
Console.WriteLine("Standard out:");
Console.WriteLine(task.GetNodeFile(Constants.StandardOutFileName).ReadAsString());
}
// Print out some timing info
timer.Stop();
Console.WriteLine();
Console.WriteLine("Sample end: {0}", DateTime.Now);
Console.WriteLine("Elapsed time: {0}", timer.Elapsed);
// Clean up Storage resources
if (container.DeleteIfExists())
{
Console.WriteLine("Container [{0}] deleted.", inputContainerName);
}
else
{
Console.WriteLine("Container [{0}] does not exist, skipping deletion.", inputContainerName);
}
// Clean up Batch resources (if the user so chooses)
Console.WriteLine();
Console.Write("Delete job? [yes] no: ");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.JobOperations.DeleteJob(JobId);
}
Console.Write("Delete pool? [yes] no: ");
response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.PoolOperations.DeletePool(PoolId);
}
}
}
finally
{
Console.WriteLine();
Console.WriteLine("Sample complete, hit ENTER to exit...");
Console.ReadLine();
}
}
/// <summary>
/// Creates a <see cref="CloudPool"/> with the specified id and configures its StartTask with the
/// specified <see cref="ResourceFile"/> collection.
/// </summary>
/// <param name="batchClient">A <see cref="BatchClient"/>.</param>
/// <param name="poolId">The id of the <see cref="CloudPool"/> to create.</param>
/// <param name="resourceFiles">A collection of <see cref="ResourceFile"/> objects representing blobs within
/// a Storage account container. The StartTask will download these files from Storage prior to execution.</param>
/// <returns>A <see cref="System.Threading.Tasks.Task"/> object that represents the asynchronous operation.</returns>
private static async Task CreatePoolAsync(BatchClient batchClient, string poolId, IList<ResourceFile> resourceFiles)
{
Console.WriteLine("Creating pool [{0}]...", poolId);
// Create the unbound pool. Until we call CloudPool.Commit() or CommitAsync(), no pool is actually created in the
// Batch service. This CloudPool instance is therefore considered "unbound," and we can modify its properties.
CloudPool pool = batchClient.PoolOperations.CreatePool(
poolId: poolId,
targetDedicated: 3, // 3 compute nodes
virtualMachineSize: "small", // single-core, 1.75 GB memory, 225 GB disk
cloudServiceConfiguration: new CloudServiceConfiguration(osFamily: "4")); // Windows Server 2012 R2
// Create and assign the StartTask that will be executed when compute nodes join the pool.
// In this case, we copy the StartTask's resource files (that will be automatically downloaded
// to the node by the StartTask) into the shared directory that all tasks will have access to.
pool.StartTask = new StartTask
{
// Specify a command line for the StartTask that copies the task application files to the
// node's shared directory. Every compute node in a Batch pool is configured with a number
// of pre-defined environment variables that can be referenced by commands or applications
// run by tasks.
// Since a successful execution of robocopy can return a non-zero exit code (e.g. 1 when one or
// more files were successfully copied) we need to manually exit with a 0 for Batch to recognize
// StartTask execution success.
CommandLine = "cmd /c (robocopy %AZ_BATCH_TASK_WORKING_DIR% %AZ_BATCH_NODE_SHARED_DIR%) ^& IF %ERRORLEVEL% LEQ 1 exit 0",
ResourceFiles = resourceFiles,
WaitForSuccess = true
};
await pool.CommitAsync();
}
private async static Task TranscribeAsync(BatchClient client, Uri[] audioFiles, bool isContainerUrls)
{
Console.WriteLine("Deleting all existing completed transcriptions.");
// get all transcriptions for the subscription
PaginatedTranscriptions paginatedTranscriptions = null;
do
{
if (paginatedTranscriptions == null)
{
paginatedTranscriptions = await client.GetTranscriptionsAsync().ConfigureAwait(false);
}
else
{
paginatedTranscriptions = await client.GetTranscriptionsAsync(paginatedTranscriptions.NextLink).ConfigureAwait(false);
}
// delete all pre-existing completed transcriptions. If transcriptions are still running or not started, they will not be deleted
foreach (var transcriptionToDelete in paginatedTranscriptions.Values)
{
// delete a transcription
await client.DeleteTranscriptionAsync(transcriptionToDelete.Self).ConfigureAwait(false);
Console.WriteLine($"Deleted transcription {transcriptionToDelete.Self}");
}
}while (paginatedTranscriptions.NextLink != null);
// <transcriptiondefinition>
var newTranscription = new Transcription
{
DisplayName = DisplayName,
Locale = Locale,
ContentUrls = audioFiles,
//ContentContainerUrl = ContentAzureBlobContainer,
Model = CustomModel,
Properties = new TranscriptionProperties
{
DestinationContainerUrl = null,
IsWordLevelTimestampsEnabled = false,
TimeToLive = TimeSpan.FromDays(1)
}
};
if (isContainerUrls)
{
newTranscription.ContentUrls = null;
newTranscription.ContentContainerUrl = audioFiles[0];
newTranscription.Properties.DestinationContainerUrl = audioFiles[1];
}
newTranscription = await client.CreateTranscriptionAsync(newTranscription).ConfigureAwait(false);
Console.WriteLine($"Created transcription {newTranscription.Self}");
// </transcriptiondefinition>
// get the transcription Id from the location URI
var createdTranscriptions = new List <Uri> {
newTranscription.Self
};
Console.WriteLine("Checking status.");
// get the status of our transcriptions periodically and log results
int completed = 0, running = 0, notStarted = 0;
while (completed < 1)
{
completed = 0; running = 0; notStarted = 0;
// get all transcriptions for the user
paginatedTranscriptions = null;
do
{
// <transcriptionstatus>
if (paginatedTranscriptions == null)
{
paginatedTranscriptions = await client.GetTranscriptionsAsync().ConfigureAwait(false);
}
else
{
paginatedTranscriptions = await client.GetTranscriptionsAsync(paginatedTranscriptions.NextLink).ConfigureAwait(false);
}
// delete all pre-existing completed transcriptions. If transcriptions are still running or not started, they will not be deleted
foreach (var transcription in paginatedTranscriptions.Values)
{
switch (transcription.Status)
{
case "Failed":
case "Succeeded":
// we check to see if it was one of the transcriptions we created from this client.
if (!createdTranscriptions.Contains(transcription.Self))
{
// not created form here, continue
continue;
}
completed++;
// if the transcription was successful, check the results
if (transcription.Status == "Succeeded")
{
Console.WriteLine($"\t\tTranscirption {transcription.DisplayName} Succeeded.");
// BUGBUG: TODO: The destination container public access level is set to OFF.
// Hence the call to GetTranscriptionResultAsync(Uri) is failing with "resource not found" error.
// so disabling the code below.
// Alternatively, checking for isContainerUrls & use audioFiles[1] to get the container and change its public access level.
if (false)
{
var paginatedfiles = await client.GetTranscriptionFilesAsync(transcription.Links.Files).ConfigureAwait(false);
var resultFile = paginatedfiles.Values.FirstOrDefault(f => f.Kind == ArtifactKind.Transcription);
var result = await client.GetTranscriptionResultAsync(new Uri(resultFile.Links.ContentUrl)).ConfigureAwait(false);
Console.WriteLine("Transcription succeeded. Results: ");
Console.WriteLine(JsonConvert.SerializeObject(result, SpeechJsonContractResolver.WriterSettings));
}
}
else
{
Console.WriteLine("Transcription failed. Status: {0}", transcription.Properties.Error.Message);
}
break;
case "Running":
running++;
break;
case "NotStarted":
notStarted++;
break;
}
}
// for each transcription in the list we check the status
Console.WriteLine(string.Format("Transcriptions status: {0} completed, {1} running, {2} not started yet", completed, running, notStarted));
}while (paginatedTranscriptions.NextLink != null);
// </transcriptionstatus>
// check again after 1 minute
await Task.Delay(TimeSpan.FromMinutes(1)).ConfigureAwait(false);
}
Console.WriteLine("Press any key...");
Console.ReadKey();
}
/// <summary>
/// Creates tasks to process each of the specified input files, and submits them to the
/// specified job for execution.
/// </summary>
/// <param name="batchClient">A <see cref="BatchClient"/>.</param>
/// <param name="jobId">The id of the job to which the tasks should be added.</param>
/// <param name="inputFiles">A collection of <see cref="ResourceFile"/> objects representing the input files to be
/// processed by the tasks executed on the compute nodes.</param>
/// <param name="outputContainerSasUrl">The shared access signature URL for the container within Azure Storage that
/// will receive the output files created by the tasks.</param>
/// <returns>A collection of the submitted tasks.</returns>
private static async Task<List<CloudTask>> AddTasksAsync(BatchClient batchClient, string jobId, List<ResourceFile> inputFiles, string outputContainerSasUrl)
{
Console.WriteLine("Adding {0} tasks to job [{1}]...", inputFiles.Count, jobId);
// Create a collection to hold the tasks that we'll be adding to the job
List<CloudTask> tasks = new List<CloudTask>();
// Create each of the tasks. Because we copied the task application to the
// node's shared directory with the pool's StartTask, we can access it via
// the shared directory on whichever node each task will run.
foreach (ResourceFile inputFile in inputFiles)
{
string taskId = "topNtask" + inputFiles.IndexOf(inputFile);
string taskCommandLine = String.Format("cmd /c %AZ_BATCH_NODE_SHARED_DIR%\\TaskApplication.exe {0} 3 \"{1}\"", inputFile.FilePath, outputContainerSasUrl);
CloudTask task = new CloudTask(taskId, taskCommandLine);
task.ResourceFiles = new List<ResourceFile> { inputFile };
tasks.Add(task);
}
// Add the tasks as a collection opposed to a separate AddTask call for each. Bulk task submission
// helps to ensure efficient underlying API calls to the Batch service.
await batchClient.JobOperations.AddTaskAsync(jobId, tasks);
return tasks;
}
static async Task Main(string[] args)
{
// Show the user the batch the app is attaching to
Console.WriteLine("URL: {0}, Name: {1}, Key: {2}", batchAccountUrl, batchAccountName, batchAccountKey);
// Show the user the accounts they are attaching to
Console.WriteLine("BATCH URL: {0}, Name: {1}, Key: {2}", batchAccountUrl, batchAccountName, batchAccountKey);
Console.WriteLine("Storage Name: {0}, Key: {1}", storageAccountName, storageAccountKey);
// Construct the Storage account connection string
string storageConnectionString = String.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}",
storageAccountName, storageAccountKey);
// Retrieve the storage account
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString);
// Create the blob client, for use in obtaining references to blob storage containers
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Use the blob client to create the containers in blob storage
const string inputContainerName = "input";
const string outputContainerName = "output";
await CreateContainerIfNotExistAsync(blobClient, inputContainerName);
await CreateContainerIfNotExistAsync(blobClient, outputContainerName);
// RESOURCE FILE SETUP
// Add *.mp4 files into the \<solutiondir>\InputFiles folder.
string inputPath = Path.Combine(Environment.CurrentDirectory, "InputFiles");
List <string> inputFilePaths = new List <string>(
Directory.GetFileSystemEntries(inputPath, "*.mp4", SearchOption.TopDirectoryOnly));
// Upload data files.
// Upload the data files using UploadResourceFilesToContainer(). This data will be
// processed by each of the tasks that are executed on the compute nodes within the pool.
List <ResourceFile> inputFiles = await UploadFilesToContainerAsync(blobClient, inputContainerName, inputFilePaths);
// Obtain a shared access signature that provides write access to the output container to which
// the tasks will upload their output.
string outputContainerSasUrl = GetContainerSasUrl(blobClient, outputContainerName, SharedAccessBlobPermissions.Write);
// The batch client requires a BatchSharedKeyCredentials object to open a connection
var sharedKeyCredentials = new BatchSharedKeyCredentials(batchAccountUrl, batchAccountName, batchAccountKey);
using (BatchClient batchClient = BatchClient.Open(sharedKeyCredentials))
{
// The batchClient object can reference all the application packages, and get a summary of their details
foreach (var app in batchClient.ApplicationOperations.ListApplicationSummaries())
{
// For each application package, print to the console their name and version number
Console.WriteLine("Installed app: {0}({1})", app.Id, app.Versions[0]);
}
// Create the Batch pool, which contains the compute nodes that execute the tasks.
await CreateBatchPoolAsync(batchClient, PoolId);
// Create the job that runs the tasks.
await CreateJobAsync(batchClient, JobId, PoolId);
// Create a collection of tasks and add them to the Batch job.
await AddTasksAsync(batchClient, JobId, inputFiles, outputContainerSasUrl);
// Monitor task success or failure, specifying a maximum amount of time to wait for
// the tasks to complete.
await MonitorTasksAsync(batchClient, JobId, TimeSpan.FromMinutes(30));
// Delete input container in storage
Console.WriteLine("Deleting container [{0}]...", inputContainerName);
CloudBlobContainer container = blobClient.GetContainerReference(inputContainerName);
await container.DeleteIfExistsAsync();
// Clean up the job (if the user so chooses)
Console.WriteLine();
Console.Write("Delete job? [yes] no: ");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
await batchClient.JobOperations.DeleteJobAsync(JobId);
}
// Clean up the pool (if the user so chooses - do not delete the pool if new batches of videos are ready to process)
Console.Write("Delete pool? [yes] no: ");
response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
Console.WriteLine("Deleting pool ...");
await batchClient.PoolOperations.DeletePoolAsync(PoolId);
Console.WriteLine("Pool deleted.");
}
}
}
/// <summary>
/// Creates a CloudPool with a single compute node associated with the Batch account.
/// </summary>
/// <param name="batchClient">A fully initialized <see cref="BatchClient"/>.</param>
/// <param name="poolId">The ID of the <see cref="CloudPool"/>.</param>
private async static Task CreatePool(BatchClient batchClient, string poolId)
{
// Create and configure an unbound pool with the specified ID
CloudPool pool = batchClient.PoolOperations.CreatePool(poolId: poolId,
osFamily: "3",
virtualMachineSize: "small",
targetDedicated: 1);
await GettingStartedCommon.CreatePoolIfNotExistAsync(batchClient, pool);
}
private static async Task MainAsync(string[] args)
{
const string poolId = "JobPrepReleaseSamplePool";
const string jobId = "JobPrepReleaseSampleJob";
// Location of the file that the job tasks will work with, a text file in the
// node's "shared" directory.
const string taskOutputFile = "%AZ_BATCH_NODE_SHARED_DIR%\\job_prep_and_release.txt";
// The job prep task will write the node ID to the text file in the shared directory
const string jobPrepCmdLine = "cmd /c echo %AZ_BATCH_NODE_ID% tasks: >" + taskOutputFile;
// Each task then echoes its ID to the same text file
const string taskCmdLine = "cmd /c echo %AZ_BATCH_TASK_ID% >> " + taskOutputFile;
// The job release task will then delete the text file from the shared directory
const string jobReleaseCmdLine = "cmd /c del " + taskOutputFile;
// Configure your AccountSettings in the Microsoft.Azure.Batch.Samples.Common project within this solution
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(AccountSettings.Default.BatchServiceUrl,
AccountSettings.Default.BatchAccountName,
AccountSettings.Default.BatchAccountKey);
// Initialize the BatchClient for access to your Batch account
using (BatchClient batchClient = await BatchClient.OpenAsync(cred))
{
// Create a CloudPool (or obtain an existing pool with the specified ID)
CloudPool pool = await ArticleHelpers.CreatePoolIfNotExistAsync(batchClient,
poolId,
"small",
2,
1);
// Create a CloudJob (or obtain an existing job with the specified ID)
CloudJob job = await SampleHelpers.GetJobIfExistAsync(batchClient, jobId);
if (job == null)
{
Console.WriteLine("Job {0} not found, creating...", jobId);
CloudJob unboundJob = batchClient.JobOperations.CreateJob(jobId, new PoolInformation()
{
PoolId = poolId
});
// Configure and assign the job preparation task
unboundJob.JobPreparationTask = new JobPreparationTask {
CommandLine = jobPrepCmdLine
};
// Configure and assign the job release task
unboundJob.JobReleaseTask = new JobReleaseTask {
CommandLine = jobReleaseCmdLine
};
await unboundJob.CommitAsync();
// Get the bound version of the job with all of its properties populated
job = await batchClient.JobOperations.GetJobAsync(jobId);
}
// Create the tasks that the job will execute
List <CloudTask> tasks = new List <CloudTask>();
for (int i = 1; i <= 8; i++)
{
string taskId = "task" + i.ToString().PadLeft(3, '0');
string taskCommandLine = taskCmdLine;
CloudTask task = new CloudTask(taskId, taskCommandLine);
tasks.Add(task);
}
// Add the tasks in one API call as opposed to a separate AddTask call for each. Bulk task
// submission helps to ensure efficient underlying API calls to the Batch service.
Console.WriteLine("Submitting tasks and awaiting completion...");
await batchClient.JobOperations.AddTaskAsync(job.Id, tasks);
// Wait for the tasks to complete before proceeding. The long timeout here is to allow time
// for the nodes within the pool to be created and started if the pool had not yet been created.
await batchClient.Utilities.CreateTaskStateMonitor().WhenAll(
job.ListTasks(),
TaskState.Completed,
TimeSpan.FromMinutes(30));
Console.WriteLine("All tasks completed.");
Console.WriteLine();
// Print the contents of the shared text file modified by the job preparation and other tasks.
ODATADetailLevel nodeDetail = new ODATADetailLevel(selectClause: "id, state");
IPagedEnumerable <ComputeNode> nodes = batchClient.PoolOperations.ListComputeNodes(pool.Id, nodeDetail);
await nodes.ForEachAsync(async (node) =>
{
// Check to ensure that the node is Idle before attempting to pull the text file.
// If the pool was just created, there is a chance that another node completed all
// of the tasks prior to the other node(s) completing their startup procedure.
if (node.State == ComputeNodeState.Idle)
{
NodeFile sharedTextFile = await node.GetNodeFileAsync("shared\\job_prep_and_release.txt");
Console.WriteLine("Contents of {0} on {1}:", sharedTextFile.Path, node.Id);
Console.WriteLine("-------------------------------------------");
Console.WriteLine(await sharedTextFile.ReadAsStringAsync());
}
});
// Terminate the job to mark it as Completed; this will initiate the Job Release Task on any node
// that executed job tasks. Note that the Job Release Task is also executed when a job is deleted,
// thus you need not call Terminate if you typically delete your jobs upon task completion.
await batchClient.JobOperations.TerminateJobAsync(job.Id);
// Wait for the job to reach state "Completed." Note that this wait is not typically necessary in
// production code, but is done here to enable the checking of the release tasks exit code below.
await ArticleHelpers.WaitForJobToReachStateAsync(batchClient, job.Id, JobState.Completed, TimeSpan.FromMinutes(2));
// Print the exit codes of the prep and release tasks by obtaining their execution info
List <JobPreparationAndReleaseTaskExecutionInformation> prepReleaseInfo = await batchClient.JobOperations.ListJobPreparationAndReleaseTaskStatus(job.Id).ToListAsync();
foreach (JobPreparationAndReleaseTaskExecutionInformation info in prepReleaseInfo)
{
Console.WriteLine();
Console.WriteLine("{0}: ", info.ComputeNodeId);
// If no tasks were scheduled to run on the node, the JobPreparationTaskExecutionInformation will be null
if (info.JobPreparationTaskExecutionInformation != null)
{
Console.WriteLine(" Prep task exit code: {0}", info.JobPreparationTaskExecutionInformation.ExitCode);
}
// If no tasks were scheduled to run on the node, the JobReleaseTaskExecutionInformation will be null
if (info.JobReleaseTaskExecutionInformation != null)
{
Console.WriteLine(" Release task exit code: {0}", info.JobReleaseTaskExecutionInformation.ExitCode);
}
}
// Clean up the resources we've created in the Batch account
Console.WriteLine();
Console.WriteLine("Delete job? [yes] no");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
// Note that deleting the job will execute the job release task if the job was not previously terminated
await batchClient.JobOperations.DeleteJobAsync(job.Id);
}
Console.WriteLine("Delete pool? [yes] no");
response = Console.ReadLine();
if (response != "n" && response != "no")
{
await batchClient.PoolOperations.DeletePoolAsync(pool.Id);
}
}
}
/// <summary>
/// Prints task information to the console for each of the nodes in the specified pool.
/// </summary>
/// <param name="poolId">The ID of the <see cref="CloudPool"/> containing the nodes whose task information
/// should be printed to the console.</param>
private static void PrintNodeTasks(BatchClient batchClient, string poolId)
{
ODATADetailLevel nodeDetail = new ODATADetailLevel(selectClause: "id,recentTasks");
// Obtain and print the task information for each of the compute nodes in the pool.
foreach (ComputeNode node in batchClient.PoolOperations.ListComputeNodes(poolId, nodeDetail))
{
Console.WriteLine();
Console.WriteLine(node.Id + " tasks:");
if (node.RecentTasks != null && node.RecentTasks.Any())
{
foreach (TaskInformation task in node.RecentTasks)
{
Console.WriteLine("\t{0}: {1}", task.TaskId, task.TaskState);
}
}
else
{
// No tasks found for the node
Console.WriteLine("\tNone");
}
}
}
internal static async Task ProcessSucceededTranscriptionAsync(BatchClient client, PostTranscriptionServiceBusMessage serviceBusMessage, Guid transcriptionId, string jobName, ILogger log)
{
log.LogInformation($"Got succeeded transcription for job {jobName}");
var jsonContainer = FetchTranscriptionEnvironmentVariables.JsonResultOutputContainer;
var transcriptionFiles = await client.GetTranscriptionFilesAsync(transcriptionId).ConfigureAwait(false);
var resultFiles = transcriptionFiles.Values.Where(t => t.Kind == TranscriptionFileKind.Transcription);
var containsMultipleTranscriptions = resultFiles.Skip(1).Any();
var textAnalyticsKey = FetchTranscriptionEnvironmentVariables.TextAnalyticsKey;
var textAnalyticsRegion = FetchTranscriptionEnvironmentVariables.TextAnalyticsRegion;
var textAnalyticsInfoProvided = !string.IsNullOrEmpty(textAnalyticsKey) &&
!string.IsNullOrEmpty(textAnalyticsRegion) &&
!textAnalyticsRegion.Equals("none", StringComparison.OrdinalIgnoreCase);
var textAnalytics = textAnalyticsInfoProvided ? new TextAnalytics(serviceBusMessage.Locale, textAnalyticsKey, textAnalyticsRegion, log) : null;
var generalErrorsStringBuilder = new StringBuilder();
foreach (var resultFile in resultFiles)
{
var fileName = string.Empty;
try
{
var transcriptionResultJson = string.Empty;
using (var webClient = new WebClient())
{
transcriptionResultJson = webClient.DownloadString(resultFile.Links.ContentUrl);
}
var transcriptionResult = JsonConvert.DeserializeObject <SpeechTranscript>(transcriptionResultJson);
fileName = StorageUtilities.GetFileNameFromUri(new Uri(transcriptionResult.Source));
log.LogInformation($"Filename is {fileName}");
if (transcriptionResult.RecognizedPhrases == null || transcriptionResult.RecognizedPhrases.All(phrase => !phrase.RecognitionStatus.Equals("Success", StringComparison.Ordinal)))
{
continue;
}
var textAnalyticsErrors = new List <string>();
if (serviceBusMessage.AddSentimentAnalysis)
{
var sentimentErrors = await textAnalytics.AddSentimentToTranscriptAsync(transcriptionResult).ConfigureAwait(false);
textAnalyticsErrors.AddRange(sentimentErrors);
}
if (serviceBusMessage.AddEntityRedaction)
{
var entityRedactionErrors = await textAnalytics.RedactEntitiesAsync(transcriptionResult).ConfigureAwait(false);
textAnalyticsErrors.AddRange(entityRedactionErrors);
}
var editedTranscriptionResultJson = JsonConvert.SerializeObject(transcriptionResult, Newtonsoft.Json.Formatting.Indented);
// Store transcript json:
var jsonFileName = $"{fileName}.json";
await StorageUtilities.WriteTextFileToBlobAsync(FetchTranscriptionEnvironmentVariables.AzureWebJobsStorage, editedTranscriptionResultJson, jsonContainer, jsonFileName, log).ConfigureAwait(false);
if (FetchTranscriptionEnvironmentVariables.CreateHtmlResultFile)
{
var htmlContainer = FetchTranscriptionEnvironmentVariables.HtmlResultOutputContainer;
var htmlFileName = $"{fileName}.html";
var displayResults = TranscriptionToHtml.ToHTML(transcriptionResult, jobName);
await StorageUtilities.WriteTextFileToBlobAsync(FetchTranscriptionEnvironmentVariables.AzureWebJobsStorage, displayResults, htmlContainer, htmlFileName, log).ConfigureAwait(false);
}
if (textAnalyticsErrors.Any())
{
var distinctErrors = textAnalyticsErrors.Distinct();
var errorMessage = $"File {(string.IsNullOrEmpty(fileName) ? "unknown" : fileName)}:\n{string.Join('\n', distinctErrors)}";
generalErrorsStringBuilder.AppendLine(errorMessage);
}
if (FetchTranscriptionEnvironmentVariables.UseSqlDatabase)
{
var duration = XmlConvert.ToTimeSpan(transcriptionResult.Duration);
var approximatedCost = CostEstimation.GetCostEstimation(
duration,
transcriptionResult.CombinedRecognizedPhrases.Count(),
serviceBusMessage.UsesCustomModel,
serviceBusMessage.AddSentimentAnalysis,
serviceBusMessage.AddEntityRedaction);
var dbConnectionString = FetchTranscriptionEnvironmentVariables.DatabaseConnectionString;
using var dbConnector = new DatabaseConnector(log, dbConnectionString);
await dbConnector.StoreTranscriptionAsync(
containsMultipleTranscriptions?Guid.NewGuid() : transcriptionId,
serviceBusMessage.Locale,
string.IsNullOrEmpty(fileName)?jobName : fileName,
(float)approximatedCost,
transcriptionResult).ConfigureAwait(false);
}
}
catch (Exception e)
{
if (string.IsNullOrEmpty(fileName) && e is ArgumentNullException)
{
var errorMessage = $"Transcription file name is unknown, failed with message: {e.Message}";
log.LogError(errorMessage);
generalErrorsStringBuilder.AppendLine(errorMessage);
continue;
}
else if (e is JsonException || e is SqlException)
{
var errorTxtname = fileName + ".txt";
var errorMessage = $"Transcription result processing failed with exception: {e.Message}";
log.LogError(errorMessage);
await StorageUtilities.WriteTextFileToBlobAsync(
FetchTranscriptionEnvironmentVariables.AzureWebJobsStorage,
errorMessage,
FetchTranscriptionEnvironmentVariables.ErrorReportOutputContainer,
errorTxtname,
log).ConfigureAwait(false);
continue;
}
throw;
}
}
var errors = generalErrorsStringBuilder.ToString();
if (!string.IsNullOrEmpty(errors))
{
var errorTxtname = jobName + ".txt";
await StorageUtilities.WriteTextFileToBlobAsync(
FetchTranscriptionEnvironmentVariables.AzureWebJobsStorage,
errors,
FetchTranscriptionEnvironmentVariables.ErrorReportOutputContainer,
errorTxtname,
log).ConfigureAwait(false);
}
// Delete trace from service
client.DeleteTranscriptionAsync(transcriptionId).ConfigureAwait(false).GetAwaiter().GetResult();
}
public JobSubmitter(BatchClient batchClient)
{
this.batchClient = batchClient;
this.batchClient.CustomBehaviors.Add(RetryPolicyProvider.ExponentialRetryProvider(TimeSpan.FromSeconds(5), 3));
}
public static async Task <bool> GetTranscripts(PostTranscriptionServiceBusMessage serviceBusMessage, ILogger log)
{
if (serviceBusMessage == null)
{
throw new ArgumentNullException(nameof(serviceBusMessage));
}
var jobName = serviceBusMessage.JobName;
var transcriptionId = serviceBusMessage.TranscriptionLocation.Split('/').LastOrDefault();
var transcriptionGuid = new Guid(transcriptionId);
log.LogInformation($"Received transcription {transcriptionGuid} with name {jobName} from service bus message.");
var serviceBusConnectionString = FetchTranscriptionEnvironmentVariables.FetchTranscriptionServiceBusConnectionString;
var reenqueueingTimeInSeconds = serviceBusMessage.ReenqueueingTimeInSeconds;
log.LogInformation($"Re-enqueueing time for messages: {reenqueueingTimeInSeconds} seconds.");
log.LogInformation($"Subscription location: {serviceBusMessage.Subscription.LocationUri.AbsoluteUri}");
var client = new BatchClient(serviceBusMessage.Subscription.SubscriptionKey, serviceBusMessage.Subscription.LocationUri.AbsoluteUri, log);
try
{
var transcription = await client.GetTranscriptionAsync(transcriptionGuid).ConfigureAwait(false);
switch (transcription.Status)
{
case "Failed":
await ProcessFailedTranscriptionAsync(client, transcription, transcriptionGuid, jobName, log).ConfigureAwait(false);
break;
case "Succeeded":
await ProcessSucceededTranscriptionAsync(client, serviceBusMessage, transcriptionGuid, jobName, log).ConfigureAwait(false);
break;
case "Running":
var runningMessage = serviceBusMessage.RetryMessage();
log.LogInformation("Transcription running, retrying message - retry count: " + runningMessage.RetryCount);
ServiceBusUtilities.SendServiceBusMessageAsync(serviceBusConnectionString, runningMessage.CreateMessageString(), log, reenqueueingTimeInSeconds).GetAwaiter().GetResult();
break;
case "NotStarted":
var notStartedMessage = serviceBusMessage.RetryMessage();
var initialDelayInSeconds = serviceBusMessage.InitialDelayInSeconds;
// If the transcription is not started yet, the job will take at least length of the audio:
var notStartedReenqueueingTime = Math.Max(initialDelayInSeconds, reenqueueingTimeInSeconds);
log.LogInformation("Transcription not started, retrying message - retry count: " + notStartedMessage.RetryCount);
ServiceBusUtilities.SendServiceBusMessageAsync(serviceBusConnectionString, notStartedMessage.CreateMessageString(), log, notStartedReenqueueingTime).GetAwaiter().GetResult();
break;
}
}
catch (WebException e)
{
if (BatchClient.IsThrottledOrTimeoutStatusCode(((HttpWebResponse)e.Response).StatusCode))
{
var timeoutMessage = serviceBusMessage.RetryMessage();
log.LogInformation("Timeout or throttled, retrying message - retry count: " + timeoutMessage.RetryCount);
ServiceBusUtilities.SendServiceBusMessageAsync(serviceBusConnectionString, timeoutMessage.CreateMessageString(), log, reenqueueingTimeInSeconds).GetAwaiter().GetResult();
return(false);
}
throw;
}
catch (TimeoutException e)
{
var timeoutMessage = serviceBusMessage.RetryMessage();
log.LogInformation($"Timeout - re-enqueueing fetch transcription message. Exception message: {e.Message}");
ServiceBusUtilities.SendServiceBusMessageAsync(serviceBusConnectionString, timeoutMessage.CreateMessageString(), log, reenqueueingTimeInSeconds).GetAwaiter().GetResult();
return(false);
}
return(true);
}
/// <summary>
/// Deletes the pools and jobs specified.
/// </summary>
/// <param name="batchClient">The <see cref="BatchClient"/> to use to delete the pools and jobs</param>
/// <param name="jobIds">The job ids to delete.</param>
/// <param name="poolIds">The pool ids to delete.</param>
/// <returns>An asynchronous <see cref="Task"/> representing the operation.</returns>
public static async Task DeleteBatchResourcesAsync(BatchClient batchClient, List<string> jobIds, List<string> poolIds)
{
// Delete the jobs
foreach (string jobId in jobIds)
{
Console.WriteLine("Deleting job: {0}", jobId);
await batchClient.JobOperations.DeleteJobAsync(jobId);
}
foreach (string poolId in poolIds)
{
Console.WriteLine("Deleting pool: {0}", poolId);
await batchClient.PoolOperations.DeletePoolAsync(poolId);
}
}
private static async Task ProcessFailedTranscriptionAsync(string transcriptionLocation, string subscriptionKey, TranscriptionStartedMessage serviceBusMessage, Transcription transcription, string jobName, ILogger log)
{
var safeErrorCode = transcription?.Properties?.Error?.Code ?? "unknown";
var safeErrorMessage = transcription?.Properties?.Error?.Message ?? "unknown";
var logMessage = $"Got failed transcription for job {jobName} with error {safeErrorMessage} (Error code: {safeErrorCode}).";
log.LogInformation(logMessage);
var transcriptionFiles = await BatchClient.GetTranscriptionFilesAsync(transcriptionLocation, subscriptionKey, log).ConfigureAwait(false);
var errorReportOutput = logMessage;
var reportFile = transcriptionFiles.Values.Where(t => t.Kind == TranscriptionFileKind.TranscriptionReport).FirstOrDefault();
if (reportFile?.Links?.ContentUrl != null)
{
var reportFileContent = await BatchClient.GetTranscriptionReportFileFromSasAsync(reportFile.Links.ContentUrl, log).ConfigureAwait(false);
errorReportOutput += $"\nReport file: \n {JsonConvert.SerializeObject(reportFileContent)}";
}
var errorOutputContainer = FetchTranscriptionEnvironmentVariables.ErrorReportOutputContainer;
await StorageConnectorInstance.WriteTextFileToBlobAsync(errorReportOutput, errorOutputContainer, $"jobs/{jobName}.txt", log).ConfigureAwait(false);
var retryAudioFile = IsRetryableError(safeErrorCode);
foreach (var audio in serviceBusMessage.AudioFileInfos)
{
var fileName = StorageConnector.GetFileNameFromUri(new Uri(audio.FileUrl));
if (retryAudioFile && audio.RetryCount < FetchTranscriptionEnvironmentVariables.RetryLimit)
{
log.LogInformation($"Retrying transcription with name {fileName} - retry count: {audio.RetryCount}");
var sbMessage = new ServiceBusMessage
{
Data = new Data
{
Url = new Uri(audio.FileUrl)
},
EventType = "BlobCreated",
RetryCount = audio.RetryCount + 1
};
var audioFileMessage = new Message(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(sbMessage)));
await ServiceBusUtilities.SendServiceBusMessageAsync(StartQueueClientInstance, audioFileMessage, log, TimeSpan.FromMinutes(1)).ConfigureAwait(false);
}
else
{
var message = $"Failed transcription with name {fileName} in job {jobName} after {audio.RetryCount} retries with error: {safeErrorMessage} (Error: {safeErrorCode}).";
await StorageConnectorInstance.WriteTextFileToBlobAsync(message, errorOutputContainer, $"{fileName}.txt", log).ConfigureAwait(false);
await StorageConnectorInstance.MoveFileAsync(
FetchTranscriptionEnvironmentVariables.AudioInputContainer,
fileName,
FetchTranscriptionEnvironmentVariables.ErrorFilesOutputContainer,
fileName,
log).ConfigureAwait(false);
}
}
await BatchClient.DeleteTranscriptionAsync(transcriptionLocation, subscriptionKey, log).ConfigureAwait(false);
}
private async Task SubmitReducerTaskAsync(BatchClient batchClient, string containerSas)
{
Console.WriteLine("Adding the reducer task: {0}", Constants.ReducerTaskId);
CloudTask unboundReducerTask = new CloudTask(Constants.ReducerTaskId, Constants.ReducerTaskExecutable);
//The set of files (exes, dlls and configuration files) required to run the reducer task.
List<ResourceFile> reducerTaskResourceFiles = SampleHelpers.GetResourceFiles(containerSas, Constants.RequiredExecutableFiles);
unboundReducerTask.ResourceFiles = reducerTaskResourceFiles;
//Send the request to the Batch Service to add the reducer task.
await batchClient.JobOperations.AddTaskAsync(this.jobId, unboundReducerTask);
}
private static async Task ProcessSucceededTranscriptionAsync(string transcriptionLocation, string subscriptionKey, TranscriptionStartedMessage serviceBusMessage, string jobName, ILogger log)
{
log.LogInformation($"Got succeeded transcription for job {jobName}");
var jsonContainer = FetchTranscriptionEnvironmentVariables.JsonResultOutputContainer;
var textAnalyticsKey = FetchTranscriptionEnvironmentVariables.TextAnalyticsKey;
var textAnalyticsRegion = FetchTranscriptionEnvironmentVariables.TextAnalyticsRegion;
var transcriptionFiles = await BatchClient.GetTranscriptionFilesAsync(transcriptionLocation, subscriptionKey, log).ConfigureAwait(false);
log.LogInformation($"Received transcription files.");
var resultFiles = transcriptionFiles.Values.Where(t => t.Kind == TranscriptionFileKind.Transcription);
var containsMultipleTranscriptions = resultFiles.Skip(1).Any();
var textAnalyticsInfoProvided = !string.IsNullOrEmpty(textAnalyticsKey) &&
!string.IsNullOrEmpty(textAnalyticsRegion) &&
!textAnalyticsRegion.Equals("none", StringComparison.OrdinalIgnoreCase);
var textAnalytics = textAnalyticsInfoProvided ? new TextAnalytics(serviceBusMessage.Locale, textAnalyticsKey, textAnalyticsRegion, log) : null;
var generalErrorsStringBuilder = new StringBuilder();
foreach (var resultFile in resultFiles)
{
log.LogInformation($"Getting result for file {resultFile.Name}");
var transcriptionResult = await BatchClient.GetSpeechTranscriptFromSasAsync(resultFile.Links.ContentUrl, log).ConfigureAwait(false);
if (string.IsNullOrEmpty(transcriptionResult.Source))
{
var errorMessage = $"Transcription source is unknown, skipping evaluation.";
log.LogError(errorMessage);
generalErrorsStringBuilder.AppendLine(errorMessage);
continue;
}
var fileName = StorageConnector.GetFileNameFromUri(new Uri(transcriptionResult.Source));
if (transcriptionResult.RecognizedPhrases != null && transcriptionResult.RecognizedPhrases.All(phrase => phrase.RecognitionStatus.Equals("Success", StringComparison.Ordinal)))
{
var textAnalyticsErrors = new List <string>();
if (FetchTranscriptionEnvironmentVariables.SentimentAnalysisSetting != SentimentAnalysisSetting.None)
{
var sentimentErrors = await textAnalytics.AddSentimentToTranscriptAsync(transcriptionResult, FetchTranscriptionEnvironmentVariables.SentimentAnalysisSetting).ConfigureAwait(false);
textAnalyticsErrors.AddRange(sentimentErrors);
}
if (FetchTranscriptionEnvironmentVariables.EntityRedactionSetting != EntityRedactionSetting.None)
{
var entityRedactionErrors = await textAnalytics.RedactEntitiesAsync(transcriptionResult, FetchTranscriptionEnvironmentVariables.EntityRedactionSetting).ConfigureAwait(false);
textAnalyticsErrors.AddRange(entityRedactionErrors);
}
if (textAnalyticsErrors.Any())
{
var distinctErrors = textAnalyticsErrors.Distinct();
var errorMessage = $"File {(string.IsNullOrEmpty(fileName) ? "unknown" : fileName)}:\n{string.Join('\n', distinctErrors)}";
generalErrorsStringBuilder.AppendLine(errorMessage);
}
}
var editedTranscriptionResultJson = JsonConvert.SerializeObject(
transcriptionResult,
Newtonsoft.Json.Formatting.Indented,
new JsonSerializerSettings
{
NullValueHandling = NullValueHandling.Ignore
});
var jsonFileName = $"{fileName}.json";
await StorageConnectorInstance.WriteTextFileToBlobAsync(editedTranscriptionResultJson, jsonContainer, jsonFileName, log).ConfigureAwait(false);
if (FetchTranscriptionEnvironmentVariables.CreateHtmlResultFile)
{
var htmlContainer = FetchTranscriptionEnvironmentVariables.HtmlResultOutputContainer;
var htmlFileName = $"{fileName}.html";
var displayResults = TranscriptionToHtml.ToHtml(transcriptionResult, jobName);
await StorageConnectorInstance.WriteTextFileToBlobAsync(displayResults, htmlContainer, htmlFileName, log).ConfigureAwait(false);
}
if (FetchTranscriptionEnvironmentVariables.UseSqlDatabase)
{
var duration = XmlConvert.ToTimeSpan(transcriptionResult.Duration);
var approximatedCost = CostEstimation.GetCostEstimation(
duration,
transcriptionResult.CombinedRecognizedPhrases.Count(),
serviceBusMessage.UsesCustomModel,
FetchTranscriptionEnvironmentVariables.SentimentAnalysisSetting,
FetchTranscriptionEnvironmentVariables.EntityRedactionSetting);
var jobId = containsMultipleTranscriptions ? Guid.NewGuid() : new Guid(transcriptionLocation.Split('/').LastOrDefault());
var dbConnectionString = FetchTranscriptionEnvironmentVariables.DatabaseConnectionString;
using var dbConnector = new DatabaseConnector(log, dbConnectionString);
await dbConnector.StoreTranscriptionAsync(
jobId,
serviceBusMessage.Locale,
string.IsNullOrEmpty(fileName)?jobName : fileName,
(float)approximatedCost,
transcriptionResult).ConfigureAwait(false);
}
}
var generalErrors = generalErrorsStringBuilder.ToString();
if (!string.IsNullOrEmpty(generalErrors))
{
var errorTxtname = $"jobs/{jobName}.txt";
await StorageConnectorInstance.WriteTextFileToBlobAsync(
generalErrors,
FetchTranscriptionEnvironmentVariables.ErrorReportOutputContainer,
errorTxtname,
log).ConfigureAwait(false);
}
var reportFile = transcriptionFiles.Values.Where(t => t.Kind == TranscriptionFileKind.TranscriptionReport).FirstOrDefault();
var reportFileContent = await BatchClient.GetTranscriptionReportFileFromSasAsync(reportFile.Links.ContentUrl, log).ConfigureAwait(false);
await ProcessReportFileAsync(reportFileContent, log).ConfigureAwait(false);
BatchClient.DeleteTranscriptionAsync(transcriptionLocation, subscriptionKey, log).ConfigureAwait(false).GetAwaiter().GetResult();
}
private async Task SubmitMapperTasksAsync(BatchClient batchClient, string containerSas)
{
Console.WriteLine("Submitting {0} mapper tasks.", this.configurationSettings.NumberOfMapperTasks);
//The collection of tasks to add to the Batch Service.
List<CloudTask> tasksToAdd = new List<CloudTask>();
for (int i = 0; i < this.configurationSettings.NumberOfMapperTasks; i++)
{
string taskId = Helpers.GetMapperTaskId(i);
string fileBlobName = Helpers.GetSplitFileName(i);
string fileBlobPath = SampleHelpers.ConstructBlobSource(containerSas, fileBlobName);
string commandLine = string.Format("{0} {1}", Constants.MapperTaskExecutable, fileBlobPath);
CloudTask unboundMapperTask = new CloudTask(taskId, commandLine);
//The set of files (exes, dlls and configuration files) required to run the mapper task.
IReadOnlyList<string> mapperTaskRequiredFiles = Constants.RequiredExecutableFiles;
List<ResourceFile> mapperTaskResourceFiles = SampleHelpers.GetResourceFiles(containerSas, mapperTaskRequiredFiles);
unboundMapperTask.ResourceFiles = mapperTaskResourceFiles;
tasksToAdd.Add(unboundMapperTask);
}
//Submit the unbound task collection to the Batch Service.
//Use the AddTask method which takes a collection of CloudTasks for the best performance.
await batchClient.JobOperations.AddTaskAsync(this.jobId, tasksToAdd);
}
public static async Task ProcessTranscriptionJobAsync(TranscriptionStartedMessage serviceBusMessage, ILogger log)
{
if (serviceBusMessage == null)
{
throw new ArgumentNullException(nameof(serviceBusMessage));
}
var subscriptionKey = FetchTranscriptionEnvironmentVariables.AzureSpeechServicesKey;
var jobName = serviceBusMessage.JobName;
var transcriptionLocation = serviceBusMessage.TranscriptionLocation;
log.LogInformation($"Received transcription at {transcriptionLocation} with name {jobName} from service bus message.");
var messageDelayTime = GetMessageDelayTime(serviceBusMessage.PollingCounter);
serviceBusMessage.PollingCounter += 1;
try
{
var transcription = await BatchClient.GetTranscriptionAsync(transcriptionLocation, subscriptionKey, log).ConfigureAwait(false);
log.LogInformation($"Polled {serviceBusMessage.PollingCounter} time(s) for results in total, delay job for {messageDelayTime.TotalMinutes} minutes if not completed.");
switch (transcription.Status)
{
case "Failed":
await ProcessFailedTranscriptionAsync(transcriptionLocation, subscriptionKey, serviceBusMessage, transcription, jobName, log).ConfigureAwait(false);
break;
case "Succeeded":
await ProcessSucceededTranscriptionAsync(transcriptionLocation, subscriptionKey, serviceBusMessage, jobName, log).ConfigureAwait(false);
break;
case "Running":
log.LogInformation($"Transcription running, polling again after {messageDelayTime.TotalMinutes} minutes.");
await ServiceBusUtilities.SendServiceBusMessageAsync(FetchQueueClientInstance, serviceBusMessage.CreateMessageString(), log, messageDelayTime).ConfigureAwait(false);
break;
case "NotStarted":
log.LogInformation($"Transcription not started, polling again after {messageDelayTime.TotalMinutes} minutes.");
await ServiceBusUtilities.SendServiceBusMessageAsync(FetchQueueClientInstance, serviceBusMessage.CreateMessageString(), log, messageDelayTime).ConfigureAwait(false);
break;
}
}
catch (WebException e)
{
if (e.Response != null && BatchClient.IsThrottledOrTimeoutStatusCode(((HttpWebResponse)e.Response).StatusCode))
{
log.LogInformation("Timeout or throttled, retrying message.");
await ServiceBusUtilities.SendServiceBusMessageAsync(FetchQueueClientInstance, serviceBusMessage.CreateMessageString(), log, messageDelayTime).ConfigureAwait(false);
}
else
{
var errorMessage = $"Fetch Transcription in job with name {jobName} failed with WebException {e} and message {e.Message}.";
log.LogError($"{errorMessage}");
await RetryOrFailJobAsync(serviceBusMessage, errorMessage, jobName, transcriptionLocation, subscriptionKey, log).ConfigureAwait(false);
}
}
catch (TimeoutException e)
{
log.LogInformation($"Timeout - re-enqueueing fetch transcription message. Exception message: {e.Message}");
await ServiceBusUtilities.SendServiceBusMessageAsync(FetchQueueClientInstance, serviceBusMessage.CreateMessageString(), log, messageDelayTime).ConfigureAwait(false);
}
catch (Exception e)
{
var errorMessage = $"Fetch Transcription in job with name {jobName} failed with Exception {e} and message {e.Message}.";
log.LogError($"{errorMessage}");
await RetryOrFailJobAsync(serviceBusMessage, errorMessage, jobName, transcriptionLocation, subscriptionKey, log).ConfigureAwait(false);
}
}
