/// <summary>
/// Runs the reducer task.
/// </summary>
public async Task RunAsync()
{
//Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchCredentials credentials = new BatchCredentials(
this.configurationSettings.BatchAccountName,
this.configurationSettings.BatchAccountKey);
using (IBatchClient batchClient = BatchClient.Connect(this.configurationSettings.BatchServiceUrl, credentials))
{
using (IWorkItemManager workItemManager = batchClient.OpenWorkItemManager())
{
//Gather each Mapper tasks output and write it to standard out.
for (int i = 0; i < this.configurationSettings.NumberOfMapperTasks; i++)
{
string mapperTaskName = Helpers.GetMapperTaskName(i);
//Download the standard out from each mapper task.
ITaskFile taskFile = await workItemManager.GetTaskFileAsync(
this.workItemName,
this.jobName,
mapperTaskName,
Microsoft.Azure.Batch.Constants.StandardOutFileName);
string taskFileString = await taskFile.ReadAsStringAsync();
Console.WriteLine(taskFileString);
Console.WriteLine();
}
}
}
}
public static void Main(string[] args)
{
// This will boost parallel submission speed for REST APIs. If your use requires many simultaneous service calls set this number to something large, such as 100.
// See: http://msdn.microsoft.com/en-us/library/system.net.servicepointmanager.defaultconnectionlimit%28v=vs.110%29.aspx for more info.
System.Net.ServicePointManager.DefaultConnectionLimit = 20;
// Get an instance of the BatchClient for a given Azure Batch account.
BatchCredentials cred = new BatchCredentials(BatchAccount, BatchKey);
using (IBatchClient client = BatchClient.Connect(Url, cred))
{
// if you want to put a retry policy in place, enable it here
// the built-in policies are No Retry (default), Linear Retry, and Exponential Retry
//client.CustomBehaviors.Add(new SetRetryPolicy(new Microsoft.Azure.Batch.Protocol.LinearRetry()));
ListPools(client);
ListWorkItems(client);
CreatePoolIfNotExist(client, Program.PoolName);
AddWork(client);
ListPools(client);
ListWorkItems(client);
AddWorkWithFileStaging(client);
ListPools(client);
ListWorkItems(client);
SubmitLargeNumberOfTasks(client);
}
Console.WriteLine("Press return to exit...");
Console.ReadLine();
}
/// <summary>
/// Allows a client to unregister itself from event notification.
/// </summary>
/// <param name="Client">Reference to the IBatchClient interface to be removed</param>
public void UnregisterClient(IBatchClient Client)
{
//
// Indicate to the thread a client should be removed
m_EventClient.Enqueue(Client);
m_CommandQueue.Enqueue(Command.UnregisterClient);
wh_CommandEvent.Set();
}
public int CompareTo(IBatchClient other)
{
if (other == this)
{
return(0);
}
return(1);
}
public BatchService(string baseUrl, BatchCredentials credentials)
{
this.Client = BatchClient.Connect(baseUrl, credentials);
this.BaseUri = new Uri(baseUrl);
this.Credentials = credentials;
this.retryPolicy = new LinearRetry(TimeSpan.FromSeconds(10), 5);
this.Client.CustomBehaviors.Add(new SetRetryPolicy(this.retryPolicy));
this.Client.CustomBehaviors.Add(new RequestInterceptor((req) => { req.MaximumExecutionTime = TimeSpan.FromMinutes(2); }));
}
public BatchService(string baseUrl, BatchCredentials credentials)
{
this.Client = BatchClient.Connect(baseUrl, credentials);
this.BaseUri = new Uri(baseUrl);
this.Credentials = credentials;
this.retryPolicy = new LinearRetry(TimeSpan.FromSeconds(10), 5);
this.Client.CustomBehaviors.Add(new SetRetryPolicy(this.retryPolicy));
this.Client.CustomBehaviors.Add(new RequestInterceptor((req) => { req.MaximumExecutionTime = TimeSpan.FromMinutes(2); }));
}
private static void ListPools(IBatchClient client)
{
using (var pm = client.OpenPoolManager())
{
Console.WriteLine("Listing Pools\n=============");
// Using optional select clause to return only the name and state.
IEnumerable <ICloudPool> pools = pm.ListPools(new ODATADetailLevel(selectClause: "name,state"));
foreach (var p in pools)
{
Console.WriteLine("pool " + p.Name + " is " + p.State);
}
Console.WriteLine();
}
}
private static void ListWorkItems(IBatchClient client)
{
// All Workitem, Job, and Task related operation start from WorkItemManager
using (var wm = client.OpenWorkItemManager())
{
Console.WriteLine("Listing Workitems\n=================");
IEnumerable <ICloudWorkItem> wis = wm.ListWorkItems();
foreach (var w in wis)
{
Console.WriteLine("Workitem: " + w.Name + " State:" + w.State);
}
Console.WriteLine();
}
}
private static void AddWork(IBatchClient client)
{
using (IWorkItemManager wm = client.OpenWorkItemManager())
{
//The toolbox contains some helper mechanisms to ease submission and monitoring of tasks.
IToolbox toolbox = client.OpenToolbox();
// to submit a batch of tasks, the TaskSubmissionHelper is useful.
ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, Program.PoolName);
// workitem is uniquely identified by its name so we will use a timestamp as suffix
taskSubmissionHelper.WorkItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss");
Console.WriteLine("Creating work item: {0}", taskSubmissionHelper.WorkItemName);
// add 2 quick tasks. Tasks within a job must have unique names
taskSubmissionHelper.AddTask(new CloudTask("task1", "hostname"));
taskSubmissionHelper.AddTask(new CloudTask("task2", "cmd /c dir /s"));
//Commit the tasks to the Batch Service
IJobCommitUnboundArtifacts artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts;
// TaskSubmissionHelper commit artifacts returns the workitem and job name
ICloudJob job = wm.GetJob(artifacts.WorkItemName, artifacts.JobName);
Console.WriteLine("Waiting for all tasks to complete on work item: {0}, Job: {1} ...", artifacts.WorkItemName, artifacts.JobName);
//We use the task state monitor to monitor the state of our tasks -- in this case we will wait for them all to complete.
ITaskStateMonitor taskStateMonitor = toolbox.CreateTaskStateMonitor();
// blocking wait on the list of tasks until all tasks reach completed state
bool timedOut = taskStateMonitor.WaitAll(job.ListTasks(), TaskState.Completed, new TimeSpan(0, 20, 0));
if (timedOut)
{
throw new TimeoutException("Timed out waiting for tasks");
}
// dump task output
foreach (var t in job.ListTasks())
{
Console.WriteLine("Task " + t.Name + " says:\n" + t.GetTaskFile(Constants.StandardOutFileName).ReadAsString());
}
// remember to delete the workitem before exiting
Console.WriteLine("Deleting work item: {0}", artifacts.WorkItemName);
wm.DeleteWorkItem(artifacts.WorkItemName);
}
}
private void CopyRDPStream(Stream destinationStream, IBatchClient client, string poolName, string vmName,
PSVM vm, IEnumerable <BatchClientBehavior> additionalBehaviors = null)
{
if (vm == null)
{
using (IPoolManager poolManager = client.OpenPoolManager())
{
poolManager.GetRDPFile(poolName, vmName, destinationStream, additionalBehaviors);
}
}
else
{
vm.omObject.GetRDPFile(destinationStream, additionalBehaviors);
}
}
private static void DeletePool(IBatchClient client, string poolName)
{
using (var pm = client.OpenPoolManager())
{
try
{
pm.DeletePool(poolName);
}
catch (AggregateException ex)
{
ex.Handle(x =>
(0 == string.Compare((x as BatchException)?.RequestInformation?.AzureError?.Code ?? "",
"PoolNotFound",
StringComparison.InvariantCulture)));
}
}
}
private static void DeletePool(IBatchClient client, string poolName)
{
using (var pm = client.OpenPoolManager())
{
try
{
pm.DeletePool(poolName);
}
catch (AggregateException ex)
{
ex.Handle(x =>
(0 == string.Compare((x as BatchException)?.RequestInformation?.AzureError?.Code ?? "",
"PoolNotFound",
StringComparison.InvariantCulture)));
}
}
}
/// <summary>
/// Allows code to register itself to receive notifications of batch processing events
/// </summary>
/// <param name="Client">Reference to the IBatchClient interface</param>
public void RegisterClient(IBatchClient Client)
{
//
// Upon new registration, let the client know about all pending batch processes.
foreach (BatchProcess Process in m_Queue)
{
Client.AddProcess(
Process,
GPDatabaseUtils.FieldValue(Process.ProjectID, "tblProject", "Name"),
Process.Profile.Name,
Process.TimeAdded,
Process.TimeStarted,
Process.IsStarted);
}
//
// Indicate to the thread a new client has been added
m_EventClient.Enqueue(Client);
m_CommandQueue.Enqueue(Command.RegisterClient);
wh_CommandEvent.Set();
}
private static void CreatePoolIfNotExist(IBatchClient client, string poolName)
{
// All Pool and VM operation starts from PoolManager
using (IPoolManager pm = client.OpenPoolManager())
{
// go through all the pools and see if it already exists
bool found = false;
foreach (ICloudPool p in pm.ListPools())
{
// pools are uniquely identified by their name
if (string.Equals(p.Name, poolName))
{
Console.WriteLine("Using existing pool {0}", poolName);
found = true;
if (!p.ListVMs().Any())
{
Console.WriteLine("There are no VMs in this pool. No tasks will be run until at least one VM has been added via resizing.");
Console.WriteLine("Resizing pool to add 3 VMs. This might take a while...");
p.Resize(3);
}
break;
}
}
if (!found)
{
Console.WriteLine("Creating pool: {0}", poolName);
// if pool not found, call CreatePool
//You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
ICloudPool pool = pm.CreatePool(poolName, targetDedicated: 3, vmSize: "small", osFamily: "3");
pool.Commit();
}
}
}
private static void CreatePoolIfNotExist(IBatchClient client, string poolName)
{
// All Pool and VM operation starts from PoolManager
using (var pm = client.OpenPoolManager())
{
var found = false;
try
{
var p = pm.GetPool(poolName, new ODATADetailLevel(selectClause: "name"));
found = true;
if (!p.ListVMs(new ODATADetailLevel(selectClause: "name")).Any())
{
Console.WriteLine(
"There are no VMs in this pool. No tasks will be run until at least one VM has been added via resizing.");
Console.WriteLine("Resizing pool to add 3 VMs. This might take a while...");
p.Resize(3);
}
}
catch (AggregateException ex)
{
ex.Handle(x =>
(0 == string.Compare((x as BatchException)?.RequestInformation?.AzureError?.Code ?? "",
"PoolNotFound",
StringComparison.InvariantCulture)));
}
if (!found)
{
Console.WriteLine("Creating pool: {0}", poolName);
// if pool not found, call CreatePool
// You can learn more about os families and versions at:
// http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
var pool = pm.CreatePool(poolName, targetDedicated: 5, vmSize: "small", osFamily: "3");
pool.Commit();
}
}
}
private static void CreatePoolIfNotExist(IBatchClient client, string poolName)
{
// All Pool and VM operation starts from PoolManager
using (IPoolManager pm = client.OpenPoolManager())
{
// go through all the pools and see if it already exists
bool found = false;
foreach (ICloudPool p in pm.ListPools())
{
// pools are uniquely identified by their name
if (string.Equals(p.Name, poolName))
{
Console.WriteLine("Using existing pool {0}", poolName);
found = true;
if (!p.ListVMs().Any <IVM>())
{
Console.WriteLine("There are no VMs in this pool. No tasks will be run until at least one VM has been added via resizing.");
Console.WriteLine("Resizing pool to add 3 VMs. This might take a while...");
p.Resize(3);
}
break;
}
}
if (!found)
{
Console.WriteLine("Creating pool: {0}", poolName);
// if pool not found, call CreatePool
//You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
ICloudPool pool = pm.CreatePool(poolName, targetDedicated: 3, vmSize: "small", osFamily: "3");
pool.Commit();
}
}
}
private static void AddWork(IBatchClient client)
{
using (var wm = client.OpenWorkItemManager())
{
//The toolbox contains some helper mechanisms to ease submission and monitoring of tasks.
var toolbox = client.OpenToolbox();
//Create a work item
var workItemName = Environment.GetEnvironmentVariable("USERNAME") +
DateTime.Now.ToString("yyyyMMdd-HHmmss");
Console.WriteLine("Creating work item {0}", workItemName);
var cloudWorkItem = wm.CreateWorkItem(workItemName);
cloudWorkItem.JobExecutionEnvironment = new JobExecutionEnvironment {
PoolName = PoolName
};
cloudWorkItem.JobSpecification = new JobSpecification()
{
JobPreparationTask = new JobPreparationTask()
{
Name = "jobprep",
CommandLine = "cmd /c ping 127.0.0.1"
}
};
cloudWorkItem.Commit();
//Wait for an active job
var maxJobCreationTimeout = TimeSpan.FromSeconds(90);
var jobCreationStartTime = DateTime.Now;
var jobCreationTimeoutTime = jobCreationStartTime.Add(maxJobCreationTimeout);
cloudWorkItem = wm.GetWorkItem(workItemName);
Console.WriteLine("Waiting for a job to become active...");
while (cloudWorkItem.ExecutionInformation?.RecentJob == null)
{
cloudWorkItem.Refresh();
if (DateTime.Now > jobCreationTimeoutTime)
{
throw new TimeoutException("Timed out waiting for job.");
}
Thread.Sleep(TimeSpan.FromSeconds(5));
}
var jobName = cloudWorkItem.ExecutionInformation.RecentJob.Name;
Console.WriteLine("Found job {0}. Adding task objects...", jobName);
var job = wm.GetJob(workItemName, jobName);
job.AddTask(new CloudTask("task1", "hostname"));
job.AddTask(new CloudTask("task2", "cmd /c dir /s ..\\.."));
var haveEnoughSuccessJobPrepTask = WaitAllJobPrepTask(
client,
PoolName,
workItemName,
jobName,
(succ, fail) => succ >= 2 || fail > 0, // stop indicator
(succ, fail) => succ >= 2, // success indicator
TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5), 10
);
if (haveEnoughSuccessJobPrepTask) // got enough nodes, continue
{
//We use the task state monitor to monitor the state of our tasks -- in this case we will wait for them all to complete.
var taskStateMonitor = toolbox.CreateTaskStateMonitor();
// blocking wait on the list of tasks until all tasks reach completed state
var timedOut = taskStateMonitor.WaitAll(job.ListTasks(), TaskState.Completed,
TimeSpan.FromMinutes(20));
if (timedOut)
{
throw new TimeoutException("Timed out waiting for tasks");
}
// dump task output
foreach (var t in job.ListTasks())
{
Console.WriteLine("Task " + t.Name + " output:\n" +
t.GetTaskFile(Constants.StandardOutFileName).ReadAsString());
}
}
else // too much failure, abort
{
Console.WriteLine("too much failure, abort.");
}
// remember to delete the workitem before exiting
Console.WriteLine("Deleting work item: {0}", workItemName);
wm.DeleteWorkItem(workItemName);
}
}
private static void ListWorkItems(IBatchClient client)
{
// All Workitem, Job, and Task related operation start from WorkItemManager
using (var wm = client.OpenWorkItemManager())
{
Console.WriteLine("Listing Workitems\n=================");
IEnumerable<ICloudWorkItem> wis = wm.ListWorkItems();
foreach (var w in wis)
{
Console.WriteLine("Workitem: " + w.Name + " State:" + w.State);
}
Console.WriteLine();
}
}
private static bool WaitAllJobPrepTask(
IBatchClient client,
string pool,
string workitem,
string job,
Func<int, int, bool> stop,
Func<int, int, bool> succeed,
TimeSpan timeout,
TimeSpan freq,
int parallelism = 5)
{
using (var pm = client.OpenPoolManager())
{
using (var vms = new BlockingCollection<string>())
{
var cts = new CancellationTokenSource();
var ct = cts.Token;
var tasks = new List<Task>();
var succeededJobPrepTasks = 0;
var failedJobPrepTasks = 0;
// find all VM in the Pool and put them in a queue
tasks.Add(Task.Run(async () =>
{
Log("Start adding vms to query queue.");
// list all vms and enqueue
// pool manager currently does nothing in Dispose()
// ReSharper disable once AccessToDisposedClosure
var p = pm.GetPool(PoolName);
var en = p.ListVMs(new ODATADetailLevel(selectClause: "name")).GetAsyncEnumerator();
while (await en.MoveNextAsync())
{
Log("Add vm to query queue.", en.Current.Name);
// If vms is disposed, exception will be thrown and the thread is aborted
// ReSharper disable once AccessToDisposedClosure
vms.Add(en.Current.Name, ct);
if (ct.IsCancellationRequested)
break;
}
}, ct));
// launch N threads to get job prep task status of the nodes in the queue
for (var i = 0; i < parallelism; i++)
{
tasks.Add(Task.Run(() =>
{
while (!ct.IsCancellationRequested)
{
string vm;
try
{
// If vms is disposed, exception will be thrown and the thread is aborted
// ReSharper disable once AccessToDisposedClosure
vm = vms.Take(ct);
}
catch (ObjectDisposedException) // if vms is disposed, abort the thread
{
break;
}
catch (OperationCanceledException) // if token cancelled, abort the thread
{
break;
}
Log("Start polling vm", vm);
try
{
// pool manager currently does nothing in Dispose()
// ReSharper disable once AccessToDisposedClosure
var state = pm.GetJobPreparationTaskStatus(pool, vm, workitem, job);
Log("got vm state", vm);
if (state.State == JobPreparationTaskState.Completed)
// job prep task completed on the thread
{
if (state.ExitCode == 0)
{
Interlocked.Increment(ref succeededJobPrepTasks);
}
else
{
Interlocked.Increment(ref failedJobPrepTasks);
}
Log("==== job prep done ====", vm);
if (stop(succeededJobPrepTasks, failedJobPrepTasks))
{
Log("==== polling done, cancel all threads. ====");
cts.Cancel();
}
}
else // job prep still running
{
Log("State is not completed", vm);
// Sleep for some time and put the VM back to the queue
Task.Run(() =>
{
Task.Delay(freq, ct); // if we hit TaskCanceledException here, abort the thread
if (!ct.IsCancellationRequested)
{
Log("Add vm back to query queue", vm);
vms.Add(vm, ct); // if we hit OperationCanceledException here, abort the thread
}
}, ct);
}
}
catch (AggregateException ex)
{
// Error happens when query job prep task info. Ignore JobPreparationTaskNotRunOnTVM
// sinc it means the task has not been run. Add the vm back to the query queue and continue.
// Other exception will be re-thrown.
Log("Exception happend when querying prep task status.", vm);
ex.Handle(x =>
{
if (0 !=
string.Compare(
(x as BatchException)?.RequestInformation?.AzureError?.Code ?? "",
"JobPreparationTaskNotRunOnTVM",
StringComparison.InvariantCulture))
{
return false;
}
Task.Run(() =>
{
Task.Delay(freq, ct);
// if we hit TaskCanceledException here, abort the thread
if (!ct.IsCancellationRequested)
{
Log("Add vm back to query queue", vm);
vms.Add(vm, ct);
// if we hit OperationCanceledException here, abort the thread
}
}, ct);
return true;
});
}
}
Log("Thread cancel.");
}, ct));
}
Task.WaitAll(tasks.ToArray(), timeout);
cts.Cancel();
Log("All done.");
return succeed(succeededJobPrepTasks, failedJobPrepTasks);
}
}
}
private static bool WaitAllJobPrepTask(
IBatchClient client,
string pool,
string workitem,
string job,
Func <int, int, bool> stop,
Func <int, int, bool> succeed,
TimeSpan timeout,
TimeSpan freq,
int parallelism = 5)
{
using (var pm = client.OpenPoolManager())
{
using (var vms = new BlockingCollection <string>())
{
var cts = new CancellationTokenSource();
var ct = cts.Token;
var tasks = new List <Task>();
var succeededJobPrepTasks = 0;
var failedJobPrepTasks = 0;
// find all VM in the Pool and put them in a queue
tasks.Add(Task.Run(async() =>
{
Log("Start adding vms to query queue.");
// list all vms and enqueue
// pool manager currently does nothing in Dispose()
// ReSharper disable once AccessToDisposedClosure
var p = pm.GetPool(PoolName);
var en = p.ListVMs(new ODATADetailLevel(selectClause: "name")).GetAsyncEnumerator();
while (await en.MoveNextAsync())
{
Log("Add vm to query queue.", en.Current.Name);
// If vms is disposed, exception will be thrown and the thread is aborted
// ReSharper disable once AccessToDisposedClosure
vms.Add(en.Current.Name, ct);
if (ct.IsCancellationRequested)
{
break;
}
}
}, ct));
// launch N threads to get job prep task status of the nodes in the queue
for (var i = 0; i < parallelism; i++)
{
tasks.Add(Task.Run(() =>
{
while (!ct.IsCancellationRequested)
{
string vm;
try
{
// If vms is disposed, exception will be thrown and the thread is aborted
// ReSharper disable once AccessToDisposedClosure
vm = vms.Take(ct);
}
catch (ObjectDisposedException) // if vms is disposed, abort the thread
{
break;
}
catch (OperationCanceledException) // if token cancelled, abort the thread
{
break;
}
Log("Start polling vm", vm);
try
{
// pool manager currently does nothing in Dispose()
// ReSharper disable once AccessToDisposedClosure
var state = pm.GetJobPreparationTaskStatus(pool, vm, workitem, job);
Log("got vm state", vm);
if (state.State == JobPreparationTaskState.Completed)
// job prep task completed on the thread
{
if (state.ExitCode == 0)
{
Interlocked.Increment(ref succeededJobPrepTasks);
}
else
{
Interlocked.Increment(ref failedJobPrepTasks);
}
Log("==== job prep done ====", vm);
if (stop(succeededJobPrepTasks, failedJobPrepTasks))
{
Log("==== polling done, cancel all threads. ====");
cts.Cancel();
}
}
else // job prep still running
{
Log("State is not completed", vm);
// Sleep for some time and put the VM back to the queue
Task.Run(() =>
{
Task.Delay(freq, ct); // if we hit TaskCanceledException here, abort the thread
if (!ct.IsCancellationRequested)
{
Log("Add vm back to query queue", vm);
vms.Add(vm, ct); // if we hit OperationCanceledException here, abort the thread
}
}, ct);
}
}
catch (AggregateException ex)
{
// Error happens when query job prep task info. Ignore JobPreparationTaskNotRunOnTVM
// sinc it means the task has not been run. Add the vm back to the query queue and continue.
// Other exception will be re-thrown.
Log("Exception happend when querying prep task status.", vm);
ex.Handle(x =>
{
if (0 !=
string.Compare(
(x as BatchException)?.RequestInformation?.AzureError?.Code ?? "",
"JobPreparationTaskNotRunOnTVM",
StringComparison.InvariantCulture))
{
return(false);
}
Task.Run(() =>
{
Task.Delay(freq, ct);
// if we hit TaskCanceledException here, abort the thread
if (!ct.IsCancellationRequested)
{
Log("Add vm back to query queue", vm);
vms.Add(vm, ct);
// if we hit OperationCanceledException here, abort the thread
}
}, ct);
return(true);
});
}
}
Log("Thread cancel.");
}, ct));
}
Task.WaitAll(tasks.ToArray(), timeout);
cts.Cancel();
Log("All done.");
return(succeed(succeededJobPrepTasks, failedJobPrepTasks));
}
}
}
/// <summary>
/// Submit a work item with tasks which have dependant files.
/// The files are automatically uploaded to Azure Storage using the FileStaging feature of the Azure.Batch client library.
/// </summary>
/// <param name="client"></param>
private static void AddWorkWithFileStaging(IBatchClient client)
{
using (IWorkItemManager wm = client.OpenWorkItemManager())
{
IToolbox toolbox = client.OpenToolbox();
ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, PoolName);
taskSubmissionHelper.WorkItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss");
Console.WriteLine("Creating work item: {0}", taskSubmissionHelper.WorkItemName);
ICloudTask taskToAdd1 = new CloudTask("task_with_file1", "cmd /c type *.txt");
ICloudTask taskToAdd2 = new CloudTask("task_with_file2", "cmd /c dir /s");
//Set up a collection of files to be staged -- these files will be uploaded to Azure Storage
//when the tasks are submitted to the Azure Batch service.
taskToAdd1.FilesToStage = new List<IFileStagingProvider>();
taskToAdd2.FilesToStage = new List<IFileStagingProvider>();
// generate a local file in temp directory
Process cur = Process.GetCurrentProcess();
string path = Path.Combine(Environment.GetEnvironmentVariable("TEMP"), cur.Id + ".txt");
File.WriteAllText(path, "hello from " + cur.Id);
// add file as task dependency so it'll be uploaded to storage before task
// is submitted and download onto the VM before task starts execution
FileToStage file = new FileToStage(path, new StagingStorageAccount(StorageAccount, StorageKey, StorageBlobEndpoint));
taskToAdd1.FilesToStage.Add(file);
taskToAdd2.FilesToStage.Add(file); // filetostage object can be reused
taskSubmissionHelper.AddTask(taskToAdd1);
taskSubmissionHelper.AddTask(taskToAdd2);
IJobCommitUnboundArtifacts artifacts = null;
bool errors = false;
try
{
//Stage the files to Azure Storage and add the tasks to Azure Batch.
artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts;
}
catch (AggregateException ae)
{
errors = true;
// Go through all exceptions and dump useful information
ae.Handle(x =>
{
if (x is BatchException)
{
BatchException be = x as BatchException;
if (null != be.RequestInformation && null != be.RequestInformation.AzureError)
{
// Write the server side error information
Console.Error.WriteLine(be.RequestInformation.AzureError.Code);
Console.Error.WriteLine(be.RequestInformation.AzureError.Message.Value);
if (null != be.RequestInformation.AzureError.Values)
{
foreach (var v in be.RequestInformation.AzureError.Values)
{
Console.Error.WriteLine(v.Key + " : " + v.Value);
}
}
}
}
else
{
Console.WriteLine(x);
}
// Indicate that the error has been handled
return true;
});
}
// if there is no exception, wait for job response
if (!errors)
{
List<ICloudTask> tasksToMonitorForCompletion = wm.ListTasks(artifacts.WorkItemName, artifacts.JobName).ToList();
Console.WriteLine("Waiting for all tasks to complete on work item: {0}, Job: {1} ...", artifacts.WorkItemName, artifacts.JobName);
client.OpenToolbox().CreateTaskStateMonitor().WaitAll(tasksToMonitorForCompletion, TaskState.Completed, TimeSpan.FromMinutes(30));
foreach (ICloudTask task in wm.ListTasks(artifacts.WorkItemName, artifacts.JobName))
{
Console.WriteLine("Task " + task.Name + " says:\n" + task.GetTaskFile(Constants.StandardOutFileName).ReadAsString());
Console.WriteLine(task.GetTaskFile(Constants.StandardErrorFileName).ReadAsString());
}
}
Console.WriteLine("Deleting work item: {0}", artifacts.WorkItemName);
wm.DeleteWorkItem(artifacts.WorkItemName); //Don't forget to delete the work item before you exit
}
}
private static void AddWork(IBatchClient client)
{
using (var wm = client.OpenWorkItemManager())
{
//The toolbox contains some helper mechanisms to ease submission and monitoring of tasks.
var toolbox = client.OpenToolbox();
//Create a work item
var workItemName = Environment.GetEnvironmentVariable("USERNAME") +
DateTime.Now.ToString("yyyyMMdd-HHmmss");
Console.WriteLine("Creating work item {0}", workItemName);
var cloudWorkItem = wm.CreateWorkItem(workItemName);
cloudWorkItem.JobExecutionEnvironment = new JobExecutionEnvironment { PoolName = PoolName };
cloudWorkItem.JobSpecification = new JobSpecification()
{
JobPreparationTask = new JobPreparationTask()
{
Name = "jobprep",
CommandLine = "cmd /c ping 127.0.0.1"
}
};
cloudWorkItem.Commit();
//Wait for an active job
var maxJobCreationTimeout = TimeSpan.FromSeconds(90);
var jobCreationStartTime = DateTime.Now;
var jobCreationTimeoutTime = jobCreationStartTime.Add(maxJobCreationTimeout);
cloudWorkItem = wm.GetWorkItem(workItemName);
Console.WriteLine("Waiting for a job to become active...");
while (cloudWorkItem.ExecutionInformation?.RecentJob == null)
{
cloudWorkItem.Refresh();
if (DateTime.Now > jobCreationTimeoutTime)
{
throw new TimeoutException("Timed out waiting for job.");
}
Thread.Sleep(TimeSpan.FromSeconds(5));
}
var jobName = cloudWorkItem.ExecutionInformation.RecentJob.Name;
Console.WriteLine("Found job {0}. Adding task objects...", jobName);
var job = wm.GetJob(workItemName, jobName);
job.AddTask(new CloudTask("task1", "hostname"));
job.AddTask(new CloudTask("task2", "cmd /c dir /s ..\\.."));
var haveEnoughSuccessJobPrepTask = WaitAllJobPrepTask(
client,
PoolName,
workItemName,
jobName,
(succ, fail) => succ >= 2 || fail > 0, // stop indicator
(succ, fail) => succ >= 2, // success indicator
TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5), 10
);
if (haveEnoughSuccessJobPrepTask) // got enough nodes, continue
{
//We use the task state monitor to monitor the state of our tasks -- in this case we will wait for them all to complete.
var taskStateMonitor = toolbox.CreateTaskStateMonitor();
// blocking wait on the list of tasks until all tasks reach completed state
var timedOut = taskStateMonitor.WaitAll(job.ListTasks(), TaskState.Completed,
TimeSpan.FromMinutes(20));
if (timedOut)
{
throw new TimeoutException("Timed out waiting for tasks");
}
// dump task output
foreach (var t in job.ListTasks())
{
Console.WriteLine("Task " + t.Name + " output:\n" +
t.GetTaskFile(Constants.StandardOutFileName).ReadAsString());
}
}
else // too much failure, abort
{
Console.WriteLine("too much failure, abort.");
}
// remember to delete the workitem before exiting
Console.WriteLine("Deleting work item: {0}", workItemName);
wm.DeleteWorkItem(workItemName);
}
}
/// <summary>
/// Submit a large number of tasks to the Batch Service.
/// </summary>
/// <param name="client">The batch client.</param>
private static void SubmitLargeNumberOfTasks(IBatchClient client)
{
const int taskCountToCreate = 5000;
// In order to simulate a "large" task object which has many properties set (such as resource files, environment variables, etc)
// we create a big environment variable so we have a big task object.
char[] env = new char[2048];
for (int i = 0; i < env.Length; i++)
{
env[i] = 'a';
}
string envStr = new string(env);
using (IWorkItemManager wm = client.OpenWorkItemManager())
{
//Create a work item
string workItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss");
Console.WriteLine("Creating work item {0}", workItemName);
ICloudWorkItem cloudWorkItem = wm.CreateWorkItem(workItemName);
cloudWorkItem.JobExecutionEnvironment = new JobExecutionEnvironment() {PoolName = PoolName}; //Specify the pool to run on
cloudWorkItem.Commit();
//Wait for an active job
TimeSpan maxJobCreationTimeout = TimeSpan.FromSeconds(90);
DateTime jobCreationStartTime = DateTime.Now;
DateTime jobCreationTimeoutTime = jobCreationStartTime.Add(maxJobCreationTimeout);
cloudWorkItem = wm.GetWorkItem(workItemName);
Console.WriteLine("Waiting for a job to become active...");
while (cloudWorkItem.ExecutionInformation == null || cloudWorkItem.ExecutionInformation.RecentJob == null)
{
cloudWorkItem.Refresh();
if (DateTime.Now > jobCreationTimeoutTime)
{
throw new Exception("Timed out waiting for job.");
}
Thread.Sleep(TimeSpan.FromSeconds(5));
}
string jobName = cloudWorkItem.ExecutionInformation.RecentJob.Name;
Console.WriteLine("Found job {0}. Adding task objects...", jobName);
//Generate a large number of tasks to submit
List<ICloudTask> tasksToSubmit = new List<ICloudTask>();
for (int i = 0; i < taskCountToCreate; i++)
{
ICloudTask task = new CloudTask("echo" + i.ToString("D5"), "echo");
List<IEnvironmentSetting> environmentSettings = new List<IEnvironmentSetting>();
environmentSettings.Add(new EnvironmentSetting("envone", envStr));
task.EnvironmentSettings = environmentSettings;
tasksToSubmit.Add(task);
}
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
//This will result in at most 10 simultaneous Bulk Add requests to the Batch Service.
MaxDegreeOfParallelism = 10
};
Console.WriteLine("Submitting {0} tasks to work item: {1}, job: {2}, on pool: {3}",
taskCountToCreate,
cloudWorkItem.Name,
jobName,
cloudWorkItem.JobExecutionEnvironment.PoolName);
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
//Use the AddTask overload which supports a list of tasks for best AddTask performence - internally this method performs an
//intelligent submission of tasks in batches in order to limit the number of REST API calls made to the Batch Service.
wm.AddTask(cloudWorkItem.Name, jobName, tasksToSubmit, parallelOptions);
stopwatch.Stop();
Console.WriteLine("Submitted {0} tasks in {1}", taskCountToCreate, stopwatch.Elapsed);
//Delete the work item to ensure the tasks are cleaned up
wm.DeleteWorkItem(workItemName);
}
}
private static void CreatePoolIfNotExist(IBatchClient client, string poolName)
{
// All Pool and VM operation starts from PoolManager
using (var pm = client.OpenPoolManager())
{
var found = false;
try
{
var p = pm.GetPool(poolName, new ODATADetailLevel(selectClause: "name"));
found = true;
if (!p.ListVMs(new ODATADetailLevel(selectClause: "name")).Any())
{
Console.WriteLine(
"There are no VMs in this pool. No tasks will be run until at least one VM has been added via resizing.");
Console.WriteLine("Resizing pool to add 3 VMs. This might take a while...");
p.Resize(3);
}
}
catch (AggregateException ex)
{
ex.Handle(x =>
(0 == string.Compare((x as BatchException)?.RequestInformation?.AzureError?.Code ?? "",
"PoolNotFound",
StringComparison.InvariantCulture)));
}
if (!found)
{
Console.WriteLine("Creating pool: {0}", poolName);
// if pool not found, call CreatePool
// You can learn more about os families and versions at:
// http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
var pool = pm.CreatePool(poolName, targetDedicated: 5, vmSize: "small", osFamily: "3");
pool.Commit();
}
}
}
private static void ListPools(IBatchClient client)
{
using (var pm = client.OpenPoolManager())
{
Console.WriteLine("Listing Pools\n=============");
// Using optional select clause to return only the name and state.
IEnumerable<ICloudPool> pools = pm.ListPools(new ODATADetailLevel(selectClause: "name,state"));
foreach (var p in pools)
{
Console.WriteLine("pool " + p.Name + " is " + p.State);
}
Console.WriteLine();
}
}
public static void JobMain(string[] args)
{
//Load the configuration
TopNWordsConfiguration configuration = TopNWordsConfiguration.LoadConfigurationFromAppConfig();
StagingStorageAccount stagingStorageAccount = new StagingStorageAccount(
configuration.StorageAccountName,
configuration.StorageAccountKey,
configuration.StorageAccountBlobEndpoint);
IBatchClient client = BatchClient.Connect(configuration.BatchServiceUrl, new BatchCredentials(configuration.BatchAccountName, configuration.BatchAccountKey));
string stagingContainer = null;
//Create a pool (if user hasn't provided one)
if (configuration.ShouldCreatePool)
{
using (IPoolManager pm = client.OpenPoolManager())
{
//OSFamily 4 == OS 2012 R2
//You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
ICloudPool pool = pm.CreatePool(configuration.PoolName, targetDedicated: configuration.PoolSize, osFamily: "4", vmSize: "small");
Console.WriteLine("Adding pool {0}", configuration.PoolName);
pool.Commit();
}
}
try
{
using (IWorkItemManager wm = client.OpenWorkItemManager())
{
IToolbox toolbox = client.OpenToolbox();
//Use the TaskSubmissionHelper to help us create a WorkItem and add tasks to it.
ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, configuration.PoolName);
taskSubmissionHelper.WorkItemName = configuration.WorkItemName;
FileToStage topNWordExe = new FileToStage(TopNWordsExeName, stagingStorageAccount);
FileToStage storageDll = new FileToStage(StorageClientDllName, stagingStorageAccount);
string bookFileUri = UploadBookFileToCloudBlob(configuration, configuration.BookFileName);
Console.WriteLine("{0} uploaded to cloud", configuration.BookFileName);
for (int i = 1; i <= configuration.NumberOfTasks; i++)
{
ICloudTask task = new CloudTask("task_no_" + i, String.Format("{0} --Task {1} {2} {3} {4}",
TopNWordsExeName,
bookFileUri,
configuration.NumberOfTopWords,
configuration.StorageAccountName,
configuration.StorageAccountKey));
//This is the list of files to stage to a container -- for each TaskSubmissionHelper one container is created and
//files all resolve to Azure Blobs by their name (so two tasks with the same named file will create just 1 blob in
//the TaskSubmissionHelper's container).
task.FilesToStage = new List <IFileStagingProvider>
{
topNWordExe,
storageDll
};
taskSubmissionHelper.AddTask(task);
}
//Commit all the tasks to the Batch Service.
IJobCommitUnboundArtifacts artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts;
foreach (var fileStagingArtifact in artifacts.FileStagingArtifacts)
{
SequentialFileStagingArtifact stagingArtifact = fileStagingArtifact.Value as SequentialFileStagingArtifact;
if (stagingArtifact != null)
{
stagingContainer = stagingArtifact.BlobContainerCreated;
Console.WriteLine("Uploaded files to container: {0} -- you will be charged for their storage unless you delete them.",
stagingArtifact.BlobContainerCreated);
}
}
//Get the job to monitor status.
ICloudJob job = wm.GetJob(artifacts.WorkItemName, artifacts.JobName);
Console.Write("Waiting for tasks to complete ...");
// Wait 1 minute for all tasks to reach the completed state
client.OpenToolbox().CreateTaskStateMonitor().WaitAll(job.ListTasks(), TaskState.Completed, TimeSpan.FromMinutes(20));
Console.WriteLine("Done.");
foreach (ICloudTask task in job.ListTasks())
{
Console.WriteLine("Task " + task.Name + " says:\n" + task.GetTaskFile(Constants.StandardOutFileName).ReadAsString());
Console.WriteLine(task.GetTaskFile(Constants.StandardErrorFileName).ReadAsString());
}
}
}
finally
{
//Delete the pool that we created
if (configuration.ShouldCreatePool)
{
using (IPoolManager pm = client.OpenPoolManager())
{
Console.WriteLine("Deleting pool: {0}", configuration.PoolName);
pm.DeletePool(configuration.PoolName);
}
}
//Delete the workitem that we created
if (configuration.ShouldDeleteWorkItem)
{
using (IWorkItemManager wm = client.OpenWorkItemManager())
{
Console.WriteLine("Deleting work item: {0}", configuration.WorkItemName);
wm.DeleteWorkItem(configuration.WorkItemName);
}
}
//Delete the containers we created
if (configuration.ShouldDeleteContainer)
{
DeleteContainers(configuration, stagingContainer);
}
}
}
/// <summary>
/// Runs the job manager task.
/// </summary>
public async Task RunAsync()
{
Console.WriteLine("JobManager for account: {0}, work item: {1}, job: {2} has started...",
this.accountName,
this.workItemName,
this.jobName);
Console.WriteLine();
Console.WriteLine("JobManager running with the following settings: ");
Console.WriteLine("----------------------------------------");
Console.WriteLine(this.configurationSettings.ToString());
//Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchCredentials batchCredentials = new BatchCredentials(
this.configurationSettings.BatchAccountName,
this.configurationSettings.BatchAccountKey);
using (IBatchClient batchClient = BatchClient.Connect(this.configurationSettings.BatchServiceUrl, batchCredentials))
{
using (IWorkItemManager workItemManager = batchClient.OpenWorkItemManager())
{
IToolbox toolbox = batchClient.OpenToolbox();
//Construct a container SAS to provide the Batch Service access to the files required to
//run the mapper and reducer tasks.
string containerSas = Helpers.ConstructContainerSas(
this.configurationSettings.StorageAccountName,
this.configurationSettings.StorageAccountKey,
this.configurationSettings.StorageServiceUrl,
this.configurationSettings.BlobContainer);
//
// Submit mapper tasks.
//
Console.WriteLine("Submitting {0} mapper tasks.", this.configurationSettings.NumberOfMapperTasks);
//The collection of tasks to add to the Batch Service.
List <ICloudTask> tasksToAdd = new List <ICloudTask>();
for (int i = 0; i < this.configurationSettings.NumberOfMapperTasks; i++)
{
string taskName = Helpers.GetMapperTaskName(i);
string fileBlobName = Helpers.GetSplitFileName(i);
string fileBlobPath = Helpers.ConstructBlobSource(containerSas, fileBlobName);
string commandLine = string.Format("{0} -MapperTask {1}", Constants.TextSearchExe, fileBlobPath);
ICloudTask unboundMapperTask = new CloudTask(taskName, commandLine);
//The set of files (exe's, dll's and configuration files) required to run the mapper task.
IReadOnlyList <string> mapperTaskRequiredFiles = Constants.RequiredExecutableFiles;
List <IResourceFile> mapperTaskResourceFiles = Helpers.GetResourceFiles(containerSas, mapperTaskRequiredFiles);
unboundMapperTask.ResourceFiles = mapperTaskResourceFiles;
tasksToAdd.Add(unboundMapperTask);
}
//Submit the unbound task collection to the Batch Service.
//Use the AddTask method which takes a collection of ICloudTasks for the best performance.
await workItemManager.AddTaskAsync(this.workItemName, this.jobName, tasksToAdd);
//
// Wait for the mapper tasks to complete.
//
Console.WriteLine("Waiting for the mapper tasks to complete...");
//List all the mapper tasks using a name filter.
DetailLevel mapperTaskNameFilter = new ODATADetailLevel()
{
FilterClause = string.Format("startswith(name, '{0}')", Constants.MapperTaskPrefix)
};
List <ICloudTask> tasksToMonitor = workItemManager.ListTasks(
this.workItemName,
this.jobName,
detailLevel: mapperTaskNameFilter).ToList();
//Use the task state monitor to wait for the tasks to complete.
ITaskStateMonitor taskStateMonitor = toolbox.CreateTaskStateMonitor();
bool timedOut = await taskStateMonitor.WaitAllAsync(tasksToMonitor, TaskState.Completed, TimeSpan.FromMinutes(5));
//Get the list of mapper tasks in order to analyze their state and ensure they completed successfully.
IEnumerableAsyncExtended <ICloudTask> asyncEnumerable = workItemManager.ListTasks(
this.workItemName,
this.jobName,
detailLevel: mapperTaskNameFilter);
IAsyncEnumerator <ICloudTask> asyncEnumerator = asyncEnumerable.GetAsyncEnumerator();
//Dump the status of each mapper task.
while (await asyncEnumerator.MoveNextAsync())
{
ICloudTask cloudTask = asyncEnumerator.Current;
Console.WriteLine("Task {0} is in state: {1}", cloudTask.Name, cloudTask.State);
await Helpers.CheckForTaskSuccessAsync(cloudTask, dumpStandardOutOnTaskSuccess : false);
Console.WriteLine();
}
//If not all the tasks reached the desired state within the timeout then the job manager
//cannot continue.
if (timedOut)
{
const string errorMessage = "Mapper tasks did not complete within expected timeout.";
Console.WriteLine(errorMessage);
throw new TimeoutException(errorMessage);
}
//
// Create the reducer task.
//
string reducerTaskCommandLine = string.Format("{0} -ReducerTask", Constants.TextSearchExe);
Console.WriteLine("Adding the reducer task: {0}", Constants.ReducerTaskName);
ICloudTask unboundReducerTask = new CloudTask(Constants.ReducerTaskName, reducerTaskCommandLine);
//The set of files (exe's, dll's and configuration files) required to run the reducer task.
List <IResourceFile> reducerTaskResourceFiles = Helpers.GetResourceFiles(containerSas, Constants.RequiredExecutableFiles);
unboundReducerTask.ResourceFiles = reducerTaskResourceFiles;
//Send the request to the Batch Service to add the reducer task.
await workItemManager.AddTaskAsync(this.workItemName, this.jobName, unboundReducerTask);
//
//Wait for the reducer task to complete.
//
//Get the bound reducer task and monitor it for completion.
ICloudTask boundReducerTask = await workItemManager.GetTaskAsync(this.workItemName, this.jobName, Constants.ReducerTaskName);
timedOut = await taskStateMonitor.WaitAllAsync(new List <ICloudTask> {
boundReducerTask
}, TaskState.Completed, TimeSpan.FromMinutes(2));
//Refresh the reducer task to get the most recent information about it from the Batch Service.
await boundReducerTask.RefreshAsync();
//Dump the reducer tasks exit code and scheduling error for debugging purposes.
await Helpers.CheckForTaskSuccessAsync(boundReducerTask, dumpStandardOutOnTaskSuccess : true);
//Handle the possibilty that the reducer task did not complete in the expected timeout.
if (timedOut)
{
const string errorMessage = "Reducer task did not complete within expected timeout.";
Console.WriteLine("Task {0} is in state: {1}", boundReducerTask.Name, boundReducerTask.State);
Console.WriteLine(errorMessage);
throw new TimeoutException(errorMessage);
}
//The job manager has completed.
Console.WriteLine("JobManager completed successfully.");
}
}
}
private static void AddWork(IBatchClient client)
{
using (IWorkItemManager wm = client.OpenWorkItemManager())
{
//The toolbox contains some helper mechanisms to ease submission and monitoring of tasks.
IToolbox toolbox = client.OpenToolbox();
// to submit a batch of tasks, the TaskSubmissionHelper is useful.
ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, PoolName);
// workitem is uniquely identified by its name so we will use a timestamp as suffix
taskSubmissionHelper.WorkItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss");
Console.WriteLine("Creating work item: {0}", taskSubmissionHelper.WorkItemName);
// add 2 quick tasks. Tasks within a job must have unique names
taskSubmissionHelper.AddTask(new CloudTask("task1", "hostname"));
taskSubmissionHelper.AddTask(new CloudTask("task2", "cmd /c dir /s"));
//Commit the tasks to the Batch Service
IJobCommitUnboundArtifacts artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts;
// TaskSubmissionHelper commit artifacts returns the workitem and job name
if (artifacts != null)
{
ICloudJob job = wm.GetJob(artifacts.WorkItemName, artifacts.JobName);
Console.WriteLine("Waiting for all tasks to complete on work item: {0}, Job: {1} ...",
artifacts.WorkItemName, artifacts.JobName);
//We use the task state monitor to monitor the state of our tasks -- in this case we will wait for them all to complete.
ITaskStateMonitor taskStateMonitor = toolbox.CreateTaskStateMonitor();
// blocking wait on the list of tasks until all tasks reach completed state
bool timedOut = taskStateMonitor.WaitAll(job.ListTasks(), TaskState.Completed,
new TimeSpan(0, 20, 0));
if (timedOut)
{
throw new TimeoutException("Timed out waiting for tasks");
}
// dump task output
foreach (var t in job.ListTasks())
{
Console.WriteLine("Task " + t.Name + " says:\n" +
t.GetTaskFile(Constants.StandardOutFileName).ReadAsString());
}
// remember to delete the workitem before exiting
Console.WriteLine("Deleting work item: {0}", artifacts.WorkItemName);
wm.DeleteWorkItem(artifacts.WorkItemName);
}
}
}
/// <summary>
/// Submit a work item with tasks which have dependant files.
/// The files are automatically uploaded to Azure Storage using the FileStaging feature of the Azure.Batch client library.
/// </summary>
/// <param name="client"></param>
private static void AddWorkWithFileStaging(IBatchClient client)
{
using (IWorkItemManager wm = client.OpenWorkItemManager())
{
IToolbox toolbox = client.OpenToolbox();
ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, Program.PoolName);
taskSubmissionHelper.WorkItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss");
Console.WriteLine("Creating work item: {0}", taskSubmissionHelper.WorkItemName);
ICloudTask taskToAdd1 = new CloudTask("task_with_file1", "cmd /c type *.txt");
ICloudTask taskToAdd2 = new CloudTask("task_with_file2", "cmd /c dir /s");
//Set up a collection of files to be staged -- these files will be uploaded to Azure Storage
//when the tasks are submitted to the Azure Batch service.
taskToAdd1.FilesToStage = new List <IFileStagingProvider>();
taskToAdd2.FilesToStage = new List <IFileStagingProvider>();
// generate a local file in temp directory
Process cur = Process.GetCurrentProcess();
string path = Path.Combine(Environment.GetEnvironmentVariable("TEMP"), cur.Id.ToString() + ".txt");
System.IO.File.WriteAllText(path, "hello from " + cur.Id.ToString());
// add file as task dependency so it'll be uploaded to storage before task
// is submitted and download onto the VM before task starts execution
FileToStage file = new FileToStage(path, new StagingStorageAccount(Program.StorageAccount, Program.StorageKey, Program.StorageBlobEndpoint));
taskToAdd1.FilesToStage.Add(file);
taskToAdd2.FilesToStage.Add(file); // filetostage object can be reused
taskSubmissionHelper.AddTask(taskToAdd1);
taskSubmissionHelper.AddTask(taskToAdd2);
IJobCommitUnboundArtifacts artifacts = null;
bool errors = false;
try
{
//Stage the files to Azure Storage and add the tasks to Azure Batch.
artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts;
}
catch (AggregateException ae)
{
errors = true;
// Go through all exceptions and dump useful information
ae.Handle((x) =>
{
if (x is BatchException)
{
BatchException be = x as BatchException;
if (null != be.RequestInformation && null != be.RequestInformation.AzureError)
{
// Write the server side error information
Console.Error.WriteLine(be.RequestInformation.AzureError.Code);
Console.Error.WriteLine(be.RequestInformation.AzureError.Message.Value);
if (null != be.RequestInformation.AzureError.Values)
{
foreach (var v in be.RequestInformation.AzureError.Values)
{
Console.Error.WriteLine(v.Key + " : " + v.Value);
}
}
}
}
else
{
Console.WriteLine(x);
}
// Indicate that the error has been handled
return(true);
});
}
// if there is no exception, wait for job response
if (!errors)
{
List <ICloudTask> tasksToMonitorForCompletion = wm.ListTasks(artifacts.WorkItemName, artifacts.JobName).ToList();
Console.WriteLine("Waiting for all tasks to complete on work item: {0}, Job: {1} ...", artifacts.WorkItemName, artifacts.JobName);
client.OpenToolbox().CreateTaskStateMonitor().WaitAll(tasksToMonitorForCompletion, TaskState.Completed, TimeSpan.FromMinutes(30));
foreach (ICloudTask task in wm.ListTasks(artifacts.WorkItemName, artifacts.JobName))
{
Console.WriteLine("Task " + task.Name + " says:\n" + task.GetTaskFile(Constants.StandardOutFileName).ReadAsString());
Console.WriteLine(task.GetTaskFile(Constants.StandardErrorFileName).ReadAsString());
}
}
Console.WriteLine("Deleting work item: {0}", artifacts.WorkItemName);
wm.DeleteWorkItem(artifacts.WorkItemName); //Don't forget to delete the work item before you exit
}
}
/// <summary>
/// Populates Azure Storage with the required files, and
/// submits the work item to the Azure Batch service.
/// </summary>
public async Task RunAsync()
{
Console.WriteLine("Running with the following settings: ");
Console.WriteLine("----------------------------------------");
Console.WriteLine(this.configurationSettings.ToString());
//Upload resources if required.
if (this.configurationSettings.ShouldUploadResources)
{
Console.WriteLine("Splitting file: {0} into {1} subfiles",
Constants.TextFilePath,
this.configurationSettings.NumberOfMapperTasks);
//Split the text file into the correct number of files for consumption by the mapper tasks.
FileSplitter splitter = new FileSplitter();
List <string> mapperTaskFiles = await splitter.SplitAsync(
Constants.TextFilePath,
this.configurationSettings.NumberOfMapperTasks);
await this.UploadResourcesAsync(mapperTaskFiles);
}
//Generate a SAS for the container.
string containerSasUrl = Helpers.ConstructContainerSas(
this.configurationSettings.StorageAccountName,
this.configurationSettings.StorageAccountKey,
this.configurationSettings.StorageServiceUrl,
this.configurationSettings.BlobContainer);
//Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchCredentials batchCredentials = new BatchCredentials(
this.configurationSettings.BatchAccountName,
this.configurationSettings.BatchAccountKey);
using (IBatchClient batchClient = BatchClient.Connect(this.configurationSettings.BatchServiceUrl, batchCredentials))
{
using (IWorkItemManager workItemManager = batchClient.OpenWorkItemManager())
{
//Create the unbound work item in local memory. An object which exists only in local memory (and not on the Batch Service) is "unbound".
string workItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.UtcNow.ToString("yyyyMMdd-HHmmss");
ICloudWorkItem unboundWorkItem = workItemManager.CreateWorkItem(workItemName);
//
// Construct the work item properties in local memory before commiting them to the Batch Service.
//
//Allow enough VMs in the pool to run each mapper task, and 1 extra to run the job manager.
int numberOfPoolVMs = 1 + this.configurationSettings.NumberOfMapperTasks;
//Define the pool specification for the pool which the work item will run on.
IPoolSpecification poolSpecification = new PoolSpecification()
{
TargetDedicated = numberOfPoolVMs,
VMSize = "small",
//You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
OSFamily = "4",
TargetOSVersion = "*"
};
//Use the auto pool feature of the Batch Service to create a pool when the work item is created.
//This creates a new pool for each work item which is added.
IAutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification()
{
AutoPoolNamePrefix = "TextSearchPool",
KeepAlive = false,
PoolLifeTimeOption = PoolLifeTimeOption.WorkItem,
PoolSpecification = poolSpecification
};
//Define the execution environment for this work item -- it will run on the pool defined by the auto pool specification above.
unboundWorkItem.JobExecutionEnvironment = new JobExecutionEnvironment()
{
AutoPoolSpecification = autoPoolSpecification
};
//Define the job manager for this work item. This job manager will run when any job is created and will submit the tasks for
//the work item. The job manager is the executable which manages the lifetime of the job
//and all tasks which should run for the job. In this case, the job manager submits the mapper and reducer tasks.
string jobManagerCommandLine = string.Format("{0} -JobManagerTask", Constants.TextSearchExe);
List <IResourceFile> jobManagerResourceFiles = Helpers.GetResourceFiles(containerSasUrl, Constants.RequiredExecutableFiles);
const string jobManagerTaskName = "JobManager";
unboundWorkItem.JobSpecification = new JobSpecification()
{
JobManager = new JobManager()
{
ResourceFiles = jobManagerResourceFiles,
CommandLine = jobManagerCommandLine,
//Determines if the job should terminate when the job manager process exits
KillJobOnCompletion = false,
Name = jobManagerTaskName
}
};
try
{
//Commit the unbound work item to the Batch Service.
Console.WriteLine("Adding work item: {0} to the Batch Service.", unboundWorkItem.Name);
await unboundWorkItem.CommitAsync(); //Issues a request to the Batch Service to add the work item which was defined above.
//
// Wait for the job manager task to complete.
//
//An object which is backed by a corresponding Batch Service object is "bound."
ICloudWorkItem boundWorkItem = await workItemManager.GetWorkItemAsync(workItemName);
//Wait for the job to be created automatically by the Batch Service.
string boundJobName = await Helpers.WaitForActiveJobAsync(boundWorkItem);
ICloudTask boundJobManagerTask = await workItemManager.GetTaskAsync(
workItemName,
boundJobName,
jobManagerTaskName);
TimeSpan maxJobCompletionTimeout = TimeSpan.FromMinutes(30);
IToolbox toolbox = batchClient.OpenToolbox();
ITaskStateMonitor monitor = toolbox.CreateTaskStateMonitor();
bool timedOut = await monitor.WaitAllAsync(new List <ICloudTask> {
boundJobManagerTask
}, TaskState.Completed, maxJobCompletionTimeout);
Console.WriteLine("Done waiting for job manager task.");
await boundJobManagerTask.RefreshAsync();
//Check to ensure the job manager task exited successfully.
await Helpers.CheckForTaskSuccessAsync(boundJobManagerTask, dumpStandardOutOnTaskSuccess : true);
if (timedOut)
{
throw new TimeoutException(string.Format("Timed out waiting for job manager task to complete."));
}
}
catch (AggregateException e)
{
e.Handle(
(innerE) =>
{
//We print all the inner exceptions for debugging purposes.
Console.WriteLine(innerE.ToString());
return(false);
});
throw;
}
catch (Exception e)
{
Console.WriteLine("Hit unexpected exception: {0}", e.ToString());
throw;
}
finally
{
//Delete the work item.
//This will delete the auto pool associated with the work item as long as the pool
//keep alive property is set to false.
if (this.configurationSettings.ShouldDeleteWorkItem)
{
Console.WriteLine("Deleting work item {0}", workItemName);
workItemManager.DeleteWorkItem(workItemName);
}
//Note that there were files uploaded to a container specified in the
//configuration file. This container will not be deleted or cleaned up by this sample.
}
}
}
}
/// <summary>
/// This will take the basic data provided about the account, upload the necessary information to the account, and schedule a job.
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void btnStart_Click(object sender, RoutedEventArgs e)
{
BatchCredentials credentials = new BatchCredentials(txtBAccountName.Text, txtBAccountKey.Text);
IBatchClient bClient = BatchClient.Connect(SampleConstants.BatchSvcEndpoint, credentials);
// Setting a retry policy adds robustness against an individual call timing out. When using a policy, by default all recoverable failures are retried.
bClient.CustomBehaviors.Add(new SetRetryPolicy(new ExponentialRetry(TimeSpan.FromSeconds(5), 5)));
// Create a unique workitem name; don't forget to delete these when you're done
string workItemName = SampleConstants.WorkItemNamePrefix + Guid.NewGuid().ToString();
// Identify the pre-existing pool of VMs that will run the tasks. An Autopool specification
// is fine but there is the delay associated with the creation of the pool along with waiting
// for the VMs to reach Idle state before tasks are running. You can use Batch Explorer to
// pre-create the pool and then resize it to the desired size and number of VMs.
JobExecutionEnvironment jee = new JobExecutionEnvironment()
{
PoolName = PoolName
};
// Next, create the JobManager instance describing the environment settings and resources it
// needs to run
JobManager jobMgr = new JobManager()
{
Name = "JM1",
CommandLine = SampleConstants.JobManager,
// NOTE: We do not in general recommend that customers put their secrets on the command line or as environmental variables, as
// these are not a secure locations. This was done for the simplicity of the sample.
EnvironmentSettings = new List <IEnvironmentSetting>()
{
{ new EnvironmentSetting(SampleConstants.EnvWorkItemName, workItemName) },
{ new EnvironmentSetting(SampleConstants.EnvBatchAccountKeyName, txtBAccountKey.Text) }
},
// In many cases you will want KillJobOnCompletion to be set to 'TRUE' - this allows the previous job to finish before
// a recurrence is scheduled. As an alternative, you can set this to 'FALSE' and use MaxWallClockTime as shown below,
// which will instead ensure that every recurrence happens.
KillJobOnCompletion = true
};
// Create a list of resource files that are needed to run JobManager.exe. A shared access signature key specifying
// readonly access is used so the JobManager program will have access to the resource files when it is started
// on a VM.
var sasPrefix = Helpers.ConstructContainerSas(
txtSAccountName.Text,
txtSAccountKey.Text,
"core.windows.net",
txtSContainerName.Text);
jobMgr.ResourceFiles = Helpers.GetResourceFiles(sasPrefix, SampleConstants.JobManagerFiles);
// Create the job specification, identifying that this job has a job manager associated with it
JobSpecification jobSpec = new JobSpecification()
{
JobManager = jobMgr
};
// Set up the desired recurrence or start time schedule.
WorkItemSchedule wiSchedule = new WorkItemSchedule();
if (rdoOnce.IsChecked == true)
{
// Set information if the task is to be run once.
DateTime runOnce = (DateTime)(dpkDate.SelectedDate);
runOnce = runOnce.AddHours(cbxHourO.SelectedIndex);
runOnce = runOnce.AddMinutes(cbxMinuteO.SelectedIndex);
wiSchedule.DoNotRunUntil = runOnce;
}
else
{
// Set information if the task is to be recurring.
TimeSpan recurring = new TimeSpan(cbxHourR.SelectedIndex, cbxMinuteR.SelectedIndex, 0);
wiSchedule.RecurrenceInterval = recurring;
TimeSpan countback = new TimeSpan(0, 0, 30);
jobSpec.JobConstraints = new JobConstraints(recurring.Subtract(countback), null);
}
// Upload files and create workitem.
UploadFiles();
using (IWorkItemManager wiMgr = bClient.OpenWorkItemManager())
{
ICloudWorkItem workItem = wiMgr.CreateWorkItem(workItemName);
workItem.JobExecutionEnvironment = jee;
workItem.Schedule = wiSchedule;
workItem.JobSpecification = jobSpec;
try
{
workItem.Commit();
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString());
}
}
// Remember to clean up your workitems and jobs
}
/// <summary>
/// Submit a large number of tasks to the Batch Service.
/// </summary>
/// <param name="client">The batch client.</param>
private static void SubmitLargeNumberOfTasks(IBatchClient client)
{
const int taskCountToCreate = 5000;
// In order to simulate a "large" task object which has many properties set (such as resource files, environment variables, etc)
// we create a big environment variable so we have a big task object.
char[] env = new char[2048];
for (int i = 0; i < env.Length; i++)
{
env[i] = 'a';
}
string envStr = new string(env);
using (IWorkItemManager wm = client.OpenWorkItemManager())
{
//Create a work item
string workItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss");
Console.WriteLine("Creating work item {0}", workItemName);
ICloudWorkItem cloudWorkItem = wm.CreateWorkItem(workItemName);
cloudWorkItem.JobExecutionEnvironment = new JobExecutionEnvironment()
{
PoolName = PoolName
}; //Specify the pool to run on
cloudWorkItem.Commit();
//Wait for an active job
TimeSpan maxJobCreationTimeout = TimeSpan.FromSeconds(90);
DateTime jobCreationStartTime = DateTime.Now;
DateTime jobCreationTimeoutTime = jobCreationStartTime.Add(maxJobCreationTimeout);
cloudWorkItem = wm.GetWorkItem(workItemName);
Console.WriteLine("Waiting for a job to become active...");
while (cloudWorkItem.ExecutionInformation == null || cloudWorkItem.ExecutionInformation.RecentJob == null)
{
cloudWorkItem.Refresh();
if (DateTime.Now > jobCreationTimeoutTime)
{
throw new Exception("Timed out waiting for job.");
}
Thread.Sleep(TimeSpan.FromSeconds(5));
}
string jobName = cloudWorkItem.ExecutionInformation.RecentJob.Name;
Console.WriteLine("Found job {0}. Adding task objects...", jobName);
//Generate a large number of tasks to submit
List <ICloudTask> tasksToSubmit = new List <ICloudTask>();
for (int i = 0; i < taskCountToCreate; i++)
{
ICloudTask task = new CloudTask("echo" + i.ToString("D5"), "echo");
List <IEnvironmentSetting> environmentSettings = new List <IEnvironmentSetting>();
environmentSettings.Add(new EnvironmentSetting("envone", envStr));
task.EnvironmentSettings = environmentSettings;
tasksToSubmit.Add(task);
}
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
//This will result in at most 10 simultaneous Bulk Add requests to the Batch Service.
MaxDegreeOfParallelism = 10
};
Console.WriteLine("Submitting {0} tasks to work item: {1}, job: {2}, on pool: {3}",
taskCountToCreate,
cloudWorkItem.Name,
jobName,
cloudWorkItem.JobExecutionEnvironment.PoolName);
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
//Use the AddTask overload which supports a list of tasks for best AddTask performence - internally this method performs an
//intelligent submission of tasks in batches in order to limit the number of REST API calls made to the Batch Service.
wm.AddTask(cloudWorkItem.Name, jobName, tasksToSubmit, parallelOptions);
stopwatch.Stop();
Console.WriteLine("Submitted {0} tasks in {1}", taskCountToCreate, stopwatch.Elapsed);
//Delete the work item to ensure the tasks are cleaned up
wm.DeleteWorkItem(workItemName);
}
}
private void CopyRDPStream(Stream destinationStream, IBatchClient client, string poolName, string vmName,
PSVM vm, IEnumerable<BatchClientBehavior> additionalBehaviors = null)
{
if (vm == null)
{
using (IPoolManager poolManager = client.OpenPoolManager())
{
poolManager.GetRDPFile(poolName, vmName, destinationStream, additionalBehaviors);
}
}
else
{
vm.omObject.GetRDPFile(destinationStream, additionalBehaviors);
}
}