/// <summary>
/// Lists the Jobs matching the specified filter options
/// </summary>
/// <param name="options">The options to use when querying for Jobs</param>
/// <returns>The Jobs matching the specified filter options</returns>
public IEnumerable <PSCloudJob> ListJobs(ListJobOptions options)
{
if (options == null)
{
throw new ArgumentNullException("options");
}
if (string.IsNullOrEmpty(options.WorkItemName) && options.WorkItem == null)
{
throw new ArgumentNullException(Resources.GBJ_NoWorkItem);
}
string wiName = options.WorkItem == null ? options.WorkItemName : options.WorkItem.Name;
// Get the single Job matching the specified name
if (!string.IsNullOrEmpty(options.JobName))
{
WriteVerbose(string.Format(Resources.GBJ_GetByName, options.JobName, wiName));
using (IWorkItemManager wiManager = options.Context.BatchOMClient.OpenWorkItemManager())
{
ICloudJob job = wiManager.GetJob(wiName, options.JobName, additionalBehaviors: options.AdditionalBehaviors);
PSCloudJob psJob = new PSCloudJob(job);
return(new PSCloudJob[] { psJob });
}
}
// List Jobs using the specified filter
else
{
if (options.MaxCount <= 0)
{
options.MaxCount = Int32.MaxValue;
}
ODATADetailLevel odata = null;
if (!string.IsNullOrEmpty(options.Filter))
{
WriteVerbose(string.Format(Resources.GBJ_GetByOData, wiName, options.MaxCount));
odata = new ODATADetailLevel(filterClause: options.Filter);
}
else
{
WriteVerbose(string.Format(Resources.GBJ_GetNoFilter, wiName, options.MaxCount));
}
using (IWorkItemManager wiManager = options.Context.BatchOMClient.OpenWorkItemManager())
{
IEnumerableAsyncExtended <ICloudJob> jobs = wiManager.ListJobs(wiName, odata, options.AdditionalBehaviors);
Func <ICloudJob, PSCloudJob> mappingFunction = j => { return(new PSCloudJob(j)); };
return(new PSAsyncEnumerable <PSCloudJob, ICloudJob>(jobs, mappingFunction).Take(options.MaxCount));
}
}
}
private static void AddWork(IBatchClient client)
{
using (IWorkItemManager wm = client.OpenWorkItemManager())
{
//The toolbox contains some helper mechanisms to ease submission and monitoring of tasks.
IToolbox toolbox = client.OpenToolbox();
// to submit a batch of tasks, the TaskSubmissionHelper is useful.
ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, Program.PoolName);
// workitem is uniquely identified by its name so we will use a timestamp as suffix
taskSubmissionHelper.WorkItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss");
Console.WriteLine("Creating work item: {0}", taskSubmissionHelper.WorkItemName);
// add 2 quick tasks. Tasks within a job must have unique names
taskSubmissionHelper.AddTask(new CloudTask("task1", "hostname"));
taskSubmissionHelper.AddTask(new CloudTask("task2", "cmd /c dir /s"));
//Commit the tasks to the Batch Service
IJobCommitUnboundArtifacts artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts;
// TaskSubmissionHelper commit artifacts returns the workitem and job name
ICloudJob job = wm.GetJob(artifacts.WorkItemName, artifacts.JobName);
Console.WriteLine("Waiting for all tasks to complete on work item: {0}, Job: {1} ...", artifacts.WorkItemName, artifacts.JobName);
//We use the task state monitor to monitor the state of our tasks -- in this case we will wait for them all to complete.
ITaskStateMonitor taskStateMonitor = toolbox.CreateTaskStateMonitor();
// blocking wait on the list of tasks until all tasks reach completed state
bool timedOut = taskStateMonitor.WaitAll(job.ListTasks(), TaskState.Completed, new TimeSpan(0, 20, 0));
if (timedOut)
{
throw new TimeoutException("Timed out waiting for tasks");
}
// dump task output
foreach (var t in job.ListTasks())
{
Console.WriteLine("Task " + t.Name + " says:\n" + t.GetTaskFile(Constants.StandardOutFileName).ReadAsString());
}
// remember to delete the workitem before exiting
Console.WriteLine("Deleting work item: {0}", artifacts.WorkItemName);
wm.DeleteWorkItem(artifacts.WorkItemName);
}
}
/// <summary>
/// This is the client that creates workitem and submits tasks.
/// </summary>
/// <param name="args"></param>
public static void SubmitTasks(string[] args)
{
config = Config.ParseConfig();
//Upload resources if specified
if (config.UploadResources)
{
//Upload ImgProc.exe, Batch.dll and the Storage Client
ImgProcUtils.UploadFileToBlob(Constants.StorageClientDllName, config.ResourceContainerSAS);
ImgProcUtils.UploadFileToBlob(Constants.ImgProcExeName, config.ResourceContainerSAS);
ImgProcUtils.UploadFileToBlob(Constants.BatchClientDllName, config.ResourceContainerSAS);
Console.WriteLine("Done uploading files to blob");
}
try
{
using (IWorkItemManager wm = config.Client.OpenWorkItemManager())
{
IToolbox toolbox = config.Client.OpenToolbox();
//Use the task submission helper to ease creation of workitem and addition of tasks, as well as management of resource file staging.
ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, config.PoolName);
taskSubmissionHelper.WorkItemName = config.WorkitemName;
//Compute the number of images each task should process
int numImgsPerTask = (int)Math.Round(config.NumInputBlobs / (decimal)config.NumTasks);
for (int i = 0; i < config.NumTasks; i++)
{
ICloudTask task = new CloudTask(
name: "task_no_" + i,
commandline: string.Format("{0} --Task {1} thumb{2}", Constants.ImgProcExeName, config.OutputContainerSAS, i));
Console.WriteLine("Generating task: {0}", task.Name);
task.FilesToStage = new List <IFileStagingProvider>();
int start = i * numImgsPerTask;
int end;
if (i < config.NumTasks - 1)
{
end = ((i + 1) * numImgsPerTask) - 1;
}
else
{
end = config.NumInputBlobs - 1;
}
//Generate and set up the list of files to be processed by this task
for (int j = start; j < end; j++)
{
string input = GetTempFilePath(j);
ImgProcUtils.GenerateImages(input, string.Format("{0}", j));
task.FilesToStage.Add(new FileToStage(input, new StagingStorageAccount(config.StorageAccount, config.StorageKey, config.StorageBlobEndpoint)));
}
task.ResourceFiles = ImgProcUtils.GetResourceFiles(config.ResourceContainerSAS);
taskSubmissionHelper.AddTask(task);
}
IJobCommitUnboundArtifacts artifacts = null;
try
{
Console.WriteLine("Submitting {0} tasks to the Batch Service", config.NumTasks);
//Submit the tasks to the Batch Service
artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts;
}
catch (AggregateException ae)
{
// Go through all exceptions and dump useful information
ae.Handle(x =>
{
if (x is BatchException)
{
BatchException be = x as BatchException;
if (null != be.RequestInformation && null != be.RequestInformation.AzureError)
{
// Write the server side error information
Console.Error.WriteLine(be.RequestInformation.AzureError.Code);
Console.Error.WriteLine(be.RequestInformation.AzureError.Message.Value);
if (null != be.RequestInformation.AzureError.Values)
{
foreach (var v in be.RequestInformation.AzureError.Values)
{
Console.Error.WriteLine(v.Key + " : " + v.Value);
}
}
}
}
// Indicate that the error has been handled
return(true);
});
}
DateTime starttime = DateTime.Now;
//Wait for the job to complete
if (config.WaitForCompletion)
{
ICloudJob job = wm.GetJob(artifacts.WorkItemName, artifacts.JobName);
Console.WriteLine("Waiting for tasks to complete...");
// Wait up to 15 minutes for all tasks to reach the completed state
config.Client.OpenToolbox().CreateTaskStateMonitor().WaitAll(job.ListTasks(), TaskState.Completed, new TimeSpan(0, 15, 0));
DateTime endtime = DateTime.Now;
Console.WriteLine("Time taken for processing the images : {0} sec", endtime.Subtract(starttime).TotalSeconds);
}
}
}
finally
{
//Delete the workitem that we created
if (config.DeleteWorkitem &&
config.WaitForCompletion)
{
Console.WriteLine("Press any key to delete the workitem . . .");
Console.ReadKey();
config.Client.OpenWorkItemManager().DeleteWorkItem(config.WorkitemName);
}
}
}
public static void JobMain(string[] args)
{
//Load the configuration
TopNWordsConfiguration configuration = TopNWordsConfiguration.LoadConfigurationFromAppConfig();
StagingStorageAccount stagingStorageAccount = new StagingStorageAccount(
configuration.StorageAccountName,
configuration.StorageAccountKey,
configuration.StorageAccountBlobEndpoint);
IBatchClient client = BatchClient.Connect(configuration.BatchServiceUrl, new BatchCredentials(configuration.BatchAccountName, configuration.BatchAccountKey));
string stagingContainer = null;
//Create a pool (if user hasn't provided one)
if (configuration.ShouldCreatePool)
{
using (IPoolManager pm = client.OpenPoolManager())
{
//OSFamily 4 == OS 2012 R2
//You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
ICloudPool pool = pm.CreatePool(configuration.PoolName, targetDedicated: configuration.PoolSize, osFamily: "4", vmSize: "small");
Console.WriteLine("Adding pool {0}", configuration.PoolName);
pool.Commit();
}
}
try
{
using (IWorkItemManager wm = client.OpenWorkItemManager())
{
IToolbox toolbox = client.OpenToolbox();
//Use the TaskSubmissionHelper to help us create a WorkItem and add tasks to it.
ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, configuration.PoolName);
taskSubmissionHelper.WorkItemName = configuration.WorkItemName;
FileToStage topNWordExe = new FileToStage(TopNWordsExeName, stagingStorageAccount);
FileToStage storageDll = new FileToStage(StorageClientDllName, stagingStorageAccount);
string bookFileUri = UploadBookFileToCloudBlob(configuration, configuration.BookFileName);
Console.WriteLine("{0} uploaded to cloud", configuration.BookFileName);
for (int i = 1; i <= configuration.NumberOfTasks; i++)
{
ICloudTask task = new CloudTask("task_no_" + i, String.Format("{0} --Task {1} {2} {3} {4}",
TopNWordsExeName,
bookFileUri,
configuration.NumberOfTopWords,
configuration.StorageAccountName,
configuration.StorageAccountKey));
//This is the list of files to stage to a container -- for each TaskSubmissionHelper one container is created and
//files all resolve to Azure Blobs by their name (so two tasks with the same named file will create just 1 blob in
//the TaskSubmissionHelper's container).
task.FilesToStage = new List <IFileStagingProvider>
{
topNWordExe,
storageDll
};
taskSubmissionHelper.AddTask(task);
}
//Commit all the tasks to the Batch Service.
IJobCommitUnboundArtifacts artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts;
foreach (var fileStagingArtifact in artifacts.FileStagingArtifacts)
{
SequentialFileStagingArtifact stagingArtifact = fileStagingArtifact.Value as SequentialFileStagingArtifact;
if (stagingArtifact != null)
{
stagingContainer = stagingArtifact.BlobContainerCreated;
Console.WriteLine("Uploaded files to container: {0} -- you will be charged for their storage unless you delete them.",
stagingArtifact.BlobContainerCreated);
}
}
//Get the job to monitor status.
ICloudJob job = wm.GetJob(artifacts.WorkItemName, artifacts.JobName);
Console.Write("Waiting for tasks to complete ...");
// Wait 1 minute for all tasks to reach the completed state
client.OpenToolbox().CreateTaskStateMonitor().WaitAll(job.ListTasks(), TaskState.Completed, TimeSpan.FromMinutes(20));
Console.WriteLine("Done.");
foreach (ICloudTask task in job.ListTasks())
{
Console.WriteLine("Task " + task.Name + " says:\n" + task.GetTaskFile(Constants.StandardOutFileName).ReadAsString());
Console.WriteLine(task.GetTaskFile(Constants.StandardErrorFileName).ReadAsString());
}
}
}
finally
{
//Delete the pool that we created
if (configuration.ShouldCreatePool)
{
using (IPoolManager pm = client.OpenPoolManager())
{
Console.WriteLine("Deleting pool: {0}", configuration.PoolName);
pm.DeletePool(configuration.PoolName);
}
}
//Delete the workitem that we created
if (configuration.ShouldDeleteWorkItem)
{
using (IWorkItemManager wm = client.OpenWorkItemManager())
{
Console.WriteLine("Deleting work item: {0}", configuration.WorkItemName);
wm.DeleteWorkItem(configuration.WorkItemName);
}
}
//Delete the containers we created
if (configuration.ShouldDeleteContainer)
{
DeleteContainers(configuration, stagingContainer);
}
}
}