/// <summary>
/// Initialises the uploader and batch client. Asks user for an Azure licence file and saves the credentials
/// if the credentials have not previously been set.
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void GetCredentials(object sender, EventArgs e)
{
if (AzureCredentialsSetup.CredentialsExist())
{
// store credentials
storageAuth = StorageCredentials.FromConfiguration();
batchAuth = BatchCredentials.FromConfiguration();
storageAccount = new CloudStorageAccount(new Microsoft.Azure.Storage.Auth.StorageCredentials(storageAuth.Account, storageAuth.Key), true);
uploader = new FileUploader(storageAccount);
var sharedCredentials = new Microsoft.Azure.Batch.Auth.BatchSharedKeyCredentials(batchAuth.Url, batchAuth.Account, batchAuth.Key);
try
{
batchCli = BatchClient.Open(sharedCredentials);
}
catch (UriFormatException)
{
ShowErrorMessage("Error opening Azure Batch client: credentials are invalid.");
AzureCredentialsSetup cred = new AzureCredentialsSetup();
cred.Finished += GetCredentials;
}
catch (Exception ex)
{
ShowError(ex);
}
}
else
{
// ask user for a credentials file
AzureCredentialsSetup cred = new AzureCredentialsSetup();
cred.Finished += GetCredentials;
}
}
/// <summary>
/// Runs the reducer task.
/// </summary>
public async Task RunAsync()
{
//Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchCredentials credentials = new BatchCredentials(
this.configurationSettings.BatchAccountName,
this.configurationSettings.BatchAccountKey);
using (IBatchClient batchClient = BatchClient.Connect(this.configurationSettings.BatchServiceUrl, credentials))
{
using (IWorkItemManager workItemManager = batchClient.OpenWorkItemManager())
{
//Gather each Mapper tasks output and write it to standard out.
for (int i = 0; i < this.configurationSettings.NumberOfMapperTasks; i++)
{
string mapperTaskName = Helpers.GetMapperTaskName(i);
//Download the standard out from each mapper task.
ITaskFile taskFile = await workItemManager.GetTaskFileAsync(
this.workItemName,
this.jobName,
mapperTaskName,
Batch.Constants.StandardOutFileName);
string taskFileString = await taskFile.ReadAsStringAsync();
Console.WriteLine(taskFileString);
Console.WriteLine();
}
}
}
}
public static void Main(string[] args)
{
// This will boost parallel submission speed for REST APIs. If your use requires many simultaneous service calls set this number to something large, such as 100.
// See: http://msdn.microsoft.com/en-us/library/system.net.servicepointmanager.defaultconnectionlimit%28v=vs.110%29.aspx for more info.
System.Net.ServicePointManager.DefaultConnectionLimit = 20;
// Get an instance of the BatchClient for a given Azure Batch account.
BatchCredentials cred = new BatchCredentials(BatchAccount, BatchKey);
using (IBatchClient client = BatchClient.Connect(Url, cred))
{
// if you want to put a retry policy in place, enable it here
// the built-in policies are No Retry (default), Linear Retry, and Exponential Retry
//client.CustomBehaviors.Add(new SetRetryPolicy(new Microsoft.Azure.Batch.Protocol.LinearRetry()));
ListPools(client);
ListWorkItems(client);
CreatePoolIfNotExist(client, Program.PoolName);
AddWork(client);
ListPools(client);
ListWorkItems(client);
AddWorkWithFileStaging(client);
ListPools(client);
ListWorkItems(client);
SubmitLargeNumberOfTasks(client);
}
Console.WriteLine("Press return to exit...");
Console.ReadLine();
}
/// <summary>
/// Runs the reducer task.
/// </summary>
public async Task RunAsync()
{
//Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchCredentials credentials = new BatchCredentials(
this.configurationSettings.BatchAccountName,
this.configurationSettings.BatchAccountKey);
using (IBatchClient batchClient = BatchClient.Connect(this.configurationSettings.BatchServiceUrl, credentials))
{
using (IWorkItemManager workItemManager = batchClient.OpenWorkItemManager())
{
//Gather each Mapper tasks output and write it to standard out.
for (int i = 0; i < this.configurationSettings.NumberOfMapperTasks; i++)
{
string mapperTaskName = Helpers.GetMapperTaskName(i);
//Download the standard out from each mapper task.
ITaskFile taskFile = await workItemManager.GetTaskFileAsync(
this.workItemName,
this.jobName,
mapperTaskName,
Microsoft.Azure.Batch.Constants.StandardOutFileName);
string taskFileString = await taskFile.ReadAsStringAsync();
Console.WriteLine(taskFileString);
Console.WriteLine();
}
}
}
}
/// <summary>
/// Initialises the Azure credentials, batch client and blob client. Asks user for an Azure
/// licence file and saves the credentials if the credentials have not previously been set.
/// Once credentials are saved, it starts the job load worker.
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
public void GetCredentials()
{
if (AzureCredentialsSetup.CredentialsExist())
{
// store credentials
storageCredentials = StorageCredentials.FromConfiguration();
batchCredentials = BatchCredentials.FromConfiguration();
storageAccount = new CloudStorageAccount(new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(storageCredentials.Account, storageCredentials.Key), true);
var sharedCredentials = new Microsoft.Azure.Batch.Auth.BatchSharedKeyCredentials(batchCredentials.Url, batchCredentials.Account, batchCredentials.Key);
batchClient = BatchClient.Open(sharedCredentials);
blobClient = storageAccount.CreateCloudBlobClient();
blobClient.DefaultRequestOptions.RetryPolicy = new Microsoft.WindowsAzure.Storage.RetryPolicies.LinearRetry(TimeSpan.FromSeconds(3), 10);
if (!FetchJobs.IsBusy)
{
FetchJobs.RunWorkerAsync();
}
}
else
{
// ask user for a credentials file
AzureCredentialsSetup cred = new AzureCredentialsSetup();
cred.Finished += (sender, e) => GetCredentials();
}
}
public BlastConfigurationManager(
StorageCredentials storageCredentials,
BatchCredentials batchCredentials)
{
_storageCredentials = storageCredentials;
_batchCredentials = batchCredentials;
}
public static void Main(string[] args)
{
// This will boost parallel submission speed for REST APIs. If your use requires many simultaneous service calls set this number to something large, such as 100.
// See: http://msdn.microsoft.com/en-us/library/system.net.servicepointmanager.defaultconnectionlimit%28v=vs.110%29.aspx for more info.
System.Net.ServicePointManager.DefaultConnectionLimit = 20;
// Get an instance of the BatchClient for a given Azure Batch account.
BatchCredentials cred = new BatchCredentials(BatchAccount, BatchKey);
using (IBatchClient client = BatchClient.Connect(Url, cred))
{
// if you want to put a retry policy in place, enable it here
// the built-in policies are No Retry (default), Linear Retry, and Exponential Retry
//client.CustomBehaviors.Add(new SetRetryPolicy(new Microsoft.Azure.Batch.Protocol.LinearRetry()));
ListPools(client);
ListWorkItems(client);
CreatePoolIfNotExist(client, PoolName);
AddWork(client);
ListPools(client);
ListWorkItems(client);
AddWorkWithFileStaging(client);
ListPools(client);
ListWorkItems(client);
SubmitLargeNumberOfTasks(client);
}
Console.WriteLine("Press return to exit...");
Console.ReadLine();
}
public DatabaseImportManager(BlastConfiguration configuration)
{
_configuration = configuration;
_tableStorageProvider = configuration.TableStorageProvider;
_blobStorageProvider = configuration.BlobStorageProvider;
_batchClient = configuration.BatchClient;
_storageCredentials = configuration.StorageCredentials;
_batchCredentials = configuration.BatchCredentials;
}
public BatchService(string baseUrl, BatchCredentials credentials)
{
this.Client = BatchClient.Connect(baseUrl, credentials);
this.BaseUri = new Uri(baseUrl);
this.Credentials = credentials;
this.retryPolicy = new LinearRetry(TimeSpan.FromSeconds(10), 5);
this.Client.CustomBehaviors.Add(new SetRetryPolicy(this.retryPolicy));
this.Client.CustomBehaviors.Add(new RequestInterceptor((req) => { req.MaximumExecutionTime = TimeSpan.FromMinutes(2); }));
}
public BatchService(string baseUrl, BatchCredentials credentials)
{
this.Client = BatchClient.Connect(baseUrl, credentials);
this.BaseUri = new Uri(baseUrl);
this.Credentials = credentials;
this.retryPolicy = new LinearRetry(TimeSpan.FromSeconds(10), 5);
this.Client.CustomBehaviors.Add(new SetRetryPolicy(this.retryPolicy));
this.Client.CustomBehaviors.Add(new RequestInterceptor((req) => { req.MaximumExecutionTime = TimeSpan.FromMinutes(2); }));
}
public AzureAnalysisProvider(BlastConfiguration configuration, IDatabaseProvider databaseProvider)
{
_configuration = configuration;
_databaseProvider = databaseProvider;
_tableStorageProvider = configuration.TableStorageProvider;
_blobStorageProvider = configuration.BlobStorageProvider;
_batchClient = configuration.BatchClient;
_storageCredentials = configuration.StorageCredentials;
_batchCredentials = configuration.BatchCredentials;
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="dispName"></param>
/// <param name="zipFileSas"></param>
/// <param name="packagePath"></param>
/// <param name="packageVersion"></param>
/// <param name="recipient"></param>
/// <param name="batch"></param>
/// <param name="storage"></param>
/// <param name="pool"></param>
public APSIMJob(string dispName, string zipFileSas, string packagePath, string packageVersion, string recipient, BatchCredentials batch, StorageCredentials storage, PoolSettings pool)
{
DisplayName = dispName;
ModelZipFileSas = zipFileSas;
ApplicationPackagePath = packagePath;
ApsimApplicationPackageVersion = packageVersion;
Recipient = recipient;
BatchAuth = batch;
StorageAuth = storage;
PoolInfo = pool;
}
public static void Main(string[] args)
{
// This will boost parallel submission speed for REST APIs. If your use requires many simultaneous service calls set this number to something large, such as 100.
// See: http://msdn.microsoft.com/en-us/library/system.net.servicepointmanager.defaultconnectionlimit%28v=vs.110%29.aspx for more info.
ServicePointManager.DefaultConnectionLimit = 20;
if (File.Exists(CredentialFile))
{
var lines = File.ReadAllLines(CredentialFile);
if (lines.Length >= 2)
{
_batchAccount = lines[0];
_batchKey = lines[1];
}
}
// Get an instance of the BatchClient for a given Azure Batch account.
var cred = new BatchCredentials(_batchAccount, _batchKey);
using (var client = BatchClient.Connect(Url, cred))
{
// if you want to put a retry policy in place, enable it here
// the built-in policies are No Retry (default), Linear Retry, and Exponential Retry
//client.CustomBehaviors.Add(new SetRetryPolicy(new Microsoft.Azure.Batch.Protocol.LinearRetry()));
ListPools(client);
ListWorkItems(client);
CreatePoolIfNotExist(client, PoolName);
AddWork(client);
ListPools(client);
ListWorkItems(client);
Console.WriteLine(
"Remember to delete the pool if you are done. Otherwise you'll still be charged for the running VM.");
while (true)
{
Console.Write("Do you want to delete the pool? [y/N]:");
string line = Console.ReadLine();
if (string.IsNullOrEmpty(line) || 0 == string.Compare(line, "n", true, CultureInfo.InvariantCulture))
{
break;
}
if (0 == string.Compare(line, "y", true, CultureInfo.InvariantCulture))
{
DeletePool(client, PoolName);
break;
}
}
}
}
public static void Main(string[] args)
{
// This will boost parallel submission speed for REST APIs. If your use requires many simultaneous service calls set this number to something large, such as 100.
// See: http://msdn.microsoft.com/en-us/library/system.net.servicepointmanager.defaultconnectionlimit%28v=vs.110%29.aspx for more info.
ServicePointManager.DefaultConnectionLimit = 20;
if (File.Exists(CredentialFile))
{
var lines = File.ReadAllLines(CredentialFile);
if (lines.Length >= 2)
{
_batchAccount = lines[0];
_batchKey = lines[1];
}
}
// Get an instance of the BatchClient for a given Azure Batch account.
var cred = new BatchCredentials(_batchAccount, _batchKey);
using (var client = BatchClient.Connect(Url, cred))
{
// if you want to put a retry policy in place, enable it here
// the built-in policies are No Retry (default), Linear Retry, and Exponential Retry
//client.CustomBehaviors.Add(new SetRetryPolicy(new Microsoft.Azure.Batch.Protocol.LinearRetry()));
ListPools(client);
ListWorkItems(client);
CreatePoolIfNotExist(client, PoolName);
AddWork(client);
ListPools(client);
ListWorkItems(client);
Console.WriteLine(
"Remember to delete the pool if you are done. Otherwise you'll still be charged for the running VM.");
while (true)
{
Console.Write("Do you want to delete the pool? [y/N]:");
string line = Console.ReadLine();
if (string.IsNullOrEmpty(line) || 0 == string.Compare(line, "n", true, CultureInfo.InvariantCulture))
{
break;
}
if (0 == string.Compare(line, "y", true, CultureInfo.InvariantCulture))
{
DeletePool(client, PoolName);
break;
}
}
}
}
public JobManager(
BatchCredentials batchCredentials,
StorageCredentials storageCredentials,
ITaskProvider taskProvider)
{
_batchClient = BatchClient.Open(
new BatchSharedKeyCredentials(
batchCredentials.Url,
batchCredentials.Account,
batchCredentials.Key));
_batchClient.CustomBehaviors.Add(
RetryPolicyProvider.LinearRetryProvider(TimeSpan.FromSeconds(3), 10));
_taskProvider = taskProvider;
_storageAccount = new CloudStorageAccount(
new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(
storageCredentials.Account,
storageCredentials.Key),
true);
}
static void Main(string[] args)
{
// open a file for tracing
var traceFile = File.Open(SampleConstants.JMTraceFile, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite);
var traceWriter = new StreamWriter(traceFile);
traceWriter.AutoFlush = true;
try
{
// pick up hints from the environment as populated by the program run on the client machine
var workItemName = System.Environment.GetEnvironmentVariable(SampleConstants.EnvWorkItemName);
if (workItemName == null)
{
traceWriter.WriteLine("Failed to get work item name from environment");
return;
}
else
{
traceWriter.WriteLine("Our WI Name: " + workItemName);
}
BatchCredentials credentials = new BatchCredentials(
System.Environment.GetEnvironmentVariable(SampleConstants.EnvWataskAccountName), // Some basic useful elements are preset as environment variables.
System.Environment.GetEnvironmentVariable(SampleConstants.EnvBatchAccountKeyName)
);
var bClient = BatchClient.Connect(SampleConstants.BatchSvcEndpoint, credentials);
bClient.CustomBehaviors.Add(new SetRetryPolicy(new ExponentialRetry(TimeSpan.FromSeconds(5), 5)));
using (var wiMgr = bClient.OpenWorkItemManager())
{
var ourWorkItem = wiMgr.GetWorkItem(workItemName);
// Since is this a job manager task, its current job will be the most recent job. This is not
// true for non-JM tasks since the most recent job could have run a while ago.
string jobName = null;
if (ourWorkItem.ExecutionInformation != null && ourWorkItem.ExecutionInformation.RecentJob != null)
{
jobName = ourWorkItem.ExecutionInformation.RecentJob.Name;
traceWriter.WriteLine("Our job name: " + jobName);
}
else
{
traceWriter.WriteLine("Failed to get jobname from workitem - exiting");
return;
}
// Schedule tasks on the job. Ping is being used as the basic task
List <CloudTask> tasksToAdd = new List <CloudTask>();
for (int i = 0; i < 5; i++)
{
string taskName = "pingTask" + i;
var task = new CloudTask(taskName, "ping 127.0.0.1 -n 8");
traceWriter.WriteLine("Adding task: " + taskName);
tasksToAdd.Add(task);
}
wiMgr.AddTask(workItemName, jobName, tasksToAdd);
// Monitor the current jobs to see when they are done, and then exit the job manager. Monitoring the tasks
// for completion is necessary if you are using KillJobOnCompletion = TRUE, as otherwise when the job manager
// exits it will kill all of the tasks that still running.
//
// Occasionally a task may get killed and requeued during an upgrade or hardware failure, including the job manager
// task. The job manager will be re-run in this case. Robustness against this was not added into the sample for
// simplicity, but should be added into any production code.
while (true)
{
// Get list of current tasks in queue. Total count includes the job manager so when
// we get to one left then we can exit.
var taskList = wiMgr.ListTasks(workItemName, jobName);
var tasksStillRunning = taskList.Count(task => task.State != TaskState.Completed);
if (tasksStillRunning > 1)
{
traceWriter.WriteLine("{0} tasks still running", tasksStillRunning);
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(5));
}
else
{
break;
}
}
traceWriter.WriteLine("All done!");
}
}
catch (Exception e)
{
traceWriter.WriteLine("Exception:");
traceWriter.WriteLine(e.ToString());
}
finally
{
traceWriter.Flush();
traceFile.Close();
}
}
/// <summary>
/// Runs the job manager task.
/// </summary>
public async Task RunAsync()
{
Console.WriteLine("JobManager for account: {0}, work item: {1}, job: {2} has started...",
this.accountName,
this.workItemName,
this.jobName);
Console.WriteLine();
Console.WriteLine("JobManager running with the following settings: ");
Console.WriteLine("----------------------------------------");
Console.WriteLine(this.configurationSettings.ToString());
//Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchCredentials batchCredentials = new BatchCredentials(
this.configurationSettings.BatchAccountName,
this.configurationSettings.BatchAccountKey);
using (IBatchClient batchClient = BatchClient.Connect(this.configurationSettings.BatchServiceUrl, batchCredentials))
{
using (IWorkItemManager workItemManager = batchClient.OpenWorkItemManager())
{
IToolbox toolbox = batchClient.OpenToolbox();
//Construct a container SAS to provide the Batch Service access to the files required to
//run the mapper and reducer tasks.
string containerSas = Helpers.ConstructContainerSas(
this.configurationSettings.StorageAccountName,
this.configurationSettings.StorageAccountKey,
this.configurationSettings.StorageServiceUrl,
this.configurationSettings.BlobContainer);
//
// Submit mapper tasks.
//
Console.WriteLine("Submitting {0} mapper tasks.", this.configurationSettings.NumberOfMapperTasks);
//The collection of tasks to add to the Batch Service.
List <ICloudTask> tasksToAdd = new List <ICloudTask>();
for (int i = 0; i < this.configurationSettings.NumberOfMapperTasks; i++)
{
string taskName = Helpers.GetMapperTaskName(i);
string fileBlobName = Helpers.GetSplitFileName(i);
string fileBlobPath = Helpers.ConstructBlobSource(containerSas, fileBlobName);
string commandLine = string.Format("{0} -MapperTask {1}", Constants.TextSearchExe, fileBlobPath);
ICloudTask unboundMapperTask = new CloudTask(taskName, commandLine);
//The set of files (exe's, dll's and configuration files) required to run the mapper task.
IReadOnlyList <string> mapperTaskRequiredFiles = Constants.RequiredExecutableFiles;
List <IResourceFile> mapperTaskResourceFiles = Helpers.GetResourceFiles(containerSas, mapperTaskRequiredFiles);
unboundMapperTask.ResourceFiles = mapperTaskResourceFiles;
tasksToAdd.Add(unboundMapperTask);
}
//Submit the unbound task collection to the Batch Service.
//Use the AddTask method which takes a collection of ICloudTasks for the best performance.
await workItemManager.AddTaskAsync(this.workItemName, this.jobName, tasksToAdd);
//
// Wait for the mapper tasks to complete.
//
Console.WriteLine("Waiting for the mapper tasks to complete...");
//List all the mapper tasks using a name filter.
DetailLevel mapperTaskNameFilter = new ODATADetailLevel()
{
FilterClause = string.Format("startswith(name, '{0}')", Constants.MapperTaskPrefix)
};
List <ICloudTask> tasksToMonitor = workItemManager.ListTasks(
this.workItemName,
this.jobName,
detailLevel: mapperTaskNameFilter).ToList();
//Use the task state monitor to wait for the tasks to complete.
ITaskStateMonitor taskStateMonitor = toolbox.CreateTaskStateMonitor();
bool timedOut = await taskStateMonitor.WaitAllAsync(tasksToMonitor, TaskState.Completed, TimeSpan.FromMinutes(5));
//Get the list of mapper tasks in order to analyze their state and ensure they completed successfully.
IEnumerableAsyncExtended <ICloudTask> asyncEnumerable = workItemManager.ListTasks(
this.workItemName,
this.jobName,
detailLevel: mapperTaskNameFilter);
IAsyncEnumerator <ICloudTask> asyncEnumerator = asyncEnumerable.GetAsyncEnumerator();
//Dump the status of each mapper task.
while (await asyncEnumerator.MoveNextAsync())
{
ICloudTask cloudTask = asyncEnumerator.Current;
Console.WriteLine("Task {0} is in state: {1}", cloudTask.Name, cloudTask.State);
await Helpers.CheckForTaskSuccessAsync(cloudTask, dumpStandardOutOnTaskSuccess : false);
Console.WriteLine();
}
//If not all the tasks reached the desired state within the timeout then the job manager
//cannot continue.
if (timedOut)
{
const string errorMessage = "Mapper tasks did not complete within expected timeout.";
Console.WriteLine(errorMessage);
throw new TimeoutException(errorMessage);
}
//
// Create the reducer task.
//
string reducerTaskCommandLine = string.Format("{0} -ReducerTask", Constants.TextSearchExe);
Console.WriteLine("Adding the reducer task: {0}", Constants.ReducerTaskName);
ICloudTask unboundReducerTask = new CloudTask(Constants.ReducerTaskName, reducerTaskCommandLine);
//The set of files (exe's, dll's and configuration files) required to run the reducer task.
List <IResourceFile> reducerTaskResourceFiles = Helpers.GetResourceFiles(containerSas, Constants.RequiredExecutableFiles);
unboundReducerTask.ResourceFiles = reducerTaskResourceFiles;
//Send the request to the Batch Service to add the reducer task.
await workItemManager.AddTaskAsync(this.workItemName, this.jobName, unboundReducerTask);
//
//Wait for the reducer task to complete.
//
//Get the bound reducer task and monitor it for completion.
ICloudTask boundReducerTask = await workItemManager.GetTaskAsync(this.workItemName, this.jobName, Constants.ReducerTaskName);
timedOut = await taskStateMonitor.WaitAllAsync(new List <ICloudTask> {
boundReducerTask
}, TaskState.Completed, TimeSpan.FromMinutes(2));
//Refresh the reducer task to get the most recent information about it from the Batch Service.
await boundReducerTask.RefreshAsync();
//Dump the reducer tasks exit code and scheduling error for debugging purposes.
await Helpers.CheckForTaskSuccessAsync(boundReducerTask, dumpStandardOutOnTaskSuccess : true);
//Handle the possibilty that the reducer task did not complete in the expected timeout.
if (timedOut)
{
const string errorMessage = "Reducer task did not complete within expected timeout.";
Console.WriteLine("Task {0} is in state: {1}", boundReducerTask.Name, boundReducerTask.State);
Console.WriteLine(errorMessage);
throw new TimeoutException(errorMessage);
}
//The job manager has completed.
Console.WriteLine("JobManager completed successfully.");
}
}
}
/// <summary>
/// Populates Azure Storage with the required files, and
/// submits the work item to the Azure Batch service.
/// </summary>
public async Task RunAsync()
{
Console.WriteLine("Running with the following settings: ");
Console.WriteLine("----------------------------------------");
Console.WriteLine(this.configurationSettings.ToString());
//Upload resources if required.
if (this.configurationSettings.ShouldUploadResources)
{
Console.WriteLine("Splitting file: {0} into {1} subfiles",
Constants.TextFilePath,
this.configurationSettings.NumberOfMapperTasks);
//Split the text file into the correct number of files for consumption by the mapper tasks.
FileSplitter splitter = new FileSplitter();
List<string> mapperTaskFiles = await splitter.SplitAsync(
Constants.TextFilePath,
this.configurationSettings.NumberOfMapperTasks);
await this.UploadResourcesAsync(mapperTaskFiles);
}
//Generate a SAS for the container.
string containerSasUrl = Helpers.ConstructContainerSas(
this.configurationSettings.StorageAccountName,
this.configurationSettings.StorageAccountKey,
this.configurationSettings.StorageServiceUrl,
this.configurationSettings.BlobContainer);
//Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchCredentials batchCredentials = new BatchCredentials(
this.configurationSettings.BatchAccountName,
this.configurationSettings.BatchAccountKey);
using (IBatchClient batchClient = BatchClient.Connect(this.configurationSettings.BatchServiceUrl, batchCredentials))
{
using (IWorkItemManager workItemManager = batchClient.OpenWorkItemManager())
{
//Create the unbound work item in local memory. An object which exists only in local memory (and not on the Batch Service) is "unbound".
string workItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.UtcNow.ToString("yyyyMMdd-HHmmss");
ICloudWorkItem unboundWorkItem = workItemManager.CreateWorkItem(workItemName);
//
// Construct the work item properties in local memory before commiting them to the Batch Service.
//
//Allow enough VMs in the pool to run each mapper task, and 1 extra to run the job manager.
int numberOfPoolVMs = 1 + this.configurationSettings.NumberOfMapperTasks;
//Define the pool specification for the pool which the work item will run on.
IPoolSpecification poolSpecification = new PoolSpecification()
{
TargetDedicated = numberOfPoolVMs,
VMSize = "small",
//You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
OSFamily = "4",
TargetOSVersion = "*"
};
//Use the auto pool feature of the Batch Service to create a pool when the work item is created.
//This creates a new pool for each work item which is added.
IAutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification()
{
AutoPoolNamePrefix = "TextSearchPool",
KeepAlive = false,
PoolLifeTimeOption = PoolLifeTimeOption.WorkItem,
PoolSpecification = poolSpecification
};
//Define the execution environment for this work item -- it will run on the pool defined by the auto pool specification above.
unboundWorkItem.JobExecutionEnvironment = new JobExecutionEnvironment()
{
AutoPoolSpecification = autoPoolSpecification
};
//Define the job manager for this work item. This job manager will run when any job is created and will submit the tasks for
//the work item. The job manager is the executable which manages the lifetime of the job
//and all tasks which should run for the job. In this case, the job manager submits the mapper and reducer tasks.
string jobManagerCommandLine = string.Format("{0} -JobManagerTask", Constants.TextSearchExe);
List<IResourceFile> jobManagerResourceFiles = Helpers.GetResourceFiles(containerSasUrl, Constants.RequiredExecutableFiles);
const string jobManagerTaskName = "JobManager";
unboundWorkItem.JobSpecification = new JobSpecification()
{
JobManager = new JobManager()
{
ResourceFiles = jobManagerResourceFiles,
CommandLine = jobManagerCommandLine,
//Determines if the job should terminate when the job manager process exits
KillJobOnCompletion = false,
Name = jobManagerTaskName
}
};
try
{
//Commit the unbound work item to the Batch Service.
Console.WriteLine("Adding work item: {0} to the Batch Service.", unboundWorkItem.Name);
await unboundWorkItem.CommitAsync(); //Issues a request to the Batch Service to add the work item which was defined above.
//
// Wait for the job manager task to complete.
//
//An object which is backed by a corresponding Batch Service object is "bound."
ICloudWorkItem boundWorkItem = await workItemManager.GetWorkItemAsync(workItemName);
//Wait for the job to be created automatically by the Batch Service.
string boundJobName = await Helpers.WaitForActiveJobAsync(boundWorkItem);
ICloudTask boundJobManagerTask = await workItemManager.GetTaskAsync(
workItemName,
boundJobName,
jobManagerTaskName);
TimeSpan maxJobCompletionTimeout = TimeSpan.FromMinutes(30);
IToolbox toolbox = batchClient.OpenToolbox();
ITaskStateMonitor monitor = toolbox.CreateTaskStateMonitor();
bool timedOut = await monitor.WaitAllAsync(new List<ICloudTask> { boundJobManagerTask }, TaskState.Completed, maxJobCompletionTimeout);
Console.WriteLine("Done waiting for job manager task.");
await boundJobManagerTask.RefreshAsync();
//Check to ensure the job manager task exited successfully.
await Helpers.CheckForTaskSuccessAsync(boundJobManagerTask, dumpStandardOutOnTaskSuccess: true);
if (timedOut)
{
throw new TimeoutException(string.Format("Timed out waiting for job manager task to complete."));
}
}
catch (AggregateException e)
{
e.Handle(
(innerE) =>
{
//We print all the inner exceptions for debugging purposes.
Console.WriteLine(innerE.ToString());
return false;
});
throw;
}
catch (Exception e)
{
Console.WriteLine("Hit unexpected exception: {0}", e.ToString());
throw;
}
finally
{
//Delete the work item.
//This will delete the auto pool associated with the work item as long as the pool
//keep alive property is set to false.
if (this.configurationSettings.ShouldDeleteWorkItem)
{
Console.WriteLine("Deleting work item {0}", workItemName);
workItemManager.DeleteWorkItem(workItemName);
}
//Note that there were files uploaded to a container specified in the
//configuration file. This container will not be deleted or cleaned up by this sample.
}
}
}
}
static void Main(string[] args)
{
// open a file for tracing
var traceFile = File.Open(SampleConstants.JMTraceFile, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite);
var traceWriter = new StreamWriter(traceFile);
traceWriter.AutoFlush = true;
try
{
// pick up hints from the environment as populated by the program run on the client machine
var workItemName = Environment.GetEnvironmentVariable(SampleConstants.EnvWorkItemName);
if (workItemName == null)
{
traceWriter.WriteLine("Failed to get work item name from environment");
return;
}
else
{
traceWriter.WriteLine("Our WI Name: " + workItemName);
}
BatchCredentials credentials = new BatchCredentials(
Environment.GetEnvironmentVariable(SampleConstants.EnvWataskAccountName), // Some basic useful elements are preset as environment variables.
Environment.GetEnvironmentVariable(SampleConstants.EnvBatchAccountKeyName)
);
var bClient = BatchClient.Connect(SampleConstants.BatchSvcEndpoint, credentials);
bClient.CustomBehaviors.Add(new SetRetryPolicy(new ExponentialRetry(TimeSpan.FromSeconds(5), 5)));
using (var wiMgr = bClient.OpenWorkItemManager())
{
var ourWorkItem = wiMgr.GetWorkItem(workItemName);
// Since is this a job manager task, its current job will be the most recent job. This is not
// true for non-JM tasks since the most recent job could have run a while ago.
string jobName;
if (ourWorkItem.ExecutionInformation != null && ourWorkItem.ExecutionInformation.RecentJob != null)
{
jobName = ourWorkItem.ExecutionInformation.RecentJob.Name;
traceWriter.WriteLine("Our job name: " + jobName);
}
else
{
traceWriter.WriteLine("Failed to get jobname from workitem - exiting");
return;
}
// Schedule tasks on the job. Ping is being used as the basic task
List<CloudTask> tasksToAdd = new List<CloudTask>();
for (int i = 0; i < 5; i++)
{
string taskName = "pingTask" + i;
var task = new CloudTask(taskName, "ping 127.0.0.1 -n 8");
traceWriter.WriteLine("Adding task: " + taskName);
tasksToAdd.Add(task);
}
wiMgr.AddTask(workItemName, jobName, tasksToAdd);
// Monitor the current jobs to see when they are done, and then exit the job manager. Monitoring the tasks
// for completion is necessary if you are using KillJobOnCompletion = TRUE, as otherwise when the job manager
// exits it will kill all of the tasks that still running.
//
// Occasionally a task may get killed and requeued during an upgrade or hardware failure, including the job manager
// task. The job manager will be re-run in this case. Robustness against this was not added into the sample for
// simplicity, but should be added into any production code.
while (true)
{
// Get list of current tasks in queue. Total count includes the job manager so when
// we get to one left then we can exit.
var taskList = wiMgr.ListTasks(workItemName, jobName);
var tasksStillRunning = taskList.Count(task => task.State != TaskState.Completed);
if (tasksStillRunning > 1)
{
traceWriter.WriteLine("{0} tasks still running", tasksStillRunning);
Thread.Sleep(TimeSpan.FromSeconds(5));
}
else
{
break;
}
}
traceWriter.WriteLine("All done!");
}
}
catch (Exception e)
{
traceWriter.WriteLine("Exception:");
traceWriter.WriteLine(e.ToString());
}
finally
{
traceWriter.Flush();
traceFile.Close();
}
}
/// <summary>
/// Runs the job manager task.
/// </summary>
public async Task RunAsync()
{
Console.WriteLine("JobManager for account: {0}, work item: {1}, job: {2} has started...",
this.accountName,
this.workItemName,
this.jobName);
Console.WriteLine();
Console.WriteLine("JobManager running with the following settings: ");
Console.WriteLine("----------------------------------------");
Console.WriteLine(this.configurationSettings.ToString());
//Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchCredentials batchCredentials = new BatchCredentials(
this.configurationSettings.BatchAccountName,
this.configurationSettings.BatchAccountKey);
using (IBatchClient batchClient = BatchClient.Connect(this.configurationSettings.BatchServiceUrl, batchCredentials))
{
using (IWorkItemManager workItemManager = batchClient.OpenWorkItemManager())
{
IToolbox toolbox = batchClient.OpenToolbox();
//Construct a container SAS to provide the Batch Service access to the files required to
//run the mapper and reducer tasks.
string containerSas = Helpers.ConstructContainerSas(
this.configurationSettings.StorageAccountName,
this.configurationSettings.StorageAccountKey,
this.configurationSettings.StorageServiceUrl,
this.configurationSettings.BlobContainer);
//
// Submit mapper tasks.
//
Console.WriteLine("Submitting {0} mapper tasks.", this.configurationSettings.NumberOfMapperTasks);
//The collection of tasks to add to the Batch Service.
List<ICloudTask> tasksToAdd = new List<ICloudTask>();
for (int i = 0; i < this.configurationSettings.NumberOfMapperTasks; i++)
{
string taskName = Helpers.GetMapperTaskName(i);
string fileBlobName = Helpers.GetSplitFileName(i);
string fileBlobPath = Helpers.ConstructBlobSource(containerSas, fileBlobName);
string commandLine = string.Format("{0} -MapperTask {1}", Constants.TextSearchExe, fileBlobPath);
ICloudTask unboundMapperTask = new CloudTask(taskName, commandLine);
//The set of files (exe's, dll's and configuration files) required to run the mapper task.
IReadOnlyList<string> mapperTaskRequiredFiles = Constants.RequiredExecutableFiles;
List<IResourceFile> mapperTaskResourceFiles = Helpers.GetResourceFiles(containerSas, mapperTaskRequiredFiles);
unboundMapperTask.ResourceFiles = mapperTaskResourceFiles;
tasksToAdd.Add(unboundMapperTask);
}
//Submit the unbound task collection to the Batch Service.
//Use the AddTask method which takes a collection of ICloudTasks for the best performance.
await workItemManager.AddTaskAsync(this.workItemName, this.jobName, tasksToAdd);
//
// Wait for the mapper tasks to complete.
//
Console.WriteLine("Waiting for the mapper tasks to complete...");
//List all the mapper tasks using a name filter.
DetailLevel mapperTaskNameFilter = new ODATADetailLevel()
{
FilterClause = string.Format("startswith(name, '{0}')", Constants.MapperTaskPrefix)
};
List<ICloudTask> tasksToMonitor = workItemManager.ListTasks(
this.workItemName,
this.jobName,
detailLevel: mapperTaskNameFilter).ToList();
//Use the task state monitor to wait for the tasks to complete.
ITaskStateMonitor taskStateMonitor = toolbox.CreateTaskStateMonitor();
bool timedOut = await taskStateMonitor.WaitAllAsync(tasksToMonitor, TaskState.Completed, TimeSpan.FromMinutes(5));
//Get the list of mapper tasks in order to analyze their state and ensure they completed successfully.
IEnumerableAsyncExtended<ICloudTask> asyncEnumerable = workItemManager.ListTasks(
this.workItemName,
this.jobName,
detailLevel: mapperTaskNameFilter);
IAsyncEnumerator<ICloudTask> asyncEnumerator = asyncEnumerable.GetAsyncEnumerator();
//Dump the status of each mapper task.
while (await asyncEnumerator.MoveNextAsync())
{
ICloudTask cloudTask = asyncEnumerator.Current;
Console.WriteLine("Task {0} is in state: {1}", cloudTask.Name, cloudTask.State);
await Helpers.CheckForTaskSuccessAsync(cloudTask, dumpStandardOutOnTaskSuccess: false);
Console.WriteLine();
}
//If not all the tasks reached the desired state within the timeout then the job manager
//cannot continue.
if (timedOut)
{
const string errorMessage = "Mapper tasks did not complete within expected timeout.";
Console.WriteLine(errorMessage);
throw new TimeoutException(errorMessage);
}
//
// Create the reducer task.
//
string reducerTaskCommandLine = string.Format("{0} -ReducerTask", Constants.TextSearchExe);
Console.WriteLine("Adding the reducer task: {0}", Constants.ReducerTaskName);
ICloudTask unboundReducerTask = new CloudTask(Constants.ReducerTaskName, reducerTaskCommandLine);
//The set of files (exe's, dll's and configuration files) required to run the reducer task.
List<IResourceFile> reducerTaskResourceFiles = Helpers.GetResourceFiles(containerSas, Constants.RequiredExecutableFiles);
unboundReducerTask.ResourceFiles = reducerTaskResourceFiles;
//Send the request to the Batch Service to add the reducer task.
await workItemManager.AddTaskAsync(this.workItemName, this.jobName, unboundReducerTask);
//
//Wait for the reducer task to complete.
//
//Get the bound reducer task and monitor it for completion.
ICloudTask boundReducerTask = await workItemManager.GetTaskAsync(this.workItemName, this.jobName, Constants.ReducerTaskName);
timedOut = await taskStateMonitor.WaitAllAsync(new List<ICloudTask> {boundReducerTask}, TaskState.Completed, TimeSpan.FromMinutes(2));
//Refresh the reducer task to get the most recent information about it from the Batch Service.
await boundReducerTask.RefreshAsync();
//Dump the reducer tasks exit code and scheduling error for debugging purposes.
await Helpers.CheckForTaskSuccessAsync(boundReducerTask, dumpStandardOutOnTaskSuccess: true);
//Handle the possibilty that the reducer task did not complete in the expected timeout.
if (timedOut)
{
const string errorMessage = "Reducer task did not complete within expected timeout.";
Console.WriteLine("Task {0} is in state: {1}", boundReducerTask.Name, boundReducerTask.State);
Console.WriteLine(errorMessage);
throw new TimeoutException(errorMessage);
}
//The job manager has completed.
Console.WriteLine("JobManager completed successfully.");
}
}
}
/// <summary>
/// Populates Azure Storage with the required files, and
/// submits the work item to the Azure Batch service.
/// </summary>
public async Task RunAsync()
{
Console.WriteLine("Running with the following settings: ");
Console.WriteLine("----------------------------------------");
Console.WriteLine(this.configurationSettings.ToString());
//Upload resources if required.
if (this.configurationSettings.ShouldUploadResources)
{
Console.WriteLine("Splitting file: {0} into {1} subfiles",
Constants.TextFilePath,
this.configurationSettings.NumberOfMapperTasks);
//Split the text file into the correct number of files for consumption by the mapper tasks.
FileSplitter splitter = new FileSplitter();
List <string> mapperTaskFiles = await splitter.SplitAsync(
Constants.TextFilePath,
this.configurationSettings.NumberOfMapperTasks);
await this.UploadResourcesAsync(mapperTaskFiles);
}
//Generate a SAS for the container.
string containerSasUrl = Helpers.ConstructContainerSas(
this.configurationSettings.StorageAccountName,
this.configurationSettings.StorageAccountKey,
this.configurationSettings.StorageServiceUrl,
this.configurationSettings.BlobContainer);
//Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchCredentials batchCredentials = new BatchCredentials(
this.configurationSettings.BatchAccountName,
this.configurationSettings.BatchAccountKey);
using (IBatchClient batchClient = BatchClient.Connect(this.configurationSettings.BatchServiceUrl, batchCredentials))
{
using (IWorkItemManager workItemManager = batchClient.OpenWorkItemManager())
{
//Create the unbound work item in local memory. An object which exists only in local memory (and not on the Batch Service) is "unbound".
string workItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.UtcNow.ToString("yyyyMMdd-HHmmss");
ICloudWorkItem unboundWorkItem = workItemManager.CreateWorkItem(workItemName);
//
// Construct the work item properties in local memory before commiting them to the Batch Service.
//
//Allow enough VMs in the pool to run each mapper task, and 1 extra to run the job manager.
int numberOfPoolVMs = 1 + this.configurationSettings.NumberOfMapperTasks;
//Define the pool specification for the pool which the work item will run on.
IPoolSpecification poolSpecification = new PoolSpecification()
{
TargetDedicated = numberOfPoolVMs,
VMSize = "small",
//You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
OSFamily = "4",
TargetOSVersion = "*"
};
//Use the auto pool feature of the Batch Service to create a pool when the work item is created.
//This creates a new pool for each work item which is added.
IAutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification()
{
AutoPoolNamePrefix = "TextSearchPool",
KeepAlive = false,
PoolLifeTimeOption = PoolLifeTimeOption.WorkItem,
PoolSpecification = poolSpecification
};
//Define the execution environment for this work item -- it will run on the pool defined by the auto pool specification above.
unboundWorkItem.JobExecutionEnvironment = new JobExecutionEnvironment()
{
AutoPoolSpecification = autoPoolSpecification
};
//Define the job manager for this work item. This job manager will run when any job is created and will submit the tasks for
//the work item. The job manager is the executable which manages the lifetime of the job
//and all tasks which should run for the job. In this case, the job manager submits the mapper and reducer tasks.
string jobManagerCommandLine = string.Format("{0} -JobManagerTask", Constants.TextSearchExe);
List <IResourceFile> jobManagerResourceFiles = Helpers.GetResourceFiles(containerSasUrl, Constants.RequiredExecutableFiles);
const string jobManagerTaskName = "JobManager";
unboundWorkItem.JobSpecification = new JobSpecification()
{
JobManager = new JobManager()
{
ResourceFiles = jobManagerResourceFiles,
CommandLine = jobManagerCommandLine,
//Determines if the job should terminate when the job manager process exits
KillJobOnCompletion = false,
Name = jobManagerTaskName
}
};
try
{
//Commit the unbound work item to the Batch Service.
Console.WriteLine("Adding work item: {0} to the Batch Service.", unboundWorkItem.Name);
await unboundWorkItem.CommitAsync(); //Issues a request to the Batch Service to add the work item which was defined above.
//
// Wait for the job manager task to complete.
//
//An object which is backed by a corresponding Batch Service object is "bound."
ICloudWorkItem boundWorkItem = await workItemManager.GetWorkItemAsync(workItemName);
//Wait for the job to be created automatically by the Batch Service.
string boundJobName = await Helpers.WaitForActiveJobAsync(boundWorkItem);
ICloudTask boundJobManagerTask = await workItemManager.GetTaskAsync(
workItemName,
boundJobName,
jobManagerTaskName);
TimeSpan maxJobCompletionTimeout = TimeSpan.FromMinutes(30);
IToolbox toolbox = batchClient.OpenToolbox();
ITaskStateMonitor monitor = toolbox.CreateTaskStateMonitor();
bool timedOut = await monitor.WaitAllAsync(new List <ICloudTask> {
boundJobManagerTask
}, TaskState.Completed, maxJobCompletionTimeout);
Console.WriteLine("Done waiting for job manager task.");
await boundJobManagerTask.RefreshAsync();
//Check to ensure the job manager task exited successfully.
await Helpers.CheckForTaskSuccessAsync(boundJobManagerTask, dumpStandardOutOnTaskSuccess : true);
if (timedOut)
{
throw new TimeoutException(string.Format("Timed out waiting for job manager task to complete."));
}
}
catch (AggregateException e)
{
e.Handle(
(innerE) =>
{
//We print all the inner exceptions for debugging purposes.
Console.WriteLine(innerE.ToString());
return(false);
});
throw;
}
catch (Exception e)
{
Console.WriteLine("Hit unexpected exception: {0}", e.ToString());
throw;
}
finally
{
//Delete the work item.
//This will delete the auto pool associated with the work item as long as the pool
//keep alive property is set to false.
if (this.configurationSettings.ShouldDeleteWorkItem)
{
Console.WriteLine("Deleting work item {0}", workItemName);
workItemManager.DeleteWorkItem(workItemName);
}
//Note that there were files uploaded to a container specified in the
//configuration file. This container will not be deleted or cleaned up by this sample.
}
}
}
}
/// <summary>
/// This will take the basic data provided about the account, upload the necessary information to the account, and schedule a job.
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void btnStart_Click(object sender, RoutedEventArgs e)
{
BatchCredentials credentials = new BatchCredentials(txtBAccountName.Text, txtBAccountKey.Text);
IBatchClient bClient = BatchClient.Connect(SampleConstants.BatchSvcEndpoint, credentials);
// Setting a retry policy adds robustness against an individual call timing out. When using a policy, by default all recoverable failures are retried.
bClient.CustomBehaviors.Add(new SetRetryPolicy(new ExponentialRetry(TimeSpan.FromSeconds(5), 5)));
// Create a unique workitem name; don't forget to delete these when you're done
string workItemName = SampleConstants.WorkItemNamePrefix + Guid.NewGuid().ToString();
// Identify the pre-existing pool of VMs that will run the tasks. An Autopool specification
// is fine but there is the delay associated with the creation of the pool along with waiting
// for the VMs to reach Idle state before tasks are running. You can use Batch Explorer to
// pre-create the pool and then resize it to the desired size and number of VMs.
JobExecutionEnvironment jee = new JobExecutionEnvironment()
{
PoolName = PoolName
};
// Next, create the JobManager instance describing the environment settings and resources it
// needs to run
JobManager jobMgr = new JobManager()
{
Name = "JM1",
CommandLine = SampleConstants.JobManager,
// NOTE: We do not in general recommend that customers put their secrets on the command line or as environmental variables, as
// these are not a secure locations. This was done for the simplicity of the sample.
EnvironmentSettings = new List <IEnvironmentSetting>()
{
{ new EnvironmentSetting(SampleConstants.EnvWorkItemName, workItemName) },
{ new EnvironmentSetting(SampleConstants.EnvBatchAccountKeyName, txtBAccountKey.Text) }
},
// In many cases you will want KillJobOnCompletion to be set to 'TRUE' - this allows the previous job to finish before
// a recurrence is scheduled. As an alternative, you can set this to 'FALSE' and use MaxWallClockTime as shown below,
// which will instead ensure that every recurrence happens.
KillJobOnCompletion = true
};
// Create a list of resource files that are needed to run JobManager.exe. A shared access signature key specifying
// readonly access is used so the JobManager program will have access to the resource files when it is started
// on a VM.
var sasPrefix = Helpers.ConstructContainerSas(
txtSAccountName.Text,
txtSAccountKey.Text,
"core.windows.net",
txtSContainerName.Text);
jobMgr.ResourceFiles = Helpers.GetResourceFiles(sasPrefix, SampleConstants.JobManagerFiles);
// Create the job specification, identifying that this job has a job manager associated with it
JobSpecification jobSpec = new JobSpecification()
{
JobManager = jobMgr
};
// Set up the desired recurrence or start time schedule.
WorkItemSchedule wiSchedule = new WorkItemSchedule();
if (rdoOnce.IsChecked == true)
{
// Set information if the task is to be run once.
DateTime runOnce = (DateTime)(dpkDate.SelectedDate);
runOnce = runOnce.AddHours(cbxHourO.SelectedIndex);
runOnce = runOnce.AddMinutes(cbxMinuteO.SelectedIndex);
wiSchedule.DoNotRunUntil = runOnce;
}
else
{
// Set information if the task is to be recurring.
TimeSpan recurring = new TimeSpan(cbxHourR.SelectedIndex, cbxMinuteR.SelectedIndex, 0);
wiSchedule.RecurrenceInterval = recurring;
TimeSpan countback = new TimeSpan(0, 0, 30);
jobSpec.JobConstraints = new JobConstraints(recurring.Subtract(countback), null);
}
// Upload files and create workitem.
UploadFiles();
using (IWorkItemManager wiMgr = bClient.OpenWorkItemManager())
{
ICloudWorkItem workItem = wiMgr.CreateWorkItem(workItemName);
workItem.JobExecutionEnvironment = jee;
workItem.Schedule = wiSchedule;
workItem.JobSpecification = jobSpec;
try
{
workItem.Commit();
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString());
}
}
// Remember to clean up your workitems and jobs
}
/// <summary>
/// This will take the basic data provided about the account, upload the necessary information to the account, and schedule a job.
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void btnStart_Click(object sender, RoutedEventArgs e)
{
BatchCredentials credentials = new BatchCredentials(txtBAccountName.Text, txtBAccountKey.Text);
IBatchClient bClient = BatchClient.Connect(SampleConstants.BatchSvcEndpoint, credentials);
// Setting a retry policy adds robustness against an individual call timing out. When using a policy, by default all recoverable failures are retried.
bClient.CustomBehaviors.Add(new SetRetryPolicy(new ExponentialRetry(TimeSpan.FromSeconds(5), 5)));
// Create a unique workitem name; don't forget to delete these when you're done
string workItemName = SampleConstants.WorkItemNamePrefix + Guid.NewGuid().ToString();
// Identify the pre-existing pool of VMs that will run the tasks. An Autopool specification
// is fine but there is the delay associated with the creation of the pool along with waiting
// for the VMs to reach Idle state before tasks are running. You can use Batch Explorer to
// pre-create the pool and then resize it to the desired size and number of VMs.
JobExecutionEnvironment jee = new JobExecutionEnvironment()
{
PoolName = PoolName
};
// Next, create the JobManager instance describing the environment settings and resources it
// needs to run
JobManager jobMgr = new JobManager()
{
Name = "JM1",
CommandLine = SampleConstants.JobManager,
// NOTE: We do not in general recommend that customers put their secrets on the command line or as environmental variables, as
// these are not a secure locations. This was done for the simplicity of the sample.
EnvironmentSettings = new List<IEnvironmentSetting>() {
{ new EnvironmentSetting( SampleConstants.EnvWorkItemName, workItemName ) },
{ new EnvironmentSetting( SampleConstants.EnvBatchAccountKeyName, txtBAccountKey.Text) }
},
// In many cases you will want KillJobOnCompletion to be set to 'TRUE' - this allows the previous job to finish before
// a recurrence is scheduled. As an alternative, you can set this to 'FALSE' and use MaxWallClockTime as shown below,
// which will instead ensure that every recurrence happens.
KillJobOnCompletion = true
};
// Create a list of resource files that are needed to run JobManager.exe. A shared access signature key specifying
// readonly access is used so the JobManager program will have access to the resource files when it is started
// on a VM.
var sasPrefix = Helpers.ConstructContainerSas(
txtSAccountName.Text,
txtSAccountKey.Text,
"core.windows.net",
txtSContainerName.Text);
jobMgr.ResourceFiles = Helpers.GetResourceFiles(sasPrefix, SampleConstants.JobManagerFiles);
// Create the job specification, identifying that this job has a job manager associated with it
JobSpecification jobSpec = new JobSpecification()
{
JobManager = jobMgr
};
// Set up the desired recurrence or start time schedule.
WorkItemSchedule wiSchedule = new WorkItemSchedule();
if (rdoOnce.IsChecked == true)
{
// Set information if the task is to be run once.
if (dpkDate.SelectedDate != null)
{
DateTime runOnce = (DateTime)(dpkDate.SelectedDate);
runOnce = runOnce.AddHours(cbxHourO.SelectedIndex);
runOnce = runOnce.AddMinutes(cbxMinuteO.SelectedIndex);
wiSchedule.DoNotRunUntil = runOnce;
}
}
else
{
// Set information if the task is to be recurring.
TimeSpan recurring = new TimeSpan(cbxHourR.SelectedIndex, cbxMinuteR.SelectedIndex, 0);
wiSchedule.RecurrenceInterval = recurring;
TimeSpan countback = new TimeSpan(0, 0, 30);
jobSpec.JobConstraints = new JobConstraints(recurring.Subtract(countback), null);
}
// Upload files and create workitem.
UploadFiles();
using (IWorkItemManager wiMgr = bClient.OpenWorkItemManager())
{
ICloudWorkItem workItem = wiMgr.CreateWorkItem(workItemName);
workItem.JobExecutionEnvironment = jee;
workItem.Schedule = wiSchedule;
workItem.JobSpecification = jobSpec;
try
{
workItem.Commit();
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString());
}
}
// Remember to clean up your workitems and jobs
}
