private void NewWorkItem(IIncomingEmailMessage message) { var workItemUpdates = new Dictionary <string, string>(); InitWorkItemFields(message, workItemUpdates); var workItemId = _workItemManager.CreateWorkItem(workItemUpdates); Logger.InfoFormat("Added new work item {0} for message with subject: {1} (conversation index:{2})", workItemId, message.Subject, message.ConversationId); try { // Since the work item *has* been created, failures in this stage are not treated as critical var overrides = new OverridesExtractor(_config).GetOverrides(message); TryApplyFieldOverrides(overrides, workItemId); ProcessAttachments(message, workItemId); if (_config.WorkItemSettings.AttachOriginalMessage) { AttachMessageToWorkItem(message, workItemId, "OriginalMessage"); } } catch (Exception ex) { Logger.ErrorFormat("Exception caught while applying settings to work item {0}\n{1}", workItemId, ex); } var workItem = _workItemManager.GetWorkItemFields(workItemId); _ackEmailHandler.SendAckEmail(message, workItem); }
private void NewWorkItem(IIncomingEmailMessage message) { var workItemUpdates = new Dictionary <string, string>(); InitWorkItemFields(message, workItemUpdates); var workItemId = _workItemManager.CreateWorkItem(workItemUpdates); Logger.InfoFormat("Added new work item {0} for message with subject: {1} (conversation index:{2})", workItemId, message.Subject, message.ConversationIndex); try { // Since the work item *has* been created, failures in this stage are not treated as critical TryApplyFieldOverrides(message, workItemId); ProcessAttachments(message, workItemId); if (_config.WorkItemSettings.AttachOriginalMessage) { string originalMessageFile = message.SaveToFile(); _workItemManager.AttachFiles(workItemId, new List <string> { originalMessageFile }); } } catch (Exception ex) { Logger.ErrorFormat("Exception caught while applying settings to work item {0}\n{1}", workItemId, ex); } _ackEmailHandler.SendAckEmail(message, workItemId.ToString(CultureInfo.InvariantCulture)); }
private void NewWorkItem(IIncomingEmailMessage message) { var workItemUpdates = new Dictionary <string, string>(); InitWorkItemFields(message, workItemUpdates); var overrides = new OverridesExtractor(_config).GetOverrides(message); var workItemId = _workItemManager.CreateWorkItem(workItemUpdates, overrides); Logger.InfoFormat("Added new work item {0} for message with subject: {1} (conversation index:{2})", workItemId, message.Subject, message.ConversationIndex); if (_config.WorkItemSettings.AttachOriginalMessage) { try { ProcessAttachments(message, workItemId); string originalMessageFile = message.SaveToFile(); _workItemManager.AttachFiles(workItemId, new List <string> { originalMessageFile }); } catch (Exception ex) { Logger.ErrorFormat("Exception caught while applying settings to work item {0}\n{1}", workItemId, ex); } } _ackEmailHandler.SendAckEmail(message, workItemId.ToString(CultureInfo.InvariantCulture)); }
/// <summary> /// Builds a PSCloudWorkItem for testing /// </summary> public static PSCloudWorkItem CreatePSCloudWorkItem() { BatchAccountContext context = CreateBatchContextWithKeys(); using (IWorkItemManager wiManager = context.BatchOMClient.OpenWorkItemManager()) { ICloudWorkItem workItem = wiManager.CreateWorkItem("testWorkItem"); return(new PSCloudWorkItem(workItem)); } }
/// <summary> /// Creates a new WorkItem /// </summary> /// <param name="parameters">The parameters to use when creating the WorkItem</param> public void CreateWorkItem(NewWorkItemParameters parameters) { if (parameters == null) { throw new ArgumentNullException("parameters"); } if (string.IsNullOrWhiteSpace(parameters.WorkItemName)) { throw new ArgumentNullException("WorkItemName"); } using (IWorkItemManager wiManager = parameters.Context.BatchOMClient.OpenWorkItemManager()) { ICloudWorkItem workItem = wiManager.CreateWorkItem(parameters.WorkItemName); if (parameters.Schedule != null) { workItem.Schedule = parameters.Schedule.omObject; } if (parameters.JobSpecification != null) { Utils.Utils.JobSpecificationSyncCollections(parameters.JobSpecification); workItem.JobSpecification = parameters.JobSpecification.omObject; } if (parameters.JobExecutionEnvironment != null) { Utils.Utils.JobExecutionEnvironmentSyncCollections(parameters.JobExecutionEnvironment); workItem.JobExecutionEnvironment = parameters.JobExecutionEnvironment.omObject; } if (parameters.Metadata != null) { workItem.Metadata = new List <IMetadataItem>(); foreach (DictionaryEntry d in parameters.Metadata) { MetadataItem metadata = new MetadataItem(d.Key.ToString(), d.Value.ToString()); workItem.Metadata.Add(metadata); } } WriteVerbose(string.Format(Resources.NBWI_CreatingWorkItem, parameters.WorkItemName)); workItem.Commit(parameters.AdditionalBehaviors); } }
/// <summary> /// Creates a test WorkItem for use in Scenario tests. /// TODO: Replace with new WorkItem client method when it exists. /// </summary> public static void CreateTestWorkItem(BatchAccountContext context, string workItemName, TimeSpan?recurrenceInterval) { if (HttpMockServer.Mode == HttpRecorderMode.Record) { using (IWorkItemManager wiManager = context.BatchOMClient.OpenWorkItemManager()) { ICloudWorkItem workItem = wiManager.CreateWorkItem(workItemName); workItem.JobExecutionEnvironment = new Azure.Batch.JobExecutionEnvironment(); workItem.JobExecutionEnvironment.PoolName = "testPool"; if (recurrenceInterval != null) { workItem.Schedule = new Azure.Batch.WorkItemSchedule(); workItem.Schedule.RecurrenceInterval = recurrenceInterval; } workItem.Commit(); } } }
/// <summary> /// Populates Azure Storage with the required files, and /// submits the work item to the Azure Batch service. /// </summary> public async Task RunAsync() { Console.WriteLine("Running with the following settings: "); Console.WriteLine("----------------------------------------"); Console.WriteLine(this.configurationSettings.ToString()); //Upload resources if required. if (this.configurationSettings.ShouldUploadResources) { Console.WriteLine("Splitting file: {0} into {1} subfiles", Constants.TextFilePath, this.configurationSettings.NumberOfMapperTasks); //Split the text file into the correct number of files for consumption by the mapper tasks. FileSplitter splitter = new FileSplitter(); List <string> mapperTaskFiles = await splitter.SplitAsync( Constants.TextFilePath, this.configurationSettings.NumberOfMapperTasks); await this.UploadResourcesAsync(mapperTaskFiles); } //Generate a SAS for the container. string containerSasUrl = Helpers.ConstructContainerSas( this.configurationSettings.StorageAccountName, this.configurationSettings.StorageAccountKey, this.configurationSettings.StorageServiceUrl, this.configurationSettings.BlobContainer); //Set up the Batch Service credentials used to authenticate with the Batch Service. BatchCredentials batchCredentials = new BatchCredentials( this.configurationSettings.BatchAccountName, this.configurationSettings.BatchAccountKey); using (IBatchClient batchClient = BatchClient.Connect(this.configurationSettings.BatchServiceUrl, batchCredentials)) { using (IWorkItemManager workItemManager = batchClient.OpenWorkItemManager()) { //Create the unbound work item in local memory. An object which exists only in local memory (and not on the Batch Service) is "unbound". string workItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.UtcNow.ToString("yyyyMMdd-HHmmss"); ICloudWorkItem unboundWorkItem = workItemManager.CreateWorkItem(workItemName); // // Construct the work item properties in local memory before commiting them to the Batch Service. // //Allow enough VMs in the pool to run each mapper task, and 1 extra to run the job manager. int numberOfPoolVMs = 1 + this.configurationSettings.NumberOfMapperTasks; //Define the pool specification for the pool which the work item will run on. IPoolSpecification poolSpecification = new PoolSpecification() { TargetDedicated = numberOfPoolVMs, VMSize = "small", //You can learn more about os families and versions at: //http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx OSFamily = "4", TargetOSVersion = "*" }; //Use the auto pool feature of the Batch Service to create a pool when the work item is created. //This creates a new pool for each work item which is added. IAutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification() { AutoPoolNamePrefix = "TextSearchPool", KeepAlive = false, PoolLifeTimeOption = PoolLifeTimeOption.WorkItem, PoolSpecification = poolSpecification }; //Define the execution environment for this work item -- it will run on the pool defined by the auto pool specification above. unboundWorkItem.JobExecutionEnvironment = new JobExecutionEnvironment() { AutoPoolSpecification = autoPoolSpecification }; //Define the job manager for this work item. This job manager will run when any job is created and will submit the tasks for //the work item. The job manager is the executable which manages the lifetime of the job //and all tasks which should run for the job. In this case, the job manager submits the mapper and reducer tasks. string jobManagerCommandLine = string.Format("{0} -JobManagerTask", Constants.TextSearchExe); List <IResourceFile> jobManagerResourceFiles = Helpers.GetResourceFiles(containerSasUrl, Constants.RequiredExecutableFiles); const string jobManagerTaskName = "JobManager"; unboundWorkItem.JobSpecification = new JobSpecification() { JobManager = new JobManager() { ResourceFiles = jobManagerResourceFiles, CommandLine = jobManagerCommandLine, //Determines if the job should terminate when the job manager process exits KillJobOnCompletion = false, Name = jobManagerTaskName } }; try { //Commit the unbound work item to the Batch Service. Console.WriteLine("Adding work item: {0} to the Batch Service.", unboundWorkItem.Name); await unboundWorkItem.CommitAsync(); //Issues a request to the Batch Service to add the work item which was defined above. // // Wait for the job manager task to complete. // //An object which is backed by a corresponding Batch Service object is "bound." ICloudWorkItem boundWorkItem = await workItemManager.GetWorkItemAsync(workItemName); //Wait for the job to be created automatically by the Batch Service. string boundJobName = await Helpers.WaitForActiveJobAsync(boundWorkItem); ICloudTask boundJobManagerTask = await workItemManager.GetTaskAsync( workItemName, boundJobName, jobManagerTaskName); TimeSpan maxJobCompletionTimeout = TimeSpan.FromMinutes(30); IToolbox toolbox = batchClient.OpenToolbox(); ITaskStateMonitor monitor = toolbox.CreateTaskStateMonitor(); bool timedOut = await monitor.WaitAllAsync(new List <ICloudTask> { boundJobManagerTask }, TaskState.Completed, maxJobCompletionTimeout); Console.WriteLine("Done waiting for job manager task."); await boundJobManagerTask.RefreshAsync(); //Check to ensure the job manager task exited successfully. await Helpers.CheckForTaskSuccessAsync(boundJobManagerTask, dumpStandardOutOnTaskSuccess : true); if (timedOut) { throw new TimeoutException(string.Format("Timed out waiting for job manager task to complete.")); } } catch (AggregateException e) { e.Handle( (innerE) => { //We print all the inner exceptions for debugging purposes. Console.WriteLine(innerE.ToString()); return(false); }); throw; } catch (Exception e) { Console.WriteLine("Hit unexpected exception: {0}", e.ToString()); throw; } finally { //Delete the work item. //This will delete the auto pool associated with the work item as long as the pool //keep alive property is set to false. if (this.configurationSettings.ShouldDeleteWorkItem) { Console.WriteLine("Deleting work item {0}", workItemName); workItemManager.DeleteWorkItem(workItemName); } //Note that there were files uploaded to a container specified in the //configuration file. This container will not be deleted or cleaned up by this sample. } } } }
/// <summary> /// This will take the basic data provided about the account, upload the necessary information to the account, and schedule a job. /// </summary> /// <param name="sender"></param> /// <param name="e"></param> private void btnStart_Click(object sender, RoutedEventArgs e) { BatchCredentials credentials = new BatchCredentials(txtBAccountName.Text, txtBAccountKey.Text); IBatchClient bClient = BatchClient.Connect(SampleConstants.BatchSvcEndpoint, credentials); // Setting a retry policy adds robustness against an individual call timing out. When using a policy, by default all recoverable failures are retried. bClient.CustomBehaviors.Add(new SetRetryPolicy(new ExponentialRetry(TimeSpan.FromSeconds(5), 5))); // Create a unique workitem name; don't forget to delete these when you're done string workItemName = SampleConstants.WorkItemNamePrefix + Guid.NewGuid().ToString(); // Identify the pre-existing pool of VMs that will run the tasks. An Autopool specification // is fine but there is the delay associated with the creation of the pool along with waiting // for the VMs to reach Idle state before tasks are running. You can use Batch Explorer to // pre-create the pool and then resize it to the desired size and number of VMs. JobExecutionEnvironment jee = new JobExecutionEnvironment() { PoolName = PoolName }; // Next, create the JobManager instance describing the environment settings and resources it // needs to run JobManager jobMgr = new JobManager() { Name = "JM1", CommandLine = SampleConstants.JobManager, // NOTE: We do not in general recommend that customers put their secrets on the command line or as environmental variables, as // these are not a secure locations. This was done for the simplicity of the sample. EnvironmentSettings = new List <IEnvironmentSetting>() { { new EnvironmentSetting(SampleConstants.EnvWorkItemName, workItemName) }, { new EnvironmentSetting(SampleConstants.EnvBatchAccountKeyName, txtBAccountKey.Text) } }, // In many cases you will want KillJobOnCompletion to be set to 'TRUE' - this allows the previous job to finish before // a recurrence is scheduled. As an alternative, you can set this to 'FALSE' and use MaxWallClockTime as shown below, // which will instead ensure that every recurrence happens. KillJobOnCompletion = true }; // Create a list of resource files that are needed to run JobManager.exe. A shared access signature key specifying // readonly access is used so the JobManager program will have access to the resource files when it is started // on a VM. var sasPrefix = Helpers.ConstructContainerSas( txtSAccountName.Text, txtSAccountKey.Text, "core.windows.net", txtSContainerName.Text); jobMgr.ResourceFiles = Helpers.GetResourceFiles(sasPrefix, SampleConstants.JobManagerFiles); // Create the job specification, identifying that this job has a job manager associated with it JobSpecification jobSpec = new JobSpecification() { JobManager = jobMgr }; // Set up the desired recurrence or start time schedule. WorkItemSchedule wiSchedule = new WorkItemSchedule(); if (rdoOnce.IsChecked == true) { // Set information if the task is to be run once. DateTime runOnce = (DateTime)(dpkDate.SelectedDate); runOnce = runOnce.AddHours(cbxHourO.SelectedIndex); runOnce = runOnce.AddMinutes(cbxMinuteO.SelectedIndex); wiSchedule.DoNotRunUntil = runOnce; } else { // Set information if the task is to be recurring. TimeSpan recurring = new TimeSpan(cbxHourR.SelectedIndex, cbxMinuteR.SelectedIndex, 0); wiSchedule.RecurrenceInterval = recurring; TimeSpan countback = new TimeSpan(0, 0, 30); jobSpec.JobConstraints = new JobConstraints(recurring.Subtract(countback), null); } // Upload files and create workitem. UploadFiles(); using (IWorkItemManager wiMgr = bClient.OpenWorkItemManager()) { ICloudWorkItem workItem = wiMgr.CreateWorkItem(workItemName); workItem.JobExecutionEnvironment = jee; workItem.Schedule = wiSchedule; workItem.JobSpecification = jobSpec; try { workItem.Commit(); } catch (Exception ex) { MessageBox.Show(ex.ToString()); } } // Remember to clean up your workitems and jobs }
/// <summary> /// Submit a large number of tasks to the Batch Service. /// </summary> /// <param name="client">The batch client.</param> private static void SubmitLargeNumberOfTasks(IBatchClient client) { const int taskCountToCreate = 5000; // In order to simulate a "large" task object which has many properties set (such as resource files, environment variables, etc) // we create a big environment variable so we have a big task object. char[] env = new char[2048]; for (int i = 0; i < env.Length; i++) { env[i] = 'a'; } string envStr = new string(env); using (IWorkItemManager wm = client.OpenWorkItemManager()) { //Create a work item string workItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss"); Console.WriteLine("Creating work item {0}", workItemName); ICloudWorkItem cloudWorkItem = wm.CreateWorkItem(workItemName); cloudWorkItem.JobExecutionEnvironment = new JobExecutionEnvironment() { PoolName = PoolName }; //Specify the pool to run on cloudWorkItem.Commit(); //Wait for an active job TimeSpan maxJobCreationTimeout = TimeSpan.FromSeconds(90); DateTime jobCreationStartTime = DateTime.Now; DateTime jobCreationTimeoutTime = jobCreationStartTime.Add(maxJobCreationTimeout); cloudWorkItem = wm.GetWorkItem(workItemName); Console.WriteLine("Waiting for a job to become active..."); while (cloudWorkItem.ExecutionInformation == null || cloudWorkItem.ExecutionInformation.RecentJob == null) { cloudWorkItem.Refresh(); if (DateTime.Now > jobCreationTimeoutTime) { throw new Exception("Timed out waiting for job."); } Thread.Sleep(TimeSpan.FromSeconds(5)); } string jobName = cloudWorkItem.ExecutionInformation.RecentJob.Name; Console.WriteLine("Found job {0}. Adding task objects...", jobName); //Generate a large number of tasks to submit List <ICloudTask> tasksToSubmit = new List <ICloudTask>(); for (int i = 0; i < taskCountToCreate; i++) { ICloudTask task = new CloudTask("echo" + i.ToString("D5"), "echo"); List <IEnvironmentSetting> environmentSettings = new List <IEnvironmentSetting>(); environmentSettings.Add(new EnvironmentSetting("envone", envStr)); task.EnvironmentSettings = environmentSettings; tasksToSubmit.Add(task); } BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions() { //This will result in at most 10 simultaneous Bulk Add requests to the Batch Service. MaxDegreeOfParallelism = 10 }; Console.WriteLine("Submitting {0} tasks to work item: {1}, job: {2}, on pool: {3}", taskCountToCreate, cloudWorkItem.Name, jobName, cloudWorkItem.JobExecutionEnvironment.PoolName); Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); //Use the AddTask overload which supports a list of tasks for best AddTask performence - internally this method performs an //intelligent submission of tasks in batches in order to limit the number of REST API calls made to the Batch Service. wm.AddTask(cloudWorkItem.Name, jobName, tasksToSubmit, parallelOptions); stopwatch.Stop(); Console.WriteLine("Submitted {0} tasks in {1}", taskCountToCreate, stopwatch.Elapsed); //Delete the work item to ensure the tasks are cleaned up wm.DeleteWorkItem(workItemName); } }
/// <summary> /// Creates a work item with the specified work item options. /// </summary> /// <param name="options">The options describing the work item to create.</param> /// <returns></returns> public async Task CreateWorkItemAsync(CreateWorkItemOptions options) { try { using (IWorkItemManager workItemManager = this.Client.OpenWorkItemManager()) { ICloudWorkItem unboundWorkItem = workItemManager.CreateWorkItem(options.WorkItemName); IJobExecutionEnvironment jobExecutionEnvironment = new JobExecutionEnvironment(); if (options.UseAutoPool.HasValue && options.UseAutoPool.Value) { IAutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification() { AutoPoolNamePrefix = options.AutoPoolPrefix, KeepAlive = options.KeepAlive, PoolLifeTimeOption = options.LifeTimeOption.Equals("Job", StringComparison.OrdinalIgnoreCase) ? PoolLifeTimeOption.Job : PoolLifeTimeOption.WorkItem }; jobExecutionEnvironment.AutoPoolSpecification = autoPoolSpecification; } else { jobExecutionEnvironment.PoolName = options.PoolName; } unboundWorkItem.JobExecutionEnvironment = jobExecutionEnvironment; unboundWorkItem.JobSpecification = new JobSpecification() { Priority = options.Priority }; // TODO: These are read only unboundWorkItem.JobSpecification.JobConstraints = new JobConstraints(options.MaxWallClockTime, options.MaxRetryCount); if (options.CreateSchedule.HasValue && options.CreateSchedule.Value == true) { IWorkItemSchedule schedule = new WorkItemSchedule() { DoNotRunAfter = options.DoNotRunAfter, DoNotRunUntil = options.DoNotRunUntil, RecurrenceInterval = options.RecurrenceInterval, StartWindow = options.StartWindow }; unboundWorkItem.Schedule = schedule; } if (options.CreateJobManager.HasValue && options.CreateJobManager.Value == true) { IJobManager jobManager = new JobManager() { CommandLine = options.CommandLine, KillJobOnCompletion = options.KillOnCompletion, Name = options.JobManagerName }; jobManager.TaskConstraints = new TaskConstraints(options.MaxTaskWallClockTime, options.RetentionTime, options.MaxTaskRetryCount); unboundWorkItem.JobSpecification.JobManager = jobManager; } await unboundWorkItem.CommitAsync(); } } catch { throw; } }