private async System.Threading.Tasks.Task <List <JobModel> > ListJobsAsync(int min, int max) { List <JobModel> results = new List <JobModel>(); string uppperBound = string.Format("job-{0:D10}", max); string lowerBound = string.Format("job-{0:D10}", min); // TODO: Need to use the detail level to limit amount of data we download from the server // TODO: and increase the performace of listing jobs in the UI //var detailLevel = new ODATADetailLevel() { SelectClause = "name,state,creationTime" }; //if (string.IsNullOrEmpty(detailLevel.FilterClause)) //{ // detailLevel.FilterClause = string.Empty; //} //detailLevel.FilterClause += string.Format("(jobName le '{0}' and jobName gt '{1}')", uppperBound, lowerBound); //IEnumerableAsyncExtended<ICloudJob> jobList = this.WorkItem.ListJobs(detailLevel); // END TODO IEnumerableAsyncExtended <ICloudJob> jobList = this.WorkItem.ListJobs(); IAsyncEnumerator <ICloudJob> asyncEnumerator = jobList.GetAsyncEnumerator(); while (await asyncEnumerator.MoveNextAsync()) { results.Add(new JobModel(this, asyncEnumerator.Current)); } return(results); }
private async System.Threading.Tasks.Task <List <TvmModel> > ListVMsAsync() { List <TvmModel> results = new List <TvmModel>(); IEnumerableAsyncExtended <IVM> jobList = this.Pool.ListVMs(OptionsModel.Instance.ListDetailLevel); IAsyncEnumerator <IVM> asyncEnumerator = jobList.GetAsyncEnumerator(); while (await asyncEnumerator.MoveNextAsync()) { results.Add(new TvmModel(this, asyncEnumerator.Current)); } return(results); }
/// <summary> /// Lists the files on this TVM /// </summary> /// <returns></returns> private async System.Threading.Tasks.Task <List <ITaskFile> > ListFilesAsync() { List <ITaskFile> results = new List <ITaskFile>(); IEnumerableAsyncExtended <ITaskFile> vmFiles = this.VM.ListVMFiles(recursive: true); IAsyncEnumerator <ITaskFile> asyncEnumerator = vmFiles.GetAsyncEnumerator(); while (await asyncEnumerator.MoveNextAsync()) { results.Add(asyncEnumerator.Current); } return(results); }
private async System.Threading.Tasks.Task <List <TaskModel> > ListTasksAsync() { List <TaskModel> results = new List <TaskModel>(); IEnumerableAsyncExtended <ICloudTask> taskList = this.Job.ListTasks(OptionsModel.Instance.ListDetailLevel); IAsyncEnumerator <ICloudTask> asyncEnumerator = taskList.GetAsyncEnumerator(); while (await asyncEnumerator.MoveNextAsync()) { results.Add(new TaskModel(this, asyncEnumerator.Current)); } return(results); }
/// <summary> /// Refreshes the status of the VMs from the Batch service. /// </summary> /// <returns></returns> private async Task RefreshVMsAsync() { // Get the list of pool VM's - can't use this because IVM does not support RecentTasks property yet DetailLevel detailLevel = new ODATADetailLevel() { SelectClause = "recentTasks,state,name" }; IEnumerableAsyncExtended <IVM> vmEnumerableAsync = this.Pool.ListVMs(detailLevel); IAsyncEnumerator <IVM> enumerator = vmEnumerableAsync.GetAsyncEnumerator(); List <IVM> vmList = new List <IVM>(); while (await enumerator.MoveNextAsync()) { vmList.Add(enumerator.Current); } this.RunningTasks = 0; this.SchedulableVMs = 0; foreach (IVM poolVM in vmList) { if (poolVM.State == TVMState.Idle || poolVM.State == TVMState.Running) { this.SchedulableVMs++; } if (poolVM.State == TVMState.Running && this.Pool.MaxTasksPerVM == 1) { this.RunningTasks++; } else if (this.Pool.MaxTasksPerVM > 1) { IEnumerable <TaskInformation> taskInfoList = poolVM.RecentTasks; if (taskInfoList != null) { foreach (TaskInformation ti in taskInfoList) { if (ti.TaskState == TaskState.Running) { this.RunningTasks++; } } } } } Interlocked.Exchange(ref this.vms, vmList); //Threadsafe swap }
private async System.Threading.Tasks.Task <List <JobModel> > ListJobsAsync() { List <JobModel> results = new List <JobModel>(); var detailLevel = new ODATADetailLevel() { SelectClause = "name,state,creationTime" }; IEnumerableAsyncExtended <ICloudJob> jobList = this.WorkItem.ListJobs(detailLevel); IAsyncEnumerator <ICloudJob> asyncEnumerator = jobList.GetAsyncEnumerator(); while (await asyncEnumerator.MoveNextAsync()) { results.Add(new JobModel(this, asyncEnumerator.Current)); } return(results); }
IEnumerator System.Collections.IEnumerable.GetEnumerator() { return(new PSAsyncEnumerator <T1, T2>(omAsyncEnumerable.GetAsyncEnumerator(), this.mappingFunction)); }
/// <summary> /// Runs the job manager task. /// </summary> public async Task RunAsync() { Console.WriteLine("JobManager for account: {0}, work item: {1}, job: {2} has started...", this.accountName, this.workItemName, this.jobName); Console.WriteLine(); Console.WriteLine("JobManager running with the following settings: "); Console.WriteLine("----------------------------------------"); Console.WriteLine(this.configurationSettings.ToString()); //Set up the Batch Service credentials used to authenticate with the Batch Service. BatchCredentials batchCredentials = new BatchCredentials( this.configurationSettings.BatchAccountName, this.configurationSettings.BatchAccountKey); using (IBatchClient batchClient = BatchClient.Connect(this.configurationSettings.BatchServiceUrl, batchCredentials)) { using (IWorkItemManager workItemManager = batchClient.OpenWorkItemManager()) { IToolbox toolbox = batchClient.OpenToolbox(); //Construct a container SAS to provide the Batch Service access to the files required to //run the mapper and reducer tasks. string containerSas = Helpers.ConstructContainerSas( this.configurationSettings.StorageAccountName, this.configurationSettings.StorageAccountKey, this.configurationSettings.StorageServiceUrl, this.configurationSettings.BlobContainer); // // Submit mapper tasks. // Console.WriteLine("Submitting {0} mapper tasks.", this.configurationSettings.NumberOfMapperTasks); //The collection of tasks to add to the Batch Service. List <ICloudTask> tasksToAdd = new List <ICloudTask>(); for (int i = 0; i < this.configurationSettings.NumberOfMapperTasks; i++) { string taskName = Helpers.GetMapperTaskName(i); string fileBlobName = Helpers.GetSplitFileName(i); string fileBlobPath = Helpers.ConstructBlobSource(containerSas, fileBlobName); string commandLine = string.Format("{0} -MapperTask {1}", Constants.TextSearchExe, fileBlobPath); ICloudTask unboundMapperTask = new CloudTask(taskName, commandLine); //The set of files (exe's, dll's and configuration files) required to run the mapper task. IReadOnlyList <string> mapperTaskRequiredFiles = Constants.RequiredExecutableFiles; List <IResourceFile> mapperTaskResourceFiles = Helpers.GetResourceFiles(containerSas, mapperTaskRequiredFiles); unboundMapperTask.ResourceFiles = mapperTaskResourceFiles; tasksToAdd.Add(unboundMapperTask); } //Submit the unbound task collection to the Batch Service. //Use the AddTask method which takes a collection of ICloudTasks for the best performance. await workItemManager.AddTaskAsync(this.workItemName, this.jobName, tasksToAdd); // // Wait for the mapper tasks to complete. // Console.WriteLine("Waiting for the mapper tasks to complete..."); //List all the mapper tasks using a name filter. DetailLevel mapperTaskNameFilter = new ODATADetailLevel() { FilterClause = string.Format("startswith(name, '{0}')", Constants.MapperTaskPrefix) }; List <ICloudTask> tasksToMonitor = workItemManager.ListTasks( this.workItemName, this.jobName, detailLevel: mapperTaskNameFilter).ToList(); //Use the task state monitor to wait for the tasks to complete. ITaskStateMonitor taskStateMonitor = toolbox.CreateTaskStateMonitor(); bool timedOut = await taskStateMonitor.WaitAllAsync(tasksToMonitor, TaskState.Completed, TimeSpan.FromMinutes(5)); //Get the list of mapper tasks in order to analyze their state and ensure they completed successfully. IEnumerableAsyncExtended <ICloudTask> asyncEnumerable = workItemManager.ListTasks( this.workItemName, this.jobName, detailLevel: mapperTaskNameFilter); IAsyncEnumerator <ICloudTask> asyncEnumerator = asyncEnumerable.GetAsyncEnumerator(); //Dump the status of each mapper task. while (await asyncEnumerator.MoveNextAsync()) { ICloudTask cloudTask = asyncEnumerator.Current; Console.WriteLine("Task {0} is in state: {1}", cloudTask.Name, cloudTask.State); await Helpers.CheckForTaskSuccessAsync(cloudTask, dumpStandardOutOnTaskSuccess : false); Console.WriteLine(); } //If not all the tasks reached the desired state within the timeout then the job manager //cannot continue. if (timedOut) { const string errorMessage = "Mapper tasks did not complete within expected timeout."; Console.WriteLine(errorMessage); throw new TimeoutException(errorMessage); } // // Create the reducer task. // string reducerTaskCommandLine = string.Format("{0} -ReducerTask", Constants.TextSearchExe); Console.WriteLine("Adding the reducer task: {0}", Constants.ReducerTaskName); ICloudTask unboundReducerTask = new CloudTask(Constants.ReducerTaskName, reducerTaskCommandLine); //The set of files (exe's, dll's and configuration files) required to run the reducer task. List <IResourceFile> reducerTaskResourceFiles = Helpers.GetResourceFiles(containerSas, Constants.RequiredExecutableFiles); unboundReducerTask.ResourceFiles = reducerTaskResourceFiles; //Send the request to the Batch Service to add the reducer task. await workItemManager.AddTaskAsync(this.workItemName, this.jobName, unboundReducerTask); // //Wait for the reducer task to complete. // //Get the bound reducer task and monitor it for completion. ICloudTask boundReducerTask = await workItemManager.GetTaskAsync(this.workItemName, this.jobName, Constants.ReducerTaskName); timedOut = await taskStateMonitor.WaitAllAsync(new List <ICloudTask> { boundReducerTask }, TaskState.Completed, TimeSpan.FromMinutes(2)); //Refresh the reducer task to get the most recent information about it from the Batch Service. await boundReducerTask.RefreshAsync(); //Dump the reducer tasks exit code and scheduling error for debugging purposes. await Helpers.CheckForTaskSuccessAsync(boundReducerTask, dumpStandardOutOnTaskSuccess : true); //Handle the possibilty that the reducer task did not complete in the expected timeout. if (timedOut) { const string errorMessage = "Reducer task did not complete within expected timeout."; Console.WriteLine("Task {0} is in state: {1}", boundReducerTask.Name, boundReducerTask.State); Console.WriteLine(errorMessage); throw new TimeoutException(errorMessage); } //The job manager has completed. Console.WriteLine("JobManager completed successfully."); } } }