Exemple #1
0
        /// <summary>
        /// Lists the Tasks matching the specified filter options
        /// </summary>
        /// <param name="options">The options to use when querying for Tasks</param>
        /// <returns>The Tasks matching the specified filter options</returns>
        public IEnumerable <PSCloudTask> ListTasks(ListTaskOptions options)
        {
            if (options == null)
            {
                throw new ArgumentNullException("options");
            }

            if ((string.IsNullOrEmpty(options.WorkItemName) || string.IsNullOrEmpty(options.JobName)) && options.Job == null)
            {
                throw new ArgumentNullException(Resources.GBT_NoJob);
            }

            // Get the single Task matching the specified name
            if (!string.IsNullOrEmpty(options.TaskName))
            {
                WriteVerbose(string.Format(Resources.GBT_GetByName, options.TaskName, options.JobName, options.WorkItemName));
                using (IWorkItemManager wiManager = options.Context.BatchOMClient.OpenWorkItemManager())
                {
                    ICloudTask  task   = wiManager.GetTask(options.WorkItemName, options.JobName, options.TaskName, additionalBehaviors: options.AdditionalBehaviors);
                    PSCloudTask psTask = new PSCloudTask(task);
                    return(new PSCloudTask[] { psTask });
                }
            }
            // List Tasks using the specified filter
            else
            {
                if (options.MaxCount <= 0)
                {
                    options.MaxCount = Int32.MaxValue;
                }
                string           jName = options.Job == null ? options.JobName : options.Job.Name;
                ODATADetailLevel odata = null;
                if (!string.IsNullOrEmpty(options.Filter))
                {
                    WriteVerbose(string.Format(Resources.GBT_GetByOData, jName, options.MaxCount));
                    odata = new ODATADetailLevel(filterClause: options.Filter);
                }
                else
                {
                    WriteVerbose(string.Format(Resources.GBT_GetNoFilter, jName, options.MaxCount));
                }

                IEnumerableAsyncExtended <ICloudTask> tasks = null;
                if (options.Job != null)
                {
                    tasks = options.Job.omObject.ListTasks(odata, options.AdditionalBehaviors);
                }
                else
                {
                    using (IWorkItemManager wiManager = options.Context.BatchOMClient.OpenWorkItemManager())
                    {
                        tasks = wiManager.ListTasks(options.WorkItemName, options.JobName, odata, options.AdditionalBehaviors);
                    }
                }
                Func <ICloudTask, PSCloudTask> mappingFunction = t => { return(new PSCloudTask(t)); };
                return(new PSAsyncEnumerable <PSCloudTask, ICloudTask>(tasks, mappingFunction).Take(options.MaxCount));
            }
        }
Exemple #2
0
        /// <summary>
        /// Runs the job manager task.
        /// </summary>
        public async Task RunAsync()
        {
            Console.WriteLine("JobManager for account: {0}, work item: {1}, job: {2} has started...",
                              this.accountName,
                              this.workItemName,
                              this.jobName);
            Console.WriteLine();

            Console.WriteLine("JobManager running with the following settings: ");
            Console.WriteLine("----------------------------------------");
            Console.WriteLine(this.configurationSettings.ToString());

            //Set up the Batch Service credentials used to authenticate with the Batch Service.
            BatchCredentials batchCredentials = new BatchCredentials(
                this.configurationSettings.BatchAccountName,
                this.configurationSettings.BatchAccountKey);

            using (IBatchClient batchClient = BatchClient.Connect(this.configurationSettings.BatchServiceUrl, batchCredentials))
            {
                using (IWorkItemManager workItemManager = batchClient.OpenWorkItemManager())
                {
                    IToolbox toolbox = batchClient.OpenToolbox();

                    //Construct a container SAS to provide the Batch Service access to the files required to
                    //run the mapper and reducer tasks.
                    string containerSas = Helpers.ConstructContainerSas(
                        this.configurationSettings.StorageAccountName,
                        this.configurationSettings.StorageAccountKey,
                        this.configurationSettings.StorageServiceUrl,
                        this.configurationSettings.BlobContainer);

                    //
                    // Submit mapper tasks.
                    //
                    Console.WriteLine("Submitting {0} mapper tasks.", this.configurationSettings.NumberOfMapperTasks);

                    //The collection of tasks to add to the Batch Service.
                    List <ICloudTask> tasksToAdd = new List <ICloudTask>();

                    for (int i = 0; i < this.configurationSettings.NumberOfMapperTasks; i++)
                    {
                        string taskName     = Helpers.GetMapperTaskName(i);
                        string fileBlobName = Helpers.GetSplitFileName(i);
                        string fileBlobPath = Helpers.ConstructBlobSource(containerSas, fileBlobName);

                        string     commandLine       = string.Format("{0} -MapperTask {1}", Constants.TextSearchExe, fileBlobPath);
                        ICloudTask unboundMapperTask = new CloudTask(taskName, commandLine);

                        //The set of files (exe's, dll's and configuration files) required to run the mapper task.
                        IReadOnlyList <string> mapperTaskRequiredFiles = Constants.RequiredExecutableFiles;

                        List <IResourceFile> mapperTaskResourceFiles = Helpers.GetResourceFiles(containerSas, mapperTaskRequiredFiles);

                        unboundMapperTask.ResourceFiles = mapperTaskResourceFiles;

                        tasksToAdd.Add(unboundMapperTask);
                    }

                    //Submit the unbound task collection to the Batch Service.
                    //Use the AddTask method which takes a collection of ICloudTasks for the best performance.
                    await workItemManager.AddTaskAsync(this.workItemName, this.jobName, tasksToAdd);

                    //
                    // Wait for the mapper tasks to complete.
                    //
                    Console.WriteLine("Waiting for the mapper tasks to complete...");

                    //List all the mapper tasks using a name filter.
                    DetailLevel mapperTaskNameFilter = new ODATADetailLevel()
                    {
                        FilterClause = string.Format("startswith(name, '{0}')", Constants.MapperTaskPrefix)
                    };

                    List <ICloudTask> tasksToMonitor = workItemManager.ListTasks(
                        this.workItemName,
                        this.jobName,
                        detailLevel: mapperTaskNameFilter).ToList();

                    //Use the task state monitor to wait for the tasks to complete.
                    ITaskStateMonitor taskStateMonitor = toolbox.CreateTaskStateMonitor();

                    bool timedOut = await taskStateMonitor.WaitAllAsync(tasksToMonitor, TaskState.Completed, TimeSpan.FromMinutes(5));

                    //Get the list of mapper tasks in order to analyze their state and ensure they completed successfully.
                    IEnumerableAsyncExtended <ICloudTask> asyncEnumerable = workItemManager.ListTasks(
                        this.workItemName,
                        this.jobName,
                        detailLevel: mapperTaskNameFilter);
                    IAsyncEnumerator <ICloudTask> asyncEnumerator = asyncEnumerable.GetAsyncEnumerator();

                    //Dump the status of each mapper task.
                    while (await asyncEnumerator.MoveNextAsync())
                    {
                        ICloudTask cloudTask = asyncEnumerator.Current;

                        Console.WriteLine("Task {0} is in state: {1}", cloudTask.Name, cloudTask.State);

                        await Helpers.CheckForTaskSuccessAsync(cloudTask, dumpStandardOutOnTaskSuccess : false);

                        Console.WriteLine();
                    }

                    //If not all the tasks reached the desired state within the timeout then the job manager
                    //cannot continue.
                    if (timedOut)
                    {
                        const string errorMessage = "Mapper tasks did not complete within expected timeout.";
                        Console.WriteLine(errorMessage);

                        throw new TimeoutException(errorMessage);
                    }

                    //
                    // Create the reducer task.
                    //
                    string reducerTaskCommandLine = string.Format("{0} -ReducerTask", Constants.TextSearchExe);

                    Console.WriteLine("Adding the reducer task: {0}", Constants.ReducerTaskName);
                    ICloudTask unboundReducerTask = new CloudTask(Constants.ReducerTaskName, reducerTaskCommandLine);

                    //The set of files (exe's, dll's and configuration files) required to run the reducer task.
                    List <IResourceFile> reducerTaskResourceFiles = Helpers.GetResourceFiles(containerSas, Constants.RequiredExecutableFiles);

                    unboundReducerTask.ResourceFiles = reducerTaskResourceFiles;

                    //Send the request to the Batch Service to add the reducer task.
                    await workItemManager.AddTaskAsync(this.workItemName, this.jobName, unboundReducerTask);

                    //
                    //Wait for the reducer task to complete.
                    //

                    //Get the bound reducer task and monitor it for completion.
                    ICloudTask boundReducerTask = await workItemManager.GetTaskAsync(this.workItemName, this.jobName, Constants.ReducerTaskName);

                    timedOut = await taskStateMonitor.WaitAllAsync(new List <ICloudTask> {
                        boundReducerTask
                    }, TaskState.Completed, TimeSpan.FromMinutes(2));

                    //Refresh the reducer task to get the most recent information about it from the Batch Service.
                    await boundReducerTask.RefreshAsync();

                    //Dump the reducer tasks exit code and scheduling error for debugging purposes.
                    await Helpers.CheckForTaskSuccessAsync(boundReducerTask, dumpStandardOutOnTaskSuccess : true);

                    //Handle the possibilty that the reducer task did not complete in the expected timeout.
                    if (timedOut)
                    {
                        const string errorMessage = "Reducer task did not complete within expected timeout.";

                        Console.WriteLine("Task {0} is in state: {1}", boundReducerTask.Name, boundReducerTask.State);

                        Console.WriteLine(errorMessage);
                        throw new TimeoutException(errorMessage);
                    }

                    //The job manager has completed.
                    Console.WriteLine("JobManager completed successfully.");
                }
            }
        }
        /// <summary>
        /// Submit a work item with tasks which have dependant files.
        /// The files are automatically uploaded to Azure Storage using the FileStaging feature of the Azure.Batch client library.
        /// </summary>
        /// <param name="client"></param>
        private static void AddWorkWithFileStaging(IBatchClient client)
        {
            using (IWorkItemManager wm = client.OpenWorkItemManager())
            {
                IToolbox toolbox = client.OpenToolbox();
                ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, Program.PoolName);

                taskSubmissionHelper.WorkItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss");

                Console.WriteLine("Creating work item: {0}", taskSubmissionHelper.WorkItemName);

                ICloudTask taskToAdd1 = new CloudTask("task_with_file1", "cmd /c type *.txt");
                ICloudTask taskToAdd2 = new CloudTask("task_with_file2", "cmd /c dir /s");

                //Set up a collection of files to be staged -- these files will be uploaded to Azure Storage
                //when the tasks are submitted to the Azure Batch service.
                taskToAdd1.FilesToStage = new List <IFileStagingProvider>();
                taskToAdd2.FilesToStage = new List <IFileStagingProvider>();

                // generate a local file in temp directory
                Process cur  = Process.GetCurrentProcess();
                string  path = Path.Combine(Environment.GetEnvironmentVariable("TEMP"), cur.Id.ToString() + ".txt");
                System.IO.File.WriteAllText(path, "hello from " + cur.Id.ToString());

                // add file as task dependency so it'll be uploaded to storage before task
                // is submitted and download onto the VM before task starts execution
                FileToStage file = new FileToStage(path, new StagingStorageAccount(Program.StorageAccount, Program.StorageKey, Program.StorageBlobEndpoint));
                taskToAdd1.FilesToStage.Add(file);
                taskToAdd2.FilesToStage.Add(file); // filetostage object can be reused

                taskSubmissionHelper.AddTask(taskToAdd1);
                taskSubmissionHelper.AddTask(taskToAdd2);

                IJobCommitUnboundArtifacts artifacts = null;
                bool errors = false;

                try
                {
                    //Stage the files to Azure Storage and add the tasks to Azure Batch.
                    artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts;
                }
                catch (AggregateException ae)
                {
                    errors = true;
                    // Go through all exceptions and dump useful information
                    ae.Handle((x) =>
                    {
                        if (x is BatchException)
                        {
                            BatchException be = x as BatchException;
                            if (null != be.RequestInformation && null != be.RequestInformation.AzureError)
                            {
                                // Write the server side error information
                                Console.Error.WriteLine(be.RequestInformation.AzureError.Code);
                                Console.Error.WriteLine(be.RequestInformation.AzureError.Message.Value);
                                if (null != be.RequestInformation.AzureError.Values)
                                {
                                    foreach (var v in be.RequestInformation.AzureError.Values)
                                    {
                                        Console.Error.WriteLine(v.Key + " : " + v.Value);
                                    }
                                }
                            }
                        }
                        else
                        {
                            Console.WriteLine(x);
                        }
                        // Indicate that the error has been handled
                        return(true);
                    });
                }

                // if there is no exception, wait for job response
                if (!errors)
                {
                    List <ICloudTask> tasksToMonitorForCompletion = wm.ListTasks(artifacts.WorkItemName, artifacts.JobName).ToList();

                    Console.WriteLine("Waiting for all tasks to complete on work item: {0}, Job: {1} ...", artifacts.WorkItemName, artifacts.JobName);
                    client.OpenToolbox().CreateTaskStateMonitor().WaitAll(tasksToMonitorForCompletion, TaskState.Completed, TimeSpan.FromMinutes(30));

                    foreach (ICloudTask task in wm.ListTasks(artifacts.WorkItemName, artifacts.JobName))
                    {
                        Console.WriteLine("Task " + task.Name + " says:\n" + task.GetTaskFile(Constants.StandardOutFileName).ReadAsString());
                        Console.WriteLine(task.GetTaskFile(Constants.StandardErrorFileName).ReadAsString());
                    }
                }

                Console.WriteLine("Deleting work item: {0}", artifacts.WorkItemName);
                wm.DeleteWorkItem(artifacts.WorkItemName); //Don't forget to delete the work item before you exit
            }
        }