Exemplo n.º 1
0
        private static void AddWork(IBatchClient client)
        {
            using (IWorkItemManager wm = client.OpenWorkItemManager())
            {
                //The toolbox contains some helper mechanisms to ease submission and monitoring of tasks.
                IToolbox toolbox = client.OpenToolbox();

                // to submit a batch of tasks, the TaskSubmissionHelper is useful.
                ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, Program.PoolName);

                // workitem is uniquely identified by its name so we will use a timestamp as suffix
                taskSubmissionHelper.WorkItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss");

                Console.WriteLine("Creating work item: {0}", taskSubmissionHelper.WorkItemName);

                // add 2 quick tasks. Tasks within a job must have unique names
                taskSubmissionHelper.AddTask(new CloudTask("task1", "hostname"));
                taskSubmissionHelper.AddTask(new CloudTask("task2", "cmd /c dir /s"));

                //Commit the tasks to the Batch Service
                IJobCommitUnboundArtifacts artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts;

                // TaskSubmissionHelper commit artifacts returns the workitem and job name
                ICloudJob job = wm.GetJob(artifacts.WorkItemName, artifacts.JobName);

                Console.WriteLine("Waiting for all tasks to complete on work item: {0}, Job: {1} ...", artifacts.WorkItemName, artifacts.JobName);

                //We use the task state monitor to monitor the state of our tasks -- in this case we will wait for them all to complete.
                ITaskStateMonitor taskStateMonitor = toolbox.CreateTaskStateMonitor();

                // blocking wait on the list of tasks until all tasks reach completed state
                bool timedOut = taskStateMonitor.WaitAll(job.ListTasks(), TaskState.Completed, new TimeSpan(0, 20, 0));

                if (timedOut)
                {
                    throw new TimeoutException("Timed out waiting for tasks");
                }

                // dump task output
                foreach (var t in job.ListTasks())
                {
                    Console.WriteLine("Task " + t.Name + " says:\n" + t.GetTaskFile(Constants.StandardOutFileName).ReadAsString());
                }

                // remember to delete the workitem before exiting
                Console.WriteLine("Deleting work item: {0}", artifacts.WorkItemName);
                wm.DeleteWorkItem(artifacts.WorkItemName);
            }
        }
        /// <summary>
        /// Waits for the specified task to complete
        /// </summary>
        public static void WaitForTaskCompletion(BatchController controller, BatchAccountContext context, string workItemName, string jobName, string taskName)
        {
            YieldInjectionInterceptor interceptor = CreateHttpRecordingInterceptor();

            BatchClientBehavior[] behaviors = new BatchClientBehavior[] { interceptor };
            BatchClient           client    = new BatchClient(controller.BatchManagementClient, controller.ResourceManagementClient);

            ListTaskOptions options = new ListTaskOptions(context, workItemName, jobName, null, behaviors)
            {
                TaskName = taskName
            };
            IEnumerable <PSCloudTask> tasks = client.ListTasks(options);

            ITaskStateMonitor monitor = context.BatchOMClient.OpenToolbox().CreateTaskStateMonitor();

            monitor.WaitAll(tasks.Select(t => t.omObject), TaskState.Completed, TimeSpan.FromMinutes(2), null, behaviors);
        }
Exemplo n.º 3
0
        /// <summary>
        /// Populates Azure Storage with the required files, and
        /// submits the work item to the Azure Batch service.
        /// </summary>
        public async Task RunAsync()
        {
            Console.WriteLine("Running with the following settings: ");
            Console.WriteLine("----------------------------------------");
            Console.WriteLine(this.configurationSettings.ToString());

            //Upload resources if required.
            if (this.configurationSettings.ShouldUploadResources)
            {
                Console.WriteLine("Splitting file: {0} into {1} subfiles",
                                  Constants.TextFilePath,
                                  this.configurationSettings.NumberOfMapperTasks);

                //Split the text file into the correct number of files for consumption by the mapper tasks.
                FileSplitter  splitter        = new FileSplitter();
                List <string> mapperTaskFiles = await splitter.SplitAsync(
                    Constants.TextFilePath,
                    this.configurationSettings.NumberOfMapperTasks);

                await this.UploadResourcesAsync(mapperTaskFiles);
            }

            //Generate a SAS for the container.
            string containerSasUrl = Helpers.ConstructContainerSas(
                this.configurationSettings.StorageAccountName,
                this.configurationSettings.StorageAccountKey,
                this.configurationSettings.StorageServiceUrl,
                this.configurationSettings.BlobContainer);

            //Set up the Batch Service credentials used to authenticate with the Batch Service.
            BatchCredentials batchCredentials = new BatchCredentials(
                this.configurationSettings.BatchAccountName,
                this.configurationSettings.BatchAccountKey);

            using (IBatchClient batchClient = BatchClient.Connect(this.configurationSettings.BatchServiceUrl, batchCredentials))
            {
                using (IWorkItemManager workItemManager = batchClient.OpenWorkItemManager())
                {
                    //Create the unbound work item in local memory.  An object which exists only in local memory (and not on the Batch Service) is "unbound".
                    string workItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.UtcNow.ToString("yyyyMMdd-HHmmss");

                    ICloudWorkItem unboundWorkItem = workItemManager.CreateWorkItem(workItemName);

                    //
                    // Construct the work item properties in local memory before commiting them to the Batch Service.
                    //

                    //Allow enough VMs in the pool to run each mapper task, and 1 extra to run the job manager.
                    int numberOfPoolVMs = 1 + this.configurationSettings.NumberOfMapperTasks;

                    //Define the pool specification for the pool which the work item will run on.
                    IPoolSpecification poolSpecification = new PoolSpecification()
                    {
                        TargetDedicated = numberOfPoolVMs,
                        VMSize          = "small",
                        //You can learn more about os families and versions at:
                        //http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
                        OSFamily        = "4",
                        TargetOSVersion = "*"
                    };

                    //Use the auto pool feature of the Batch Service to create a pool when the work item is created.
                    //This creates a new pool for each work item which is added.
                    IAutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification()
                    {
                        AutoPoolNamePrefix = "TextSearchPool",
                        KeepAlive          = false,
                        PoolLifeTimeOption = PoolLifeTimeOption.WorkItem,
                        PoolSpecification  = poolSpecification
                    };

                    //Define the execution environment for this work item -- it will run on the pool defined by the auto pool specification above.
                    unboundWorkItem.JobExecutionEnvironment = new JobExecutionEnvironment()
                    {
                        AutoPoolSpecification = autoPoolSpecification
                    };

                    //Define the job manager for this work item.  This job manager will run when any job is created and will submit the tasks for
                    //the work item.  The job manager is the executable which manages the lifetime of the job
                    //and all tasks which should run for the job.  In this case, the job manager submits the mapper and reducer tasks.
                    string jobManagerCommandLine = string.Format("{0} -JobManagerTask", Constants.TextSearchExe);
                    List <IResourceFile> jobManagerResourceFiles = Helpers.GetResourceFiles(containerSasUrl, Constants.RequiredExecutableFiles);
                    const string         jobManagerTaskName      = "JobManager";

                    unboundWorkItem.JobSpecification = new JobSpecification()
                    {
                        JobManager = new JobManager()
                        {
                            ResourceFiles = jobManagerResourceFiles,
                            CommandLine   = jobManagerCommandLine,

                            //Determines if the job should terminate when the job manager process exits
                            KillJobOnCompletion = false,

                            Name = jobManagerTaskName
                        }
                    };

                    try
                    {
                        //Commit the unbound work item to the Batch Service.
                        Console.WriteLine("Adding work item: {0} to the Batch Service.", unboundWorkItem.Name);
                        await unboundWorkItem.CommitAsync(); //Issues a request to the Batch Service to add the work item which was defined above.

                        //
                        // Wait for the job manager task to complete.
                        //

                        //An object which is backed by a corresponding Batch Service object is "bound."
                        ICloudWorkItem boundWorkItem = await workItemManager.GetWorkItemAsync(workItemName);

                        //Wait for the job to be created automatically by the Batch Service.
                        string boundJobName = await Helpers.WaitForActiveJobAsync(boundWorkItem);

                        ICloudTask boundJobManagerTask = await workItemManager.GetTaskAsync(
                            workItemName,
                            boundJobName,
                            jobManagerTaskName);

                        TimeSpan maxJobCompletionTimeout = TimeSpan.FromMinutes(30);

                        IToolbox          toolbox = batchClient.OpenToolbox();
                        ITaskStateMonitor monitor = toolbox.CreateTaskStateMonitor();
                        bool timedOut             = await monitor.WaitAllAsync(new List <ICloudTask> {
                            boundJobManagerTask
                        }, TaskState.Completed, maxJobCompletionTimeout);

                        Console.WriteLine("Done waiting for job manager task.");

                        await boundJobManagerTask.RefreshAsync();

                        //Check to ensure the job manager task exited successfully.
                        await Helpers.CheckForTaskSuccessAsync(boundJobManagerTask, dumpStandardOutOnTaskSuccess : true);

                        if (timedOut)
                        {
                            throw new TimeoutException(string.Format("Timed out waiting for job manager task to complete."));
                        }
                    }
                    catch (AggregateException e)
                    {
                        e.Handle(
                            (innerE) =>
                        {
                            //We print all the inner exceptions for debugging purposes.
                            Console.WriteLine(innerE.ToString());
                            return(false);
                        });
                        throw;
                    }
                    catch (Exception e)
                    {
                        Console.WriteLine("Hit unexpected exception: {0}", e.ToString());
                        throw;
                    }
                    finally
                    {
                        //Delete the work item.
                        //This will delete the auto pool associated with the work item as long as the pool
                        //keep alive property is set to false.
                        if (this.configurationSettings.ShouldDeleteWorkItem)
                        {
                            Console.WriteLine("Deleting work item {0}", workItemName);
                            workItemManager.DeleteWorkItem(workItemName);
                        }

                        //Note that there were files uploaded to a container specified in the
                        //configuration file.  This container will not be deleted or cleaned up by this sample.
                    }
                }
            }
        }
Exemplo n.º 4
0
        /// <summary>
        /// Runs the job manager task.
        /// </summary>
        public async Task RunAsync()
        {
            Console.WriteLine("JobManager for account: {0}, work item: {1}, job: {2} has started...",
                              this.accountName,
                              this.workItemName,
                              this.jobName);
            Console.WriteLine();

            Console.WriteLine("JobManager running with the following settings: ");
            Console.WriteLine("----------------------------------------");
            Console.WriteLine(this.configurationSettings.ToString());

            //Set up the Batch Service credentials used to authenticate with the Batch Service.
            BatchCredentials batchCredentials = new BatchCredentials(
                this.configurationSettings.BatchAccountName,
                this.configurationSettings.BatchAccountKey);

            using (IBatchClient batchClient = BatchClient.Connect(this.configurationSettings.BatchServiceUrl, batchCredentials))
            {
                using (IWorkItemManager workItemManager = batchClient.OpenWorkItemManager())
                {
                    IToolbox toolbox = batchClient.OpenToolbox();

                    //Construct a container SAS to provide the Batch Service access to the files required to
                    //run the mapper and reducer tasks.
                    string containerSas = Helpers.ConstructContainerSas(
                        this.configurationSettings.StorageAccountName,
                        this.configurationSettings.StorageAccountKey,
                        this.configurationSettings.StorageServiceUrl,
                        this.configurationSettings.BlobContainer);

                    //
                    // Submit mapper tasks.
                    //
                    Console.WriteLine("Submitting {0} mapper tasks.", this.configurationSettings.NumberOfMapperTasks);

                    //The collection of tasks to add to the Batch Service.
                    List <ICloudTask> tasksToAdd = new List <ICloudTask>();

                    for (int i = 0; i < this.configurationSettings.NumberOfMapperTasks; i++)
                    {
                        string taskName     = Helpers.GetMapperTaskName(i);
                        string fileBlobName = Helpers.GetSplitFileName(i);
                        string fileBlobPath = Helpers.ConstructBlobSource(containerSas, fileBlobName);

                        string     commandLine       = string.Format("{0} -MapperTask {1}", Constants.TextSearchExe, fileBlobPath);
                        ICloudTask unboundMapperTask = new CloudTask(taskName, commandLine);

                        //The set of files (exe's, dll's and configuration files) required to run the mapper task.
                        IReadOnlyList <string> mapperTaskRequiredFiles = Constants.RequiredExecutableFiles;

                        List <IResourceFile> mapperTaskResourceFiles = Helpers.GetResourceFiles(containerSas, mapperTaskRequiredFiles);

                        unboundMapperTask.ResourceFiles = mapperTaskResourceFiles;

                        tasksToAdd.Add(unboundMapperTask);
                    }

                    //Submit the unbound task collection to the Batch Service.
                    //Use the AddTask method which takes a collection of ICloudTasks for the best performance.
                    await workItemManager.AddTaskAsync(this.workItemName, this.jobName, tasksToAdd);

                    //
                    // Wait for the mapper tasks to complete.
                    //
                    Console.WriteLine("Waiting for the mapper tasks to complete...");

                    //List all the mapper tasks using a name filter.
                    DetailLevel mapperTaskNameFilter = new ODATADetailLevel()
                    {
                        FilterClause = string.Format("startswith(name, '{0}')", Constants.MapperTaskPrefix)
                    };

                    List <ICloudTask> tasksToMonitor = workItemManager.ListTasks(
                        this.workItemName,
                        this.jobName,
                        detailLevel: mapperTaskNameFilter).ToList();

                    //Use the task state monitor to wait for the tasks to complete.
                    ITaskStateMonitor taskStateMonitor = toolbox.CreateTaskStateMonitor();

                    bool timedOut = await taskStateMonitor.WaitAllAsync(tasksToMonitor, TaskState.Completed, TimeSpan.FromMinutes(5));

                    //Get the list of mapper tasks in order to analyze their state and ensure they completed successfully.
                    IEnumerableAsyncExtended <ICloudTask> asyncEnumerable = workItemManager.ListTasks(
                        this.workItemName,
                        this.jobName,
                        detailLevel: mapperTaskNameFilter);
                    IAsyncEnumerator <ICloudTask> asyncEnumerator = asyncEnumerable.GetAsyncEnumerator();

                    //Dump the status of each mapper task.
                    while (await asyncEnumerator.MoveNextAsync())
                    {
                        ICloudTask cloudTask = asyncEnumerator.Current;

                        Console.WriteLine("Task {0} is in state: {1}", cloudTask.Name, cloudTask.State);

                        await Helpers.CheckForTaskSuccessAsync(cloudTask, dumpStandardOutOnTaskSuccess : false);

                        Console.WriteLine();
                    }

                    //If not all the tasks reached the desired state within the timeout then the job manager
                    //cannot continue.
                    if (timedOut)
                    {
                        const string errorMessage = "Mapper tasks did not complete within expected timeout.";
                        Console.WriteLine(errorMessage);

                        throw new TimeoutException(errorMessage);
                    }

                    //
                    // Create the reducer task.
                    //
                    string reducerTaskCommandLine = string.Format("{0} -ReducerTask", Constants.TextSearchExe);

                    Console.WriteLine("Adding the reducer task: {0}", Constants.ReducerTaskName);
                    ICloudTask unboundReducerTask = new CloudTask(Constants.ReducerTaskName, reducerTaskCommandLine);

                    //The set of files (exe's, dll's and configuration files) required to run the reducer task.
                    List <IResourceFile> reducerTaskResourceFiles = Helpers.GetResourceFiles(containerSas, Constants.RequiredExecutableFiles);

                    unboundReducerTask.ResourceFiles = reducerTaskResourceFiles;

                    //Send the request to the Batch Service to add the reducer task.
                    await workItemManager.AddTaskAsync(this.workItemName, this.jobName, unboundReducerTask);

                    //
                    //Wait for the reducer task to complete.
                    //

                    //Get the bound reducer task and monitor it for completion.
                    ICloudTask boundReducerTask = await workItemManager.GetTaskAsync(this.workItemName, this.jobName, Constants.ReducerTaskName);

                    timedOut = await taskStateMonitor.WaitAllAsync(new List <ICloudTask> {
                        boundReducerTask
                    }, TaskState.Completed, TimeSpan.FromMinutes(2));

                    //Refresh the reducer task to get the most recent information about it from the Batch Service.
                    await boundReducerTask.RefreshAsync();

                    //Dump the reducer tasks exit code and scheduling error for debugging purposes.
                    await Helpers.CheckForTaskSuccessAsync(boundReducerTask, dumpStandardOutOnTaskSuccess : true);

                    //Handle the possibilty that the reducer task did not complete in the expected timeout.
                    if (timedOut)
                    {
                        const string errorMessage = "Reducer task did not complete within expected timeout.";

                        Console.WriteLine("Task {0} is in state: {1}", boundReducerTask.Name, boundReducerTask.State);

                        Console.WriteLine(errorMessage);
                        throw new TimeoutException(errorMessage);
                    }

                    //The job manager has completed.
                    Console.WriteLine("JobManager completed successfully.");
                }
            }
        }