public JobModel(WorkItemModel parentWorkItem, ICloudJob job)
        {
            this.Job             = job;
            this.Tasks           = new List <TaskModel>();
            this.LastUpdatedTime = DateTime.UtcNow;
            this.ParentWorkItem  = parentWorkItem;

            this.TaskCollection = CollectionViewSource.GetDefaultView(this.Tasks);
            this.UpdateTaskView();
        }
        public WorkItemModel(ICloudWorkItem workItem, ICloudJob mostRecentJob)
        {
            this.WorkItem        = workItem;
            this.LastUpdatedTime = DateTime.UtcNow;

            // create a new job list and add the most recent job to the list
            JobModel recentJob = new JobModel(this, mostRecentJob);

            this.Jobs = new List <JobModel> {
                recentJob
            };

            smallestJobId = WorkItemModel.GetJobNumberFromJobName(recentJob.Name);
        }
Beispiel #3
0
        /// <summary>
        /// Lists the Jobs matching the specified filter options
        /// </summary>
        /// <param name="options">The options to use when querying for Jobs</param>
        /// <returns>The Jobs matching the specified filter options</returns>
        public IEnumerable <PSCloudJob> ListJobs(ListJobOptions options)
        {
            if (options == null)
            {
                throw new ArgumentNullException("options");
            }

            if (string.IsNullOrEmpty(options.WorkItemName) && options.WorkItem == null)
            {
                throw new ArgumentNullException(Resources.GBJ_NoWorkItem);
            }
            string wiName = options.WorkItem == null ? options.WorkItemName : options.WorkItem.Name;

            // Get the single Job matching the specified name
            if (!string.IsNullOrEmpty(options.JobName))
            {
                WriteVerbose(string.Format(Resources.GBJ_GetByName, options.JobName, wiName));
                using (IWorkItemManager wiManager = options.Context.BatchOMClient.OpenWorkItemManager())
                {
                    ICloudJob  job   = wiManager.GetJob(wiName, options.JobName, additionalBehaviors: options.AdditionalBehaviors);
                    PSCloudJob psJob = new PSCloudJob(job);
                    return(new PSCloudJob[] { psJob });
                }
            }
            // List Jobs using the specified filter
            else
            {
                if (options.MaxCount <= 0)
                {
                    options.MaxCount = Int32.MaxValue;
                }
                ODATADetailLevel odata = null;
                if (!string.IsNullOrEmpty(options.Filter))
                {
                    WriteVerbose(string.Format(Resources.GBJ_GetByOData, wiName, options.MaxCount));
                    odata = new ODATADetailLevel(filterClause: options.Filter);
                }
                else
                {
                    WriteVerbose(string.Format(Resources.GBJ_GetNoFilter, wiName, options.MaxCount));
                }

                using (IWorkItemManager wiManager = options.Context.BatchOMClient.OpenWorkItemManager())
                {
                    IEnumerableAsyncExtended <ICloudJob> jobs            = wiManager.ListJobs(wiName, odata, options.AdditionalBehaviors);
                    Func <ICloudJob, PSCloudJob>         mappingFunction = j => { return(new PSCloudJob(j)); };
                    return(new PSAsyncEnumerable <PSCloudJob, ICloudJob>(jobs, mappingFunction).Take(options.MaxCount));
                }
            }
        }
        private static void AddWork(IBatchClient client)
        {
            using (IWorkItemManager wm = client.OpenWorkItemManager())
            {
                //The toolbox contains some helper mechanisms to ease submission and monitoring of tasks.
                IToolbox toolbox = client.OpenToolbox();

                // to submit a batch of tasks, the TaskSubmissionHelper is useful.
                ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, Program.PoolName);

                // workitem is uniquely identified by its name so we will use a timestamp as suffix
                taskSubmissionHelper.WorkItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss");

                Console.WriteLine("Creating work item: {0}", taskSubmissionHelper.WorkItemName);

                // add 2 quick tasks. Tasks within a job must have unique names
                taskSubmissionHelper.AddTask(new CloudTask("task1", "hostname"));
                taskSubmissionHelper.AddTask(new CloudTask("task2", "cmd /c dir /s"));

                //Commit the tasks to the Batch Service
                IJobCommitUnboundArtifacts artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts;

                // TaskSubmissionHelper commit artifacts returns the workitem and job name
                ICloudJob job = wm.GetJob(artifacts.WorkItemName, artifacts.JobName);

                Console.WriteLine("Waiting for all tasks to complete on work item: {0}, Job: {1} ...", artifacts.WorkItemName, artifacts.JobName);

                //We use the task state monitor to monitor the state of our tasks -- in this case we will wait for them all to complete.
                ITaskStateMonitor taskStateMonitor = toolbox.CreateTaskStateMonitor();

                // blocking wait on the list of tasks until all tasks reach completed state
                bool timedOut = taskStateMonitor.WaitAll(job.ListTasks(), TaskState.Completed, new TimeSpan(0, 20, 0));

                if (timedOut)
                {
                    throw new TimeoutException("Timed out waiting for tasks");
                }

                // dump task output
                foreach (var t in job.ListTasks())
                {
                    Console.WriteLine("Task " + t.Name + " says:\n" + t.GetTaskFile(Constants.StandardOutFileName).ReadAsString());
                }

                // remember to delete the workitem before exiting
                Console.WriteLine("Deleting work item: {0}", artifacts.WorkItemName);
                wm.DeleteWorkItem(artifacts.WorkItemName);
            }
        }
        public WorkItemModel(ICloudWorkItem workItem, ICloudJob mostRecentJob)
        {
            this.WorkItem = workItem;
            this.LastUpdatedTime = DateTime.UtcNow;

            // create a new job list and add the most recent job to the list
            JobModel recentJob = new JobModel(this, mostRecentJob);
            this.Jobs = new List<JobModel> { recentJob };

            smallestJobId = WorkItemModel.GetJobNumberFromJobName(recentJob.Name);
        }
        public JobModel(WorkItemModel parentWorkItem, ICloudJob job)
        {
            this.Job = job;
            this.Tasks = new List<TaskModel>();
            this.LastUpdatedTime = DateTime.UtcNow;
            this.ParentWorkItem = parentWorkItem;

            this.TaskCollection = CollectionViewSource.GetDefaultView(this.Tasks);
            this.UpdateTaskView();
        }
Beispiel #7
0
        /// <summary>
        /// This is the client that creates workitem and submits tasks.
        /// </summary>
        /// <param name="args"></param>
        public static void SubmitTasks(string[] args)
        {
            config = Config.ParseConfig();

            //Upload resources if specified
            if (config.UploadResources)
            {
                //Upload ImgProc.exe, Batch.dll and the Storage Client
                ImgProcUtils.UploadFileToBlob(Constants.StorageClientDllName, config.ResourceContainerSAS);
                ImgProcUtils.UploadFileToBlob(Constants.ImgProcExeName, config.ResourceContainerSAS);
                ImgProcUtils.UploadFileToBlob(Constants.BatchClientDllName, config.ResourceContainerSAS);
                Console.WriteLine("Done uploading files to blob");
            }

            try
            {
                using (IWorkItemManager wm = config.Client.OpenWorkItemManager())
                {
                    IToolbox toolbox = config.Client.OpenToolbox();

                    //Use the task submission helper to ease creation of workitem and addition of tasks, as well as management of resource file staging.
                    ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, config.PoolName);
                    taskSubmissionHelper.WorkItemName = config.WorkitemName;

                    //Compute the number of images each task should process
                    int numImgsPerTask = (int)Math.Round(config.NumInputBlobs / (decimal)config.NumTasks);

                    for (int i = 0; i < config.NumTasks; i++)
                    {
                        ICloudTask task = new CloudTask(
                            name: "task_no_" + i,
                            commandline: string.Format("{0} --Task {1} thumb{2}", Constants.ImgProcExeName, config.OutputContainerSAS, i));

                        Console.WriteLine("Generating task: {0}", task.Name);

                        task.FilesToStage = new List <IFileStagingProvider>();

                        int start = i * numImgsPerTask;
                        int end;
                        if (i < config.NumTasks - 1)
                        {
                            end = ((i + 1) * numImgsPerTask) - 1;
                        }
                        else
                        {
                            end = config.NumInputBlobs - 1;
                        }

                        //Generate and set up the list of files to be processed by this task
                        for (int j = start; j < end; j++)
                        {
                            string input = GetTempFilePath(j);
                            ImgProcUtils.GenerateImages(input, string.Format("{0}", j));
                            task.FilesToStage.Add(new FileToStage(input, new StagingStorageAccount(config.StorageAccount, config.StorageKey, config.StorageBlobEndpoint)));
                        }
                        task.ResourceFiles = ImgProcUtils.GetResourceFiles(config.ResourceContainerSAS);
                        taskSubmissionHelper.AddTask(task);
                    }

                    IJobCommitUnboundArtifacts artifacts = null;
                    try
                    {
                        Console.WriteLine("Submitting {0} tasks to the Batch Service", config.NumTasks);

                        //Submit the tasks to the Batch Service
                        artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts;
                    }
                    catch (AggregateException ae)
                    {
                        // Go through all exceptions and dump useful information
                        ae.Handle(x =>
                        {
                            if (x is BatchException)
                            {
                                BatchException be = x as BatchException;
                                if (null != be.RequestInformation && null != be.RequestInformation.AzureError)
                                {
                                    // Write the server side error information
                                    Console.Error.WriteLine(be.RequestInformation.AzureError.Code);
                                    Console.Error.WriteLine(be.RequestInformation.AzureError.Message.Value);
                                    if (null != be.RequestInformation.AzureError.Values)
                                    {
                                        foreach (var v in be.RequestInformation.AzureError.Values)
                                        {
                                            Console.Error.WriteLine(v.Key + " : " + v.Value);
                                        }
                                    }
                                }
                            }
                            // Indicate that the error has been handled
                            return(true);
                        });
                    }

                    DateTime starttime = DateTime.Now;

                    //Wait for the job to complete
                    if (config.WaitForCompletion)
                    {
                        ICloudJob job = wm.GetJob(artifacts.WorkItemName, artifacts.JobName);

                        Console.WriteLine("Waiting for tasks to complete...");

                        // Wait up to 15 minutes for all tasks to reach the completed state
                        config.Client.OpenToolbox().CreateTaskStateMonitor().WaitAll(job.ListTasks(), TaskState.Completed, new TimeSpan(0, 15, 0));

                        DateTime endtime = DateTime.Now;

                        Console.WriteLine("Time taken for processing the images : {0} sec", endtime.Subtract(starttime).TotalSeconds);
                    }
                }
            }
            finally
            {
                //Delete the workitem that we created
                if (config.DeleteWorkitem &&
                    config.WaitForCompletion)
                {
                    Console.WriteLine("Press any key to delete the workitem . . .");
                    Console.ReadKey();
                    config.Client.OpenWorkItemManager().DeleteWorkItem(config.WorkitemName);
                }
            }
        }
Beispiel #8
0
        public static void JobMain(string[] args)
        {
            //Load the configuration
            TopNWordsConfiguration configuration = TopNWordsConfiguration.LoadConfigurationFromAppConfig();

            StagingStorageAccount stagingStorageAccount = new StagingStorageAccount(
                configuration.StorageAccountName,
                configuration.StorageAccountKey,
                configuration.StorageAccountBlobEndpoint);

            IBatchClient client           = BatchClient.Connect(configuration.BatchServiceUrl, new BatchCredentials(configuration.BatchAccountName, configuration.BatchAccountKey));
            string       stagingContainer = null;

            //Create a pool (if user hasn't provided one)
            if (configuration.ShouldCreatePool)
            {
                using (IPoolManager pm = client.OpenPoolManager())
                {
                    //OSFamily 4 == OS 2012 R2
                    //You can learn more about os families and versions at:
                    //http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
                    ICloudPool pool = pm.CreatePool(configuration.PoolName, targetDedicated: configuration.PoolSize, osFamily: "4", vmSize: "small");
                    Console.WriteLine("Adding pool {0}", configuration.PoolName);
                    pool.Commit();
                }
            }

            try
            {
                using (IWorkItemManager wm = client.OpenWorkItemManager())
                {
                    IToolbox toolbox = client.OpenToolbox();

                    //Use the TaskSubmissionHelper to help us create a WorkItem and add tasks to it.
                    ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, configuration.PoolName);
                    taskSubmissionHelper.WorkItemName = configuration.WorkItemName;

                    FileToStage topNWordExe = new FileToStage(TopNWordsExeName, stagingStorageAccount);
                    FileToStage storageDll  = new FileToStage(StorageClientDllName, stagingStorageAccount);

                    string bookFileUri = UploadBookFileToCloudBlob(configuration, configuration.BookFileName);
                    Console.WriteLine("{0} uploaded to cloud", configuration.BookFileName);

                    for (int i = 1; i <= configuration.NumberOfTasks; i++)
                    {
                        ICloudTask task = new CloudTask("task_no_" + i, String.Format("{0} --Task {1} {2} {3} {4}",
                                                                                      TopNWordsExeName,
                                                                                      bookFileUri,
                                                                                      configuration.NumberOfTopWords,
                                                                                      configuration.StorageAccountName,
                                                                                      configuration.StorageAccountKey));

                        //This is the list of files to stage to a container -- for each TaskSubmissionHelper one container is created and
                        //files all resolve to Azure Blobs by their name (so two tasks with the same named file will create just 1 blob in
                        //the TaskSubmissionHelper's container).
                        task.FilesToStage = new List <IFileStagingProvider>
                        {
                            topNWordExe,
                            storageDll
                        };

                        taskSubmissionHelper.AddTask(task);
                    }

                    //Commit all the tasks to the Batch Service.
                    IJobCommitUnboundArtifacts artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts;

                    foreach (var fileStagingArtifact in artifacts.FileStagingArtifacts)
                    {
                        SequentialFileStagingArtifact stagingArtifact = fileStagingArtifact.Value as SequentialFileStagingArtifact;
                        if (stagingArtifact != null)
                        {
                            stagingContainer = stagingArtifact.BlobContainerCreated;
                            Console.WriteLine("Uploaded files to container: {0} -- you will be charged for their storage unless you delete them.",
                                              stagingArtifact.BlobContainerCreated);
                        }
                    }

                    //Get the job to monitor status.
                    ICloudJob job = wm.GetJob(artifacts.WorkItemName, artifacts.JobName);

                    Console.Write("Waiting for tasks to complete ...");
                    // Wait 1 minute for all tasks to reach the completed state
                    client.OpenToolbox().CreateTaskStateMonitor().WaitAll(job.ListTasks(), TaskState.Completed, TimeSpan.FromMinutes(20));
                    Console.WriteLine("Done.");

                    foreach (ICloudTask task in job.ListTasks())
                    {
                        Console.WriteLine("Task " + task.Name + " says:\n" + task.GetTaskFile(Constants.StandardOutFileName).ReadAsString());
                        Console.WriteLine(task.GetTaskFile(Constants.StandardErrorFileName).ReadAsString());
                    }
                }
            }
            finally
            {
                //Delete the pool that we created
                if (configuration.ShouldCreatePool)
                {
                    using (IPoolManager pm = client.OpenPoolManager())
                    {
                        Console.WriteLine("Deleting pool: {0}", configuration.PoolName);
                        pm.DeletePool(configuration.PoolName);
                    }
                }

                //Delete the workitem that we created
                if (configuration.ShouldDeleteWorkItem)
                {
                    using (IWorkItemManager wm = client.OpenWorkItemManager())
                    {
                        Console.WriteLine("Deleting work item: {0}", configuration.WorkItemName);
                        wm.DeleteWorkItem(configuration.WorkItemName);
                    }
                }

                //Delete the containers we created
                if (configuration.ShouldDeleteContainer)
                {
                    DeleteContainers(configuration, stagingContainer);
                }
            }
        }