public static ConcurrentChangeTrackedList <T> TransformEnumerableToConcurrentList(IEnumerable <T> enumerable)
        {
            if (enumerable == null)
            {
                return(null);
            }

            ConcurrentChangeTrackedList <T> list = new ConcurrentChangeTrackedList <T>(enumerable);

            return(list);
        }
        /// <summary>
        /// Applies the <paramref name="objectCreationFunc"/> to each item in <paramref name="items"/> and returns a non-threadsafe collection containing the results.
        /// </summary>
        /// <typeparam name="TIn">The type of the input collection.</typeparam>
        /// <typeparam name="TOut">The type of the output collection.</typeparam>
        /// <param name="items">The collection to convert.</param>
        /// <param name="objectCreationFunc">The function used to created each <typeparamref name="TOut"/> type object.</param>
        /// <returns>A non-threadsafe collection containing the results of the conversion, or null if <paramref name="items"/> was null.</returns>
        internal static IList <TOut> CollectionToThreadSafeCollection <TIn, TOut>(
            IEnumerable <TIn> items,
            Func <TIn, TOut> objectCreationFunc)
            where TIn : class
            where TOut : class
        {
            ConcurrentChangeTrackedList <TOut> result = UtilitiesInternal.ConvertCollection(
                items,
                objectCreationFunc,
                (convertedItemsEnumerable) => new ConcurrentChangeTrackedList <TOut>(convertedItemsEnumerable));

            return(result);
        }
示例#3
0
        /// <summary>
        /// Performs file staging and also issues the AddTaskCollection request for the set of tasks to add.
        /// </summary>
        /// <param name="tasksToAdd">The set of tasks to add.</param>
        /// <param name="namingFragment"></param>
        /// <returns></returns>
        private async Task StageFilesAndAddTasks(
            Dictionary <string, TrackedCloudTask> tasksToAdd,
            string namingFragment)
        {
            List <Models.TaskAddParameter> protoTasksToAdd = new List <Models.TaskAddParameter>();

            this.CheckForCancellationOrTimeoutAndThrow();

            //
            // Perform file staging
            //

            // list of all files to be staged across all Tasks
            List <IFileStagingProvider> allFiles = new List <IFileStagingProvider>();

            // collect all files to be staged
            foreach (TrackedCloudTask trackedCloudTask in tasksToAdd.Values)
            {
                if (trackedCloudTask.Task.FilesToStage != null)
                {
                    // add in the files for the current task
                    allFiles.AddRange(trackedCloudTask.Task.FilesToStage);
                }
            }

            //This dictonary is only for the purpose of this batch add
            ConcurrentDictionary <Type, IFileStagingArtifact> legStagingArtifacts = new ConcurrentDictionary <Type, IFileStagingArtifact>();

            //Add the file staging artifacts for this let to the overall bag so as to allow customers to track the file staging progress
            this._customerVisibleFileStagingArtifacts.Add(legStagingArtifacts);

            // now we have all files, send off to file staging machine
            System.Threading.Tasks.Task fileStagingTask = FileStagingUtils.StageFilesAsync(allFiles, legStagingArtifacts, namingFragment);

            // wait for file staging async task
            await fileStagingTask.ConfigureAwait(continueOnCapturedContext : false);

            // now update each non-finalized Task with its new ResourceFiles
            foreach (TrackedCloudTask taskToAdd in tasksToAdd.Values)
            {
                //Update the resource files if the task hasn't already been finalized
                if (taskToAdd.Task.FilesToStage != null)
                {
                    foreach (IFileStagingProvider curFile in taskToAdd.Task.FilesToStage)
                    {
                        IEnumerable <ResourceFile> curStagedFiles = curFile.StagedFiles;

                        if (null != curStagedFiles && !((IReadOnly)taskToAdd.Task).IsReadOnly)
                        {
                            //TODO: There is a threading issue here -- lock this property down somehow?
                            if (taskToAdd.Task.ResourceFiles == null)
                            {
                                taskToAdd.Task.ResourceFiles = new List <ResourceFile>();
                            }

                            foreach (ResourceFile curStagedFile in curStagedFiles)
                            {
                                taskToAdd.Task.ResourceFiles.Add(curStagedFile);
                            }
                        }
                    }

                    //Mark the file staging collection as read only just incase there's another reference to it
                    ConcurrentChangeTrackedList <IFileStagingProvider> filesToStageListImpl =
                        taskToAdd.Task.FilesToStage as ConcurrentChangeTrackedList <IFileStagingProvider>;

                    filesToStageListImpl.IsReadOnly = true; //Set read only
                }

                Models.TaskAddParameter protoTask = taskToAdd.GetProtocolTask();
                protoTasksToAdd.Add(protoTask);
            }

            this.CheckForCancellationOrTimeoutAndThrow();

            //
            // Fire the protocol add collection request
            //
            try
            {
                var asyncTask = this._jobOperations.ParentBatchClient.ProtocolLayer.AddTaskCollection(
                    this._jobId,
                    protoTasksToAdd,
                    this._behaviorManager,
                    this._parallelOptions.CancellationToken);

                var response = await asyncTask.ConfigureAwait(continueOnCapturedContext : false);

                //
                // Process the results of the add task collection request
                //
                this.ProcessProtocolAddTaskResults(response.Body.Value, tasksToAdd);
            }
            catch (Common.BatchException e)
            {
                if (e.InnerException is Models.BatchErrorException)
                {
                    Models.BatchError error = ((Models.BatchErrorException)e.InnerException).Body;
                    int currLength          = tasksToAdd.Count;
                    if (error.Code == Common.BatchErrorCodeStrings.RequestBodyTooLarge && currLength != 1)
                    {
                        // Our chunk sizes were too large to fit in a request, so universally reduce size
                        // This is an internal error due to us using greedy initial maximum chunk size,
                        //   so do not increment retry counter.
                        {
                            int newLength   = currLength / 2;
                            int tmpMaxTasks = this._maxTasks;
                            while (newLength < tmpMaxTasks)
                            {
                                tmpMaxTasks = Interlocked.CompareExchange(ref this._maxTasks, newLength, tmpMaxTasks);
                            }
                            foreach (TrackedCloudTask trackedTask in tasksToAdd.Values)
                            {
                                this._remainingTasksToAdd.Enqueue(trackedTask);
                            }
                            return;
                        }
                    }
                }
                throw;
            }
        }
示例#4
0
        /// <summary>
        /// Performs file staging and also issues the AddTaskCollection request for the set of tasks to add.
        /// </summary>
        /// <param name="tasksToAdd">The set of tasks to add.</param>
        /// <param name="namingFragment"></param>
        /// <returns></returns>
        private async Task StageFilesAndAddTasks(
            Dictionary <string, TrackedCloudTask> tasksToAdd,
            string namingFragment)
        {
            List <Models.TaskAddParameter> protoTasksToAdd = new List <Models.TaskAddParameter>();

            this.CheckForCancellationOrTimeoutAndThrow();

            //
            // Perform file staging
            //

            // list of all files to be staged across all Tasks
            List <IFileStagingProvider> allFiles = new List <IFileStagingProvider>();

            // collect all files to be staged
            foreach (TrackedCloudTask trackedCloudTask in tasksToAdd.Values)
            {
                if (trackedCloudTask.Task.FilesToStage != null)
                {
                    // add in the files for the current task
                    allFiles.AddRange(trackedCloudTask.Task.FilesToStage);
                }
            }

            //This dictonary is only for the purpose of this batch add
            ConcurrentDictionary <Type, IFileStagingArtifact> legStagingArtifacts = new ConcurrentDictionary <Type, IFileStagingArtifact>();

            //Add the file staging artifacts for this let to the overall bag so as to allow customers to track the file staging progress
            this._customerVisibleFileStagingArtifacts.Add(legStagingArtifacts);

            // now we have all files, send off to file staging machine
            System.Threading.Tasks.Task fileStagingTask = FileStagingLinkedSources.StageFilesAsync(allFiles, legStagingArtifacts, namingFragment);

            // wait for file staging async task
            await fileStagingTask.ConfigureAwait(continueOnCapturedContext : false);

            // now update each non-finalized Task with its new ResourceFiles
            foreach (TrackedCloudTask taskToAdd in tasksToAdd.Values)
            {
                //Update the resource files if the task hasn't already been finalized
                if (taskToAdd.Task.FilesToStage != null)
                {
                    foreach (IFileStagingProvider curFile in taskToAdd.Task.FilesToStage)
                    {
                        IEnumerable <ResourceFile> curStagedFiles = curFile.StagedFiles;

                        if (null != curStagedFiles && !((IReadOnly)taskToAdd.Task).IsReadOnly)
                        {
                            //TODO: There is a threading issue here -- lock this property down somehow?
                            if (taskToAdd.Task.ResourceFiles == null)
                            {
                                taskToAdd.Task.ResourceFiles = new List <ResourceFile>();
                            }

                            foreach (ResourceFile curStagedFile in curStagedFiles)
                            {
                                taskToAdd.Task.ResourceFiles.Add(curStagedFile);
                            }
                        }
                    }

                    //Mark the file staging collection as read only just incase there's another reference to it
                    ConcurrentChangeTrackedList <IFileStagingProvider> filesToStageListImpl =
                        taskToAdd.Task.FilesToStage as ConcurrentChangeTrackedList <IFileStagingProvider>;

                    filesToStageListImpl.IsReadOnly = true; //Set read only
                }

                Models.TaskAddParameter protoTask = taskToAdd.GetProtocolTask();
                protoTasksToAdd.Add(protoTask);
            }

            this.CheckForCancellationOrTimeoutAndThrow();

            //
            // Fire the protocol add collection request
            //
            var asyncTask = this._jobOperations.ParentBatchClient.ProtocolLayer.AddTaskCollection(
                this._jobId,
                protoTasksToAdd,
                this._behaviorManager,
                this._parallelOptions.CancellationToken);

            var response = await asyncTask.ConfigureAwait(continueOnCapturedContext : false);

            //
            // Process the results of the add task collection request
            //
            this.ProcessProtocolAddTaskResults(response.Body.Value, tasksToAdd);
        }