public static ConcurrentChangeTrackedList <T> TransformEnumerableToConcurrentList(IEnumerable <T> enumerable)
{
if (enumerable == null)
{
return(null);
}
ConcurrentChangeTrackedList <T> list = new ConcurrentChangeTrackedList <T>(enumerable);
return(list);
}
/// <summary>
/// Applies the <paramref name="objectCreationFunc"/> to each item in <paramref name="items"/> and returns a non-threadsafe collection containing the results.
/// </summary>
/// <typeparam name="TIn">The type of the input collection.</typeparam>
/// <typeparam name="TOut">The type of the output collection.</typeparam>
/// <param name="items">The collection to convert.</param>
/// <param name="objectCreationFunc">The function used to created each <typeparamref name="TOut"/> type object.</param>
/// <returns>A non-threadsafe collection containing the results of the conversion, or null if <paramref name="items"/> was null.</returns>
internal static IList <TOut> CollectionToThreadSafeCollection <TIn, TOut>(
IEnumerable <TIn> items,
Func <TIn, TOut> objectCreationFunc)
where TIn : class
where TOut : class
{
ConcurrentChangeTrackedList <TOut> result = UtilitiesInternal.ConvertCollection(
items,
objectCreationFunc,
(convertedItemsEnumerable) => new ConcurrentChangeTrackedList <TOut>(convertedItemsEnumerable));
return(result);
}
/// <summary>
/// Performs file staging and also issues the AddTaskCollection request for the set of tasks to add.
/// </summary>
/// <param name="tasksToAdd">The set of tasks to add.</param>
/// <param name="namingFragment"></param>
/// <returns></returns>
private async Task StageFilesAndAddTasks(
Dictionary <string, TrackedCloudTask> tasksToAdd,
string namingFragment)
{
List <Models.TaskAddParameter> protoTasksToAdd = new List <Models.TaskAddParameter>();
this.CheckForCancellationOrTimeoutAndThrow();
//
// Perform file staging
//
// list of all files to be staged across all Tasks
List <IFileStagingProvider> allFiles = new List <IFileStagingProvider>();
// collect all files to be staged
foreach (TrackedCloudTask trackedCloudTask in tasksToAdd.Values)
{
if (trackedCloudTask.Task.FilesToStage != null)
{
// add in the files for the current task
allFiles.AddRange(trackedCloudTask.Task.FilesToStage);
}
}
//This dictonary is only for the purpose of this batch add
ConcurrentDictionary <Type, IFileStagingArtifact> legStagingArtifacts = new ConcurrentDictionary <Type, IFileStagingArtifact>();
//Add the file staging artifacts for this let to the overall bag so as to allow customers to track the file staging progress
this._customerVisibleFileStagingArtifacts.Add(legStagingArtifacts);
// now we have all files, send off to file staging machine
System.Threading.Tasks.Task fileStagingTask = FileStagingUtils.StageFilesAsync(allFiles, legStagingArtifacts, namingFragment);
// wait for file staging async task
await fileStagingTask.ConfigureAwait(continueOnCapturedContext : false);
// now update each non-finalized Task with its new ResourceFiles
foreach (TrackedCloudTask taskToAdd in tasksToAdd.Values)
{
//Update the resource files if the task hasn't already been finalized
if (taskToAdd.Task.FilesToStage != null)
{
foreach (IFileStagingProvider curFile in taskToAdd.Task.FilesToStage)
{
IEnumerable <ResourceFile> curStagedFiles = curFile.StagedFiles;
if (null != curStagedFiles && !((IReadOnly)taskToAdd.Task).IsReadOnly)
{
//TODO: There is a threading issue here -- lock this property down somehow?
if (taskToAdd.Task.ResourceFiles == null)
{
taskToAdd.Task.ResourceFiles = new List <ResourceFile>();
}
foreach (ResourceFile curStagedFile in curStagedFiles)
{
taskToAdd.Task.ResourceFiles.Add(curStagedFile);
}
}
}
//Mark the file staging collection as read only just incase there's another reference to it
ConcurrentChangeTrackedList <IFileStagingProvider> filesToStageListImpl =
taskToAdd.Task.FilesToStage as ConcurrentChangeTrackedList <IFileStagingProvider>;
filesToStageListImpl.IsReadOnly = true; //Set read only
}
Models.TaskAddParameter protoTask = taskToAdd.GetProtocolTask();
protoTasksToAdd.Add(protoTask);
}
this.CheckForCancellationOrTimeoutAndThrow();
//
// Fire the protocol add collection request
//
try
{
var asyncTask = this._jobOperations.ParentBatchClient.ProtocolLayer.AddTaskCollection(
this._jobId,
protoTasksToAdd,
this._behaviorManager,
this._parallelOptions.CancellationToken);
var response = await asyncTask.ConfigureAwait(continueOnCapturedContext : false);
//
// Process the results of the add task collection request
//
this.ProcessProtocolAddTaskResults(response.Body.Value, tasksToAdd);
}
catch (Common.BatchException e)
{
if (e.InnerException is Models.BatchErrorException)
{
Models.BatchError error = ((Models.BatchErrorException)e.InnerException).Body;
int currLength = tasksToAdd.Count;
if (error.Code == Common.BatchErrorCodeStrings.RequestBodyTooLarge && currLength != 1)
{
// Our chunk sizes were too large to fit in a request, so universally reduce size
// This is an internal error due to us using greedy initial maximum chunk size,
// so do not increment retry counter.
{
int newLength = currLength / 2;
int tmpMaxTasks = this._maxTasks;
while (newLength < tmpMaxTasks)
{
tmpMaxTasks = Interlocked.CompareExchange(ref this._maxTasks, newLength, tmpMaxTasks);
}
foreach (TrackedCloudTask trackedTask in tasksToAdd.Values)
{
this._remainingTasksToAdd.Enqueue(trackedTask);
}
return;
}
}
}
throw;
}
}
/// <summary>
/// Performs file staging and also issues the AddTaskCollection request for the set of tasks to add.
/// </summary>
/// <param name="tasksToAdd">The set of tasks to add.</param>
/// <param name="namingFragment"></param>
/// <returns></returns>
private async Task StageFilesAndAddTasks(
Dictionary <string, TrackedCloudTask> tasksToAdd,
string namingFragment)
{
List <Models.TaskAddParameter> protoTasksToAdd = new List <Models.TaskAddParameter>();
this.CheckForCancellationOrTimeoutAndThrow();
//
// Perform file staging
//
// list of all files to be staged across all Tasks
List <IFileStagingProvider> allFiles = new List <IFileStagingProvider>();
// collect all files to be staged
foreach (TrackedCloudTask trackedCloudTask in tasksToAdd.Values)
{
if (trackedCloudTask.Task.FilesToStage != null)
{
// add in the files for the current task
allFiles.AddRange(trackedCloudTask.Task.FilesToStage);
}
}
//This dictonary is only for the purpose of this batch add
ConcurrentDictionary <Type, IFileStagingArtifact> legStagingArtifacts = new ConcurrentDictionary <Type, IFileStagingArtifact>();
//Add the file staging artifacts for this let to the overall bag so as to allow customers to track the file staging progress
this._customerVisibleFileStagingArtifacts.Add(legStagingArtifacts);
// now we have all files, send off to file staging machine
System.Threading.Tasks.Task fileStagingTask = FileStagingLinkedSources.StageFilesAsync(allFiles, legStagingArtifacts, namingFragment);
// wait for file staging async task
await fileStagingTask.ConfigureAwait(continueOnCapturedContext : false);
// now update each non-finalized Task with its new ResourceFiles
foreach (TrackedCloudTask taskToAdd in tasksToAdd.Values)
{
//Update the resource files if the task hasn't already been finalized
if (taskToAdd.Task.FilesToStage != null)
{
foreach (IFileStagingProvider curFile in taskToAdd.Task.FilesToStage)
{
IEnumerable <ResourceFile> curStagedFiles = curFile.StagedFiles;
if (null != curStagedFiles && !((IReadOnly)taskToAdd.Task).IsReadOnly)
{
//TODO: There is a threading issue here -- lock this property down somehow?
if (taskToAdd.Task.ResourceFiles == null)
{
taskToAdd.Task.ResourceFiles = new List <ResourceFile>();
}
foreach (ResourceFile curStagedFile in curStagedFiles)
{
taskToAdd.Task.ResourceFiles.Add(curStagedFile);
}
}
}
//Mark the file staging collection as read only just incase there's another reference to it
ConcurrentChangeTrackedList <IFileStagingProvider> filesToStageListImpl =
taskToAdd.Task.FilesToStage as ConcurrentChangeTrackedList <IFileStagingProvider>;
filesToStageListImpl.IsReadOnly = true; //Set read only
}
Models.TaskAddParameter protoTask = taskToAdd.GetProtocolTask();
protoTasksToAdd.Add(protoTask);
}
this.CheckForCancellationOrTimeoutAndThrow();
//
// Fire the protocol add collection request
//
var asyncTask = this._jobOperations.ParentBatchClient.ProtocolLayer.AddTaskCollection(
this._jobId,
protoTasksToAdd,
this._behaviorManager,
this._parallelOptions.CancellationToken);
var response = await asyncTask.ConfigureAwait(continueOnCapturedContext : false);
//
// Process the results of the add task collection request
//
this.ProcessProtocolAddTaskResults(response.Body.Value, tasksToAdd);
}
