/// <summary>
/// Stages the files listed in the <see cref="FilesToStage"/> list.
/// </summary>
/// <param name="allFileStagingArtifacts">An optional collection to customize and receive information about the file staging process.
/// For more information see <see cref="IFileStagingArtifact"/>.</param>
/// <returns>A <see cref="System.Threading.Tasks.Task"/> that represents the asynchronous operation.</returns>
/// <remarks>The staging operation runs asynchronously.</remarks>
public async System.Threading.Tasks.Task StageFilesAsync(ConcurrentDictionary <Type, IFileStagingArtifact> allFileStagingArtifacts = null)
{
// stage all these files
// TODO: align this copy with threadsafe implementation of the IList<>
List <IFileStagingProvider> allFiles = this.FilesToStage == null ? new List <IFileStagingProvider>() : new List <IFileStagingProvider>(this.FilesToStage);
//TODO: There is a threading issue doing this - expose a change tracking box directly and use a lock?
if (this.FilesToStage != null && this.ResourceFiles == null)
{
this.ResourceFiles = new List <ResourceFile>(); //We're about to have some resource files
}
// now we have all files, send off to file staging machine
System.Threading.Tasks.Task fileStagingTask = FileStagingUtils.StageFilesAsync(allFiles, allFileStagingArtifacts);
// wait for file staging async task
await fileStagingTask.ConfigureAwait(continueOnCapturedContext : false);
// now update Task with its new ResourceFiles
foreach (IFileStagingProvider curFile in allFiles)
{
IEnumerable <ResourceFile> curStagedFiles = curFile.StagedFiles;
foreach (ResourceFile curStagedFile in curStagedFiles)
{
this.ResourceFiles.Add(curStagedFile);
}
}
}
/// <summary>
/// Performs file staging and also issues the AddTaskCollection request for the set of tasks to add.
/// </summary>
/// <param name="tasksToAdd">The set of tasks to add.</param>
/// <param name="namingFragment"></param>
/// <returns></returns>
private async Task StageFilesAndAddTasks(
Dictionary <string, TrackedCloudTask> tasksToAdd,
string namingFragment)
{
List <Models.TaskAddParameter> protoTasksToAdd = new List <Models.TaskAddParameter>();
this.CheckForCancellationOrTimeoutAndThrow();
//
// Perform file staging
//
// list of all files to be staged across all Tasks
List <IFileStagingProvider> allFiles = new List <IFileStagingProvider>();
// collect all files to be staged
foreach (TrackedCloudTask trackedCloudTask in tasksToAdd.Values)
{
if (trackedCloudTask.Task.FilesToStage != null)
{
// add in the files for the current task
allFiles.AddRange(trackedCloudTask.Task.FilesToStage);
}
}
//This dictonary is only for the purpose of this batch add
ConcurrentDictionary <Type, IFileStagingArtifact> legStagingArtifacts = new ConcurrentDictionary <Type, IFileStagingArtifact>();
//Add the file staging artifacts for this let to the overall bag so as to allow customers to track the file staging progress
this._customerVisibleFileStagingArtifacts.Add(legStagingArtifacts);
// now we have all files, send off to file staging machine
System.Threading.Tasks.Task fileStagingTask = FileStagingUtils.StageFilesAsync(allFiles, legStagingArtifacts, namingFragment);
// wait for file staging async task
await fileStagingTask.ConfigureAwait(continueOnCapturedContext : false);
// now update each non-finalized Task with its new ResourceFiles
foreach (TrackedCloudTask taskToAdd in tasksToAdd.Values)
{
//Update the resource files if the task hasn't already been finalized
if (taskToAdd.Task.FilesToStage != null)
{
foreach (IFileStagingProvider curFile in taskToAdd.Task.FilesToStage)
{
IEnumerable <ResourceFile> curStagedFiles = curFile.StagedFiles;
if (null != curStagedFiles && !((IReadOnly)taskToAdd.Task).IsReadOnly)
{
//TODO: There is a threading issue here -- lock this property down somehow?
if (taskToAdd.Task.ResourceFiles == null)
{
taskToAdd.Task.ResourceFiles = new List <ResourceFile>();
}
foreach (ResourceFile curStagedFile in curStagedFiles)
{
taskToAdd.Task.ResourceFiles.Add(curStagedFile);
}
}
}
//Mark the file staging collection as read only just incase there's another reference to it
ConcurrentChangeTrackedList <IFileStagingProvider> filesToStageListImpl =
taskToAdd.Task.FilesToStage as ConcurrentChangeTrackedList <IFileStagingProvider>;
filesToStageListImpl.IsReadOnly = true; //Set read only
}
Models.TaskAddParameter protoTask = taskToAdd.GetProtocolTask();
protoTasksToAdd.Add(protoTask);
}
this.CheckForCancellationOrTimeoutAndThrow();
//
// Fire the protocol add collection request
//
try
{
var asyncTask = this._jobOperations.ParentBatchClient.ProtocolLayer.AddTaskCollection(
this._jobId,
protoTasksToAdd,
this._behaviorManager,
this._parallelOptions.CancellationToken);
var response = await asyncTask.ConfigureAwait(continueOnCapturedContext : false);
//
// Process the results of the add task collection request
//
this.ProcessProtocolAddTaskResults(response.Body.Value, tasksToAdd);
}
catch (Common.BatchException e)
{
if (e.InnerException is Models.BatchErrorException)
{
Models.BatchError error = ((Models.BatchErrorException)e.InnerException).Body;
int currLength = tasksToAdd.Count;
if (error.Code == Common.BatchErrorCodeStrings.RequestBodyTooLarge && currLength != 1)
{
// Our chunk sizes were too large to fit in a request, so universally reduce size
// This is an internal error due to us using greedy initial maximum chunk size,
// so do not increment retry counter.
{
int newLength = currLength / 2;
int tmpMaxTasks = this._maxTasks;
while (newLength < tmpMaxTasks)
{
tmpMaxTasks = Interlocked.CompareExchange(ref this._maxTasks, newLength, tmpMaxTasks);
}
foreach (TrackedCloudTask trackedTask in tasksToAdd.Values)
{
this._remainingTasksToAdd.Enqueue(trackedTask);
}
return;
}
}
}
throw;
}
}