private async Task OnGetResponseCompleted(Task<WebResponse> task)
{
if (this._isCancelled)
{
return;
}
// get the response
HttpWebResponse response;
try
{
response = (HttpWebResponse)await task.ConfigureAwait(false);
}
catch (Exception e)
{
this.InvokeOnErrorHandler(ErrorMessages.WebPageLoadError);
return;
}
if (response.StatusCode != HttpStatusCode.OK)
{
this.InvokeOnErrorHandler((int)response.StatusCode + " " + response.StatusDescription);
return;
}
// response stream
Stream stream;
if (this.OnProgressChanged != null && response.Headers.AllKeys.Any(key => key == "Content-Length"))
{
var progressStream = new ProgressStream(response.GetResponseStream(), response.ContentLength);
progressStream.OnProgressChanged = v => this.InvokeInUiThread(() => this.OnProgressChanged(v));
stream = progressStream;
}
else
{
stream = response.GetResponseStream();
}
if (!this._isCancelled)
{
this.OnStreamDownloaded(stream);
}
}
public async Task<bool> TryWithUacFallback(Task task, string info) {
if (!Tools.UacHelper.CheckUac()) {
await task.ConfigureAwait(false);
return false;
}
Exception e;
try {
await task.ConfigureAwait(false);
return false;
} catch (UnauthorizedAccessException ex) {
e = ex;
}
var report =
await
_dialogManager.MessageBox(
new MessageBoxDialogParams(
$"The application failed to write to the path, probably indicating permission issues\nWould you like to restart the application Elevated?\n\n {info}\n{e.Message}",
"Restart the application elevated?", SixMessageBoxButton.YesNo)).ConfigureAwait(false) ==
SixMessageBoxResult.Yes;
if (!report)
throw e;
RestartWithUacInclEnvironmentCommandLine();
return true;
}
public async Task<bool> TryWithUacFallback(Task task, string info) {
if (!Tools.Processes.Uac.CheckUac()) {
await task.ConfigureAwait(false);
return false;
}
Exception e;
try {
await task.ConfigureAwait(false);
return false;
} catch (UnauthorizedAccessException ex) {
e = ex;
}
var report = await _dialogManager.MessageBoxAsync(new MessageBoxDialogParams(
String.Format(
"The application failed to write to the path, probably indicating permission issues\nWould you like to restart the application Elevated?\n\n {0}\n{1}",
info, e.Message),
"Restart the application elevated?", SixMessageBoxButton.YesNo)) == SixMessageBoxResult.Yes;
UsageCounter.ReportUsage("Dialog - Restart the application elevated: {0}".FormatWith(report));
if (!report)
throw e;
RestartWithUacInclEnvironmentCommandLine();
return true;
}
/// <summary>
/// Automatically calls the <see cref="BaseCommand{T}.InitializeAsync(AsyncPackage)"/> method for every command that has the <see cref="CommandAttribute"/> applied.
/// </summary>
/// <param name="package"></param>
/// <param name="assemblies"></param>
/// <returns>A collection of the command instances</returns>
public static async Task <IEnumerable <object> > RegisterCommandsAsync(this AsyncPackage package, params Assembly[] assemblies)
{
List <Assembly> assembliesList = assemblies.ToList();
Assembly packageAssembly = package.GetType().Assembly;
if (!assembliesList.Contains(packageAssembly))
{
assembliesList.Add(packageAssembly);
}
Type baseCommandType = typeof(BaseCommand <>);
IEnumerable <Type> commandTypes = assembliesList.SelectMany(x => x.GetTypes())
.Where(x =>
!x.IsAbstract &&
x.IsAssignableToGenericType(baseCommandType) &&
x.GetCustomAttribute <CommandAttribute>() != null);
List <object> commands = new();
foreach (Type?commandType in commandTypes)
{
MethodInfo initializeAsyncMethod = commandType.GetMethod(
nameof(BaseCommand <object> .InitializeAsync),
BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy);
Task initializeAsyncTask = (Task)initializeAsyncMethod.Invoke(null, new object[] { package });
await initializeAsyncTask.ConfigureAwait(false);
object command = initializeAsyncTask.GetType().GetProperty("Result").GetValue(initializeAsyncTask);
commands.Add(command);
}
return(commands);
}
public async Task Do(Action act, TaskScheduler scheduler = null)
{
Exception lastException = null;
int retryCount = -1;
TimeSpan wait;
while (true)
{
try
{
var task = new Task(act);
task.Start(scheduler);
await task.ConfigureAwait(false);
break;
}
catch (OutOfMemoryException)
{
throw;
}
catch (Exception ex)
{
lastException = ex;
}
retryCount++;
if (!GetShouldRetry(retryCount, lastException, out wait))
{
ExceptionDispatchInfo.Capture(lastException).Throw();
}
else
{
await Task.Delay(wait).ConfigureAwait(false);
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="ConfiguredTaskAwaiter"/> struct.
/// </summary>
internal ConfiguredTaskAwaiter(CoyoteRuntime runtime, SystemTasks.Task awaitedTask,
bool continueOnCapturedContext)
{
this.Runtime = runtime;
this.AwaitedTask = awaitedTask;
this.Awaiter = awaitedTask.ConfigureAwait(continueOnCapturedContext).GetAwaiter();
}
public static async Task ProcessLogFileWrapperAsync(string filePath, string toolFormat, bool promptOnLogConversions, bool cleanErrors, bool openInEditor)
{
var taskStatusCenterService = (IVsTaskStatusCenterService)Package.GetGlobalService(typeof(SVsTaskStatusCenterService));
var taskProgressData = new TaskProgressData
{
CanBeCanceled = false,
ProgressText = null,
};
string fileName = Path.GetFileName(filePath);
var taskHandlerOptions = new TaskHandlerOptions
{
ActionsAfterCompletion = CompletionActions.None,
TaskSuccessMessage = string.Format(CultureInfo.CurrentCulture, Resources.CompletedProcessingLogFileFormat, fileName),
Title = string.Format(CultureInfo.CurrentCulture, Resources.ProcessingLogFileFormat, fileName),
};
ITaskHandler taskHandler = taskStatusCenterService.PreRegister(taskHandlerOptions, taskProgressData);
Task task = ProcessLogFileAsync(filePath, toolFormat, promptOnLogConversions, cleanErrors, openInEditor);
taskHandler.RegisterTask(task);
await task.ConfigureAwait(continueOnCapturedContext : false);
}
internal static async System.Threading.Tasks.Task StageFilesAsync(List <IFileStagingProvider> filesToStage, ConcurrentDictionary <Type, IFileStagingArtifact> allFileStagingArtifacts)
{
using (System.Threading.Tasks.Task asyncTask = FileStagingLinkedSources.StageFilesAsync(filesToStage, allFileStagingArtifacts, string.Empty))
{
await asyncTask.ConfigureAwait(continueOnCapturedContext : false);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="ConfiguredTaskAwaiter"/> struct.
/// </summary>
internal ConfiguredTaskAwaiter(TaskController taskController, SystemTasks.Task awaitedTask,
bool continueOnCapturedContext)
{
this.TaskController = taskController;
this.AwaitedTask = awaitedTask;
this.Awaiter = awaitedTask.ConfigureAwait(continueOnCapturedContext).GetAwaiter();
}
/// <summary>
/// Commits all pending changes to this <see cref="CloudJobSchedule" /> to the Azure Batch service.
/// </summary>
/// <param name="additionalBehaviors">A collection of <see cref="BatchClientBehavior"/> instances that are applied to the Batch service request after the <see cref="CustomBehaviors"/>.</param>
/// <param name="cancellationToken">A <see cref="CancellationToken"/> for controlling the lifetime of the asynchronous operation.</param>
/// <returns>A <see cref="System.Threading.Tasks.Task"/> that represents the asynchronous operation.</returns>
/// <remarks>
/// <para>
/// Updates an existing <see cref="CloudJobSchedule"/> on the Batch service by replacing its properties with the properties of this <see cref="CloudJobSchedule"/> which have been changed.
/// Unchanged properties are ignored.
/// All changes since the last time this entity was retrieved from the Batch service (either via <see cref="Refresh"/>, <see cref="JobScheduleOperations.GetJobSchedule"/>,
/// or <see cref="JobScheduleOperations.ListJobSchedules"/>) are applied.
/// Properties which are explicitly set to null will cause an exception because the Batch service does not support partial updates which set a property to null.
/// If you need to set a property to null, use <see cref="Commit"/>.
/// </para>
/// <para>This operation runs asynchronously.</para>
/// </remarks>
public async System.Threading.Tasks.Task CommitChangesAsync(
IEnumerable <BatchClientBehavior> additionalBehaviors = null,
CancellationToken cancellationToken = default(CancellationToken))
{
UtilitiesInternal.ThrowOnUnbound(this.propertyContainer.BindingState);
// after this no prop access is allowed
this.propertyContainer.IsReadOnly = true;
// craft the bahavior manager for this call
BehaviorManager bhMgr = new BehaviorManager(this.CustomBehaviors, additionalBehaviors);
Models.JobSpecification jobSpecification = this.propertyContainer.JobSpecificationProperty.GetTransportObjectIfChanged <JobSpecification, Models.JobSpecification>();
Models.Schedule schedule = this.propertyContainer.ScheduleProperty.GetTransportObjectIfChanged <Schedule, Models.Schedule>();
Models.MetadataItem[] metadata = this.propertyContainer.MetadataProperty.GetTransportObjectIfChanged <MetadataItem, Models.MetadataItem>();
System.Threading.Tasks.Task asyncJobScheduleUpdate =
this.parentBatchClient.ProtocolLayer.PatchJobSchedule(
this.Id,
jobSpecification,
metadata,
schedule,
bhMgr,
cancellationToken);
await asyncJobScheduleUpdate.ConfigureAwait(continueOnCapturedContext : false);
}
/// <summary>
/// Stages the files listed in the <see cref="FilesToStage"/> list.
/// </summary>
/// <param name="allFileStagingArtifacts">An optional collection to customize and receive information about the file staging process.
/// For more information see <see cref="IFileStagingArtifact"/>.</param>
/// <returns>A <see cref="System.Threading.Tasks.Task"/> that represents the asynchronous operation.</returns>
/// <remarks>The staging operation runs asynchronously.</remarks>
public async System.Threading.Tasks.Task StageFilesAsync(ConcurrentDictionary <Type, IFileStagingArtifact> allFileStagingArtifacts = null)
{
// stage all these files
// TODO: align this copy with threadsafe implementation of the IList<>
List <IFileStagingProvider> allFiles = this.FilesToStage == null ? new List <IFileStagingProvider>() : new List <IFileStagingProvider>(this.FilesToStage);
//TODO: There is a threading issue doing this - expose a change tracking box directly and use a lock?
if (this.FilesToStage != null && this.ResourceFiles == null)
{
this.ResourceFiles = new List <ResourceFile>(); //We're about to have some resource files
}
// now we have all files, send off to file staging machine
System.Threading.Tasks.Task fileStagingTask = FileStagingUtils.StageFilesAsync(allFiles, allFileStagingArtifacts);
// wait for file staging async task
await fileStagingTask.ConfigureAwait(continueOnCapturedContext : false);
// now update Task with its new ResourceFiles
foreach (IFileStagingProvider curFile in allFiles)
{
IEnumerable <ResourceFile> curStagedFiles = curFile.StagedFiles;
foreach (ResourceFile curStagedFile in curStagedFiles)
{
this.ResourceFiles.Add(curStagedFile);
}
}
}
private async Task RunTaskAsync(Func <CancellationToken, Task> getTaskToRun, string taskTitle, string errorMessage)
{
if (IsOperationInProgress)
{
return;
}
try
{
ITaskHandler handler = await _taskStatusCenterService.CreateTaskHandlerAsync(taskTitle);
CancellationToken internalToken = RegisterCancellationToken(handler.UserCancellation);
lock (_lockObject)
{
_currentOperationTask = getTaskToRun(internalToken);
handler.RegisterTask(_currentOperationTask);
}
await _currentOperationTask.ConfigureAwait(false);
}
catch (Exception ex)
{
Logger.LogEvent(errorMessage + Environment.NewLine + ex.Message, LogLevel.Operation);
Telemetry.TrackException(nameof(RunTaskAsync), ex);
}
}
private async void DebugPrintAnalyticsOutput(Task<string> resultTask, Dictionary<string, string> hitData)
{
using (var form = new FormUrlEncodedContent(hitData))
{
var result = await resultTask.ConfigureAwait(false);
var formData = await form.ReadAsStringAsync().ConfigureAwait(false);
Debug.WriteLine($"Request: POST {_serverUrl} Data: {formData}");
Debug.WriteLine($"Output of analytics: {result}");
}
}
private async Task GetRequestStreamCallback(Task<Stream> task, HttpWebRequest request)
{
Stream postStream = await task.ConfigureAwait(false);
this.WriteMultipartObject(postStream, this.Parameters);
postStream.Close();
var responseTask = request.GetResponseAsync();
await GetResponseCallback(responseTask);
}
private static async Task Callback(Task<WebResponse> task, WebRequest request)
{
try
{
var response = await task.ConfigureAwait(false);
}
catch (WebException)
{
}
}
public async Task PostStart(IBus bus, Task<BusReady> busReady)
{
if (_log.IsDebugEnabled)
_log.DebugFormat("Job Service Starting: {0}", _jobService.InputAddress);
await busReady.ConfigureAwait(false);
if (_log.IsInfoEnabled)
_log.InfoFormat("Job Service Started: {0}", _jobService.InputAddress);
}
/// <summary>
/// Authenticate using the secret for the specified client from the key store
/// </summary>
/// <param name="clientId">The active directory client id for the application.</param>
/// <param name="audience">The intended audience for authentication</param>
/// <param name="context">The AD AuthenticationContext to use</param>
/// <returns></returns>
public async Task<AuthenticationResult> AuthenticateAsync(string clientId, string audience, AuthenticationContext context)
{
var task = new Task<SecureString>(() =>
{
return ServicePrincipalKeyStore.GetKey(clientId, _tenantId);
});
task.Start();
var key = await task.ConfigureAwait(false);
return await context.AcquireTokenAsync(audience, new ClientCredential(clientId, key));
}
/// <summary>
/// Awaits the given task without having the device going to sleep.
/// </summary>
/// <returns>A task to await.</returns>
/// <param name="action">Task to run.</param>
/// <param name="task">Task.</param>
public async Task DoWithoutSleepAsync(Task task)
{
try
{
ActivateAutoSleepMode(false);
await task.ConfigureAwait(false);
}
finally
{
ActivateAutoSleepMode(true);
}
}
/// <summary>
/// Deletes this <see cref="CloudJobSchedule" />.
/// </summary>
/// <param name="additionalBehaviors">A collection of <see cref="BatchClientBehavior"/> instances that are applied to the Batch service request after the <see cref="CustomBehaviors"/>.</param>
/// <param name="cancellationToken">A <see cref="CancellationToken"/> for controlling the lifetime of the asynchronous operation.</param>
/// <returns>A <see cref="System.Threading.Tasks.Task"/> that represents the asynchronous operation.</returns>
/// <remarks>
/// <para>The delete operation requests that the job schedule be deleted. The request puts the schedule in the <see cref="Common.JobScheduleState.Deleting"/> state.
/// The Batch service will delete any existing jobs and tasks under the schedule, including any active job, and perform the actual job schedule deletion without any further client action.</para>
/// <para>The delete operation runs asynchronously.</para>
/// </remarks>
public async System.Threading.Tasks.Task DeleteAsync(IEnumerable <BatchClientBehavior> additionalBehaviors = null, CancellationToken cancellationToken = default(CancellationToken))
{
// throw if if this object is unbound
UtilitiesInternal.ThrowOnUnbound(this.propertyContainer.BindingState);
// craft the behavior manager for this call
BehaviorManager bhMgr = new BehaviorManager(this.CustomBehaviors, additionalBehaviors);
System.Threading.Tasks.Task asyncTask = this.parentBatchClient.ProtocolLayer.DeleteJobSchedule(this.Id, bhMgr, cancellationToken);
await asyncTask.ConfigureAwait(continueOnCapturedContext : false);
}
/// <summary>
/// Initializes a new instance of the <see cref="ConfiguredTaskAwaiter"/> struct.
/// </summary>
internal ConfiguredTaskAwaiter(SystemTask awaitedTask, bool continueOnCapturedContext)
{
if (RuntimeProvider.TryGetFromSynchronizationContext(out CoyoteRuntime runtime))
{
// Force the continuation to run on the current context so that it can be controlled.
continueOnCapturedContext = true;
}
this.AwaitedTask = awaitedTask;
this.Awaiter = awaitedTask.ConfigureAwait(continueOnCapturedContext).GetAwaiter();
this.Runtime = runtime;
}
/// <summary>
/// Authenticate using certificate thumbprint from the datastore
/// </summary>
/// <param name="clientId">The active directory client id for the application.</param>
/// <param name="audience">The intended audience for authentication</param>
/// <param name="context">The AD AuthenticationContext to use</param>
/// <returns></returns>
public async Task<AuthenticationResult> AuthenticateAsync(string clientId, string audience, AuthenticationContext context)
{
var task = new Task<X509Certificate2>(() =>
{
return AzureSession.DataStore.GetCertificate(this._certificateThumbprint);
});
task.Start();
var certificate = await task.ConfigureAwait(false);
return await context.AcquireTokenAsync(
audience,
new ClientAssertionCertificate(clientId, certificate));
}
/// <summary>
/// Initializes a new instance of the <see cref="ConfiguredTaskAwaiter"/> struct.
/// </summary>
internal ConfiguredTaskAwaiter(CoyoteRuntime runtime, SystemTasks.Task awaitedTask,
bool continueOnCapturedContext)
{
if (runtime?.SchedulingPolicy != SchedulingPolicy.None)
{
// Force the continuation to run on the current context so that it can be controlled.
continueOnCapturedContext = true;
}
this.Runtime = runtime;
this.AwaitedTask = awaitedTask;
this.Awaiter = awaitedTask.ConfigureAwait(continueOnCapturedContext).GetAwaiter();
}
public async Task PostStart(IBus bus, Task<BusReady> busReady)
{
if (_log.IsDebugEnabled)
_log.DebugFormat("Quartz Scheduler Starting: {0} ({1}/{2})", _schedulerEndpointAddress, _scheduler.SchedulerName, _scheduler.SchedulerInstanceId);
await busReady.ConfigureAwait(false);
_scheduler.JobFactory = new MassTransitJobFactory(bus);
_scheduler.Start();
if (_log.IsInfoEnabled)
_log.InfoFormat("Quartz Scheduler Started: {0} ({1}/{2})", _schedulerEndpointAddress, _scheduler.SchedulerName, _scheduler.SchedulerInstanceId);
}
/// <summary>
/// Adds a single task to this <see cref="CloudJob"/>. To add multiple tasks,
/// use <see cref="JobOperations.AddTaskAsync(string,IEnumerable{CloudTask},BatchClientParallelOptions,ConcurrentBag{ConcurrentDictionary{Type, IFileStagingArtifact}},TimeSpan?,IEnumerable{BatchClientBehavior})">JobOperations.AddTaskAsync</see>.
/// </summary>
/// <param name="taskToAdd">The <see cref="CloudTask"/> to add.</param>
/// <param name="allFileStagingArtifacts">An optional collection to customize and receive information about the file staging process (see <see cref="CloudTask.FilesToStage"/>).
/// For more information see <see cref="IFileStagingArtifact"/>.</param>
/// <param name="additionalBehaviors">A collection of <see cref="BatchClientBehavior"/> instances that are applied to the Batch service request after the <see cref="CustomBehaviors"/>.</param>
/// <param name="cancellationToken">A <see cref="CancellationToken"/> for controlling the lifetime of the asynchronous operation.</param>
/// <returns>A <see cref="System.Threading.Tasks.Task"/> that represents the asynchronous operation.</returns>
/// <remarks>
/// <para>
/// Each call to this method incurs a request to the Batch service. Therefore, using this method to add
/// multiple tasks is less efficient than using a bulk add method, and can incur HTTP connection restrictions.
/// If you are performing many of these operations in parallel and are seeing client side timeouts (a <see cref="TaskCanceledException"/>), please see
/// http://msdn.microsoft.com/en-us/library/system.net.servicepointmanager.defaultconnectionlimit%28v=vs.110%29.aspx
/// or use
/// <see cref="JobOperations.AddTaskAsync(string,IEnumerable{CloudTask},BatchClientParallelOptions,ConcurrentBag{ConcurrentDictionary{Type, IFileStagingArtifact}},TimeSpan?,IEnumerable{Microsoft.Azure.Batch.BatchClientBehavior})"/>.
/// </para>
/// <para>The add task operation runs asynchronously.</para>
/// </remarks>
public async System.Threading.Tasks.Task AddTaskAsync(
CloudTask taskToAdd,
ConcurrentDictionary <Type, IFileStagingArtifact> allFileStagingArtifacts = null,
IEnumerable <BatchClientBehavior> additionalBehaviors = null,
CancellationToken cancellationToken = default(CancellationToken))
{
UtilitiesInternal.ThrowOnUnbound(this.propertyContainer.BindingState);
// craft the behavior manager for this call
BehaviorManager behaveMgr = new BehaviorManager(this.CustomBehaviors, additionalBehaviors);
System.Threading.Tasks.Task asyncTask = this.parentBatchClient.JobOperations.AddTaskAsyncImpl(this.Id, taskToAdd, behaveMgr, cancellationToken, allFileStagingArtifacts);
await asyncTask.ConfigureAwait(continueOnCapturedContext : false);
}
public async void StartMonitoring(Task preTask)
{
LogHelper.Logger.InfoFormat("{0} A ring is set up: {1}", m_host.FullName, DisplayName);
await preTask.ConfigureAwait(false);
LogHelper.Logger.InfoFormat("{0} Ring pretask done. Starting check loop for {1}", m_host.FullName, DisplayName);
while (true)
{
await Task.Delay(m_host.m_dataflowOptions.MonitorInterval).ConfigureAwait(false);
bool empty = false;
if (m_hb.NoHeartbeatDuring(() => { empty = this.IsRingEmpty(); }) && empty)
{
LogHelper.Logger.DebugFormat("{0} 1st level empty ring check passed for {1}", m_host.FullName, this.DisplayName);
bool noHeartbeat = await m_hb.NoHeartbeatDuring(
async () =>
{
//trigger batch
foreach (var batchedFlow in m_ringNodes.OfType<IBatchedDataflow>())
{
batchedFlow.TriggerBatch();
}
await Task.Delay(m_host.m_dataflowOptions.MonitorInterval).ConfigureAwait(false);
empty = this.IsRingEmpty();
}).ConfigureAwait(false);
if (noHeartbeat && empty)
{
LogHelper.Logger.DebugFormat("{0} 2nd level empty ring check passed for {1}", m_host.FullName, this.DisplayName);
m_hb.Complete(); //start the completion domino :)
break;
}
else
{
//batched flow dumped some hidden elements to the flow, go back to the loop
LogHelper.Logger.DebugFormat("{0} New elements entered the ring by batched flows ({1}). Will fall back to 1st level empty ring check.", m_host.FullName, this.DisplayName);
}
}
}
LogHelper.Logger.InfoFormat("{0} Ring completion detected! Completion triggered on heartbeat node: {1}", m_host.FullName, DisplayName);
}
public async Task Bug1360227_AddTasksBatchCancelation(bool useJobOperations)
{
const string testName = "Bug1360227_AddTasksBatchCancelation";
const int taskCount = 322;
await SynchronizationContextHelper.RunTestAsync(async() =>
{
using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync())
{
using (CancellationTokenSource source = new CancellationTokenSource())
{
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
MaxDegreeOfParallelism = 2,
CancellationToken = source.Token
};
System.Threading.Tasks.Task t = this.AddTasksSimpleTestAsync(
batchCli,
testName,
taskCount,
parallelOptions,
useJobOperations: useJobOperations);
Thread.Sleep(TimeSpan.FromSeconds(.3)); //Wait till we get into the workflow
this.testOutputHelper.WriteLine("Canceling the work flow");
source.Cancel();
try
{
await t.ConfigureAwait(false);
}
catch (Exception e)
{
//This is expected to throw one of two possible exception types...
if (!(e is TaskCanceledException) && !(e is OperationCanceledException))
{
throw new ThrowsException(typeof(TaskCanceledException), e);
}
}
}
}
},
TestTimeout);
}
private async Task<ClientResponse> GetClientResponseAsync(Task<HttpResponseMessage> responseTask)
{
// get the response
HttpResponseMessage response = await responseTask.ConfigureAwait(false);
// get the content
HttpContent httpContent = response.Content;
byte[] content = await httpContent.ReadAsByteArrayAsync().ConfigureAwait(false);
// construct the output
return new ClientResponse
{
Content = content,
Headers = response.Headers,
StatusCode = response.StatusCode
};
}
/// <summary>
/// Stages all files in the queue
/// </summary>
private async static System.Threading.Tasks.Task StageFilesAsync(List <IFileStagingProvider> filesToStage, SequentialFileStagingArtifact seqArtifacts)
{
foreach (IFileStagingProvider currentFile in filesToStage)
{
// for "retry" and/or "double calls" we ignore files that have already been staged
if (null == currentFile.StagedFiles)
{
FileToStage fts = currentFile as FileToStage;
if (null != fts)
{
System.Threading.Tasks.Task stageTask = StageOneFileAsync(fts, seqArtifacts);
await stageTask.ConfigureAwait(continueOnCapturedContext : false);
}
}
}
}
/// <summary>
/// Begins asynchronous call to return the contents of the file as a string.
/// </summary>
/// <param name="encoding">The encoding used to interpret the file data. If no value or null is specified, UTF8 is used.</param>
/// <param name="additionalBehaviors">A collection of BatchClientBehavior instances that are applied after the CustomBehaviors on the current object.</param>
/// <param name="cancellationToken">A <see cref="CancellationToken"/> for controlling the lifetime of the asynchronous operation.</param>
/// <returns>A <see cref="System.Threading.Tasks.Task"/> object that represents the asynchronous operation.</returns>
public async Task <string> ReadAsStringAsync(
Encoding encoding = null,
IEnumerable <BatchClientBehavior> additionalBehaviors = null,
CancellationToken cancellationToken = default(CancellationToken))
{
using (Stream streamToUse = new MemoryStream())
{
// get the data
System.Threading.Tasks.Task asyncTask = CopyToStreamAsync(streamToUse, additionalBehaviors, cancellationToken);
// wait for completion
await asyncTask.ConfigureAwait(continueOnCapturedContext : false);
streamToUse.Seek(0, SeekOrigin.Begin); //We just wrote to this stream, have to seek to the beginning
// convert to string
string result = UtilitiesInternal.StreamToString(streamToUse, encoding);
return(result);
}
}
public async Task <bool> MoveNextAsync(CancellationToken cancellationToken = default(CancellationToken))
{
try
{
// move to next item in current batch
++_currentIndex;
// if the index fits in the current batch, return true
if ((null != _currentBatch) && (_currentIndex < _currentBatch.Length)) // we have results in memory, just advance index
{
return(true);
}
// if we are out of data then return false
if (_skipHandler.AtLeastOneCallMade && !_skipHandler.ThereIsMoreData)
{
return(false);
}
// at this point we need to call for more data
System.Threading.Tasks.Task asyncTask = this.GetNextBatchFromServerAsync(_skipHandler, cancellationToken);
// wait for it to complete
await asyncTask.ConfigureAwait(continueOnCapturedContext : false);
// if we have new data in cache, serve it up
if ((null != _currentBatch) && (_currentBatch.Length > 0))
{
_currentIndex = 0;
return(true);
}
}
catch
{
throw; // set breakpoint here
}
return(false);
}
private async Task GetResponseCallback(Task<WebResponse> task)
{
// get the response
HttpWebResponse response;
try
{
response = (HttpWebResponse)await task.ConfigureAwait(false);
}
catch (Exception e)
{
if (e.InnerException != null && e.InnerException.Message.StartsWith("[net_WebHeaderInvalidControlChars]"))
{
// not an exception, everything is ok
this.InvokeInUiThread(() => this.OnCompleted(null));
}
else
{
this.InvokeOnErrorHandler(ErrorMessages.HttpPostError);
}
return;
}
if (response.StatusCode != HttpStatusCode.OK)
{
this.InvokeOnErrorHandler((int)response.StatusCode + " " + response.StatusDescription);
return;
}
// response stream
using (Stream stream = response.GetResponseStream())
{
using (StreamReader reader = new StreamReader(stream))
{
string str = reader.ReadToEnd();
this.InvokeInUiThread(() => this.OnCompleted(str));
}
}
}
/// <summary>
/// Starts an asynchronous call to stage the given files.
/// </summary>
private static async System.Threading.Tasks.Task StageFilesInternalAsync(List <IFileStagingProvider> filesToStage, IFileStagingArtifact fileStagingArtifact)
{
if (null == filesToStage)
{
throw new ArgumentNullException("filesToStage");
}
if (null == fileStagingArtifact)
{
throw new ArgumentNullException("filesStagingArtifact");
}
SequentialFileStagingArtifact seqArtifact = (SequentialFileStagingArtifact)fileStagingArtifact;
// is there any work to do?
if (null == FindAtLeastOne(filesToStage))
{
return; // now work to do. none of the files belong to this provider
}
// is there any work to do
if ((null == filesToStage) || (filesToStage.Count <= 0))
{
return; // we are done
}
// create a Run task to create the blob containers if needed
System.Threading.Tasks.Task createContainerTask = System.Threading.Tasks.Task.Run(async() => { await CreateDefaultBlobContainerAndSASIfNeededReturnAsync(filesToStage, seqArtifact); });
// wait for container to be created
await createContainerTask.ConfigureAwait(continueOnCapturedContext : false);
// begin staging the files
System.Threading.Tasks.Task stageTask = StageFilesAsync(filesToStage, seqArtifact);
// wait for files to be staged
await stageTask.ConfigureAwait(continueOnCapturedContext : false);
}
private async Task<IProjectMetric> InnerCalculate(Project project, Task<Compilation> compilationTask, Solution solution)
{
if (project == null)
{
return null;
}
var compilation = await compilationTask.ConfigureAwait(false);
var metricsTask = _metricsCalculator.Calculate(project, solution);
IEnumerable<string> dependencies;
if (solution != null)
{
var dependencyGraph = solution.GetProjectDependencyGraph();
dependencies = dependencyGraph.GetProjectsThatThisProjectTransitivelyDependsOn(project.Id)
.Select<ProjectId, Project>(id => solution.GetProject(id))
.SelectMany(x => x.MetadataReferences.Select(y => y.Display).Concat(new[] { x.AssemblyName }));
}
else
{
dependencies = project.AllProjectReferences.SelectMany(x => x.Aliases)
.Concat(project.MetadataReferences.Select(y => y.Display));
}
var assemblyTypes = compilation.Assembly.TypeNames;
var metrics = (await metricsTask.ConfigureAwait(false)).AsArray();
var internalTypesUsed = from metric in metrics
from coupling in metric.ClassCouplings
where coupling.Assembly == project.AssemblyName
select coupling;
var relationalCohesion = (internalTypesUsed.Count() + 1.0) / assemblyTypes.Count;
return new ProjectMetric(project.Name, metrics, dependencies, relationalCohesion);
}
public static ConfiguredTaskAwaitable ContinueOnAnyContext(this Task @this)
{
Ensure.ArgumentNotNull(@this, nameof(@this));
return(@this.ConfigureAwait(continueOnCapturedContext: false));
}
private async Task WriteAttributeStringAsyncHelper(Task task, string value)
{
await task.ConfigureAwait(false);
await WriteStringAsync(value).ConfigureAwait(false);
await WriteEndAttributeAsync().ConfigureAwait(false);
}
internal static void UnsafeOnCompleted(Task task, Action continuation)
{
var captureContext = NeedToConfigure();
var awaiter = task.ConfigureAwait(captureContext).GetAwaiter();
awaiter.UnsafeOnCompleted(continuation);
}
private async Task WriteStartAttributeAsync_NoAdvanceState(Task task, string prefix, string localName, string namespaceName) {
try {
await task.ConfigureAwait(false);
await WriteStartAttributeAsync_NoAdvanceState(prefix, localName, namespaceName).ConfigureAwait(false);
}
catch {
currentState = State.Error;
throw;
}
}
private static async Task WriteMessageCore(ValueTask writeMessageTask)
{
await writeMessageTask.ConfigureAwait(false);
GrpcEventSource.Log.MessageSent();
}
/// <summary>
/// Performs file staging and also issues the AddTaskCollection request for the set of tasks to add.
/// </summary>
/// <param name="tasksToAdd">The set of tasks to add.</param>
/// <param name="namingFragment"></param>
/// <returns></returns>
private async Task StageFilesAndAddTasks(
Dictionary <string, TrackedCloudTask> tasksToAdd,
string namingFragment)
{
List <Models.TaskAddParameter> protoTasksToAdd = new List <Models.TaskAddParameter>();
this.CheckForCancellationOrTimeoutAndThrow();
//
// Perform file staging
//
// list of all files to be staged across all Tasks
List <IFileStagingProvider> allFiles = new List <IFileStagingProvider>();
// collect all files to be staged
foreach (TrackedCloudTask trackedCloudTask in tasksToAdd.Values)
{
if (trackedCloudTask.Task.FilesToStage != null)
{
// add in the files for the current task
allFiles.AddRange(trackedCloudTask.Task.FilesToStage);
}
}
//This dictonary is only for the purpose of this batch add
ConcurrentDictionary <Type, IFileStagingArtifact> legStagingArtifacts = new ConcurrentDictionary <Type, IFileStagingArtifact>();
//Add the file staging artifacts for this let to the overall bag so as to allow customers to track the file staging progress
this._customerVisibleFileStagingArtifacts.Add(legStagingArtifacts);
// now we have all files, send off to file staging machine
System.Threading.Tasks.Task fileStagingTask = FileStagingLinkedSources.StageFilesAsync(allFiles, legStagingArtifacts, namingFragment);
// wait for file staging async task
await fileStagingTask.ConfigureAwait(continueOnCapturedContext : false);
// now update each non-finalized Task with its new ResourceFiles
foreach (TrackedCloudTask taskToAdd in tasksToAdd.Values)
{
//Update the resource files if the task hasn't already been finalized
if (taskToAdd.Task.FilesToStage != null)
{
foreach (IFileStagingProvider curFile in taskToAdd.Task.FilesToStage)
{
IEnumerable <ResourceFile> curStagedFiles = curFile.StagedFiles;
if (null != curStagedFiles && !((IReadOnly)taskToAdd.Task).IsReadOnly)
{
//TODO: There is a threading issue here -- lock this property down somehow?
if (taskToAdd.Task.ResourceFiles == null)
{
taskToAdd.Task.ResourceFiles = new List <ResourceFile>();
}
foreach (ResourceFile curStagedFile in curStagedFiles)
{
taskToAdd.Task.ResourceFiles.Add(curStagedFile);
}
}
}
//Mark the file staging collection as read only just incase there's another reference to it
ConcurrentChangeTrackedList <IFileStagingProvider> filesToStageListImpl =
taskToAdd.Task.FilesToStage as ConcurrentChangeTrackedList <IFileStagingProvider>;
filesToStageListImpl.IsReadOnly = true; //Set read only
}
Models.TaskAddParameter protoTask = taskToAdd.GetProtocolTask();
protoTasksToAdd.Add(protoTask);
}
this.CheckForCancellationOrTimeoutAndThrow();
//
// Fire the protocol add collection request
//
var asyncTask = this._jobOperations.ParentBatchClient.ProtocolLayer.AddTaskCollection(
this._jobId,
protoTasksToAdd,
this._behaviorManager,
this._parallelOptions.CancellationToken);
var response = await asyncTask.ConfigureAwait(continueOnCapturedContext : false);
//
// Process the results of the add task collection request
//
this.ProcessProtocolAddTaskResults(response.Body.Value, tasksToAdd);
}
private static async Task _CallTaskFuncWhenFinish(Task task, Func<Task> func) {
await task.ConfigureAwait(false);
await func().ConfigureAwait(false);
}
private async Task _TryReturnTask(Task task) {
try {
await task.ConfigureAwait(false);
}
catch {
currentState = State.Error;
throw;
}
}
public void OnCompleted(Action action)
{
_task.ConfigureAwait(false).GetAwaiter().OnCompleted(action);
}
private async Task _SequenceRun(Task task, Func<Task> nextTaskFun) {
try {
await task.ConfigureAwait(false);
await nextTaskFun().ConfigureAwait(false);
}
catch {
currentState = State.Error;
throw;
}
}
private async Task<bool> _ReadAsync_ReadAhead(Task task)
{
await task.ConfigureAwait(false);
_validationState = ValidatingReaderState.Read;
return true;
}
private async Task WriteBase64Async_NoAdvanceState(Task task, byte[] buffer, int index, int count) {
try {
await task.ConfigureAwait(false);
await writer.WriteBase64Async(buffer, index, count).ConfigureAwait(false);
}
catch {
currentState = State.Error;
throw;
}
}
private async Task WriteStringAsync_NoAdvanceState(Task task, string text) {
try {
await task.ConfigureAwait(false);
await WriteStringAsync_NoAdvanceState(text).ConfigureAwait(false);
}
catch {
currentState = State.Error;
throw;
}
}
/// <summary>
/// Stage a single file.
/// </summary>
private async static System.Threading.Tasks.Task StageOneFileAsync(FileToStage stageThisFile, SequentialFileStagingArtifact seqArtifacts)
{
StagingStorageAccount storecreds = stageThisFile.StagingStorageAccount;
string containerName = seqArtifacts.BlobContainerCreated;
// TODO: this flattens all files to the top of the compute node/task relative file directory. solve the hiearchy problem (virt dirs?)
string blobName = Path.GetFileName(stageThisFile.LocalFileToStage);
// Create the storage account with the connection string.
CloudStorageAccount storageAccount = new CloudStorageAccount(
new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(storecreds.StorageAccount, storecreds.StorageAccountKey),
blobEndpoint: storecreds.BlobUri,
queueEndpoint: null,
tableEndpoint: null,
fileEndpoint: null);
CloudBlobClient client = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = client.GetContainerReference(containerName);
ICloudBlob blob = container.GetBlockBlobReference(blobName);
bool doesBlobExist;
try
{
// fetch attributes so we can compare file lengths
System.Threading.Tasks.Task fetchTask = blob.FetchAttributesAsync();
await fetchTask.ConfigureAwait(continueOnCapturedContext : false);
doesBlobExist = true;
}
catch (StorageException scex)
{
// check to see if blob does not exist
if ((int)System.Net.HttpStatusCode.NotFound == scex.RequestInformation.HttpStatusCode)
{
doesBlobExist = false;
}
else
{
throw; // unknown exception, throw to caller
}
}
bool mustUploadBlob = true; // we do not re-upload blobs if they have already been uploaded
if (doesBlobExist) // if the blob exists, compare
{
FileInfo fi = new FileInfo(stageThisFile.LocalFileToStage);
// since we don't have a hash of the contents... we check length
if (blob.Properties.Length == fi.Length)
{
mustUploadBlob = false;
}
}
if (mustUploadBlob)
{
// upload the file
System.Threading.Tasks.Task uploadTask = blob.UploadFromFileAsync(stageThisFile.LocalFileToStage);
await uploadTask.ConfigureAwait(continueOnCapturedContext : false);
}
// get the SAS for the blob
string blobSAS = ConstructBlobSource(seqArtifacts.DefaultContainerSAS, blobName);
string nodeFileName = stageThisFile.NodeFileName;
// create a new ResourceFile and populate it. This file is now staged!
stageThisFile.StagedFiles = new ResourceFile[] { ResourceFile.FromUrl(blobSAS, nodeFileName) };
}
/// <summary>
/// Transfer the results of the <paramref name="outer"/> task's inner task to the <paramref name="completionSource"/>.
/// </summary>
/// <param name="completionSource">The completion source to which results should be transfered.</param>
/// <param name="outer">
/// The outer task that when completed will yield an inner task whose results we want marshaled to the <paramref name="completionSource"/>.
/// </param>
private static void TransferAsynchronously <TResult, TInner>(TaskCompletionSource <TResult> completionSource, Task <TInner> outer) where TInner : Task
{
Action[] callback = { null };
// Create a continuation delegate. For performance reasons, we reuse the same delegate/closure across multiple
// continuations; by using .ConfigureAwait(false).GetAwaiter().UnsafeOnComplete(action), in most cases
// this delegate can be stored directly into the Task's continuation field, eliminating the need for additional
// allocations. Thus, this whole Unwrap operation generally results in four allocations: one for the TaskCompletionSource,
// one for the returned task, one for the delegate, and one for the closure. Since the delegate is used
// across multiple continuations, we use the callback variable as well to indicate which continuation we're in:
// if the callback is non-null, then we're processing the continuation for the outer task and use the callback
// object as the continuation off of the inner task; if the callback is null, then we're processing the
// inner task.
callback[0] = delegate
{
Debug.Assert(outer.IsCompleted);
if (callback[0] != null)
{
// Process the outer task's completion
// Clear out the callback field to indicate that any future invocations should
// be for processing the inner task, but store away a local copy in case we need
// to use it as the continuation off of the outer task.
var innerCallback = callback[0];
callback[0] = null;
var result = true;
switch (outer.Status)
{
case TaskStatus.Canceled:
case TaskStatus.Faulted:
// The outer task has completed as canceled or faulted; transfer that
// status to the completion source, and we're done.
result = completionSource.TrySetFromTask(outer);
break;
case TaskStatus.RanToCompletion:
Task inner = outer.Result;
if (inner == null)
{
// The outer task completed successfully, but with a null inner task;
// cancel the completionSource, and we're done.
result = completionSource.TrySetCanceled();
}
else if (inner.IsCompleted)
{
// The inner task also completed! Transfer the results, and we're done.
result = completionSource.TrySetFromTask(inner);
}
else
{
// Run this delegate again once the inner task has completed.
inner.ConfigureAwait(false).GetAwaiter().UnsafeOnCompleted(innerCallback);
}
break;
}
Debug.Assert(result);
}
else
{
// Process the inner task's completion. All we need to do is transfer its results
// to the completion source.
Debug.Assert(outer.Status == TaskStatus.RanToCompletion);
Debug.Assert(outer.Result.IsCompleted);
completionSource.TrySetFromTask(outer.Result);
}
};
// Kick things off by hooking up the callback as the task's continuation
outer.ConfigureAwait(false).GetAwaiter().UnsafeOnCompleted(callback[0]);
}
private async Task _AdvanceStateAsync_ReturnWhenFinish(Task task, State newState) {
await task.ConfigureAwait(false);
currentState = newState;
}
private static async Task<bool> _ContinueBoolTaskFuncWhenFalse(Task<bool> task, Func<Task<bool>> func) {
if (await task.ConfigureAwait(false))
return true;
else
return await func().ConfigureAwait(false);
}
/// <summary>
/// Performs file staging and also issues the AddTaskCollection request for the set of tasks to add.
/// </summary>
/// <param name="tasksToAdd">The set of tasks to add.</param>
/// <param name="namingFragment"></param>
/// <returns></returns>
private async Task StageFilesAndAddTasks(
Dictionary <string, TrackedCloudTask> tasksToAdd,
string namingFragment)
{
List <Models.TaskAddParameter> protoTasksToAdd = new List <Models.TaskAddParameter>();
this.CheckForCancellationOrTimeoutAndThrow();
//
// Perform file staging
//
// list of all files to be staged across all Tasks
List <IFileStagingProvider> allFiles = new List <IFileStagingProvider>();
// collect all files to be staged
foreach (TrackedCloudTask trackedCloudTask in tasksToAdd.Values)
{
if (trackedCloudTask.Task.FilesToStage != null)
{
// add in the files for the current task
allFiles.AddRange(trackedCloudTask.Task.FilesToStage);
}
}
//This dictonary is only for the purpose of this batch add
ConcurrentDictionary <Type, IFileStagingArtifact> legStagingArtifacts = new ConcurrentDictionary <Type, IFileStagingArtifact>();
//Add the file staging artifacts for this let to the overall bag so as to allow customers to track the file staging progress
this._customerVisibleFileStagingArtifacts.Add(legStagingArtifacts);
// now we have all files, send off to file staging machine
System.Threading.Tasks.Task fileStagingTask = FileStagingUtils.StageFilesAsync(allFiles, legStagingArtifacts, namingFragment);
// wait for file staging async task
await fileStagingTask.ConfigureAwait(continueOnCapturedContext : false);
// now update each non-finalized Task with its new ResourceFiles
foreach (TrackedCloudTask taskToAdd in tasksToAdd.Values)
{
//Update the resource files if the task hasn't already been finalized
if (taskToAdd.Task.FilesToStage != null)
{
foreach (IFileStagingProvider curFile in taskToAdd.Task.FilesToStage)
{
IEnumerable <ResourceFile> curStagedFiles = curFile.StagedFiles;
if (null != curStagedFiles && !((IReadOnly)taskToAdd.Task).IsReadOnly)
{
//TODO: There is a threading issue here -- lock this property down somehow?
if (taskToAdd.Task.ResourceFiles == null)
{
taskToAdd.Task.ResourceFiles = new List <ResourceFile>();
}
foreach (ResourceFile curStagedFile in curStagedFiles)
{
taskToAdd.Task.ResourceFiles.Add(curStagedFile);
}
}
}
//Mark the file staging collection as read only just incase there's another reference to it
ConcurrentChangeTrackedList <IFileStagingProvider> filesToStageListImpl =
taskToAdd.Task.FilesToStage as ConcurrentChangeTrackedList <IFileStagingProvider>;
filesToStageListImpl.IsReadOnly = true; //Set read only
}
Models.TaskAddParameter protoTask = taskToAdd.GetProtocolTask();
protoTasksToAdd.Add(protoTask);
}
this.CheckForCancellationOrTimeoutAndThrow();
//
// Fire the protocol add collection request
//
try
{
var asyncTask = this._jobOperations.ParentBatchClient.ProtocolLayer.AddTaskCollection(
this._jobId,
protoTasksToAdd,
this._behaviorManager,
this._parallelOptions.CancellationToken);
var response = await asyncTask.ConfigureAwait(continueOnCapturedContext : false);
//
// Process the results of the add task collection request
//
this.ProcessProtocolAddTaskResults(response.Body.Value, tasksToAdd);
}
catch (Common.BatchException e)
{
if (e.InnerException is Models.BatchErrorException)
{
Models.BatchError error = ((Models.BatchErrorException)e.InnerException).Body;
int currLength = tasksToAdd.Count;
if (error.Code == Common.BatchErrorCodeStrings.RequestBodyTooLarge && currLength != 1)
{
// Our chunk sizes were too large to fit in a request, so universally reduce size
// This is an internal error due to us using greedy initial maximum chunk size,
// so do not increment retry counter.
{
int newLength = currLength / 2;
int tmpMaxTasks = this._maxTasks;
while (newLength < tmpMaxTasks)
{
tmpMaxTasks = Interlocked.CompareExchange(ref this._maxTasks, newLength, tmpMaxTasks);
}
foreach (TrackedCloudTask trackedTask in tasksToAdd.Values)
{
this._remainingTasksToAdd.Enqueue(trackedTask);
}
return;
}
}
}
throw;
}
}
/// <summary>
/// This is the worker thread that loads the datasources from disk.
/// </summary>
private void ReadLevelsProc()
{
ReadDataSource("BuiltInWorlds", Genres.BuiltInWorlds, StorageSource.TitleSpace);
ReadDataSource("MyWorlds", Genres.MyWorlds, StorageSource.UserSpace);
ReadDataSource("Downloads", Genres.Downloads, StorageSource.UserSpace);
#if NETFX_CORE
working = false;
#else
thread = null;
#endif
for (; ;)
{
if (!running || !BokuGame.Running)
{
break;
}
// Wait for a wake-up signal
#if NETFX_CORE
if (!signal.WaitOne(10))
{
continue;
}
#else
if (!signal.WaitOne(10, false))
{
continue;
}
#endif
// Process all queued thumbnail load requests.
for (; ;)
{
lock (Synch)
{
if (thumbnailQueue.Count == 0 || !running || !BokuGame.Running)
{
break;
}
LevelMetadata level = thumbnailQueue[thumbnailQueue.Count - 1];
thumbnailQueue.RemoveAt(thumbnailQueue.Count - 1);
try
{
string texFilename = BokuGame.Settings.MediaPath + Utils.FolderNameFromFlags(level.Genres) + level.WorldId.ToString();
Stream texStream = OpenTextureStream(texFilename);
if (texStream != null)
{
level.ThumbnailBytes = new byte[texStream.Length];
texStream.Read(level.ThumbnailBytes, 0, (int)texStream.Length);
Storage4.Close(texStream);
thumbnailCompletions.Add(level);
}
}
catch { }
}
// Let the main thread have the cpu so it can deliver the thumbnail to the level.
#if NETFX_CORE
{
System.Threading.Tasks.Task delayTask = System.Threading.Tasks.Task.Delay(1);
delayTask.ConfigureAwait(false);
delayTask.Wait();
}
#else
Thread.Sleep(1);
#endif
}
}
}
/// <summary>
/// Attempts to download the file.
/// </summary>
/// <param name="uri">The parsed <see cref="Uri"/> of the request.</param>
/// <param name="filename">The filename to use when downloading the file.</param>
private void Download(Uri uri)
{
// The main reason to use HttpClient vs WebClient is because we can pass a message handler for unit tests to mock
using (HttpClient client = new HttpClient(HttpMessageHandler ?? new HttpClientHandler(), disposeHandler: true))
{
// Only get the response without downloading the file so we can determine if the file is already up-to-date
using (HttpResponseMessage response = client.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead, _cancellationTokenSource.Token)
.ConfigureAwait(continueOnCapturedContext: false)
.GetAwaiter()
.GetResult())
{
try
{
response.EnsureSuccessStatusCode();
}
catch (HttpRequestException e)
{
// HttpRequestException does not have the status code so its wrapped and thrown here so that later on we can determine
// if a retry is possible based on the status code
throw new CustomHttpRequestException(e.Message, e.InnerException, response.StatusCode);
}
if (!TryGetFileName(response, out string filename))
{
Log.LogErrorFromResources("DownloadFile.ErrorUnknownFileName", SourceUrl, nameof(DestinationFileName));
return;
}
DirectoryInfo destinationDirectory = Directory.CreateDirectory(DestinationFolder.ItemSpec);
FileInfo destinationFile = new FileInfo(Path.Combine(destinationDirectory.FullName, filename));
// The file is considered up-to-date if its the same length. This could be inaccurate, we can consider alternatives in the future
if (ShouldSkip(response, destinationFile))
{
Log.LogMessageFromResources(MessageImportance.Normal, "DownloadFile.DidNotDownloadBecauseOfFileMatch", SourceUrl, destinationFile.FullName, nameof(SkipUnchangedFiles), "true");
DownloadedFile = new TaskItem(destinationFile.FullName);
return;
}
try
{
using (FileStream target = new FileStream(destinationFile.FullName, FileMode.Create, FileAccess.Write, FileShare.None))
{
Log.LogMessageFromResources(MessageImportance.High, "DownloadFile.Downloading", SourceUrl, destinationFile.FullName, response.Content.Headers.ContentLength);
Task task = response.Content.CopyToAsync(target);
task.ConfigureAwait(continueOnCapturedContext: false);
task.Wait(_cancellationTokenSource.Token);
DownloadedFile = new TaskItem(destinationFile.FullName);
}
}
finally
{
if (DownloadedFile == null)
{
// Delete the file if anything goes wrong during download. This could be destructive but we don't want to leave
// partially downloaded files on disk either. Alternatively we could download to a temporary location and copy
// on success but we are concerned about the added I/O
destinationFile.Delete();
}
}
}
}
}
private async Task<bool> _ReadAsync_Read(Task<bool> task)
{
if (await task.ConfigureAwait(false))
{
await ProcessReaderEventAsync().ConfigureAwait(false);
return true;
}
else
{
_validator.EndValidation();
if (_coreReader.EOF)
{
_validationState = ValidatingReaderState.EOF;
}
return false;
}
}
private async Task WriteStartElementAsync_FinishWrite(Task t, string prefix, string localName, string ns) {
try {
await t.ConfigureAwait(false);
WriteStartElementAsync_FinishWrite(prefix, localName, ns);
}
catch {
currentState = State.Error;
throw;
}
}
private async Task _AdvanceStateAsync_ContinueWhenFinish(Task task, State newState, Token token) {
await task.ConfigureAwait(false);
currentState = newState;
await AdvanceStateAsync(token).ConfigureAwait(false);
}
internal static async System.Threading.Tasks.Task StageFilesAsync(List <IFileStagingProvider> filesToStage, ConcurrentDictionary <Type, IFileStagingArtifact> allFileStagingArtifacts, string namingFragment)
{
try
{
if (null == allFileStagingArtifacts)
{
throw new ArgumentNullException("allFileStagingArtifacts");
}
if (allFileStagingArtifacts.Count > 0)
{
throw new ArgumentOutOfRangeException("allFileStagingArtifacts.Count");
}
// first we get the buckets. One for each file staging provider that contains only the files for that provider.
Dictionary <Type, List <IFileStagingProvider> > bucketByProviders = BucketizeFileStagingProviders(filesToStage);
// missing artifacts will be instantiated and stored here temporarily
Dictionary <Type, IFileStagingArtifact> pendingArtifactsToAdd = new Dictionary <Type, IFileStagingArtifact>();
// detect any missing staging artifacts. Each bucket must have a staging artifact.
foreach (Type curProviderType in bucketByProviders.Keys)
{
IFileStagingArtifact curProviderArtifact;
// if no artifact was passed in, instantiate one and have it added
if (!allFileStagingArtifacts.TryGetValue(curProviderType, out curProviderArtifact))
{
// we need to have the staging provider create an artifact instance
// so first we retrieve the list of files and ask one of them
List <IFileStagingProvider> filesForProviderType;
if (bucketByProviders.TryGetValue(curProviderType, out filesForProviderType))
{
Debug.Assert(filesForProviderType.Count > 0); // to be in a bucket means there must be at least one.
IFileStagingProvider curProviderAsInterface = filesForProviderType[0];
IFileStagingArtifact newArtifactFreshFromProvider = curProviderAsInterface.CreateStagingArtifact();
// give the file stager the naming fragment if it does not already have one by default
if (string.IsNullOrEmpty(newArtifactFreshFromProvider.NamingFragment) && !string.IsNullOrEmpty(namingFragment))
{
newArtifactFreshFromProvider.NamingFragment = namingFragment;
}
pendingArtifactsToAdd.Add(curProviderType, newArtifactFreshFromProvider);
}
}
}
// add missing artifacts to collection
foreach (Type curProvderType in pendingArtifactsToAdd.Keys)
{
IFileStagingArtifact curArtifact;
if (pendingArtifactsToAdd.TryGetValue(curProvderType, out curArtifact))
{
allFileStagingArtifacts.TryAdd(curProvderType, curArtifact);
}
}
// now we have buckets of files for each provider and artifacts for each provider
// start tasks for each provider
// list of all running providers
List <System.Threading.Tasks.Task> runningProviders = new List <System.Threading.Tasks.Task>();
// start a task for each FileStagingProvider
foreach (List <IFileStagingProvider> curProviderFilesToStage in bucketByProviders.Values)
{
Debug.Assert(curProviderFilesToStage.Count > 0);
IFileStagingProvider anyInstance = curProviderFilesToStage[0]; // had to be at least one to get here.
System.Threading.Tasks.Task providerTask; // this is the async task for this provider
IFileStagingArtifact stagingArtifact; // artifact for this provider
if (allFileStagingArtifacts.TryGetValue(anyInstance.GetType(), out stagingArtifact)) // register the staging artifact
{
providerTask = anyInstance.StageFilesAsync(curProviderFilesToStage, stagingArtifact);
runningProviders.Add(providerTask);
}
else
{
Debug.Assert(true, "The staging artifacts collection is somehow missing an artifact for " + anyInstance.GetType().ToString());
}
}
//
// the individual tasks were created above
// now a-wait for them all to finish
//
System.Threading.Tasks.Task[] runningArray = runningProviders.ToArray();
System.Threading.Tasks.Task allRunningTasks = System.Threading.Tasks.Task.WhenAll(runningArray);
// actual a-wait for all the providers
await allRunningTasks.ConfigureAwait(continueOnCapturedContext : false);
}
catch (Exception ex)
{
if (null != ex)
{
throw; // TODO: trace here?
}
}
}