private async Task DownloadFiles(FileDownloadManagerContext context, IEnumerable<FileDownloadManagerItem> items) { var client = new HttpClient(); client.DefaultRequestHeaders.CacheControl = new CacheControlHeaderValue(); client.DefaultRequestHeaders.CacheControl.NoCache = true; client.DefaultRequestHeaders.Add("Connection", "Keep-alive"); if (context.Timeout != null) client.Timeout = context.Timeout; IEnumerable<Task> downloadTasksQuery = from item in items select ProcessUrl(context, client, item); // now execute the bunch List<Task> downloadTasks = downloadTasksQuery.ToList(); while (downloadTasks.Count > 0) { // identify the first task that completes Task firstFinishedTask = await Task.WhenAny(downloadTasks); // process only once downloadTasks.Remove(firstFinishedTask); await firstFinishedTask; } }
private async Task DownloadFiles(FileDownloadManagerContext context, IEnumerable <FileDownloadManagerItem> items) { var client = new HttpClient(); client.DefaultRequestHeaders.CacheControl = new CacheControlHeaderValue(); client.DefaultRequestHeaders.CacheControl.NoCache = true; client.DefaultRequestHeaders.Add("Connection", "Keep-alive"); if (context.Timeout != null) { client.Timeout = context.Timeout; } IEnumerable <Task> downloadTasksQuery = from item in items select ProcessUrl(context, client, item); // now execute the bunch List <Task> downloadTasks = downloadTasksQuery.ToList(); while (downloadTasks.Count > 0) { // identify the first task that completes Task firstFinishedTask = await Task.WhenAny(downloadTasks); // process only once downloadTasks.Remove(firstFinishedTask); await firstFinishedTask; } }
private async Task ProcessUrl(FileDownloadManagerContext context, HttpClient client, FileDownloadManagerItem item) { try { //HttpResponseMessage response = await client.GetAsync(item.Url, HttpCompletionOption.ResponseHeadersRead); //Task<Stream> task = response.Content.ReadAsStreamAsync(); Task <Stream> task = client.GetStreamAsync(item.Url); await task; int count; bool canceled = false; byte[] bytes = new byte[_bufferSize]; using (var srcStream = task.Result) using (var dstStream = File.OpenWrite(item.Path)) { while ((count = srcStream.Read(bytes, 0, bytes.Length)) != 0 && !canceled) { dstStream.Write(bytes, 0, count); if (context.CancellationToken != null && context.CancellationToken.IsCancellationRequested) { canceled = true; } } } item.Success = (!task.IsFaulted && !canceled); } catch (Exception exception) { try { item.Success = false; item.ErrorMessage = exception.ToAllMessages(); var webExc = exception.InnerException as WebException; if (webExc != null) { item.ExceptionStatus = webExc.Status; } if (context.Logger != null) { context.Logger.Error(item.ToString(), exception); } } catch { } } }
private async Task ProcessUrl(FileDownloadManagerContext context, HttpClient client, FileDownloadManagerItem item) { try { Task<Stream> task = client.GetStreamAsync(item.Url); await task; int count; bool canceled = false; byte[] bytes = new byte[_bufferSize]; using (var srcStream = task.Result) using (var dstStream = File.OpenWrite(item.Path)) { while ((count = srcStream.Read(bytes, 0, bytes.Length)) != 0 && !canceled) { dstStream.Write(bytes, 0, count); if (context.CancellationToken != null && context.CancellationToken.IsCancellationRequested) canceled = true; } } item.Success = (!task.IsFaulted && !canceled); } catch (Exception exc) { item.Success = false; item.ErrorMessage = exc.ToAllMessages(); var webExc = exc.InnerException as WebException; if (webExc != null) item.ExceptionStatus = webExc.Status; if (context.Logger != null) context.Logger.Error(item.ToString(), exc); } }
private async Task DownloadFiles(FileDownloadManagerContext context, IEnumerable<FileDownloadManagerItem> items) { try { using (var client = new HttpClient()) { client.DefaultRequestHeaders.CacheControl = new CacheControlHeaderValue(); client.DefaultRequestHeaders.CacheControl.NoCache = true; client.DefaultRequestHeaders.Add("Connection", "Keep-alive"); if (context.Timeout.TotalMilliseconds > 0 && context.Timeout != Timeout.InfiniteTimeSpan) client.Timeout = context.Timeout; IEnumerable<Task> downloadTasksQuery = from item in items select ProcessUrl(context, client, item); // now execute the bunch List<Task> downloadTasks = downloadTasksQuery.ToList(); while (downloadTasks.Count > 0) { // identify the first task that completes Task firstFinishedTask = await Task.WhenAny(downloadTasks); // process only once downloadTasks.Remove(firstFinishedTask); await firstFinishedTask; } } } catch (Exception exception) { if (context.Logger != null) context.Logger.ErrorsAll(exception); } }
private async Task ProcessUrl(FileDownloadManagerContext context, HttpClient client, FileDownloadManagerItem item) { try { var count = 0; var canceled = false; var bytes = new byte[_bufferSize]; using (var response = await client.GetAsync(item.Url)) { if (response.IsSuccessStatusCode) { if (response.Content.Headers.ContentType != null) { var contentType = response.Content.Headers.ContentType.MediaType; if (contentType.HasValue() && !contentType.IsCaseInsensitiveEqual(item.MimeType)) { // Update mime type and local path. var extension = MimeTypes.MapMimeTypeToExtension(contentType).NullEmpty() ?? ".jpg"; item.MimeType = contentType; item.Path = Path.ChangeExtension(item.Path, extension.EnsureStartsWith(".")); } } //Task <Stream> task = client.GetStreamAsync(item.Url); Task <Stream> task = response.Content.ReadAsStreamAsync(); await task; using (var srcStream = task.Result) using (var dstStream = File.Open(item.Path, FileMode.Create)) { while ((count = srcStream.Read(bytes, 0, bytes.Length)) != 0 && !canceled) { dstStream.Write(bytes, 0, count); if (context.CancellationToken != null && context.CancellationToken.IsCancellationRequested) { canceled = true; } } } item.Success = !task.IsFaulted && !canceled; } else { item.Success = false; item.ErrorMessage = response.StatusCode.ToString(); } } } catch (Exception ex) { try { item.Success = false; item.ErrorMessage = ex.ToAllMessages(); var webExc = ex.InnerException as WebException; if (webExc != null) { item.ExceptionStatus = webExc.Status; } if (context.Logger != null) { context.Logger.Error(ex, item.ToString()); } } catch { } } }
/// <summary> /// Starts asynchronous download of files and saves them to disk /// </summary> /// <param name="context">Download context</param> /// <param name="items">Items to be downloaded</param> public async Task DownloadAsync(FileDownloadManagerContext context, IEnumerable <FileDownloadManagerItem> items) { await DownloadFilesAsync(context, items); }
/// <summary> /// Starts asynchronous download of files and saves them to disk /// </summary> /// <param name="context">Download context</param> /// <param name="items">Items to be downloaded</param> public async Task DownloadAsync(FileDownloadManagerContext context, IEnumerable<FileDownloadManagerItem> items) { await DownloadFiles(context, items); }