private async Task DownloadFiles(FileDownloadManagerContext context, IEnumerable<FileDownloadManagerItem> items)
{
var client = new HttpClient();
client.DefaultRequestHeaders.CacheControl = new CacheControlHeaderValue();
client.DefaultRequestHeaders.CacheControl.NoCache = true;
client.DefaultRequestHeaders.Add("Connection", "Keep-alive");
if (context.Timeout != null)
client.Timeout = context.Timeout;
IEnumerable<Task> downloadTasksQuery =
from item in items
select ProcessUrl(context, client, item);
// now execute the bunch
List<Task> downloadTasks = downloadTasksQuery.ToList();
while (downloadTasks.Count > 0)
{
// identify the first task that completes
Task firstFinishedTask = await Task.WhenAny(downloadTasks);
// process only once
downloadTasks.Remove(firstFinishedTask);
await firstFinishedTask;
}
}
private async Task DownloadFiles(FileDownloadManagerContext context, IEnumerable <FileDownloadManagerItem> items)
{
var client = new HttpClient();
client.DefaultRequestHeaders.CacheControl = new CacheControlHeaderValue();
client.DefaultRequestHeaders.CacheControl.NoCache = true;
client.DefaultRequestHeaders.Add("Connection", "Keep-alive");
if (context.Timeout != null)
{
client.Timeout = context.Timeout;
}
IEnumerable <Task> downloadTasksQuery =
from item in items
select ProcessUrl(context, client, item);
// now execute the bunch
List <Task> downloadTasks = downloadTasksQuery.ToList();
while (downloadTasks.Count > 0)
{
// identify the first task that completes
Task firstFinishedTask = await Task.WhenAny(downloadTasks);
// process only once
downloadTasks.Remove(firstFinishedTask);
await firstFinishedTask;
}
}
private async Task ProcessUrl(FileDownloadManagerContext context, HttpClient client, FileDownloadManagerItem item)
{
try
{
//HttpResponseMessage response = await client.GetAsync(item.Url, HttpCompletionOption.ResponseHeadersRead);
//Task<Stream> task = response.Content.ReadAsStreamAsync();
Task <Stream> task = client.GetStreamAsync(item.Url);
await task;
int count;
bool canceled = false;
byte[] bytes = new byte[_bufferSize];
using (var srcStream = task.Result)
using (var dstStream = File.OpenWrite(item.Path))
{
while ((count = srcStream.Read(bytes, 0, bytes.Length)) != 0 && !canceled)
{
dstStream.Write(bytes, 0, count);
if (context.CancellationToken != null && context.CancellationToken.IsCancellationRequested)
{
canceled = true;
}
}
}
item.Success = (!task.IsFaulted && !canceled);
}
catch (Exception exception)
{
try
{
item.Success = false;
item.ErrorMessage = exception.ToAllMessages();
var webExc = exception.InnerException as WebException;
if (webExc != null)
{
item.ExceptionStatus = webExc.Status;
}
if (context.Logger != null)
{
context.Logger.Error(item.ToString(), exception);
}
}
catch { }
}
}
private async Task ProcessUrl(FileDownloadManagerContext context, HttpClient client, FileDownloadManagerItem item)
{
try
{
Task<Stream> task = client.GetStreamAsync(item.Url);
await task;
int count;
bool canceled = false;
byte[] bytes = new byte[_bufferSize];
using (var srcStream = task.Result)
using (var dstStream = File.OpenWrite(item.Path))
{
while ((count = srcStream.Read(bytes, 0, bytes.Length)) != 0 && !canceled)
{
dstStream.Write(bytes, 0, count);
if (context.CancellationToken != null && context.CancellationToken.IsCancellationRequested)
canceled = true;
}
}
item.Success = (!task.IsFaulted && !canceled);
}
catch (Exception exc)
{
item.Success = false;
item.ErrorMessage = exc.ToAllMessages();
var webExc = exc.InnerException as WebException;
if (webExc != null)
item.ExceptionStatus = webExc.Status;
if (context.Logger != null)
context.Logger.Error(item.ToString(), exc);
}
}
private async Task DownloadFiles(FileDownloadManagerContext context, IEnumerable<FileDownloadManagerItem> items)
{
try
{
using (var client = new HttpClient())
{
client.DefaultRequestHeaders.CacheControl = new CacheControlHeaderValue();
client.DefaultRequestHeaders.CacheControl.NoCache = true;
client.DefaultRequestHeaders.Add("Connection", "Keep-alive");
if (context.Timeout.TotalMilliseconds > 0 && context.Timeout != Timeout.InfiniteTimeSpan)
client.Timeout = context.Timeout;
IEnumerable<Task> downloadTasksQuery =
from item in items
select ProcessUrl(context, client, item);
// now execute the bunch
List<Task> downloadTasks = downloadTasksQuery.ToList();
while (downloadTasks.Count > 0)
{
// identify the first task that completes
Task firstFinishedTask = await Task.WhenAny(downloadTasks);
// process only once
downloadTasks.Remove(firstFinishedTask);
await firstFinishedTask;
}
}
}
catch (Exception exception)
{
if (context.Logger != null)
context.Logger.ErrorsAll(exception);
}
}
private async Task ProcessUrl(FileDownloadManagerContext context, HttpClient client, FileDownloadManagerItem item)
{
try
{
var count = 0;
var canceled = false;
var bytes = new byte[_bufferSize];
using (var response = await client.GetAsync(item.Url))
{
if (response.IsSuccessStatusCode)
{
if (response.Content.Headers.ContentType != null)
{
var contentType = response.Content.Headers.ContentType.MediaType;
if (contentType.HasValue() && !contentType.IsCaseInsensitiveEqual(item.MimeType))
{
// Update mime type and local path.
var extension = MimeTypes.MapMimeTypeToExtension(contentType).NullEmpty() ?? ".jpg";
item.MimeType = contentType;
item.Path = Path.ChangeExtension(item.Path, extension.EnsureStartsWith("."));
}
}
//Task <Stream> task = client.GetStreamAsync(item.Url);
Task <Stream> task = response.Content.ReadAsStreamAsync();
await task;
using (var srcStream = task.Result)
using (var dstStream = File.Open(item.Path, FileMode.Create))
{
while ((count = srcStream.Read(bytes, 0, bytes.Length)) != 0 && !canceled)
{
dstStream.Write(bytes, 0, count);
if (context.CancellationToken != null && context.CancellationToken.IsCancellationRequested)
{
canceled = true;
}
}
}
item.Success = !task.IsFaulted && !canceled;
}
else
{
item.Success = false;
item.ErrorMessage = response.StatusCode.ToString();
}
}
}
catch (Exception ex)
{
try
{
item.Success = false;
item.ErrorMessage = ex.ToAllMessages();
var webExc = ex.InnerException as WebException;
if (webExc != null)
{
item.ExceptionStatus = webExc.Status;
}
if (context.Logger != null)
{
context.Logger.Error(ex, item.ToString());
}
}
catch { }
}
}
/// <summary>
/// Starts asynchronous download of files and saves them to disk
/// </summary>
/// <param name="context">Download context</param>
/// <param name="items">Items to be downloaded</param>
public async Task DownloadAsync(FileDownloadManagerContext context, IEnumerable <FileDownloadManagerItem> items)
{
await DownloadFilesAsync(context, items);
}
/// <summary>
/// Starts asynchronous download of files and saves them to disk
/// </summary>
/// <param name="context">Download context</param>
/// <param name="items">Items to be downloaded</param>
public async Task DownloadAsync(FileDownloadManagerContext context, IEnumerable<FileDownloadManagerItem> items)
{
await DownloadFiles(context, items);
}
