private async Task <HttpResponseMessage> SendDownloadRequestAsync(DownloadItem downloadItem, string url, string referer, long?startPosition = null)
{
bool partialDownload = startPosition.HasValue && startPosition.Value > 0;
if (!partialDownload)
{
Logger.Debug($"Requesting {url}");
}
else
{
Logger.Debug($"Requesting {url}, range: {startPosition.Value} - end.");
}
HttpRequestMessage request;
try
{
request = new HttpRequestMessage(HttpMethod.Get, url);
}
catch (Exception exception)
{
Logger.Exception(exception);
ReportError(downloadItem, localization.GetLogLineRequestError(Uri.UnescapeDataString(url)));
return(null);
}
request.Headers.UserAgent.ParseAdd(USER_AGENT);
if (downloadItem.Cookies.Any())
{
request.Headers.Add("Cookie", GenerateCookieHeader(downloadItem.Cookies));
}
if (partialDownload)
{
request.Headers.Range = new RangeHeaderValue(startPosition.Value, null);
}
if (!String.IsNullOrEmpty(referer))
{
request.Headers.Referrer = new Uri(referer);
}
string requestHeaders = request.Headers.ToString().TrimEnd();
Logger.Debug("Request headers:", requestHeaders);
StringBuilder requestLogBuilder = new StringBuilder();
requestLogBuilder.Append(localization.LogLineRequest);
requestLogBuilder.AppendLine(":");
requestLogBuilder.Append("GET ");
requestLogBuilder.AppendLine(url);
requestLogBuilder.AppendLine(requestHeaders);
AddLogLine(downloadItem, DownloadItemLogLineType.DEBUG, requestLogBuilder.ToString().TrimEnd());
HttpResponseMessage response;
try
{
response = await SendRequestAsync(request, downloadItem.CancellationToken);
}
catch (TimeoutException)
{
Logger.Debug("Download timeout.");
ReportStatusChange(downloadItem, DownloadItemStatus.RETRY_DELAY, DownloadItemLogLineType.INFORMATIONAL,
localization.LogLineServerResponseTimeout);
return(null);
}
catch (AggregateException aggregateException)
{
if (downloadItem.CancellationToken.IsCancellationRequested)
{
Logger.Debug("File download has been cancelled.");
}
else
{
Logger.Exception(aggregateException);
ReportError(downloadItem, localization.LogLineUnexpectedError);
}
return(null);
}
catch (Exception exception)
{
Logger.Exception(exception);
ReportError(downloadItem, localization.LogLineUnexpectedError);
return(null);
}
Logger.Debug($"Response status code: {(int)response.StatusCode} {response.StatusCode}.");
string responseHeaders = response.Headers.ToString().TrimEnd();
string responseContentHeaders = response.Content.Headers.ToString().TrimEnd();
Logger.Debug("Response headers:", responseHeaders, responseContentHeaders);
StringBuilder responseLogBuilder = new StringBuilder();
responseLogBuilder.Append(localization.LogLineResponse);
responseLogBuilder.AppendLine(":");
responseLogBuilder.Append((int)response.StatusCode);
responseLogBuilder.Append(" ");
responseLogBuilder.AppendLine(response.StatusCode.ToString());
if (!String.IsNullOrEmpty(responseHeaders))
{
responseLogBuilder.AppendLine(responseHeaders);
}
if (!String.IsNullOrEmpty(responseContentHeaders))
{
responseLogBuilder.AppendLine(responseContentHeaders);
}
AddLogLine(downloadItem, DownloadItemLogLineType.DEBUG, responseLogBuilder.ToString().TrimEnd());
if (response.Headers.TryGetValues("Set-Cookie", out IEnumerable <string> cookieHeaders))
{
Uri uri = new Uri(url);
foreach (string cookieHeader in cookieHeaders)
{
AppendCookies(downloadItem.Cookies, uri, cookieHeader);
}
}
return(response);
}
private async Task DownloadFileAsync(DownloadItem downloadItem, FileStream destinationFileStream)
{
string url = downloadItem.DirectFileUrl;
string referer = downloadItem.Referer;
long startPosition = destinationFileStream.Position;
bool partialDownload = startPosition > 0;
bool isRedirect;
int redirectCount = 0;
HttpResponseMessage response;
do
{
response = await SendDownloadRequestAsync(downloadItem, url, referer, startPosition);
if (response == null)
{
return;
}
isRedirect = response.StatusCode == HttpStatusCode.MovedPermanently || response.StatusCode == HttpStatusCode.Redirect ||
response.StatusCode == HttpStatusCode.TemporaryRedirect;
if (isRedirect)
{
referer = url;
url = response.Headers.Location.AbsoluteUri;
AddLogLine(downloadItem, DownloadItemLogLineType.INFORMATIONAL, localization.GetLogLineRedirect(Uri.UnescapeDataString(url)));
redirectCount++;
if (redirectCount == MAX_DOWNLOAD_REDIRECT_COUNT)
{
ReportError(downloadItem, localization.LogLineTooManyRedirects);
return;
}
}
}while (isRedirect);
if (response.StatusCode != HttpStatusCode.OK && response.StatusCode != HttpStatusCode.PartialContent)
{
string statusCode = $"{(int)response.StatusCode} {response.StatusCode}";
Logger.Debug($"Server returned non-successful status code: {statusCode}.");
string messageText = localization.GetLogLineNonSuccessfulStatusCode(statusCode);
if (response.StatusCode == HttpStatusCode.InternalServerError || response.StatusCode == HttpStatusCode.BadGateway ||
response.StatusCode == HttpStatusCode.ServiceUnavailable || response.StatusCode == HttpStatusCode.GatewayTimeout)
{
ReportStatusChange(downloadItem, DownloadItemStatus.RETRY_DELAY, DownloadItemLogLineType.INFORMATIONAL, messageText);
}
else
{
ReportError(downloadItem, messageText);
}
return;
}
if (response.Content.Headers.ContentType != null && response.Content.Headers.ContentType.MediaType.CompareOrdinalIgnoreCase("text/html"))
{
Logger.Debug($"Server returned HTML page instead of the file.");
ReportError(downloadItem, localization.LogLineHtmlPageReturned);
return;
}
if (partialDownload && response.StatusCode == HttpStatusCode.OK)
{
Logger.Debug("Server doesn't support partial downloads.");
ReportError(downloadItem, localization.LogLineNoPartialDownloadSupport);
return;
}
long?contentLength = response.Content.Headers.ContentLength;
if (!contentLength.HasValue)
{
Logger.Debug("Server did not return Content-Length value.");
AddLogLine(downloadItem, DownloadItemLogLineType.INFORMATIONAL, localization.LogLineNoContentLengthWarning);
}
long?remainingSize = contentLength;
lock (downloadQueueLock)
{
downloadItem.DownloadedFileSize = startPosition;
if (remainingSize.HasValue)
{
downloadItem.TotalFileSize = startPosition + remainingSize;
}
else
{
downloadItem.TotalFileSize = null;
}
ReportChange(downloadItem);
}
if (remainingSize == 0)
{
Logger.Debug($"File download is complete.");
lock (downloadQueueLock)
{
downloadItem.Status = DownloadItemStatus.COMPLETED;
}
return;
}
if (partialDownload)
{
if (remainingSize.HasValue)
{
Logger.Debug($"Remaining download size is {remainingSize.Value} bytes.");
AddLogLine(downloadItem, DownloadItemLogLineType.INFORMATIONAL,
localization.GetLogLineResumingFileDownloadKnownFileSize(remainingSize.Value));
}
else
{
Logger.Debug($"Remaining download size is unknown.");
AddLogLine(downloadItem, DownloadItemLogLineType.INFORMATIONAL, localization.LogLineResumingFileDownloadUnknownFileSize);
}
}
else
{
if (remainingSize.HasValue)
{
Logger.Debug($"Download file size is {remainingSize.Value} bytes.");
AddLogLine(downloadItem, DownloadItemLogLineType.INFORMATIONAL,
localization.GetLogLineStartingFileDownloadKnownFileSize(remainingSize.Value));
}
else
{
Logger.Debug($"Download file size is unknown.");
AddLogLine(downloadItem, DownloadItemLogLineType.INFORMATIONAL, localization.LogLineStartingFileDownloadUnknownFileSize);
}
}
Stream downloadStream = await response.Content.ReadAsStreamAsync();
byte[] buffer = new byte[4096];
long downloadedBytes = 0;
while (!downloadItem.CancellationToken.IsCancellationRequested)
{
int bytesRead;
try
{
bytesRead = downloadStream.Read(buffer, 0, buffer.Length);
}
catch (IOException ioException)
{
bool expectedError = false;
if (ioException.InnerException != null)
{
if (ioException.InnerException is WebException webException)
{
switch (webException.Status)
{
case WebExceptionStatus.Timeout:
Logger.Debug("Download timeout.");
ReportStatusChange(downloadItem, DownloadItemStatus.RETRY_DELAY, DownloadItemLogLineType.INFORMATIONAL,
localization.LogLineServerResponseTimeout);
expectedError = true;
break;
case WebExceptionStatus.RequestCanceled:
Logger.Debug("File download has been cancelled.");
expectedError = true;
break;
}
}
}
if (!expectedError)
{
Logger.Exception(ioException);
ReportError(downloadItem, localization.LogLineUnexpectedError);
}
break;
}
catch (Exception exception)
{
if (downloadItem.CancellationToken.IsCancellationRequested)
{
Logger.Debug("File download has been cancelled.");
break;
}
Logger.Exception(exception);
ReportError(downloadItem, localization.LogLineUnexpectedError);
break;
}
if (bytesRead == 0)
{
bool isCompleted = !remainingSize.HasValue || downloadedBytes == remainingSize.Value;
Logger.Debug($"File download is {(isCompleted ? "complete" : "incomplete")}.");
if (isCompleted)
{
lock (downloadQueueLock)
{
downloadItem.Status = DownloadItemStatus.COMPLETED;
}
}
else
{
ReportError(downloadItem, localization.LogLineDownloadIncompleteError);
}
break;
}
try
{
destinationFileStream.Write(buffer, 0, bytesRead);
}
catch (Exception exception)
{
Logger.Exception(exception);
ReportError(downloadItem, localization.LogLineFileWriteError);
break;
}
downloadedBytes += bytesRead;
downloadItem.DownloadedFileSize = startPosition + downloadedBytes;
ReportChange(downloadItem);
}
}
private async Task DownloadFileAsync(DownloadItem downloadItem)
{
try
{
AddLogLine(downloadItem, DownloadItemLogLineType.INFORMATIONAL,
localization.GetLogLineDownloadingFile(Uri.UnescapeDataString(downloadItem.DirectFileUrl)));
if (!Directory.Exists(downloadItem.DownloadDirectory))
{
try
{
Directory.CreateDirectory(downloadItem.DownloadDirectory);
}
catch (Exception exception)
{
Logger.Exception(exception);
ReportError(downloadItem, localization.GetLogLineCannotCreateDownloadDirectory(downloadItem.DownloadDirectory));
}
}
string fileName = downloadItem.FileCreated ? downloadItem.FileName : GenerateFileName(downloadItem.DownloadDirectory, downloadItem.FileName,
downloadItem.Md5Hash);
string targetFilePath = Path.Combine(downloadItem.DownloadDirectory, fileName);
string partFilePath = targetFilePath + ".part";
bool isCompleted;
bool isDeleteRequested;
FileStream destinationFileStream;
try
{
destinationFileStream = new FileStream(partFilePath, FileMode.Append, FileAccess.Write, FileShare.None);
}
catch (Exception exception)
{
Logger.Exception(exception);
ReportError(downloadItem, localization.GetLogLineCannotCreateOrOpenFile(partFilePath));
return;
}
using (destinationFileStream)
{
lock (downloadQueueLock)
{
downloadItem.FileName = fileName;
downloadItem.FileCreated = true;
downloadItem.FileHandleOpened = true;
SaveDownloadQueue();
}
await DownloadFileAsync(downloadItem, destinationFileStream);
lock (downloadQueueLock)
{
isCompleted = downloadItem.Status == DownloadItemStatus.COMPLETED;
isDeleteRequested = downloadItem.Status == DownloadItemStatus.REMOVED;
SaveDownloadQueue();
}
}
if (isCompleted)
{
bool moveFileError = false;
try
{
File.Move(partFilePath, targetFilePath);
}
catch (Exception exception)
{
Logger.Debug($"File rename error: partFilePath = {partFilePath}, targetFilePath = {targetFilePath}");
Logger.Exception(exception);
ReportError(downloadItem, localization.GetLogLineCannotRenamePartFile(partFilePath, targetFilePath));
moveFileError = true;
}
if (!moveFileError)
{
ReportStatusChange(downloadItem, DownloadItemStatus.COMPLETED, DownloadItemLogLineType.COMPLETED, localization.LogLineCompleted);
}
}
else if (isDeleteRequested)
{
try
{
File.Delete(partFilePath);
}
catch (Exception exception)
{
Logger.Debug($"Part file delete error: partFilePath = {partFilePath}");
Logger.Exception(exception);
}
}
lock (downloadQueueLock)
{
downloadItem.FileHandleOpened = false;
}
}
catch (Exception exception)
{
Logger.Exception(exception);
ReportError(downloadItem, localization.LogLineUnexpectedError);
}
}
private Task StartDownloadTask()
{
return(Task.Run(async() =>
{
while (true)
{
DownloadItem downloadItem = null;
if (isShuttingDown)
{
return;
}
if (httpClient == null || isInOfflineMode)
{
downloadTaskResetEvent.WaitOne();
}
else
{
lock (downloadQueueLock)
{
if (downloadItem == null || downloadItem.Status != DownloadItemStatus.DOWNLOADING)
{
downloadItem = downloadQueue.FirstOrDefault(item => item.Status == DownloadItemStatus.QUEUED ||
item.Status == DownloadItemStatus.DOWNLOADING || item.Status == DownloadItemStatus.RETRY_DELAY);
if (downloadItem != null)
{
ReportStatusChange(downloadItem, DownloadItemStatus.DOWNLOADING, DownloadItemLogLineType.INFORMATIONAL,
localization.LogLineStarted);
SaveDownloadQueue();
}
}
}
if (downloadItem == null)
{
downloadTaskResetEvent.WaitOne();
}
else
{
if (!downloadItem.FileCreated || downloadItem.RestartSessionOnTimeout)
{
string url = downloadItem.DownloadPageUrl;
string referer = null;
if (!String.IsNullOrWhiteSpace(downloadItem.DownloadTransformations))
{
foreach (string transformationName in downloadItem.DownloadTransformations.Split(',').
Select(transformation => transformation.Trim()))
{
if (String.IsNullOrEmpty(transformationName))
{
continue;
}
string pageContent;
try
{
pageContent = await DownloadPageAsync(downloadItem, url, referer);
}
catch (TaskCanceledException)
{
Logger.Debug("Page download has been cancelled.");
break;
}
if (pageContent == null)
{
break;
}
try
{
referer = url;
url = ExecuteTransformation(pageContent, transformationName);
}
catch (Exception exception)
{
Logger.Debug($"Transformation {transformationName} threw an exception.");
Logger.Exception(exception);
ReportError(downloadItem, localization.GetLogLineTransformationError(transformationName));
break;
}
bool isUrlValid;
if (String.IsNullOrWhiteSpace(url))
{
isUrlValid = false;
}
else if (!url.ToLower().StartsWith("http://") && !url.ToLower().StartsWith("https://"))
{
isUrlValid = false;
}
else
{
url = url.Replace("<", "%3C").Replace(">", "%3E");
isUrlValid = Uri.IsWellFormedUriString(url, UriKind.Absolute);
}
if (!isUrlValid)
{
Logger.Debug($"Transformation {transformationName} failed, returned string:", url);
Logger.Debug("Page:", pageContent);
ReportError(downloadItem, localization.GetLogLineTransformationReturnedIncorrectUrl(transformationName));
break;
}
}
}
if (downloadItem.CancellationToken.IsCancellationRequested)
{
continue;
}
bool skipFileDownload = false;
lock (downloadQueueLock)
{
if (downloadItem.Status == DownloadItemStatus.DOWNLOADING)
{
downloadItem.DirectFileUrl = url;
downloadItem.Referer = referer;
SaveDownloadQueue();
}
else
{
SaveDownloadQueue();
if (downloadItem.Status == DownloadItemStatus.RETRY_DELAY)
{
skipFileDownload = true;
}
else
{
continue;
}
}
}
if (!skipFileDownload)
{
await DownloadFileAsync(downloadItem);
}
}
else
{
await DownloadFileAsync(downloadItem);
}
bool retryDelay;
CancellationToken cancellationToken;
int currentAttempt;
lock (downloadQueueLock)
{
retryDelay = downloadItem.Status == DownloadItemStatus.RETRY_DELAY;
cancellationToken = downloadItem.CancellationToken;
currentAttempt = downloadItem.CurrentAttempt;
}
if (retryDelay)
{
if (currentAttempt == downloadSettings.Attempts)
{
lock (downloadQueueLock)
{
ReportError(downloadItem, localization.LogLineMaximumDownloadAttempts);
SaveDownloadQueue();
}
}
else
{
AddLogLine(downloadItem, DownloadItemLogLineType.INFORMATIONAL,
localization.GetLogLineRetryDelay(downloadSettings.RetryDelay));
bool isCancelled = false;
try
{
await Task.Delay(TimeSpan.FromSeconds(downloadSettings.RetryDelay), cancellationToken);
}
catch (TaskCanceledException)
{
isCancelled = true;
}
if (!isCancelled)
{
lock (downloadQueueLock)
{
downloadItem.Status = DownloadItemStatus.DOWNLOADING;
currentAttempt++;
downloadItem.CurrentAttempt = currentAttempt;
if (downloadItem.RestartSessionOnTimeout)
{
downloadItem.Referer = null;
}
AddLogLine(downloadItem, DownloadItemLogLineType.INFORMATIONAL,
localization.GetLogLineAttempt(currentAttempt, downloadSettings.Attempts));
}
}
}
}
}
}
}
}));
}
private void ReportError(DownloadItem downloadItem, string errorMessage)
{
ReportStatusChange(downloadItem, DownloadItemStatus.ERROR, DownloadItemLogLineType.ERROR, errorMessage);
}
public DownloadItemChangedEventArgs(DownloadItem changedDownloadItem)
{
ChangedDownloadItem = changedDownloadItem.Clone();
}
