public override async Task <bool> PrepareAsync(CookieAwareWebClient client, CancellationToken cancellation)
{
for (var i = 0; i < 5; i++)
{
var str = await client.DownloadStringTaskAsync(Url);
if (cancellation.IsCancellationRequested)
{
return(false);
}
var f = Regex.Match(str, @"<div class=""fileName"">([^<]+)");
FileName = f.Success ? f.Groups[1].Value : null;
var m = Regex.Match(str, @"(https?:\/\/download[^""']+)");
if (!m.Success)
{
return(false);
}
Url = m.Success ? m.Groups[1].Value : null;
if (Url != null)
{
Logging.Debug("HEAD request is coming…");
try {
using (client.SetMethod("HEAD"))
using (client.SetAutoRedirect(false)) {
await client.DownloadStringTaskAsync(Url);
Logging.Debug("Done");
}
} catch (Exception e) {
Logging.Warning(e);
return(true);
}
var contentType = client.ResponseHeaders?.Get("Content-Type");
Logging.Debug("Content-Type: " + contentType);
if (contentType?.IndexOf("text/html") != -1)
{
Logging.Debug("Redirect to web-page detected! Let’s try again");
continue;
}
}
return(true);
}
Logging.Warning("Too many redirects!");
return(false);
}
public override async Task <bool> PrepareAsync(CookieAwareWebClient client, CancellationToken cancellation)
{
if (OptionFailImmediately)
{
throw new NotSupportedException();
}
async Task Login()
{
var login = SettingsHolder.Content.RdLogin;
var password = SettingsHolder.Content.RdPassword;
var loginParams = string.IsNullOrWhiteSpace(login) || string.IsNullOrWhiteSpace(password)
? InternalUtils.GetRdLoginParams()
: new NameValueCollection {
["login"] = login,
["password"] = password,
};
Logging.Debug($"Forbidden! Trying to login with provided params ({loginParams["login"]})");
var result = (await client.UploadValuesTaskAsync("http://www.racedepartment.com/login/login", loginParams)).ToUtf8String();
var error = Regex.Match(result, @"<div class=""errorPanel""><span class=""errors"">([\s\S]+?)(?:</span>\s*)?</div>");
if (error.Success)
{
throw new Exception(error.Groups[1].Value);
}
}
using (client.SetProxy(SettingsHolder.Content.RdProxy)) {
var downloadPage = await client.DownloadStringTaskAsync(Url);
if (cancellation.IsCancellationRequested)
{
return(false);
}
var match = Regex.Match(downloadPage, @"href=""(downloads/[^""]+\?version=[^""]+)");
if (!match.Success)
{
NonfatalError.Notify(ToolsStrings.Common_CannotDownloadFile, ToolsStrings.DirectLoader_RdChanged);
return(false);
}
if (Regex.IsMatch(downloadPage, @"""inner"">\s*Login to download this mod"))
{
await Login();
}
var url = "http://www.racedepartment.com/" + HttpUtility.HtmlDecode(match.Groups[1].Value);
// Why, RD, why?!
try {
using (client.SetMethod("HEAD")) {
await client.DownloadStringTaskAsync(url);
}
} catch (WebException e) when((e.Response as HttpWebResponse)?.StatusCode == HttpStatusCode.Forbidden)
{
await Login();
}
Url = url;
Logging.Write("RaceDepartment download link: " + Url);
}
return(true);
}
public override async Task <bool> PrepareAsync(CookieAwareWebClient client, CancellationToken cancellation)
{
Logging.Debug(Url);
if (!Url.Contains("://drive.google.com/uc?", StringComparison.OrdinalIgnoreCase))
{
return(true);
}
// First of all, let’s see if there is an HTML-file under that link
Logging.Debug("HEAD request is coming…");
try {
using (client.SetMethod("HEAD"))
using (client.SetAutoRedirect(false)) {
await client.DownloadStringTaskAsync(Url);
Logging.Debug("Done");
}
} catch (Exception e) {
Logging.Warning(e);
}
// If file is freely available to download, server should redirect user to downloading
var location = client.ResponseHeaders?.Get("Location");
if (location != null)
{
Url = location;
Logging.Debug("Download URL is ready: " + location);
return(true);
}
Logging.Debug("Loading page…");
var downloadPage = await client.DownloadStringTaskAsync(Url);
if (cancellation.IsCancellationRequested)
{
return(false);
}
if (client.ResponseHeaders?.Get("Content-Type").Contains("text/html", StringComparison.OrdinalIgnoreCase) == false)
{
return(true);
}
var match = Regex.Match(downloadPage, @"href=""(/uc\?export=download[^""]+)", RegexOptions.IgnoreCase);
if (!match.Success)
{
NonfatalError.Notify(ToolsStrings.Common_CannotDownloadFile, ToolsStrings.DirectLoader_GoogleDriveChanged);
return(false);
}
Url = "https://drive.google.com" + HttpUtility.HtmlDecode(match.Groups[1].Value);
Logging.Write("Google Drive download link: " + Url);
var fileNameMatch = Regex.Match(downloadPage, @"/<span class=""uc-name-size""><a[^>]*>([^<]+)");
FileName = fileNameMatch.Success ? fileNameMatch.Groups[1].Value : null;
try {
var totalSizeMatch = Regex.Match(downloadPage, @"</a> \((\d+(?:\.\d+)?)([KMGT])\)</span> ");
if (totalSizeMatch.Success)
{
var value = double.Parse(totalSizeMatch.Groups[1].Value, CultureInfo.InvariantCulture);
var unit = totalSizeMatch.Groups[2].Value;
switch (unit.ToLowerInvariant())
{
case "k":
value *= 1024;
break;
case "m":
value *= 1024 * 1024;
break;
case "g":
value *= 1024 * 1024 * 1024;
break;
case "t":
value *= 1024d * 1024 * 1024 * 1024;
break;
}
TotalSize = (long)value;
}
} catch (Exception) {
// ignored
}
return(true);
}
private async Task <string> DownloadResumeSupportAsync([NotNull] CookieAwareWebClient client,
[NotNull] FlexibleLoaderGetPreferredDestinationCallback getPreferredDestination,
[CanBeNull] FlexibleLoaderReportDestinationCallback reportDestination, [CanBeNull] Func <bool> checkIfPaused,
IProgress <long> progress, CancellationToken cancellation)
{
// Common variables
string filename = null, selectedDestination = null, actualFootprint = null;
Stream remoteData = null;
var resumeSupported = ResumeSupported;
try {
// Read resume-related data and remove it to avoid conflicts
var resumeDestination = CacheStorage.Get <string>(_keyDestination);
var resumePartiallyLoadedFilename = CacheStorage.Get <string>(_keyPartiallyLoadedFilename);
var resumeLastWriteDate = CacheStorage.Get <DateTime?>(_keyLastWriteDate);
var resumePreviousFootprint = CacheStorage.Get <string>(_keyFootprint);
ClearResumeData();
// Collect known information for destination callback
var information = FlexibleLoaderMetaInformation.FromLoader(this);
// Opening stream to read…
var headRequest = HeadRequestSupported && resumeDestination != null;
using (headRequest ? client.SetMethod("HEAD") : null) {
Logging.Warning($"Initial request: {(headRequest ? "HEAD" : "GET")}");
remoteData = await client.OpenReadTaskAsync(Url);
}
cancellation.ThrowIfCancellationRequested();
// Maybe we’ll be lucky enough to load the most accurate data
if (client.ResponseHeaders != null)
{
if (long.TryParse(client.ResponseHeaders[HttpResponseHeader.ContentLength] ?? "",
NumberStyles.Any, CultureInfo.InvariantCulture, out var length))
{
TotalSize = information.TotalSize = length;
}
if (TryGetFileName(client.ResponseHeaders, out var fileName))
{
FileName = information.FileName = fileName;
}
// For example, Google Drive responds with “none” and yet allows to download file partially,
// so this header will only be checked if value is not defined.
if (resumeSupported == null)
{
var accept = client.ResponseHeaders[HttpResponseHeader.AcceptRanges] ?? "";
if (accept.Contains("bytes"))
{
resumeSupported = true;
}
else if (accept.Contains("none"))
{
resumeSupported = false;
}
}
client.LogResponseHeaders();
}
// Was the file partially loaded before?
var partiallyLoaded = ResumeSupported != false && resumePartiallyLoadedFilename != null
? new FileInfo(FileUtils.EnsureFilenameIsValid(resumePartiallyLoadedFilename)) : null;
if (partiallyLoaded != null)
{
Logging.Warning("Not finished: " + partiallyLoaded);
}
// Does it still exist
if (partiallyLoaded?.Exists != true)
{
Logging.Warning($"Partially downloaded file “{partiallyLoaded?.FullName}” does not exist");
partiallyLoaded = null;
}
// If so, wasn’t it changed since the last time?
if (partiallyLoaded?.LastWriteTime > resumeLastWriteDate + TimeSpan.FromMinutes(5))
{
Logging.Warning($"Partially downloaded file is newer that it should be: {partiallyLoaded.LastWriteTime}, expected: {resumeLastWriteDate}");
partiallyLoaded = null;
}
// Looks like file is partially downloaded, but let’s ensure link still leads to the same content
actualFootprint = GetFootprint(information, client.ResponseHeaders);
if (partiallyLoaded != null && resumePreviousFootprint != actualFootprint)
{
Logging.Warning($"Footprints don’t match: {resumePreviousFootprint}≠{actualFootprint}");
partiallyLoaded = null;
}
// Let’s check where to load data, which is potentially the most actual data at this point
var destination = getPreferredDestination(Url, information);
selectedDestination = destination.Filename;
if (partiallyLoaded != null && (!destination.CanResumeDownload || !FileUtils.ArePathsEqual(selectedDestination, resumeDestination)))
{
Logging.Warning($"Different destination chosen: {selectedDestination} (before: {resumeDestination})");
partiallyLoaded = null;
}
// TODO: Check that header?
// Where to write?
// ReSharper disable once MergeConditionalExpression
filename = partiallyLoaded != null ? partiallyLoaded.FullName : FileUtils.EnsureUnique(true, destination.Filename);
reportDestination?.Invoke(filename);
// Set cancellation token
cancellation.Register(o => client.CancelAsync(), null);
// Open write stream
if (partiallyLoaded != null)
{
var rangeFrom = partiallyLoaded.Length;
using (client.SetRange(new Tuple <long, long>(rangeFrom, -1))) {
Logging.Warning($"Trying to resume download from {rangeFrom} bytes…");
remoteData.Dispose();
remoteData = await client.OpenReadTaskAsync(Url);
cancellation.ThrowIfCancellationRequested();
client.LogResponseHeaders();
// It’s unknown if resume is supported or not at this point
if (resumeSupported == null)
{
var bytes = new byte[16];
var firstBytes = await remoteData.ReadAsync(bytes, 0, bytes.Length);
cancellation.ThrowIfCancellationRequested();
if (CouldBeBeginningOfAFile(bytes))
{
using (var file = File.Create(filename)) {
Logging.Warning("File beginning found, restart download");
file.Write(bytes, 0, firstBytes);
await CopyToAsync(remoteData, file, checkIfPaused, progress, cancellation);
cancellation.ThrowIfCancellationRequested();
}
Logging.Write("Download finished");
return(filename);
}
rangeFrom += firstBytes;
}
using (var file = new FileStream(filename, FileMode.Append, FileAccess.Write)) {
await CopyToAsync(remoteData, file, checkIfPaused, new Progress <long>(v => {
progress?.Report(v + rangeFrom);
}), cancellation);
cancellation.ThrowIfCancellationRequested();
}
}
}
else
{
if (headRequest)
{
Logging.Warning("Re-open request to be GET");
remoteData.Dispose();
remoteData = await client.OpenReadTaskAsync(Url);
}
using (var file = File.Create(filename)) {
Logging.Debug("Downloading the whole file…");
await CopyToAsync(remoteData, file, checkIfPaused, progress, cancellation);
cancellation.ThrowIfCancellationRequested();
}
}
Logging.Write("Download finished");
return(filename);
} catch (Exception e) when(e is WebException || e.IsCancelled())
{
Logging.Write("Download is interrupted! Saving details to resume later…");
var download = filename == null ? null : new FileInfo(filename);
if (download?.Exists == true && filename.Length > 0)
{
CacheStorage.Set(_keyDestination, selectedDestination);
CacheStorage.Set(_keyPartiallyLoadedFilename, filename);
CacheStorage.Set(_keyFootprint, actualFootprint);
CacheStorage.Set(_keyLastWriteDate, download.LastWriteTime);
}
else
{
ClearResumeData();
}
throw;
} finally {
remoteData?.Dispose();
}
}
