public override async Task <bool> PrepareAsync(CookieAwareWebClient client, CancellationToken cancellation) { for (var i = 0; i < 5; i++) { var str = await client.DownloadStringTaskAsync(Url); if (cancellation.IsCancellationRequested) { return(false); } var f = Regex.Match(str, @"<div class=""fileName"">([^<]+)"); FileName = f.Success ? f.Groups[1].Value : null; var m = Regex.Match(str, @"(https?:\/\/download[^""']+)"); if (!m.Success) { return(false); } Url = m.Success ? m.Groups[1].Value : null; if (Url != null) { Logging.Debug("HEAD request is coming…"); try { using (client.SetMethod("HEAD")) using (client.SetAutoRedirect(false)) { await client.DownloadStringTaskAsync(Url); Logging.Debug("Done"); } } catch (Exception e) { Logging.Warning(e); return(true); } var contentType = client.ResponseHeaders?.Get("Content-Type"); Logging.Debug("Content-Type: " + contentType); if (contentType?.IndexOf("text/html") != -1) { Logging.Debug("Redirect to web-page detected! Let’s try again"); continue; } } return(true); } Logging.Warning("Too many redirects!"); return(false); }
public override async Task <bool> PrepareAsync(CookieAwareWebClient client, CancellationToken cancellation) { if (OptionFailImmediately) { throw new NotSupportedException(); } async Task Login() { var login = SettingsHolder.Content.RdLogin; var password = SettingsHolder.Content.RdPassword; var loginParams = string.IsNullOrWhiteSpace(login) || string.IsNullOrWhiteSpace(password) ? InternalUtils.GetRdLoginParams() : new NameValueCollection { ["login"] = login, ["password"] = password, }; Logging.Debug($"Forbidden! Trying to login with provided params ({loginParams["login"]})"); var result = (await client.UploadValuesTaskAsync("http://www.racedepartment.com/login/login", loginParams)).ToUtf8String(); var error = Regex.Match(result, @"<div class=""errorPanel""><span class=""errors"">([\s\S]+?)(?:</span>\s*)?</div>"); if (error.Success) { throw new Exception(error.Groups[1].Value); } } using (client.SetProxy(SettingsHolder.Content.RdProxy)) { var downloadPage = await client.DownloadStringTaskAsync(Url); if (cancellation.IsCancellationRequested) { return(false); } var match = Regex.Match(downloadPage, @"href=""(downloads/[^""]+\?version=[^""]+)"); if (!match.Success) { NonfatalError.Notify(ToolsStrings.Common_CannotDownloadFile, ToolsStrings.DirectLoader_RdChanged); return(false); } if (Regex.IsMatch(downloadPage, @"""inner"">\s*Login to download this mod")) { await Login(); } var url = "http://www.racedepartment.com/" + HttpUtility.HtmlDecode(match.Groups[1].Value); // Why, RD, why?! try { using (client.SetMethod("HEAD")) { await client.DownloadStringTaskAsync(url); } } catch (WebException e) when((e.Response as HttpWebResponse)?.StatusCode == HttpStatusCode.Forbidden) { await Login(); } Url = url; Logging.Write("RaceDepartment download link: " + Url); } return(true); }
public override async Task <bool> PrepareAsync(CookieAwareWebClient client, CancellationToken cancellation) { Logging.Debug(Url); if (!Url.Contains("://drive.google.com/uc?", StringComparison.OrdinalIgnoreCase)) { return(true); } // First of all, let’s see if there is an HTML-file under that link Logging.Debug("HEAD request is coming…"); try { using (client.SetMethod("HEAD")) using (client.SetAutoRedirect(false)) { await client.DownloadStringTaskAsync(Url); Logging.Debug("Done"); } } catch (Exception e) { Logging.Warning(e); } // If file is freely available to download, server should redirect user to downloading var location = client.ResponseHeaders?.Get("Location"); if (location != null) { Url = location; Logging.Debug("Download URL is ready: " + location); return(true); } Logging.Debug("Loading page…"); var downloadPage = await client.DownloadStringTaskAsync(Url); if (cancellation.IsCancellationRequested) { return(false); } if (client.ResponseHeaders?.Get("Content-Type").Contains("text/html", StringComparison.OrdinalIgnoreCase) == false) { return(true); } var match = Regex.Match(downloadPage, @"href=""(/uc\?export=download[^""]+)", RegexOptions.IgnoreCase); if (!match.Success) { NonfatalError.Notify(ToolsStrings.Common_CannotDownloadFile, ToolsStrings.DirectLoader_GoogleDriveChanged); return(false); } Url = "https://drive.google.com" + HttpUtility.HtmlDecode(match.Groups[1].Value); Logging.Write("Google Drive download link: " + Url); var fileNameMatch = Regex.Match(downloadPage, @"/<span class=""uc-name-size""><a[^>]*>([^<]+)"); FileName = fileNameMatch.Success ? fileNameMatch.Groups[1].Value : null; try { var totalSizeMatch = Regex.Match(downloadPage, @"</a> \((\d+(?:\.\d+)?)([KMGT])\)</span> "); if (totalSizeMatch.Success) { var value = double.Parse(totalSizeMatch.Groups[1].Value, CultureInfo.InvariantCulture); var unit = totalSizeMatch.Groups[2].Value; switch (unit.ToLowerInvariant()) { case "k": value *= 1024; break; case "m": value *= 1024 * 1024; break; case "g": value *= 1024 * 1024 * 1024; break; case "t": value *= 1024d * 1024 * 1024 * 1024; break; } TotalSize = (long)value; } } catch (Exception) { // ignored } return(true); }
private async Task <string> DownloadResumeSupportAsync([NotNull] CookieAwareWebClient client, [NotNull] FlexibleLoaderGetPreferredDestinationCallback getPreferredDestination, [CanBeNull] FlexibleLoaderReportDestinationCallback reportDestination, [CanBeNull] Func <bool> checkIfPaused, IProgress <long> progress, CancellationToken cancellation) { // Common variables string filename = null, selectedDestination = null, actualFootprint = null; Stream remoteData = null; var resumeSupported = ResumeSupported; try { // Read resume-related data and remove it to avoid conflicts var resumeDestination = CacheStorage.Get <string>(_keyDestination); var resumePartiallyLoadedFilename = CacheStorage.Get <string>(_keyPartiallyLoadedFilename); var resumeLastWriteDate = CacheStorage.Get <DateTime?>(_keyLastWriteDate); var resumePreviousFootprint = CacheStorage.Get <string>(_keyFootprint); ClearResumeData(); // Collect known information for destination callback var information = FlexibleLoaderMetaInformation.FromLoader(this); // Opening stream to read… var headRequest = HeadRequestSupported && resumeDestination != null; using (headRequest ? client.SetMethod("HEAD") : null) { Logging.Warning($"Initial request: {(headRequest ? "HEAD" : "GET")}"); remoteData = await client.OpenReadTaskAsync(Url); } cancellation.ThrowIfCancellationRequested(); // Maybe we’ll be lucky enough to load the most accurate data if (client.ResponseHeaders != null) { if (long.TryParse(client.ResponseHeaders[HttpResponseHeader.ContentLength] ?? "", NumberStyles.Any, CultureInfo.InvariantCulture, out var length)) { TotalSize = information.TotalSize = length; } if (TryGetFileName(client.ResponseHeaders, out var fileName)) { FileName = information.FileName = fileName; } // For example, Google Drive responds with “none” and yet allows to download file partially, // so this header will only be checked if value is not defined. if (resumeSupported == null) { var accept = client.ResponseHeaders[HttpResponseHeader.AcceptRanges] ?? ""; if (accept.Contains("bytes")) { resumeSupported = true; } else if (accept.Contains("none")) { resumeSupported = false; } } client.LogResponseHeaders(); } // Was the file partially loaded before? var partiallyLoaded = ResumeSupported != false && resumePartiallyLoadedFilename != null ? new FileInfo(FileUtils.EnsureFilenameIsValid(resumePartiallyLoadedFilename)) : null; if (partiallyLoaded != null) { Logging.Warning("Not finished: " + partiallyLoaded); } // Does it still exist if (partiallyLoaded?.Exists != true) { Logging.Warning($"Partially downloaded file “{partiallyLoaded?.FullName}” does not exist"); partiallyLoaded = null; } // If so, wasn’t it changed since the last time? if (partiallyLoaded?.LastWriteTime > resumeLastWriteDate + TimeSpan.FromMinutes(5)) { Logging.Warning($"Partially downloaded file is newer that it should be: {partiallyLoaded.LastWriteTime}, expected: {resumeLastWriteDate}"); partiallyLoaded = null; } // Looks like file is partially downloaded, but let’s ensure link still leads to the same content actualFootprint = GetFootprint(information, client.ResponseHeaders); if (partiallyLoaded != null && resumePreviousFootprint != actualFootprint) { Logging.Warning($"Footprints don’t match: {resumePreviousFootprint}≠{actualFootprint}"); partiallyLoaded = null; } // Let’s check where to load data, which is potentially the most actual data at this point var destination = getPreferredDestination(Url, information); selectedDestination = destination.Filename; if (partiallyLoaded != null && (!destination.CanResumeDownload || !FileUtils.ArePathsEqual(selectedDestination, resumeDestination))) { Logging.Warning($"Different destination chosen: {selectedDestination} (before: {resumeDestination})"); partiallyLoaded = null; } // TODO: Check that header? // Where to write? // ReSharper disable once MergeConditionalExpression filename = partiallyLoaded != null ? partiallyLoaded.FullName : FileUtils.EnsureUnique(true, destination.Filename); reportDestination?.Invoke(filename); // Set cancellation token cancellation.Register(o => client.CancelAsync(), null); // Open write stream if (partiallyLoaded != null) { var rangeFrom = partiallyLoaded.Length; using (client.SetRange(new Tuple <long, long>(rangeFrom, -1))) { Logging.Warning($"Trying to resume download from {rangeFrom} bytes…"); remoteData.Dispose(); remoteData = await client.OpenReadTaskAsync(Url); cancellation.ThrowIfCancellationRequested(); client.LogResponseHeaders(); // It’s unknown if resume is supported or not at this point if (resumeSupported == null) { var bytes = new byte[16]; var firstBytes = await remoteData.ReadAsync(bytes, 0, bytes.Length); cancellation.ThrowIfCancellationRequested(); if (CouldBeBeginningOfAFile(bytes)) { using (var file = File.Create(filename)) { Logging.Warning("File beginning found, restart download"); file.Write(bytes, 0, firstBytes); await CopyToAsync(remoteData, file, checkIfPaused, progress, cancellation); cancellation.ThrowIfCancellationRequested(); } Logging.Write("Download finished"); return(filename); } rangeFrom += firstBytes; } using (var file = new FileStream(filename, FileMode.Append, FileAccess.Write)) { await CopyToAsync(remoteData, file, checkIfPaused, new Progress <long>(v => { progress?.Report(v + rangeFrom); }), cancellation); cancellation.ThrowIfCancellationRequested(); } } } else { if (headRequest) { Logging.Warning("Re-open request to be GET"); remoteData.Dispose(); remoteData = await client.OpenReadTaskAsync(Url); } using (var file = File.Create(filename)) { Logging.Debug("Downloading the whole file…"); await CopyToAsync(remoteData, file, checkIfPaused, progress, cancellation); cancellation.ThrowIfCancellationRequested(); } } Logging.Write("Download finished"); return(filename); } catch (Exception e) when(e is WebException || e.IsCancelled()) { Logging.Write("Download is interrupted! Saving details to resume later…"); var download = filename == null ? null : new FileInfo(filename); if (download?.Exists == true && filename.Length > 0) { CacheStorage.Set(_keyDestination, selectedDestination); CacheStorage.Set(_keyPartiallyLoadedFilename, filename); CacheStorage.Set(_keyFootprint, actualFootprint); CacheStorage.Set(_keyLastWriteDate, download.LastWriteTime); } else { ClearResumeData(); } throw; } finally { remoteData?.Dispose(); } }