public override async Task <bool> PrepareAsync(CookieAwareWebClient client, CancellationToken cancellation) { for (var i = 0; i < 5; i++) { var str = await client.DownloadStringTaskAsync(Url); if (cancellation.IsCancellationRequested) { return(false); } var f = Regex.Match(str, @"<div class=""fileName"">([^<]+)"); FileName = f.Success ? f.Groups[1].Value : null; var m = Regex.Match(str, @"(https?:\/\/download[^""']+)"); if (!m.Success) { return(false); } Url = m.Success ? m.Groups[1].Value : null; if (Url != null) { Logging.Debug("HEAD request is coming…"); try { using (client.SetMethod("HEAD")) using (client.SetAutoRedirect(false)) { await client.DownloadStringTaskAsync(Url); Logging.Debug("Done"); } } catch (Exception e) { Logging.Warning(e); return(true); } var contentType = client.ResponseHeaders?.Get("Content-Type"); Logging.Debug("Content-Type: " + contentType); if (contentType?.IndexOf("text/html") != -1) { Logging.Debug("Redirect to web-page detected! Let’s try again"); continue; } } return(true); } Logging.Warning("Too many redirects!"); return(false); }
public override async Task <bool> PrepareAsync(CookieAwareWebClient client, CancellationToken cancellation) { try { var step1 = await client.DownloadStringTaskAsync(Url); if (cancellation.IsCancellationRequested) { return(false); } var step2 = (await client.UploadValuesTaskAsync(Url, @"POST", GetFormData(step1, @"//div[@id='plans_free']//form"))).ToUtf8String(); if (cancellation.IsCancellationRequested) { return(false); } var step3 = (await client.UploadValuesTaskAsync(Url, @"POST", GetFormData(step2, @"//form[@name='F1']"))).ToUtf8String(); if (cancellation.IsCancellationRequested) { return(false); } Url = GetDocument(step3).DocumentNode.SelectSingleNode(@"//a[contains(@href, 'sharemods.com/cgi-bin')]")?.Attributes[@"href"]?.Value; return(Url != null); } catch (Exception e) { Logging.Error(e); throw new InformativeException(ToolsStrings.Common_CannotDownloadFile, "Sharemods.com has changed."); } }
public override async Task <bool> PrepareAsync(CookieAwareWebClient client, CancellationToken cancellation) { var description = await client.DownloadStringTaskAsync( "https://cloud-api.yandex.net:443/v1/disk/public/resources/download?public_key=" + HttpUtility.UrlEncode(Url)); if (cancellation.IsCancellationRequested) { return(false); } Url = (string)JObject.Parse(description)["href"]; Logging.Write("Yandex Disk download link: " + Url); try { var query = HttpUtility.ParseQueryString(Url); FileName = query["filename"]; if (FlexibleParser.TryParseLong(query["fsize"], out var size)) { TotalSize = size; } } catch (Exception) { // ignored } return(true); }
protected override async Task <string> GetRedirectOverrideAsync(string url, CookieAwareWebClient client, CancellationToken cancellation) { using (client.SetUserAgent("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")) { var redirectTo = Reverse(Regex.Match(await client.DownloadStringTaskAsync(url), @"ysmm\s=\s'(.*?)'").Groups[1].Value); if (cancellation.IsCancellationRequested) { return(redirectTo); } if (Test(redirectTo)) { redirectTo = Unwrap(await client.DownloadStringTaskAsync(redirectTo)) ?? redirectTo; if (cancellation.IsCancellationRequested) { return(redirectTo); } } using (var stream = await client.OpenReadTaskAsync(redirectTo)) { if (cancellation.IsCancellationRequested) { return(redirectTo); } if (client.ResponseHeaders?.Get("Content-Type").Contains(@"text/html", StringComparison.OrdinalIgnoreCase) == true) { redirectTo = Unwrap((await stream.ReadAsBytesAsync()).ToUtf8String()) ?? redirectTo; } } return(redirectTo); } string Unwrap(string html) { var doc = new HtmlDocument(); doc.LoadHtml(html); return(doc.DocumentNode.Descendants(@"a") .FirstOrDefault(x => x.InnerText.Contains(@"click"))? .Attributes[@"href"]?.Value); } }
protected override async Task <string> GetRedirectOverrideAsync(string url, CookieAwareWebClient client, CancellationToken cancellation) { using (client.SetAccept("application/json")) { var response = await client.DownloadStringTaskAsync(url.AddQueryParameter("json")); if (cancellation.IsCancellationRequested) { return(null); } return(JObject.Parse(response).GetStringValueOnly("request")); } }
protected override async Task <string> GetRedirect(string url, CookieAwareWebClient client, CancellationToken cancellation) { var downloadUrl = url; if (!DownloadTest(downloadUrl)) { var itemPage = await client.DownloadStringTaskAsync(url); if (cancellation.IsCancellationRequested) { return(null); } var downloadUrlMatch = Regex.Match(itemPage, @"href=""(/[^""]+/download)"""); if (!downloadUrlMatch.Success) { NonfatalError.Notify(ToolsStrings.Common_CannotDownloadFile, ToolsStrings.DirectLoader_AssettoDbChanged); return(null); } downloadUrl = "http://assetto-db.com" + HttpUtility.HtmlDecode(downloadUrlMatch.Groups[1].Value); } var downloadPage = await client.DownloadStringTaskAsync(downloadUrl); if (cancellation.IsCancellationRequested) { return(null); } var match = Regex.Match(downloadPage, @"\bwindow\.location='([^']+)'"); if (!match.Success) { NonfatalError.Notify(ToolsStrings.Common_CannotDownloadFile, ToolsStrings.DirectLoader_AssettoDbChanged); return(null); } return(match.Groups[1].Value); }
protected override async Task <string> GetRedirectOverrideAsync(string url, CookieAwareWebClient client, CancellationToken cancellation) { var id = TestRegex.Match(url).Groups[1].Value; if (string.IsNullOrWhiteSpace(id)) { return(null); } var data = await client.DownloadStringTaskAsync( $@"https://www.googleapis.com/youtube/v3/videos?part=snippet&id={id}&key={InternalUtils.GetYouTubeApiKey().Item1}"); if (cancellation.IsCancellationRequested) { return(null); } try { var links = JObject.Parse(data)[@"items"][0][@"snippet"][@"description"].ToString().GetUrls().ToList(); var supported = links.FirstOrDefault(FlexibleLoader.IsSupportedFileStorage); if (supported != null) { return(supported); } foreach (var link in links) { if (cancellation.IsCancellationRequested) { return(null); } if (await FlexibleLoader.IsSupportedAsync(link, cancellation)) { return(link); } } } catch (Exception e) { Logging.Warning(e); throw; } return(null); }
protected override async Task <string> GetRedirectOverrideAsync(string url, CookieAwareWebClient client, CancellationToken cancellation) { var downloadPage = await client.DownloadStringTaskAsync(url); if (cancellation.IsCancellationRequested) { return(null); } var downloadUrlMatch = Regex.Match(downloadPage, @"\bhref=""(https?://(?:www\.)?acstuff\.ru/s/[^""]+)", RegexOptions.IgnoreCase); if (!downloadUrlMatch.Success) { NonfatalError.Notify(ToolsStrings.Common_CannotDownloadFile, "ACDriftingPro.com has changed."); return(null); } return(HttpUtility.HtmlDecode(downloadUrlMatch.Groups[1].Value)); }
protected override async Task <string> GetRedirectOverrideAsync(string url, CookieAwareWebClient client, CancellationToken cancellation) { var downloadPage = await client.DownloadStringTaskAsync(url); if (cancellation.IsCancellationRequested) { return(null); } var versionMatch = Regex.Match(downloadPage, @"""spec"">[\s\S]+?V\. ([^<]+)"); OverrideVersion = versionMatch.Success ? versionMatch.Groups[1].Value : null; var downloadUrlMatch = Regex.Match(downloadPage, @"<p class=""download""><a href=""([^""]+)"); if (!downloadUrlMatch.Success) { NonfatalError.Notify(ToolsStrings.Common_CannotDownloadFile, ToolsStrings.DirectLoader_AcClubChanged); return(null); } return(HttpUtility.HtmlDecode(downloadUrlMatch.Groups[1].Value)); }
public override async Task <bool> PrepareAsync(CookieAwareWebClient client, CancellationToken cancellation) { var str = await client.DownloadStringTaskAsync(Url); if (cancellation.IsCancellationRequested) { return(false); } var f = Regex.Match(str, @"<div class=""fileName"">([^<]+)"); FileName = f.Success ? f.Groups[1].Value : null; var m = Regex.Match(str, @"(http:\/\/download[^""']+)"); if (!m.Success) { return(false); } Url = m.Success ? m.Groups[1].Value : null; return(true); }
public override async Task <bool> PrepareAsync(CookieAwareWebClient client, CancellationToken cancellation) { Logging.Debug(Url); if (!Url.Contains("://drive.google.com/uc?", StringComparison.OrdinalIgnoreCase)) { return(true); } // First of all, let’s see if there is an HTML-file under that link Logging.Debug("GET request is coming…"); string webPageContent; using (client.SetAutoRedirect(false)) using (var stream = await client.OpenReadTaskAsync(Url)) { if (cancellation.IsCancellationRequested) { return(false); } // If file is freely available to download, server should redirect user to downloading var location = client.ResponseHeaders?.Get("Location"); if (location != null) { Url = location; FileName = new Uri(Url, UriKind.RelativeOrAbsolute).GetQueryParam("id"); Logging.Debug("Download URL is ready: " + location); client.LogResponseHeaders(); return(true); } Logging.Debug("Content-Type: " + client.ResponseHeaders?.Get("Content-Type")); if (client.ResponseHeaders?.Get("Content-Type").Contains("text/html", StringComparison.OrdinalIgnoreCase) == false) { return(true); } // Looks like it’s a webpage, now we need to download and parse it webPageContent = (await stream.ReadAsBytesAsync()).ToUtf8String(); if (cancellation.IsCancellationRequested) { return(false); } Logging.Debug("…done"); } var doc = new HtmlDocument(); doc.LoadHtml(webPageContent); var link = doc.DocumentNode.SelectSingleNode(@"//a[contains(@href, 'export=download')]").Attributes[@"href"].Value; if (link == null) { NonfatalError.Notify(ToolsStrings.Common_CannotDownloadFile, ToolsStrings.DirectLoader_GoogleDriveChanged); return(false); } Url = @"https://drive.google.com" + HttpUtility.HtmlDecode(link); FileName = HttpUtility.HtmlDecode(doc.DocumentNode.SelectSingleNode(@"//span[@class='uc-name-size']/a")?.InnerText?.Trim()); Logging.Write($"Google Drive download link: {Url}"); try { var totalSize = HttpUtility.HtmlDecode( doc.DocumentNode.SelectSingleNode(@"//span[@class='uc-name-size']/text()")?.InnerText?.Trim(' ', '(', ')')); Logging.Write($"Total size: {totalSize}"); if (totalSize != null && LocalizationHelper.TryParseReadableSize(totalSize, null, out var size)) { Logging.Write($"Parsed size: {size} bytes"); TotalSize = size; } } catch (Exception e) { Logging.Warning(e); } if (OptionManualRedirect) { using (client.SetDebugMode(OptionDebugMode)) using (client.SetAutoRedirect(false)) { var redirect = await client.DownloadStringTaskAsync(Url); Logging.Debug(redirect); if (!redirect.Contains("<TITLE>Moved Temporarily</TITLE>")) { NonfatalError.Notify(ToolsStrings.Common_CannotDownloadFile, ToolsStrings.DirectLoader_GoogleDriveChanged); return(false); } var redirectMatch = Regex.Match(redirect, @"href=""([^""]+)", RegexOptions.IgnoreCase); if (!redirectMatch.Success) { NonfatalError.Notify(ToolsStrings.Common_CannotDownloadFile, ToolsStrings.DirectLoader_GoogleDriveChanged); return(false); } Url = HttpUtility.HtmlDecode(redirectMatch.Groups[1].Value); Logging.Debug(Url); } } return(true); }
public override async Task <bool> PrepareAsync(CookieAwareWebClient client, CancellationToken cancellation) { if (OptionFailImmediately) { throw new NotSupportedException(); } async Task Login() { var login = SettingsHolder.Content.RdLogin; var password = SettingsHolder.Content.RdPassword; var loginParams = string.IsNullOrWhiteSpace(login) || string.IsNullOrWhiteSpace(password) ? InternalUtils.GetRdLoginParams() : new NameValueCollection { ["login"] = login, ["password"] = password, }; Logging.Debug($"Forbidden! Trying to login with provided params ({loginParams["login"]})"); var result = (await client.UploadValuesTaskAsync("http://www.racedepartment.com/login/login", loginParams)).ToUtf8String(); var error = Regex.Match(result, @"<div class=""errorPanel""><span class=""errors"">([\s\S]+?)(?:</span>\s*)?</div>"); if (error.Success) { throw new Exception(error.Groups[1].Value); } } using (client.SetProxy(SettingsHolder.Content.RdProxy)) { var downloadPage = await client.DownloadStringTaskAsync(Url); if (cancellation.IsCancellationRequested) { return(false); } var match = Regex.Match(downloadPage, @"href=""(downloads/[^""]+\?version=[^""]+)"); if (!match.Success) { NonfatalError.Notify(ToolsStrings.Common_CannotDownloadFile, ToolsStrings.DirectLoader_RdChanged); return(false); } if (Regex.IsMatch(downloadPage, @"""inner"">\s*Login to download this mod")) { await Login(); } var url = "http://www.racedepartment.com/" + HttpUtility.HtmlDecode(match.Groups[1].Value); // Why, RD, why?! try { using (client.SetMethod("HEAD")) { await client.DownloadStringTaskAsync(url); } } catch (WebException e) when((e.Response as HttpWebResponse)?.StatusCode == HttpStatusCode.Forbidden) { await Login(); } Url = url; Logging.Write("RaceDepartment download link: " + Url); } return(true); }
protected override async Task <string> GetRedirect(string url, CookieAwareWebClient client, CancellationToken cancellation) { using (client.SetUserAgent("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")) { return(Encoding.UTF8.GetString(Convert.FromBase64String(Reverse( Regex.Match(await client.DownloadStringTaskAsync(url), @"ysmm\s=\s'(.*?)'").Groups[1].Value))).Remove(0, 2)); } }
public override async Task <bool> PrepareAsync(CookieAwareWebClient client, CancellationToken cancellation) { Logging.Debug(Url); if (!Url.Contains("://drive.google.com/uc?", StringComparison.OrdinalIgnoreCase)) { return(true); } // First of all, let’s see if there is an HTML-file under that link Logging.Debug("HEAD request is coming…"); try { using (client.SetMethod("HEAD")) using (client.SetAutoRedirect(false)) { await client.DownloadStringTaskAsync(Url); Logging.Debug("Done"); } } catch (Exception e) { Logging.Warning(e); } // If file is freely available to download, server should redirect user to downloading var location = client.ResponseHeaders?.Get("Location"); if (location != null) { Url = location; Logging.Debug("Download URL is ready: " + location); return(true); } Logging.Debug("Loading page…"); var downloadPage = await client.DownloadStringTaskAsync(Url); if (cancellation.IsCancellationRequested) { return(false); } if (client.ResponseHeaders?.Get("Content-Type").Contains("text/html", StringComparison.OrdinalIgnoreCase) == false) { return(true); } var match = Regex.Match(downloadPage, @"href=""(/uc\?export=download[^""]+)", RegexOptions.IgnoreCase); if (!match.Success) { NonfatalError.Notify(ToolsStrings.Common_CannotDownloadFile, ToolsStrings.DirectLoader_GoogleDriveChanged); return(false); } Url = "https://drive.google.com" + HttpUtility.HtmlDecode(match.Groups[1].Value); Logging.Write("Google Drive download link: " + Url); var fileNameMatch = Regex.Match(downloadPage, @"/<span class=""uc-name-size""><a[^>]*>([^<]+)"); FileName = fileNameMatch.Success ? fileNameMatch.Groups[1].Value : null; try { var totalSizeMatch = Regex.Match(downloadPage, @"</a> \((\d+(?:\.\d+)?)([KMGT])\)</span> "); if (totalSizeMatch.Success) { var value = double.Parse(totalSizeMatch.Groups[1].Value, CultureInfo.InvariantCulture); var unit = totalSizeMatch.Groups[2].Value; switch (unit.ToLowerInvariant()) { case "k": value *= 1024; break; case "m": value *= 1024 * 1024; break; case "g": value *= 1024 * 1024 * 1024; break; case "t": value *= 1024d * 1024 * 1024 * 1024; break; } TotalSize = (long)value; } } catch (Exception) { // ignored } return(true); }
//////////////////////////////////////////////////////////////////////// public async System.Threading.Tasks.Task <object> TimTick(object[] parameters) { string password = ""; byte[] decryptedArr = new byte[1]; NameValueCollection formData = new NameValueCollection(); //while (true) { if (TimTickEnabled == true) { Uri uriStr; string autoIncrementIndexOfData = cntAutoInc.ToString(); string InputString = ""; if (password == "" && GlobalClass.TestSecretCloud() == true) { password = GlobalClass.ReadSecretCloud(); //Sha256 } //******************* INITIATE PHPSESSION *************************** CookieAwareWebClient webClient = new CookieAwareWebClient(); webClient.Encoding = System.Text.Encoding.Default; formData.Clear(); formData["username"] = "******"; formData["password"] = ""; if ((formData["password"] == "") && GlobalClass.TestPasswordCloud() == true) { formData["password"] = GlobalClass.ReadPasswordCloud(); //Sha256 } Uri.TryCreate("http://your.url.here/lo.php", UriKind.RelativeOrAbsolute, out uriStr); Task <byte[]> loTaskString = webClient.UploadValuesTaskAsync(uriStr, "POST", formData); await loTaskString; byte[] responseBytes = loTaskString.Result; string responseHTML = Encoding.UTF8.GetString(responseBytes); if (Uri.TryCreate("http://your.url.here", UriKind.RelativeOrAbsolute, out uriStr) == false) { System.Diagnostics.Debug.WriteLine("NO"); } foreach (Cookie cookie in webClient.CookieContainer.GetCookies(uriStr)) { System.Diagnostics.Debug.WriteLine(cookie.Name); System.Diagnostics.Debug.WriteLine(cookie.Value); } //******************* INITIATE PHPSESSION *************************** //******************* DOWNLOAD XML FILE LIST *************************** Uri.TryCreate("http://your.url.here/filelist.php", UriKind.RelativeOrAbsolute, out uriStr); Task <string> fileListTaskString = webClient.DownloadStringTaskAsync(uriStr); await fileListTaskString; string fileList = fileListTaskString.Result; DataSet ds = new DataSet(); byte[] byteArray = Encoding.UTF8.GetBytes(fileList); ds.ReadXml(new MemoryStream(byteArray)); if (ds.Tables.Count > 0) { var result = ds.Tables[0]; } dataGridView1.DataSource = ds.Tables[0]; //******************* DOWNLOAD XML FILE LIST *************************** //******************* CHECK MAX ID *************************** DataTable dtMaxId = ds.Tables[0]; using (var ms = new MemoryStream()) { int cnt = dtMaxId.Rows.Count; Int64 newID = Convert.ToInt64(dtMaxId.Rows[cnt - 1]["id"]); autoIncrementIndexOfData = newID.ToString(); if (lastId < newID) { lastId = newID; Debug.WriteLine(autoIncrementIndexOfData); //******************* CHECK TIME *************************** DateTime tim = Convert.ToDateTime(dtMaxId.Rows[cnt - 1]["time"]); { Debug.WriteLine(tim); } double timCmop = Convert.ToDouble((DateTime.UtcNow - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalSeconds); if (!((Convert.ToDouble(((tim.ToUniversalTime().AddSeconds(1)) - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalSeconds) - timCmop) >= 0.0)) { //last picture is oler than ten seconds Debug.WriteLine("No newId"); return(0); } //******************* CHECK TIME *************************** } else //no new Id { Debug.WriteLine("No newId"); return(0); } } //******************* CHECK MAX ID *************************** //******************* CHECK TIME *************************** //DataTable dtMaxTime = ds.Tables[0]; //using (var ms = new MemoryStream()) //{ // int cnt = dtMaxTime.Rows.Count; // DateTime tim = Convert.ToDateTime(dtMaxTime.Rows[cnt - 1]["time"]); // { // Debug.WriteLine(tim); // } //} //******************* CHECK TIME *************************** //******************* DOWNLOAD IV *************************** Task <byte []> ivArrTaskByte = webClient.DownloadDataTaskAsync("http://your.url.here/listiv.php?id=" + autoIncrementIndexOfData); await ivArrTaskByte; byte[] ivArr = ivArrTaskByte.Result; //******************* DOWNLOAD IV *************************** //******************* DOWNLOAD LENGTH *************************** Task <byte[]> lengthTaskByte = webClient.DownloadDataTaskAsync("http://your.url.here/getLen.php?id=" + autoIncrementIndexOfData); await lengthTaskByte; byte[] length = lengthTaskByte.Result; //filesize from files int fileSize = 0; foreach (byte l in length) { fileSize += (byte)(l - (byte)(0x30)); fileSize *= 10; } fileSize /= 10; System.Diagnostics.Debug.WriteLine("filesize: " + fileSize.ToString()); //******************* DOWNLOAD LENGTH *************************** //******************* DOWNL DATA *************************** Uri.TryCreate("http://your.url.here/dwnl_delete.php?id=" + autoIncrementIndexOfData, UriKind.RelativeOrAbsolute, out uriStr); Task <string> InputTaskString = webClient.DownloadStringTaskAsync(uriStr); await InputTaskString; InputString = InputTaskString.Result; //******************* DOWNL DATA *************************** // Create sha256 hash SHA256 mySHA256 = SHA256Managed.Create(); byte[] key = mySHA256.ComputeHash(Encoding.ASCII.GetBytes(password)); try { string decrypted = this.DecryptString(InputString, key, ivArr, fileSize); decryptedArr = Convert.FromBase64String(decrypted); //******************* DECRYPT *************************** //******************* CHECK SIGN *************************** Encoding encoding = Encoding.UTF8; DataTable dt = ds.Tables[0]; BinaryFormatter bf = new BinaryFormatter(); string dbSignStr = ""; using (var ms = new MemoryStream()) { foreach (DataRow row in dt.Rows) { if (row["id"].ToString() == autoIncrementIndexOfData) { Debug.WriteLine(row["sign"].ToString()); bf.Serialize(ms, row["sign"]); dbSignStr = row["sign"].ToString(); } } byte[] signObj = ms.ToArray(); } StringBuilder sBuilder = new StringBuilder(); using (HMACSHA256 hmac = new HMACSHA256(key)) { var hash = hmac.ComputeHash(StringToStream(decrypted)); // Create a new Stringbuilder to collect the bytes and create a string. // Loop through each byte of the hashed data // and format each one as a hexadecimal string. for (int i = 0; i < hash.Length; i++) { sBuilder.Append(hash[i].ToString("x2")); } // Return the hexadecimal string. Debug.WriteLine(sBuilder.ToString()); } //******************* CHECK SIGN *************************** if (sBuilder.ToString() == dbSignStr) //check signature { //SIGNATURE CHECK SUCCESSFUL try { using (System.IO.StreamWriter file = new System.IO.StreamWriter(dwnlPath, true)) { await file.WriteLineAsync(Encoding.UTF8.GetChars(decryptedArr)); } } catch (Exception ex) { //GlobalClass.Log("ERROR @ WriteFileToPath" + ex.ToString()); } //******************* CHECK SIGN *************************** } } catch { } finally { } try { Stream newStream = ByteArrToStream(decryptedArr); // Stretches the image to fit the pictureBox. if (pictureBox1.Image == null) { pictureBox1.SizeMode = PictureBoxSizeMode.StretchImage; } if (pictureBox1.Image != null) { pictureBox1.Image.Dispose(); } using (Image MyImage = Image.FromStream(newStream)) { pictureBox1.Image = (Image)MyImage.Clone(); pictureBox1.Update(); pictureBox1.Refresh(); } } catch (Exception ex) { System.Diagnostics.Debug.WriteLine(ex.ToString()); } } else //false { //stop await System.Threading.Tasks.Task.Delay(1000); return(0); } } return(0); }