private bool IsCookiePresent(string name) { var rawCookies = CookieHeader.Split(';'); IDictionary <string, string> cookies = rawCookies .Where(e => e.Contains('=')) .ToDictionary((e) => e.Split('=')[0].Trim(), (e) => e.Split('=')[1].Trim()); return(cookies.ContainsKey(name)); }
public void TestParser() { CookieCollection c; CookieParser parser = new CookieParser(); CookieHeader header = (CookieHeader)parser.Parse("Cookie", new StringReader("name: \"Value\";cookie2: value2;cookieName;last:one")); Assert.Equal("Value", header.Cookies["name"].Value); Assert.Equal("value2", header.Cookies["cookie2"].Value); //Assert.Equal(string.Empty, header.Cookies["cookieName"].Value); Assert.Equal("one", header.Cookies["last"].Value); }
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { var releases = new List <ReleaseInfo>(); var searchString = query.GetQueryString(); var prevCook = CookieHeader + ""; var searchParams = new Dictionary <string, string> { { "do", "search" }, { "category", "0" }, { "include_dead_torrents", "no" } }; if (query.IsImdbQuery) { searchParams.Add("keywords", query.ImdbID); searchParams.Add("search_type", "t_both"); } else { searchParams.Add("keywords", searchString); searchParams.Add("search_type", "t_name"); } var searchPage = await RequestWithCookiesAndRetryAsync( SearchUrl, CookieHeader, RequestType.POST, null, searchParams); // Occasionally the cookies become invalid, login again if that happens if (searchPage.IsRedirect) { await ApplyConfiguration(null); searchPage = await RequestWithCookiesAndRetryAsync( SearchUrl, CookieHeader, RequestType.POST, null, searchParams); } try { var parser = new HtmlParser(); var dom = parser.ParseDocument(searchPage.ContentString); var rows = dom.QuerySelectorAll("table#sortabletable > tbody > tr:has(div > a[href*=\"details.php?id=\"])"); foreach (var row in rows) { var release = new ReleaseInfo(); var qDetails = row.QuerySelector("div > a[href*=\"details.php?id=\"]"); var qTitle = qDetails; // #7975 release.Title = qTitle.TextContent; release.Guid = new Uri(row.QuerySelector("td:nth-of-type(3) a").GetAttribute("href")); release.Link = release.Guid; release.Comments = new Uri(qDetails.GetAttribute("href")); //08-08-2015 12:51 release.PublishDate = DateTime.ParseExact( row.QuerySelectorAll("td:nth-of-type(2) div").Last().TextContent.Trim(), "dd-MM-yyyy H:mm", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal); release.Seeders = ParseUtil.CoerceInt(row.QuerySelector("td:nth-of-type(7)").TextContent); release.Peers = release.Seeders + ParseUtil.CoerceInt(row.QuerySelector("td:nth-of-type(8)").TextContent.Trim()); release.Size = ReleaseInfo.GetBytes(row.QuerySelector("td:nth-of-type(5)").TextContent.Trim()); var qBanner = row.QuerySelector("td:nth-of-type(2) .tooltip-content img"); if (qBanner != null) { release.BannerUrl = new Uri(qBanner.GetAttribute("src")); } var cat = row.QuerySelector("td:nth-of-type(1) a").GetAttribute("href"); var catSplit = cat.LastIndexOf('='); if (catSplit > -1) { cat = cat.Substring(catSplit + 1); } release.Category = MapTrackerCatToNewznab(cat); var grabs = row.QuerySelector("td:nth-child(6)").TextContent; release.Grabs = ParseUtil.CoerceInt(grabs); if (row.QuerySelector("img[alt^=\"Free Torrent\"]") != null) { release.DownloadVolumeFactor = 0; } else if (row.QuerySelector("img[alt^=\"Silver Torrent\"]") != null) { release.DownloadVolumeFactor = 0.5; } else { release.DownloadVolumeFactor = 1; } if (row.QuerySelector("img[alt^=\"x2 Torrent\"]") != null) { release.UploadVolumeFactor = 2; } else { release.UploadVolumeFactor = 1; } releases.Add(release); } } catch (Exception ex) { OnParseError(searchPage.ContentString, ex); } if (!CookieHeader.Trim().Equals(prevCook.Trim())) { SaveConfig(); } return(releases); }
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { var releases = new List <ReleaseInfo>(); var searchString = query.GetQueryString(); var prevCook = CookieHeader + ""; var searchStringIsImdbQuery = (ParseUtil.GetImdbID(searchString) != null); // If we have no query use the RSS Page as their server is slow enough at times! // ~15.01.2019 they removed the description tag making the RSS feed almost useless, we don't use it for now. See #4458 // if (false && query.IsTest || string.IsNullOrWhiteSpace(searchString)) /* * if (false) * { * var rssPage = await RequestStringWithCookiesAndRetry(string.Format(RSSUrl, configData.RSSKey.Value)); * try * { * if (rssPage.Content.EndsWith("\0")) * { * rssPage.Content = rssPage.Content.Substring(0, rssPage.Content.Length - 1); * } * rssPage.Content = RemoveInvalidXmlChars(rssPage.Content); * var rssDoc = XDocument.Parse(rssPage.Content); * * foreach (var item in rssDoc.Descendants("item")) * { * var title = item.Descendants("title").First().Value; * var description = item.Descendants("description").First().Value; * var link = item.Descendants("link").First().Value; * var category = item.Descendants("category").First().Value; * var date = item.Descendants("pubDate").First().Value; * * var torrentIdMatch = Regex.Match(link, "(?<=id=)(\\d)*"); * var torrentId = torrentIdMatch.Success ? torrentIdMatch.Value : string.Empty; * if (string.IsNullOrWhiteSpace(torrentId)) * throw new Exception("Missing torrent id"); * * var infoMatch = Regex.Match(description, @"Category:\W(?<cat>.*)\W\/\WSeeders:\W(?<seeders>[\d\,]*)\W\/\WLeechers:\W(?<leechers>[\d\,]*)\W\/\WSize:\W(?<size>[\d\.]*\W\S*)"); * if (!infoMatch.Success) * throw new Exception("Unable to find info"); * * var release = new ReleaseInfo * { * Title = title, * Description = title, * Guid = new Uri(string.Format(DownloadUrl, torrentId)), * Comments = new Uri(string.Format(CommentUrl, torrentId)), * PublishDate = DateTime.ParseExact(date, "yyyy-MM-dd H:mm:ss", CultureInfo.InvariantCulture), //2015-08-08 21:20:31 * Link = new Uri(string.Format(DownloadUrl, torrentId)), * Seeders = ParseUtil.CoerceInt(infoMatch.Groups["seeders"].Value), * Peers = ParseUtil.CoerceInt(infoMatch.Groups["leechers"].Value), * Size = ReleaseInfo.GetBytes(infoMatch.Groups["size"].Value), * Category = MapTrackerCatToNewznab(category) * }; * * release.Peers += release.Seeders; * releases.Add(release); * } * } * catch (Exception ex) * { * logger.Error("XSpeeds: Error while parsing the RSS feed:"); * logger.Error(rssPage.Content); * throw ex; * } * } */ //if (query.IsTest || !string.IsNullOrWhiteSpace(searchString)) /* * if (searchString.Length < 3 && !query.IsTest) * { * OnParseError("", new Exception("Minimum search length is 3")); * return releases; * } */ var searchParams = new Dictionary <string, string> { { "do", "search" }, { "category", "0" }, { "include_dead_torrents", "no" } }; if (query.IsImdbQuery) { searchParams.Add("keywords", query.ImdbID); searchParams.Add("search_type", "t_both"); } else { searchParams.Add("keywords", searchString); if (searchStringIsImdbQuery) { searchParams.Add("search_type", "t_both"); } else { searchParams.Add("search_type", "t_name"); } } var searchPage = await PostDataWithCookiesAndRetry(SearchUrl, searchParams, CookieHeader); // Occasionally the cookies become invalid, login again if that happens if (searchPage.IsRedirect) { await ApplyConfiguration(null); searchPage = await PostDataWithCookiesAndRetry(SearchUrl, searchParams, CookieHeader); } try { CQ dom = searchPage.Content; var rows = dom["table#sortabletable > tbody > tr:has(div > a[href*=\"details.php?id=\"])"]; foreach (var row in rows) { var release = new ReleaseInfo(); var qRow = row.Cq(); var qDetails = qRow.Find("div > a[href*=\"details.php?id=\"]"); // details link, release name get's shortened if it's to long var qTitle = qRow.Find("td:eq(1) .tooltip-content div:eq(0)"); // use Title from tooltip if (!qTitle.Any()) // fallback to Details link if there's no tooltip { qTitle = qDetails; } release.Title = qTitle.Text(); release.Guid = new Uri(qRow.Find("td:eq(2) a").Attr("href")); release.Link = release.Guid; release.Comments = new Uri(qDetails.Attr("href")); release.PublishDate = DateTime.ParseExact(qRow.Find("td:eq(1) div").Last().Text().Trim(), "dd-MM-yyyy H:mm", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal); //08-08-2015 12:51 release.Seeders = ParseUtil.CoerceInt(qRow.Find("td:eq(6)").Text()); release.Peers = release.Seeders + ParseUtil.CoerceInt(qRow.Find("td:eq(7)").Text().Trim()); release.Size = ReleaseInfo.GetBytes(qRow.Find("td:eq(4)").Text().Trim()); var qBanner = qRow.Find("td:eq(1) .tooltip-content img").First(); if (qBanner.Length > 0) { release.BannerUrl = new Uri(qBanner.Attr("src")); } var cat = row.Cq().Find("td:eq(0) a").First().Attr("href"); var catSplit = cat.LastIndexOf('='); if (catSplit > -1) { cat = cat.Substring(catSplit + 1); } release.Category = MapTrackerCatToNewznab(cat); var grabs = qRow.Find("td:nth-child(6)").Text(); release.Grabs = ParseUtil.CoerceInt(grabs); if (qRow.Find("img[alt^=\"Free Torrent\"]").Length >= 1) { release.DownloadVolumeFactor = 0; } else if (qRow.Find("img[alt^=\"Silver Torrent\"]").Length >= 1) { release.DownloadVolumeFactor = 0.5; } else { release.DownloadVolumeFactor = 1; } if (qRow.Find("img[alt^=\"x2 Torrent\"]").Length >= 1) { release.UploadVolumeFactor = 2; } else { release.UploadVolumeFactor = 1; } releases.Add(release); } } catch (Exception ex) { OnParseError(searchPage.Content, ex); } if (!CookieHeader.Trim().Equals(prevCook.Trim())) { SaveConfig(); } return(releases); }
public async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { var releases = new List <ReleaseInfo>(); var searchString = query.GetQueryString(); var prevCook = CookieHeader + ""; // If we have no query use the RSS Page as their server is slow enough at times! if (string.IsNullOrWhiteSpace(searchString)) { var rssPage = await RequestStringWithCookiesAndRetry(string.Format(RSSUrl, configData.RSSKey.Value)); if (rssPage.Content.EndsWith("\0")) { rssPage.Content = rssPage.Content.Substring(0, rssPage.Content.Length - 1); } rssPage.Content = RemoveInvalidXmlChars(rssPage.Content); var rssDoc = XDocument.Parse(rssPage.Content); foreach (var item in rssDoc.Descendants("item")) { var title = item.Descendants("title").First().Value; var description = item.Descendants("description").First().Value; var link = item.Descendants("link").First().Value; var category = item.Descendants("category").First().Value; var date = item.Descendants("pubDate").First().Value; var torrentIdMatch = Regex.Match(link, "(?<=id=)(\\d)*"); var torrentId = torrentIdMatch.Success ? torrentIdMatch.Value : string.Empty; if (string.IsNullOrWhiteSpace(torrentId)) { throw new Exception("Missing torrent id"); } var infoMatch = Regex.Match(description, @"Category:\W(?<cat>.*)\W\/\WSeeders:\W(?<seeders>[\d\,]*)\W\/\WLeechers:\W(?<leechers>[\d\,]*)\W\/\WSize:\W(?<size>[\d\.]*\W\S*)"); if (!infoMatch.Success) { throw new Exception("Unable to find info"); } var release = new ReleaseInfo { Title = title, Description = title, Guid = new Uri(string.Format(DownloadUrl, torrentId)), Comments = new Uri(string.Format(CommentUrl, torrentId)), PublishDate = DateTime.ParseExact(date, "yyyy-MM-dd H:mm:ss", CultureInfo.InvariantCulture), //2015-08-08 21:20:31 Link = new Uri(string.Format(DownloadUrl, torrentId)), Seeders = ParseUtil.CoerceInt(infoMatch.Groups["seeders"].Value), Peers = ParseUtil.CoerceInt(infoMatch.Groups["leechers"].Value), Size = ReleaseInfo.GetBytes(infoMatch.Groups["size"].Value), Category = MapTrackerCatToNewznab(category) }; release.Peers += release.Seeders; releases.Add(release); } } else { if (searchString.Length < 3) { OnParseError("", new Exception("Minimum search length is 3")); return(releases); } var searchParams = new Dictionary <string, string> { { "do", "search" }, { "keywords", searchString }, { "search_type", "t_name" }, { "category", "0" }, { "include_dead_torrents", "no" } }; var pairs = new Dictionary <string, string> { { "username", configData.Username.Value }, { "password", configData.Password.Value } }; var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, CookieHeader, true, null, SiteLink, true); if (!result.Cookies.Trim().Equals(prevCook.Trim())) { result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, result.Cookies, true, SearchUrl, SiteLink, true); } CookieHeader = result.Cookies; var attempt = 1; var searchPage = await PostDataWithCookiesAndRetry(SearchUrl, searchParams, CookieHeader); while (searchPage.IsRedirect && attempt < 3) { // add any cookies var cookieString = CookieHeader; if (searchPage.Cookies != null) { cookieString += " " + searchPage.Cookies; // resolve cookie conflicts - really no need for this as the webclient will handle it var expression = new Regex(@"([^\s]+)=([^=]+)(?:\s|$)"); var cookieDIctionary = new Dictionary <string, string>(); var matches = expression.Match(cookieString); while (matches.Success) { if (matches.Groups.Count > 2) { cookieDIctionary[matches.Groups[1].Value] = matches.Groups[2].Value; } matches = matches.NextMatch(); } cookieString = string.Join(" ", cookieDIctionary.Select(kv => kv.Key.ToString() + "=" + kv.Value.ToString()).ToArray()); } CookieHeader = cookieString; attempt++; searchPage = await PostDataWithCookiesAndRetry(SearchUrl, searchParams, CookieHeader); } try { CQ dom = searchPage.Content; var rows = dom["table#sortabletable > tbody > tr:has(div > a[href*=\"details.php?id=\"])"]; foreach (var row in rows) { var release = new ReleaseInfo(); var qRow = row.Cq(); var qDetails = qRow.Find("div > a[href*=\"details.php?id=\"]"); // details link, release name get's shortened if it's to long var qTitle = qRow.Find("td:eq(1) .tooltip-content div:eq(0)"); // use Title from tooltip if (!qTitle.Any()) // fallback to Details link if there's no tooltip { qTitle = qDetails; } release.Title = qTitle.Text(); release.Guid = new Uri(qRow.Find("td:eq(2) a").Attr("href")); release.Link = release.Guid; release.Comments = new Uri(qDetails.Attr("href")); release.PublishDate = DateTime.ParseExact(qRow.Find("td:eq(1) div").Last().Text().Trim(), "dd-MM-yyyy H:mm", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal); //08-08-2015 12:51 release.Seeders = ParseUtil.CoerceInt(qRow.Find("td:eq(6)").Text()); release.Peers = release.Seeders + ParseUtil.CoerceInt(qRow.Find("td:eq(7)").Text().Trim()); release.Size = ReleaseInfo.GetBytes(qRow.Find("td:eq(4)").Text().Trim()); var cat = row.Cq().Find("td:eq(0) a").First().Attr("href"); var catSplit = cat.LastIndexOf('='); if (catSplit > -1) { cat = cat.Substring(catSplit + 1); } release.Category = MapTrackerCatToNewznab(cat); // If its not apps or audio we can only mark as general TV if (release.Category == 0) { release.Category = 5030; } var grabs = qRow.Find("td:nth-child(6)").Text(); release.Grabs = ParseUtil.CoerceInt(grabs); if (qRow.Find("img[alt^=\"Free Torrent\"]").Length >= 1) { release.DownloadVolumeFactor = 0; } else if (qRow.Find("img[alt^=\"Silver Torrent\"]").Length >= 1) { release.DownloadVolumeFactor = 0.5; } else { release.DownloadVolumeFactor = 1; } if (qRow.Find("img[alt^=\"x2 Torrent\"]").Length >= 1) { release.UploadVolumeFactor = 2; } else { release.UploadVolumeFactor = 1; } releases.Add(release); } } catch (Exception ex) { OnParseError(searchPage.Content, ex); } } if (!CookieHeader.Trim().Equals(prevCook.Trim())) { SaveConfig(); } return(releases); }
public bool GetResponse() { try { bool completed = false; int tryCount = 0; // enable unsafe header parsing if needed if (AllowUnsafeHeader) { SetAllowUnsafeHeaderParsing(true); } // In .NET 4.0 default transport level security standard is TLS 1.1, // Some endpoints will reject this older insecure standard. ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12; // setup some request properties Request.Proxy = WebRequest.DefaultWebProxy; Request.Proxy.Credentials = CredentialCache.DefaultCredentials; if (UserAgent != null) { Request.UserAgent = UserAgent; } Request.Method = Method; Request.Accept = Accept; Request.Headers["Accept-Encoding"] = "gzip, deflate"; Request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip; if (AcceptLanguage != null) { Request.Headers["Accept-Language"] = AcceptLanguage; } Request.CookieContainer = new CookieContainer(); while (!completed) { tryCount++; Request.Timeout = Timeout + (TimeoutIncrement * tryCount); if (CookieHeader != null) { Request.CookieContainer.SetCookies(Request.RequestUri, CookieHeader.Replace(';', ',')); } try { Response = (HttpWebResponse)Request.GetResponse(); completed = true; } catch (WebException e) { // Skip retry logic on protocol errors if (e.Status == WebExceptionStatus.ProtocolError) { HttpStatusCode statusCode = ((HttpWebResponse)e.Response).StatusCode; switch (statusCode) { // Currently the only exception is the service temporarily unavailable status // So keep retrying when this is the case case HttpStatusCode.ServiceUnavailable: break; // all other status codes mostly indicate problems that won't be // solved within the retry period so fail these immediately default: Logger.Error("ScriptableScraperProvider: Connection failed: URL={0}, Status={1}, Description={2}.", requestUrl, statusCode, ((HttpWebResponse)e.Response).StatusDescription); return(false); } } // Return when hitting maximum retries. if (tryCount == MaxRetries) { Logger.Warn("ScriptableScraperProvider: Connection failed: Reached retry limit of " + MaxRetries + ". URL=" + requestUrl); return(false); } // If we did not experience a timeout but some other error // use the timeout value as a pause between retries if (e.Status != WebExceptionStatus.Timeout) { Thread.Sleep(Timeout + (TimeoutIncrement * tryCount)); } } catch (NotSupportedException e) { Logger.Error("ScriptableScraperProvider: Connection failed.", e); return(false); } catch (ProtocolViolationException e) { Logger.Error("ScriptableScraperProvider: Connection failed.", e); return(false); } catch (InvalidOperationException e) { Logger.Error("ScriptableScraperProvider: Connection failed.", e); return(false); } finally { // disable unsafe header parsing if it was enabled if (AllowUnsafeHeader) { SetAllowUnsafeHeaderParsing(false); } } } // persist the cookie header CookieHeader = Request.CookieContainer.GetCookieHeader(Request.RequestUri); // Debug Logger.Debug("ScriptableScraperProvider: GetResponse: URL={0}, UserAgent={1}, CookieHeader={2}, Accept={3}", requestUrl, UserAgent, CookieHeader, Accept); // disable unsafe header parsing if it was enabled if (AllowUnsafeHeader) { SetAllowUnsafeHeaderParsing(false); } return(true); } catch (Exception e) { Logger.Warn("ScriptableScraperProvider: Unexpected error getting http response from '{0}'", e, requestUrl); return(false); } }
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { var releases = new List <ReleaseInfo>(); NameValueCollection qParams = new NameValueCollection(); qParams.Add("tor[text]", query.GetQueryString()); qParams.Add("tor[srchIn][title]", "true"); qParams.Add("tor[srchIn][author]", "true"); qParams.Add("tor[searchType]", "all"); qParams.Add("tor[searchIn]", "torrents"); qParams.Add("tor[hash]", ""); qParams.Add("tor[sortType]", "default"); qParams.Add("tor[startNumber]", "0"); List <string> catList = MapTorznabCapsToTrackers(query); if (catList.Any()) { foreach (string cat in catList) { qParams.Add("tor[cat][]", cat); } } else { qParams.Add("tor[cat][]", "0"); } string urlSearch = SearchUrl; if (qParams.Count > 0) { urlSearch += $"?{qParams.GetQueryString()}"; } var response = await RequestStringWithCookiesAndRetry(urlSearch); if (response.Status == System.Net.HttpStatusCode.Forbidden || CookieHeader.Contains("pass=deleted")) { // re-login await ApplyConfiguration(null); response = await RequestStringWithCookiesAndRetry(urlSearch); } try { CQ dom = response.Content; var rows = dom["table[class='newTorTable'] > tbody > tr[id^=\"tdr\"]"]; foreach (IDomObject row in rows) { CQ torrentData = row.OuterHTML; CQ cells = row.Cq().Find("td"); string tid = torrentData.Attr("id").Substring(4); var qTitle = torrentData.Find("a[class='title']").First(); string title = qTitle.Text().Trim(); var details = new Uri(SiteLink + qTitle.Attr("href")); string author = torrentData.Find("a[class='author']").First().Text().Trim(); Uri link = new Uri(SiteLink + "tor/download.php?tid=" + tid); // DL Link is no long available directly, build it ourself long files = ParseUtil.CoerceLong(cells.Elements.ElementAt(4).Cq().Find("a").Text()); long size = ReleaseInfo.GetBytes(cells.Elements.ElementAt(4).Cq().Text().Split('[')[1].TrimEnd(']')); int seeders = ParseUtil.CoerceInt(cells.Elements.ElementAt(6).Cq().Find("p").ElementAt(0).Cq().Text()); int leechers = ParseUtil.CoerceInt(cells.Elements.ElementAt(6).Cq().Find("p").ElementAt(1).Cq().Text()); long grabs = ParseUtil.CoerceLong(cells.Elements.ElementAt(6).Cq().Find("p").ElementAt(2).Cq().Text()); bool freeleech = torrentData.Find("img[alt=\"freeleech\"]").Any(); string pubDateStr = cells.Elements.ElementAt(5).Cq().Text().Split('[')[0]; DateTime publishDate = DateTime.Parse(pubDateStr).ToLocalTime(); long category = 0; string cat = torrentData.Find("a[class='newCatLink']").First().Attr("href").Remove(0, "/tor/browse.php?tor[cat][]]=".Length); long.TryParse(cat, out category); var release = new ReleaseInfo(); release.Title = String.IsNullOrEmpty(author) ? title : String.Format("{0} by {1}", title, author); release.Guid = link; release.Link = link; release.PublishDate = publishDate; release.Files = files; release.Size = size; release.Description = release.Title; release.Seeders = seeders; release.Peers = seeders + leechers; release.Grabs = grabs; release.MinimumRatio = 1; release.MinimumSeedTime = 172800; release.Category = MapTrackerCatToNewznab(category.ToString()); release.Comments = details; if (freeleech) { release.DownloadVolumeFactor = 0; } else { release.DownloadVolumeFactor = 1; } release.UploadVolumeFactor = 1; releases.Add(release); } } catch (Exception ex) { OnParseError(response.Content, ex); } return(releases); }