public async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { List <ReleaseInfo> releases = new List <ReleaseInfo>(); var searchString = query.GetQueryString(); var searchUrl = SearchUrl; var queryCollection = new NameValueCollection(); queryCollection.Add("action", "basic"); if (!string.IsNullOrWhiteSpace(searchString)) { queryCollection.Add("searchstr", searchString); } foreach (var cat in MapTorznabCapsToTrackers(query)) { queryCollection.Add("filter_cat[" + cat + "]", "1"); } searchUrl += queryCollection.GetQueryString(); var results = await RequestStringWithCookiesAndRetry(searchUrl); // Occasionally the cookies become invalid, login again if that happens if (results.IsRedirect) { await ApplyConfiguration(null); results = await RequestStringWithCookiesAndRetry(searchUrl); } try { CQ dom = results.Content; var rows = dom["#torrent_table > tbody > tr.torrent"]; foreach (var row in rows) { CQ qRow = row.Cq(); var release = new ReleaseInfo(); release.MinimumRatio = 1; release.MinimumSeedTime = 172800; var catStr = row.ChildElements.ElementAt(0).FirstElementChild.GetAttribute("href").Split(new char[] { '[', ']' })[1]; release.Category = MapTrackerCatToNewznab(catStr); var qLink = row.ChildElements.ElementAt(1).Cq().Children("a")[0].Cq(); var linkStr = qLink.Attr("href"); release.Comments = new Uri(BaseUrl + "/" + linkStr); release.Guid = release.Comments; var qDownload = row.ChildElements.ElementAt(1).Cq().Find("a[title='Download']")[0].Cq(); release.Link = new Uri(BaseUrl + "/" + qDownload.Attr("href")); var dateStr = row.ChildElements.ElementAt(3).Cq().Text().Trim().Replace(" and", ""); release.PublishDate = DateTimeUtil.FromTimeAgo(dateStr); var sizeStr = row.ChildElements.ElementAt(4).Cq().Text(); release.Size = ReleaseInfo.GetBytes(sizeStr); release.Files = ParseUtil.CoerceInt(row.ChildElements.ElementAt(2).Cq().Text().Trim()); release.Grabs = ParseUtil.CoerceInt(row.ChildElements.ElementAt(6).Cq().Text().Trim()); release.Seeders = ParseUtil.CoerceInt(row.ChildElements.ElementAt(7).Cq().Text().Trim()); release.Peers = ParseUtil.CoerceInt(row.ChildElements.ElementAt(8).Cq().Text().Trim()) + release.Seeders; var grabs = qRow.Find("td:nth-child(6)").Text(); release.Grabs = ParseUtil.CoerceInt(grabs); if (qRow.Find("strong:contains(\"Freeleech!\")").Length >= 1) { release.DownloadVolumeFactor = 0; } else { release.DownloadVolumeFactor = 1; } release.UploadVolumeFactor = 1; var title = qRow.Find("td:nth-child(2)"); title.Find("span, strong, div, br").Remove(); release.Title = ParseUtil.NormalizeMultiSpaces(title.Text().Replace(" - ]", "]")); releases.Add(release); } } catch (Exception ex) { OnParseError(results.Content, ex); } return(releases); }
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { List <ReleaseInfo> releases = new List <ReleaseInfo>(); List <string> searchStrings = new List <string>(new string[] { query.GetQueryString() }); if (string.IsNullOrEmpty(query.Episode) && (query.Season > 0)) { // Tracker naming rules: If query is for a whole season, "Season #" instead of "S##". searchStrings.Add((query.SanitizedSearchTerm + " " + string.Format("\"Season {0}\"", query.Season)).Trim()); } List <string> categories = MapTorznabCapsToTrackers(query); List <string> request_urls = new List <string>(); foreach (var searchString in searchStrings) { var queryCollection = new NameValueCollection(); queryCollection.Add("action", "basic"); if (!string.IsNullOrWhiteSpace(searchString)) { queryCollection.Add("searchstr", searchString); } foreach (var cat in categories) { queryCollection.Add("filter_cat[" + cat + "]", "1"); } request_urls.Add(SearchUrl + queryCollection.GetQueryString()); } IEnumerable <Task <WebClientStringResult> > downloadTasksQuery = from url in request_urls select RequestStringWithCookiesAndRetry(url); WebClientStringResult[] responses = await Task.WhenAll(downloadTasksQuery.ToArray()); for (int i = 0; i < searchStrings.Count(); i++) { var results = responses[i]; // Occasionally the cookies become invalid, login again if that happens if (results.IsRedirect) { await ApplyConfiguration(null); results = await RequestStringWithCookiesAndRetry(request_urls[i]); } try { CQ dom = results.Content; var rows = dom["#torrent_table > tbody > tr.torrent"]; foreach (var row in rows) { CQ qRow = row.Cq(); var release = new ReleaseInfo(); release.MinimumRatio = 1; release.MinimumSeedTime = 172800; var catStr = row.ChildElements.ElementAt(0).FirstElementChild.GetAttribute("href").Split(new char[] { '[', ']' })[1]; release.Category = MapTrackerCatToNewznab(catStr); var qLink = row.ChildElements.ElementAt(1).Cq().Children("a")[0].Cq(); var linkStr = qLink.Attr("href"); release.Comments = new Uri(BaseUrl + "/" + linkStr); release.Guid = release.Comments; var qDownload = row.ChildElements.ElementAt(1).Cq().Find("a[title='Download']")[0].Cq(); release.Link = new Uri(BaseUrl + "/" + qDownload.Attr("href")); var dateStr = row.ChildElements.ElementAt(3).Cq().Text().Trim().Replace(" and", ""); release.PublishDate = DateTimeUtil.FromTimeAgo(dateStr); var sizeStr = row.ChildElements.ElementAt(4).Cq().Text(); release.Size = ReleaseInfo.GetBytes(sizeStr); release.Files = ParseUtil.CoerceInt(row.ChildElements.ElementAt(2).Cq().Text().Trim()); release.Grabs = ParseUtil.CoerceInt(row.ChildElements.ElementAt(6).Cq().Text().Trim()); release.Seeders = ParseUtil.CoerceInt(row.ChildElements.ElementAt(7).Cq().Text().Trim()); release.Peers = ParseUtil.CoerceInt(row.ChildElements.ElementAt(8).Cq().Text().Trim()) + release.Seeders; var grabs = qRow.Find("td:nth-child(6)").Text(); release.Grabs = ParseUtil.CoerceInt(grabs); if (qRow.Find("strong:contains(\"Freeleech!\")").Length >= 1) { release.DownloadVolumeFactor = 0; } else { release.DownloadVolumeFactor = 1; } release.UploadVolumeFactor = 1; var title = qRow.Find("td:nth-child(2)"); title.Find("span, strong, div, br").Remove(); release.Title = ParseUtil.NormalizeMultiSpaces(title.Text().Replace(" - ]", "]")); if (catStr == "10") //change "Season #" to "S##" for TV shows { release.Title = Regex.Replace(release.Title, @"Season (\d+)", m => string.Format("S{0:00}", Int32.Parse(m.Groups[1].Value))); } releases.Add(release); } } catch (Exception ex) { OnParseError(results.Content, ex); } } return(releases); }
public void should_normalize_multiple_spaces(string original, string newString) { ParseUtil.NormalizeMultiSpaces(original).Should().Be(newString); }
public IList <ReleaseInfo> ParseResponse(IndexerResponse indexerResponse) { var torrentInfos = new List <ReleaseInfo>(); var parser = new HtmlParser(); var dom = parser.ParseDocument(indexerResponse.Content); var rows = dom.QuerySelectorAll("#torrent_table > tbody > tr.torrent"); foreach (var row in rows) { var release = new TorrentInfo(); release.MinimumRatio = 1; release.MinimumSeedTime = 172800; // 48 hours var catStr = row.Children[0].FirstElementChild.GetAttribute("href").Split(new[] { '[', ']' })[1]; release.Categories = _categories.MapTrackerCatToNewznab(catStr); var qDetails = row.Children[1].QuerySelector("a[title='View Torrent']"); release.InfoUrl = _settings.BaseUrl + qDetails.GetAttribute("href"); release.Guid = release.InfoUrl; var qDownload = row.Children[1].QuerySelector("a[title='Download']"); release.DownloadUrl = _settings.BaseUrl + qDownload.GetAttribute("href"); var dateStr = row.Children[3].TextContent.Trim().Replace(" and", ""); release.PublishDate = DateTimeUtil.FromTimeAgo(dateStr); var sizeStr = row.Children[4].TextContent; release.Size = ParseUtil.GetBytes(sizeStr); release.Files = ParseUtil.CoerceInt(row.Children[2].TextContent.Trim()); release.Seeders = ParseUtil.CoerceInt(row.Children[7].TextContent.Trim()); release.Peers = ParseUtil.CoerceInt(row.Children[8].TextContent.Trim()) + release.Seeders; var grabs = row.QuerySelector("td:nth-child(6)").TextContent; release.Grabs = ParseUtil.CoerceInt(grabs); if (row.QuerySelector("strong:contains(\"Freeleech!\")") != null) { release.DownloadVolumeFactor = 0; } else { release.DownloadVolumeFactor = 1; } release.UploadVolumeFactor = 1; var title = row.QuerySelector("td:nth-child(2)"); foreach (var element in title.QuerySelectorAll("span, strong, div, br")) { element.Remove(); } release.Title = ParseUtil.NormalizeMultiSpaces(title.TextContent.Replace(" - ]", "]")); //change "Season #" to "S##" for TV shows if (catStr == "10") { release.Title = Regex.Replace(release.Title, @"Season (\d+)", m => string.Format("S{0:00}", int.Parse(m.Groups[1].Value))); } torrentInfos.Add(release); } return(torrentInfos.ToArray()); }