public static string GetInfoHashFromMagnet(string magnet) { try { var xt = ParseUtil.GetArgumentFromQueryString(magnet.ToString(), "xt"); return(xt.Split(':').Last()); // remove prefix urn:btih: } catch (Exception) { return(null); } }
protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query) { var releases = new List<ReleaseInfo>(); var searchUrl = BrowseUrl; var searchString = query.GetQueryString(); var queryCollection = new NameValueCollection { {"searchstr", searchString}, {"order_by", "time"}, {"order_way", "desc"}, {"action", "basic"}, {"searchsubmit", "1"} }; var i = 0; foreach (var cat in MapTorznabCapsToTrackers(query)) { queryCollection.Add("artistcheck["+i+"]", cat); i++; } searchUrl += "?" + queryCollection.GetQueryString(); var results = await RequestStringWithCookies(searchUrl); try { string RowsSelector = ".torrent_table > tbody > tr"; var SearchResultParser = new HtmlParser(); var SearchResultDocument = SearchResultParser.Parse(results.Content); var Rows = SearchResultDocument.QuerySelectorAll(RowsSelector); bool stickyGroup = false; string CategoryStr; ICollection<int> GroupCategory = null; string GroupTitle = null; //Nullable<DateTime> GroupPublishDate = null; foreach (var Row in Rows) { if (Row.ClassList.Contains("torrent")) { // garbage rows continue; } else if (Row.ClassList.Contains("group")) { stickyGroup = Row.ClassList.Contains("sticky"); var dispalyname = Row.QuerySelector("#displayname"); var qCat = Row.QuerySelector("td.cats_col > div"); CategoryStr = qCat.GetAttribute("title"); var qArtistLink = dispalyname.QuerySelector("#groupplatform > a"); if (qArtistLink != null) CategoryStr = ParseUtil.GetArgumentFromQueryString(qArtistLink.GetAttribute("href"), "artistname"); GroupCategory = MapTrackerCatToNewznab(CategoryStr); var qDetailsLink = dispalyname.QuerySelector("#groupname > a"); GroupTitle = qDetailsLink.TextContent; } else if (Row.ClassList.Contains("group_torrent")) { if (Row.QuerySelector("td.edition_info") != null) // ignore edition rows continue; var release = new ReleaseInfo(); release.MinimumRatio = 1; release.MinimumSeedTime = 80 * 3600; var qDetailsLink = Row.QuerySelector("a[href^=\"torrents.php?id=\"]"); var qDescription = qDetailsLink.QuerySelector("span.torrent_info_tags"); var qDLLink = Row.QuerySelector("a[href^=\"torrents.php?action=download\"]"); var qTime = Row.QuerySelector("span.time"); // some users have an extra colum (8), we can't use nth-last-child var qSize = Row.QuerySelector("td:nth-child(4)"); var qGrabs = Row.QuerySelector("td:nth-child(5)"); var qSeeders = Row.QuerySelector("td:nth-child(6)"); var qLeechers = Row.QuerySelector("td:nth-child(7)"); var qFreeLeech = Row.QuerySelector("strong.freeleech_label"); var qNeutralLeech = Row.QuerySelector("strong.neutralleech_label"); var RowTitle = Row.GetAttribute("title"); var Time = qTime.GetAttribute("title"); release.PublishDate = DateTime.SpecifyKind(DateTime.ParseExact(Time, "MMM dd yyyy, HH:mm", CultureInfo.InvariantCulture), DateTimeKind.Unspecified).ToLocalTime(); release.Category = GroupCategory; if (qDescription != null) release.Description = qDescription.TextContent; release.Title = qDetailsLink.TextContent; release.Title = release.Title.Replace(", Freeleech!", ""); release.Title = release.Title.Replace(", Neutral Leech!", ""); if (stickyGroup) // AND match for sticky releases if ((query.ImdbID == null || !TorznabCaps.SupportsImdbSearch) && !query.MatchQueryStringAND(release.Title)) continue; var Size = qSize.TextContent; if (string.IsNullOrEmpty(Size)) // external links, example BlazBlue: Calamity Trigger Manual - Guide [GameDOX - External Link] continue; release.Size = ReleaseInfo.GetBytes(Size); release.Link = new Uri(SiteLink + qDLLink.GetAttribute("href")); release.Comments = new Uri(SiteLink + qDetailsLink.GetAttribute("href")); release.Guid = release.Link; release.Grabs = ParseUtil.CoerceLong(qGrabs.TextContent); release.Seeders = ParseUtil.CoerceInt(qSeeders.TextContent); release.Peers = ParseUtil.CoerceInt(qLeechers.TextContent) + release.Seeders; if (qFreeLeech != null) { release.UploadVolumeFactor = 1; release.DownloadVolumeFactor = 0; } else if (qNeutralLeech != null) { release.UploadVolumeFactor = 0; release.DownloadVolumeFactor = 0; } else { release.UploadVolumeFactor = 1; release.DownloadVolumeFactor = 1; } releases.Add(release); } } } catch (Exception ex) { OnParseError(results.Content, ex); } return releases; }
private async Task <IEnumerable <ReleaseInfo> > performRegularQuery(TorznabQuery query, string hebName = null) { var releases = new List <ReleaseInfo>(); var searchurls = new List <string>(); var searchUrl = SearchUrl; var queryCollection = new NameValueCollection(); var searchString = query.GetQueryString(); if (query.IsImdbQuery) { searchString = query.ImdbID; } if (hebName != null) { searchString = hebName + " - עונה " + query.Season + " פרק " + query.Episode; } searchUrl += "?"; if (!string.IsNullOrWhiteSpace(searchString)) { var strEncoded = WebUtilityHelpers.UrlEncode(searchString, Encoding); searchUrl += "&query=" + strEncoded + "&matchquery=any"; } foreach (var cat in MapTorznabCapsToTrackers(query)) { searchUrl += "&c[]=" + cat; } var data = await RequestStringWithCookiesAndRetry(searchUrl); try { CQ dom = data.Content; var rows = dom["tr.box_torrent"]; foreach (var row in rows) { CQ qRow = row.Cq(); var release = new ReleaseInfo(); var main_title_link = qRow.Find("div.main_title > a"); release.Title = main_title_link.Attr("longtitle"); if (release.Title.IsNullOrEmptyOrWhitespace()) { release.Title = main_title_link.Text(); } release.MinimumRatio = 1; release.MinimumSeedTime = 172800; int seeders, peers; if (ParseUtil.TryCoerceInt(qRow.Find("td:nth-child(7) > div").Text(), out seeders)) { release.Seeders = seeders; if (ParseUtil.TryCoerceInt(qRow.Find("td:nth-child(8) > div").Text(), out peers)) { release.Peers = peers + release.Seeders; } } release.Grabs = ParseUtil.CoerceLong(qRow.Find("td:nth-child(5)").Text().Replace(",", "")); release.Seeders = ParseUtil.CoerceInt(qRow.Find("td:nth-child(6)").Text().Replace(",", "")); release.Peers = ParseUtil.CoerceInt(qRow.Find("td:nth-child(7)").Text().Replace(",", "")) + release.Seeders; string fullSize = qRow.Find("td:nth-child(4)").Text(); release.Size = ReleaseInfo.GetBytes(fullSize); release.Comments = new Uri(SiteLink + qRow.Find("a.threadlink[href]").Attr("href")); release.Link = new Uri(SiteLink + qRow.Find("a:has(div.dlimg)").Attr("href")); release.Guid = release.Comments; try { release.BannerUrl = new Uri(qRow.Find("a[imgsrc]").Attr("imgsrc")); } catch (Exception) { // do nothing, some releases have invalid banner URLs, ignore the banners in this case } var dateStringAll = qRow.Find("div.up_info2")[0].ChildNodes.Last().ToString(); var dateParts = dateStringAll.Split(' '); string dateString = dateParts[dateParts.Length - 2] + " " + dateParts[dateParts.Length - 1]; release.PublishDate = DateTime.ParseExact(dateString, "dd/MM/yy HH:mm", CultureInfo.InvariantCulture); string categoryLink = qRow.Find("a[href^=\"/browse.php?cat=\"]").Attr("href"); var catid = ParseUtil.GetArgumentFromQueryString(categoryLink, "cat"); release.Category = MapTrackerCatToNewznab(catid); if (qRow.Find("a[href^=\"?freeleech=1\"]").Length >= 1) { release.DownloadVolumeFactor = 0; } else { release.DownloadVolumeFactor = 1; } release.UploadVolumeFactor = 1; var sub_title = qRow.Find("div.sub_title"); var imdb_link = sub_title.Find("span.imdb-inline > a"); release.Imdb = ParseUtil.GetLongFromString(imdb_link.Attr("href")); sub_title.Find("span.imdb-inline").Remove(); release.Description = sub_title.Text(); releases.Add(release); } } catch (Exception ex) { OnParseError(data.Content, ex); } return(releases); }
List <ReleaseInfo> parseTorrents(WebClientStringResult results, String seasonep, TorznabQuery query, int already_founded, int limit) { var releases = new List <ReleaseInfo>(); try { CQ dom = results.Content; ReleaseInfo release; var rows = dom[".box_torrent_all"].Find(".box_torrent"); // Check torrents only till we reach the query Limit for (int i = 0; (i < rows.Length && ((already_founded + releases.Count) < limit)); i++) { try { CQ qRow = rows[i].Cq(); var key = dom["link[rel=alternate]"].First().Attr("href").Split('=').Last(); release = new ReleaseInfo(); var torrentTxt = qRow.Find(".torrent_txt, .torrent_txt2").Find("a").Get(0); //if (torrentTxt == null) continue; release.Title = torrentTxt.GetAttribute("title"); release.Description = qRow.Find("span").Get(0).GetAttribute("title") + " " + qRow.Find("a.infolink").Text(); release.MinimumRatio = 1; release.MinimumSeedTime = 172800; release.DownloadVolumeFactor = 0; release.UploadVolumeFactor = 1; string downloadLink = SiteLink + torrentTxt.GetAttribute("href"); string downloadId = downloadLink.Substring(downloadLink.IndexOf("&id=") + 4); release.Link = new Uri(SiteLink.ToString() + "torrents.php?action=download&id=" + downloadId + "&key=" + key); release.Comments = new Uri(SiteLink.ToString() + "torrents.php?action=details&id=" + downloadId); release.Guid = new Uri(release.Comments.ToString() + "#comments");; release.Seeders = ParseUtil.CoerceInt(qRow.Find(".box_s2").Find("a").First().Text()); release.Peers = ParseUtil.CoerceInt(qRow.Find(".box_l2").Find("a").First().Text()) + release.Seeders; var imdblink = qRow.Find("a[href*=\".imdb.com/title\"]").Attr("href"); release.Imdb = ParseUtil.GetLongFromString(imdblink); var banner = qRow.Find("img.infobar_ico").Attr("onmouseover"); if (banner != null) { Regex BannerRegEx = new Regex(@"mutat\('(.*?)', '", RegexOptions.Compiled); var BannerMatch = BannerRegEx.Match(banner); var bannerurl = BannerMatch.Groups[1].Value; release.BannerUrl = new Uri(bannerurl); } release.PublishDate = DateTime.Parse(qRow.Find(".box_feltoltve2").Get(0).InnerHTML.Replace("<br />", " "), CultureInfo.InvariantCulture); string[] sizeSplit = qRow.Find(".box_meret2").Get(0).InnerText.Split(' '); release.Size = ReleaseInfo.GetBytes(sizeSplit[1].ToLower(), ParseUtil.CoerceFloat(sizeSplit[0])); string catlink = qRow.Find("a:has(img[class='categ_link'])").First().Attr("href"); string cat = ParseUtil.GetArgumentFromQueryString(catlink, "tipus"); release.Category = MapTrackerCatToNewznab(cat); /* if the release name not contains the language we add it because it is know from category */ if (cat.Contains("hun") && !release.Title.Contains("hun")) { release.Title += ".hun"; } if (seasonep == null) { releases.Add(release); } else { if (query.MatchQueryStringAND(release.Title, null, seasonep)) { /* For sonnar if the search querry was english the title must be english also so we need to change the Description and Title */ var temp = release.Title; // releasedata everithing after Name.S0Xe0X String releasedata = release.Title.Split(new[] { seasonep }, StringSplitOptions.None)[1].Trim(); /* if the release name not contains the language we add it because it is know from category */ if (cat.Contains("hun") && !releasedata.Contains("hun")) { releasedata += ".hun"; } // release description contains [imdb: ****] but we only need the data before it for title String[] description = { release.Description, "" }; if (release.Description.Contains("[imdb:")) { description = release.Description.Split('['); description[1] = "[" + description[1]; } release.Title = (description[0].Trim() + "." + seasonep.Trim() + "." + releasedata.Trim('.')).Replace(' ', '.'); // if search is done for S0X than we dont want to put . between S0X and E0X Match match = Regex.Match(releasedata, @"^E\d\d?"); if (seasonep.Length == 3 && match.Success) { release.Title = (description[0].Trim() + "." + seasonep.Trim() + releasedata.Trim('.')).Replace(' ', '.'); } // add back imdb points to the description [imdb: 8.7] release.Description = temp + " " + description[1]; release.Description = release.Description.Trim(); releases.Add(release); } } } catch (FormatException ex) { logger.Error("Problem of parsing Torrent:" + rows[i].InnerHTML); logger.Error("Exception was the following:" + ex); } } } catch (Exception ex) { OnParseError(results.Content, ex); } return(releases); }
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { var releases = new List <ReleaseInfo>(); var searchString = query.GetQueryString(); var queryCollection = new Dictionary <string, string>(); queryCollection.Add("search_type", "t_name"); queryCollection.Add("do", "search"); queryCollection.Add("keywords", searchString); queryCollection.Add("category", "0"); // multi cat search not supported var results = await PostDataWithCookies(BrowseUrl, queryCollection); if (results.IsRedirect) { // re-login await ApplyConfiguration(null); results = await PostDataWithCookies(BrowseUrl, queryCollection); } try { var RowsSelector = "table[id='sortabletable'] > tbody > tr"; var SearchResultParser = new HtmlParser(); var SearchResultDocument = SearchResultParser.Parse(results.Content); var Rows = SearchResultDocument.QuerySelectorAll(RowsSelector); var lastDate = DateTime.Now; foreach (var Row in Rows.Skip(1)) { var release = new ReleaseInfo(); release.MinimumRatio = 1; release.MinimumSeedTime = 0; var category = Row.QuerySelector("td:nth-child(1) > a"); var title = Row.QuerySelector("td:nth-child(2) a"); var added = Row.QuerySelector("td:nth-child(2) > div:has(span[style=\"float: right;\"])"); if (added == null) // not a torrent line { continue; } var pretime = added.QuerySelector("font.mkprettytime"); var tooltip = Row.QuerySelector("td:nth-child(2) > div.tooltip-content"); var link = Row.QuerySelector("td:nth-child(3)").QuerySelector("a"); var comments = Row.QuerySelector("td:nth-child(2)").QuerySelector("a"); var Size = Row.QuerySelector("td:nth-child(5)"); var Grabs = Row.QuerySelector("td:nth-child(6)").QuerySelector("a"); var Seeders = Row.QuerySelector("td:nth-child(7)").QuerySelector("a"); var Leechers = Row.QuerySelector("td:nth-child(8)").QuerySelector("a"); var categoryIdparts = category.GetAttribute("href").Split('-'); var categoryId = categoryIdparts[categoryIdparts.Length - 1].Replace(".ts", ""); release.Title = title.TextContent; release.Category = MapTrackerCatToNewznab(categoryId); release.Link = new Uri(link.GetAttribute("href")); release.Comments = new Uri(comments.GetAttribute("href")); release.Guid = release.Link; release.Size = ReleaseInfo.GetBytes(Size.TextContent); release.Seeders = ParseUtil.CoerceInt(Seeders.TextContent); release.Peers = ParseUtil.CoerceInt(Leechers.TextContent) + release.Seeders; release.Grabs = ParseUtil.CoerceLong(Grabs.TextContent); if (TorrentHTTPSMode) { var linkHttps = Row.QuerySelector("td:nth-child(4)").QuerySelector("a").GetAttribute("href"); var idTorrent = ParseUtil.GetArgumentFromQueryString(linkHttps, "id"); release.Link = new Uri($"{SiteLink}download.php?id={idTorrent}&type=ssl"); } if (added.QuerySelector("img[alt^=\"TORRENT GRATUIT\"]") != null) { release.DownloadVolumeFactor = 0; } else if (added.QuerySelector("img[alt^=\"TORRENT SILVER\"]") != null) { release.DownloadVolumeFactor = 0.5; } else { release.DownloadVolumeFactor = 1; } if (added.QuerySelector("img[alt^=\"TORRENT X2\"]") != null) { release.UploadVolumeFactor = 2; } else { release.UploadVolumeFactor = 1; } if (tooltip != null) { var banner = tooltip.QuerySelector("img"); if (banner != null) { release.BannerUrl = new Uri(banner.GetAttribute("src")); banner.Remove(); } tooltip.QuerySelector("div:contains(\"Total Hits: \")").Remove(); var longtitle = tooltip.QuerySelectorAll("div").First(); release.Title = longtitle.TextContent; longtitle.Remove(); var desc = tooltip.TextContent.Trim(); if (!string.IsNullOrWhiteSpace(desc)) { release.Description = desc; } } // if even the tooltip title is shortened we use the URL if (release.Title.EndsWith("...")) { var tregex = new Regex(@"/([^/]+)-s-\d+\.ts"); var tmatch = tregex.Match(release.Comments.ToString()); release.Title = tmatch.Groups[1].Value; } if (pretime != null) { if (release.Description == null) { release.Description = pretime.TextContent; } else { release.Description += "<br>\n" + pretime.TextContent; } release.PublishDate = lastDate; } else { release.PublishDate = DateTime.ParseExact(added.TextContent.Trim(), "dd.M.yyyy HH:mm", CultureInfo.InvariantCulture); lastDate = release.PublishDate; } releases.Add(release); } } catch (Exception ex) { OnParseError(results.Content, ex); } return(releases); }
private List <ReleaseInfo> ParseTorrents(WebResult results, string episodeString, TorznabQuery query, int alreadyFound, int limit, int previouslyParsedOnPage) { var releases = new List <ReleaseInfo>(); try { var parser = new HtmlParser(); var dom = parser.ParseDocument(results.ContentString); var rows = dom.QuerySelectorAll(".box_torrent").Skip(previouslyParsedOnPage).Take(limit - alreadyFound); var key = ParseUtil.GetArgumentFromQueryString( dom.QuerySelector("link[rel=alternate]").GetAttribute("href"), "key"); // Check torrents only till we reach the query Limit foreach (var row in rows) { try { var torrentTxt = row.QuerySelector(".torrent_txt, .torrent_txt2").QuerySelector("a"); //if (torrentTxt == null) continue; var infoLink = row.QuerySelector("a.infolink"); var imdbId = ParseUtil.GetLongFromString(infoLink?.GetAttribute("href")); var desc = row.QuerySelector("span")?.GetAttribute("title") + " " + infoLink?.TextContent; var torrentLink = SiteLink + torrentTxt.GetAttribute("href"); var downloadId = ParseUtil.GetArgumentFromQueryString(torrentLink, "id"); //Build site links var baseLink = SiteLink + "torrents.php?action=details&id=" + downloadId; var downloadLink = SiteLink + "torrents.php?action=download&id=" + downloadId; var commentsUri = new Uri(baseLink + "#comments"); var guidUri = new Uri(baseLink); var linkUri = new Uri(QueryHelpers.AddQueryString(downloadLink, "key", key)); var seeders = ParseUtil.CoerceInt(row.QuerySelector(".box_s2 a").TextContent); var leechers = ParseUtil.CoerceInt(row.QuerySelector(".box_l2 a").TextContent); var publishDate = DateTime.Parse( row.QuerySelector(".box_feltoltve2").InnerHtml.Replace("<br>", " "), CultureInfo.InvariantCulture); var sizeSplit = row.QuerySelector(".box_meret2").TextContent.Split(' '); var size = ReleaseInfo.GetBytes(sizeSplit[1].ToLower(), ParseUtil.CoerceFloat(sizeSplit[0])); var catLink = row.QuerySelector("a:has(img[class='categ_link'])").GetAttribute("href"); var cat = ParseUtil.GetArgumentFromQueryString(catLink, "tipus"); var title = torrentTxt.GetAttribute("title"); // if the release name does not contain the language we add from the category if (cat.Contains("hun") && !title.ToLower().Contains("hun")) { title += ".hun"; } // Minimum seed time is 48 hours + 24 minutes (.4 hours) per GB of torrent size if downloaded in full. // Or a 1.0 ratio on the torrent var seedTime = TimeSpan.FromHours(48) + TimeSpan.FromMinutes(24 * ReleaseInfo.GigabytesFromBytes(size).Value); var release = new ReleaseInfo { Title = title, Description = desc.Trim(), MinimumRatio = 1, MinimumSeedTime = (long)seedTime.TotalSeconds, DownloadVolumeFactor = 0, UploadVolumeFactor = 1, Link = linkUri, Comments = commentsUri, Guid = guidUri, Seeders = seeders, Peers = leechers + seeders, Imdb = imdbId, PublishDate = publishDate, Size = size, Category = MapTrackerCatToNewznab(cat) }; var banner = row.QuerySelector("img.infobar_ico")?.GetAttribute("onmouseover"); if (banner != null) { // static call to Regex.Match caches the pattern, so we aren't recompiling every loop. var bannerMatch = Regex.Match(banner, @"mutat\('(.*?)', '", RegexOptions.Compiled); release.BannerUrl = new Uri(bannerMatch.Groups[1].Value); } //TODO there is room for improvement here. if (episodeString != null && query.MatchQueryStringAND(release.Title, queryStringOverride: episodeString) && !query.IsImdbQuery) { // For Sonarr if the search query was english the title must be english also // The description holds the alternate language name // so we need to swap title and description names var tempTitle = release.Title; // releaseData everything after Name.S0Xe0X var releaseIndex = tempTitle.IndexOf(episodeString, StringComparison.OrdinalIgnoreCase) + episodeString.Length; var releaseData = tempTitle.Substring(releaseIndex).Trim(); // release description contains [imdb: ****] but we only need the data before it for title var description = new[] { release.Description, "" }; if (release.Description.Contains("[imdb:")) { description = release.Description.Split('['); description[1] = "[" + description[1]; } var match = Regex.Match(releaseData, @"^E\d\d?"); // if search is done for S0X than we don't want to put . between S0X and E0X var episodeSeparator = episodeString.Length == 3 && match.Success ? null : "."; release.Title = (description[0].Trim() + "." + episodeString.Trim() + episodeSeparator + releaseData.Trim('.')).Replace(' ', '.'); // add back imdb points to the description [imdb: 8.7] release.Description = tempTitle + " " + description[1]; release.Description = release.Description.Trim(); } releases.Add(release); } catch (FormatException ex) { logger.Error("Problem of parsing Torrent:" + row.InnerHtml); logger.Error("Exception was the following:" + ex); } } } catch (Exception ex) { OnParseError(results.ContentString, ex); } return(releases); }
private async Task <List <ReleaseInfo> > PerformQueryAsync(TorznabQuery query, string episodeString) { var releases = new List <ReleaseInfo>(); var pairs = new NameValueCollection { { "nyit_sorozat_resz", "true" }, { "tipus", "kivalasztottak_kozott" }, { "submit.x", "1" }, { "submit.y", "1" }, { "submit", "Ok" } }; if (query.IsImdbQuery) { pairs.Add("miben", "imdb"); pairs.Add("mire", query.ImdbID); } else { pairs.Add("miben", "name"); pairs.Add("mire", episodeString == null ? query.GetQueryString() : query.SanitizedSearchTerm); } var cats = MapTorznabCapsToTrackers(query); if (cats.Count == 0) { cats = GetAllTrackerCategories(); } if (!configData.Hungarian.Value) { cats.RemoveAll(cat => cat.Contains("_hun")); } if (!configData.English.Value) { cats = cats.Except(_languageCats).ToList(); } pairs.Add("kivalasztott_tipus[]", string.Join(",", cats)); var results = await RequestWithCookiesAndRetryAsync( SearchUrl, null, RequestType.POST, null, pairs.ToEnumerable(true)); var parser = new HtmlParser(); var dom = parser.ParseDocument(results.ContentString); // find number of torrents / page var torrentPerPage = dom.QuerySelectorAll(".box_torrent").Length; if (torrentPerPage == 0) { return(releases); } var startPage = (query.Offset / torrentPerPage) + 1; var previouslyParsedOnPage = query.Offset % torrentPerPage; // find page links in the bottom var lastPageLink = dom.QuerySelectorAll("div[id=pager_bottom] a[href*=oldal]") .LastOrDefault()?.GetAttribute("href"); var pages = int.TryParse(ParseUtil.GetArgumentFromQueryString(lastPageLink, "oldal"), out var lastPage) ? lastPage : 1; var limit = query.Limit; if (limit == 0) { limit = 100; } if (startPage == 1) { releases = ParseTorrents(results, episodeString, query, releases.Count, limit, previouslyParsedOnPage); previouslyParsedOnPage = 0; startPage++; } // Check all the pages for the torrents. // The starting index is 2. (the first one is the original where we parse out the pages.) for (var page = startPage; page <= pages && releases.Count < limit; page++) { pairs["oldal"] = page.ToString(); results = await RequestWithCookiesAndRetryAsync( SearchUrl, null, RequestType.POST, null, pairs.ToEnumerable(true)); releases.AddRange(ParseTorrents(results, episodeString, query, releases.Count, limit, previouslyParsedOnPage)); previouslyParsedOnPage = 0; } return(releases); }
public async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { var releases = new List <ReleaseInfo>(); var searchString = query.GetQueryString(); var pairs = new List <KeyValuePair <string, string> >(); pairs.Add(new KeyValuePair <string, string>("nyit_sorozat_resz", "true")); pairs.Add(new KeyValuePair <string, string>("miben", "name")); pairs.Add(new KeyValuePair <string, string>("tipus", "kivalasztottak_kozott")); pairs.Add(new KeyValuePair <string, string>("submit.x", "1")); pairs.Add(new KeyValuePair <string, string>("submit.y", "1")); pairs.Add(new KeyValuePair <string, string>("submit", "Ok")); pairs.Add(new KeyValuePair <string, string>("mire", searchString)); var cats = MapTorznabCapsToTrackers(query); if (cats.Count == 0) { cats = GetAllTrackerCategories(); } foreach (var lcat in LanguageCats) { if (!configData.Hungarian.Value) { cats.Remove(lcat + "_hun"); } if (!configData.English.Value) { cats.Remove(lcat); } } foreach (var cat in cats) { pairs.Add(new KeyValuePair <string, string>("kivalasztott_tipus[]", cat)); } var results = await PostDataWithCookiesAndRetry(SearchUrl, pairs); try { CQ dom = results.Content; ReleaseInfo release; var rows = dom[".box_torrent_all"].Find(".box_torrent"); foreach (var row in rows) { CQ qRow = row.Cq(); var key = dom ["link[rel=alternate]"].First().Attr("href").Split('=').Last(); release = new ReleaseInfo(); var torrentTxt = qRow.Find(".torrent_txt, .torrent_txt2").Find("a").Get(0); //if (torrentTxt == null) continue; release.Title = torrentTxt.GetAttribute("title"); release.Description = qRow.Find("div.siterank").Text(); release.MinimumRatio = 1; release.MinimumSeedTime = 172800; release.DownloadVolumeFactor = 0; release.UploadVolumeFactor = 1; string downloadLink = SiteLink + torrentTxt.GetAttribute("href"); string downloadId = downloadLink.Substring(downloadLink.IndexOf("&id=") + 4); release.Link = new Uri(SiteLink.ToString() + "torrents.php?action=download&id=" + downloadId + "&key=" + key); release.Comments = new Uri(SiteLink.ToString() + "torrents.php?action=details&id=" + downloadId); release.Guid = new Uri(release.Comments.ToString() + "#comments");; release.Seeders = ParseUtil.CoerceInt(qRow.Find(".box_s2").Find("a").First().Text()); release.Peers = ParseUtil.CoerceInt(qRow.Find(".box_l2").Find("a").First().Text()) + release.Seeders; var imdblink = qRow.Find("a[href*=\".imdb.com/title\"]").Attr("href"); release.Imdb = ParseUtil.GetLongFromString(imdblink); var banner = qRow.Find("img.infobar_ico").Attr("onmouseover"); if (banner != null) { Regex BannerRegEx = new Regex(@"mutat\('(.*?)', '", RegexOptions.Compiled); var BannerMatch = BannerRegEx.Match(banner); var bannerurl = BannerMatch.Groups[1].Value; release.BannerUrl = new Uri(bannerurl); } release.PublishDate = DateTime.Parse(qRow.Find(".box_feltoltve2").Get(0).InnerHTML.Replace("<br />", " "), CultureInfo.InvariantCulture); string[] sizeSplit = qRow.Find(".box_meret2").Get(0).InnerText.Split(' '); release.Size = ReleaseInfo.GetBytes(sizeSplit[1].ToLower(), ParseUtil.CoerceFloat(sizeSplit[0])); string catlink = qRow.Find("a:has(img[class='categ_link'])").First().Attr("href"); string cat = ParseUtil.GetArgumentFromQueryString(catlink, "tipus"); release.Category = MapTrackerCatToNewznab(cat); releases.Add(release); } } catch (Exception ex) { OnParseError(results.Content, ex); } return(releases); }
private async Task <IEnumerable <ReleaseInfo> > PerformRegularQueryAsync(TorznabQuery query, string hebName = null) { var releases = new List <ReleaseInfo>(); var searchUrl = SearchUrl; var searchString = query.GetQueryString(); if (query.IsImdbQuery) { searchString = query.ImdbID; } if (hebName != null) { searchString = hebName + " - עונה " + query.Season + " פרק " + query.Episode; } searchUrl += "?"; if (!string.IsNullOrWhiteSpace(searchString)) { var strEncoded = WebUtilityHelpers.UrlEncode(searchString, Encoding); searchUrl += "&query=" + strEncoded + "&matchquery=any"; } searchUrl = MapTorznabCapsToTrackers(query).Aggregate(searchUrl, (current, cat) => $"{current}&c[]={cat}"); var data = await RequestWithCookiesAndRetryAsync(searchUrl); try { var parser = new HtmlParser(); var dom = parser.ParseDocument(data.ContentString); var rows = dom.QuerySelectorAll("tr.box_torrent"); foreach (var row in rows) { var release = new ReleaseInfo(); var mainTitleLink = row.QuerySelector("div.main_title > a"); release.Title = mainTitleLink.GetAttribute("longtitle"); if (string.IsNullOrWhiteSpace(release.Title)) { release.Title = mainTitleLink.TextContent; } release.MinimumRatio = 1; release.MinimumSeedTime = 172800; // 48 hours release.Grabs = ParseUtil.CoerceLong(row.QuerySelector("td:nth-child(5)").TextContent.Replace(",", "")); release.Seeders = ParseUtil.CoerceInt(row.QuerySelector("td:nth-child(6)").TextContent.Replace(",", "")); release.Peers = ParseUtil.CoerceInt(row.QuerySelector("td:nth-child(7)").TextContent.Replace(",", "")) + release.Seeders; var fullSize = row.QuerySelector("td:nth-child(4)").TextContent; release.Size = ReleaseInfo.GetBytes(fullSize); release.Comments = new Uri(SiteLink + row.QuerySelector("a.threadlink[href]").GetAttribute("href")); release.Link = new Uri(SiteLink + row.QuerySelector("a:has(div.dlimg)").GetAttribute("href")); release.Guid = release.Comments; //some releases have invalid banner URLs, ignore the banners in this case if (Uri.TryCreate(row.QuerySelector("a[imgsrc]").GetAttribute("imgsrc"), UriKind.Absolute, out var banner)) { release.BannerUrl = banner; } var dateStringAll = row.QuerySelector("div.up_info2").ChildNodes.Last().TextContent; var dateParts = dateStringAll.Split(' '); var dateString = dateParts[dateParts.Length - 2] + " " + dateParts[dateParts.Length - 1]; release.PublishDate = DateTime.ParseExact(dateString, "dd/MM/yy HH:mm", CultureInfo.InvariantCulture); var categoryLink = row.QuerySelector("a[href^=\"/browse.php?cat=\"]").GetAttribute("href"); var catid = ParseUtil.GetArgumentFromQueryString(categoryLink, "cat"); release.Category = MapTrackerCatToNewznab(catid); if (row.QuerySelector("a[href^=\"?freeleech=1\"]") != null) { release.DownloadVolumeFactor = 0; } else { release.DownloadVolumeFactor = 1; } release.UploadVolumeFactor = 1; var subTitle = row.QuerySelector("div.sub_title"); var imdbLink = subTitle.QuerySelector("span.imdb-inline > a"); if (imdbLink != null) { release.Imdb = ParseUtil.GetLongFromString(imdbLink.GetAttribute("href")); } release.Description = subTitle.FirstChild.TextContent; releases.Add(release); } } catch (Exception ex) { OnParseError(data.ContentString, ex); } return(releases); }
protected async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query, String seasonep) { var releases = new List <ReleaseInfo>(); var searchString = query.GetQueryString(); var pairs = new List <KeyValuePair <string, string> >(); if (seasonep != null) { searchString = query.SanitizedSearchTerm; } pairs.Add(new KeyValuePair <string, string>("nyit_sorozat_resz", "true")); pairs.Add(new KeyValuePair <string, string>("miben", "name")); pairs.Add(new KeyValuePair <string, string>("tipus", "kivalasztottak_kozott")); pairs.Add(new KeyValuePair <string, string>("submit.x", "1")); pairs.Add(new KeyValuePair <string, string>("submit.y", "1")); pairs.Add(new KeyValuePair <string, string>("submit", "Ok")); pairs.Add(new KeyValuePair <string, string>("mire", searchString)); var cats = MapTorznabCapsToTrackers(query); if (cats.Count == 0) { cats = GetAllTrackerCategories(); } foreach (var lcat in LanguageCats) { if (!configData.Hungarian.Value) { cats.Remove(lcat + "_hun"); } if (!configData.English.Value) { cats.Remove(lcat); } } foreach (var cat in cats) { pairs.Add(new KeyValuePair <string, string>("kivalasztott_tipus[]", cat)); } var results = await PostDataWithCookiesAndRetry(SearchUrl, pairs); try { CQ dom = results.Content; ReleaseInfo release; var rows = dom[".box_torrent_all"].Find(".box_torrent"); foreach (var row in rows) { CQ qRow = row.Cq(); var key = dom["link[rel=alternate]"].First().Attr("href").Split('=').Last(); release = new ReleaseInfo(); var torrentTxt = qRow.Find(".torrent_txt, .torrent_txt2").Find("a").Get(0); //if (torrentTxt == null) continue; release.Title = torrentTxt.GetAttribute("title"); release.Description = qRow.Find("div.siterank").Text(); release.MinimumRatio = 1; release.MinimumSeedTime = 172800; release.DownloadVolumeFactor = 0; release.UploadVolumeFactor = 1; string downloadLink = SiteLink + torrentTxt.GetAttribute("href"); string downloadId = downloadLink.Substring(downloadLink.IndexOf("&id=") + 4); release.Link = new Uri(SiteLink.ToString() + "torrents.php?action=download&id=" + downloadId + "&key=" + key); release.Comments = new Uri(SiteLink.ToString() + "torrents.php?action=details&id=" + downloadId); release.Guid = new Uri(release.Comments.ToString() + "#comments");; release.Seeders = ParseUtil.CoerceInt(qRow.Find(".box_s2").Find("a").First().Text()); release.Peers = ParseUtil.CoerceInt(qRow.Find(".box_l2").Find("a").First().Text()) + release.Seeders; var imdblink = qRow.Find("a[href*=\".imdb.com/title\"]").Attr("href"); release.Imdb = ParseUtil.GetLongFromString(imdblink); var banner = qRow.Find("img.infobar_ico").Attr("onmouseover"); if (banner != null) { Regex BannerRegEx = new Regex(@"mutat\('(.*?)', '", RegexOptions.Compiled); var BannerMatch = BannerRegEx.Match(banner); var bannerurl = BannerMatch.Groups[1].Value; release.BannerUrl = new Uri(bannerurl); } release.PublishDate = DateTime.Parse(qRow.Find(".box_feltoltve2").Get(0).InnerHTML.Replace("<br />", " "), CultureInfo.InvariantCulture); string[] sizeSplit = qRow.Find(".box_meret2").Get(0).InnerText.Split(' '); release.Size = ReleaseInfo.GetBytes(sizeSplit[1].ToLower(), ParseUtil.CoerceFloat(sizeSplit[0])); string catlink = qRow.Find("a:has(img[class='categ_link'])").First().Attr("href"); string cat = ParseUtil.GetArgumentFromQueryString(catlink, "tipus"); release.Category = MapTrackerCatToNewznab(cat); if (seasonep == null) { releases.Add(release); } else { if (query.MatchQueryStringAND(release.Title, null, seasonep)) { /* For sonnar if the search querry was english the title must be english also so we need to change the Description and Title */ var temp = release.Title; // releasedata everithing after Name.S0Xe0X String releasedata = release.Title.Split(new[] { seasonep }, StringSplitOptions.None)[1].Trim(); /* if the release name not contains the language we add it because it is know from category */ if (cat.Contains("hun") && !releasedata.Contains("hun")) { releasedata += ".hun"; } // release description contains [imdb: ****] but we only need the data before it for title String[] description = { release.Description, "" }; if (release.Description.Contains("[imdb:")) { description = release.Description.Split('['); description[1] = "[" + description[1]; } release.Title = (description[0].Trim() + "." + seasonep.Trim() + "." + releasedata.Trim('.')).Replace(' ', '.'); // if search is done for S0X than we dont want to put . between S0X and E0X Match match = Regex.Match(releasedata, @"^E\d\d?"); if (seasonep.Length == 3 && match.Success) { release.Title = (description[0].Trim() + "." + seasonep.Trim() + releasedata.Trim('.')).Replace(' ', '.'); } // add back imdb points to the description [imdb: 8.7] release.Description = temp + " " + description[1]; release.Description = release.Description.Trim(); releases.Add(release); } } } } catch (Exception ex) { OnParseError(results.Content, ex); } return(releases); }
protected string ApplyFilters(string data, List <FilterBlock> filters, Dictionary <string, object> variables = null) { if (filters == null) { return(data); } foreach (var filter in filters) { switch (filter.Name) { case "querystring": var param = (string)filter.Args; data = ParseUtil.GetArgumentFromQueryString(data, param); break; case "timeparse": case "dateparse": var layout = (string)filter.Args; try { var date = DateTimeUtil.ParseDateTimeGoLang(data, layout); data = date.ToString(DateTimeUtil.Rfc1123ZPattern); } catch (InvalidDateException ex) { _logger.Debug(ex.Message); } break; case "regexp": var pattern = (string)filter.Args; var regexp = new Regex(pattern); var match = regexp.Match(data); data = match.Groups[1].Value; break; case "re_replace": var regexpreplace_pattern = (string)filter.Args[0]; var regexpreplace_replacement = (string)filter.Args[1]; regexpreplace_replacement = ApplyGoTemplateText(regexpreplace_replacement, variables); var regexpreplace_regex = new Regex(regexpreplace_pattern); data = regexpreplace_regex.Replace(data, regexpreplace_replacement); break; case "split": var sep = (string)filter.Args[0]; var pos = (string)filter.Args[1]; var posInt = int.Parse(pos); var strParts = data.Split(sep[0]); if (posInt < 0) { posInt += strParts.Length; } data = strParts[posInt]; break; case "replace": var from = (string)filter.Args[0]; var to = (string)filter.Args[1]; to = ApplyGoTemplateText(to, variables); data = data.Replace(from, to); break; case "trim": var cutset = (string)filter.Args; if (cutset != null) { data = data.Trim(cutset[0]); } else { data = data.Trim(); } break; case "prepend": var prependstr = (string)filter.Args; data = ApplyGoTemplateText(prependstr, variables) + data; break; case "append": var str = (string)filter.Args; data += ApplyGoTemplateText(str, variables); break; case "tolower": data = data.ToLower(); break; case "toupper": data = data.ToUpper(); break; case "urldecode": data = data.UrlDecode(_encoding); break; case "urlencode": data = data.UrlEncode(_encoding); break; case "timeago": case "reltime": data = DateTimeUtil.FromTimeAgo(data).ToString(DateTimeUtil.Rfc1123ZPattern); break; case "fuzzytime": data = DateTimeUtil.FromUnknown(data).ToString(DateTimeUtil.Rfc1123ZPattern); break; case "validfilename": data = StringUtil.MakeValidFileName(data, '_', false); break; case "diacritics": var diacriticsOp = (string)filter.Args; if (diacriticsOp == "replace") { // Should replace diacritics charcaters with their base character // It's not perfect, e.g. "ŠĐĆŽ - šđčćž" becomes "SĐCZ-sđccz" var stFormD = data.Normalize(NormalizationForm.FormD); var len = stFormD.Length; var sb = new StringBuilder(); for (var i = 0; i < len; i++) { var uc = System.Globalization.CharUnicodeInfo.GetUnicodeCategory(stFormD[i]); if (uc != System.Globalization.UnicodeCategory.NonSpacingMark) { sb.Append(stFormD[i]); } } data = sb.ToString().Normalize(NormalizationForm.FormC); } else { throw new Exception("unsupported diacritics filter argument"); } break; case "jsonjoinarray": var jsonjoinarrayJSONPath = (string)filter.Args[0]; var jsonjoinarraySeparator = (string)filter.Args[1]; var jsonjoinarrayO = JObject.Parse(data); var jsonjoinarrayOResult = jsonjoinarrayO.SelectToken(jsonjoinarrayJSONPath); var jsonjoinarrayOResultStrings = jsonjoinarrayOResult.Select(j => j.ToString()); data = string.Join(jsonjoinarraySeparator, jsonjoinarrayOResultStrings); break; case "hexdump": // this is mainly for debugging invisible special char related issues var hexData = string.Join("", data.Select(c => c + "(" + ((int)c).ToString("X2") + ")")); _logger.Debug(string.Format("CardigannIndexer ({0}): strdump: {1}", _definition.Id, hexData)); break; case "strdump": // for debugging var debugData = data.Replace("\r", "\\r").Replace("\n", "\\n").Replace("\xA0", "\\xA0"); var strTag = (string)filter.Args; if (strTag != null) { strTag = string.Format("({0}):", strTag); } else { strTag = ":"; } _logger.Debug(string.Format("CardigannIndexer ({0}): strdump{1} {2}", _definition.Id, strTag, debugData)); break; default: break; } } return(data); }
public IList <ReleaseInfo> ParseResponse(IndexerResponse indexerResponse) { var torrentInfos = new List <ReleaseInfo>(); var rowsSelector = ".torrent_table > tbody > tr"; var searchResultParser = new HtmlParser(); var searchResultDocument = searchResultParser.ParseDocument(indexerResponse.Content); var rows = searchResultDocument.QuerySelectorAll(rowsSelector); var stickyGroup = false; string categoryStr; ICollection <IndexerCategory> groupCategory = null; string groupTitle = null; foreach (var row in rows) { if (row.ClassList.Contains("torrent")) { // garbage rows continue; } else if (row.ClassList.Contains("group")) { stickyGroup = row.ClassList.Contains("sticky"); var dispalyname = row.QuerySelector("#displayname"); var qCat = row.QuerySelector("td.cats_col > div"); categoryStr = qCat.GetAttribute("title"); var qArtistLink = dispalyname.QuerySelector("#groupplatform > a"); if (qArtistLink != null) { categoryStr = ParseUtil.GetArgumentFromQueryString(qArtistLink.GetAttribute("href"), "artistname"); } groupCategory = _categories.MapTrackerCatToNewznab(categoryStr); var qDetailsLink = dispalyname.QuerySelector("#groupname > a"); groupTitle = qDetailsLink.TextContent; } else if (row.ClassList.Contains("group_torrent")) { if (row.QuerySelector("td.edition_info") != null) { continue; } var sizeString = row.QuerySelector("td:nth-child(4)").TextContent; if (string.IsNullOrEmpty(sizeString)) { continue; } var qDetailsLink = row.QuerySelector("a[href^=\"torrents.php?id=\"]"); var title = qDetailsLink.TextContent.Replace(", Freeleech!", "").Replace(", Neutral Leech!", ""); //if (stickyGroup && (query.ImdbID == null || !NewznabStandardCategory.MovieSearchImdbAvailable) && !query.MatchQueryStringAND(title)) // AND match for sticky releases //{ // continue; //} var qDescription = qDetailsLink.QuerySelector("span.torrent_info_tags"); var qDLLink = row.QuerySelector("a[href^=\"torrents.php?action=download\"]"); var qTime = row.QuerySelector("span.time"); var qGrabs = row.QuerySelector("td:nth-child(5)"); var qSeeders = row.QuerySelector("td:nth-child(6)"); var qLeechers = row.QuerySelector("td:nth-child(7)"); var qFreeLeech = row.QuerySelector("strong.freeleech_label"); var qNeutralLeech = row.QuerySelector("strong.neutralleech_label"); var time = qTime.GetAttribute("title"); var link = _settings.BaseUrl + qDLLink.GetAttribute("href"); var seeders = ParseUtil.CoerceInt(qSeeders.TextContent); var publishDate = DateTime.SpecifyKind( DateTime.ParseExact(time, "MMM dd yyyy, HH:mm", CultureInfo.InvariantCulture), DateTimeKind.Unspecified).ToLocalTime(); var details = _settings.BaseUrl + qDetailsLink.GetAttribute("href"); var grabs = ParseUtil.CoerceInt(qGrabs.TextContent); var leechers = ParseUtil.CoerceInt(qLeechers.TextContent); var size = ParseUtil.GetBytes(sizeString); var release = new TorrentInfo { MinimumRatio = 1, MinimumSeedTime = 288000, //80 hours Categories = groupCategory, PublishDate = publishDate, Size = size, InfoUrl = details, DownloadUrl = link, Guid = link, Grabs = grabs, Seeders = seeders, Peers = leechers + seeders, Title = title, Description = qDescription?.TextContent, UploadVolumeFactor = qNeutralLeech is null ? 1 : 0, DownloadVolumeFactor = qFreeLeech != null || qNeutralLeech != null ? 0 : 1 }; torrentInfos.Add(release); } } return(torrentInfos.ToArray()); }