protected abstract Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query);
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { List <ReleaseInfo> releases = new List <ReleaseInfo>(); List <string> searchStrings = new List <string>(new string[] { query.GetQueryString() }); if (string.IsNullOrEmpty(query.Episode) && (query.Season > 0)) { // Tracker naming rules: If query is for a whole season, "Season #" instead of "S##". searchStrings.Add((query.SanitizedSearchTerm + " " + string.Format("\"Season {0}\"", query.Season)).Trim()); } List <string> categories = MapTorznabCapsToTrackers(query); List <string> request_urls = new List <string>(); foreach (var searchString in searchStrings) { var queryCollection = new NameValueCollection(); queryCollection.Add("action", "basic"); if (!string.IsNullOrWhiteSpace(searchString)) { queryCollection.Add("searchstr", searchString); } foreach (var cat in categories) { queryCollection.Add("filter_cat[" + cat + "]", "1"); } request_urls.Add(SearchUrl + queryCollection.GetQueryString()); } IEnumerable <Task <WebClientStringResult> > downloadTasksQuery = from url in request_urls select RequestStringWithCookiesAndRetry(url); WebClientStringResult[] responses = await Task.WhenAll(downloadTasksQuery.ToArray()); for (int i = 0; i < searchStrings.Count(); i++) { var results = responses[i]; // Occasionally the cookies become invalid, login again if that happens if (results.IsRedirect) { await ApplyConfiguration(null); results = await RequestStringWithCookiesAndRetry(request_urls[i]); } try { CQ dom = results.Content; var rows = dom["#torrent_table > tbody > tr.torrent"]; foreach (var row in rows) { CQ qRow = row.Cq(); var release = new ReleaseInfo(); release.MinimumRatio = 1; release.MinimumSeedTime = 172800; var catStr = row.ChildElements.ElementAt(0).FirstElementChild.GetAttribute("href").Split(new char[] { '[', ']' })[1]; release.Category = MapTrackerCatToNewznab(catStr); var qLink = row.ChildElements.ElementAt(1).Cq().Children("a")[0].Cq(); var linkStr = qLink.Attr("href"); release.Comments = new Uri(BaseUrl + "/" + linkStr); release.Guid = release.Comments; var qDownload = row.ChildElements.ElementAt(1).Cq().Find("a[title='Download']")[0].Cq(); release.Link = new Uri(BaseUrl + "/" + qDownload.Attr("href")); var dateStr = row.ChildElements.ElementAt(3).Cq().Text().Trim().Replace(" and", ""); release.PublishDate = DateTimeUtil.FromTimeAgo(dateStr); var sizeStr = row.ChildElements.ElementAt(4).Cq().Text(); release.Size = ReleaseInfo.GetBytes(sizeStr); release.Files = ParseUtil.CoerceInt(row.ChildElements.ElementAt(2).Cq().Text().Trim()); release.Grabs = ParseUtil.CoerceInt(row.ChildElements.ElementAt(6).Cq().Text().Trim()); release.Seeders = ParseUtil.CoerceInt(row.ChildElements.ElementAt(7).Cq().Text().Trim()); release.Peers = ParseUtil.CoerceInt(row.ChildElements.ElementAt(8).Cq().Text().Trim()) + release.Seeders; var grabs = qRow.Find("td:nth-child(6)").Text(); release.Grabs = ParseUtil.CoerceInt(grabs); if (qRow.Find("strong:contains(\"Freeleech!\")").Length >= 1) { release.DownloadVolumeFactor = 0; } else { release.DownloadVolumeFactor = 1; } release.UploadVolumeFactor = 1; var title = qRow.Find("td:nth-child(2)"); title.Find("span, strong, div, br").Remove(); release.Title = ParseUtil.NormalizeMultiSpaces(title.Text().Replace(" - ]", "]")); if (catStr == "10") //change "Season #" to "S##" for TV shows { release.Title = Regex.Replace(release.Title, @"Season (\d+)", m => string.Format("S{0:00}", Int32.Parse(m.Groups[1].Value))); } releases.Add(release); } } catch (Exception ex) { OnParseError(results.Content, ex); } } return(releases); }
private List <ReleaseInfo> ParseResponse(TorznabQuery query, string htmlResponse) { var releases = new List <ReleaseInfo>(); try { var parser = new HtmlParser(); var dom = parser.ParseDocument(htmlResponse); var table = dom.QuerySelector("table.movehere"); if (table == null) { return(releases); // no results } var headerColumns = table.QuerySelectorAll("tbody > tr > td.cat_Head") .Select(x => x.TextContent).ToList(); var categoryIndex = headerColumns.FindIndex(x => x.Equals("Type")); var nameIndex = headerColumns.FindIndex(x => x.Equals("Name")); var sizeIndex = headerColumns.FindIndex(x => x.Equals("Size")); var seedersIndex = headerColumns.FindIndex(x => x.Equals("Seeders")); var leechersIndex = headerColumns.FindIndex(x => x.Equals("Leechers")); var rows = dom.QuerySelectorAll("tr.browse"); foreach (var row in rows) { // TODO convert to initializer var release = new ReleaseInfo(); release.MinimumRatio = 1; release.MinimumSeedTime = 259200; // 72 hours var catId = "82"; // default var qCatLink = row.Children[categoryIndex].QuerySelector("a"); if (qCatLink != null) { catId = new Regex(@"\?cat=(\d*)").Match(qCatLink.GetAttribute("href")).Groups[1].ToString().Trim(); } release.Category = MapTrackerCatToNewznab(catId); var qDescCol = row.Children[nameIndex]; var qLink = qDescCol.QuerySelector("a"); release.Title = qLink.TextContent; if (!query.MatchQueryStringAND(release.Title)) { continue; } release.Details = new Uri(SiteLink + "/" + qLink.GetAttribute("href")); release.Guid = release.Details; var torrentId = qLink.GetAttribute("href").Split('=')[1]; release.Link = new Uri(string.Format(DownloadUrl, torrentId)); release.PublishDate = DateTimeUtil.FromTimeAgo(qDescCol.ChildNodes.Last().TextContent); var sizeStr = row.Children[sizeIndex].TextContent; release.Size = ReleaseInfo.GetBytes(sizeStr); release.Seeders = ParseUtil.CoerceInt(row.Children[seedersIndex].TextContent.Trim()); release.Peers = ParseUtil.CoerceInt(row.Children[leechersIndex].TextContent.Trim()) + release.Seeders; release.DownloadVolumeFactor = row.QuerySelector("font > b:contains(Freeleech)") != null ? 0 : 1; release.UploadVolumeFactor = 1; releases.Add(release); } } catch (Exception ex) { OnParseError(htmlResponse, ex); } return(releases); }
public async Task<ReleaseInfo[]> PerformQuery(TorznabQuery query) { List<ReleaseInfo> releases = new List<ReleaseInfo>(); foreach (var title in query.ShowTitles ?? new string[] { "%20" }) { var searchString = title + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = string.Format(SearchUrl, HttpUtility.UrlEncode(searchString)); var message = new HttpRequestMessage(); message.Method = HttpMethod.Get; message.RequestUri = new Uri(episodeSearchUrl); message.Headers.TryAddWithoutValidation("Authorization", await GetAuthToken()); var response = await client.SendAsync(message); var results = await response.Content.ReadAsStringAsync(); var jsonResult = JObject.Parse(results); try { var items = (JArray)jsonResult["torrents"]; foreach (var item in items) { var release = new ReleaseInfo(); release.MinimumRatio = 1; release.MinimumSeedTime = 172800; var torrentId = (string)item["id"]; release.Link = new Uri(string.Format(DownloadUrl, torrentId)); release.Title = (string)item["name"]; release.Description = release.Title; release.Comments = new Uri(string.Format(CommentsUrl, (string)item["rewritename"])); release.Guid = release.Comments; var dateUtc = DateTime.ParseExact((string)item["added"], "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture); release.PublishDate = DateTime.SpecifyKind(dateUtc, DateTimeKind.Utc).ToLocalTime(); release.Seeders = ParseUtil.CoerceInt((string)item["seeders"]); release.Peers = ParseUtil.CoerceInt((string)item["leechers"]) + release.Seeders; release.Size = ParseUtil.CoerceLong((string)item["size"]); releases.Add(release); } } catch (Exception ex) { OnResultParsingError(this, results, ex); throw ex; } } return releases.ToArray(); }
public async Task<ReleaseInfo[]> PerformQuery(TorznabQuery query) { List<ReleaseInfo> releases = new List<ReleaseInfo>(); foreach (var title in query.ShowTitles ?? new string[] { string.Empty }) { var searchString = title + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = SearchUrl + HttpUtility.UrlEncode(searchString); string results; if (Program.IsWindows) { var request = CreateHttpRequest(new Uri(episodeSearchUrl)); request.Method = HttpMethod.Get; var response = await client.SendAsync(request); results = await response.Content.ReadAsStringAsync(); } else { var response = await CurlHelper.GetAsync(episodeSearchUrl, cookieHeader); results = Encoding.UTF8.GetString(response.Content); } try { var json = JObject.Parse(results); foreach (JObject r in json["response"]["results"]) { DateTime pubDate = DateTime.MinValue; double dateNum; if (double.TryParse((string)r["groupTime"], out dateNum)) pubDate = UnixTimestampToDateTime(dateNum); var groupName = (string)r["groupName"]; if (r["torrents"] is JArray) { foreach (JObject t in r["torrents"]) { var release = new ReleaseInfo(); release.PublishDate = pubDate; release.Title = groupName; release.Description = groupName; FillReleaseInfoFromJson(release, t); releases.Add(release); } } else { var release = new ReleaseInfo(); release.PublishDate = pubDate; release.Title = groupName; release.Description = groupName; FillReleaseInfoFromJson(release, r); releases.Add(release); } } } catch (Exception ex) { OnResultParsingError(this, results, ex); throw ex; } } return releases.ToArray(); }
public async Task<ReleaseInfo[]> PerformQuery(TorznabQuery query) { List<ReleaseInfo> releases = new List<ReleaseInfo>(); foreach (var title in query.ShowTitles ?? new string[] { string.Empty }) { var searchString = title + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = string.Format(SearchUrl, HttpUtility.UrlEncode(searchString)); var results = await client.GetStringAsync(episodeSearchUrl); try { CQ dom = results; var rows = dom["table.torrenttable > tbody > tr.browse_color"]; foreach (var row in rows) { var release = new ReleaseInfo(); release.MinimumRatio = 1; release.MinimumSeedTime = 172800; var qRow = row.Cq(); var qLink = row.ChildElements.ElementAt(2).FirstChild.Cq(); release.Link = new Uri(BaseUrl + "/" + qLink.Attr("href")); var torrentID = qLink.Attr("href").Split('=').Last(); var descCol = row.ChildElements.ElementAt(3); var qCommentLink = descCol.FirstChild.Cq(); release.Title = qCommentLink.Text(); release.Description = release.Title; release.Comments = new Uri(BaseUrl + "/" + qCommentLink.Attr("href")); release.Guid = release.Comments; var dateStr = descCol.ChildElements.Last().Cq().Text().Split('|').Last().ToLowerInvariant().Replace("ago.", "").Trim(); var dateParts = dateStr.Split(new char[] { ' ', ' ' }, StringSplitOptions.RemoveEmptyEntries); var timeSpan = TimeSpan.Zero; for (var i = 0; i < dateParts.Length / 2; i++) { var timeVal = ParseUtil.CoerceInt(dateParts[i * 2]); var timeUnit = dateParts[i * 2 + 1]; if (timeUnit.Contains("year")) timeSpan += TimeSpan.FromDays(365 * timeVal); else if (timeUnit.Contains("month")) timeSpan += TimeSpan.FromDays(30 * timeVal); else if (timeUnit.Contains("day")) timeSpan += TimeSpan.FromDays(timeVal); else if (timeUnit.Contains("hour")) timeSpan += TimeSpan.FromHours(timeVal); else if (timeUnit.Contains("min")) timeSpan += TimeSpan.FromMinutes(timeVal); } release.PublishDate = DateTime.SpecifyKind(DateTime.Now - timeSpan, DateTimeKind.Local); var sizeEl = row.ChildElements.ElementAt(7); var sizeVal = ParseUtil.CoerceFloat(sizeEl.ChildNodes.First().NodeValue); var sizeUnit = sizeEl.ChildNodes.Last().NodeValue; release.Size = ReleaseInfo.GetBytes(sizeUnit, sizeVal); release.Seeders = ParseUtil.CoerceInt(row.ChildElements.ElementAt(9).Cq().Text()); release.Peers = ParseUtil.CoerceInt(row.ChildElements.ElementAt(10).Cq().Text()) + release.Seeders; releases.Add(release); } } catch (Exception ex) { OnResultParsingError(this, results, ex); throw ex; } } return releases.ToArray(); }
async Task<ReleaseInfo[]> PerformQuery(TorznabQuery query, string baseUrl) { List<ReleaseInfo> releases = new List<ReleaseInfo>(); List<string> searchUrls = new List<string>(); foreach (var title in query.ShowTitles ?? new string[] { string.Empty }) { var searchString = title + " " + query.GetEpisodeSearchString(); var queryStr = HttpUtility.UrlEncode(searchString); var episodeSearchUrl = baseUrl + string.Format(SearchUrl, queryStr); var episodeSearchUrl2 = baseUrl + string.Format(SearchUrl2, queryStr); searchUrls.Add(episodeSearchUrl); searchUrls.Add(episodeSearchUrl2); } foreach (var episodeSearchUrl in searchUrls) { var message = new HttpRequestMessage { Method = HttpMethod.Get, RequestUri = new Uri(baseUrl + SwitchSingleViewUrl) }; message.Headers.Referrer = new Uri(episodeSearchUrl); string results; if (Program.IsWindows) { var response = await client.SendAsync(message); results = await response.Content.ReadAsStringAsync(); } else { var response = await CurlHelper.GetAsync(baseUrl + SwitchSingleViewUrl, null, episodeSearchUrl); results = Encoding.UTF8.GetString(response.Content); } try { CQ dom = results; var rows = dom["#searchResult > tbody > tr"]; foreach (var row in rows) { var release = new ReleaseInfo(); CQ qLink = row.ChildElements.ElementAt(1).Cq().Children("a").First(); release.MinimumRatio = 1; release.MinimumSeedTime = 172800; release.Title = qLink.Text().Trim(); release.Description = release.Title; release.Comments = new Uri(baseUrl + "/" + qLink.Attr("href").TrimStart('/')); release.Guid = release.Comments; var timeString = row.ChildElements.ElementAt(2).Cq().Text(); if (timeString.Contains("mins ago")) release.PublishDate = (DateTime.Now - TimeSpan.FromMinutes(int.Parse(timeString.Split(' ')[0]))); else if (timeString.Contains("Today")) release.PublishDate = (DateTime.UtcNow - TimeSpan.FromHours(2) - TimeSpan.Parse(timeString.Split(' ')[1])).ToLocalTime(); else if (timeString.Contains("Y-day")) release.PublishDate = (DateTime.UtcNow - TimeSpan.FromHours(26) - TimeSpan.Parse(timeString.Split(' ')[1])).ToLocalTime(); else if (timeString.Contains(':')) { var utc = DateTime.ParseExact(timeString, "MM-dd HH:mm", CultureInfo.InvariantCulture) - TimeSpan.FromHours(2); release.PublishDate = DateTime.SpecifyKind(utc, DateTimeKind.Utc).ToLocalTime(); } else { var utc = DateTime.ParseExact(timeString, "MM-dd yyyy", CultureInfo.InvariantCulture) - TimeSpan.FromHours(2); release.PublishDate = DateTime.SpecifyKind(utc, DateTimeKind.Utc).ToLocalTime(); } var downloadCol = row.ChildElements.ElementAt(3).Cq().Find("a"); release.MagnetUri = new Uri(downloadCol.Attr("href")); release.InfoHash = release.MagnetUri.ToString().Split(':')[3].Split('&')[0]; var sizeString = row.ChildElements.ElementAt(4).Cq().Text().Split(' '); var sizeVal = float.Parse(sizeString[0], CultureInfo.InvariantCulture); var sizeUnit = sizeString[1]; release.Size = ReleaseInfo.GetBytes(sizeUnit, sizeVal); release.Seeders = int.Parse(row.ChildElements.ElementAt(5).Cq().Text()); release.Peers = int.Parse(row.ChildElements.ElementAt(6).Cq().Text()) + release.Seeders; releases.Add(release); } } catch (Exception ex) { OnResultParsingError(this, results, ex); throw ex; } } return releases.ToArray(); }
private async Task <IEnumerable <ReleaseInfo> > PerformQuery(Uri siteLink, TorznabQuery query, int attempts) { var releases = new List <ReleaseInfo>(); _includeVo = ((BoolItem)configData.GetDynamic("IncludeVo")).Value; _filterMovies = ((BoolItem)configData.GetDynamic("FilterMovies")).Value; _dailyNow = DateTime.Now; _dailyResultIdx = 0; bool rssMode = string.IsNullOrEmpty(query.SanitizedSearchTerm); if (rssMode) { int pg = 1; while (pg <= _maxDailyPages) { Uri url = new Uri(siteLink, string.Format(_dailyUrl, pg)); var results = await RequestStringWithCookiesAndRetry(url.AbsoluteUri); await FollowIfRedirect(results); var items = ParseDailyContent(results.Content); if (items == null || !items.Any()) { break; } releases.AddRange(items); //Check if we need to go to next page bool recentFound = _mostRecentRelease != null && items.Any(r => r.Title == _mostRecentRelease.Title && r.Link.AbsoluteUri == _mostRecentRelease.Link.AbsoluteUri); if (pg == 1) { _mostRecentRelease = (NewpctRelease)items.First().Clone(); } if (recentFound) { break; } pg++; } } else { bool isTvSearch = query.Categories == null || query.Categories.Length == 0 || query.Categories.Any(c => _allTvCategories.Contains(c)); if (isTvSearch) { releases.AddRange(await TvSearch(siteLink, query)); } bool isMovieSearch = query.Categories == null || query.Categories.Length == 0 || query.Categories.Any(c => _allMoviesCategories.Contains(c)); if (isMovieSearch) { releases.AddRange(await MovieSearch(siteLink, query)); } } return(releases); }
private async Task <IEnumerable <ReleaseInfo> > TvSearch(Uri siteLink, TorznabQuery query) { List <ReleaseInfo> newpctReleases = null; string seriesName = query.SanitizedSearchTerm; int? season = query.Season > 0 ? (int?)query.Season : null; int? episode = null; if (!string.IsNullOrWhiteSpace(query.Episode) && int.TryParse(query.Episode, out int episodeTemp)) { episode = episodeTemp; } //If query has no season/episode info, try to parse title if (season == null && episode == null) { Match searchMatch = _searchStringRegex.Match(query.SanitizedSearchTerm); if (searchMatch.Success) { seriesName = searchMatch.Groups[1].Value.Trim(); season = int.Parse(searchMatch.Groups[2].Value); episode = searchMatch.Groups[4].Success ? (int?)int.Parse(searchMatch.Groups[4].Value) : null; } } //Try to reuse cache lock (cache) { var cachedResult = cache.FirstOrDefault(i => i.Query == seriesName.ToLower()); if (cachedResult != null) { newpctReleases = cachedResult.Results.Select(r => (ReleaseInfo)r.Clone()).ToList(); } } if (newpctReleases == null) { newpctReleases = new List <ReleaseInfo>(); //Search series url foreach (Uri seriesListUrl in SeriesListUris(siteLink, seriesName)) { newpctReleases.AddRange(await GetReleasesFromUri(seriesListUrl, seriesName)); } //Sonarr removes "the" from shows. If there is nothing try prepending "the" if (newpctReleases.Count == 0 && !(seriesName.ToLower().StartsWith("the"))) { seriesName = "The " + seriesName; foreach (Uri seriesListUrl in SeriesListUris(siteLink, seriesName)) { newpctReleases.AddRange(await GetReleasesFromUri(seriesListUrl, seriesName)); } } //Cache ALL episodes lock (cache) { cache.Add(new CachedQueryResult(seriesName.ToLower(), newpctReleases)); } } //Filter only episodes needed return(newpctReleases.Where(r => { NewpctRelease nr = r as NewpctRelease; return ( nr.Season.HasValue != season.HasValue || //Can't determine if same season nr.Season.HasValue && season.Value == nr.Season.Value && //Same season and ... ( nr.Episode.HasValue != episode.HasValue || //Can't determine if same episode nr.Episode.HasValue && ( nr.Episode.Value == episode.Value || //Same episode nr.EpisodeTo.HasValue && episode.Value >= nr.Episode.Value && episode.Value <= nr.EpisodeTo.Value //Episode in interval ) ) ); })); }
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { var releases = new List <ReleaseInfo>(); var searchString = query.GetQueryString(); var queryCollection = new List <KeyValuePair <string, string> >(); int page = 0; if (!string.IsNullOrWhiteSpace(searchString)) { queryCollection.Add("term", searchString); } else { // no term execute latest search var result = await SendAPIRequestLatest(); try { // this time is a jarray JArray json = (JArray)result; foreach (var torrent in json) { // add release releases.Add(makeRelease(torrent)); } } catch (Exception ex) { OnParseError(result.ToString(), ex); } return(releases); } var cats = MapTorznabCapsToTrackers(query); if (cats.Count > 0) { queryCollection.Add("category", string.Join(",", cats)); } else { queryCollection.Add("category", "0"); // set ALL category } // lazy horrible page initialization queryCollection.Add("page", page.ToString()); do { // update page number queryCollection.RemoveAt(queryCollection.Count - 1); // remove last elem: page number queryCollection.Add("page", (++page).ToString()); var result = await SendAPIRequest(queryCollection); try { // this time is a jobject JObject json = (JObject)result; if (json["results"] == null) { throw new Exception("Error invalid JSON response"); } // check number result if (((JArray)json["results"]).Count() == 0) { break; } foreach (var torrent in json["results"]) { // add release releases.Add(makeRelease(torrent)); } } catch (Exception ex) { OnParseError(result.ToString(), ex); } } while (page < MAX_SEARCH_PAGE_LIMIT); return(releases); }
public async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { List <ReleaseInfo> releases = new List <ReleaseInfo>(); var searchString = query.GetQueryString(); var searchUrl = SearchUrl; var queryCollection = new NameValueCollection(); queryCollection.Add("searchin", "title"); queryCollection.Add("incldead", "1"); if (!string.IsNullOrWhiteSpace(searchString)) { // use AND+wildcard operator to avoid getting to many useless results var searchStringArray = Regex.Split(searchString.Trim(), "[ _.-]+", RegexOptions.Compiled).ToList(); searchStringArray = searchStringArray.Where(x => x.Length >= 3).ToList(); // remove words with less than 3 characters searchStringArray = searchStringArray.Select(x => "+" + x).ToList(); // add AND operators+wildcards var searchStringFinal = String.Join("", searchStringArray); queryCollection.Add("search", searchStringFinal); } foreach (var cat in MapTorznabCapsToTrackers(query)) { queryCollection.Add("c" + cat, "1"); } searchUrl += "?" + queryCollection.GetQueryString(); var results = await RequestStringWithCookiesAndRetry(searchUrl); try { CQ dom = results.Content; var rows = dom["table.tt > tbody > tr"]; foreach (var row in rows.Skip(1)) { var release = new ReleaseInfo(); release.MinimumRatio = 1; release.MinimumSeedTime = 72 * 60 * 60; var qRow = row.Cq(); var qDetailsLink = qRow.Find("a[href^=details.php?id=]").First(); release.Title = qDetailsLink.Text().Trim(); // HoT search returns should support AND search but it simply doesn't work, so we AND filter it manualy if (!query.MatchQueryStringAND(release.Title)) { continue; } var qCatLink = qRow.Find("a[href^=browse.php?cat=]").First(); var qSeeders = qRow.Find("td:eq(8)"); var qLeechers = qRow.Find("td:eq(9)"); var qDownloadLink = qRow.Find("a[href^=download.php]").First(); var qTimeAgo = qRow.Find("td:eq(5)"); var qSize = qRow.Find("td:eq(6)"); var catStr = qCatLink.Attr("href").Split('=')[1]; release.Category = MapTrackerCatToNewznab(catStr); release.Link = new Uri(SiteLink + qDownloadLink.Attr("href")); release.Comments = new Uri(SiteLink + qDetailsLink.Attr("href")); release.Guid = release.Link; var sizeStr = qSize.Text(); release.Size = ReleaseInfo.GetBytes(sizeStr); release.Seeders = ParseUtil.CoerceInt(qSeeders.Text()); release.Peers = ParseUtil.CoerceInt(qLeechers.Text()) + release.Seeders; var dateStr = qTimeAgo.Text(); DateTime pubDateUtc; var Timeparts = dateStr.Split(new char[] { ' ' }, 2)[1]; if (dateStr.StartsWith("Today ")) { pubDateUtc = DateTime.SpecifyKind(DateTime.UtcNow.Date, DateTimeKind.Unspecified) + DateTime.ParseExact(dateStr.Split(new char[] { ' ' }, 2)[1], "hh:mm tt", System.Globalization.CultureInfo.InvariantCulture).TimeOfDay; } else if (dateStr.StartsWith("Yesterday ")) { pubDateUtc = DateTime.SpecifyKind(DateTime.UtcNow.Date, DateTimeKind.Unspecified) + DateTime.ParseExact(dateStr.Split(new char[] { ' ' }, 2)[1], "hh:mm tt", System.Globalization.CultureInfo.InvariantCulture).TimeOfDay - TimeSpan.FromDays(1); } else { pubDateUtc = DateTime.SpecifyKind(DateTime.ParseExact(dateStr, "MMM d yyyy hh:mm tt", CultureInfo.InvariantCulture), DateTimeKind.Unspecified); } release.PublishDate = pubDateUtc.ToLocalTime(); releases.Add(release); } } catch (Exception ex) { OnParseError(results.Content, ex); } return(releases); }
protected async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query, string seasonep) { var releases = new List <ReleaseInfo>(); var searchString = query.GetQueryString(); var pairs = new List <KeyValuePair <string, string> >(); if (seasonep != null) { searchString = query.SanitizedSearchTerm; } pairs.Add(new KeyValuePair <string, string>("nyit_sorozat_resz", "true")); pairs.Add(new KeyValuePair <string, string>("miben", "name")); pairs.Add(new KeyValuePair <string, string>("tipus", "kivalasztottak_kozott")); pairs.Add(new KeyValuePair <string, string>("submit.x", "1")); pairs.Add(new KeyValuePair <string, string>("submit.y", "1")); pairs.Add(new KeyValuePair <string, string>("submit", "Ok")); pairs.Add(new KeyValuePair <string, string>("mire", searchString)); var cats = MapTorznabCapsToTrackers(query); if (cats.Count == 0) { cats = GetAllTrackerCategories(); } foreach (var lcat in LanguageCats) { if (!configData.Hungarian.Value) { cats.Remove(lcat + "_hun"); } if (!configData.English.Value) { cats.Remove(lcat); } } foreach (var cat in cats) { pairs.Add(new KeyValuePair <string, string>("kivalasztott_tipus[]", cat)); } var results = await PostDataWithCookiesAndRetry(SearchUrl, pairs); CQ dom = results.Content; var numVal = 0; // find number of torrents / page var torrent_per_page = dom[".box_torrent_all"].Find(".box_torrent").Length; if (torrent_per_page == 0) { return(releases); } var start_page = (query.Offset / torrent_per_page) + 1; var previously_parsed_on_page = query.Offset - (start_page * torrent_per_page) + 1; //+1 because indexing start from 0 if (previously_parsed_on_page < 0) { previously_parsed_on_page = query.Offset; } // find pagelinks in the bottom var pagelinks = dom["div[id=pager_bottom]"].Find("a"); if (pagelinks.Length > 0) { // If there are several pages find the link for the latest one for (var i = pagelinks.Length - 1; i > 0; i--) { var last_page_link = (pagelinks[i].Cq().Attr("href")).Trim(); if (last_page_link.Contains("oldal")) { var match = Regex.Match(last_page_link, @"(?<=[\?,&]oldal=)(\d+)(?=&)"); numVal = int.Parse(match.Value); break; } } } var limit = query.Limit; if (limit == 0) { limit = 100; } if (start_page == 1) { releases = parseTorrents(results, seasonep, query, releases.Count, limit, previously_parsed_on_page); previously_parsed_on_page = 0; start_page++; } // Check all the pages for the torrents. // The starting index is 2. (the first one is the original where we parse out the pages.) for (var i = start_page; (i <= numVal && releases.Count < limit); i++) { pairs.Add(new KeyValuePair <string, string>("oldal", i.ToString())); results = await PostDataWithCookiesAndRetry(SearchUrl, pairs); releases.AddRange(parseTorrents(results, seasonep, query, releases.Count, limit, previously_parsed_on_page)); previously_parsed_on_page = 0; pairs.Remove(new KeyValuePair <string, string>("oldal", i.ToString())); } return(releases); }
public async Task<ReleaseInfo[]> PerformQuery(TorznabQuery query) { List<ReleaseInfo> releases = new List<ReleaseInfo>(); foreach (var title in query.ShowTitles ?? new string[] { string.Empty }) { var searchString = title + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = SearchUrl + HttpUtility.UrlEncode(searchString); var results = await client.GetStringAsync(episodeSearchUrl); try { CQ dom = results; dom["#needseed"].Remove(); var rows = dom["table[width='750'] > tbody"].Children(); foreach (var row in rows.Skip(1)) { var release = new ReleaseInfo(); var qRow = row.Cq(); var qLink = qRow.Children().ElementAt(2).Cq().Children("a").First(); release.MinimumRatio = 1; release.MinimumSeedTime = 172800; release.Title = qLink.Attr("title"); release.Description = release.Title; release.Guid = new Uri(BaseUrl + qLink.Attr("href")); release.Comments = release.Guid; release.Link = new Uri(string.Format(DownloadUrl, qLink.Attr("href").Split('=')[1])); var dateString = qRow.Children().ElementAt(5).Cq().Text().Trim(); var pubDate = DateTime.ParseExact(dateString, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture); release.PublishDate = DateTime.SpecifyKind(pubDate, DateTimeKind.Local); var sizeCol = qRow.Children().ElementAt(6); var sizeVal = sizeCol.ChildNodes[0].NodeValue; var sizeUnit = sizeCol.ChildNodes[2].NodeValue; release.Size = ReleaseInfo.GetBytes(sizeUnit, ParseUtil.CoerceFloat(sizeVal)); release.Seeders = ParseUtil.CoerceInt(qRow.Children().ElementAt(8).Cq().Text().Trim()); release.Peers = ParseUtil.CoerceInt(qRow.Children().ElementAt(9).Cq().Text().Trim()) + release.Seeders; releases.Add(release); } } catch (Exception ex) { OnResultParsingError(this, results, ex); throw ex; } } return releases.ToArray(); }
public async Task<ReleaseInfo[]> PerformQuery(TorznabQuery query) { List<ReleaseInfo> releases = new List<ReleaseInfo>(); foreach (var title in query.ShowTitles ?? new string[] { string.Empty }) { var searchString = title + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = string.Format(SearchUrl, HttpUtility.UrlEncode(searchString)); var results = await client.GetStringAsync(episodeSearchUrl); try { CQ dom = results; var rows = dom["#torrent_table > tbody > tr.torrent"]; foreach (var row in rows) { CQ qRow = row.Cq(); var release = new ReleaseInfo(); release.MinimumRatio = 1; release.MinimumSeedTime = 172800; release.Title = qRow.Find(".torrent_name_link").Text(); release.Description = release.Title; release.Guid = new Uri(BaseUrl + "/" + qRow.Find(".torrent_name_link").Parent().Attr("href")); release.Comments = release.Guid; release.Link = new Uri(BaseUrl + "/" + qRow.Find(".torrent_handle_links > a").First().Attr("href")); var dateStr = qRow.Find(".time").Text().Trim(); if (dateStr.ToLower().Contains("just now")) release.PublishDate = DateTime.Now; else { var dateParts = dateStr.Split(' '); var dateValue = int.Parse(dateParts[0]); TimeSpan ts = TimeSpan.Zero; if (dateStr.Contains("sec")) ts = TimeSpan.FromSeconds(dateValue); else if (dateStr.Contains("min")) ts = TimeSpan.FromMinutes(dateValue); else if (dateStr.Contains("hour")) ts = TimeSpan.FromHours(dateValue); else if (dateStr.Contains("day")) ts = TimeSpan.FromDays(dateValue); else if (dateStr.Contains("week")) ts = TimeSpan.FromDays(dateValue * 7); else if (dateStr.Contains("month")) ts = TimeSpan.FromDays(dateValue * 30); else if (dateStr.Contains("year")) ts = TimeSpan.FromDays(dateValue * 365); release.PublishDate = DateTime.Now - ts; } var sizeStr = qRow.Find(".size")[0].ChildNodes[0].NodeValue.Trim(); var sizeParts = sizeStr.Split(' '); release.Size = ReleaseInfo.GetBytes(sizeParts[1], float.Parse(sizeParts[0], NumberStyles.AllowThousands)); release.Seeders = int.Parse(qRow.Children().ElementAt(6).InnerText.Trim(), NumberStyles.AllowThousands); release.Peers = int.Parse(qRow.Children().ElementAt(7).InnerText.Trim(), NumberStyles.AllowThousands) + release.Seeders; releases.Add(release); } } catch (Exception ex) { OnResultParsingError(this, results, ex); throw ex; } } return releases.ToArray(); }
private async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query, int attempts) { var releases = new List <ReleaseInfo>(); bool rssMode = string.IsNullOrEmpty(query.SanitizedSearchTerm); Uri siteLinkUri = new Uri(configData.SiteLink.Value); if (rssMode) { int pg = 1; while (pg <= _maxDailyPages) { Uri url = new Uri(siteLinkUri, string.Format(_dailyUrl, pg)); var results = await RequestStringWithCookies(url.AbsoluteUri); var items = ParseDailyContent(results.Content); if (items == null || !items.Any()) { break; } releases.AddRange(items); //Check if we need to go to next page bool recentFound = _mostRecentRelease != null && items.Any(r => r.Title == _mostRecentRelease.Title && r.Link.AbsoluteUri == _mostRecentRelease.Link.AbsoluteUri); if (pg == 1) { _mostRecentRelease = (ReleaseInfo)items.First().Clone(); } if (recentFound) { break; } pg++; } } else { //Only tv search supported. (newpct web search is useless) bool isTvSearch = query.Categories == null || query.Categories.Length == 0 || query.Categories.Any(c => _allTvCategories.Contains(c)); if (isTvSearch) { var newpctReleases = new List <ReleaseInfo>(); string seriesName = query.SanitizedSearchTerm; int? season = query.Season > 0 ? (int?)query.Season : null; int? episode = null; if (!string.IsNullOrWhiteSpace(query.Episode) && int.TryParse(query.Episode, out int episodeTemp)) { episode = episodeTemp; } //If query has no season/episode info, try to parse title if (season == null && episode == null) { Match searchMatch = _searchStringRegex.Match(query.SanitizedSearchTerm); if (searchMatch.Success) { seriesName = searchMatch.Groups[1].Value.Trim(); season = int.Parse(searchMatch.Groups[2].Value); episode = searchMatch.Groups[4].Success ? (int?)int.Parse(searchMatch.Groups[4].Value) : null; } } //Try to reuse cache bool cacheFound = false; lock (cache) { CleanCache(); var cachedResult = cache.FirstOrDefault(i => i.Query == seriesName.ToLower()); if (cachedResult != null && cachedResult.Results != null) { cacheFound = true; newpctReleases = cachedResult.Results.Where(r => (r as NewpctRelease) != null).ToList(); if (!newpctReleases.Any() && cachedResult.Results.Any()) { cacheFound = false; } } } if (!cacheFound) { IEnumerable <string> lettersUrl; if (!((BoolItem)configData.GetDynamic("IncludeVo")).Value) { lettersUrl = _seriesLetterUrls; } else { lettersUrl = _seriesLetterUrls.Concat(_seriesVOLetterUrls); } string seriesLetter = !char.IsDigit(seriesName[0]) ? seriesName[0].ToString() : "0-9"; //Search series url foreach (string urlFormat in lettersUrl) { Uri seriesListUrl = new Uri(siteLinkUri, string.Format(urlFormat, seriesLetter.ToLower())); var results = await RequestStringWithCookies(seriesListUrl.AbsoluteUri); //Episodes list string seriesEpisodesUrl = ParseSeriesListContent(results.Content, seriesName); if (!string.IsNullOrEmpty(seriesEpisodesUrl)) { int pg = 1; while (pg < _maxEpisodesListPages) { Uri episodesListUrl = new Uri(string.Format(_seriesUrl, seriesEpisodesUrl, pg)); results = await RequestStringWithCookies(episodesListUrl.AbsoluteUri); var items = ParseEpisodesListContent(results.Content); if (items == null || !items.Any()) { break; } newpctReleases.AddRange(items); pg++; } } } //Cache ALL episodes lock (cache) { cache.Add(new CachedQueryResult(seriesName.ToLower(), newpctReleases)); } } //Filter only episodes needed releases.AddRange(newpctReleases.Where(r => { NewpctRelease nr = r as NewpctRelease; return(nr.Season.HasValue != season.HasValue || //Can't determine if same season nr.Season.HasValue && season.Value == nr.Season.Value && //Same season and ... ( nr.Episode.HasValue != episode.HasValue || //Can't determine if same episode nr.Episode.HasValue && ( nr.Episode.Value == episode.Value || //Same episode nr.EpisodeTo.HasValue && episode.Value >= nr.Episode.Value && episode.Value <= nr.EpisodeTo.Value //Episode in interval ) )); })); } } return(releases); }
private async Task <IEnumerable <ReleaseInfo> > GetResults(TorznabQuery query, string searchType, string searchTerm) { var releases = new List <ReleaseInfo>(); var queryCollection = new NameValueCollection { { "username", ConfigData.Username.Value }, { "torrent_pass", ConfigData.Passkey.Value }, { "type", searchType }, { "searchstr", searchTerm } }; var queryCats = MapTorznabCapsToTrackers(query); if (queryCats.Count > 0) { foreach (var cat in queryCats) { queryCollection.Add(cat, "1"); } } var queryUrl = ScrapeUrl + "?" + queryCollection.GetQueryString(); // Check cache first so we don't query the server for each episode when searching for each episode in a series. lock (cache) { // Remove old cache items CleanCache(); var cachedResult = cache.Where(i => i.Query == queryUrl).FirstOrDefault(); if (cachedResult != null) { return(cachedResult.Results.Select(s => (ReleaseInfo)s.Clone()).ToArray()); } } // Get the content from the tracker var response = await RequestWithCookiesAndRetryAsync(queryUrl); if (!response.ContentString.StartsWith("{")) // not JSON => error { throw new ExceptionWithConfigData("Unexpected response (not JSON)", ConfigData); } var json = JsonConvert.DeserializeObject <dynamic>(response.ContentString); // Parse try { if (json["error"] != null) { throw new Exception(json["error"].ToString()); } var matches = (long)json["Matches"]; if (matches > 0) { var groups = (JArray)json.Groups; foreach (var group in groups) { var synonyms = new List <string>(); var posterStr = (string)group["Image"]; var poster = (string.IsNullOrWhiteSpace(posterStr) ? null : new Uri(posterStr)); var year = (int)group["Year"]; var groupName = (string)group["GroupName"]; var seriesName = (string)group["SeriesName"]; var mainTitle = WebUtility.HtmlDecode((string)group["FullName"]); if (seriesName != null) { mainTitle = seriesName; } synonyms.Add(mainTitle); if (group["Synonymns"].HasValues) { if (group["Synonymns"] is JArray) { var allSyonyms = group["Synonymns"].ToObject <List <string> >(); if (AddJapaneseTitle && allSyonyms.Count >= 1) { synonyms.Add(allSyonyms[0]); } if (AddRomajiTitle && allSyonyms.Count >= 2) { synonyms.Add(allSyonyms[1]); } if (AddAlternativeTitles && allSyonyms.Count >= 3) { synonyms.AddRange(allSyonyms[2].Split(',').Select(t => t.Trim())); } } else { var allSynonyms = group["Synonymns"].ToObject <Dictionary <int, string> >(); if (AddJapaneseTitle && allSynonyms.ContainsKey(0)) { synonyms.Add(allSynonyms[0]); } if (AddRomajiTitle && allSynonyms.ContainsKey(1)) { synonyms.Add(allSynonyms[1]); } if (AddAlternativeTitles && allSynonyms.ContainsKey(2)) { synonyms.AddRange(allSynonyms[2].Split(',').Select(t => t.Trim())); } } } List <int> category = null; var categoryName = (string)group["CategoryName"]; var description = (string)group["Description"]; foreach (var torrent in group["Torrents"]) { var releaseInfo = "S01"; string episode = null; int? season = null; var editionTitle = (string)torrent["EditionData"]["EditionTitle"]; if (!string.IsNullOrWhiteSpace(editionTitle)) { releaseInfo = WebUtility.HtmlDecode(editionTitle); } var seasonRegEx = new Regex(@"Season (\d+)", RegexOptions.Compiled); var seasonRegExMatch = seasonRegEx.Match(releaseInfo); if (seasonRegExMatch.Success) { season = ParseUtil.CoerceInt(seasonRegExMatch.Groups[1].Value); } var episodeRegEx = new Regex(@"Episode (\d+)", RegexOptions.Compiled); var episodeRegExMatch = episodeRegEx.Match(releaseInfo); if (episodeRegExMatch.Success) { episode = episodeRegExMatch.Groups[1].Value; } releaseInfo = releaseInfo.Replace("Episode ", ""); releaseInfo = releaseInfo.Replace("Season ", "S"); releaseInfo = releaseInfo.Trim(); if (PadEpisode && int.TryParse(releaseInfo, out _) && releaseInfo.Length == 1) { releaseInfo = "0" + releaseInfo; } if (FilterSeasonEpisode) { if (query.Season != 0 && season != null && season != query.Season) // skip if season doesn't match { continue; } if (query.Episode != null && episode != null && episode != query.Episode) // skip if episode doesn't match { continue; } } var torrentId = (long)torrent["ID"]; var property = ((string)torrent["Property"]).Replace(" | Freeleech", ""); var link = (string)torrent["Link"]; var linkUri = new Uri(link); var uploadTimeString = (string)torrent["UploadTime"]; var uploadTime = DateTime.ParseExact(uploadTimeString, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture); var publishDate = DateTime.SpecifyKind(uploadTime, DateTimeKind.Utc).ToLocalTime(); var details = new Uri(SiteLink + "torrent/" + torrentId + "/group"); var size = (long)torrent["Size"]; var snatched = (long)torrent["Snatched"]; var seeders = (int)torrent["Seeders"]; var leechers = (int)torrent["Leechers"]; var fileCount = (long)torrent["FileCount"]; var peers = seeders + leechers; var rawDownMultiplier = (int?)torrent["RawDownMultiplier"] ?? 0; var rawUpMultiplier = (int?)torrent["RawUpMultiplier"] ?? 0; if (searchType == "anime") { if (groupName == "TV Series" || groupName == "OVA") { category = new List <int> { TorznabCatType.TVAnime.ID } } ; // Ignore these categories as they'll cause hell with the matcher // TV Special, OVA, ONA, DVD Special, BD Special if (groupName == "Movie" || groupName == "Live Action Movie") { category = new List <int> { TorznabCatType.Movies.ID } } ; if (categoryName == "Manga" || categoryName == "Oneshot" || categoryName == "Anthology" || categoryName == "Manhwa" || categoryName == "Manhua" || categoryName == "Light Novel") { category = new List <int> { TorznabCatType.BooksComics.ID } } ; if (categoryName == "Novel" || categoryName == "Artbook") { category = new List <int> { TorznabCatType.BooksComics.ID } } ; if (categoryName == "Game" || categoryName == "Visual Novel") { if (property.Contains(" PSP ")) { category = new List <int> { TorznabCatType.ConsolePSP.ID } } ; if (property.Contains("PSX")) { category = new List <int> { TorznabCatType.ConsoleOther.ID } } ; if (property.Contains(" NES ")) { category = new List <int> { TorznabCatType.ConsoleOther.ID } } ; if (property.Contains(" PC ")) { category = new List <int> { TorznabCatType.PCGames.ID } } ; } } else if (searchType == "music") { if (categoryName == "Single" || categoryName == "EP" || categoryName == "Album" || categoryName == "Compilation" || categoryName == "Soundtrack" || categoryName == "Remix CD" || categoryName == "PV" || categoryName == "Live Album" || categoryName == "Image CD" || categoryName == "Drama CD" || categoryName == "Vocal CD") { if (property.Contains(" Lossless ")) { category = new List <int> { TorznabCatType.AudioLossless.ID } } ; else if (property.Contains("MP3")) { category = new List <int> { TorznabCatType.AudioMP3.ID } } ; else { category = new List <int> { TorznabCatType.AudioOther.ID } }; } } // We don't actually have a release name >.> so try to create one var releaseTags = property.Split("|".ToCharArray(), StringSplitOptions.RemoveEmptyEntries).ToList(); for (var i = releaseTags.Count - 1; i >= 0; i--) { releaseTags[i] = releaseTags[i].Trim(); if (string.IsNullOrWhiteSpace(releaseTags[i])) { releaseTags.RemoveAt(i); } } var releaseGroup = releaseTags.LastOrDefault(); if (releaseGroup != null && releaseGroup.Contains("(") && releaseGroup.Contains(")")) { // Skip raws if set if (releaseGroup.ToLowerInvariant().StartsWith("raw") && !AllowRaws) { continue; } var start = releaseGroup.IndexOf("(", StringComparison.Ordinal); releaseGroup = "[" + releaseGroup.Substring(start + 1, (releaseGroup.IndexOf(")", StringComparison.Ordinal) - 1) - start) + "] "; } else { releaseGroup = string.Empty; } if (!AllowRaws && releaseTags.Contains("raw", StringComparer.InvariantCultureIgnoreCase)) { continue; } var infoString = releaseTags.Aggregate("", (prev, cur) => prev + "[" + cur + "]"); var minimumSeedTime = 259200; // Additional 5 hours per GB minimumSeedTime += (int)((size / 1000000000) * 18000); foreach (var title in synonyms) { var releaseTitle = groupName == "Movie" ? $"{title} {year} {releaseGroup}{infoString}" : $"{releaseGroup}{title} {releaseInfo} {infoString}"; var guid = new Uri(details + "&nh=" + StringUtil.Hash(title)); var release = new ReleaseInfo { MinimumRatio = 1, MinimumSeedTime = minimumSeedTime, Title = releaseTitle, Details = details, Guid = guid, Link = linkUri, Poster = poster, PublishDate = publishDate, Category = category, Description = description, Size = size, Seeders = seeders, Peers = peers, Grabs = snatched, Files = fileCount, DownloadVolumeFactor = rawDownMultiplier, UploadVolumeFactor = rawUpMultiplier }; releases.Add(release); } } } } } catch (Exception ex) { OnParseError(response.ContentString, ex); } // Add to the cache lock (cache) { cache.Add(new CachedQueryResult(queryUrl, releases)); } return(releases.Select(s => (ReleaseInfo)s.Clone())); }
public async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { var releases = new List <ReleaseInfo>(); var searchurls = new List <string>(); var searchUrl = SearchUrl;// string.Format(SearchUrl, HttpUtility.UrlEncode())); var queryCollection = new NameValueCollection(); var searchString = query.GetQueryString(); foreach (var cat in MapTorznabCapsToTrackers(query)) { searchUrl += "category%5B%5D=" + cat + "&"; } if (!string.IsNullOrWhiteSpace(searchString)) { queryCollection.Add("search", searchString); } queryCollection.Add("active", "0"); queryCollection.Add("options", "0"); searchUrl += queryCollection.GetQueryString().Replace("(", "%28").Replace(")", "%29"); // maually url encode brackets to prevent "hacking" detection var results = await RequestStringWithCookiesAndRetry(searchUrl); try { CQ dom = results.Content; ReleaseInfo release; var rows = dom[".mainblockcontenttt > tbody > tr:has(a[href^=\"details.php?id=\"])"]; foreach (var row in rows) { CQ qRow = row.Cq(); release = new ReleaseInfo(); release.Title = qRow.Find("td.mainblockcontent b a").Text(); release.Description = qRow.Find("td:nth-child(3) > span").Text(); if (0 != qRow.Find("td.mainblockcontent u").Length) { var imdbStr = qRow.Find("td.mainblockcontent u").Parent().First().Attr("href").Replace("http://www.imdb.com/title/tt", "").Replace("/", ""); long imdb; if (ParseUtil.TryCoerceLong(imdbStr, out imdb)) { release.Imdb = imdb; } } release.MinimumRatio = 1; release.MinimumSeedTime = 172800; int seeders, peers; if (ParseUtil.TryCoerceInt(qRow.Find("td").Get(9).FirstChild.FirstChild.InnerText, out seeders)) { release.Seeders = seeders; if (ParseUtil.TryCoerceInt(qRow.Find("td").Get(10).FirstChild.FirstChild.InnerText, out peers)) { release.Peers = peers + release.Seeders; } } release.Grabs = ParseUtil.CoerceLong(qRow.Find("td:nth-child(12)").Text()); string fullSize = qRow.Find("td.mainblockcontent").Get(6).InnerText; release.Size = ReleaseInfo.GetBytes(fullSize); release.Guid = new Uri(SiteLink + qRow.Find("td.mainblockcontent b a").Attr("href")); release.Link = new Uri(SiteLink + qRow.Find("td.mainblockcontent").Get(3).FirstChild.GetAttribute("href")); release.Comments = new Uri(SiteLink + qRow.Find("td.mainblockcontent b a").Attr("href")); string[] dateSplit = qRow.Find("td.mainblockcontent").Get(5).InnerHTML.Split(','); string dateString = dateSplit[1].Substring(0, dateSplit[1].IndexOf('>')); release.PublishDate = DateTime.Parse(dateString, CultureInfo.InvariantCulture); string category = qRow.Find("td:eq(0) a").Attr("href").Replace("torrents.php?category=", ""); release.Category = MapTrackerCatToNewznab(category); release.UploadVolumeFactor = 1; if (qRow.Find("img[alt=\"Free Torrent\"]").Length >= 1) { release.DownloadVolumeFactor = 0; release.UploadVolumeFactor = 0; } else if (qRow.Find("img[alt=\"Silver Torrent\"]").Length >= 1) { release.DownloadVolumeFactor = 0.5; } else if (qRow.Find("img[alt=\"Bronze Torrent\"]").Length >= 1) { release.DownloadVolumeFactor = 0.75; } else if (qRow.Find("img[alt=\"Blue Torrent\"]").Length >= 1) { release.DownloadVolumeFactor = 0.25; } else { release.DownloadVolumeFactor = 1; } releases.Add(release); } } catch (Exception ex) { OnParseError(results.Content, ex); } return(releases); }
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { var releases = new List <ReleaseInfo>(); var searchUrl = BrowseUrl; var searchString = query.GetQueryString(); var cats = MapTorznabCapsToTrackers(query); string cat = "0"; if (cats.Count == 1) { cat = cats[0]; } var queryCollection = new NameValueCollection(); if (query.ImdbID != null) { queryCollection.Add("search", query.ImdbID); } else if (!string.IsNullOrWhiteSpace(searchString)) { queryCollection.Add("search", searchString); } queryCollection.Add("cat", cat); queryCollection.Add("searchin", "0"); queryCollection.Add("sort", "0"); searchUrl += "?" + queryCollection.GetQueryString(); var response = await RequestStringWithCookiesAndRetry(searchUrl, null, BrowseUrl); // Occasionally the cookies become invalid, login again if that happens if (response.IsRedirect) { await ApplyConfiguration(null); response = await RequestStringWithCookiesAndRetry(searchUrl, null, BrowseUrl); } var results = response.Content; try { CQ dom = results; var globalFreeLeech = dom.Find("div.globalFreeLeech").Any(); var rows = dom[".torrentrow"]; foreach (var row in rows) { var release = new ReleaseInfo(); var qRow = row.Cq(); var qTitleLink = qRow.Find(".torrenttable:eq(1) a").First(); release.Title = qRow.Find(".torrenttable:eq(1) b").Text(); if (query.ImdbID == null && !query.MatchQueryStringAND(release.Title)) { continue; } release.Description = qRow.Find(".torrenttable:eq(1) > span > font.small").First().Text(); var tooltip = qTitleLink.Attr("title"); if (!string.IsNullOrEmpty(tooltip)) { var ImgRegexp = new Regex("src='(.*?)'"); var ImgRegexpMatch = ImgRegexp.Match(tooltip); if (ImgRegexpMatch.Success) { release.BannerUrl = new Uri(ImgRegexpMatch.Groups[1].Value); } } release.Guid = new Uri(SiteLink + qTitleLink.Attr("href")); release.Comments = release.Guid; //22:05:3716/02/2013 var dateStr = qRow.Find(".torrenttable:eq(5)").Text().Trim() + " +0200"; release.PublishDate = DateTime.ParseExact(dateStr, "H:mm:ssdd/MM/yyyy zzz", CultureInfo.InvariantCulture); var qLink = qRow.Find("a[href^=\"download.php?id=\"]").First(); release.Link = new Uri(SiteLink + qLink.Attr("href")); var sizeStr = qRow.Find(".torrenttable:eq(6)").Text().Trim(); release.Size = ReleaseInfo.GetBytes(sizeStr); release.Seeders = ParseUtil.CoerceInt(qRow.Find(".torrenttable:eq(8)").Text().Trim()); release.Peers = ParseUtil.CoerceInt(qRow.Find(".torrenttable:eq(9)").Text().Trim()) + release.Seeders; var catId = qRow.Find(".torrenttable:eq(0) a").First().Attr("href").Substring(15); release.Category = MapTrackerCatToNewznab(catId); var grabs = qRow.Find(".torrenttable:eq(7)").First().Get(0).FirstChild; release.Grabs = ParseUtil.CoerceLong(catId); if (globalFreeLeech || row.Cq().Find("img[alt=\"FreeLeech\"]").Any()) { release.DownloadVolumeFactor = 0; } else { release.DownloadVolumeFactor = 1; } release.UploadVolumeFactor = 1; // Skip Romanian releases if (release.Category.Contains(TorznabCatType.MoviesForeign.ID) && !configData.IncludeRomanianReleases.Value) { continue; } releases.Add(release); } } catch (Exception ex) { OnParseError(results, ex); } return(releases); }
async Task<ReleaseInfo[]> PerformQuery(TorznabQuery query, string baseUrl) { List<ReleaseInfo> releases = new List<ReleaseInfo>(); List<string> searchurls = new List<string>(); foreach (var title in query.ShowTitles ?? new string[] { string.Empty }) { var searchString = title + " " + query.GetEpisodeSearchString(); for (int page = 0; page < MAXPAGES; page++) searchurls.Add(string.Format(SearchUrl, HttpUtility.UrlEncode(searchString.Trim()), page)); } foreach (string SearchUrl in searchurls) { var results = await client.GetStringAsync(SearchUrl); try { CQ dom = results; ReleaseInfo release; int rowCount = 0; var rows = dom[".mainblockcontenttt > tbody > tr"]; foreach (var row in rows) { CQ qRow = row.Cq(); if (rowCount < 2 || qRow.Children().Count() != 12) //skip 2 rows because there's an empty row & a title/sort row { rowCount++; continue; } release = new ReleaseInfo(); long? size; release.Title = qRow.Find("td.mainblockcontent b a").Text(); release.Description = release.Title; if (0 != qRow.Find("td.mainblockcontent u").Length) { var imdbStr = qRow.Find("td.mainblockcontent u").Parent().First().Attr("href").Replace("http://www.imdb.com/title/tt", "").Replace("/", ""); long imdb; if (ParseUtil.TryCoerceLong(imdbStr, out imdb)) { release.Imdb = imdb; } } release.MinimumRatio = 1; release.MinimumSeedTime = 172800; int seeders, peers; if (ParseUtil.TryCoerceInt(qRow.Find("td").Get(9).FirstChild.FirstChild.InnerText, out seeders)) { release.Seeders = seeders; if (ParseUtil.TryCoerceInt(qRow.Find("td").Get(10).FirstChild.FirstChild.InnerText, out peers)) { release.Peers = peers + release.Seeders; } } string fullSize = qRow.Find("td.mainblockcontent").Get(6).InnerText; string[] sizeSplit = fullSize.Split(' '); switch (sizeSplit[1].ToLower()) { case "kb": size = ReleaseInfo.BytesFromKB(ParseUtil.CoerceFloat(sizeSplit[0])); break; case "mb": size = ReleaseInfo.BytesFromMB(ParseUtil.CoerceFloat(sizeSplit[0])); break; case "gb": size = ReleaseInfo.BytesFromGB(ParseUtil.CoerceFloat(sizeSplit[0])); break; default: size = null; break; } release.Size = size; release.Guid = new Uri(DefaultUrl + "/" + qRow.Find("td.mainblockcontent b a").Attr("href")); release.Link = new Uri(DefaultUrl + "/" + qRow.Find("td.mainblockcontent").Get(3).FirstChild.GetAttribute("href")); release.Comments = new Uri(DefaultUrl + "/" + qRow.Find("td.mainblockcontent b a").Attr("href") + "#comments"); string[] dateSplit = qRow.Find("td.mainblockcontent").Get(5).InnerHTML.Split(','); string dateString = dateSplit[1].Substring(0, dateSplit[1].IndexOf('>')); release.PublishDate = DateTime.Parse(dateString, CultureInfo.InvariantCulture); releases.Add(release); } } catch (Exception ex) { OnResultParsingError(this, results, ex); throw ex; } } return releases.ToArray(); }
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { var releases = new List <ReleaseInfo>(); // remove operator characters / all words must be present (+ prefix) var cleanSearchString = Regex.Replace(query.GetQueryString().Trim(), "[ _.+-]+", " ", RegexOptions.Compiled); var finalSearchString = cleanSearchString.Split(' ') .Aggregate("", (current, word) => current + $"+{word} ") .Trim(); var searchUrl = SearchUrl; var queryCollection = new NameValueCollection { { "searchin", "title" }, { "incldead", "1" }, { "sort", "4" }, { "type", "desc" } }; if (!string.IsNullOrWhiteSpace(finalSearchString)) { queryCollection.Add("search", finalSearchString); } foreach (var cat in MapTorznabCapsToTrackers(query)) { queryCollection.Add("c" + cat, "1"); } searchUrl += "?" + queryCollection.GetQueryString(); var response = await RequestStringWithCookiesAndRetry(searchUrl); var results = response.Content; if (!results.Contains("/logout.php?")) { await ApplyConfiguration(null); response = await RequestStringWithCookiesAndRetry(searchUrl); results = response.Content; } try { var parser = new HtmlParser(); var dom = parser.ParseDocument(results); var rows = dom.QuerySelectorAll("table.table-bordered > tbody > tr.browse_color"); foreach (var row in rows) { var release = new ReleaseInfo(); release.MinimumRatio = 1; release.MinimumSeedTime = 72 * 60 * 60; var qCatLink = row.QuerySelector("a[href^=\"browse.php?cat=\"]"); var catStr = qCatLink.GetAttribute("href").Split('=')[1]; release.Category = MapTrackerCatToNewznab(catStr); var qDetailsLink = row.QuerySelector("a[href^=\"details.php?id=\"]"); var qDetailsTitle = row.QuerySelector("td:has(a[href^=\"details.php?id=\"]) b"); release.Title = qDetailsTitle.TextContent.Trim(); var qDlLink = row.QuerySelector("a[href^=\"download.php?torrent=\"]"); release.Link = new Uri(SiteLink + qDlLink.GetAttribute("href")); release.Comments = new Uri(SiteLink + qDetailsLink.GetAttribute("href")); release.Guid = release.Comments; var qColumns = row.QuerySelectorAll("td"); release.Files = ParseUtil.CoerceInt(qColumns[4].TextContent); release.PublishDate = DateTimeUtil.FromUnknown(qColumns[5].TextContent); release.Size = ReleaseInfo.GetBytes(qColumns[6].TextContent); release.Grabs = ParseUtil.CoerceInt(qColumns[7].TextContent.Replace("Times", "")); release.Seeders = ParseUtil.CoerceInt(qColumns[8].TextContent); release.Peers = ParseUtil.CoerceInt(qColumns[9].TextContent) + release.Seeders; var qImdb = row.QuerySelector("a[href*=\"www.imdb.com\"]"); if (qImdb != null) { var deRefUrl = qImdb.GetAttribute("href"); release.Imdb = ParseUtil.GetImdbID(WebUtility.UrlDecode(deRefUrl).Split('/').Last()); } release.DownloadVolumeFactor = row.QuerySelector("img[src*=\"freedownload\"]") != null ? 0 : 1; release.UploadVolumeFactor = 1; releases.Add(release); } } catch (Exception ex) { OnParseError(results, ex); } return(releases); }
public async Task<ReleaseInfo[]> PerformQuery(TorznabQuery query) { List<ReleaseInfo> releases = new List<ReleaseInfo>(); foreach (var title in query.ShowTitles ?? new string[] { string.Empty }) { var searchString = title + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = string.Format(SearchUrl, HttpUtility.UrlEncode(searchString)); var message = new HttpRequestMessage(); message.Method = HttpMethod.Get; message.RequestUri = new Uri(episodeSearchUrl); var response = await client.SendAsync(message); var results = await response.Content.ReadAsStringAsync(); try { CQ dom = results; var rows = dom[".results_index ul"]; foreach (var row in rows) { var release = new ReleaseInfo(); CQ qRow = row.Cq(); CQ qLink = qRow.Find("li.torrents_name > .torrents_name_link").First(); CQ qDlLink = qRow.Find("li.torrents_download > a").First(); release.MinimumRatio = 1; release.MinimumSeedTime = 172800; release.Title = qLink.Text().Trim(); release.Description = release.Title; release.Comments = new Uri(BaseUrl + "/" + qLink.Attr("href").TrimStart('/')); release.Guid = release.Comments; release.Link = new Uri(BaseUrl + "/" + qDlLink.Attr("href").TrimStart('/')); release.PublishDate = DateTime.Now; release.Seeders = ParseUtil.CoerceInt(qRow.Find("li.torrents_seeders").Text()); release.Peers = ParseUtil.CoerceInt(qRow.Find("li.torrents_leechers").Text()) + release.Seeders; var sizeParts = qRow.Find("li.torrents_size").Text().Split(' '); var sizeVal = ParseUtil.CoerceFloat(sizeParts[0]); var sizeUnit = sizeParts[1]; release.Size = ReleaseInfo.GetBytes(sizeUnit, sizeVal); releases.Add(release); } } catch (Exception ex) { OnResultParsingError(this, results, ex); throw ex; } } return releases.ToArray(); }
private async Task <IEnumerable <ReleaseInfo> > performRegularQuery(TorznabQuery query, string hebName = null) { var releases = new List <ReleaseInfo>(); var searchurls = new List <string>(); var searchUrl = SearchUrl; var queryCollection = new NameValueCollection(); var searchString = query.GetQueryString(); if (hebName != null) { searchString = hebName + " - עונה " + query.Season + " פרק " + query.Episode; } int categoryCounter = 1; foreach (var cat in MapTorznabCapsToTrackers(query)) { searchUrl += "c" + categoryCounter.ToString() + "=" + cat + "&"; categoryCounter++; } if (string.IsNullOrWhiteSpace(searchString)) { searchUrl = SiteLink + "index.php?name=torrents"; } else { var strEncoded = HttpUtility.UrlEncode(searchString, Encoding.GetEncoding("Windows-1255")); searchUrl += "text=" + strEncoded + "&category=0&search=1"; } var data = await RequestStringWithCookiesAndRetry(searchUrl); try { CQ dom = data.Content; ReleaseInfo release; int rowCount = 0; var rows = dom["#collapseobj_module_17 > tr"]; foreach (var row in rows) { CQ qRow = row.Cq(); if (rowCount < 1 || qRow.Children().Count() != 9) //skip 1 row because there's an empty row { rowCount++; continue; } release = new ReleaseInfo(); release.Description = qRow.Find("td:nth-child(2) > a").Text();; if (hebName != null) { release.Title = query.SearchTerm + " " + release.Description.Substring(release.Description.IndexOf(string.Format("S{0:D2}E{1:D2}", query.Season, int.Parse(query.Episode)))); } else { const string DELIMITER = " | "; release.Title = release.Description.Substring(release.Description.IndexOf(DELIMITER) + DELIMITER.Length); } release.MinimumRatio = 1; release.MinimumSeedTime = 172800; int seeders, peers; if (ParseUtil.TryCoerceInt(qRow.Find("td:nth-child(7) > div").Text(), out seeders)) { release.Seeders = seeders; if (ParseUtil.TryCoerceInt(qRow.Find("td:nth-child(8) > div").Text(), out peers)) { release.Peers = peers + release.Seeders; } } string fullSize = qRow.Find("td:nth-child(5) > div").Text(); release.Size = ReleaseInfo.GetBytes(fullSize); release.Guid = new Uri(qRow.Find("td:nth-child(2) > a").Attr("href")); release.Link = new Uri(SiteLink + qRow.Find("td:nth-child(3) > a").Attr("href")); release.Comments = release.Guid; string[] dateSplit = qRow.Find("td:nth-child(2) > span.torrentstime").Text().Split(' '); string dateString = dateSplit[1] + " " + dateSplit[3]; release.PublishDate = DateTime.ParseExact(dateString, "dd-MM-yy HH:mm", CultureInfo.InvariantCulture); string category = qRow.Find("script:nth-child(1)").Text(); int index = category.IndexOf("category="); if (index == -1) { /// Other type category = "17"; } else { category = category.Substring(index + "category=".Length, 2); if (category[1] == '\\') { category = category[0].ToString(); } } release.Category = MapTrackerCatToNewznab(category); var grabs = qRow.Find("td:nth-child(6)").Text(); release.Grabs = ParseUtil.CoerceInt(grabs); if (qRow.Find("img[src=\"/images/FL.png\"]").Length >= 1) { release.DownloadVolumeFactor = 0; } else { release.DownloadVolumeFactor = 1; } release.UploadVolumeFactor = 1; releases.Add(release); } } catch (Exception ex) { OnParseError(data.Content, ex); } return(releases); }
public async Task<ReleaseInfo[]> PerformQuery(TorznabQuery query) { return await PerformQuery(query, BaseUrl); }
private async Task <IEnumerable <MTReleaseInfo> > FetchMoviesBasedOnLongestWord(TorznabQuery query) { var originalSearch = query.SearchTerm; var regexStr = ".*" + originalSearch.Replace(" ", ".*") + ".*"; var regex = new Regex(regexStr, RegexOptions.IgnoreCase); query.SearchTerm = LongestWord(query); var movies = await FetchMovies(query); return(movies.Where(m => regex.Match(m.Title).Success)); }
public async Task<ReleaseInfo[]> PerformQuery(TorznabQuery query) { // The result list var releases = new ConcurrentBag<ReleaseInfo>(); var titles = query.ShowTitles ?? new string[] { query.SearchTerm??string.Empty }; var tasks = titles.Select(async item => { foreach (var result in await GetResults(item)) releases.Add(result); }); await Task.WhenAll(tasks); return releases.ToArray(); }
private List <ReleaseInfo> ParseReleases(WebResult response, TorznabQuery query) { var releases = new List <ReleaseInfo>(); try { var parser = new HtmlParser(); var dom = parser.ParseDocument(response.ContentString); var rows = dom.QuerySelectorAll("div.home_post_cont"); foreach (var row in rows) { var qImg = row.QuerySelector("img"); if (qImg == null) { continue; // skip results without image } var title = qImg.GetAttribute("title"); if (!CheckTitleMatchWords(query.GetQueryString(), title)) { continue; // skip if it doesn't contain all words } title += " MULTi LATiN SPANiSH 1080p BDRip x264"; var poster = new Uri(GetAbsoluteUrl(qImg.GetAttribute("src"))); var extract = row.QuerySelector("noscript").InnerHtml.Split('\''); Uri link = null; foreach (var part in extract) { if (part.StartsWith(SiteLink) && part.EndsWith("/")) { link = new Uri(GetAbsoluteUrl(part)); break; } } var release = new ReleaseInfo { Title = title, Link = link, Details = link, Guid = link, Category = new List <int> { TorznabCatType.MoviesHD.ID }, Poster = poster, Size = 2147483648, // 2 GB Files = 1, Seeders = 1, Peers = 2, DownloadVolumeFactor = 0, UploadVolumeFactor = 1 }; releases.Add(release); } } catch (Exception ex) { OnParseError(response.ContentString, ex); } return(releases); }
/// <summary> /// Execute our search query /// </summary> /// <param name="query">Query</param> /// <returns>Releases</returns> protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { var releases = new List <ReleaseInfo>(); var searchTerm = query.GetEpisodeSearchString() + " " + query.SanitizedSearchTerm; // use episode search string first, see issue #1202 searchTerm = searchTerm.Trim(); searchTerm = searchTerm.ToLower(); if (EnhancedAnime && query.HasSpecifiedCategories && (query.Categories.Contains(TorznabCatType.TVAnime.ID) || query.Categories.Contains(100032) || query.Categories.Contains(100101) || query.Categories.Contains(100110))) { var regex = new Regex(" ([0-9]+)"); searchTerm = regex.Replace(searchTerm, " E$1"); } // Check cache first so we don't query the server (if search term used or not in dev mode) if (!DevMode && !string.IsNullOrEmpty(searchTerm)) { lock (cache) { // Remove old cache items CleanCache(); // Search in cache var cachedResult = cache.FirstOrDefault(i => i.Query == searchTerm); if (cachedResult != null) { return(cachedResult.Results.Select(s => (ReleaseInfo)s.Clone()).ToArray()); } } } // Build our query var request = BuildQuery(searchTerm, query, ApiEndpoint); // Getting results & Store content var results = await QueryExec(request); try { // Deserialize our Json Response var xthorResponse = JsonConvert.DeserializeObject <XthorResponse>(results); // Check Tracker's State CheckApiState(xthorResponse.error); // If contains torrents if (xthorResponse.torrents != null) { // Adding each torrent row to releases // Exclude hidden torrents (category 106, example => search 'yoda' in the API) #10407 releases.AddRange(xthorResponse.torrents .Where(torrent => torrent.category != 106).Select(torrent => { //issue #3847 replace multi keyword if (!string.IsNullOrEmpty(ReplaceMulti)) { var regex = new Regex("(?i)([\\.\\- ])MULTI([\\.\\- ])"); torrent.name = regex.Replace(torrent.name, "$1" + ReplaceMulti + "$2"); } // issue #8759 replace vostfr and subfrench with English if (ConfigData.Vostfr.Value) { torrent.name = torrent.name.Replace("VOSTFR", "ENGLISH").Replace("SUBFRENCH", "ENGLISH"); } var publishDate = DateTimeUtil.UnixTimestampToDateTime(torrent.added); //TODO replace with download link? var guid = new Uri(TorrentDetailsUrl.Replace("{id}", torrent.id.ToString())); var details = new Uri(TorrentDetailsUrl.Replace("{id}", torrent.id.ToString())); var link = new Uri(torrent.download_link); var release = new ReleaseInfo { // Mapping data Category = MapTrackerCatToNewznab(torrent.category.ToString()), Title = torrent.name, Seeders = torrent.seeders, Peers = torrent.seeders + torrent.leechers, MinimumRatio = 1, MinimumSeedTime = 345600, PublishDate = publishDate, Size = torrent.size, Grabs = torrent.times_completed, Files = torrent.numfiles, UploadVolumeFactor = 1, DownloadVolumeFactor = (torrent.freeleech == 1 ? 0 : 1), Guid = guid, Details = details, Link = link, TMDb = torrent.tmdb_id }; //TODO make consistent with other trackers if (DevMode) { Output(release.ToString()); } return(release); })); } } catch (Exception ex) { OnParseError("Unable to parse result \n" + ex.StackTrace, ex); } // Return found releases return(releases); }
public async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { var releases = new List <ReleaseInfo>(); var searchString = query.GetQueryString(); var searchUrl = SearchUrl; // If query is empty, use the RSS Feed if (string.IsNullOrWhiteSpace(searchString)) { var rssPage = await RequestStringWithCookiesAndRetry(RSSUrl + configData.RSSKey.Value); var rssDoc = XDocument.Parse(rssPage.Content); foreach (var item in rssDoc.Descendants("item")) { var title = item.Descendants("title").First().Value; var description = item.Descendants("description").First().Value; var link = item.Descendants("link").First().Value; var date = item.Descendants("pubDate").First().Value; var torrentIdMatch = Regex.Match(link, "(?<=download\\.php/)([a-zA-z0-9]*)"); var torrentId = torrentIdMatch.Success ? torrentIdMatch.Value : string.Empty; if (string.IsNullOrWhiteSpace(torrentId)) { throw new Exception("Missing torrent id"); } var infoMatch = Regex.Match(description, @"Category:\W(?<cat>.*)\W\n\WSize:\W(?<size>.*)\n\WStatus:\W(?<seeders>.*)\Wseeder(.*)\Wand\W(?<leechers>.*)\Wleecher(.*)\n\WAdded:\W(?<added>.*)\n\WDescription:"); if (!infoMatch.Success) { throw new Exception("Unable to find info"); } var imdbMatch = Regex.Match(description, "(?<=http://www.imdb.com/title/tt)([0-9]*)"); long?imdbID = null; if (imdbMatch.Success) { long l; if (long.TryParse(imdbMatch.Value, out l)) { imdbID = l; } } var release = new ReleaseInfo() { Title = title, Description = title, Guid = new Uri(string.Format(DetailsURL, torrentId)), Comments = new Uri(string.Format(DetailsURL, torrentId) + "&tocomm=1"), PublishDate = DateTime.ParseExact(infoMatch.Groups["added"].Value, "yyyy-MM-dd H:mm:ss", CultureInfo.InvariantCulture), //2015-08-08 21:20:31 Link = new Uri(link), Seeders = ParseUtil.CoerceInt(infoMatch.Groups["seeders"].Value == "no" ? "0" : infoMatch.Groups["seeders"].Value), Peers = ParseUtil.CoerceInt(infoMatch.Groups["leechers"].Value == "no" ? "0" : infoMatch.Groups["leechers"].Value), Size = ReleaseInfo.GetBytes(infoMatch.Groups["size"].Value), Category = MapTrackerCatToNewznab(infoMatch.Groups["cat"].Value), Imdb = imdbID }; // if unknown category, set to "other" if (release.Category == 0) { release.Category = 7000; } release.Peers += release.Seeders; releases.Add(release); } } else { searchUrl += "?titleonly=1&search=" + HttpUtility.UrlEncode(searchString); string.Format(SearchUrl, HttpUtility.UrlEncode(searchString)); var cats = MapTorznabCapsToTrackers(query); if (cats.Count > 0) { foreach (var cat in cats) { searchUrl += "&c" + cat + "=1"; } } var results = await RequestStringWithCookiesAndRetry(searchUrl); try { CQ dom = results.Content; // table header is the first <tr> in table body, get all rows except this CQ qRows = dom["#torrents-table > tbody > tr:not(:first-child)"]; foreach (var row in qRows) { var release = new ReleaseInfo(); var qRow = row.Cq(); var debug = qRow.Html(); release.MinimumRatio = 1; release.MinimumSeedTime = 172800; CQ qLink = qRow.Find(".br_right > a").First(); release.Guid = new Uri(SiteLink + qLink.Attr("href")); release.Comments = new Uri(SiteLink + qLink.Attr("href") + "&tocomm=1"); release.Title = qLink.Find("b").Text(); release.Description = release.Title; release.Link = new Uri(SiteLink + qRow.Find("td:nth-child(4) > a").Attr("href")); var dateString = qRow.Find("td:nth-child(6) nobr")[0].InnerText.Trim(); //"2015-04-25 23:38:12" //"yyyy-MMM-dd hh:mm:ss" release.PublishDate = DateTime.ParseExact(dateString, "yyyy-MM-ddHH:mm:ss", CultureInfo.InvariantCulture); var sizeStr = qRow.Children().ElementAt(6).InnerText.Trim(); release.Size = ReleaseInfo.GetBytes(sizeStr); release.Seeders = ParseUtil.CoerceInt(qRow.Find("td:nth-child(9)").Text()); release.Peers = release.Seeders + ParseUtil.CoerceInt(qRow.Find("td:nth-child(10)").Text()); var category = qRow.Find(".br_type > a").Attr("href").Replace("browse.php?cat=", string.Empty); release.Category = MapTrackerCatToNewznab(category); releases.Add(release); } } catch (Exception ex) { OnParseError(results.Content, ex); } } return(releases); }
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { var releases = new List <ReleaseInfo>(); var searchString = query.GetQueryString(); var queryUrl = SearchUrl; var queryCollection = new NameValueCollection(); if (!string.IsNullOrWhiteSpace(query.ImdbID) && query.ImdbID.ToLower().StartsWith("tt")) { queryCollection.Add("search", query.ImdbID); } else { if (!string.IsNullOrWhiteSpace(searchString)) { queryCollection.Add("search", searchString); } } foreach (var cat in MapTorznabCapsToTrackers(query)) { queryCollection.Add("c" + cat, "1"); } if (queryCollection.Count > 0) { queryUrl += "?" + queryCollection.GetQueryString(); } var results = await RequestStringWithCookiesAndRetry(queryUrl); // Check for being logged out if (results.IsRedirect) { if (results.RedirectingTo.Contains("login.php")) { throw new ExceptionWithConfigData("Login failed, please reconfigure the tracker to update the cookies", configData); } else { throw new ExceptionWithConfigData(string.Format("Got a redirect to {0}, please adjust your the alternative link", results.RedirectingTo), configData); } } try { CQ dom = results.Content; var rows = dom["#torrentTable > tbody > tr.browse"]; foreach (var row in rows) { CQ qRow = row.Cq(); var release = new ReleaseInfo(); release.MinimumRatio = 1; release.MinimumSeedTime = 172800; release.Title = qRow.Find(".torrentName").Text(); if ((query.ImdbID == null || !TorznabCaps.SupportsImdbSearch) && !query.MatchQueryStringAND(release.Title)) { continue; } release.Guid = new Uri(SiteLink + qRow.Find(".torrentName").Attr("href")); release.Comments = release.Guid; release.Link = new Uri(SiteLink + qRow.Find(".dlLinksInfo > a").Attr("href")); var sizeStr = qRow.Find(".sizeInfo").Text(); release.Size = ReleaseInfo.GetBytes(sizeStr); var dateStr = qRow.Find(".ulInfo").Text().Split('|').Last().Trim(); var agoIdx = dateStr.IndexOf("ago"); if (agoIdx > -1) { dateStr = dateStr.Substring(0, agoIdx); } release.PublishDate = DateTimeUtil.FromTimeAgo(dateStr); release.Seeders = ParseUtil.CoerceInt(qRow.Find(".seedersInfo").Text()); release.Peers = ParseUtil.CoerceInt(qRow.Find(".leechersInfo").Text()) + release.Seeders; var cat = qRow.Find("td:eq(0) a").First().Attr("href").Split('#')[0].Substring(15);//browse.php?cat=24 release.Category = MapTrackerCatToNewznab(cat); if (qRow.Find("span.flTags").Length >= 1) { release.DownloadVolumeFactor = 0; } else { release.DownloadVolumeFactor = 1; } release.UploadVolumeFactor = 1; releases.Add(release); } } catch (Exception ex) { OnParseError(results.Content, ex); } return(releases); }
public async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { var releases = new List <ReleaseInfo>(); string qryString = query.GetQueryString(); Match matchQry = new Regex(@".*\s[Ss]{1}\d{2}$").Match(qryString); if (matchQry.Success) { //If search string ends in S## eg. S03 (season search) add an asterix to search term qryString += "*"; } var episodeSearchUrl = SearchUrl + HttpUtility.UrlEncode(qryString); WebClientStringResult response = await RequestStringWithCookiesAndRetry(episodeSearchUrl); try { string decodedResponse = WebUtility.HtmlDecode(response.Content); var json = JObject.Parse(decodedResponse); foreach (JObject r in json["response"]["results"]) { DateTime pubDate = DateTime.MinValue; double dateNum; if (double.TryParse((string)r["groupTime"], out dateNum)) { pubDate = DateTimeUtil.UnixTimestampToDateTime(dateNum); pubDate = DateTime.SpecifyKind(pubDate, DateTimeKind.Utc).ToLocalTime(); } string groupName = (string)r["groupName"]; if (r["torrents"] is JArray) { string showName = (string)r["artist"]; foreach (JObject t in r["torrents"]) { var release = new ReleaseInfo(); release.PublishDate = pubDate; release.Title = $"{showName} {groupName}"; release.Description = $"{showName} {groupName}"; FillReleaseInfoFromJson(release, t); releases.Add(release); } } else { var release = new ReleaseInfo(); release.PublishDate = pubDate; release.Title = groupName; release.Description = groupName; FillReleaseInfoFromJson(release, r); releases.Add(release); } } } catch (Exception ex) { OnParseError(response.Content, ex); } return(releases); }
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { var releases = new List <ReleaseInfo>(); var searchString = query.GetQueryString(); WebClientStringResult results = null; var queryCollection = new NameValueCollection(); queryCollection.Add("st", "0"); queryCollection.Add("sd", "d"); queryCollection.Add("sk", "t"); queryCollection.Add("tracker_search", "torrent"); queryCollection.Add("t", "0"); queryCollection.Add("submit", "Search"); queryCollection.Add("sr", "topics"); //queryCollection.Add("sr", "posts"); //queryCollection.Add("ch", "99999"); // if the search string is empty use the getnew view if (string.IsNullOrWhiteSpace(searchString)) { queryCollection.Add("search_id", "active_topics"); queryCollection.Add("ot", "1"); } else // use the normal search { searchString = searchString.Replace("-", " "); queryCollection.Add("keywords", searchString); queryCollection.Add("sf", "titleonly"); queryCollection.Add("sr", "topics"); queryCollection.Add("pt", "t"); queryCollection.Add("ot", "1"); } var searchUrl = SearchUrl + "?" + queryCollection.GetQueryString(); results = await RequestStringWithCookies(searchUrl); if (!results.Content.Contains("ucp.php?mode=logout")) { await ApplyConfiguration(null); results = await RequestStringWithCookies(searchUrl); } try { string RowsSelector = "ul.topics > li.row"; var ResultParser = new HtmlParser(); var SearchResultDocument = ResultParser.ParseDocument(results.Content); var Rows = SearchResultDocument.QuerySelectorAll(RowsSelector); foreach (var Row in Rows) { try { var release = new ReleaseInfo(); release.MinimumRatio = 1; release.MinimumSeedTime = 0; var qDetailsLink = Row.QuerySelector("a.topictitle"); release.Title = qDetailsLink.TextContent; release.Comments = new Uri(SiteLink + qDetailsLink.GetAttribute("href")); release.Guid = release.Comments; var detailsResult = await RequestStringWithCookies(SiteLink + qDetailsLink.GetAttribute("href")); var DetailsResultDocument = ResultParser.ParseDocument(detailsResult.Content); var qDownloadLink = DetailsResultDocument.QuerySelector("table.table2 > tbody > tr > td > a[href^=\"/download/torrent.php?id\"]"); release.Link = new Uri(SiteLink + qDownloadLink.GetAttribute("href").TrimStart('/')); release.Seeders = ParseUtil.CoerceInt(Row.QuerySelector("span.seed").TextContent); release.Peers = ParseUtil.CoerceInt(Row.QuerySelector("span.leech").TextContent) + release.Seeders; release.Grabs = ParseUtil.CoerceLong(Row.QuerySelector("span.complet").TextContent); var author = Row.QuerySelector("dd.lastpost > span"); var timestr = author.TextContent.Split('\n')[4].Trim(); release.PublishDate = DateTimeUtil.FromUnknown(timestr, "UK"); var forum = Row.QuerySelector("a[href^=\"./viewforum.php?f=\"]"); var forumid = forum.GetAttribute("href").Split('=')[1]; release.Category = MapTrackerCatToNewznab(forumid); var size = Row.QuerySelector("dl.row-item > dt > div.list-inner > div[style^=\"float:right\"]").TextContent; size = size.Replace("GiB", "GB"); size = size.Replace("MiB", "MB"); size = size.Replace("KiB", "KB"); size = size.Replace("ГБ", "GB"); size = size.Replace("МБ", "MB"); size = size.Replace("КБ", "KB"); release.Size = ReleaseInfo.GetBytes(size); release.DownloadVolumeFactor = 1; release.UploadVolumeFactor = 1; releases.Add(release); } catch (Exception ex) { logger.Error(string.Format("{0}: Error while parsing row '{1}':\n\n{2}", ID, Row.OuterHtml, ex)); } } } catch (Exception ex) { OnParseError(results.Content, ex); } return(releases); }
/// <summary> /// Execute our search query /// </summary> /// <param name="query">Query</param> /// <returns>Releases</returns> public async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { var releases = new List <ReleaseInfo>(); var torrentRowList = new List <CQ>(); var searchTerm = query.GetQueryString(); var searchUrl = SearchUrl; int nbResults = 0; int pageLinkCount = 0; // Check cache first so we don't query the server (if search term used or not in dev mode) if (!DevMode && !string.IsNullOrEmpty(searchTerm)) { lock (cache) { // Remove old cache items CleanCache(); // Search in cache var cachedResult = cache.Where(i => i.Query == searchTerm).FirstOrDefault(); if (cachedResult != null) { return(cachedResult.Results.Select(s => (ReleaseInfo)s.Clone()).ToArray()); } } } // Build our query var request = buildQuery(searchTerm, query, searchUrl); // Getting results & Store content WebClientStringResult results = await queryExec(request); fDom = results.Content; try { // Find torrent rows var firstPageRows = findTorrentRows(); // Add them to torrents list torrentRowList.AddRange(firstPageRows.Select(fRow => fRow.Cq())); // Check if there are pagination links at bottom Boolean pagination = (fDom[".linkbox > a"].Length != 0); // If pagination available if (pagination) { // Calculate numbers of pages available for this search query (Based on number results and number of torrents on first page) pageLinkCount = ParseUtil.CoerceInt(Regex.Match(fDom[".linkbox > a"].Last().Attr("href").ToString(), @"\d+").Value); // Calculate average number of results (based on torrents rows lenght on first page) nbResults = firstPageRows.Count() * pageLinkCount; } else { // Check if we have a minimum of one result if (firstPageRows.Length >= 1) { // Retrieve total count on our alone page nbResults = firstPageRows.Count(); pageLinkCount = 1; } else { output("\nNo result found for your query, please try another search term ...\n", "info"); // No result found for this query return(releases); } } output("\nFound " + nbResults + " result(s) (+/- " + firstPageRows.Length + ") in " + pageLinkCount + " page(s) for this query !"); output("\nThere are " + firstPageRows.Length + " results on the first page !"); // If we have a term used for search and pagination result superior to one if (!string.IsNullOrWhiteSpace(query.GetQueryString()) && pageLinkCount > 1) { // Starting with page #2 for (int i = 2; i <= Math.Min(Int32.Parse(ConfigData.Pages.Value), pageLinkCount); i++) { output("\nProcessing page #" + i); // Request our page latencyNow(); // Build our query var pageRequest = buildQuery(searchTerm, query, searchUrl, i); // Getting results & Store content WebClientStringResult pageResults = await queryExec(pageRequest); // Assign response fDom = pageResults.Content; // Process page results var additionalPageRows = findTorrentRows(); // Add them to torrents list torrentRowList.AddRange(additionalPageRows.Select(fRow => fRow.Cq())); } } else { // No search term, maybe testing... so registring autkey and torrentpass for future uses string infosData = firstPageRows.First().Find("td:eq(3) > a").Attr("href"); IList <string> infosList = infosData.Split('&').Select(s => s.Trim()).Where(s => s != String.Empty).ToList(); IList <string> infosTracker = infosList.Select(s => s.Split(new[] { '=' }, 2)[1].Trim()).ToList(); output("\nStoring Authkey for future uses..."); ConfigData.AuthKey.Value = infosTracker[2]; output("\nStoring TorrentPass for future uses..."); ConfigData.TorrentPass.Value = infosTracker[3]; } // Loop on results foreach (CQ tRow in torrentRowList) { output("\n=>> Torrent #" + (releases.Count + 1)); // ID int id = ParseUtil.CoerceInt(Regex.Match(tRow.Find("td:eq(1) > a").Attr("href").ToString(), @"\d+").Value); output("ID: " + id); // Release Name string name = tRow.Find("td:eq(1) > a").Text().ToString(); output("Release: " + name); // Category string categoryID = tRow.Find("td:eq(0) > a").Attr("href").Replace("torrents.php?cat[]=", String.Empty); output("Category: " + MapTrackerCatToNewznab(categoryID) + " (" + categoryID + ")"); // Seeders int seeders = ParseUtil.CoerceInt(Regex.Match(tRow.Find("td:eq(5)").Text(), @"\d+").Value); output("Seeders: " + seeders); // Leechers int leechers = ParseUtil.CoerceInt(Regex.Match(tRow.Find("td:eq(6)").Text(), @"\d+").Value); output("Leechers: " + leechers); // Completed int completed = ParseUtil.CoerceInt(Regex.Match(tRow.Find("td:eq(5)").Text(), @"\d+").Value); output("Completed: " + completed); // Size string sizeStr = tRow.Find("td:eq(4)").Text().Replace("Go", "gb").Replace("Mo", "mb").Replace("Ko", "kb"); long size = ReleaseInfo.GetBytes(sizeStr); output("Size: " + sizeStr + " (" + size + " bytes)"); // Publish DateToString IList <string> clockList = tRow.Find("td:eq(2) > span").Text().Replace("Il y a", "").Split(',').Select(s => s.Trim()).Where(s => s != String.Empty).ToList(); var date = agoToDate(clockList); output("Released on: " + date.ToLocalTime()); // Torrent Details URL Uri detailsLink = new Uri(TorrentDescriptionUrl + id); output("Details: " + detailsLink.AbsoluteUri); // Torrent Comments URL Uri commentsLink = new Uri(TorrentCommentUrl + id); output("Comments Link: " + commentsLink.AbsoluteUri); // Torrent Download URL Uri downloadLink = new Uri(TorrentDownloadUrl.Replace("{id}", id.ToString()).Replace("{auth_key}", ConfigData.AuthKey.Value).Replace("{torrent_pass}", ConfigData.TorrentPass.Value)); output("Download Link: " + downloadLink.AbsoluteUri); // Building release infos var release = new ReleaseInfo(); release.Category = MapTrackerCatToNewznab(categoryID.ToString()); release.Title = name; release.Seeders = seeders; release.Peers = seeders + leechers; release.MinimumRatio = 1; release.MinimumSeedTime = 172800; release.PublishDate = date; release.Size = size; release.Guid = detailsLink; release.Comments = commentsLink; release.Link = downloadLink; // freeleech if (tRow.Find("img[alt=\"Freeleech\"]").Length >= 1) { release.DownloadVolumeFactor = 0; } else { release.DownloadVolumeFactor = 1; } release.UploadVolumeFactor = 1; releases.Add(release); } } catch (Exception ex) { OnParseError("Error, unable to parse result \n" + ex.StackTrace, ex); } // Return found releases return(releases); }
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { return(await PerformQuery(query, 0)); }
/// <summary> /// Execute our search query /// </summary> /// <param name="query">Query</param> /// <returns>Releases</returns> protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { var releases = new List <ReleaseInfo>(); var torrentRowList = new List <CQ>(); var searchTerm = query.GetQueryString(); var searchUrl = SearchUrl; int nbResults = 0; int pageLinkCount = 0; // Check cache first so we don't query the server (if search term used or not in dev mode) if (!DevMode && !string.IsNullOrEmpty(searchTerm)) { lock (cache) { // Remove old cache items CleanCache(); // Search in cache var cachedResult = cache.Where(i => i.Query == searchTerm).FirstOrDefault(); if (cachedResult != null) { return(cachedResult.Results.Select(s => (ReleaseInfo)s.Clone()).ToArray()); } } } // Add emulated XHR request emulatedBrowserHeaders.Add("X-Requested-With", "XMLHttpRequest"); // Build our query var request = buildQuery(searchTerm, query, searchUrl); // Getting results & Store content fDom = await queryExec(request); try { // Find number of results nbResults = ParseUtil.CoerceInt(Regex.Match(fDom["div.ajaxtotaltorrentcount"].Text(), @"\d+").Value); // Find torrent rows var firstPageRows = findTorrentRows(); // Add them to torrents list torrentRowList.AddRange(firstPageRows.Select(fRow => fRow.Cq())); // Check if there are pagination links at bottom Boolean pagination = (nbResults != 0); // If pagination available if (pagination) { // Calculate numbers of pages available for this search query (Based on number results and number of torrents on first page) pageLinkCount = (int)Math.Ceiling((double)nbResults / firstPageRows.Length); } else { // Check if we have a minimum of one result if (firstPageRows.Length >= 1) { // Set page count arbitrary to one pageLinkCount = 1; } else { output("\nNo result found for your query, please try another search term ...\n", "info"); // No result found for this query return(releases); } } output("\nFound " + nbResults + " result(s) in " + pageLinkCount + " page(s) for this query !"); output("\nThere are " + firstPageRows.Length + " results on the first page !"); // If we have a term used for search and pagination result superior to one if (!string.IsNullOrWhiteSpace(query.GetQueryString()) && pageLinkCount > 1) { // Starting with page #2 for (int i = 2; i <= Math.Min(Int32.Parse(ConfigData.Pages.Value), pageLinkCount); i++) { output("\nProcessing page #" + i); // Request our page latencyNow(); // Build our query var pageRequest = buildQuery(searchTerm, query, searchUrl, i); // Getting results & Store content fDom = await queryExec(pageRequest); // Process page results var additionalPageRows = findTorrentRows(); // Add them to torrents list torrentRowList.AddRange(additionalPageRows.Select(fRow => fRow.Cq())); } } // Loop on results foreach (CQ tRow in torrentRowList) { output("\n=>> Torrent #" + (releases.Count + 1)); // Release Name string name = tRow.Find(".torrent-h3 > h3 > a").Attr("title").ToString(); output("Release: " + name); // Category string categoryID = tRow.Find(".category > img").Attr("src").Split('/').Last().ToString(); string categoryName = tRow.Find(".category > img").Attr("title").ToString(); output("Category: " + MapTrackerCatToNewznab(mediaToCategory(categoryID, categoryName)).First().ToString() + " (" + categoryName + ")"); // Uploader string uploader = tRow.Find(".uploader > span > a").Attr("title").ToString(); output("Uploader: " + uploader); // Seeders int seeders = ParseUtil.CoerceInt(Regex.Match(tRow.Find(".seeders")[0].LastChild.ToString(), @"\d+").Value); output("Seeders: " + seeders); // Leechers int leechers = ParseUtil.CoerceInt(Regex.Match(tRow.Find(".leechers")[0].LastChild.ToString(), @"\d+").Value); output("Leechers: " + leechers); // Completed int completed = ParseUtil.CoerceInt(Regex.Match(tRow.Find(".completed")[0].LastChild.ToString(), @"\d+").Value); output("Completed: " + completed); // Comments int comments = ParseUtil.CoerceInt(Regex.Match(tRow.Find(".comments")[0].LastChild.ToString(), @"\d+").Value); output("Comments: " + comments); // Size & Publish Date string infosData = tRow.Find(".torrent-h3 > span")[0].LastChild.ToString().Trim(); IList <string> infosList = infosData.Split('-').Select(s => s.Trim()).Where(s => s != String.Empty).ToList(); // --> Size var size = ReleaseInfo.GetBytes(infosList[1].Replace("Go", "gb").Replace("Mo", "mb").Replace("Ko", "kb")); output("Size: " + infosList[1] + " (" + size + " bytes)"); // --> Publish Date IList <string> clockList = infosList[0].Replace("Il y a", "").Split(',').Select(s => s.Trim()).Where(s => s != String.Empty).ToList(); var clock = agoToDate(clockList); output("Released on: " + clock.ToString()); // Torrent Details URL string details = tRow.Find(".torrent-h3 > h3 > a").Attr("href").ToString().TrimStart('/'); Uri detailsLink = new Uri(SiteLink + details); output("Details: " + detailsLink.AbsoluteUri); // Torrent Comments URL Uri commentsLink = new Uri(SiteLink + details + "#tab_2"); output("Comments Link: " + commentsLink.AbsoluteUri); // Torrent Download URL string download = tRow.Find(".download-item > a").Attr("href").ToString().TrimStart('/'); Uri downloadLink = new Uri(SiteLink + download); output("Download Link: " + downloadLink.AbsoluteUri); // Freeleech int downloadVolumeFactor = 1; if (tRow.Find(".fl-item").Length >= 1) { downloadVolumeFactor = 0; output("FreeLeech =)"); } // Building release infos var release = new ReleaseInfo() { Category = MapTrackerCatToNewznab(mediaToCategory(categoryID, categoryName)), Title = name, Seeders = seeders, Peers = seeders + leechers, MinimumRatio = 1, MinimumSeedTime = 345600, PublishDate = clock, Size = size, Guid = detailsLink, Comments = commentsLink, Link = downloadLink, UploadVolumeFactor = 1, DownloadVolumeFactor = downloadVolumeFactor }; releases.Add(release); } } catch (Exception ex) { OnParseError("Error, unable to parse result \n" + ex.StackTrace, ex); } finally { // Remove our XHR request header emulatedBrowserHeaders.Remove("X-Requested-With"); } // Return found releases return(releases); }
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { // TODO verify this code is necessary for TZ data or if builtin exist var startTransition = TimeZoneInfo.TransitionTime.CreateFloatingDateRule( new DateTime(1, 1, 1, 3, 0, 0), 3, 5, DayOfWeek.Sunday); var endTransition = TimeZoneInfo.TransitionTime.CreateFloatingDateRule( new DateTime(1, 1, 1, 4, 0, 0), 10, 5, DayOfWeek.Sunday); var delta = new TimeSpan(1, 0, 0); var adjustment = TimeZoneInfo.AdjustmentRule.CreateAdjustmentRule( new DateTime(1999, 10, 1), DateTime.MaxValue.Date, delta, startTransition, endTransition); TimeZoneInfo.AdjustmentRule[] adjustments = { adjustment }; var germanyTz = TimeZoneInfo.CreateCustomTimeZone( "W. Europe Standard Time", new TimeSpan(1, 0, 0), "(GMT+01:00) W. Europe Standard Time", "W. Europe Standard Time", "W. Europe DST Time", adjustments); var releases = new List <ReleaseInfo>(); var searchString = query.GetQueryString(); var searchUrl = IndexUrl; var queryCollection = new NameValueCollection { { "strWebValue", "torrent" }, { "strWebAction", "search" }, { "sort", "torrent_added" }, { "by", "d" }, { "type", "2" }, // 0 active, 1 inactive, 2 all { "do_search", "suchen" }, { "time", "0" }, // 0 any, 1 1day, 2 1week, 3 30days, 4 90days { "details", "title" } // title, info, descr, all }; if (!string.IsNullOrWhiteSpace(searchString)) { queryCollection.Add("searchstring", searchString); } foreach (var cat in MapTorznabCapsToTrackers(query)) { queryCollection.Add("dirs" + cat, "1"); } searchUrl += "?" + queryCollection.GetQueryString(); var response = await RequestWithCookiesAsync(searchUrl); var titleRegexp = new Regex(@"^return buildTable\('(.*?)',\s+"); try { var parser = new HtmlParser(); var dom = parser.ParseDocument(response.ContentString); var rows = dom.QuerySelectorAll("table.torrenttable > tbody > tr"); foreach (var row in rows.Skip(1)) { var qColumn1 = row.QuerySelectorAll("td.column1"); var qColumn2 = row.QuerySelectorAll("td.column2"); var qDetailsLink = row.QuerySelector("a[href^=\"index.php?strWebValue=torrent&strWebAction=details\"]"); var qCatLink = row.QuerySelector("a[href^=\"index.php?strWebValue=torrent&strWebAction=search&dir=\"]"); var qDlLink = row.QuerySelector("a[href^=\"index.php?strWebValue=torrent&strWebAction=download&id=\"]"); var qDateStr = row.QuerySelector("font:has(a)"); var catStr = qCatLink.GetAttribute("href").Split('=')[3].Split('#')[0]; var link = new Uri(SiteLink + qDlLink.GetAttribute("href")); var dateStr = qDateStr.TextContent; var split = dateStr.IndexOf("Uploader", StringComparison.OrdinalIgnoreCase); dateStr = dateStr.Substring(0, split > 0 ? split : dateStr.Length).Trim().Replace("Heute", "Today") .Replace("Gestern", "Yesterday"); var dateGerman = DateTimeUtil.FromUnknown(dateStr); double downloadFactor; if (row.QuerySelector("img[src=\"themes/images/freeleech.png\"]") != null || row.QuerySelector("img[src=\"themes/images/onlyup.png\"]") != null) { downloadFactor = 0; } else if (row.QuerySelector("img[src=\"themes/images/DL50.png\"]") != null) { downloadFactor = 0.5; } else { downloadFactor = 1; } var title = titleRegexp.Match(qDetailsLink.GetAttribute("onmouseover")).Groups[1].Value; var details = new Uri(SiteLink + qDetailsLink.GetAttribute("href")); var size = ReleaseInfo.GetBytes(qColumn2[1].TextContent); var seeders = ParseUtil.CoerceInt(qColumn1[3].TextContent); var leechers = ParseUtil.CoerceInt(qColumn2[3].TextContent); var publishDate = TimeZoneInfo.ConvertTime(dateGerman, germanyTz, TimeZoneInfo.Local); var release = new ReleaseInfo { MinimumRatio = 0.8, MinimumSeedTime = 0, Title = title, Category = MapTrackerCatToNewznab(catStr), Details = details, Link = link, Guid = link, Size = size, Seeders = seeders, Peers = leechers + seeders, PublishDate = publishDate, DownloadVolumeFactor = downloadFactor, UploadVolumeFactor = 1 }; releases.Add(release); } } catch (Exception ex) { OnParseError(response.ContentString, ex); } return(releases); }
public async Task<ReleaseInfo[]> PerformQuery(TorznabQuery query, string baseUrl) { List<ReleaseInfo> releases = new List<ReleaseInfo>(); foreach (var title in query.ShowTitles ?? new string[] { "2015" }) { var searchString = title + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = baseUrl + string.Format(SearchUrl, HttpUtility.UrlEncode(searchString.Trim())); var results = await client.GetStringAsync(episodeSearchUrl); try { var jResults = JObject.Parse(results); foreach (JObject result in (JArray)jResults["torrents"]) { var release = new ReleaseInfo(); release.MinimumRatio = 1; release.MinimumSeedTime = 172800; release.Title = (string)result["torrent_title"]; release.Description = release.Title; release.Seeders = (int)result["seeds"]; release.Peers = (int)result["leeches"] + release.Seeders; release.Size = (long)result["size"]; // "Apr 2, 2015", "Apr 12, 2015" (note the spacing) var dateString = string.Join(" ", ((string)result["upload_date"]).Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries)); release.PublishDate = DateTime.ParseExact(dateString, "MMM d, yyyy", CultureInfo.InvariantCulture); release.Guid = new Uri((string)result["page"]); release.Comments = release.Guid; release.InfoHash = (string)result["torrent_hash"]; release.MagnetUri = new Uri((string)result["magnet_uri"]); release.Link = new Uri(string.Format("{0}{1}{2}", baseUrl, DownloadUrl, release.InfoHash)); releases.Add(release); } } catch (Exception ex) { OnResultParsingError(this, results, ex); throw ex; } } return releases.ToArray(); }
async Task<ReleaseInfo[]> PerformQuery(TorznabQuery query, string baseUrl) { List<ReleaseInfo> releases = new List<ReleaseInfo>(); foreach (var title in query.ShowTitles ?? new string[] { string.Empty }) { var searchString = title + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = string.Format(SearchUrl, HttpUtility.UrlEncode(searchString.Trim())); XmlDocument xmlDoc = new XmlDocument(); string xml = string.Empty; WebClient wc = getWebClient(); try { using (wc) { xml = wc.DownloadString(episodeSearchUrl); xmlDoc.LoadXml(xml); } ReleaseInfo release; TorrentzHelper td; string serie_title; foreach (XmlNode node in xmlDoc.GetElementsByTagName("item")) { release = new ReleaseInfo(); release.MinimumRatio = 1; release.MinimumSeedTime = 172800; serie_title = node.SelectSingleNode("title").InnerText; release.Title = serie_title; release.Comments = new Uri(node.SelectSingleNode("link").InnerText); release.Category = node.SelectSingleNode("category").InnerText; release.Guid = new Uri(node.SelectSingleNode("guid").InnerText); release.PublishDate = DateTime.Parse(node.SelectSingleNode("pubDate").InnerText, CultureInfo.InvariantCulture); td = new TorrentzHelper(node.SelectSingleNode("description").InnerText); release.Description = td.Description; release.InfoHash = td.hash; release.Size = td.Size; release.Seeders = td.Seeders; release.Peers = td.Peers + release.Seeders; release.MagnetUri = TorrentzHelper.createMagnetLink(td.hash, serie_title); releases.Add(release); } } catch (Exception ex) { OnResultParsingError(this, xml, ex); throw ex; } } return releases.ToArray(); }
public async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query, int attempts) { var releases = new List <ReleaseInfo>(); var searchString = query.GetQueryString(); var queryCollection = new NameValueCollection(); if (query.ImdbID != null) { queryCollection.Add("query_term", query.ImdbID); } else if (!string.IsNullOrWhiteSpace(searchString)) { searchString = searchString.Replace("'", ""); // ignore ' (e.g. search for america's Next Top Model) queryCollection.Add("query_term", searchString); } // This API does not seem to be working for quality=720p or quality=1080p // Only quality=3D seems to return a proper result? //var cats = string.Join(";", MapTorznabCapsToTrackers(query)); //if (!string.IsNullOrEmpty(cats)) //{ // if (cats == "45") // { // queryCollection.Add("quality", "720p"); // } // if (cats == "44") // { // queryCollection.Add("quality", "1080p"); // } // if (cats == "2050") // { // queryCollection.Add("quality", "3D"); // } //} var searchUrl = ApiEndpoint + "?" + queryCollection.GetQueryString(); var response = await RequestStringWithCookiesAndRetry(searchUrl, string.Empty); try { var jsonContent = JObject.Parse(response.Content); string result = jsonContent.Value <string>("status"); if (result != "ok") // query was not successful { return(releases.ToArray()); } var data_items = jsonContent.Value <JToken>("data"); int movie_count = data_items.Value <int>("movie_count"); if (movie_count < 1) // no results found in query { return(releases.ToArray()); } foreach (var movie_item in data_items.Value <JToken>("movies")) { var torrents = movie_item.Value <JArray>("torrents"); if (torrents == null) { continue; } foreach (var torrent_info in torrents) { var release = new ReleaseInfo(); // Append the quality to the title because thats how radarr seems to be determining the quality? // All releases are BRRips, see issue #2200 release.Title = movie_item.Value <string>("title_long") + " " + torrent_info.Value <string>("quality") + " BRRip"; var imdb = movie_item.Value <string>("imdb_code"); release.Imdb = ParseUtil.GetImdbID(imdb); // API does not provide magnet link, so, construct it string magnet_uri = "magnet:?xt=urn:btih:" + torrent_info.Value <string>("hash") + "&dn=" + movie_item.Value <string>("slug") + "&tr=udp://open.demonii.com:1337/announce" + "&tr=udp://tracker.openbittorrent.com:80" + "&tr=udp://tracker.coppersurfer.tk:6969" + "&tr=udp://glotorrents.pw:6969/announce" + "&tr=udp://tracker.opentrackr.org:1337/announce" + "&tr=udp://torrent.gresille.org:80/announce" + "&tr=udp://p4p.arenabg.com:1337&tr=udp://tracker.leechers-paradise.org:6969"; release.MagnetUri = new Uri(magnet_uri); release.InfoHash = torrent_info.Value <string>("hash"); // ex: 2015-08-16 21:25:08 +0000 var dateStr = torrent_info.Value <string>("date_uploaded"); var dateTime = DateTime.ParseExact(dateStr, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture); release.PublishDate = DateTime.SpecifyKind(dateTime, DateTimeKind.Utc).ToLocalTime(); release.Link = new Uri(torrent_info.Value <string>("url")); release.Seeders = torrent_info.Value <int>("seeds"); release.Peers = torrent_info.Value <int>("peers") + release.Seeders; release.Size = torrent_info.Value <long>("size_bytes"); release.DownloadVolumeFactor = 0; release.UploadVolumeFactor = 1; release.Comments = new Uri(movie_item.Value <string>("url")); release.BannerUrl = new Uri(movie_item.Value <string>("large_cover_image")); release.Guid = release.Link; // Hack to prevent adding non-specified catogery, since API doesn't seem to be working string categories = string.Join(";", MapTorznabCapsToTrackers(query)); if (!string.IsNullOrEmpty(categories)) { if (categories.Contains("45") || categories.Contains("2040")) { if (torrent_info.Value <string>("quality") == "720p") { release.Category = MapTrackerCatToNewznab("45"); releases.Add(release); } } if (categories.Contains("44") || categories.Contains("2040")) { if (torrent_info.Value <string>("quality") == "1080p") { release.Category = MapTrackerCatToNewznab("44"); releases.Add(release); } } if (categories.Contains("47")) { if (torrent_info.Value <string>("quality") == "3D") { release.Category = MapTrackerCatToNewznab("47"); releases.Add(release); } } } else { release.Category = MapTrackerCatToNewznab("45"); releases.Add(release); } } } } catch (Exception ex) { OnParseError(response.Content, ex); } return(releases); }
public async Task<ReleaseInfo[]> PerformQuery(TorznabQuery query) { List<ReleaseInfo> releases = new List<ReleaseInfo>(); foreach (var title in query.ShowTitles ?? new string[] { string.Empty }) { var searchString = title + " " + query.GetEpisodeSearchString(); var searchContent = GetSearchFormData(searchString); var response = await client.PostAsync(SearchUrl, searchContent); var results = await response.Content.ReadAsStringAsync(); try { CQ dom = results; var rows = dom["tr.browse"]; foreach (var row in rows) { var release = new ReleaseInfo(); release.MinimumRatio = 1; release.MinimumSeedTime = 172800; var descCol = row.ChildElements.ElementAt(1); var qDescCol = descCol.Cq(); var qLink = qDescCol.Find("a"); release.Title = qLink.Text(); release.Description = release.Title; release.Comments = new Uri(BaseUrl + "/" + qLink.Attr("href")); release.Guid = release.Comments; var torrentId = qLink.Attr("href").Split('=')[1]; release.Link = new Uri(string.Format(DownloadUrl, torrentId)); var dateStr = descCol.ChildNodes.Last().NodeValue.Trim(); var euDate = DateTime.ParseExact(dateStr, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture); var localDate = TimeZoneInfo.ConvertTimeToUtc(euDate, TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time")).ToLocalTime(); release.PublishDate = localDate; var sizeNodes = row.ChildElements.ElementAt(3).ChildNodes; var sizeVal = sizeNodes.First().NodeValue; var sizeUnit = sizeNodes.Last().NodeValue; release.Size = ReleaseInfo.GetBytes(sizeUnit, ParseUtil.CoerceFloat(sizeVal)); release.Seeders = ParseUtil.CoerceInt(row.ChildElements.ElementAt(4).Cq().Text().Trim()); release.Peers = ParseUtil.CoerceInt(row.ChildElements.ElementAt(5).Cq().Text().Trim()) + release.Seeders; releases.Add(release); } } catch (Exception ex) { OnResultParsingError(this, results, ex); throw ex; } } return releases.ToArray(); }
private async Task <IEnumerable <ReleaseInfo> > TvSearch(TorznabQuery query) { var seriesName = query.SanitizedSearchTerm; var season = query.Season > 0 ? (int?)query.Season : null; int?episode = null; if (!string.IsNullOrWhiteSpace(query.Episode) && int.TryParse(query.Episode, out var episodeTemp)) { episode = episodeTemp; } //If query has no season/episode info, try to parse title if (season == null && episode == null) { var searchMatch = _searchStringRegex.Match(query.SanitizedSearchTerm); if (searchMatch.Success) { seriesName = searchMatch.Groups[1].Value.Trim(); season = int.Parse(searchMatch.Groups[2].Value); episode = searchMatch.Groups[4].Success ? (int?)int.Parse(searchMatch.Groups[4].Value) : null; } } var releases = new List <ReleaseInfo>(); //Search series url foreach (var seriesListUrl in SeriesListUris(seriesName)) { releases.AddRange(await GetReleasesFromUri(seriesListUrl, seriesName)); } //Sonarr removes "the" from shows. If there is nothing try prepending "the" if (releases.Count == 0 && !(seriesName.ToLower().StartsWith("the"))) { seriesName = "The " + seriesName; foreach (var seriesListUrl in SeriesListUris(seriesName)) { releases.AddRange(await GetReleasesFromUri(seriesListUrl, seriesName)); } } // remove duplicates releases = releases.GroupBy(x => x.Guid).Select(y => y.First()).ToList(); //Filter only episodes needed return(releases.Where(r => { var nr = r as NewpctRelease; return ( nr.Season.HasValue != season.HasValue || //Can't determine if same season nr.Season.HasValue && season.Value == nr.Season.Value && //Same season and ... ( nr.Episode.HasValue != episode.HasValue || //Can't determine if same episode nr.Episode.HasValue && ( nr.Episode.Value == episode.Value || //Same episode nr.EpisodeTo.HasValue && episode.Value >= nr.Episode.Value && episode.Value <= nr.EpisodeTo.Value //Episode in interval ) ) ); })); }
public async Task<ReleaseInfo[]> PerformQuery(TorznabQuery query) { List<ReleaseInfo> releases = new List<ReleaseInfo>(); foreach (var title in query.ShowTitles ?? new string[] { string.Empty }) { var searchString = title + " " + query.GetEpisodeSearchString(); var searchSection = string.IsNullOrEmpty(query.Episode) ? "archive" : "browse"; var searchCategory = string.IsNullOrEmpty(query.Episode) ? "26" : "27"; var searchUrl = string.Format(SearchUrl, searchSection, searchCategory, searchString); string results; if (Program.IsWindows) { results = await client.GetStringAsync(searchUrl); } else { var response = await CurlHelper.GetAsync(searchUrl, cookieHeader); results = Encoding.UTF8.GetString(response.Content); } try { CQ dom = results; var rows = dom["#torrents-table > tbody > tr.tt_row"]; foreach (var row in rows) { CQ qRow = row.Cq(); var release = new ReleaseInfo(); release.MinimumRatio = 1; release.MinimumSeedTime = 129600; release.Title = qRow.Find(".ttr_name > a").Text(); release.Description = release.Title; release.Guid = new Uri(BaseUrl + "/" + qRow.Find(".ttr_name > a").Attr("href")); release.Comments = release.Guid; release.Link = new Uri(BaseUrl + "/" + qRow.Find(".td_dl > a").Attr("href")); var sizeStr = qRow.Find(".ttr_size").Contents()[0].NodeValue; var sizeParts = sizeStr.Split(' '); release.Size = ReleaseInfo.GetBytes(sizeParts[1], float.Parse(sizeParts[0], NumberStyles.Float | NumberStyles.AllowThousands)); var timeStr = qRow.Find(".ttr_added").Text(); DateTime time; if (DateTime.TryParseExact(timeStr, "yyyy-MM-ddHH:mm:ss", CultureInfo.InvariantCulture, DateTimeStyles.None, out time)) { release.PublishDate = time; } release.Seeders = int.Parse(qRow.Find(".ttr_seeders").Text(), NumberStyles.AllowThousands); release.Peers = int.Parse(qRow.Find(".ttr_leechers").Text(), NumberStyles.AllowThousands) + release.Seeders; releases.Add(release); } } catch (Exception ex) { OnResultParsingError(this, results, ex); throw ex; } } return releases.ToArray(); }
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { var releases = new List <ReleaseInfo>(); var searchUrl = BrowseUrl; var searchTerm = FixSearchTerm(query); var queryCollection = new NameValueCollection { { "searchstr", StripSearchString(searchTerm) }, { "order_by", "time" }, { "order_way", "desc" }, { "group_results", "1" }, { "action", "basic" }, { "searchsubmit", "1" } }; searchUrl += "?" + queryCollection.GetQueryString(); var results = await RequestWithCookiesAsync(searchUrl); try { const string rowsSelector = "table.torrent_table > tbody > tr:not(tr.colhead)"; var searchResultParser = new HtmlParser(); var searchResultDocument = searchResultParser.ParseDocument(results.ContentString); var rows = searchResultDocument.QuerySelectorAll(rowsSelector); string groupTitle = null; string groupYearStr = null; Uri groupPoster = null; string imdbLink = null; string tmdbLink = null; foreach (var row in rows) { try { // ignore sub groups info row, it's just an row with an info about the next section, something like "Dual Áudio" or "Legendado" if (row.QuerySelector(".edition_info") != null) { continue; } // some torrents has more than one link, and the one with .tooltip is the wrong one in that case, // so let's try to pick up first without the .tooltip class, // if nothing is found, then we try again without that filter var qDetailsLink = row.QuerySelector("a[href^=\"torrents.php?id=\"]:not(.tooltip)"); if (qDetailsLink == null) { qDetailsLink = row.QuerySelector("a[href^=\"torrents.php?id=\"]"); // if still can't find the right link, skip it if (qDetailsLink == null) { logger.Error($"{Id}: Error while parsing row '{row.OuterHtml}': Can't find the right details link"); continue; } } var title = StripSearchString(qDetailsLink.TextContent); var seasonEl = row.QuerySelector("a[href^=\"torrents.php?torrentid=\"]"); string seasonEp = null; if (seasonEl != null) { var seasonMatch = _EpisodeRegex.Match(seasonEl.TextContent); seasonEp = seasonMatch.Success ? seasonMatch.Value : null; } seasonEp ??= _EpisodeRegex.Match(qDetailsLink.TextContent).Value; ICollection <int> category = new List <int> { TorznabCatType.Other.ID }; string yearStr = null; if (row.ClassList.Contains("group") || row.ClassList.Contains("torrent")) // group/ungrouped headers { var qCatLink = row.QuerySelector("a[href^=\"/torrents.php?filter_cat\"]"); var torrentInfoEl = row.QuerySelector("div.torrent_info"); if (torrentInfoEl != null) { // valid for torrent grouped but that has only 1 episode yet yearStr = torrentInfoEl.GetAttribute("data-year"); } yearStr ??= qDetailsLink.NextSibling.TextContent.Trim().TrimStart('[').TrimEnd(']'); if (Uri.TryCreate(row.QuerySelector("img[alt=\"Cover\"]")?.GetAttribute("src"), UriKind.Absolute, out var posterUri)) { groupPoster = posterUri; } if (row.ClassList.Contains("group")) // group headers { groupTitle = title; groupYearStr = yearStr; imdbLink = row.QuerySelector("a[href*=\"imdb.com/title/tt\"]")?.GetAttribute("href"); tmdbLink = row.QuerySelector("a[href*=\"themoviedb.org/\"]")?.GetAttribute("href"); continue; } } var release = new ReleaseInfo { MinimumRatio = 1, MinimumSeedTime = 172800 }; var qDlLink = row.QuerySelector("a[href^=\"torrents.php?action=download\"]"); var qSize = row.QuerySelector("td:nth-last-child(4)"); var qGrabs = row.QuerySelector("td:nth-last-child(3)"); var qSeeders = row.QuerySelector("td:nth-last-child(2)"); var qLeechers = row.QuerySelector("td:nth-last-child(1)"); var qFreeLeech = row.QuerySelector("strong[title=\"Free\"]"); if (row.ClassList.Contains("group_torrent")) // torrents belonging to a group { release.Description = qDetailsLink.TextContent; release.Title = ParseTitle(groupTitle, seasonEp, groupYearStr); } else if (row.ClassList.Contains("torrent")) // standalone/un grouped torrents { release.Description = row.QuerySelector("div.torrent_info").TextContent; release.Title = ParseTitle(title, seasonEp, yearStr); imdbLink = row.QuerySelector("a[href*=\"imdb.com/title/tt\"]")?.GetAttribute("href"); tmdbLink = row.QuerySelector("a[href*=\"themoviedb.org/\"]")?.GetAttribute("href"); } release.Poster = groupPoster; release.Imdb = ParseUtil.GetLongFromString(imdbLink); release.TMDb = ParseUtil.GetLongFromString(tmdbLink); release.Category = category; release.Description = release.Description.Replace(" / Free", ""); // Remove Free Tag release.Description = release.Description.Replace("/ WEB ", "/ WEB-DL "); // Fix web/web-dl release.Description = release.Description.Replace("Full HD", "1080p"); // Handles HDR conflict release.Description = release.Description.Replace("/ HD /", "/ 720p /"); release.Description = release.Description.Replace("/ HD]", "/ 720p]"); release.Description = release.Description.Replace("4K", "2160p"); release.Description = release.Description.Replace("SD", "480p"); release.Description = release.Description.Replace("Dual Áudio", "Dual"); release.Description = release.Description.Replace("Dual Audio", "Dual"); // Adjust the description in order to can be read by Radarr and Sonarr var cleanDescription = release.Description.Trim().TrimStart('[').TrimEnd(']'); string[] titleElements; //Formats the title so it can be parsed later var stringSeparators = new[] { " / " }; titleElements = cleanDescription.Split(stringSeparators, StringSplitOptions.None); // release.Title += string.Join(" ", titleElements); release.Title = release.Title.Trim(); if (titleElements.Length < 6) { // Usually non movies / series could have less than 6 elements, eg: Books. release.Title += " " + string.Join(" ", titleElements); } else { release.Title += " " + titleElements[5] + " " + titleElements[3] + " " + titleElements[1] + " " + titleElements[2] + " " + titleElements[4] + " " + string.Join( " ", titleElements.Skip(6)); } if (Regex.IsMatch(release.Description, "(Dual|[Nn]acional|[Dd]ublado)")) { release.Title += " Brazilian"; } // This tracker does not provide an publish date to search terms (only on last 24h page) release.PublishDate = DateTime.Today; // check for previously stripped search terms if (!query.IsImdbQuery && !query.MatchQueryStringAND(release.Title, null, searchTerm)) { continue; } var size = qSize.TextContent; release.Size = ReleaseInfo.GetBytes(size); release.Link = new Uri(SiteLink + qDlLink.GetAttribute("href")); release.Details = new Uri(SiteLink + qDetailsLink.GetAttribute("href")); release.Guid = release.Link; release.Grabs = ParseUtil.CoerceLong(qGrabs.TextContent); release.Seeders = ParseUtil.CoerceInt(qSeeders.TextContent); release.Peers = ParseUtil.CoerceInt(qLeechers.TextContent) + release.Seeders; release.DownloadVolumeFactor = qFreeLeech != null ? 0 : 1; release.UploadVolumeFactor = 1; releases.Add(release); } catch (Exception ex) { logger.Error($"{Id}: Error while parsing row '{row.OuterHtml}': {ex.Message}"); } } } catch (Exception ex) { OnParseError(results.ContentString, ex); } return(releases); }
async Task<ReleaseInfo[]> PerformQuery(TorznabQuery query, string baseUrl) { List<ReleaseInfo> releases = new List<ReleaseInfo>(); string token = await GetToken(baseUrl); string searchUrl; if (query.RageID != 0) searchUrl = string.Format(baseUrl + SearchTVRageUrl, query.RageID, token); else searchUrl = string.Format(baseUrl + SearchQueryUrl, query.SearchTerm, token); var request = CreateHttpRequest(searchUrl); var response = await client.SendAsync(request); var results = await response.Content.ReadAsStringAsync(); try { var jItems = JArray.Parse(results); foreach (JObject item in jItems) { var release = new ReleaseInfo(); release.Title = (string)item["f"]; release.MagnetUri = new Uri((string)item["d"]); release.Guid = release.MagnetUri; release.PublishDate = new DateTime(1970, 1, 1); release.Size = 0; release.Seeders = 1; release.Peers = 1; release.MinimumRatio = 1; release.MinimumSeedTime = 172800; releases.Add(release); } } catch (Exception ex) { OnResultParsingError(this, results, ex); } return releases.ToArray(); }
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { var releases = new List <ReleaseInfo>(); var qc = new NameValueCollection { { "action", "basic" }, { "order_by", "time" }, { "order_way", "desc" }, { "searchtext", query.GetQueryString() } }; var searchUrl = SearchUrl + "?" + qc.GetQueryString(); var response = await RequestWithCookiesAsync(searchUrl); if (!response.ContentString.Contains("logout.php")) // re-login { await DoLogin(); response = await RequestWithCookiesAsync(searchUrl); } try { var parser = new HtmlParser(); var document = parser.ParseDocument(response.ContentString); var rows = document.QuerySelectorAll(".torrent_table > tbody > tr[class^='torrent row']"); foreach (var row in rows) { var title = row.QuerySelector("a[data-src]").GetAttribute("data-src"); if (string.IsNullOrEmpty(title) || title == "0") { title = row.QuerySelector("a[data-src]").TextContent; title = Regex.Replace(title, @"[\[\]\/]", ""); } else { if (title.Length > 5 && title.Substring(title.Length - 5).Contains(".")) { title = title.Remove(title.LastIndexOf(".", StringComparison.Ordinal)); } } var bannerStr = row.QuerySelector("img")?.GetAttribute("src"); var bannerUri = !string.IsNullOrWhiteSpace(bannerStr) ? new Uri(bannerStr) : null; var commentsUri = new Uri(SiteLink + row.QuerySelector("a[data-src]").GetAttribute("href")); var linkUri = new Uri(SiteLink + row.QuerySelector("a[href*='action=download']").GetAttribute("href")); var qColSize = row.QuerySelector("td:nth-child(3)"); var size = ReleaseInfo.GetBytes(qColSize.Children[0].TextContent); var files = ParseUtil.CoerceLong(qColSize.Children[1].TextContent.Split(':')[1].Trim()); var qPublishdate = row.QuerySelector("td:nth-child(4) span"); var publishDateStr = qPublishdate.GetAttribute("title"); var publishDate = !string.IsNullOrEmpty(publishDateStr) && publishDateStr.Contains(",") ? DateTime.ParseExact(publishDateStr, "MMM dd yyyy, HH:mm", CultureInfo.InvariantCulture) : DateTime.ParseExact(qPublishdate.TextContent.Trim(), "MMM dd yyyy, HH:mm", CultureInfo.InvariantCulture); var grabs = ParseUtil.CoerceLong(row.QuerySelector("td:nth-child(5)").TextContent); var seeds = ParseUtil.CoerceLong(row.QuerySelector("td:nth-child(6)").TextContent); var leechers = ParseUtil.CoerceLong(row.QuerySelector("td:nth-child(7)").TextContent); var release = new ReleaseInfo { Title = title, Guid = commentsUri, Comments = commentsUri, Link = linkUri, Category = new List <int> { TvCategoryParser.ParseTvShowQuality(title) }, Size = size, Files = files, PublishDate = publishDate, Grabs = grabs, Seeders = seeds, Peers = seeds + leechers, BannerUrl = bannerUri, MinimumRatio = 0, // ratioless MinimumSeedTime = 86400, // 24 hours DownloadVolumeFactor = 0, // ratioless tracker UploadVolumeFactor = 1 }; releases.Add(release); } } catch (Exception e) { OnParseError(response.ContentString, e); } return(releases); }
async Task<ReleaseInfo[]> PerformQuery(TorznabQuery query, string baseUrl) { List<ReleaseInfo> releases = new List<ReleaseInfo>(); foreach (var title in query.ShowTitles ?? new string[] { string.Empty }) { var searchString = title + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = string.Format(searchAllUrl); XmlDocument xmlDoc = new XmlDocument(); string xml = string.Empty; WebClient wc = getWebClient(); try { using (wc) { xml = wc.DownloadString(episodeSearchUrl); xmlDoc.LoadXml(xml); } ReleaseInfo release; string serie_title; foreach (XmlNode node in xmlDoc.GetElementsByTagName("item")) { release = new ReleaseInfo(); release.MinimumRatio = 1; release.MinimumSeedTime = 172800; serie_title = node.SelectSingleNode("title").InnerText; release.Title = serie_title; release.Comments = new Uri(node.SelectSingleNode("link").InnerText); release.Category = node.SelectSingleNode("title").InnerText; var test = node.SelectSingleNode("enclosure"); release.Guid = new Uri(test.Attributes["url"].Value); release.PublishDate = DateTime.Parse(node.SelectSingleNode("pubDate").InnerText, CultureInfo.InvariantCulture); release.Description = node.SelectSingleNode("description").InnerText; release.InfoHash = node.SelectSingleNode("description").InnerText; release.Size = 0; release.Seeders = 1; release.Peers = 1; release.MagnetUri = new Uri(node.SelectSingleNode("link").InnerText); releases.Add(release); } } catch (Exception ex) { OnResultParsingError(this, xml, ex); throw ex; } } return releases.ToArray(); }
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query) { var releases = new List <ReleaseInfo>(); // the order of the params is important! var qc = new List <string>(); var catList = MapTorznabCapsToTrackers(query); foreach (var cat in catList) { qc.Add(cat); } if (query.IsImdbQuery) { qc.Add("deep"); qc.Add("q"); qc.Add(query.ImdbID); } else { qc.Add("q"); qc.Add(WebUtilityHelpers.UrlEncode(query.GetQueryString(), Encoding)); } var searchUrl = SearchUrl + string.Join("/", qc); var response = await RequestWithCookiesAndRetryAsync(searchUrl); if (!response.ContentString.Contains("/logout.php")) // re-login { await DoLogin(); response = await RequestWithCookiesAndRetryAsync(searchUrl); } try { var parser = new HtmlParser(); var dom = parser.ParseDocument(response.ContentString); var rows = dom.QuerySelectorAll("div.boxContent > table > tbody > tr"); foreach (var row in rows) { var cells = row.QuerySelectorAll("td"); var title = row.QuerySelector("td[class='lft'] > div > a").TextContent.Trim(); var link = new Uri(SiteLink + row.QuerySelector("img[title='Download']").ParentElement.GetAttribute("href").TrimStart('/')); var comments = new Uri(SiteLink + row.QuerySelector("td[class='lft'] > div > a").GetAttribute("href").TrimStart('/')); var size = ReleaseInfo.GetBytes(cells[5].TextContent); var grabs = ParseUtil.CoerceInt(cells[6].TextContent); var seeders = ParseUtil.CoerceInt(cells[7].TextContent); var leechers = ParseUtil.CoerceInt(cells[8].TextContent); var pubDateStr = row.QuerySelector("span[class^='elapsedDate']").GetAttribute("title").Replace(" at", ""); var publishDate = DateTime.ParseExact(pubDateStr, "dddd, MMMM d, yyyy h:mmtt", CultureInfo.InvariantCulture); var cat = row.QuerySelector("a").GetAttribute("href").Split('/').Last(); var downloadVolumeFactor = row.QuerySelector("span:contains(\"[Freeleech]\")") != null ? 0 : 1; var release = new ReleaseInfo { Title = title, Link = link, Guid = link, Comments = comments, PublishDate = publishDate, Category = MapTrackerCatToNewznab(cat), Size = size, Grabs = grabs, Seeders = seeders, Peers = seeders + leechers, MinimumRatio = 1, MinimumSeedTime = 172800, // 48 hours DownloadVolumeFactor = downloadVolumeFactor, UploadVolumeFactor = 1 }; releases.Add(release); } } catch (Exception ex) { OnParseError(response.ContentString, ex); } return(releases); }
public async Task<ReleaseInfo[]> PerformQuery(TorznabQuery query) { List<ReleaseInfo> releases = new List<ReleaseInfo>(); foreach (var title in query.ShowTitles ?? new string[] { string.Empty }) { var searchString = title + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = string.Format(SearchUrl, HttpUtility.UrlEncode(searchString)); var results = await client.GetStringAsync(episodeSearchUrl); try { CQ dom = results; CQ qRows = dom["#torrenttable > tbody > tr"]; foreach (var row in qRows) { var release = new ReleaseInfo(); var qRow = row.Cq(); var debug = qRow.Html(); release.MinimumRatio = 1; release.MinimumSeedTime = 172800; CQ qLink = qRow.Find(".title > a").First(); release.Guid = new Uri(BaseUrl + qLink.Attr("href")); release.Comments = release.Guid; release.Title = qLink.Text(); release.Description = release.Title; release.Link = new Uri(BaseUrl + qRow.Find(".quickdownload > a").Attr("href")); var dateString = qRow.Find(".name").First()[0].ChildNodes[4].NodeValue.Replace(" on", "").Trim(); //"2015-04-25 23:38:12" //"yyyy-MMM-dd hh:mm:ss" release.PublishDate = DateTime.ParseExact(dateString, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture); var sizeStringParts = qRow.Children().ElementAt(4).InnerText.Split(' '); release.Size = ReleaseInfo.GetBytes(sizeStringParts[1], ParseUtil.CoerceFloat(sizeStringParts[0])); release.Seeders = ParseUtil.CoerceInt(qRow.Find(".seeders").Text()); release.Peers = release.Seeders + ParseUtil.CoerceInt(qRow.Find(".leechers").Text()); releases.Add(release); } } catch (Exception ex) { OnResultParsingError(this, results, ex); throw ex; } } return releases.ToArray(); }
public async Task<ReleaseInfo[]> PerformQuery(TorznabQuery query) { List<ReleaseInfo> releases = new List<ReleaseInfo>(); foreach (var title in query.ShowTitles ?? new string[] { string.Empty }) { var searchString = title + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = SearchUrl + HttpUtility.UrlEncode(searchString); var request = CreateHttpRequest(new Uri(episodeSearchUrl)); var response = await client.SendAsync(request); var results = await response.Content.ReadAsStringAsync(); try { CQ dom = results; var rows = dom["table.torrents > tbody > tr"]; foreach (var row in rows.Skip(1)) { var release = new ReleaseInfo(); var qRow = row.Cq(); var qTitleLink = qRow.Find("a.t_title").First(); release.Title = qTitleLink.Text().Trim(); release.Description = release.Title; release.Guid = new Uri(BaseUrl + qTitleLink.Attr("href")); release.Comments = release.Guid; DateTime pubDate; var descString = qRow.Find(".t_ctime").Text(); var dateString = descString.Split('|').Last().Trim(); dateString = dateString.Split(new string[] { " by " }, StringSplitOptions.None)[0]; var dateValue = ParseUtil.CoerceFloat(dateString.Split(' ')[0]); var dateUnit = dateString.Split(' ')[1]; if (dateUnit.Contains("minute")) pubDate = DateTime.Now - TimeSpan.FromMinutes(dateValue); else if (dateUnit.Contains("hour")) pubDate = DateTime.Now - TimeSpan.FromHours(dateValue); else if (dateUnit.Contains("day")) pubDate = DateTime.Now - TimeSpan.FromDays(dateValue); else if (dateUnit.Contains("week")) pubDate = DateTime.Now - TimeSpan.FromDays(7 * dateValue); else if (dateUnit.Contains("month")) pubDate = DateTime.Now - TimeSpan.FromDays(30 * dateValue); else if (dateUnit.Contains("year")) pubDate = DateTime.Now - TimeSpan.FromDays(365 * dateValue); else pubDate = DateTime.MinValue; release.PublishDate = pubDate; var qLink = row.ChildElements.ElementAt(3).Cq().Children("a"); release.Link = new Uri(BaseUrl + qLink.Attr("href")); var sizeStr = row.ChildElements.ElementAt(5).Cq().Text().Trim(); var sizeVal = ParseUtil.CoerceFloat(sizeStr.Split(' ')[0]); var sizeUnit = sizeStr.Split(' ')[1]; release.Size = ReleaseInfo.GetBytes(sizeUnit, sizeVal); release.Seeders = ParseUtil.CoerceInt(qRow.Find(".t_seeders").Text().Trim()); release.Peers = ParseUtil.CoerceInt(qRow.Find(".t_leechers").Text().Trim()) + release.Seeders; releases.Add(release); } } catch (Exception ex) { OnResultParsingError(this, results, ex); throw ex; } } return releases.ToArray(); }
public async Task<ReleaseInfo[]> PerformQuery(TorznabQuery query) { List<ReleaseInfo> releases = new List<ReleaseInfo>(); foreach (var title in query.ShowTitles ?? new string[] { string.Empty }) { var searchString = title + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = string.Format(SearchUrl, HttpUtility.UrlEncode(searchString)); var results = await client.GetStringAsync(episodeSearchUrl); try { CQ dom = results; var rows = dom["#torrentTable > tbody > tr.browse"]; foreach (var row in rows) { CQ qRow = row.Cq(); var release = new ReleaseInfo(); release.MinimumRatio = 1; release.MinimumSeedTime = 172800; release.Title = qRow.Find(".torrentName").Text(); release.Description = release.Title; release.Guid = new Uri(BaseUrl + "/" + qRow.Find(".torrentName").Attr("href")); release.Comments = release.Guid; release.Link = new Uri(BaseUrl + "/" + qRow.Find(".dlLinksInfo > a").Attr("href")); var sizeStr = qRow.Find(".sizeInfo").Text().Trim(); var sizeParts = sizeStr.Split(' '); release.Size = ReleaseInfo.GetBytes(sizeParts[1], ParseUtil.CoerceFloat(sizeParts[0])); var dateStr = qRow.Find(".ulInfo").Text().Split('|').Last().Trim(); var dateParts = dateStr.Split(' '); var dateValue = ParseUtil.CoerceInt(dateParts[0]); TimeSpan ts = TimeSpan.Zero; if (dateStr.Contains("sec")) ts = TimeSpan.FromSeconds(dateValue); else if (dateStr.Contains("min")) ts = TimeSpan.FromMinutes(dateValue); else if (dateStr.Contains("hour")) ts = TimeSpan.FromHours(dateValue); else if (dateStr.Contains("day")) ts = TimeSpan.FromDays(dateValue); else if (dateStr.Contains("week")) ts = TimeSpan.FromDays(dateValue * 7); else if (dateStr.Contains("month")) ts = TimeSpan.FromDays(dateValue * 30); else if (dateStr.Contains("year")) ts = TimeSpan.FromDays(dateValue * 365); release.PublishDate = DateTime.Now - ts; release.Seeders = ParseUtil.CoerceInt(qRow.Find(".seedersInfo").Text()); release.Peers = ParseUtil.CoerceInt(qRow.Find(".leechersInfo").Text()) + release.Seeders; releases.Add(release); } } catch (Exception ex) { OnResultParsingError(this, results, ex); throw ex; } } return releases.ToArray(); }
async Task<ReleaseInfo[]> PerformQuery(TorznabQuery query, string baseUrl) { List<ReleaseInfo> releases = new List<ReleaseInfo>(); List<string> searchUrls = new List<string>(); foreach (var title in query.ShowTitles ?? new string[] { string.Empty }) { var searchString = title + " " + query.GetEpisodeSearchString(); var queryStr = HttpUtility.UrlEncode(searchString); var episodeSearchUrl = baseUrl + string.Format(SearchUrl, queryStr); searchUrls.Add(episodeSearchUrl); } foreach (var episodeSearchUrl in searchUrls) { string results; if (Program.IsWindows) { results = await client.GetStringAsync(episodeSearchUrl); } else { var response = await CurlHelper.GetAsync(episodeSearchUrl, null, episodeSearchUrl); results = Encoding.UTF8.GetString(response.Content); } try { CQ dom = results; var rows = dom["#searchResult > tbody > tr"]; foreach (var row in rows) { var release = new ReleaseInfo(); CQ qRow = row.Cq(); CQ qLink = qRow.Find(".detName > .detLink").First(); release.MinimumRatio = 1; release.MinimumSeedTime = 172800; release.Title = qLink.Text().Trim(); release.Description = release.Title; release.Comments = new Uri(baseUrl + "/" + qLink.Attr("href").TrimStart('/')); release.Guid = release.Comments; var downloadCol = row.ChildElements.ElementAt(1).Cq().Children("a"); release.MagnetUri = new Uri(downloadCol.Attr("href")); release.InfoHash = release.MagnetUri.ToString().Split(':')[3].Split('&')[0]; var descString = qRow.Find(".detDesc").Text().Trim(); var descParts = descString.Split(','); var timeString = descParts[0].Split(' ')[1]; if (timeString.Contains("mins ago")) { release.PublishDate = (DateTime.Now - TimeSpan.FromMinutes(ParseUtil.CoerceInt(timeString.Split(' ')[0]))); } else if (timeString.Contains("Today")) { release.PublishDate = (DateTime.UtcNow - TimeSpan.FromHours(2) - TimeSpan.Parse(timeString.Split(' ')[1])).ToLocalTime(); } else if (timeString.Contains("Y-day")) { release.PublishDate = (DateTime.UtcNow - TimeSpan.FromHours(26) - TimeSpan.Parse(timeString.Split(' ')[1])).ToLocalTime(); } else if (timeString.Contains(':')) { var utc = DateTime.ParseExact(timeString, "MM-dd HH:mm", CultureInfo.InvariantCulture) - TimeSpan.FromHours(2); release.PublishDate = DateTime.SpecifyKind(utc, DateTimeKind.Utc).ToLocalTime(); } else { var utc = DateTime.ParseExact(timeString, "MM-dd yyyy", CultureInfo.InvariantCulture) - TimeSpan.FromHours(2); release.PublishDate = DateTime.SpecifyKind(utc, DateTimeKind.Utc).ToLocalTime(); } var sizeParts = descParts[1].Split(new char[] { ' ', ' ' }, StringSplitOptions.RemoveEmptyEntries); var sizeVal = ParseUtil.CoerceFloat(sizeParts[1]); var sizeUnit = sizeParts[2]; release.Size = ReleaseInfo.GetBytes(sizeUnit, sizeVal); release.Seeders = ParseUtil.CoerceInt(row.ChildElements.ElementAt(2).Cq().Text()); release.Peers = ParseUtil.CoerceInt(row.ChildElements.ElementAt(3).Cq().Text()) + release.Seeders; releases.Add(release); } } catch (Exception ex) { OnResultParsingError(this, results, ex); throw ex; } } return releases.ToArray(); }