protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var qc = new NameValueCollection
{
{ "incldead", "1" },
{ "showspam", "1" },
{ "cat", MapTorznabCapsToTrackers(query).FirstIfSingleOrDefault("0") }
};
if (query.IsImdbQuery)
{
qc.Add("search", query.ImdbID);
qc.Add("s_desc", "1");
}
else
{
qc.Add("search", query.GetQueryString());
}
var searchUrl = SearchUrl + "?" + qc.GetQueryString();
var results = await RequestWithCookiesAndRetryAsync(searchUrl);
if (results.IsRedirect) // re-login
{
await ApplyConfiguration(null);
results = await RequestWithCookiesAndRetryAsync(searchUrl);
}
try
{
var parser = new HtmlParser();
var dom = parser.ParseDocument(results.ContentString);
var rows = dom.QuerySelectorAll("table[cellpadding=2] > tbody > tr:has(td.row3)");
foreach (var row in rows)
{
var qDownloadLink = row.QuerySelector("a[href^=\"download.php\"]");
if (qDownloadLink == null)
{
throw new Exception("Download links not found. Make sure you can download from the website.");
}
var link = new Uri(SiteLink + qDownloadLink.GetAttribute("href"));
var qDetailsLink = row.QuerySelector("a[href^=\"details.php?id=\"]");
var title = qDetailsLink.GetAttribute("title").Trim();
var comments = new Uri(SiteLink + qDetailsLink.GetAttribute("href"));
var qCatLink = row.QuerySelector("a[href^=\"browse.php?cat=\"]");
var catStr = qCatLink.GetAttribute("href").Split('=')[1].Split('&')[0];
var files = ParseUtil.CoerceInt(row.Children[3].TextContent);
var publishDate = DateTimeUtil.FromTimeAgo(row.Children[5].TextContent);
var size = ReleaseInfo.GetBytes(row.Children[7].TextContent);
var grabs = ParseUtil.CoerceInt(row.Children[8].TextContent);
var seeders = ParseUtil.CoerceInt(row.Children[9].TextContent);
var leechers = ParseUtil.CoerceInt(row.Children[10].TextContent);
var ka = row.NextElementSibling.QuerySelector("table > tbody > tr:nth-child(3)");
var ulFactor = ParseUtil.CoerceDouble(ka.Children[0].TextContent.Replace("X", ""));
var dlFactor = ParseUtil.CoerceDouble(ka.Children[1].TextContent.Replace("X", ""));
var release = new ReleaseInfo
{
Title = title,
Comments = comments,
Link = link,
Guid = link,
Category = MapTrackerCatToNewznab(catStr),
PublishDate = publishDate,
Size = size,
Files = files,
Grabs = grabs,
Seeders = seeders,
Peers = leechers + seeders,
MinimumRatio = 1,
MinimumSeedTime = 172800, // 48 hours
DownloadVolumeFactor = dlFactor,
UploadVolumeFactor = ulFactor
};
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError(results.ContentString, ex);
}
return(releases);
}
public async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
searchBlock Search = Definition.Search;
// init template context
var variables = getTemplateVariablesFromConfigData();
variables[".Query.Type"] = query.QueryType;
variables[".Query.Q"] = query.SearchTerm;
variables[".Query.Series"] = null;
variables[".Query.Ep"] = query.Episode;
variables[".Query.Season"] = query.Season;
variables[".Query.Movie"] = null;
variables[".Query.Year"] = null;
variables[".Query.Limit"] = query.Limit;
variables[".Query.Offset"] = query.Offset;
variables[".Query.Extended"] = query.Extended;
variables[".Query.Categories"] = query.Categories;
variables[".Query.APIKey"] = query.ApiKey;
variables[".Query.TVDBID"] = null;
variables[".Query.TVRageID"] = query.RageID;
variables[".Query.IMDBID"] = query.ImdbID;
variables[".Query.TVMazeID"] = null;
variables[".Query.TraktID"] = null;
variables[".Query.Episode"] = query.GetEpisodeSearchString();
variables[".Categories"] = MapTorznabCapsToTrackers(query);
var KeywordTokens = new List <string>();
var KeywordTokenKeys = new List <string> {
"Q", "Series", "Movie", "Year"
};
foreach (var key in KeywordTokenKeys)
{
var Value = (string)variables[".Query." + key];
if (!string.IsNullOrWhiteSpace(Value))
{
KeywordTokens.Add(Value);
}
}
if (!string.IsNullOrWhiteSpace((string)variables[".Query.Episode"]))
{
KeywordTokens.Add((string)variables[".Query.Episode"]);
}
variables[".Query.Keywords"] = string.Join(" ", KeywordTokens);
variables[".Keywords"] = variables[".Query.Keywords"];
// build search URL
var searchUrl = SiteLink + applyGoTemplateText(Search.Path, variables) + "?";
var queryCollection = new NameValueCollection();
if (Search.Inputs != null)
{
foreach (var Input in Search.Inputs)
{
var value = applyGoTemplateText(Input.Value, variables);
if (Input.Key == "$raw")
{
searchUrl += value;
}
else
{
queryCollection.Add(Input.Key, value);
}
}
}
searchUrl += "&" + queryCollection.GetQueryString();
// send HTTP request
var response = await RequestBytesWithCookies(searchUrl);
var results = Encoding.GetEncoding("iso-8859-1").GetString(response.Content);
try
{
var SearchResultParser = new HtmlParser();
var SearchResultDocument = SearchResultParser.Parse(results);
var RowsDom = SearchResultDocument.QuerySelectorAll(Search.Rows.Selector);
List <IElement> Rows = new List <IElement>();
foreach (var RowDom in RowsDom)
{
Rows.Add(RowDom);
}
// merge following rows for After selector
var After = Definition.Search.Rows.After;
if (After > 0)
{
for (int i = 0; i < Rows.Count; i += 1)
{
var CurrentRow = Rows[i];
for (int j = 0; j < After; j += 1)
{
var MergeRowIndex = i + j + 1;
var MergeRow = Rows[MergeRowIndex];
List <INode> MergeNodes = new List <INode>();
foreach (var node in MergeRow.QuerySelectorAll("td"))
{
MergeNodes.Add(node);
}
CurrentRow.Append(MergeNodes.ToArray());
}
Rows.RemoveRange(i + 1, After);
}
}
foreach (var Row in Rows)
{
try
{
var release = new ReleaseInfo();
release.MinimumRatio = 1;
release.MinimumSeedTime = 48 * 60 * 60;
// Parse fields
foreach (var Field in Search.Fields)
{
string value = handleSelector(Field.Value, Row);
value = ParseUtil.NormalizeSpace(value);
try
{
switch (Field.Key)
{
case "download":
if (value.StartsWith("magnet:"))
{
release.MagnetUri = new Uri(value);
release.Link = release.MagnetUri;
}
else
{
release.Link = resolvePath(value);
}
break;
case "details":
var url = resolvePath(value);
release.Guid = url;
if (release.Comments == null)
{
release.Comments = url;
}
break;
case "comments":
var CommentsUrl = resolvePath(value);
release.Comments = CommentsUrl;
if (release.Guid == null)
{
release.Guid = CommentsUrl;
}
break;
case "title":
release.Title = value;
break;
case "description":
release.Description = value;
break;
case "category":
release.Category = MapTrackerCatToNewznab(value);
break;
case "size":
release.Size = ReleaseInfo.GetBytes(value);
break;
case "leechers":
if (release.Peers == null)
{
release.Peers = ParseUtil.CoerceInt(value);
}
else
{
release.Peers += ParseUtil.CoerceInt(value);
}
break;
case "seeders":
release.Seeders = ParseUtil.CoerceInt(value);
if (release.Peers == null)
{
release.Peers = release.Seeders;
}
else
{
release.Peers += release.Seeders;
}
break;
case "date":
release.PublishDate = DateTimeUtil.FromUnknown(value);
break;
case "files":
release.Files = ParseUtil.CoerceLong(value);
break;
case "grabs":
release.Grabs = ParseUtil.CoerceLong(value);
break;
case "downloadvolumefactor":
release.DownloadVolumeFactor = ParseUtil.CoerceDouble(value);
break;
case "uploadvolumefactor":
release.UploadVolumeFactor = ParseUtil.CoerceDouble(value);
break;
default:
break;
}
}
catch (Exception ex)
{
throw new Exception(string.Format("Error while parsing field={0}, selector={1}, value={2}: {3}", Field.Key, Field.Value.Selector, value, ex.Message));
}
}
// if DateHeaders is set go through the previous rows and look for the header selector
var DateHeaders = Definition.Search.Rows.Dateheaders;
if (DateHeaders != null)
{
var PrevRow = Row.PreviousElementSibling;
string value = null;
while (PrevRow != null)
{
try
{
value = handleSelector(DateHeaders, PrevRow);
break;
}
catch (Exception ex)
{
// do nothing
}
PrevRow = PrevRow.PreviousElementSibling;
}
if (value == null)
{
throw new Exception(string.Format("No date header row found for {0}", release.ToString()));
}
release.PublishDate = DateTimeUtil.FromUnknown(value);
}
releases.Add(release);
}
catch (Exception ex)
{
logger.Error(string.Format("CardigannIndexer ({0}): Error while parsing row '{1}': {2}", ID, Row.OuterHtml, ex));
}
}
}
catch (Exception ex)
{
OnParseError(results, ex);
}
return(releases);
}
public async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var searchString = query.GetQueryString();
var searchUrl = SearchUrl;
var queryCollection = new NameValueCollection();
queryCollection.Add("total", "146"); // Not sure what this is about but its required!
var cat = "0";
var queryCats = MapTorznabCapsToTrackers(query);
if (queryCats.Count == 1)
{
cat = queryCats.First().ToString();
}
queryCollection.Add("cat", cat);
queryCollection.Add("searchin", "filename");
queryCollection.Add("search", searchString);
queryCollection.Add("page", "1");
searchUrl += "?" + queryCollection.GetQueryString();
var extraHeaders = new Dictionary <string, string>()
{
{ "X-Requested-With", "XMLHttpRequest" }
};
var response = await RequestStringWithCookiesAndRetry(searchUrl, null, SearchUrlReferer, extraHeaders);
var results = response.Content;
try
{
CQ dom = results;
var rows = dom["tr"];
foreach (var row in rows.Skip(1))
{
var release = new ReleaseInfo();
var qRow = row.Cq();
var qTitleLink = qRow.Find("td:eq(1) a:eq(0)").First();
release.Title = qTitleLink.Find("strong").Text().Trim();
// If we search an get no results, we still get a table just with no info.
if (string.IsNullOrWhiteSpace(release.Title))
{
break;
}
release.Description = release.Title;
release.Guid = new Uri(qTitleLink.Attr("href"));
release.Comments = release.Guid;
var dateString = qRow.Find("td:eq(4)").Text();
release.PublishDate = DateTime.ParseExact(dateString, "dd MMM yy", CultureInfo.InvariantCulture);
var qLink = qRow.Find("td:eq(2) a");
if (qLink.Length != 0) // newbie users don't see DL links
{
release.Link = new Uri(qLink.Attr("href"));
}
else
{
// use comments link as placeholder
// null causes errors during export to torznab
// skipping the release prevents newbie users from adding the tracker (empty result)
release.Link = release.Comments;
}
var sizeStr = qRow.Find("td:eq(5)").Text();
release.Size = ReleaseInfo.GetBytes(sizeStr);
var connections = qRow.Find("td:eq(7)").Text().Trim().Split("/".ToCharArray(), StringSplitOptions.RemoveEmptyEntries);
release.Seeders = ParseUtil.CoerceInt(connections[0].Trim());
release.Peers = ParseUtil.CoerceInt(connections[1].Trim()) + release.Seeders;
release.Grabs = ParseUtil.CoerceLong(connections[2].Trim());
var rCat = row.Cq().Find("td:eq(0) a").First().Attr("href");
var rCatIdx = rCat.IndexOf("cat=");
if (rCatIdx > -1)
{
rCat = rCat.Substring(rCatIdx + 4);
}
release.Category = MapTrackerCatToNewznab(rCat);
if (qRow.Find("img[alt=\"Gold Torrent\"]").Length >= 1)
{
release.DownloadVolumeFactor = 0;
}
else if (qRow.Find("img[alt=\"Silver Torrent\"]").Length >= 1)
{
release.DownloadVolumeFactor = 0.5;
}
else
{
release.DownloadVolumeFactor = 1;
}
var ULFactorImg = qRow.Find("img[alt*=\"x Multiplier Torrent\"]");
if (ULFactorImg.Length >= 1)
{
release.UploadVolumeFactor = ParseUtil.CoerceDouble(ULFactorImg.Attr("alt").Split('x')[0]);
}
else
{
release.UploadVolumeFactor = 1;
}
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError(results, ex);
}
return(releases);
}
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var searchString = query.GetQueryString();
WebClientStringResult results = null;
var queryCollection = new NameValueCollection();
queryCollection.Add("act", "search");
queryCollection.Add("forums", "all");
queryCollection.Add("torrents", "1");
queryCollection.Add("search_in", "titles");
queryCollection.Add("result_type", "topics");
// if the search string is empty use the getnew view
if (string.IsNullOrWhiteSpace(searchString))
{
queryCollection.Add("CODE", "getnew");
queryCollection.Add("active", "1");
}
else // use the normal search
{
searchString = searchString.Replace("-", " ");
queryCollection.Add("CODE", "01");
queryCollection.Add("keywords", searchString);
}
var searchUrl = IndexUrl + "?" + queryCollection.GetQueryString();
results = await RequestStringWithCookies(searchUrl);
if (results.IsRedirect && results.RedirectingTo.Contains("CODE=show"))
{
results = await RequestStringWithCookies(results.RedirectingTo);
}
try
{
string RowsSelector = "div.borderwrap:has(div.maintitle) > table > tbody > tr:has(a[href*=\"index.php?showtopic=\"])";
var SearchResultParser = new HtmlParser();
var SearchResultDocument = SearchResultParser.ParseDocument(results.Content);
var Rows = SearchResultDocument.QuerySelectorAll(RowsSelector);
foreach (var Row in Rows)
{
try
{
var release = new ReleaseInfo();
var StatsElements = Row.QuerySelector("td:nth-child(5)");
var stats = StatsElements.TextContent.Split('·');
if (stats.Length != 3) // not a torrent
{
continue;
}
release.Seeders = ParseUtil.CoerceInt(stats[0]);
release.Peers = ParseUtil.CoerceInt(stats[1]) + release.Seeders;
release.Grabs = ParseUtil.CoerceInt(stats[2]);
release.MinimumRatio = 0.51;
release.MinimumSeedTime = 0;
var qDetailsLink = Row.QuerySelector("a[onmouseover][href*=\"index.php?showtopic=\"]");
release.Title = qDetailsLink.TextContent;
release.Comments = new Uri(qDetailsLink.GetAttribute("href"));
release.Link = release.Comments;
release.Guid = release.Link;
release.DownloadVolumeFactor = 1;
release.UploadVolumeFactor = 1;
//var id = QueryHelpers.ParseQuery(release.Comments.Query)["showtopic"].FirstOrDefault();
var id = HttpUtility.ParseQueryString(release.Comments.Query)["showtopic"];
var desc = Row.QuerySelector("span.desc");
var forange = desc.QuerySelector("font.forange");
if (forange != null)
{
var DownloadVolumeFactor = forange.QuerySelector("i:contains(\"freeleech\")");
if (DownloadVolumeFactor != null)
{
release.DownloadVolumeFactor = 0;
}
var UploadVolumeFactor = forange.QuerySelector("i:contains(\"x upload]\")");
if (UploadVolumeFactor != null)
{
release.UploadVolumeFactor = ParseUtil.CoerceDouble(UploadVolumeFactor.TextContent.Split(' ')[0].Substring(1).Replace("x", ""));
}
forange.Remove();
}
var format = desc.TextContent;
release.Title += " [" + format + "]";
var preview = SearchResultDocument.QuerySelector("div#d21-tph-preview-data-" + id);
if (preview != null)
{
release.Description = "";
foreach (var e in preview.ChildNodes)
{
if (e.NodeType == NodeType.Text)
{
release.Description += e.NodeValue;
}
else
{
release.Description += e.TextContent + "\n";
}
}
}
release.Description = WebUtility.HtmlEncode(release.Description.Trim());
release.Description = release.Description.Replace("\n", "<br>");
if (format.Contains("MP3"))
{
release.Category = new List <int> {
TorznabCatType.AudioMP3.ID
}
}
;
else if (format.Contains("AAC"))
{
release.Category = new List <int> {
TorznabCatType.AudioOther.ID
}
}
;
else if (format.Contains("Lossless"))
{
release.Category = new List <int> {
TorznabCatType.AudioLossless.ID
}
}
;
else
{
release.Category = new List <int> {
TorznabCatType.AudioOther.ID
}
};
var lastAction = Row.QuerySelector("td:nth-child(9) > span").FirstChild.NodeValue;
release.PublishDate = DateTimeUtil.FromUnknown(lastAction, "UK");
releases.Add(release);
}
catch (Exception ex)
{
logger.Error(string.Format("{0}: Error while parsing row '{1}':\n\n{2}", ID, Row.OuterHtml, ex));
}
}
}
catch (Exception ex)
{
OnParseError(results.Content, ex);
}
return(releases);
}
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
List <ReleaseInfo> releases = new List <ReleaseInfo>();
var searchString = query.GetQueryString();
var searchUrl = SearchUrl;
var queryCollection = new NameValueCollection();
queryCollection.Add("incldead", "1");
queryCollection.Add("showspam", "1");
if (!string.IsNullOrWhiteSpace(searchString))
{
queryCollection.Add("search", searchString);
}
var cats = MapTorznabCapsToTrackers(query);
string cat = "0";
if (cats.Count == 1)
{
cat = cats[0];
}
queryCollection.Add("cat", cat);
searchUrl += "?" + queryCollection.GetQueryString();
var results = await RequestStringWithCookiesAndRetry(searchUrl);
// Occasionally the cookies become invalid, login again if that happens
if (results.IsRedirect)
{
await ApplyConfiguration(null);
results = await RequestStringWithCookiesAndRetry(searchUrl);
}
try
{
CQ dom = results.Content;
var rows = dom["table[cellpadding=2] > tbody > tr:has(td.row3)"];
foreach (var row in rows)
{
var release = new ReleaseInfo();
release.MinimumRatio = 1;
release.MinimumSeedTime = 48 * 60 * 60;
var qRow = row.Cq();
var qCatLink = qRow.Find("a[href^=browse.php?cat=]").First();
var qDetailsLink = qRow.Find("a[href^=details.php?id=]").First();
var qSeeders = qRow.Find("td:eq(9)");
var qLeechers = qRow.Find("td:eq(10)");
var qDownloadLink = qRow.Find("a[href^=download.php]").First();
var qTimeAgo = qRow.Find("td:eq(5)");
var qSize = qRow.Find("td:eq(7)");
var catStr = qCatLink.Attr("href").Split('=')[1].Split('&')[0];
release.Category = MapTrackerCatToNewznab(catStr);
release.Link = new Uri(SiteLink + qDownloadLink.Attr("href"));
release.Title = qDetailsLink.Attr("title").Trim();
release.Comments = new Uri(SiteLink + qDetailsLink.Attr("href"));
release.Guid = release.Link;
var sizeStr = qSize.Text();
release.Size = ReleaseInfo.GetBytes(sizeStr);
release.Seeders = ParseUtil.CoerceInt(qSeeders.Text());
release.Peers = ParseUtil.CoerceInt(qLeechers.Text()) + release.Seeders;
var dateStr = qTimeAgo.Text();
release.PublishDate = DateTimeUtil.FromTimeAgo(dateStr);
var files = qRow.Find("td:nth-child(4)").Text();
release.Files = ParseUtil.CoerceInt(files);
var grabs = qRow.Find("td:nth-child(9)").Text();
release.Grabs = ParseUtil.CoerceInt(grabs);
var ka = qRow.Next();
var DLFactor = ka.Find("table > tbody > tr:nth-child(3) > td:nth-child(2)").Text().Replace("X", "");
var ULFactor = ka.Find("table > tbody > tr:nth-child(3) > td:nth-child(1)").Text().Replace("X", "");
release.DownloadVolumeFactor = ParseUtil.CoerceDouble(DLFactor);
release.UploadVolumeFactor = ParseUtil.CoerceDouble(ULFactor);
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError(results.Content, ex);
}
return(releases);
}
public async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var queryString = query.GetQueryString();
var url = TorrentsUrl;
WebClientStringResult results = null;
var pairs = new Dictionary <string, string> {
{ "portlet", "true" }
};
if (!string.IsNullOrWhiteSpace(queryString))
{
pairs.Add("search", queryString);
results = await PostDataWithCookiesAndRetry(SearchUrl, pairs, null, TorrentsUrl);
}
else
{
results = await PostDataWithCookiesAndRetry(TorrentsUrl, pairs, null, TorrentsUrl);
}
try
{
CQ dom = results.Content;
var rows = dom["#torrent-table tr"];
if (!string.IsNullOrWhiteSpace(queryString))
{
rows = dom["table tr"];
}
foreach (var row in rows.Skip(1))
{
var release = new ReleaseInfo();
var qRow = row.Cq();
var titleRow = qRow.Find("td:eq(2)").First();
titleRow.Children().Remove();
release.Title = titleRow.Text().Trim();
if (string.IsNullOrWhiteSpace(release.Title))
{
continue;
}
release.Description = release.Title;
var qLink = row.Cq().Find("td:eq(4) a:eq(0)");
release.Link = new Uri(SiteLink + qLink.Attr("href"));
release.Guid = release.Link;
var qLinkComm = row.Cq().Find("td:eq(4) a:eq(1)");
release.Comments = new Uri(SiteLink + qLinkComm.Attr("href"));
var dateString = qRow.Find(".datetime").Attr("data-timestamp");
release.PublishDate = DateTimeUtil.UnixTimestampToDateTime(ParseUtil.CoerceDouble(dateString));
var infoString = row.Cq().Find("td:eq(3)").Text();
release.Size = ParseUtil.CoerceLong(Regex.Match(infoString, "\\((\\d+)\\)").Value.Replace("(", "").Replace(")", ""));
var infosplit = infoString.Replace("/", string.Empty).Split(":".ToCharArray());
release.Seeders = ParseUtil.CoerceInt(infosplit[1]);
release.Peers = release.Seeders + ParseUtil.CoerceInt(infosplit[2]);
// var tags = row.Cq().Find(".label-tag").Text(); These don't see to parse - bad tags?
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError(results.Content, ex);
}
/* else
* {
* var rssUrl = SiteLink + "rss/recent?passkey=" + configData.RSSKey.Value;
*
* results = await RequestStringWithCookiesAndRetry(rssUrl);
* try
* {
* var doc = XDocument.Parse(results.Content);
* foreach (var result in doc.Descendants("item"))
* {
* var xTitle = result.Element("title").Value;
* var xLink = result.Element("link").Value;
* var xGUID = result.Element("guid").Value;
* var xDesc = result.Element("description").Value;
* var xDate = result.Element("pubDate").Value;
* var release = new ReleaseInfo();
* release.Guid =release.Link = new Uri(xLink);
* release.MinimumRatio = 1;
* release.Seeders = 1; // We are not supplied with peer info so just mark it as one.
* foreach (var element in xDesc.Split(";".ToCharArray()))
* {
* var split = element.IndexOf(':');
* if (split > -1)
* {
* var key = element.Substring(0, split).Trim();
* var value = element.Substring(split+1).Trim();
*
* switch (key)
* {
* case "Filename":
* release.Title = release.Description = value;
* break;
* }
* }
* }
*
* //"Thu, 24 Sep 2015 18:07:07 +0000"
* release.PublishDate = DateTime.ParseExact(xDate, "ddd, dd MMM yyyy HH:mm:ss +0000", CultureInfo.InvariantCulture);
*
* if (!string.IsNullOrWhiteSpace(release.Title))
* {
* releases.Add(release);
* }
* }
* }
* catch (Exception ex)
* {
* OnParseError(results.Content, ex);
* }*/
foreach (var release in releases)
{
if (release.Title.Contains("1080p") || release.Title.Contains("720p"))
{
release.Category = TorznabCatType.TVHD.ID;
}
else
{
release.Category = TorznabCatType.TVSD.ID;
}
}
return(releases);
}
public void should_parse_double_from_string(string original, double parsedInt)
{
ParseUtil.CoerceDouble(original).Should().Be(parsedInt);
}
public async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var queryString = query.GetQueryString();
var url = TorrentsUrl;
WebClientStringResult results = null;
var searchUrls = new List <string>();
if (!string.IsNullOrWhiteSpace(query.SanitizedSearchTerm))
{
var pairs = new Dictionary <string, string>();
pairs.Add("search", query.SanitizedSearchTerm);
results = await PostDataWithCookiesAndRetry(SearchUrl, pairs, null, TorrentsUrl);
results = await ReloginIfNecessary(results);
CQ dom = results.Content;
var shows = dom.Find("div.show[data-id]");
foreach (var show in shows)
{
var showUrl = ShowUrl + show.GetAttribute("data-id");
searchUrls.Add(showUrl);
}
}
else
{
searchUrls.Add(TorrentsUrl);
}
try
{
foreach (var searchUrl in searchUrls)
{
results = await RequestStringWithCookies(searchUrl);
results = await ReloginIfNecessary(results);
CQ dom = results.Content;
var rows = dom["#torrent-table tr"];
if (!string.IsNullOrWhiteSpace(queryString))
{
rows = dom["table tr"];
}
var globalFreeleech = dom.Find("span:contains(\"Freeleech until:\"):has(span.datetime)").Any();
foreach (var row in rows.Skip(1))
{
var release = new ReleaseInfo();
var qRow = row.Cq();
var titleRow = qRow.Find("td:eq(2)").First();
titleRow.Children().Remove();
release.Title = titleRow.Text().Trim();
if ((query.ImdbID == null || !TorznabCaps.SupportsImdbSearch) && !query.MatchQueryStringAND(release.Title))
{
continue;
}
var qBanner = qRow.Find("div[style^=\"cursor: pointer; background-image:url\"]");
var qBannerStyle = qBanner.Attr("style");
if (!string.IsNullOrEmpty(qBannerStyle))
{
var bannerImg = Regex.Match(qBannerStyle, @"url\('(.*?)'\);").Groups[1].Value;
release.BannerUrl = new Uri(SiteLink + bannerImg);
}
var qLink = row.Cq().Find("td:eq(4) a:eq(0)");
release.Link = new Uri(SiteLink + qLink.Attr("href"));
release.Guid = release.Link;
var qLinkComm = row.Cq().Find("td:eq(4) a:eq(1)");
release.Comments = new Uri(SiteLink + qLinkComm.Attr("href"));
var dateString = qRow.Find(".datetime").Attr("data-timestamp");
if (dateString != null)
{
release.PublishDate = DateTimeUtil.UnixTimestampToDateTime(ParseUtil.CoerceDouble(dateString)).ToLocalTime();
}
var infoString = row.Cq().Find("td:eq(3)").Text();
release.Size = ParseUtil.CoerceLong(Regex.Match(infoString, "\\((\\d+)\\)").Value.Replace("(", "").Replace(")", ""));
var infosplit = infoString.Replace("/", string.Empty).Split(":".ToCharArray());
release.Seeders = ParseUtil.CoerceInt(infosplit[1]);
release.Peers = release.Seeders + ParseUtil.CoerceInt(infosplit[2]);
if (globalFreeleech)
{
release.DownloadVolumeFactor = 0;
}
else
{
release.DownloadVolumeFactor = 1;
}
release.UploadVolumeFactor = 1;
// var tags = row.Cq().Find(".label-tag").Text(); These don't see to parse - bad tags?
releases.Add(release);
}
}
}
catch (Exception ex)
{
OnParseError(results.Content, ex);
}
/* else
* {
* var rssUrl = SiteLink + "rss/recent?passkey=" + configData.RSSKey.Value;
*
* results = await RequestStringWithCookiesAndRetry(rssUrl);
* try
* {
* var doc = XDocument.Parse(results.Content);
* foreach (var result in doc.Descendants("item"))
* {
* var xTitle = result.Element("title").Value;
* var xLink = result.Element("link").Value;
* var xGUID = result.Element("guid").Value;
* var xDesc = result.Element("description").Value;
* var xDate = result.Element("pubDate").Value;
* var release = new ReleaseInfo();
* release.Guid =release.Link = new Uri(xLink);
* release.MinimumRatio = 1;
* release.Seeders = 1; // We are not supplied with peer info so just mark it as one.
* foreach (var element in xDesc.Split(";".ToCharArray()))
* {
* var split = element.IndexOf(':');
* if (split > -1)
* {
* var key = element.Substring(0, split).Trim();
* var value = element.Substring(split+1).Trim();
*
* switch (key)
* {
* case "Filename":
* release.Title = release.Description = value;
* break;
* }
* }
* }
*
* //"Thu, 24 Sep 2015 18:07:07 +0000"
* release.PublishDate = DateTime.ParseExact(xDate, "ddd, dd MMM yyyy HH:mm:ss +0000", CultureInfo.InvariantCulture);
*
* if (!string.IsNullOrWhiteSpace(release.Title))
* {
* releases.Add(release);
* }
* }
* }
* catch (Exception ex)
* {
* OnParseError(results.Content, ex);
* }*/
foreach (var release in releases)
{
if (release.Title.Contains("1080p") || release.Title.Contains("720p"))
{
release.Category = new List <int> {
TorznabCatType.TVHD.ID
};
}
else
{
release.Category = new List <int> {
TorznabCatType.TVSD.ID
};
}
}
return(releases);
}
private string ParseFields(string value, string fieldName, TorrentInfo release, List <string> fieldModifiers, Uri searchUrlUri)
{
switch (fieldName)
{
case "download":
if (string.IsNullOrEmpty(value))
{
value = null;
release.DownloadUrl = null;
break;
}
if (value.StartsWith("magnet:"))
{
release.MagnetUrl = value;
value = release.MagnetUrl;
}
else
{
release.DownloadUrl = ResolvePath(value, searchUrlUri).AbsoluteUri;
value = release.DownloadUrl;
}
release.Guid = value;
break;
case "magnet":
var magnetUri = value;
release.MagnetUrl = magnetUri;
value = magnetUri.ToString();
break;
case "infohash":
release.InfoHash = value;
break;
case "details":
var url = ResolvePath(value, searchUrlUri)?.AbsoluteUri;
release.InfoUrl = url;
value = url.ToString();
break;
case "comments":
var commentsUrl = ResolvePath(value, searchUrlUri);
if (release.CommentUrl == null)
{
release.CommentUrl = commentsUrl.AbsoluteUri;
}
value = commentsUrl.ToString();
break;
case "title":
if (fieldModifiers.Contains("append"))
{
release.Title += value;
}
else
{
release.Title = value;
}
value = release.Title;
break;
case "description":
if (fieldModifiers.Contains("append"))
{
release.Description += value;
}
else
{
release.Description = value;
}
value = release.Description;
break;
case "category":
var cats = MapTrackerCatToNewznab(value);
if (cats.Any())
{
if (release.Categories == null || fieldModifiers.Contains("noappend"))
{
release.Categories = cats;
}
else
{
release.Categories = release.Categories.Union(cats).ToList();
}
}
value = release.Categories.ToString();
break;
case "categorydesc":
var catsDesc = MapTrackerCatDescToNewznab(value);
if (catsDesc.Any())
{
if (release.Categories == null || fieldModifiers.Contains("noappend"))
{
release.Categories = catsDesc;
}
else
{
release.Categories = release.Categories.Union(catsDesc).ToList();
}
}
value = release.Categories.ToString();
break;
case "size":
release.Size = ParseUtil.GetBytes(value);
value = release.Size.ToString();
break;
case "leechers":
var leechers = ParseUtil.CoerceLong(value);
leechers = leechers < 5000000L ? leechers : 0; // to fix #6558
if (release.Peers == null)
{
release.Peers = (int)leechers;
}
else
{
release.Peers += (int)leechers;
}
value = leechers.ToString();
break;
case "seeders":
release.Seeders = ParseUtil.CoerceInt(value);
release.Seeders = release.Seeders < 5000000L ? release.Seeders : 0; // to fix #6558
if (release.Peers == null)
{
release.Peers = release.Seeders;
}
else
{
release.Peers += release.Seeders;
}
value = release.Seeders.ToString();
break;
case "date":
release.PublishDate = DateTimeUtil.FromUnknown(value);
value = release.PublishDate.ToString(DateTimeUtil.Rfc1123ZPattern);
break;
case "files":
release.Files = ParseUtil.CoerceInt(value);
value = release.Files.ToString();
break;
case "grabs":
release.Grabs = ParseUtil.CoerceInt(value);
value = release.Grabs.ToString();
break;
case "downloadvolumefactor":
release.DownloadVolumeFactor = ParseUtil.CoerceDouble(value);
value = release.DownloadVolumeFactor.ToString();
break;
case "uploadvolumefactor":
release.UploadVolumeFactor = ParseUtil.CoerceDouble(value);
value = release.UploadVolumeFactor.ToString();
break;
case "minimumratio":
release.MinimumRatio = ParseUtil.CoerceDouble(value);
value = release.MinimumRatio.ToString();
break;
case "minimumseedtime":
release.MinimumSeedTime = ParseUtil.CoerceLong(value);
value = release.MinimumSeedTime.ToString();
break;
case "imdb":
case "imdbid":
release.ImdbId = (int)ParseUtil.GetLongFromString(value);
value = release.ImdbId.ToString();
break;
case "tmdbid":
var tmdbIDRegEx = new Regex(@"(\d+)", RegexOptions.Compiled);
var tmdbIDMatch = tmdbIDRegEx.Match(value);
var tmdbID = tmdbIDMatch.Groups[1].Value;
release.TmdbId = (int)ParseUtil.CoerceLong(tmdbID);
value = release.TmdbId.ToString();
break;
case "rageid":
var rageIDRegEx = new Regex(@"(\d+)", RegexOptions.Compiled);
var rageIDMatch = rageIDRegEx.Match(value);
var rageID = rageIDMatch.Groups[1].Value;
release.TvRageId = (int)ParseUtil.CoerceLong(rageID);
value = release.TvRageId.ToString();
break;
case "traktid":
var traktIDRegEx = new Regex(@"(\d+)", RegexOptions.Compiled);
var traktIDMatch = traktIDRegEx.Match(value);
var traktID = traktIDMatch.Groups[1].Value;
release.TraktId = (int)ParseUtil.CoerceLong(traktID);
value = release.TraktId.ToString();
break;
case "tvdbid":
var tvdbIdRegEx = new Regex(@"(\d+)", RegexOptions.Compiled);
var tvdbIdMatch = tvdbIdRegEx.Match(value);
var tvdbId = tvdbIdMatch.Groups[1].Value;
release.TvdbId = (int)ParseUtil.CoerceLong(tvdbId);
value = release.TvdbId.ToString();
break;
case "poster":
if (!string.IsNullOrWhiteSpace(value))
{
var poster = ResolvePath(value, searchUrlUri);
release.PosterUrl = poster.AbsoluteUri;
}
value = release.PosterUrl;
break;
case "genre":
release.Genres = release.Genres.Union(value.Split(',')).ToList();
value = release.Genres.ToString();
break;
case "year":
release.Year = ParseUtil.CoerceInt(value);
value = release.Year.ToString();
break;
case "author":
release.Author = value;
break;
case "booktitle":
release.BookTitle = value;
break;
case "artist":
release.Artist = value;
break;
case "album":
release.Album = value;
break;
default:
break;
}
return(value);
}
public IList <ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
{
var torrentInfos = new List <TorrentInfo>();
var parser = new HtmlParser();
var dom = parser.ParseDocument(indexerResponse.Content);
var rows = dom.QuerySelectorAll("tr");
foreach (var row in rows.Skip(1))
{
var release = new TorrentInfo();
var qTitleLink = row.QuerySelector("td:nth-of-type(2) a:nth-of-type(1)");
release.Title = qTitleLink.TextContent.Trim();
// If we search an get no results, we still get a table just with no info.
if (string.IsNullOrWhiteSpace(release.Title))
{
break;
}
release.Guid = qTitleLink.GetAttribute("href");
release.InfoUrl = release.Guid;
var dateString = row.QuerySelector("td:nth-of-type(5)").TextContent;
release.PublishDate = DateTime.ParseExact(dateString, "dd MMM yy", CultureInfo.InvariantCulture);
// newbie users don't see DL links
var qLink = row.QuerySelector("td:nth-of-type(3) a");
if (qLink != null)
{
release.DownloadUrl = qLink.GetAttribute("href");
}
else
{
// use details link as placeholder
// null causes errors during export to torznab
// skipping the release prevents newbie users from adding the tracker (empty result)
release.DownloadUrl = release.InfoUrl;
}
var sizeStr = row.QuerySelector("td:nth-of-type(6)").TextContent;
release.Size = ParseUtil.GetBytes(sizeStr);
var connections = row.QuerySelector("td:nth-of-type(8)").TextContent.Trim().Split("/".ToCharArray(), StringSplitOptions.RemoveEmptyEntries);
release.Seeders = ParseUtil.CoerceInt(connections[0].Trim());
release.Peers = ParseUtil.CoerceInt(connections[1].Trim()) + release.Seeders;
release.Grabs = ParseUtil.CoerceInt(connections[2].Trim());
var rCat = row.QuerySelector("td:nth-of-type(1) a").GetAttribute("href");
var rCatIdx = rCat.IndexOf("cat=");
if (rCatIdx > -1)
{
rCat = rCat.Substring(rCatIdx + 4);
}
release.Categories = _categories.MapTrackerCatToNewznab(rCat);
if (row.QuerySelector("img[alt=\"Gold Torrent\"]") != null)
{
release.DownloadVolumeFactor = 0;
}
else if (row.QuerySelector("img[alt=\"Silver Torrent\"]") != null)
{
release.DownloadVolumeFactor = 0.5;
}
else
{
release.DownloadVolumeFactor = 1;
}
var uLFactorImg = row.QuerySelector("img[alt*=\"x Multiplier Torrent\"]");
if (uLFactorImg != null)
{
release.UploadVolumeFactor = ParseUtil.CoerceDouble(uLFactorImg.GetAttribute("alt").Split('x')[0]);
}
else
{
release.UploadVolumeFactor = 1;
}
qTitleLink.Remove();
//release.Description = row.QuerySelector("td:nth-of-type(2)").TextContent;
torrentInfos.Add(release);
}
return(torrentInfos.ToArray());
}
public async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
searchBlock Search = Definition.Search;
// init template context
var variables = getTemplateVariablesFromConfigData();
variables[".Query.Type"] = query.QueryType;
variables[".Query.Q"] = query.SearchTerm;
variables[".Query.Series"] = null;
variables[".Query.Ep"] = query.Episode;
variables[".Query.Season"] = query.Season;
variables[".Query.Movie"] = null;
variables[".Query.Year"] = null;
variables[".Query.Limit"] = query.Limit;
variables[".Query.Offset"] = query.Offset;
variables[".Query.Extended"] = query.Extended;
variables[".Query.Categories"] = query.Categories;
variables[".Query.APIKey"] = query.ApiKey;
variables[".Query.TVDBID"] = null;
variables[".Query.TVRageID"] = query.RageID;
variables[".Query.IMDBID"] = query.ImdbID;
variables[".Query.TVMazeID"] = null;
variables[".Query.TraktID"] = null;
variables[".Query.Episode"] = query.GetEpisodeSearchString();
variables[".Categories"] = MapTorznabCapsToTrackers(query);
var KeywordTokens = new List <string>();
var KeywordTokenKeys = new List <string> {
"Q", "Series", "Movie", "Year"
};
foreach (var key in KeywordTokenKeys)
{
var Value = (string)variables[".Query." + key];
if (!string.IsNullOrWhiteSpace(Value))
{
KeywordTokens.Add(Value);
}
}
if (!string.IsNullOrWhiteSpace((string)variables[".Query.Episode"]))
{
KeywordTokens.Add((string)variables[".Query.Episode"]);
}
variables[".Query.Keywords"] = string.Join(" ", KeywordTokens);
variables[".Keywords"] = variables[".Query.Keywords"];
// build search URL
var searchUrl = SiteLink + applyGoTemplateText(Search.Path, variables) + "?";
var queryCollection = new NameValueCollection();
if (Search.Inputs != null)
{
foreach (var Input in Search.Inputs)
{
var value = applyGoTemplateText(Input.Value, variables);
if (Input.Key == "$raw")
{
searchUrl += value;
}
else
{
queryCollection.Add(Input.Key, value);
}
}
}
searchUrl += "&" + queryCollection.GetQueryString();
// send HTTP request
var response = await RequestBytesWithCookies(searchUrl);
var results = Encoding.GetEncoding("iso-8859-1").GetString(response.Content);
try
{
var SearchResultParser = new HtmlParser();
var SearchResultDocument = SearchResultParser.Parse(results);
var Rows = SearchResultDocument.QuerySelectorAll(Search.Rows.Selector);
foreach (var Row in Rows)
{
try
{
var release = new ReleaseInfo();
release.MinimumRatio = 1;
release.MinimumSeedTime = 48 * 60 * 60;
// Parse fields
foreach (var Field in Search.Fields)
{
string value = handleSelector(Field.Value, Row);
value = ParseUtil.NormalizeSpace(value);
try
{
switch (Field.Key)
{
case "download":
release.Link = resolvePath(value);
break;
case "details":
var url = resolvePath(value);
release.Guid = url;
if (release.Comments == null)
{
release.Comments = url;
}
break;
case "comments":
release.Comments = resolvePath(value);
break;
case "title":
release.Title = value;
break;
case "description":
release.Description = value;
break;
case "category":
release.Category = MapTrackerCatToNewznab(value);
break;
case "size":
release.Size = ReleaseInfo.GetBytes(value);
break;
case "leechers":
if (release.Peers == null)
{
release.Peers = ParseUtil.CoerceInt(value);
}
else
{
release.Peers += ParseUtil.CoerceInt(value);
}
break;
case "seeders":
release.Seeders = ParseUtil.CoerceInt(value);
if (release.Peers == null)
{
release.Peers = release.Seeders;
}
else
{
release.Peers += release.Seeders;
}
break;
case "date":
release.PublishDate = DateTimeUtil.FromUnknown(value);
break;
case "files":
release.Files = ParseUtil.CoerceLong(value);
break;
case "grabs":
release.Grabs = ParseUtil.CoerceLong(value);
break;
case "downloadvolumefactor":
release.DownloadVolumeFactor = ParseUtil.CoerceDouble(value);
break;
case "uploadvolumefactor":
release.UploadVolumeFactor = ParseUtil.CoerceDouble(value);
break;
default:
break;
}
}
catch (Exception ex)
{
throw new Exception(string.Format("Error while parsing field={0}, selector={1}, value={2}: {3}", Field.Key, Field.Value.Selector, value, ex.Message));
}
}
releases.Add(release);
}
catch (Exception ex)
{
logger.Error(string.Format("CardigannIndexer ({0}): Error while parsing row '{1}': {2}", ID, Row.OuterHtml, ex));
}
}
}
catch (Exception ex)
{
OnParseError(results, ex);
}
return(releases);
}
