public static Uri CreateSearchUri(string search)
{
var finalUri = SearchUriBase.AbsoluteUri;
finalUri += "?sec=buscador&valor=" + WebUtilityHelpers.UrlEncode(search, MEEncoding);
return(new Uri(finalUri));
}
public static string GetQueryString(this ICollection <KeyValuePair <string, string> > collection, Encoding encoding = null)
{
if (encoding == null)
{
encoding = Encoding.UTF8;
}
return(string.Join("&", collection.Select(a => a.Key + "=" + WebUtilityHelpers.UrlEncode(a.Value, encoding))));
}
public static Uri InfoHashToPublicMagnet(string infoHash, string title)
{
if (string.IsNullOrWhiteSpace(infoHash) || string.IsNullOrWhiteSpace(title))
{
return(null);
}
return(new Uri($"magnet:?xt=urn:btih:{infoHash}&dn={WebUtilityHelpers.UrlEncode(title, Encoding.UTF8)}&{_TrackersEncoded}"));
}
public static string GetQueryString(this NameValueCollection collection, Encoding encoding = null)
{
if (encoding == null)
{
encoding = Encoding.UTF8;
}
return(string.Join("&", collection.AllKeys.Select(a => a + "=" + WebUtilityHelpers.UrlEncode(collection[a], encoding))));
}
public void WebUtilityHelpers_UrlDecode_CorrectlyDecodes()
{
foreach (var encoding in _codePagesToTest)
{
foreach (var testString in _stringsToTest)
{
//Check our implementation of Decode in .NET Standard Matches the .NET Framework Version
var encodedString = HttpUtility.UrlEncode(testString, encoding);
var NETString = HttpUtility.UrlDecode(encodedString, encoding);
var WebUtilityString = WebUtilityHelpers.UrlDecode(encodedString, encoding);
Assert.AreEqual(NETString, WebUtilityString, $"{testString} did not match the expected decoded value after encoding with {encoding.EncodingName})");
}
}
}
public void WebUtilityHelpers_UrlEncode_CorrectlyEncodes()
{
foreach (var encoding in _codePagesToTest)
{
foreach (var testString in _stringsToTest)
{
//Check our implementation of Decode in .NET Standard Matches the .NET Framework Version
var NETString = HttpUtility.UrlEncode(testString, encoding);
var WebUtilityString = WebUtilityHelpers.UrlEncode(testString, encoding);
//Of note is that percent encoding gives lowercase values, where NET Native uses upper case this should be okay according to RFC3986 (https://tools.ietf.org/html/rfc3986#section-2.1)
var NETDecode = HttpUtility.UrlDecode(NETString);
var WebUtilityDecode = HttpUtility.UrlDecode(WebUtilityString);
Assert.AreEqual(NETDecode, WebUtilityDecode, $"{testString} did not match the expected decoded string with {encoding.EncodingName})");
}
}
}
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var templateUrl = SiteLink;
if (_language.Equals("castellano"))
{
templateUrl += "espana/";
}
templateUrl += "{0}"; // placeholder for page
var maxPages = 2; // we scrape only 2 pages for recent torrents
if (!string.IsNullOrWhiteSpace(query.GetQueryString()))
{
templateUrl += "?s=" + WebUtilityHelpers.UrlEncode(query.GetQueryString(), Encoding.UTF8);
maxPages = MaxSearchPageLimit;
}
var lastPublishDate = DateTime.Now;
for (var page = 1; page <= maxPages; page++)
{
var pageParam = page > 1 ? $"page/{page}/" : "";
var searchUrl = string.Format(templateUrl, pageParam);
var response = await RequestWithCookiesAndRetryAsync(searchUrl);
var pageReleases = ParseReleases(response, query);
// publish date is not available in the torrent list, but we add a relative date so we can sort
foreach (var release in pageReleases)
{
release.PublishDate = lastPublishDate;
lastPublishDate = lastPublishDate.AddMinutes(-1);
}
releases.AddRange(pageReleases);
if (pageReleases.Count < MaxItemsPerPage)
{
break; // this is the last page
}
}
return(releases);
}
/// <summary>
/// Build query to process
/// </summary>
/// <param name="term">Term to search</param>
/// <param name="query">Torznab Query for categories mapping</param>
/// <param name="url">Search url for provider</param>
/// <param name="page">Page number to request</param>
/// <returns>URL to query for parsing and processing results</returns>
private string BuildQuery(string term, TorznabQuery query, string url, int page = 0)
{
var parameters = new NameValueCollection();
var categoriesList = MapTorznabCapsToTrackers(query);
string searchterm = term;
// Building our tracker query
parameters.Add("incldead", "1");
parameters.Add("fullsearch", "0");
parameters.Add("scenerelease", "0");
// If search term provided
if (!string.IsNullOrWhiteSpace(query.ImdbID))
{
searchterm = "imdbsearch=" + query.ImdbID;
}
else if (!string.IsNullOrWhiteSpace(term))
{
searchterm = "search=" + WebUtilityHelpers.UrlEncode(term, Encoding.GetEncoding(28591));
}
else
{
// Showing all torrents (just for output function)
searchterm = "search=";
term = "all";
}
var CatQryStr = "";
foreach (var cat in categoriesList)
{
CatQryStr += "&" + cat;
}
// Building our query
url += "?" + searchterm + "&" + parameters.GetQueryString() + "&" + CatQryStr;
Output("\nBuilded query for \"" + term + "\"... " + url);
// Return our search url
return(url);
}
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var templateUrl = SiteLink + "{0}";
var maxPages = 2;
if (!string.IsNullOrWhiteSpace(query.GetQueryString()))
{
templateUrl += "?s=" + WebUtilityHelpers.UrlEncode(query.GetQueryString(), Encoding.UTF8);
maxPages = MaxSearchPageLimit;
}
var lastPublishDate = DateTime.Now;
for (var page = 1; page <= maxPages; page++)
{
var pageParam = page > 1 ? $"page/{page}" : "";
var searchUrl = string.Format(templateUrl, pageParam);
var response = await RequestWithCookiesAndRetryAsync(searchUrl);
var pageReleases = await ParseReleases(response, query);
foreach (var release in pageReleases)
{
release.PublishDate = lastPublishDate;
lastPublishDate = lastPublishDate.AddMinutes(-1);
}
releases.AddRange(pageReleases);
if (pageReleases.Count < MaxItemsPerPage)
{
break;
}
}
return(releases);
}
public void WebUtilityHelpers_UrlDecode_BadDecodes()
{
var decoded = WebUtilityHelpers.UrlDecode(null, Encoding.UTF8);
Assert.AreEqual("", decoded);
}
protected override async Task <WebResult> Run(WebRequest webRequest)
{
ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072;
var cookies = new CookieContainer();
if (!string.IsNullOrWhiteSpace(webRequest.Cookies))
{
// don't include the path, Scheme is needed for mono compatibility
var requestUri = new Uri(webRequest.Url);
var cookieUrl = new Uri(requestUri.Scheme + "://" + requestUri.Host);
var cookieDictionary = CookieUtil.CookieHeaderToDictionary(webRequest.Cookies);
foreach (var kv in cookieDictionary)
{
cookies.Add(cookieUrl, new Cookie(kv.Key, kv.Value));
}
}
var userAgent = webRequest.EmulateBrowser.Value ? BrowserUtil.ChromeUserAgent : "Jackett/" + configService.GetVersion();
using (var clearanceHandlr = new ClearanceHandler(userAgent))
{
clearanceHandlr.MaxTries = 10;
using (var clientHandlr = new HttpClientHandler
{
CookieContainer = cookies,
AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
UseCookies = true,
Proxy = webProxy,
UseProxy = (webProxy != null),
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
})
{
clearanceHandlr.InnerHandler = clientHandlr;
using (var client = new HttpClient(clearanceHandlr))
{
if (webRequest.EmulateBrowser == true)
{
client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent);
}
else
{
client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion());
}
HttpResponseMessage response = null;
using (var request = new HttpRequestMessage())
{
request.Headers.ExpectContinue = false;
request.RequestUri = new Uri(webRequest.Url);
if (webRequest.Headers != null)
{
foreach (var header in webRequest.Headers)
{
if (header.Key != "Content-Type")
{
request.Headers.TryAddWithoutValidation(header.Key, header.Value);
}
}
}
if (!string.IsNullOrEmpty(webRequest.Referer))
{
request.Headers.Referrer = new Uri(webRequest.Referer);
}
if (!string.IsNullOrEmpty(webRequest.RawBody))
{
var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast <KeyValuePair <string, string>?>().FirstOrDefault();
if (type.HasValue)
{
var str = new StringContent(webRequest.RawBody);
str.Headers.Remove("Content-Type");
str.Headers.Add("Content-Type", type.Value.Value);
request.Content = str;
}
else
{
request.Content = new StringContent(webRequest.RawBody);
}
request.Method = HttpMethod.Post;
}
else if (webRequest.Type == RequestType.POST)
{
if (webRequest.PostData != null)
{
request.Content = FormUrlEncodedContentWithEncoding(webRequest.PostData, webRequest.Encoding);
}
request.Method = HttpMethod.Post;
}
else
{
request.Method = HttpMethod.Get;
}
using (response = await client.SendAsync(request))
{
var result = new WebResult
{
ContentBytes = await response.Content.ReadAsByteArrayAsync()
};
foreach (var header in response.Headers)
{
var value = header.Value;
result.Headers[header.Key.ToLowerInvariant()] = value.ToArray();
}
// some cloudflare clients are using a refresh header
// Pull it out manually
if (response.StatusCode == HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh"))
{
var refreshHeaders = response.Headers.GetValues("Refresh");
var redirval = "";
var redirtime = 0;
if (refreshHeaders != null)
{
foreach (var value in refreshHeaders)
{
var start = value.IndexOf("=");
var end = value.IndexOf(";");
var len = value.Length;
if (start > -1)
{
redirval = value.Substring(start + 1);
result.RedirectingTo = redirval;
// normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
// of this cloudflare approach..don't want to alter WebResult.IsRedirect because normally
// it shoudln't include service unavailable..only if we have this redirect header.
response.StatusCode = System.Net.HttpStatusCode.Redirect;
redirtime = int.Parse(value.Substring(0, end));
System.Threading.Thread.Sleep(redirtime * 1000);
}
}
}
}
if (response.Headers.Location != null)
{
result.RedirectingTo = response.Headers.Location.ToString();
}
// Mono won't add the baseurl to relative redirects.
// e.g. a "Location: /index.php" header will result in the Uri "file:///index.php"
// See issue #1200
if (result.RedirectingTo != null && result.RedirectingTo.StartsWith("file://"))
{
// URL decoding apparently is needed to, without it e.g. Demonoid download is broken
// TODO: is it always needed (not just for relative redirects)?
var newRedirectingTo = WebUtilityHelpers.UrlDecode(result.RedirectingTo, webRequest.Encoding);
if (newRedirectingTo.StartsWith("file:////")) // Location without protocol but with host (only add scheme)
{
newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + ":");
}
else
{
newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + "://" + request.RequestUri.Host);
}
logger.Debug("[MONO relative redirect bug] Rewriting relative redirect URL from " + result.RedirectingTo + " to " + newRedirectingTo);
result.RedirectingTo = newRedirectingTo;
}
result.Status = response.StatusCode;
// Compatiblity issue between the cookie format and httpclient
// Pull it out manually ignoring the expiry date then set it manually
// http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
var responseCookies = new List <Tuple <string, string> >();
if (response.Headers.TryGetValues("set-cookie", out var cookieHeaders))
{
foreach (var value in cookieHeaders)
{
var nameSplit = value.IndexOf('=');
if (nameSplit > -1)
{
responseCookies.Add(new Tuple <string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';'))) + ";"));
}
}
var cookieBuilder = new StringBuilder();
foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1))
{
cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
}
result.Cookies = cookieBuilder.ToString().Trim();
}
ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
return(result);
}
}
}
}
}
}
private async Task <IEnumerable <ReleaseInfo> > performRegularQuery(TorznabQuery query, string hebName = null)
{
var releases = new List <ReleaseInfo>();
var searchurls = new List <string>();
var searchUrl = SearchUrl;
var queryCollection = new NameValueCollection();
var searchString = query.GetQueryString();
if (query.IsImdbQuery)
{
searchString = query.ImdbID;
}
if (hebName != null)
{
searchString = hebName + " - עונה " + query.Season + " פרק " + query.Episode;
}
searchUrl += "?";
if (!string.IsNullOrWhiteSpace(searchString))
{
var strEncoded = WebUtilityHelpers.UrlEncode(searchString, Encoding);
searchUrl += "&query=" + strEncoded + "&matchquery=any";
}
foreach (var cat in MapTorznabCapsToTrackers(query))
{
searchUrl += "&c[]=" + cat;
}
var data = await RequestStringWithCookiesAndRetry(searchUrl);
try
{
CQ dom = data.Content;
var rows = dom["tr.box_torrent"];
foreach (var row in rows)
{
CQ qRow = row.Cq();
var release = new ReleaseInfo();
var main_title_link = qRow.Find("div.main_title > a");
release.Title = main_title_link.Attr("longtitle");
if (release.Title.IsNullOrEmptyOrWhitespace())
{
release.Title = main_title_link.Text();
}
release.MinimumRatio = 1;
release.MinimumSeedTime = 172800;
int seeders, peers;
if (ParseUtil.TryCoerceInt(qRow.Find("td:nth-child(7) > div").Text(), out seeders))
{
release.Seeders = seeders;
if (ParseUtil.TryCoerceInt(qRow.Find("td:nth-child(8) > div").Text(), out peers))
{
release.Peers = peers + release.Seeders;
}
}
release.Grabs = ParseUtil.CoerceLong(qRow.Find("td:nth-child(5)").Text().Replace(",", ""));
release.Seeders = ParseUtil.CoerceInt(qRow.Find("td:nth-child(6)").Text().Replace(",", ""));
release.Peers = ParseUtil.CoerceInt(qRow.Find("td:nth-child(7)").Text().Replace(",", "")) + release.Seeders;
string fullSize = qRow.Find("td:nth-child(4)").Text();
release.Size = ReleaseInfo.GetBytes(fullSize);
release.Comments = new Uri(SiteLink + qRow.Find("a.threadlink[href]").Attr("href"));
release.Link = new Uri(SiteLink + qRow.Find("a:has(div.dlimg)").Attr("href"));
release.Guid = release.Comments;
try
{
release.BannerUrl = new Uri(qRow.Find("a[imgsrc]").Attr("imgsrc"));
}
catch (Exception)
{
// do nothing, some releases have invalid banner URLs, ignore the banners in this case
}
var dateStringAll = qRow.Find("div.up_info2")[0].ChildNodes.Last().ToString();
var dateParts = dateStringAll.Split(' ');
string dateString = dateParts[dateParts.Length - 2] + " " + dateParts[dateParts.Length - 1];
release.PublishDate = DateTime.ParseExact(dateString, "dd/MM/yy HH:mm", CultureInfo.InvariantCulture);
string categoryLink = qRow.Find("a[href^=\"/browse.php?cat=\"]").Attr("href");
var catid = ParseUtil.GetArgumentFromQueryString(categoryLink, "cat");
release.Category = MapTrackerCatToNewznab(catid);
if (qRow.Find("a[href^=\"?freeleech=1\"]").Length >= 1)
{
release.DownloadVolumeFactor = 0;
}
else
{
release.DownloadVolumeFactor = 1;
}
release.UploadVolumeFactor = 1;
var sub_title = qRow.Find("div.sub_title");
var imdb_link = sub_title.Find("span.imdb-inline > a");
release.Imdb = ParseUtil.GetLongFromString(imdb_link.Attr("href"));
sub_title.Find("span.imdb-inline").Remove();
release.Description = sub_title.Text();
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError(data.Content, ex);
}
return(releases);
}
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var searchString = query.GetQueryString();
var searchUrl = SearchUrl;
if (!string.IsNullOrWhiteSpace(searchString))
{
searchUrl += "&action=advanced&searchsubmit=1&filelist=" + WebUtilityHelpers.UrlEncode(searchString, Encoding);
}
var cats = MapTorznabCapsToTrackers(query);
if (cats.Count > 0)
{
searchUrl = cats.Aggregate(searchUrl, (url, cat) => $"{url}&filter_cat[{cat}]=1");
}
var response = await RequestWithCookiesAsync(searchUrl);
try
{
var parser = new HtmlParser();
var dom = parser.ParseDocument(response.ContentString);
var rows = dom.QuerySelectorAll("table#torrent_table > tbody > tr.torrent");
foreach (var row in rows)
{
var release = new ReleaseInfo
{
MinimumRatio = 1.0,
MinimumSeedTime = 604800 // 168 hours
};
var qCat = row.QuerySelector("div[class*=\"cats_\"]");
var catStr = qCat.GetAttribute("class").Split('_')[1];
release.Category = catStr switch
{
"movies" => MapTrackerCatToNewznab("1"),
"tv" => MapTrackerCatToNewznab("2"),
"theater" => MapTrackerCatToNewznab("3"),
"software" => MapTrackerCatToNewznab("4"),
"games" => MapTrackerCatToNewznab("5"),
"music" => MapTrackerCatToNewznab("6"),
"books" => MapTrackerCatToNewznab("7"),
"moviespacks" => MapTrackerCatToNewznab("8"),
"porno" => MapTrackerCatToNewznab("9"),
"other" => MapTrackerCatToNewznab("10"),
_ => throw new Exception("Error parsing category! Unknown cat=" + catStr),
};
var qTitle = row.QuerySelector("div.torrent_info");
release.Title = qTitle.TextContent.Trim();
var qDetailsLink = row.QuerySelector("a.torrent_name");
// I don't understand, I correctly build the poster as
// https://hebits.net/images/oRbJr/3776T8DeNsKbyUM3tsV0LBY_v_JfdfmkhShh_LguTP.jpg
// yet the dashboard shows a broken icon symbol! No idea why this does not work.
//if (!string.IsNullOrEmpty(qDetailsLink.GetAttribute("data-cover")))
//{
// release.Poster = new Uri(SiteLink.TrimEnd('/') + qDetailsLink.GetAttribute("data-cover"));
// logger.Debug("poster=" + release.Poster);
//}
release.Details = new Uri(SiteLink + qDetailsLink.GetAttribute("href"));
release.Link = new Uri(SiteLink + row.QuerySelector("a[href^=\"torrents.php?action=download&id=\"]").GetAttribute("href"));
release.Guid = release.Link;
var qDate = row.QuerySelector("span.time").GetAttribute("title");
release.PublishDate = DateTime.ParseExact(qDate, "dd/MM/yyyy, HH:mm", CultureInfo.InvariantCulture);
var qSize = row.QuerySelector("td:nth-last-child(4)").TextContent.Trim();
release.Size = ReleaseInfo.GetBytes(qSize);
release.Seeders = ParseUtil.CoerceInt(row.QuerySelector("td:nth-last-child(2)").TextContent.Trim());
release.Peers = release.Seeders + ParseUtil.CoerceInt(row.QuerySelector("td:nth-last-child(1)").TextContent.Trim());
release.Files = ParseUtil.CoerceInt(row.QuerySelector("td:nth-last-child(6)").TextContent.Trim());
release.Grabs = ParseUtil.CoerceInt(row.QuerySelector("td:nth-last-child(3)").TextContent.Trim());
release.DownloadVolumeFactor = release.Title.Contains("פריליץ")
? 0
: release.Title.Contains("חצי פריליץ")
? 0.5
: release.Title.Contains("75% פריליץ")
? 0.25
: 1;
release.UploadVolumeFactor = release.Title.Contains("העלאה משולשת")
? 3
: release.Title.Contains("העלאה כפולה")
? 2
: 1;
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError(response.ContentString, ex);
}
return(releases);
}
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var searchString = query.GetQueryString();
var queryUrl = SearchUrl;
var cats = MapTorznabCapsToTrackers(query);
if (cats.Count == 0)
{
cats = GetAllTrackerCategories();
}
var catStr = string.Join(";", cats);
queryUrl += "?" + catStr;
if (!string.IsNullOrWhiteSpace(query.ImdbID))
{
queryUrl += ";q=" + query.ImdbID;
}
else
{
queryUrl += ";q=" + WebUtilityHelpers.UrlEncode(searchString, Encoding);
}
var results = await RequestStringWithCookiesAndRetry(queryUrl);
// Check for being logged out
if (results.IsRedirect)
{
if (results.RedirectingTo.Contains("login.php"))
{
throw new ExceptionWithConfigData("Login failed, please reconfigure the tracker to update the cookies", configData);
}
else
{
throw new ExceptionWithConfigData(string.Format("Got a redirect to {0}, please adjust your the alternative link", results.RedirectingTo), configData);
}
}
try
{
dynamic json = JsonConvert.DeserializeObject <dynamic>(results.Content);
foreach (var torrent in json)
{
var release = new ReleaseInfo();
release.Title = torrent.name;
if ((query.ImdbID == null || !TorznabCaps.SupportsImdbMovieSearch) && !query.MatchQueryStringAND(release.Title))
{
continue;
}
release.MinimumRatio = 1;
release.MinimumSeedTime = 172800; // 48 hours
release.Category = MapTrackerCatToNewznab(torrent.c.ToString());
var torrentID = (long)torrent.t;
release.Comments = new Uri(SiteLink + "details.php?id=" + torrentID);
release.Guid = release.Comments;
release.Link = new Uri(SiteLink + "download.php/" + torrentID + "/" + torrentID + ".torrent");
release.PublishDate = DateTimeUtil.UnixTimestampToDateTime((long)torrent.ctime).ToLocalTime();
release.Size = (long)torrent.size;
release.Seeders = (int)torrent.seeders;
release.Peers = release.Seeders + (int)torrent.leechers;
release.Files = (long)torrent.files;
release.Grabs = (long)torrent.completed;
var imdbId = (string)torrent["imdb-id"];
release.Imdb = ParseUtil.GetImdbID(imdbId);
var downloadMultiplier = (double?)torrent["download-multiplier"];
release.DownloadVolumeFactor = downloadMultiplier ?? 1;
release.UploadVolumeFactor = 1;
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError(results.Content, ex);
}
return(releases);
}
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var searchString = query.GetQueryString();
var searchUrl = SearchUrl;
if (!string.IsNullOrWhiteSpace(searchString))
{
searchUrl += "&search=" + WebUtilityHelpers.UrlEncode(searchString, Encoding);
}
string.Format(SearchUrl, WebUtilityHelpers.UrlEncode(searchString, Encoding));
var cats = MapTorznabCapsToTrackers(query);
if (cats.Count > 0)
{
foreach (var cat in cats)
{
searchUrl += "&c" + cat + "=1";
}
}
var response = await RequestStringWithCookies(searchUrl);
try
{
CQ dom = response.Content;
var qRows = dom[".browse > div > div"];
foreach (var row in qRows)
{
var release = new ReleaseInfo();
var qRow = row.Cq();
var debug = qRow.Html();
release.MinimumRatio = 1;
release.MinimumSeedTime = 172800; // 48 hours
var qTitle = qRow.Find(".bTitle");
var titleParts = qTitle.Text().Split('/');
if (titleParts.Length >= 2)
{
release.Title = titleParts[1].Trim();
}
else
{
release.Title = titleParts[0].Trim();
}
var qDetailsLink = qTitle.Find("a[href^=\"details.php\"]");
release.Comments = new Uri(SiteLink + qDetailsLink.Attr("href"));
release.Link = new Uri(SiteLink + qRow.Find("a[href^=\"download.php\"]").Attr("href"));
release.Guid = release.Link;
var dateString = qRow.Find("div:last-child").Text().Trim();
var pattern = "\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}";
var match = Regex.Match(dateString, pattern);
if (match.Success)
{
release.PublishDate = DateTime.ParseExact(match.Value, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
}
var sizeStr = qRow.Find(".bSize").Text();
release.Size = ReleaseInfo.GetBytes(sizeStr);
release.Seeders = ParseUtil.CoerceInt(qRow.Find(".bUping").Text().Trim());
release.Peers = release.Seeders + ParseUtil.CoerceInt(qRow.Find(".bDowning").Text().Trim());
var files = qRow.Find("div.bFiles").Get(0).LastChild.ToString();
release.Files = ParseUtil.CoerceInt(files);
var grabs = qRow.Find("div.bFinish").Get(0).LastChild.ToString();
release.Grabs = ParseUtil.CoerceInt(grabs);
if (qRow.Find("img[src=\"/pic/free.jpg\"]").Length >= 1)
{
release.DownloadVolumeFactor = 0;
}
else
{
release.DownloadVolumeFactor = 1;
}
if (qRow.Find("img[src=\"/pic/triple.jpg\"]").Length >= 1)
{
release.UploadVolumeFactor = 3;
}
else if (qRow.Find("img[src=\"/pic/double.jpg\"]").Length >= 1)
{
release.UploadVolumeFactor = 2;
}
else
{
release.UploadVolumeFactor = 1;
}
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError(response.Content, ex);
}
return(releases);
}
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var searchString = query.GetQueryString();
var searchUrl = SearchUrl;
if (!string.IsNullOrWhiteSpace(searchString))
{
searchUrl += "&search=" + WebUtilityHelpers.UrlEncode(searchString, Encoding);
}
var cats = MapTorznabCapsToTrackers(query);
if (cats.Count > 0)
{
searchUrl = cats.Aggregate(searchUrl, (url, cat) => $"{url}&c{cat}=1");
}
var response = await RequestWithCookiesAsync(searchUrl);
try
{
var parser = new HtmlParser();
var dom = parser.ParseDocument(response.ContentString);
var rows = dom.QuerySelectorAll(".browse > div > div");
foreach (var row in rows)
{
var release = new ReleaseInfo();
release.MinimumRatio = 0.8;
release.MinimumSeedTime = 259200; // 72 hours
var qCatLink = row.QuerySelector("a[href^=\"browse.php?cat=\"]");
var catStr = qCatLink.GetAttribute("href").Split('=')[1];
release.Category = MapTrackerCatToNewznab(catStr);
var qTitle = row.QuerySelector(".bTitle");
var titleParts = qTitle.TextContent.Split('/');
release.Title = titleParts.Length >= 2 ? titleParts[1].Trim() : titleParts[0].Trim();
var qDetailsLink = qTitle.QuerySelector("a[href^=\"details.php\"]");
release.Details = new Uri(SiteLink + qDetailsLink.GetAttribute("href"));
release.Link = new Uri(SiteLink + row.QuerySelector("a[href^=\"download.php\"]").GetAttribute("href"));
release.Guid = release.Link;
var dateString = row.QuerySelector("div:last-child").TextContent.Trim();
var match = Regex.Match(dateString, @"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}");
if (match.Success)
{
release.PublishDate = DateTime.ParseExact(match.Value, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
}
var sizeStr = row.QuerySelector(".bSize").TextContent.Trim();
release.Size = ReleaseInfo.GetBytes(sizeStr);
release.Seeders = ParseUtil.CoerceInt(row.QuerySelector(".bUping").TextContent.Trim());
release.Peers = release.Seeders + ParseUtil.CoerceInt(row.QuerySelector(".bDowning").TextContent.Trim());
var files = row.QuerySelector("div.bFiles").ChildNodes.Last().TextContent.Trim();
release.Files = ParseUtil.CoerceInt(files);
var grabs = row.QuerySelector("div.bFinish").ChildNodes.Last().TextContent.Trim();
release.Grabs = ParseUtil.CoerceInt(grabs);
release.DownloadVolumeFactor = row.QuerySelector("img[src=\"/pic/free.jpg\"]") != null ? 0 : 1;
if (row.QuerySelector("img[src=\"/pic/triple.jpg\"]") != null)
{
release.UploadVolumeFactor = 3;
}
else if (row.QuerySelector("img[src=\"/pic/double.jpg\"]") != null)
{
release.UploadVolumeFactor = 2;
}
else
{
release.UploadVolumeFactor = 1;
}
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError(response.ContentString, ex);
}
return(releases);
}
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
// the order of the params is important!
var qc = new List <string>();
var catList = MapTorznabCapsToTrackers(query);
foreach (var cat in catList)
{
qc.Add(cat);
}
if (query.IsImdbQuery)
{
qc.Add("deep");
qc.Add("q");
qc.Add(query.ImdbID);
}
else
{
qc.Add("q");
qc.Add(WebUtilityHelpers.UrlEncode(query.GetQueryString(), Encoding));
}
var searchUrl = SearchUrl + string.Join("/", qc);
var response = await RequestWithCookiesAndRetryAsync(searchUrl);
if (!response.ContentString.Contains("/logout.php")) // re-login
{
await DoLogin();
response = await RequestWithCookiesAndRetryAsync(searchUrl);
}
try
{
var parser = new HtmlParser();
var dom = parser.ParseDocument(response.ContentString);
var rows = dom.QuerySelectorAll("div.boxContent > table > tbody > tr");
foreach (var row in rows)
{
var cells = row.QuerySelectorAll("td");
var title = row.QuerySelector("td[class='lft'] > div > a").TextContent.Trim();
var link = new Uri(SiteLink + row.QuerySelector("img[title='Download']").ParentElement.GetAttribute("href").TrimStart('/'));
var comments = new Uri(SiteLink + row.QuerySelector("td[class='lft'] > div > a").GetAttribute("href").TrimStart('/'));
var size = ReleaseInfo.GetBytes(cells[5].TextContent);
var grabs = ParseUtil.CoerceInt(cells[6].TextContent);
var seeders = ParseUtil.CoerceInt(cells[7].TextContent);
var leechers = ParseUtil.CoerceInt(cells[8].TextContent);
var pubDateStr = row.QuerySelector("span[class^='elapsedDate']").GetAttribute("title").Replace(" at", "");
var publishDate = DateTime.ParseExact(pubDateStr, "dddd, MMMM d, yyyy h:mmtt", CultureInfo.InvariantCulture);
var cat = row.QuerySelector("a").GetAttribute("href").Split('/').Last();
var downloadVolumeFactor = row.QuerySelector("span:contains(\"[Freeleech]\")") != null ? 0 : 1;
var release = new ReleaseInfo
{
Title = title,
Link = link,
Guid = link,
Comments = comments,
PublishDate = publishDate,
Category = MapTrackerCatToNewznab(cat),
Size = size,
Grabs = grabs,
Seeders = seeders,
Peers = seeders + leechers,
MinimumRatio = 1,
MinimumSeedTime = 172800, // 48 hours
DownloadVolumeFactor = downloadVolumeFactor,
UploadVolumeFactor = 1
};
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError(response.ContentString, ex);
}
return(releases);
}
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var cats = MapTorznabCapsToTrackers(query);
if (cats.Count == 0)
{
cats = GetAllTrackerCategories();
}
var catStr = string.Join(";", cats);
var searchUrl = SearchUrl + "?" + catStr;
if (query.IsImdbQuery)
{
searchUrl += ";q=" + query.ImdbID;
}
else
{
searchUrl += ";q=" + WebUtilityHelpers.UrlEncode(query.GetQueryString(), Encoding);
}
if (((BoolConfigurationItem)configData.GetDynamic("freeleech")).Value)
{
searchUrl += ";free=on";
}
var results = await RequestWithCookiesAndRetryAsync(searchUrl);
// Check for being logged out
if (results.IsRedirect)
{
if (results.RedirectingTo.Contains("login.php"))
{
throw new Exception("The user is not logged in. It is possible that the cookie has expired or you made a mistake when copying it. Please check the settings.");
}
else
{
throw new Exception($"Got a redirect to {results.RedirectingTo}, please adjust your the alternative link");
}
}
try
{
var rows = JsonConvert.DeserializeObject <dynamic>(results.ContentString);
foreach (var row in rows)
{
var title = (string)row.name;
if ((!query.IsImdbQuery || !TorznabCaps.MovieSearchImdbAvailable) && !query.MatchQueryStringAND(title))
{
continue;
}
var torrentId = (long)row.t;
var details = new Uri(SiteLink + "details.php?id=" + torrentId);
var seeders = (int)row.seeders;
var imdbId = (string)row["imdb-id"];
var downloadMultiplier = (double?)row["download-multiplier"] ?? 1;
var link = new Uri(SiteLink + "download.php/" + torrentId + "/" + torrentId + ".torrent");
var publishDate = DateTimeUtil.UnixTimestampToDateTime((long)row.ctime).ToLocalTime();
var imdb = ParseUtil.GetImdbID(imdbId);
var release = new ReleaseInfo
{
Title = title,
Details = details,
Guid = details,
Link = link,
PublishDate = publishDate,
Category = MapTrackerCatToNewznab(row.c.ToString()),
Size = (long)row.size,
Files = (long)row.files,
Grabs = (long)row.completed,
Seeders = seeders,
Peers = seeders + (int)row.leechers,
Imdb = imdb,
DownloadVolumeFactor = downloadMultiplier,
UploadVolumeFactor = 1,
MinimumRatio = 1,
MinimumSeedTime = 172800 // 48 hours
};
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError(results.ContentString, ex);
}
return(releases);
}
private async Task <IEnumerable <ReleaseInfo> > PerformRegularQueryAsync(TorznabQuery query, string hebName = null)
{
var releases = new List <ReleaseInfo>();
var searchUrl = SearchUrl;
var searchString = query.GetQueryString();
if (query.IsImdbQuery)
{
searchString = query.ImdbID;
}
if (hebName != null)
{
searchString = hebName + " - עונה " + query.Season + " פרק " + query.Episode;
}
searchUrl += "?";
if (!string.IsNullOrWhiteSpace(searchString))
{
var strEncoded = WebUtilityHelpers.UrlEncode(searchString, Encoding);
searchUrl += "&query=" + strEncoded + "&matchquery=any";
}
searchUrl = MapTorznabCapsToTrackers(query).Aggregate(searchUrl, (current, cat) => $"{current}&c[]={cat}");
var data = await RequestWithCookiesAndRetryAsync(searchUrl);
try
{
var parser = new HtmlParser();
var dom = parser.ParseDocument(data.ContentString);
var rows = dom.QuerySelectorAll("tr.box_torrent");
foreach (var row in rows)
{
var release = new ReleaseInfo();
var mainTitleLink = row.QuerySelector("div.main_title > a");
release.Title = mainTitleLink.GetAttribute("longtitle");
if (string.IsNullOrWhiteSpace(release.Title))
{
release.Title = mainTitleLink.TextContent;
}
release.MinimumRatio = 1;
release.MinimumSeedTime = 172800; // 48 hours
release.Grabs = ParseUtil.CoerceLong(row.QuerySelector("td:nth-child(5)").TextContent.Replace(",", ""));
release.Seeders = ParseUtil.CoerceInt(row.QuerySelector("td:nth-child(6)").TextContent.Replace(",", ""));
release.Peers = ParseUtil.CoerceInt(row.QuerySelector("td:nth-child(7)").TextContent.Replace(",", "")) +
release.Seeders;
var fullSize = row.QuerySelector("td:nth-child(4)").TextContent;
release.Size = ReleaseInfo.GetBytes(fullSize);
release.Comments = new Uri(SiteLink + row.QuerySelector("a.threadlink[href]").GetAttribute("href"));
release.Link = new Uri(SiteLink + row.QuerySelector("a:has(div.dlimg)").GetAttribute("href"));
release.Guid = release.Comments;
//some releases have invalid banner URLs, ignore the banners in this case
if (Uri.TryCreate(row.QuerySelector("a[imgsrc]").GetAttribute("imgsrc"),
UriKind.Absolute, out var banner))
{
release.BannerUrl = banner;
}
var dateStringAll = row.QuerySelector("div.up_info2").ChildNodes.Last().TextContent;
var dateParts = dateStringAll.Split(' ');
var dateString = dateParts[dateParts.Length - 2] + " " + dateParts[dateParts.Length - 1];
release.PublishDate = DateTime.ParseExact(dateString, "dd/MM/yy HH:mm", CultureInfo.InvariantCulture);
var categoryLink = row.QuerySelector("a[href^=\"/browse.php?cat=\"]").GetAttribute("href");
var catid = ParseUtil.GetArgumentFromQueryString(categoryLink, "cat");
release.Category = MapTrackerCatToNewznab(catid);
if (row.QuerySelector("a[href^=\"?freeleech=1\"]") != null)
{
release.DownloadVolumeFactor = 0;
}
else
{
release.DownloadVolumeFactor = 1;
}
release.UploadVolumeFactor = 1;
var subTitle = row.QuerySelector("div.sub_title");
var imdbLink = subTitle.QuerySelector("span.imdb-inline > a");
if (imdbLink != null)
{
release.Imdb = ParseUtil.GetLongFromString(imdbLink.GetAttribute("href"));
}
release.Description = subTitle.FirstChild.TextContent;
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError(data.ContentString, ex);
}
return(releases);
}
override protected async Task <WebClientByteResult> Run(WebRequest webRequest)
{
HttpResponseMessage response = null;
var request = new HttpRequestMessage();
request.Headers.ExpectContinue = false;
request.RequestUri = new Uri(webRequest.Url);
if (webRequest.EmulateBrowser == true)
{
request.Headers.UserAgent.ParseAdd(BrowserUtil.ChromeUserAgent);
}
else
{
request.Headers.UserAgent.ParseAdd("Jackett/" + configService.GetVersion());
}
// clear cookies from cookiecontainer
var oldCookies = cookies.GetCookies(request.RequestUri);
foreach (Cookie oldCookie in oldCookies)
{
oldCookie.Expired = true;
}
if (!string.IsNullOrEmpty(webRequest.Cookies))
{
// add cookies to cookiecontainer
var cookieUrl = new Uri(request.RequestUri.Scheme + "://" + request.RequestUri.Host); // don't include the path, Scheme is needed for mono compatibility
foreach (var ccookiestr in webRequest.Cookies.Split(';'))
{
var cookiestrparts = ccookiestr.Split('=');
var name = cookiestrparts[0].Trim();
if (string.IsNullOrWhiteSpace(name))
{
continue;
}
var value = "";
if (cookiestrparts.Length >= 2)
{
value = cookiestrparts[1].Trim();
}
var cookie = new Cookie(name, value);
cookies.Add(cookieUrl, cookie);
}
}
if (webRequest.Headers != null)
{
foreach (var header in webRequest.Headers)
{
if (header.Key != "Content-Type")
{
request.Headers.TryAddWithoutValidation(header.Key, header.Value);
}
}
}
if (!string.IsNullOrEmpty(webRequest.Referer))
{
request.Headers.Referrer = new Uri(webRequest.Referer);
}
if (!string.IsNullOrEmpty(webRequest.RawBody))
{
var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast <KeyValuePair <string, string>?>().FirstOrDefault();
if (type.HasValue)
{
var str = new StringContent(webRequest.RawBody);
str.Headers.Remove("Content-Type");
str.Headers.Add("Content-Type", type.Value.Value);
request.Content = str;
}
else
{
request.Content = new StringContent(webRequest.RawBody);
}
request.Method = HttpMethod.Post;
}
else if (webRequest.Type == RequestType.POST)
{
if (webRequest.PostData != null)
{
request.Content = new FormUrlEncodedContent(webRequest.PostData);
}
request.Method = HttpMethod.Post;
}
else
{
request.Method = HttpMethod.Get;
}
response = await client.SendAsync(request);
var result = new WebClientByteResult();
result.Content = await response.Content.ReadAsByteArrayAsync();
foreach (var header in response.Headers)
{
IEnumerable <string> value = header.Value;
result.Headers[header.Key.ToLowerInvariant()] = value.ToArray();
}
// some cloudflare clients are using a refresh header
// Pull it out manually
if (response.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh"))
{
var refreshHeaders = response.Headers.GetValues("Refresh");
var redirval = "";
var redirtime = 0;
if (refreshHeaders != null)
{
foreach (var value in refreshHeaders)
{
var start = value.IndexOf("=");
var end = value.IndexOf(";");
var len = value.Length;
if (start > -1)
{
redirval = value.Substring(start + 1);
result.RedirectingTo = redirval;
// normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
// of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
// it shoudln't include service unavailable..only if we have this redirect header.
response.StatusCode = System.Net.HttpStatusCode.Redirect;
redirtime = Int32.Parse(value.Substring(0, end));
System.Threading.Thread.Sleep(redirtime * 1000);
}
}
}
}
if (response.Headers.Location != null)
{
result.RedirectingTo = response.Headers.Location.ToString();
}
// Mono won't add the baseurl to relative redirects.
// e.g. a "Location: /index.php" header will result in the Uri "file:///index.php"
// See issue #1200
if (result.RedirectingTo != null && result.RedirectingTo.StartsWith("file://"))
{
// URL decoding apparently is needed to, without it e.g. Demonoid download is broken
// TODO: is it always needed (not just for relative redirects)?
var newRedirectingTo = WebUtilityHelpers.UrlDecode(result.RedirectingTo, webRequest.Encoding);
newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + "://" + request.RequestUri.Host);
logger.Debug("[MONO relative redirect bug] Rewriting relative redirect URL from " + result.RedirectingTo + " to " + newRedirectingTo);
result.RedirectingTo = newRedirectingTo;
}
result.Status = response.StatusCode;
// Compatiblity issue between the cookie format and httpclient
// Pull it out manually ignoring the expiry date then set it manually
// http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
IEnumerable <string> cookieHeaders;
var responseCookies = new List <Tuple <string, string> >();
if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
{
foreach (var value in cookieHeaders)
{
logger.Debug(value);
var nameSplit = value.IndexOf('=');
if (nameSplit > -1)
{
responseCookies.Add(new Tuple <string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';'))) + ";"));
}
}
var cookieBuilder = new StringBuilder();
foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1))
{
cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
}
result.Cookies = cookieBuilder.ToString().Trim();
}
ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
return(result);
}
public static string GetQueryString(this NameValueCollection collection, Encoding encoding = null) =>
string.Join("&", collection.AllKeys.Select(a =>
$"{a}={WebUtilityHelpers.UrlEncode(collection[a], encoding ?? Encoding.UTF8)}"));
/// <summary>
/// Build query to process
/// </summary>
/// <param name="term">Term to search</param>
/// <param name="query">Torznab Query for categories mapping</param>
/// <param name="url">Search url for provider</param>
/// <param name="page">Page number to request</param>
/// <returns>URL to query for parsing and processing results</returns>
private string BuildQuery(string term, TorznabQuery query, string url, int page = 0)
{
var parameters = new NameValueCollection();
var categoriesList = MapTorznabCapsToTrackers(query);
string searchterm = term;
// Building our tracker query
parameters.Add("searchin", "title");
parameters.Add("incldead", "0");
// If search term provided
if (!string.IsNullOrWhiteSpace(query.ImdbID))
{
searchterm = "imdbsearch=" + query.ImdbID;
}
else if (!string.IsNullOrWhiteSpace(term))
{
searchterm = "search=" + WebUtilityHelpers.UrlEncode(term, Encoding.GetEncoding(28591));
}
else
{
// Showing all torrents (just for output function)
searchterm = "search=";
term = "all";
}
// Loop on categories and change the catagories for search purposes
for (int i = 0; i < categoriesList.Count; i++)
{
// APPS
if (new[] { "63", "17", "12", "62", "64" }.Any(c => categoriesList[i].Contains(categoriesList[i])))
{
categoriesList[i] = categoriesList[i].Replace("cat=", "cats5[]=");
}
// Books
if (new[] { "54", "9" }.Any(c => categoriesList[i].Contains(categoriesList[i])))
{
categoriesList[i] = categoriesList[i].Replace("cat=", "cats6[]=");
}
// Games
if (new[] { "24", "53", "49", "51" }.Any(c => categoriesList[i].Contains(categoriesList[i])))
{
categoriesList[i] = categoriesList[i].Replace("cat=", "cats3[]=");
}
// Movies
if (new[] { "35", "42", "47", "15", "58", "16", "6", "21", "19", "22", "20", "25", "10", "23", "65" }.Any(c => categoriesList[i].Contains(categoriesList[i])))
{
categoriesList[i] = categoriesList[i].Replace("cat=", "cats1[]=");
}
// Music
if (new[] { "28", "60", "4", "59", "61", "1" }.Any(c => categoriesList[i].Contains(categoriesList[i])))
{
categoriesList[i] = categoriesList[i].Replace("cat=", "cats4[]=");
}
// Series
if (new[] { "48", "57", "11", "7", "31", "30", "32", "5", "66" }.Any(c => categoriesList[i].Contains(categoriesList[i])))
{
categoriesList[i] = categoriesList[i].Replace("cat=", "cats2[]=");
}
}
// Build category search string
var CatQryStr = "";
foreach (var cat in categoriesList)
{
CatQryStr += cat + "&";
}
// Building our query
url += "?" + CatQryStr + searchterm + "&" + parameters.GetQueryString();
Output("\nBuilded query for \"" + term + "\"... " + url);
// Return our search url
return(url);
}
public static string GetQueryString(this ICollection <KeyValuePair <string, string> > collection, Encoding encoding = null) =>
string.Join("&", collection.Select(a =>
$"{a.Key}={WebUtilityHelpers.UrlEncode(a.Value, encoding ?? Encoding.UTF8)}"));
public static string GetQueryString(this IEnumerable <KeyValuePair <string, string> > collection,
Encoding encoding = null, string separator = "&") =>
string.Join(separator,
collection.Select(a => $"{a.Key}={WebUtilityHelpers.UrlEncode(a.Value, encoding ?? Encoding.UTF8)}"));
