public IList <ReleaseInfo> ParseResponse(IndexerResponse indexerResponse) { var torrentInfos = new List <TorrentInfo>(); if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK) { throw new IndexerException(indexerResponse, $"Unexpected response status {indexerResponse.HttpResponse.StatusCode} code from API request"); } if (!indexerResponse.HttpResponse.Headers.ContentType.Contains(HttpAccept.Json.Value)) { throw new IndexerException(indexerResponse, $"Unexpected response header {indexerResponse.HttpResponse.Headers.ContentType} from API request, expected {HttpAccept.Json.Value}"); } var jsonResponse = new HttpResponse <InternetArchiveResponse>(indexerResponse.HttpResponse); foreach (var searchResult in jsonResponse.Resource.SearchResults) { var title = searchResult.Title ?? searchResult.Identifier; var downloadUrl = string.Format("{0}/download/{1}/{1}_archive.torrent", _settings.BaseUrl.TrimEnd('/'), searchResult.Identifier); var detailsUrl = string.Format("{0}/details/{1}", _settings.BaseUrl.TrimEnd('/'), searchResult.Identifier); var category = _categories.MapTrackerCatToNewznab(searchResult.MediaType ?? "other"); var release = new TorrentInfo { Categories = category, CommentUrl = detailsUrl, DownloadUrl = downloadUrl, DownloadVolumeFactor = 0, Guid = detailsUrl, Grabs = searchResult.Downloads, InfoHash = searchResult.InfoHash, InfoUrl = detailsUrl, Peers = 2, PublishDate = searchResult.PublicDate, Seeders = 1, Size = searchResult.Size, Title = title, UploadVolumeFactor = 1 }; if (!_settings.TorrentFileOnly && searchResult.InfoHash != null) { release.MagnetUrl = MagnetLinkBuilder.BuildPublicMagnetLink(searchResult.InfoHash, title); } torrentInfos.Add(release); } return(torrentInfos.ToArray()); }
public IList <ReleaseInfo> ParseResponse(IndexerResponse indexerResponse) { var torrentInfos = new List <ReleaseInfo>(); if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK) { // Remove cookie cache CookiesUpdater(null, null); throw new IndexerException(indexerResponse, $"Unexpected response status {indexerResponse.HttpResponse.StatusCode} code from API request"); } var results = JsonConvert.DeserializeObject <List <TorrentParadiseResult> >(indexerResponse.Content); if (results == null) { return(torrentInfos); } foreach (var result in results) { var magnetUrl = MagnetLinkBuilder.BuildPublicMagnetLink(result.Id, result.Text); var release = new TorrentInfo { Title = result.Text, Size = result.Size, Seeders = result.Seeders, Peers = result.Seeders + result.Leechers, InfoHash = result.Id, Guid = magnetUrl, MagnetUrl = magnetUrl, PublishDate = DateTime.UtcNow, InfoUrl = _settings.BaseUrl, DownloadVolumeFactor = 0, UploadVolumeFactor = 1, Categories = new List <IndexerCategory> { NewznabStandardCategory.Other } }; torrentInfos.Add(release); } return(torrentInfos.ToArray()); }
public IList <ReleaseInfo> ParseResponse(IndexerResponse indexerResponse) { var torrentInfos = new List <ReleaseInfo>(); var queryResponseItems = JsonConvert.DeserializeObject <List <ThePirateBayTorrent> >(indexerResponse.Content); // The API returns a single item to represent a state of no results. Avoid returning this result. if (queryResponseItems.Count == 1 && queryResponseItems.First().Id == 0) { return(torrentInfos); } foreach (var item in queryResponseItems) { var details = item.Id == 0 ? null : $"{_settings.BaseUrl}description.php?id={item.Id}"; var imdbId = string.IsNullOrEmpty(item.Imdb) ? null : ParseUtil.GetImdbID(item.Imdb); var torrentItem = new TorrentInfo { Title = item.Name, Categories = _categories.MapTrackerCatToNewznab(item.Category.ToString()), Guid = details, InfoUrl = details, InfoHash = item.InfoHash, // magnet link is auto generated from infohash PublishDate = DateTimeUtil.UnixTimestampToDateTime(item.Added), Seeders = item.Seeders, Peers = item.Seeders + item.Leechers, Size = item.Size, Files = item.NumFiles, DownloadVolumeFactor = 0, UploadVolumeFactor = 1, ImdbId = imdbId.GetValueOrDefault() }; if (item.InfoHash != null) { torrentItem.MagnetUrl = MagnetLinkBuilder.BuildPublicMagnetLink(item.InfoHash, item.Name); } torrentInfos.Add(torrentItem); } return(torrentInfos.ToArray()); }
public async Task <HttpRequest> DownloadRequest(Uri link) { Cookies = GetCookies(); var method = HttpMethod.Get; var headers = new Dictionary <string, string>(); var variables = GetBaseTemplateVariables(); AddTemplateVariablesFromUri(variables, link, ".DownloadUri"); headers = ParseCustomHeaders(_definition.Search?.Headers, variables); if (_definition.Download != null) { var download = _definition.Download; HttpResponse response = null; var request = new HttpRequestBuilder(link.ToString()) .SetCookies(Cookies ?? new Dictionary <string, string>()) .SetHeaders(headers ?? new Dictionary <string, string>()) .SetEncoding(_encoding) .Build(); request.AllowAutoRedirect = true; var beforeBlock = download.Before; if (beforeBlock != null) { if (beforeBlock.Pathselector != null) { response = await HttpClient.ExecuteProxiedAsync(request, Definition); beforeBlock.Path = MatchSelector(response, beforeBlock.Pathselector, variables); } response = await HandleRequest(beforeBlock, variables, link.ToString()); } if (download.Method == "post") { method = HttpMethod.Post; } if (download.Infohash != null) { try { if (!download.Infohash.UseBeforeResponse || download.Before == null || response == null) { response = await HttpClient.ExecuteProxiedAsync(request, Definition); } var hash = MatchSelector(response, download.Infohash.Hash, variables); if (hash == null) { throw new CardigannException($"InfoHash selectors didn't match"); } var title = MatchSelector(response, download.Infohash.Title, variables); if (title == null) { throw new CardigannException($"InfoHash selectors didn't match"); } var magnet = MagnetLinkBuilder.BuildPublicMagnetLink(hash, title); var torrentLink = ResolvePath(magnet, link); var hashDownloadRequest = new HttpRequestBuilder(torrentLink.AbsoluteUri) .SetCookies(Cookies ?? new Dictionary <string, string>()) .SetHeaders(headers ?? new Dictionary <string, string>()) .SetEncoding(_encoding) .Build(); hashDownloadRequest.Method = method; return(hashDownloadRequest); } catch (Exception) { _logger.Error("CardigannIndexer ({0}): Exception with InfoHash block with hashSelector {1} and titleSelector {2}", _definition.Id, download.Infohash.Hash.Selector, download.Infohash.Title.Selector); } } else if (download.Selectors != null) { foreach (var selector in download.Selectors) { var queryselector = ApplyGoTemplateText(selector.Selector, variables); try { if (!selector.UseBeforeResponse || download.Before == null || response == null) { response = await HttpClient.ExecuteProxiedAsync(request, Definition); } var href = MatchSelector(response, selector, variables, debugMatch: true); if (href == null) { continue; } var torrentLink = ResolvePath(href, link); if (torrentLink.Scheme != "magnet" && _definition.TestLinkTorrent) { // Test link var testLinkRequest = new HttpRequestBuilder(torrentLink.ToString()) .SetCookies(Cookies ?? new Dictionary <string, string>()) .SetHeaders(headers ?? new Dictionary <string, string>()) .SetEncoding(_encoding) .Build(); response = await HttpClient.ExecuteProxiedAsync(testLinkRequest, Definition); var content = response.Content; if (content.Length >= 1 && content[0] != 'd') { _logger.Debug("CardigannIndexer ({0}): Download selector {1}'s torrent file is invalid, retrying with next available selector", _definition.Id, queryselector); continue; } } link = torrentLink; var selectorDownloadRequest = new HttpRequestBuilder(link.AbsoluteUri) .SetCookies(Cookies ?? new Dictionary <string, string>()) .SetHeaders(headers ?? new Dictionary <string, string>()) .SetEncoding(_encoding) .Build(); selectorDownloadRequest.Method = method; return(selectorDownloadRequest); } catch (Exception e) { _logger.Error("{0} CardigannIndexer ({1}): An exception occurred while trying selector {2}, retrying with next available selector", e, _definition.Id, queryselector); throw new CardigannException(string.Format("An exception occurred while trying selector {0}", queryselector)); } } } } var downloadRequest = new HttpRequestBuilder(link.AbsoluteUri) .SetCookies(Cookies ?? new Dictionary <string, string>()) .SetHeaders(headers ?? new Dictionary <string, string>()) .SetEncoding(_encoding) .Build(); downloadRequest.Method = method; return(downloadRequest); }
public IList <ReleaseInfo> ParseResponse(IndexerResponse indexerResponse) { var releases = new List <ReleaseInfo>(); _logger.Debug("Parsing"); var indexerLogging = _configService.LogIndexerResponse; if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK) { // Remove cookie cache if (indexerResponse.HttpResponse.HasHttpRedirect && indexerResponse.HttpResponse.RedirectUrl .ContainsIgnoreCase("login.php")) { CookiesUpdater(null, null); throw new IndexerException(indexerResponse, "We are being redirected to the login page. Most likely your session expired or was killed. Try testing the indexer in the settings."); } throw new IndexerException(indexerResponse, $"Unexpected response status {indexerResponse.HttpResponse.StatusCode} code from API request"); } var results = indexerResponse.Content; var request = indexerResponse.Request as CardigannRequest; var variables = request.Variables; var search = _definition.Search; var searchUrlUri = new Uri(request.Url.FullUri); if (request.SearchPath.Response != null && request.SearchPath.Response.Type.Equals("json")) { if (request.SearchPath.Response != null && request.SearchPath.Response.NoResultsMessage != null && (request.SearchPath.Response.NoResultsMessage.Equals(results) || (request.SearchPath.Response.NoResultsMessage == string.Empty && results == string.Empty))) { return(releases); } var parsedJson = JToken.Parse(results); if (parsedJson == null) { throw new IndexerException(indexerResponse, "Error Parsing Json Response"); } if (search.Rows.Count != null) { var countVal = HandleJsonSelector(search.Rows.Count, parsedJson, variables); if (int.TryParse(countVal, out var count)) { if (count < 1) { return(releases); } } } var rowsArray = JsonParseRowsSelector(parsedJson, search.Rows.Selector); if (rowsArray == null) { throw new IndexerException(indexerResponse, "Error Parsing Rows Selector"); } foreach (var row in rowsArray) { var selObj = search.Rows.Attribute != null?row.SelectToken(search.Rows.Attribute).Value <JToken>() : row; var mulRows = search.Rows.Multiple ? selObj.Values <JObject>() : new List <JObject> { selObj.Value <JObject>() }; foreach (var mulRow in mulRows) { var release = new TorrentInfo(); foreach (var field in search.Fields) { var fieldParts = field.Key.Split('|'); var fieldName = fieldParts[0]; var fieldModifiers = new List <string>(); for (var i = 1; i < fieldParts.Length; i++) { fieldModifiers.Add(fieldParts[i]); } string value = null; var variablesKey = ".Result." + fieldName; var isOptional = OptionalFields.Contains(field.Key) || fieldModifiers.Contains("optional") || field.Value.Optional; try { var parentObj = mulRow; if (field.Value.Selector != null && field.Value.Selector.StartsWith("..")) { parentObj = row.Value <JObject>(); } value = HandleJsonSelector(field.Value, parentObj, variables, !isOptional); if (isOptional && string.IsNullOrWhiteSpace(value)) { variables[variablesKey] = null; continue; } variables[variablesKey] = ParseFields(value, fieldName, release, fieldModifiers, searchUrlUri); } catch (Exception ex) { if (!variables.ContainsKey(variablesKey)) { variables[variablesKey] = null; } if (isOptional) { variables[variablesKey] = null; continue; } throw new CardigannException(string.Format("Error while parsing field={0}, selector={1}, value={2}: {3}", field.Key, field.Value.Selector, value ?? "<null>", ex.Message)); } var filters = search.Rows.Filters; var skipRelease = ParseRowFilters(filters, release, variables, row); if (skipRelease) { continue; } } releases.Add(release); } } } else { try { IHtmlCollection <IElement> rowsDom; if (request.SearchPath.Response != null && request.SearchPath.Response.Type.Equals("xml")) { var searchResultParser = new XmlParser(); var searchResultDocument = searchResultParser.ParseDocument(results); if (search.Preprocessingfilters != null) { results = ApplyFilters(results, search.Preprocessingfilters, variables); searchResultDocument = searchResultParser.ParseDocument(results); _logger.Trace(string.Format("CardigannIndexer ({0}): result after preprocessingfilters: {1}", _definition.Id, results)); } var rowsSelector = ApplyGoTemplateText(search.Rows.Selector, variables); rowsDom = searchResultDocument.QuerySelectorAll(rowsSelector); } else { var searchResultParser = new HtmlParser(); var searchResultDocument = searchResultParser.ParseDocument(results); if (search.Preprocessingfilters != null) { results = ApplyFilters(results, search.Preprocessingfilters, variables); searchResultDocument = searchResultParser.ParseDocument(results); _logger.Trace(string.Format("CardigannIndexer ({0}): result after preprocessingfilters: {1}", _definition.Id, results)); } var rowsSelector = ApplyGoTemplateText(search.Rows.Selector, variables); rowsDom = searchResultDocument.QuerySelectorAll(rowsSelector); } var rows = new List <IElement>(); foreach (var rowDom in rowsDom) { rows.Add(rowDom); } // merge following rows for After selector var after = search.Rows.After; if (after > 0) { for (var i = 0; i < rows.Count; i += 1) { var currentRow = rows[i]; for (var j = 0; j < after; j += 1) { var mergeRowIndex = i + j + 1; var mergeRow = rows[mergeRowIndex]; var mergeNodes = new List <INode>(); foreach (var node in mergeRow.ChildNodes) { mergeNodes.Add(node); } currentRow.Append(mergeNodes.ToArray()); } rows.RemoveRange(i + 1, after); } } foreach (var row in rows) { try { var release = new TorrentInfo(); // Parse fields foreach (var field in search.Fields) { var fieldParts = field.Key.Split('|'); var fieldName = fieldParts[0]; var fieldModifiers = new List <string>(); for (var i = 1; i < fieldParts.Length; i++) { fieldModifiers.Add(fieldParts[i]); } string value = null; var variablesKey = ".Result." + fieldName; var isOptional = OptionalFields.Contains(field.Key) || fieldModifiers.Contains("optional") || field.Value.Optional; try { value = HandleSelector(field.Value, row, variables, !isOptional); if (isOptional && string.IsNullOrWhiteSpace(value)) { variables[variablesKey] = null; continue; } variables[variablesKey] = ParseFields(value, fieldName, release, fieldModifiers, searchUrlUri); } catch (Exception ex) { if (!variables.ContainsKey(variablesKey)) { variables[variablesKey] = null; } if (OptionalFields.Contains(field.Key) || fieldModifiers.Contains("optional") || field.Value.Optional) { variables[variablesKey] = null; continue; } if (indexerLogging) { _logger.Trace("Error while parsing field={0}, selector={1}, value={2}: {3}", field.Key, field.Value.Selector, value == null ? "<null>" : value, ex.Message); } } } var filters = search.Rows.Filters; var skipRelease = ParseRowFilters(filters, release, variables, row); if (skipRelease) { continue; } // if DateHeaders is set go through the previous rows and look for the header selector var dateHeaders = _definition.Search.Rows.Dateheaders; if (release.PublishDate == DateTime.MinValue && dateHeaders != null) { var prevRow = row.PreviousElementSibling; string value = null; if (prevRow == null) { // continue with parent var parent = row.ParentElement; if (parent != null) { prevRow = parent.PreviousElementSibling; } } while (prevRow != null) { var curRow = prevRow; _logger.Debug(prevRow.OuterHtml); try { value = HandleSelector(dateHeaders, curRow); break; } catch (Exception) { // do nothing } prevRow = curRow.PreviousElementSibling; if (prevRow == null) { // continue with parent var parent = curRow.ParentElement; if (parent != null) { prevRow = parent.PreviousElementSibling; } } } if (value == null && dateHeaders.Optional == false) { throw new CardigannException(string.Format("No date header row found for {0}", release.ToString())); } if (value != null) { release.PublishDate = DateTimeUtil.FromUnknown(value); } } releases.Add(release); } catch (Exception ex) { _logger.Error(ex, "CardigannIndexer ({0}): Error while parsing row '{1}':\n\n{2}", _definition.Id, row.ToHtmlPretty()); } } } catch (Exception) { // OnParseError(results, ex); throw; } } /* * if (query.Limit > 0) * { * releases = releases.Take(query.Limit).ToList(); * }*/ releases.ForEach(c => { // generate magnet link from info hash (not allowed for private sites) if (((TorrentInfo)c).MagnetUrl == null && !string.IsNullOrWhiteSpace(((TorrentInfo)c).InfoHash) && _definition.Type != "private") { ((TorrentInfo)c).MagnetUrl = MagnetLinkBuilder.BuildPublicMagnetLink(((TorrentInfo)c).InfoHash, c.Title); } // generate info hash from magnet link if (((TorrentInfo)c).MagnetUrl != null && string.IsNullOrWhiteSpace(((TorrentInfo)c).InfoHash)) { ((TorrentInfo)c).InfoHash = MagnetLinkBuilder.GetInfoHashFromMagnet(((TorrentInfo)c).MagnetUrl); } }); _logger.Debug($"Got {releases.Count} releases"); return(releases); }