public IList <ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
{
var torrentInfos = new List <TorrentInfo>();
if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
{
throw new IndexerException(indexerResponse, $"Unexpected response status {indexerResponse.HttpResponse.StatusCode} code from API request");
}
if (!indexerResponse.HttpResponse.Headers.ContentType.Contains(HttpAccept.Json.Value))
{
throw new IndexerException(indexerResponse, $"Unexpected response header {indexerResponse.HttpResponse.Headers.ContentType} from API request, expected {HttpAccept.Json.Value}");
}
var jsonResponse = new HttpResponse <InternetArchiveResponse>(indexerResponse.HttpResponse);
foreach (var searchResult in jsonResponse.Resource.SearchResults)
{
var title = searchResult.Title ?? searchResult.Identifier;
var downloadUrl = string.Format("{0}/download/{1}/{1}_archive.torrent", _settings.BaseUrl.TrimEnd('/'), searchResult.Identifier);
var detailsUrl = string.Format("{0}/details/{1}", _settings.BaseUrl.TrimEnd('/'), searchResult.Identifier);
var category = _categories.MapTrackerCatToNewznab(searchResult.MediaType ?? "other");
var release = new TorrentInfo
{
Categories = category,
CommentUrl = detailsUrl,
DownloadUrl = downloadUrl,
DownloadVolumeFactor = 0,
Guid = detailsUrl,
Grabs = searchResult.Downloads,
InfoHash = searchResult.InfoHash,
InfoUrl = detailsUrl,
Peers = 2,
PublishDate = searchResult.PublicDate,
Seeders = 1,
Size = searchResult.Size,
Title = title,
UploadVolumeFactor = 1
};
if (!_settings.TorrentFileOnly && searchResult.InfoHash != null)
{
release.MagnetUrl = MagnetLinkBuilder.BuildPublicMagnetLink(searchResult.InfoHash, title);
}
torrentInfos.Add(release);
}
return(torrentInfos.ToArray());
}
public IList <ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
{
var torrentInfos = new List <ReleaseInfo>();
if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
{
// Remove cookie cache
CookiesUpdater(null, null);
throw new IndexerException(indexerResponse, $"Unexpected response status {indexerResponse.HttpResponse.StatusCode} code from API request");
}
var results = JsonConvert.DeserializeObject <List <TorrentParadiseResult> >(indexerResponse.Content);
if (results == null)
{
return(torrentInfos);
}
foreach (var result in results)
{
var magnetUrl = MagnetLinkBuilder.BuildPublicMagnetLink(result.Id, result.Text);
var release = new TorrentInfo
{
Title = result.Text,
Size = result.Size,
Seeders = result.Seeders,
Peers = result.Seeders + result.Leechers,
InfoHash = result.Id,
Guid = magnetUrl,
MagnetUrl = magnetUrl,
PublishDate = DateTime.UtcNow,
InfoUrl = _settings.BaseUrl,
DownloadVolumeFactor = 0,
UploadVolumeFactor = 1,
Categories = new List <IndexerCategory> {
NewznabStandardCategory.Other
}
};
torrentInfos.Add(release);
}
return(torrentInfos.ToArray());
}
public IList <ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
{
var torrentInfos = new List <ReleaseInfo>();
var queryResponseItems = JsonConvert.DeserializeObject <List <ThePirateBayTorrent> >(indexerResponse.Content);
// The API returns a single item to represent a state of no results. Avoid returning this result.
if (queryResponseItems.Count == 1 && queryResponseItems.First().Id == 0)
{
return(torrentInfos);
}
foreach (var item in queryResponseItems)
{
var details = item.Id == 0 ? null : $"{_settings.BaseUrl}description.php?id={item.Id}";
var imdbId = string.IsNullOrEmpty(item.Imdb) ? null : ParseUtil.GetImdbID(item.Imdb);
var torrentItem = new TorrentInfo
{
Title = item.Name,
Categories = _categories.MapTrackerCatToNewznab(item.Category.ToString()),
Guid = details,
InfoUrl = details,
InfoHash = item.InfoHash, // magnet link is auto generated from infohash
PublishDate = DateTimeUtil.UnixTimestampToDateTime(item.Added),
Seeders = item.Seeders,
Peers = item.Seeders + item.Leechers,
Size = item.Size,
Files = item.NumFiles,
DownloadVolumeFactor = 0,
UploadVolumeFactor = 1,
ImdbId = imdbId.GetValueOrDefault()
};
if (item.InfoHash != null)
{
torrentItem.MagnetUrl = MagnetLinkBuilder.BuildPublicMagnetLink(item.InfoHash, item.Name);
}
torrentInfos.Add(torrentItem);
}
return(torrentInfos.ToArray());
}
public async Task <HttpRequest> DownloadRequest(Uri link)
{
Cookies = GetCookies();
var method = HttpMethod.Get;
var headers = new Dictionary <string, string>();
var variables = GetBaseTemplateVariables();
AddTemplateVariablesFromUri(variables, link, ".DownloadUri");
headers = ParseCustomHeaders(_definition.Search?.Headers, variables);
if (_definition.Download != null)
{
var download = _definition.Download;
HttpResponse response = null;
var request = new HttpRequestBuilder(link.ToString())
.SetCookies(Cookies ?? new Dictionary <string, string>())
.SetHeaders(headers ?? new Dictionary <string, string>())
.SetEncoding(_encoding)
.Build();
request.AllowAutoRedirect = true;
var beforeBlock = download.Before;
if (beforeBlock != null)
{
if (beforeBlock.Pathselector != null)
{
response = await HttpClient.ExecuteProxiedAsync(request, Definition);
beforeBlock.Path = MatchSelector(response, beforeBlock.Pathselector, variables);
}
response = await HandleRequest(beforeBlock, variables, link.ToString());
}
if (download.Method == "post")
{
method = HttpMethod.Post;
}
if (download.Infohash != null)
{
try
{
if (!download.Infohash.UseBeforeResponse || download.Before == null || response == null)
{
response = await HttpClient.ExecuteProxiedAsync(request, Definition);
}
var hash = MatchSelector(response, download.Infohash.Hash, variables);
if (hash == null)
{
throw new CardigannException($"InfoHash selectors didn't match");
}
var title = MatchSelector(response, download.Infohash.Title, variables);
if (title == null)
{
throw new CardigannException($"InfoHash selectors didn't match");
}
var magnet = MagnetLinkBuilder.BuildPublicMagnetLink(hash, title);
var torrentLink = ResolvePath(magnet, link);
var hashDownloadRequest = new HttpRequestBuilder(torrentLink.AbsoluteUri)
.SetCookies(Cookies ?? new Dictionary <string, string>())
.SetHeaders(headers ?? new Dictionary <string, string>())
.SetEncoding(_encoding)
.Build();
hashDownloadRequest.Method = method;
return(hashDownloadRequest);
}
catch (Exception)
{
_logger.Error("CardigannIndexer ({0}): Exception with InfoHash block with hashSelector {1} and titleSelector {2}",
_definition.Id,
download.Infohash.Hash.Selector,
download.Infohash.Title.Selector);
}
}
else if (download.Selectors != null)
{
foreach (var selector in download.Selectors)
{
var queryselector = ApplyGoTemplateText(selector.Selector, variables);
try
{
if (!selector.UseBeforeResponse || download.Before == null || response == null)
{
response = await HttpClient.ExecuteProxiedAsync(request, Definition);
}
var href = MatchSelector(response, selector, variables, debugMatch: true);
if (href == null)
{
continue;
}
var torrentLink = ResolvePath(href, link);
if (torrentLink.Scheme != "magnet" && _definition.TestLinkTorrent)
{
// Test link
var testLinkRequest = new HttpRequestBuilder(torrentLink.ToString())
.SetCookies(Cookies ?? new Dictionary <string, string>())
.SetHeaders(headers ?? new Dictionary <string, string>())
.SetEncoding(_encoding)
.Build();
response = await HttpClient.ExecuteProxiedAsync(testLinkRequest, Definition);
var content = response.Content;
if (content.Length >= 1 && content[0] != 'd')
{
_logger.Debug("CardigannIndexer ({0}): Download selector {1}'s torrent file is invalid, retrying with next available selector", _definition.Id, queryselector);
continue;
}
}
link = torrentLink;
var selectorDownloadRequest = new HttpRequestBuilder(link.AbsoluteUri)
.SetCookies(Cookies ?? new Dictionary <string, string>())
.SetHeaders(headers ?? new Dictionary <string, string>())
.SetEncoding(_encoding)
.Build();
selectorDownloadRequest.Method = method;
return(selectorDownloadRequest);
}
catch (Exception e)
{
_logger.Error("{0} CardigannIndexer ({1}): An exception occurred while trying selector {2}, retrying with next available selector", e, _definition.Id, queryselector);
throw new CardigannException(string.Format("An exception occurred while trying selector {0}", queryselector));
}
}
}
}
var downloadRequest = new HttpRequestBuilder(link.AbsoluteUri)
.SetCookies(Cookies ?? new Dictionary <string, string>())
.SetHeaders(headers ?? new Dictionary <string, string>())
.SetEncoding(_encoding)
.Build();
downloadRequest.Method = method;
return(downloadRequest);
}
public IList <ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
{
var releases = new List <ReleaseInfo>();
_logger.Debug("Parsing");
var indexerLogging = _configService.LogIndexerResponse;
if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
{
// Remove cookie cache
if (indexerResponse.HttpResponse.HasHttpRedirect && indexerResponse.HttpResponse.RedirectUrl
.ContainsIgnoreCase("login.php"))
{
CookiesUpdater(null, null);
throw new IndexerException(indexerResponse, "We are being redirected to the login page. Most likely your session expired or was killed. Try testing the indexer in the settings.");
}
throw new IndexerException(indexerResponse, $"Unexpected response status {indexerResponse.HttpResponse.StatusCode} code from API request");
}
var results = indexerResponse.Content;
var request = indexerResponse.Request as CardigannRequest;
var variables = request.Variables;
var search = _definition.Search;
var searchUrlUri = new Uri(request.Url.FullUri);
if (request.SearchPath.Response != null && request.SearchPath.Response.Type.Equals("json"))
{
if (request.SearchPath.Response != null && request.SearchPath.Response.NoResultsMessage != null && (request.SearchPath.Response.NoResultsMessage.Equals(results) || (request.SearchPath.Response.NoResultsMessage == string.Empty && results == string.Empty)))
{
return(releases);
}
var parsedJson = JToken.Parse(results);
if (parsedJson == null)
{
throw new IndexerException(indexerResponse, "Error Parsing Json Response");
}
if (search.Rows.Count != null)
{
var countVal = HandleJsonSelector(search.Rows.Count, parsedJson, variables);
if (int.TryParse(countVal, out var count))
{
if (count < 1)
{
return(releases);
}
}
}
var rowsArray = JsonParseRowsSelector(parsedJson, search.Rows.Selector);
if (rowsArray == null)
{
throw new IndexerException(indexerResponse, "Error Parsing Rows Selector");
}
foreach (var row in rowsArray)
{
var selObj = search.Rows.Attribute != null?row.SelectToken(search.Rows.Attribute).Value <JToken>() : row;
var mulRows = search.Rows.Multiple ? selObj.Values <JObject>() : new List <JObject> {
selObj.Value <JObject>()
};
foreach (var mulRow in mulRows)
{
var release = new TorrentInfo();
foreach (var field in search.Fields)
{
var fieldParts = field.Key.Split('|');
var fieldName = fieldParts[0];
var fieldModifiers = new List <string>();
for (var i = 1; i < fieldParts.Length; i++)
{
fieldModifiers.Add(fieldParts[i]);
}
string value = null;
var variablesKey = ".Result." + fieldName;
var isOptional = OptionalFields.Contains(field.Key) || fieldModifiers.Contains("optional") || field.Value.Optional;
try
{
var parentObj = mulRow;
if (field.Value.Selector != null && field.Value.Selector.StartsWith(".."))
{
parentObj = row.Value <JObject>();
}
value = HandleJsonSelector(field.Value, parentObj, variables, !isOptional);
if (isOptional && string.IsNullOrWhiteSpace(value))
{
variables[variablesKey] = null;
continue;
}
variables[variablesKey] = ParseFields(value, fieldName, release, fieldModifiers, searchUrlUri);
}
catch (Exception ex)
{
if (!variables.ContainsKey(variablesKey))
{
variables[variablesKey] = null;
}
if (isOptional)
{
variables[variablesKey] = null;
continue;
}
throw new CardigannException(string.Format("Error while parsing field={0}, selector={1}, value={2}: {3}", field.Key, field.Value.Selector, value ?? "<null>", ex.Message));
}
var filters = search.Rows.Filters;
var skipRelease = ParseRowFilters(filters, release, variables, row);
if (skipRelease)
{
continue;
}
}
releases.Add(release);
}
}
}
else
{
try
{
IHtmlCollection <IElement> rowsDom;
if (request.SearchPath.Response != null && request.SearchPath.Response.Type.Equals("xml"))
{
var searchResultParser = new XmlParser();
var searchResultDocument = searchResultParser.ParseDocument(results);
if (search.Preprocessingfilters != null)
{
results = ApplyFilters(results, search.Preprocessingfilters, variables);
searchResultDocument = searchResultParser.ParseDocument(results);
_logger.Trace(string.Format("CardigannIndexer ({0}): result after preprocessingfilters: {1}", _definition.Id, results));
}
var rowsSelector = ApplyGoTemplateText(search.Rows.Selector, variables);
rowsDom = searchResultDocument.QuerySelectorAll(rowsSelector);
}
else
{
var searchResultParser = new HtmlParser();
var searchResultDocument = searchResultParser.ParseDocument(results);
if (search.Preprocessingfilters != null)
{
results = ApplyFilters(results, search.Preprocessingfilters, variables);
searchResultDocument = searchResultParser.ParseDocument(results);
_logger.Trace(string.Format("CardigannIndexer ({0}): result after preprocessingfilters: {1}", _definition.Id, results));
}
var rowsSelector = ApplyGoTemplateText(search.Rows.Selector, variables);
rowsDom = searchResultDocument.QuerySelectorAll(rowsSelector);
}
var rows = new List <IElement>();
foreach (var rowDom in rowsDom)
{
rows.Add(rowDom);
}
// merge following rows for After selector
var after = search.Rows.After;
if (after > 0)
{
for (var i = 0; i < rows.Count; i += 1)
{
var currentRow = rows[i];
for (var j = 0; j < after; j += 1)
{
var mergeRowIndex = i + j + 1;
var mergeRow = rows[mergeRowIndex];
var mergeNodes = new List <INode>();
foreach (var node in mergeRow.ChildNodes)
{
mergeNodes.Add(node);
}
currentRow.Append(mergeNodes.ToArray());
}
rows.RemoveRange(i + 1, after);
}
}
foreach (var row in rows)
{
try
{
var release = new TorrentInfo();
// Parse fields
foreach (var field in search.Fields)
{
var fieldParts = field.Key.Split('|');
var fieldName = fieldParts[0];
var fieldModifiers = new List <string>();
for (var i = 1; i < fieldParts.Length; i++)
{
fieldModifiers.Add(fieldParts[i]);
}
string value = null;
var variablesKey = ".Result." + fieldName;
var isOptional = OptionalFields.Contains(field.Key) || fieldModifiers.Contains("optional") || field.Value.Optional;
try
{
value = HandleSelector(field.Value, row, variables, !isOptional);
if (isOptional && string.IsNullOrWhiteSpace(value))
{
variables[variablesKey] = null;
continue;
}
variables[variablesKey] = ParseFields(value, fieldName, release, fieldModifiers, searchUrlUri);
}
catch (Exception ex)
{
if (!variables.ContainsKey(variablesKey))
{
variables[variablesKey] = null;
}
if (OptionalFields.Contains(field.Key) || fieldModifiers.Contains("optional") || field.Value.Optional)
{
variables[variablesKey] = null;
continue;
}
if (indexerLogging)
{
_logger.Trace("Error while parsing field={0}, selector={1}, value={2}: {3}", field.Key, field.Value.Selector, value == null ? "<null>" : value, ex.Message);
}
}
}
var filters = search.Rows.Filters;
var skipRelease = ParseRowFilters(filters, release, variables, row);
if (skipRelease)
{
continue;
}
// if DateHeaders is set go through the previous rows and look for the header selector
var dateHeaders = _definition.Search.Rows.Dateheaders;
if (release.PublishDate == DateTime.MinValue && dateHeaders != null)
{
var prevRow = row.PreviousElementSibling;
string value = null;
if (prevRow == null)
{
// continue with parent
var parent = row.ParentElement;
if (parent != null)
{
prevRow = parent.PreviousElementSibling;
}
}
while (prevRow != null)
{
var curRow = prevRow;
_logger.Debug(prevRow.OuterHtml);
try
{
value = HandleSelector(dateHeaders, curRow);
break;
}
catch (Exception)
{
// do nothing
}
prevRow = curRow.PreviousElementSibling;
if (prevRow == null)
{
// continue with parent
var parent = curRow.ParentElement;
if (parent != null)
{
prevRow = parent.PreviousElementSibling;
}
}
}
if (value == null && dateHeaders.Optional == false)
{
throw new CardigannException(string.Format("No date header row found for {0}", release.ToString()));
}
if (value != null)
{
release.PublishDate = DateTimeUtil.FromUnknown(value);
}
}
releases.Add(release);
}
catch (Exception ex)
{
_logger.Error(ex, "CardigannIndexer ({0}): Error while parsing row '{1}':\n\n{2}", _definition.Id, row.ToHtmlPretty());
}
}
}
catch (Exception)
{
// OnParseError(results, ex);
throw;
}
}
/*
* if (query.Limit > 0)
* {
* releases = releases.Take(query.Limit).ToList();
* }*/
releases.ForEach(c =>
{
// generate magnet link from info hash (not allowed for private sites)
if (((TorrentInfo)c).MagnetUrl == null && !string.IsNullOrWhiteSpace(((TorrentInfo)c).InfoHash) && _definition.Type != "private")
{
((TorrentInfo)c).MagnetUrl = MagnetLinkBuilder.BuildPublicMagnetLink(((TorrentInfo)c).InfoHash, c.Title);
}
// generate info hash from magnet link
if (((TorrentInfo)c).MagnetUrl != null && string.IsNullOrWhiteSpace(((TorrentInfo)c).InfoHash))
{
((TorrentInfo)c).InfoHash = MagnetLinkBuilder.GetInfoHashFromMagnet(((TorrentInfo)c).MagnetUrl);
}
});
_logger.Debug($"Got {releases.Count} releases");
return(releases);
}