public virtual IList <ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
{
var releases = new List <ReleaseInfo>();
if (!PreProcess(indexerResponse))
{
return(releases);
}
using (var xmlTextReader = XmlReader.Create(new StringReader(indexerResponse.Content), new XmlReaderSettings {
DtdProcessing = DtdProcessing.Ignore, IgnoreComments = true
}))
{
var document = XDocument.Load(xmlTextReader);
var items = GetItems(document);
foreach (var item in items)
{
try
{
var reportInfo = ProcessItem(item);
releases.AddIfNotNull(reportInfo);
}
catch (Exception itemEx)
{
itemEx.Data.Add("Item", item.Title());
_logger.ErrorException("An error occurred while processing feed item from " + indexerResponse.Request.Url, itemEx);
}
}
}
return(releases);
}
public virtual IList <ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
{
_indexerResponse = indexerResponse;
var releases = new List <ReleaseInfo>();
if (!PreProcess(indexerResponse))
{
return(releases);
}
var document = LoadXmlDocument(indexerResponse);
var items = GetItems(document);
foreach (var item in items)
{
try
{
var reportInfo = ProcessItem(item);
releases.AddIfNotNull(reportInfo);
}
catch (Exception itemEx)
{
itemEx.Data.Add("Item", item.Title());
_logger.Error(itemEx, "An error occurred while processing feed item from {0}", indexerResponse.Request.Url);
}
}
return(releases);
}
protected virtual XDocument LoadXmlDocument(IndexerResponse indexerResponse)
{
try
{
var content = indexerResponse.Content;
content = ReplaceEntities.Replace(content, ReplaceEntity);
using (var xmlTextReader = XmlReader.Create(new StringReader(content), new XmlReaderSettings {
DtdProcessing = DtdProcessing.Ignore, IgnoreComments = true
}))
{
return(XDocument.Load(xmlTextReader));
}
}
catch (XmlException ex)
{
var contentSample = indexerResponse.Content.Substring(0, Math.Min(indexerResponse.Content.Length, 512));
_logger.Debug("Truncated response content (originally {0} characters): {1}", indexerResponse.Content.Length, contentSample);
ex.Data.Add("ContentLength", indexerResponse.Content.Length);
ex.Data.Add("ContentSample", contentSample);
throw;
}
}
public virtual IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
{
_indexerResponse = indexerResponse;
var releases = new List<ReleaseInfo>();
if (!PreProcess(indexerResponse))
{
return releases;
}
var document = LoadXmlDocument(indexerResponse);
var items = GetItems(document);
foreach (var item in items)
{
try
{
var reportInfo = ProcessItem(item);
releases.AddIfNotNull(reportInfo);
}
catch (Exception itemEx)
{
itemEx.Data.Add("Item", item.Title());
_logger.ErrorException("An error occurred while processing feed item from " + indexerResponse.Request.Url, itemEx);
}
}
return releases;
}
public IEnumerable <XElement> GetItems(IndexerResponse indexerResponse)
{
var document = LoadXmlDocument(indexerResponse);
var items = GetItems(document);
return(items);
}
public IEnumerable<XElement> GetItems(IndexerResponse indexerResponse)
{
var document = LoadXmlDocument(indexerResponse);
var items = GetItems(document);
return items;
}
protected virtual IList <ReleaseInfo> FetchPage(IndexerRequest request, IParseIndexerResponse parser)
{
_logger.Debug("Downloading Feed " + request.Url);
var response = new IndexerResponse(request, _httpClient.Execute(request.HttpRequest));
return(parser.ParseResponse(response).ToList());
}
protected virtual Boolean PreProcess(IndexerResponse indexerResponse)
{
if (indexerResponse.HttpResponse.StatusCode != System.Net.HttpStatusCode.OK)
{
throw new IndexerException("Indexer API call resulted in an unexpected StatusCode [{0}]", indexerResponse.HttpResponse.StatusCode);
}
return(true);
}
protected override bool PreProcess(IndexerResponse indexerResponse)
{
var document = LoadXmlDocument(indexerResponse);
var items = GetItems(document).ToList();
if (items.Count == 1 && GetTitle(items.First()).Equals("No items exist - Try again later"))
{
throw new IndexerException(indexerResponse, "No results were found");
}
return(base.PreProcess(indexerResponse));
}
protected override bool PreProcess(IndexerResponse indexerResponse)
{
var document = LoadXmlDocument(indexerResponse);
var items = GetItems(document).ToList();
if (items.Count == 1 && GetTitle(items.First()).Equals("No items exist - Try again later"))
{
throw new IndexerException(indexerResponse, "No results were found");
}
return base.PreProcess(indexerResponse);
}
protected virtual IList <ReleaseInfo> FetchPage(IndexerRequest request, IParseIndexerResponse parser)
{
var url = request.Url;
_logger.Debug("Downloading Feed " + request.Url);
var response = new IndexerResponse(request, _httpClient.Execute(request.HttpRequest));
if (response.HttpResponse.Headers.ContentType != null && response.HttpResponse.Headers.ContentType.Contains("text/html") &&
request.HttpRequest.Headers.Accept != null && !request.HttpRequest.Headers.Accept.Contains("text/html"))
{
throw new WebException("Indexer responded with html content. Site is likely blocked or unavailable.");
}
return(parser.ParseResponse(response).ToList());
}
protected virtual bool PreProcess(IndexerResponse indexerResponse)
{
if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
{
throw new IndexerException(indexerResponse, "Indexer API call resulted in an unexpected StatusCode [{0}]", indexerResponse.HttpResponse.StatusCode);
}
if (indexerResponse.HttpResponse.Headers.ContentType != null && indexerResponse.HttpResponse.Headers.ContentType.Contains("text/html") &&
indexerResponse.HttpRequest.Headers.Accept != null && !indexerResponse.HttpRequest.Headers.Accept.Contains("text/html"))
{
throw new IndexerException(indexerResponse, "Indexer responded with html content. Site is likely blocked or unavailable.");
}
return(true);
}
protected override bool PreProcess(IndexerResponse indexerResponse)
{
using (var xmlTextReader = XmlReader.Create(new StringReader(indexerResponse.Content), new XmlReaderSettings { DtdProcessing = DtdProcessing.Ignore, IgnoreComments = true }))
{
var document = XDocument.Load(xmlTextReader);
var items = GetItems(document).ToList();
if (items.Count == 1 && GetTitle(items.First()).Equals("No items exist - Try again later"))
{
throw new IndexerException(indexerResponse, "No results were found");
}
}
return base.PreProcess(indexerResponse);
}
protected override bool PreProcess(IndexerResponse indexerResponse)
{
using (var xmlTextReader = XmlReader.Create(new StringReader(indexerResponse.Content), new XmlReaderSettings {
DtdProcessing = DtdProcessing.Ignore, IgnoreComments = true
}))
{
var document = XDocument.Load(xmlTextReader);
var items = GetItems(document).ToList();
if (items.Count == 1 && GetTitle(items.First()).Equals("No items exist - Try again later"))
{
throw new IndexerException(indexerResponse, "No results were found");
}
}
return(base.PreProcess(indexerResponse));
}
protected virtual XDocument LoadXmlDocument(IndexerResponse indexerResponse)
{
try
{
using (var xmlTextReader = XmlReader.Create(new StringReader(indexerResponse.Content), new XmlReaderSettings { DtdProcessing = DtdProcessing.Ignore, IgnoreComments = true }))
{
return XDocument.Load(xmlTextReader);
}
}
catch (XmlException ex)
{
var contentSample = indexerResponse.Content.Substring(0, Math.Min(indexerResponse.Content.Length, 512));
_logger.Debug("Truncated response content (originally {0} characters): {1}", indexerResponse.Content.Length, contentSample);
ex.Data.Add("ContentLength", indexerResponse.Content.Length);
ex.Data.Add("ContentSample", contentSample);
throw;
}
}
protected virtual bool PostProcess(IndexerResponse indexerResponse, List <XElement> elements, List <ReleaseInfo> releases)
{
return(true);
}
protected virtual Boolean PreProcess(IndexerResponse indexerResponse)
{
if (indexerResponse.HttpResponse.StatusCode != System.Net.HttpStatusCode.OK)
{
throw new IndexerException(indexerResponse, "Indexer API call resulted in an unexpected StatusCode [{0}]", indexerResponse.HttpResponse.StatusCode);
}
if (indexerResponse.HttpResponse.Headers.ContentType != null && indexerResponse.HttpResponse.Headers.ContentType.Contains("text/html") &&
indexerResponse.HttpRequest.Headers.Accept != null && !indexerResponse.HttpRequest.Headers.Accept.Contains("text/html"))
{
throw new IndexerException(indexerResponse, "Indexer responded with html content. Site is likely blocked or unavailable.");
}
return true;
}