public virtual IndexerPageableRequestChain GetRecentRequests()
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(new[] { new IndexerRequest(_baseUrl, HttpAccept.Rss) });
return pageableRequests;
}
protected virtual IList <ReleaseInfo> FetchReleases(IndexerPageableRequestChain pageableRequestChain, bool isRecent = false)
{
var releases = new List <ReleaseInfo>();
var url = string.Empty;
var parser = GetParser();
try
{
var fullyUpdated = false;
ReleaseInfo lastReleaseInfo = null;
if (isRecent)
{
lastReleaseInfo = _indexerStatusService.GetLastRssSyncReleaseInfo(Definition.Id);
}
for (int i = 0; i < pageableRequestChain.Tiers; i++)
{
var pageableRequests = pageableRequestChain.GetTier(i);
foreach (var pageableRequest in pageableRequests)
{
var pagedReleases = new List <ReleaseInfo>();
foreach (var request in pageableRequest)
{
url = request.Url.ToString();
var page = FetchPage(request, parser);
pagedReleases.AddRange(page);
if (isRecent && page.Any())
{
if (lastReleaseInfo == null)
{
fullyUpdated = true;
break;
}
var oldestReleaseDate = page.Select(v => v.PublishDate).Min();
if (oldestReleaseDate < lastReleaseInfo.PublishDate || page.Any(v => v.DownloadUrl == lastReleaseInfo.DownloadUrl))
{
fullyUpdated = true;
break;
}
if (pagedReleases.Count >= MaxNumResultsPerQuery &&
oldestReleaseDate < DateTime.UtcNow - TimeSpan.FromHours(24))
{
fullyUpdated = false;
break;
}
}
else if (pagedReleases.Count >= MaxNumResultsPerQuery)
{
break;
}
if (!IsFullPage(page))
{
break;
}
}
releases.AddRange(pagedReleases);
}
if (releases.Any())
{
break;
}
}
if (isRecent && !releases.Empty())
{
var ordered = releases.OrderByDescending(v => v.PublishDate).ToList();
if (!fullyUpdated && lastReleaseInfo != null)
{
var gapStart = lastReleaseInfo.PublishDate;
var gapEnd = ordered.Last().PublishDate;
_logger.Warn("Indexer {0} rss sync didn't cover the period between {1} and {2} UTC. Search may be required.", Definition.Name, gapStart, gapEnd);
}
lastReleaseInfo = ordered.First();
_indexerStatusService.UpdateRssSyncStatus(Definition.Id, lastReleaseInfo);
}
_indexerStatusService.RecordSuccess(Definition.Id);
}
catch (WebException webException)
{
_indexerStatusService.RecordFailure(Definition.Id);
if (webException.Message.Contains("502") || webException.Message.Contains("503") ||
webException.Message.Contains("timed out"))
{
_logger.Warn("{0} server is currently unavailable. {1} {2}", this, url, webException.Message);
}
else
{
_logger.Warn("{0} {1} {2}", this, url, webException.Message);
}
}
catch (HttpException httpException)
{
if ((int)httpException.Response.StatusCode == 429)
{
_indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
_logger.Warn("API Request Limit reached for {0}", this);
}
else
{
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Warn("{0} {1}", this, httpException.Message);
}
}
catch (RequestLimitReachedException)
{
_indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
_logger.Warn("API Request Limit reached for {0}", this);
}
catch (ApiKeyException)
{
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Warn("Invalid API Key for {0} {1}", this, url);
}
catch (IndexerException ex)
{
_indexerStatusService.RecordFailure(Definition.Id);
var message = string.Format("{0} - {1}", ex.Message, url);
_logger.WarnException(message, ex);
}
catch (Exception feedEx)
{
_indexerStatusService.RecordFailure(Definition.Id);
feedEx.Data.Add("FeedUrl", url);
_logger.ErrorException("An error occurred while processing feed. " + url, feedEx);
}
return(CleanupReleases(releases));
}