public virtual IndexerPageableRequestChain GetRecentRequests() { var pageableRequests = new IndexerPageableRequestChain(); pageableRequests.Add(new[] { new IndexerRequest(_baseUrl, HttpAccept.Rss) }); return pageableRequests; }
protected virtual IList <ReleaseInfo> FetchReleases(IndexerPageableRequestChain pageableRequestChain, bool isRecent = false) { var releases = new List <ReleaseInfo>(); var url = string.Empty; var parser = GetParser(); try { var fullyUpdated = false; ReleaseInfo lastReleaseInfo = null; if (isRecent) { lastReleaseInfo = _indexerStatusService.GetLastRssSyncReleaseInfo(Definition.Id); } for (int i = 0; i < pageableRequestChain.Tiers; i++) { var pageableRequests = pageableRequestChain.GetTier(i); foreach (var pageableRequest in pageableRequests) { var pagedReleases = new List <ReleaseInfo>(); foreach (var request in pageableRequest) { url = request.Url.ToString(); var page = FetchPage(request, parser); pagedReleases.AddRange(page); if (isRecent && page.Any()) { if (lastReleaseInfo == null) { fullyUpdated = true; break; } var oldestReleaseDate = page.Select(v => v.PublishDate).Min(); if (oldestReleaseDate < lastReleaseInfo.PublishDate || page.Any(v => v.DownloadUrl == lastReleaseInfo.DownloadUrl)) { fullyUpdated = true; break; } if (pagedReleases.Count >= MaxNumResultsPerQuery && oldestReleaseDate < DateTime.UtcNow - TimeSpan.FromHours(24)) { fullyUpdated = false; break; } } else if (pagedReleases.Count >= MaxNumResultsPerQuery) { break; } if (!IsFullPage(page)) { break; } } releases.AddRange(pagedReleases); } if (releases.Any()) { break; } } if (isRecent && !releases.Empty()) { var ordered = releases.OrderByDescending(v => v.PublishDate).ToList(); if (!fullyUpdated && lastReleaseInfo != null) { var gapStart = lastReleaseInfo.PublishDate; var gapEnd = ordered.Last().PublishDate; _logger.Warn("Indexer {0} rss sync didn't cover the period between {1} and {2} UTC. Search may be required.", Definition.Name, gapStart, gapEnd); } lastReleaseInfo = ordered.First(); _indexerStatusService.UpdateRssSyncStatus(Definition.Id, lastReleaseInfo); } _indexerStatusService.RecordSuccess(Definition.Id); } catch (WebException webException) { _indexerStatusService.RecordFailure(Definition.Id); if (webException.Message.Contains("502") || webException.Message.Contains("503") || webException.Message.Contains("timed out")) { _logger.Warn("{0} server is currently unavailable. {1} {2}", this, url, webException.Message); } else { _logger.Warn("{0} {1} {2}", this, url, webException.Message); } } catch (HttpException httpException) { if ((int)httpException.Response.StatusCode == 429) { _indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1)); _logger.Warn("API Request Limit reached for {0}", this); } else { _indexerStatusService.RecordFailure(Definition.Id); _logger.Warn("{0} {1}", this, httpException.Message); } } catch (RequestLimitReachedException) { _indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1)); _logger.Warn("API Request Limit reached for {0}", this); } catch (ApiKeyException) { _indexerStatusService.RecordFailure(Definition.Id); _logger.Warn("Invalid API Key for {0} {1}", this, url); } catch (IndexerException ex) { _indexerStatusService.RecordFailure(Definition.Id); var message = string.Format("{0} - {1}", ex.Message, url); _logger.WarnException(message, ex); } catch (Exception feedEx) { _indexerStatusService.RecordFailure(Definition.Id); feedEx.Data.Add("FeedUrl", url); _logger.ErrorException("An error occurred while processing feed. " + url, feedEx); } return(CleanupReleases(releases)); }