/// <summary> /// /// </summary> /// <param name="settings"></param> /// <exception cref="ArgumentNullException"></exception> public ScoreSaberFeed(ScoreSaberFeedSettings settings) { if (settings == null) { throw new ArgumentNullException(nameof(settings), "settings cannot be null when creating a new ScoreSaberFeed."); } ScoreSaberFeedSettings = (ScoreSaberFeedSettings)settings.Clone(); Feed = ScoreSaberFeedSettings.Feed; FeedInfo = Feeds[ScoreSaberFeedSettings.Feed]; SongsPerPage = ScoreSaberFeedSettings.SongsPerPage; SearchQuery = ScoreSaberFeedSettings.SearchQuery; RankedOnly = ScoreSaberFeedSettings.RankedOnly; }
/// <summary> /// /// </summary> /// <param name="_settings"></param> /// <param name="cancellationToken"></param> /// <returns></returns> /// <exception cref="InvalidCastException">Throw when the provided settings object isn't a BeatSaverFeedSettings</exception> /// <exception cref="ArgumentNullException">Thrown when <paramref name="_settings"/> is null.</exception> public async Task <FeedResult> GetSongsFromScoreSaberAsync(ScoreSaberFeedSettings _settings, IProgress <ReaderProgress> progress, CancellationToken cancellationToken) { if (_settings == null) { throw new ArgumentNullException(nameof(_settings), "settings cannot be null for ScoreSaberReader.GetSongsFromScoreSaberAsync"); } if (!(_settings is ScoreSaberFeedSettings settings)) { throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE); } // "https://scoresaber.com/api.php?function=get-leaderboards&cat={CATKEY}&limit={LIMITKEY}&page={PAGENUMKEY}&ranked={RANKEDKEY}" int songsPerPage = settings.SongsPerPage; if (songsPerPage == 0) { songsPerPage = 100; } int pageNum = settings.StartingPage; //int maxPages = (int)Math.Ceiling(settings.MaxSongs / ((float)songsPerPage)); int maxPages = settings.MaxPages; int pagesChecked = 0; if (pageNum > 1 && maxPages != 0) { maxPages = maxPages + pageNum - 1; } //if (settings.MaxPages > 0) // maxPages = maxPages < settings.MaxPages ? maxPages : settings.MaxPages; // Take the lower limit. var feed = new ScoreSaberFeed(settings); try { feed.EnsureValidSettings(); } catch (InvalidFeedSettingsException ex) { return(new FeedResult(null, null, ex, FeedResultError.Error)); } Dictionary <string, ScrapedSong> songs = new Dictionary <string, ScrapedSong>(); var pageResults = new List <PageReadResult>(); Uri uri = feed.GetUriForPage(1); PageReadResult result = await feed.GetSongsAsync(uri, cancellationToken).ConfigureAwait(false); pageResults.Add(result); foreach (var song in result.Songs) { if (!songs.ContainsKey(song.Hash) && (songs.Count < settings.MaxSongs || settings.MaxSongs == 0)) { songs.Add(song.Hash, song); } } pagesChecked++; progress?.Report(new ReaderProgress(pagesChecked, songs.Count)); bool continueLooping = true; do { pageNum++; //int diffCount = 0; if ((maxPages > 0 && pageNum > maxPages) || (settings.MaxSongs > 0 && songs.Count >= settings.MaxSongs)) { break; } if (Utilities.IsPaused) { await Utilities.WaitUntil(() => !Utilities.IsPaused, 500).ConfigureAwait(false); } // TODO: Handle PageReadResult here uri = feed.GetUriForPage(pageNum); var pageResult = await feed.GetSongsAsync(uri, cancellationToken).ConfigureAwait(false); pageResults.Add(pageResult); if (pageResult.PageError == PageErrorType.Cancelled) { return(FeedResult.GetCancelledResult(songs, pageResults)); } int uniqueSongCount = 0; foreach (var song in pageResult.Songs) { //diffCount++; if (!songs.ContainsKey(song.Hash) && (songs.Count < settings.MaxSongs || settings.MaxSongs == 0)) { songs.Add(song.Hash, song); uniqueSongCount++; } } int prog = Interlocked.Increment(ref pagesChecked); progress?.Report(new ReaderProgress(prog, uniqueSongCount)); if (uniqueSongCount > 0) { Logger?.Debug($"Receiving {uniqueSongCount} potential songs from {pageResult.Uri}"); } else { Logger?.Debug($"Did not find any new songs on page '{pageResult.Uri}' of {Name}.{settings.FeedName}."); } if (pageResult.IsLastPage) { Logger?.Debug($"Last page reached."); continueLooping = false; } if (!pageResult.Successful) { Logger?.Debug($"Page {pageResult.Uri.ToString()} failed, ending read."); if (pageResult.Exception != null) { Logger?.Debug($"{pageResult.Exception.Message}\n{pageResult.Exception.StackTrace}"); } continueLooping = false; } //pageReadTasks.Add(GetSongsFromPageAsync(url.ToString())); if ((maxPages > 0 && pageNum >= maxPages) || (settings.MaxSongs > 0 && songs.Count >= settings.MaxSongs)) { continueLooping = false; } } while (continueLooping); if (pageResults.Any(r => r.PageError == PageErrorType.Cancelled)) { return(FeedResult.GetCancelledResult(songs, pageResults)); } return(new FeedResult(songs, pageResults)); }
public async Task <FeedResult> GetSongsFromScoreSaberAsync(ScoreSaberFeedSettings settings, CancellationToken cancellationToken) { if (settings == null) { throw new ArgumentNullException(nameof(settings), "settings cannot be null for ScoreSaberReader.GetSongsFromScoreSaberAsync"); } // "https://scoresaber.com/api.php?function=get-leaderboards&cat={CATKEY}&limit={LIMITKEY}&page={PAGENUMKEY}&ranked={RANKEDKEY}" int songsPerPage = settings.SongsPerPage; if (songsPerPage == 0) { songsPerPage = 100; } int pageNum = settings.StartingPage; //int maxPages = (int)Math.Ceiling(settings.MaxSongs / ((float)songsPerPage)); int maxPages = settings.MaxPages; if (pageNum > 1 && maxPages != 0) { maxPages = maxPages + pageNum - 1; } //if (settings.MaxPages > 0) // maxPages = maxPages < settings.MaxPages ? maxPages : settings.MaxPages; // Take the lower limit. Dictionary <string, ScrapedSong> songs = new Dictionary <string, ScrapedSong>(); StringBuilder url = new StringBuilder(Feeds[settings.Feed].BaseUrl); Dictionary <string, string> urlReplacements = new Dictionary <string, string>() { { LIMITKEY, songsPerPage.ToString() }, { PAGENUMKEY, pageNum.ToString() }, { RANKEDKEY, settings.RankedOnly ? "1" : "0" } }; if (settings.Feed == ScoreSaberFeed.Search) { urlReplacements.Add(QUERYKEY, settings.SearchQuery); } GetPageUrl(ref url, urlReplacements); var uri = new Uri(url.ToString()); string pageText = ""; var pageResults = new List <PageReadResult>(); IWebResponseMessage response = null; try { response = await WebUtils.WebClient.GetAsync(uri, cancellationToken).ConfigureAwait(false); response.EnsureSuccessStatusCode(); pageText = await response.Content.ReadAsStringAsync().ConfigureAwait(false); } catch (WebClientException ex) { string errorText = string.Empty; int statusCode = ex?.Response?.StatusCode ?? 0; if (statusCode != 0) { switch (statusCode) { case 408: errorText = "Timeout"; break; default: errorText = "Site Error"; break; } } string message = $"{errorText} getting first page in ScoreSaberReader: {uri}: {ex.Message}"; Logger?.Debug(message); // No need for a stacktrace if it's one of these errors. if (!(statusCode == 408 || statusCode == 500)) { Logger?.Debug($"{ex.Message}\n{ex.StackTrace}"); } return(new FeedResult(null, null, new FeedReaderException(message, ex, FeedReaderFailureCode.SourceFailed), FeedResultError.Error)); } catch (Exception ex) { string message = $"Uncaught error getting the first page in ScoreSaberReader.GetSongsFromScoreSaberAsync(): {ex.Message}"; return(new FeedResult(null, null, new FeedReaderException(message, ex, FeedReaderFailureCode.SourceFailed), FeedResultError.Error)); } finally { response?.Dispose(); response = null; } var result = GetSongsFromPageText(pageText, uri); pageResults.Add(result); foreach (var song in result.Songs) { if (!songs.ContainsKey(song.Hash) && (songs.Count < settings.MaxSongs || settings.MaxSongs == 0)) { songs.Add(song.Hash, song); } } bool continueLooping = true; do { pageNum++; int diffCount = 0; if ((maxPages > 0 && pageNum > maxPages) || (settings.MaxSongs > 0 && songs.Count >= settings.MaxSongs)) { break; } url.Clear(); url.Append(Feeds[settings.Feed].BaseUrl); if (!urlReplacements.ContainsKey(PAGENUMKEY)) { urlReplacements.Add(PAGENUMKEY, pageNum.ToString()); } else { urlReplacements[PAGENUMKEY] = pageNum.ToString(); } GetPageUrl(ref url, urlReplacements); uri = new Uri(url.ToString()); if (Utilities.IsPaused) { await Utilities.WaitUntil(() => !Utilities.IsPaused, 500).ConfigureAwait(false); } // TODO: Handle PageReadResult here var pageResult = await GetSongsFromPageAsync(uri, cancellationToken).ConfigureAwait(false); pageResults.Add(pageResult); foreach (var song in pageResult.Songs) { diffCount++; if (!songs.ContainsKey(song.Hash) && (songs.Count < settings.MaxSongs || settings.MaxSongs == 0)) { songs.Add(song.Hash, song); } } if (diffCount == 0) { Logger?.Debug($"No diffs found on {uri.ToString()}, should be after last page."); continueLooping = false; } //pageReadTasks.Add(GetSongsFromPageAsync(url.ToString())); if ((maxPages > 0 && pageNum >= maxPages) || (settings.MaxSongs > 0 && songs.Count >= settings.MaxSongs)) { continueLooping = false; } } while (continueLooping); return(new FeedResult(songs, pageResults)); }