// TODO: Abort early when bsaber.com is down (check if all items in block failed?)
// TODO: Make cancellationToken actually do something.
/// <summary>
/// Gets all songs from the feed defined by the provided settings.
/// </summary>
/// <param name="settings"></param>
/// <param name="cancellationToken"></param>
/// <exception cref="ArgumentNullException">Thrown when <paramref name="_settings"/> is null.</exception>
/// <exception cref="InvalidCastException">Thrown when the passed IFeedSettings isn't a BeastSaberFeedSettings.</exception>
/// <exception cref="OperationCanceledException"></exception>
/// <returns></returns>
public async override Task <FeedResult> GetSongsFromFeedAsync(IFeedSettings settings, IProgress <ReaderProgress> progress, CancellationToken cancellationToken)
{
if (cancellationToken.IsCancellationRequested)
{
return(FeedResult.CancelledResult);
}
if (settings == null)
{
throw new ArgumentNullException(nameof(settings), "settings cannot be null for BeastSaberReader.GetSongsFromFeedAsync.");
}
Dictionary <string, ScrapedSong> retDict = new Dictionary <string, ScrapedSong>();
if (!(settings is BeastSaberFeedSettings _settings))
{
throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE);
}
if (_settings.Feed != BeastSaberFeedName.CuratorRecommended && string.IsNullOrEmpty(_settings.Username))
{
_settings.Username = Username;
}
BeastSaberFeed feed = new BeastSaberFeed(_settings)
{
StoreRawData = StoreRawData
};
try
{
feed.EnsureValidSettings();
}
catch (InvalidFeedSettingsException ex)
{
return(new FeedResult(null, null, ex, FeedResultError.Error));
}
int pageIndex = settings.StartingPage;
int maxPages = _settings.MaxPages;
int pagesChecked = 0;
bool useMaxSongs = _settings.MaxSongs != 0;
bool useMaxPages = maxPages != 0;
if (useMaxPages && pageIndex > 1)
{
maxPages = maxPages + pageIndex - 1;
}
var ProcessPageBlock = new TransformBlock <int, PageReadResult>(async pageNum =>
{
return(await feed.GetSongsFromPageAsync(pageNum, cancellationToken).ConfigureAwait(false));
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = MaxConcurrency,
BoundedCapacity = MaxConcurrency,
CancellationToken = cancellationToken
//#if NETSTANDARD
// , EnsureOrdered = true
//#endif
});
bool continueLooping = true;
int itemsInBlock = 0;
List <PageReadResult> pageResults = new List <PageReadResult>(maxPages + 2);
do
{
if (cancellationToken.IsCancellationRequested)
{
continueLooping = false;
}
while (continueLooping)
{
if (Utilities.IsPaused)
{
await Utilities.WaitUntil(() => !Utilities.IsPaused, 500, cancellationToken).ConfigureAwait(false);
}
if (cancellationToken.IsCancellationRequested)
{
continueLooping = false;
break;
}
await ProcessPageBlock.SendAsync(pageIndex, cancellationToken).ConfigureAwait(false); // TODO: Need check with SongsPerPage
itemsInBlock++;
pageIndex++;
if ((pageIndex > maxPages && useMaxPages) || cancellationToken.IsCancellationRequested)
{
continueLooping = false;
}
// TODO: Better http error handling, what if only a single page is broken and returns 0 songs?
while (ProcessPageBlock.OutputCount > 0 || itemsInBlock == MaxConcurrency || !continueLooping)
{
if (cancellationToken.IsCancellationRequested)
{
continueLooping = false;
break;
}
if (itemsInBlock <= 0)
{
break;
}
await ProcessPageBlock.OutputAvailableAsync(cancellationToken).ConfigureAwait(false);
while (ProcessPageBlock.TryReceive(out PageReadResult pageResult))
{
int songsAdded = 0;
if (pageResult != null)
{
pageResults.Add(pageResult);
}
if (Utilities.IsPaused)
{
await Utilities.WaitUntil(() => !Utilities.IsPaused, 500, cancellationToken).ConfigureAwait(false);
}
itemsInBlock--;
if (pageResult.IsLastPage || pageResult == null || pageResult.Count == 0) // TODO: This will trigger if a single page has an error.
{
Logger?.Debug("Received no new songs, last page reached.");
ProcessPageBlock.Complete();
itemsInBlock = 0;
continueLooping = false;
break;
}
if (pageResult.Count > 0)
{
Logger?.Debug($"Receiving {pageResult.Count} potential songs from {pageResult.Uri}");
}
else
{
Logger?.Debug($"Did not find any songs on page '{pageResult.Uri}' of {Name}.{settings.FeedName}.");
}
// TODO: Process PageReadResults for better error feedback.
foreach (var song in pageResult.Songs)
{
if (!retDict.ContainsKey(song.Hash))
{
if (retDict.Count < settings.MaxSongs || settings.MaxSongs == 0)
{
retDict.Add(song.Hash, song);
songsAdded++;
}
if (retDict.Count >= settings.MaxSongs && useMaxSongs)
{
continueLooping = false;
}
}
}
int prog = Interlocked.Increment(ref pagesChecked);
progress?.Report(new ReaderProgress(prog, songsAdded));
if (!useMaxPages || pageIndex <= maxPages)
{
if (retDict.Count < settings.MaxSongs)
{
continueLooping = true;
}
}
}
}
}
}while (continueLooping);
if (pageResults.Any(r => r.PageError == PageErrorType.Cancelled))
{
return(FeedResult.GetCancelledResult(retDict, pageResults));
}
return(new FeedResult(retDict, pageResults));
}
/// <summary>
///
/// </summary>
/// <param name="_settings"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
/// <exception cref="InvalidCastException">Throw when the provided settings object isn't a BeatSaverFeedSettings</exception>
/// <exception cref="ArgumentNullException">Thrown when <paramref name="_settings"/> is null.</exception>
public async Task <FeedResult> GetSongsFromScoreSaberAsync(ScoreSaberFeedSettings _settings, IProgress <ReaderProgress> progress, CancellationToken cancellationToken)
{
if (_settings == null)
{
throw new ArgumentNullException(nameof(_settings), "settings cannot be null for ScoreSaberReader.GetSongsFromScoreSaberAsync");
}
if (!(_settings is ScoreSaberFeedSettings settings))
{
throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE);
}
// "https://scoresaber.com/api.php?function=get-leaderboards&cat={CATKEY}&limit={LIMITKEY}&page={PAGENUMKEY}&ranked={RANKEDKEY}"
int songsPerPage = settings.SongsPerPage;
if (songsPerPage == 0)
{
songsPerPage = 100;
}
int pageNum = settings.StartingPage;
//int maxPages = (int)Math.Ceiling(settings.MaxSongs / ((float)songsPerPage));
int maxPages = settings.MaxPages;
int pagesChecked = 0;
if (pageNum > 1 && maxPages != 0)
{
maxPages = maxPages + pageNum - 1;
}
//if (settings.MaxPages > 0)
// maxPages = maxPages < settings.MaxPages ? maxPages : settings.MaxPages; // Take the lower limit.
var feed = new ScoreSaberFeed(settings);
try
{
feed.EnsureValidSettings();
}
catch (InvalidFeedSettingsException ex)
{
return(new FeedResult(null, null, ex, FeedResultError.Error));
}
Dictionary <string, ScrapedSong> songs = new Dictionary <string, ScrapedSong>();
var pageResults = new List <PageReadResult>();
Uri uri = feed.GetUriForPage(1);
PageReadResult result = await feed.GetSongsAsync(uri, cancellationToken).ConfigureAwait(false);
pageResults.Add(result);
foreach (var song in result.Songs)
{
if (!songs.ContainsKey(song.Hash) && (songs.Count < settings.MaxSongs || settings.MaxSongs == 0))
{
songs.Add(song.Hash, song);
}
}
pagesChecked++;
progress?.Report(new ReaderProgress(pagesChecked, songs.Count));
bool continueLooping = true;
do
{
pageNum++;
//int diffCount = 0;
if ((maxPages > 0 && pageNum > maxPages) || (settings.MaxSongs > 0 && songs.Count >= settings.MaxSongs))
{
break;
}
if (Utilities.IsPaused)
{
await Utilities.WaitUntil(() => !Utilities.IsPaused, 500).ConfigureAwait(false);
}
// TODO: Handle PageReadResult here
uri = feed.GetUriForPage(pageNum);
var pageResult = await feed.GetSongsAsync(uri, cancellationToken).ConfigureAwait(false);
pageResults.Add(pageResult);
if (pageResult.PageError == PageErrorType.Cancelled)
{
return(FeedResult.GetCancelledResult(songs, pageResults));
}
int uniqueSongCount = 0;
foreach (var song in pageResult.Songs)
{
//diffCount++;
if (!songs.ContainsKey(song.Hash) && (songs.Count < settings.MaxSongs || settings.MaxSongs == 0))
{
songs.Add(song.Hash, song);
uniqueSongCount++;
}
}
int prog = Interlocked.Increment(ref pagesChecked);
progress?.Report(new ReaderProgress(prog, uniqueSongCount));
if (uniqueSongCount > 0)
{
Logger?.Debug($"Receiving {uniqueSongCount} potential songs from {pageResult.Uri}");
}
else
{
Logger?.Debug($"Did not find any new songs on page '{pageResult.Uri}' of {Name}.{settings.FeedName}.");
}
if (pageResult.IsLastPage)
{
Logger?.Debug($"Last page reached.");
continueLooping = false;
}
if (!pageResult.Successful)
{
Logger?.Debug($"Page {pageResult.Uri.ToString()} failed, ending read.");
if (pageResult.Exception != null)
{
Logger?.Debug($"{pageResult.Exception.Message}\n{pageResult.Exception.StackTrace}");
}
continueLooping = false;
}
//pageReadTasks.Add(GetSongsFromPageAsync(url.ToString()));
if ((maxPages > 0 && pageNum >= maxPages) || (settings.MaxSongs > 0 && songs.Count >= settings.MaxSongs))
{
continueLooping = false;
}
} while (continueLooping);
if (pageResults.Any(r => r.PageError == PageErrorType.Cancelled))
{
return(FeedResult.GetCancelledResult(songs, pageResults));
}
return(new FeedResult(songs, pageResults));
}