// TODO: Abort early when bsaber.com is down (check if all items in block failed?)
// TODO: Make cancellationToken actually do something.
/// <summary>
/// Gets all songs from the feed defined by the provided settings.
/// </summary>
/// <param name="settings"></param>
/// <param name="cancellationToken"></param>
/// <exception cref="ArgumentNullException">Thrown when <paramref name="_settings"/> is null.</exception>
/// <exception cref="InvalidCastException">Thrown when the passed IFeedSettings isn't a BeastSaberFeedSettings.</exception>
/// <exception cref="OperationCanceledException"></exception>
/// <returns></returns>
public async override Task <FeedResult> GetSongsFromFeedAsync(IFeedSettings settings, IProgress <ReaderProgress> progress, CancellationToken cancellationToken)
{
if (cancellationToken.IsCancellationRequested)
{
return(FeedResult.CancelledResult);
}
if (settings == null)
{
throw new ArgumentNullException(nameof(settings), "settings cannot be null for BeastSaberReader.GetSongsFromFeedAsync.");
}
Dictionary <string, ScrapedSong> retDict = new Dictionary <string, ScrapedSong>();
if (!(settings is BeastSaberFeedSettings _settings))
{
throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE);
}
if (_settings.Feed != BeastSaberFeedName.CuratorRecommended && string.IsNullOrEmpty(_settings.Username))
{
_settings.Username = Username;
}
BeastSaberFeed feed = new BeastSaberFeed(_settings)
{
StoreRawData = StoreRawData
};
try
{
feed.EnsureValidSettings();
}
catch (InvalidFeedSettingsException ex)
{
return(new FeedResult(null, null, ex, FeedResultError.Error));
}
int pageIndex = settings.StartingPage;
int maxPages = _settings.MaxPages;
int pagesChecked = 0;
bool useMaxSongs = _settings.MaxSongs != 0;
bool useMaxPages = maxPages != 0;
if (useMaxPages && pageIndex > 1)
{
maxPages = maxPages + pageIndex - 1;
}
var ProcessPageBlock = new TransformBlock <int, PageReadResult>(async pageNum =>
{
return(await feed.GetSongsFromPageAsync(pageNum, cancellationToken).ConfigureAwait(false));
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = MaxConcurrency,
BoundedCapacity = MaxConcurrency,
CancellationToken = cancellationToken
//#if NETSTANDARD
// , EnsureOrdered = true
//#endif
});
bool continueLooping = true;
int itemsInBlock = 0;
List <PageReadResult> pageResults = new List <PageReadResult>(maxPages + 2);
do
{
if (cancellationToken.IsCancellationRequested)
{
continueLooping = false;
}
while (continueLooping)
{
if (Utilities.IsPaused)
{
await Utilities.WaitUntil(() => !Utilities.IsPaused, 500, cancellationToken).ConfigureAwait(false);
}
if (cancellationToken.IsCancellationRequested)
{
continueLooping = false;
break;
}
await ProcessPageBlock.SendAsync(pageIndex, cancellationToken).ConfigureAwait(false); // TODO: Need check with SongsPerPage
itemsInBlock++;
pageIndex++;
if ((pageIndex > maxPages && useMaxPages) || cancellationToken.IsCancellationRequested)
{
continueLooping = false;
}
// TODO: Better http error handling, what if only a single page is broken and returns 0 songs?
while (ProcessPageBlock.OutputCount > 0 || itemsInBlock == MaxConcurrency || !continueLooping)
{
if (cancellationToken.IsCancellationRequested)
{
continueLooping = false;
break;
}
if (itemsInBlock <= 0)
{
break;
}
await ProcessPageBlock.OutputAvailableAsync(cancellationToken).ConfigureAwait(false);
while (ProcessPageBlock.TryReceive(out PageReadResult pageResult))
{
int songsAdded = 0;
if (pageResult != null)
{
pageResults.Add(pageResult);
}
if (Utilities.IsPaused)
{
await Utilities.WaitUntil(() => !Utilities.IsPaused, 500, cancellationToken).ConfigureAwait(false);
}
itemsInBlock--;
if (pageResult.IsLastPage || pageResult == null || pageResult.Count == 0) // TODO: This will trigger if a single page has an error.
{
Logger?.Debug("Received no new songs, last page reached.");
ProcessPageBlock.Complete();
itemsInBlock = 0;
continueLooping = false;
break;
}
if (pageResult.Count > 0)
{
Logger?.Debug($"Receiving {pageResult.Count} potential songs from {pageResult.Uri}");
}
else
{
Logger?.Debug($"Did not find any songs on page '{pageResult.Uri}' of {Name}.{settings.FeedName}.");
}
// TODO: Process PageReadResults for better error feedback.
foreach (var song in pageResult.Songs)
{
if (!retDict.ContainsKey(song.Hash))
{
if (retDict.Count < settings.MaxSongs || settings.MaxSongs == 0)
{
retDict.Add(song.Hash, song);
songsAdded++;
}
if (retDict.Count >= settings.MaxSongs && useMaxSongs)
{
continueLooping = false;
}
}
}
int prog = Interlocked.Increment(ref pagesChecked);
progress?.Report(new ReaderProgress(prog, songsAdded));
if (!useMaxPages || pageIndex <= maxPages)
{
if (retDict.Count < settings.MaxSongs)
{
continueLooping = true;
}
}
}
}
}
}while (continueLooping);
if (pageResults.Any(r => r.PageError == PageErrorType.Cancelled))
{
return(FeedResult.GetCancelledResult(retDict, pageResults));
}
return(new FeedResult(retDict, pageResults));
}
public BeastSaberFeedSettings(BeastSaberFeed feed)
{
Feed = feed;
MaxPages = 0;
StartingPage = 1;
}