public async Task <PageReadResult> GetSongsFromPageAsync(Uri uri, CancellationToken cancellationToken)
{
if (uri == null)
{
throw new ArgumentNullException(nameof(uri), "uri cannot be null in ScoreSaberReader.GetSongsFromPageAsync");
}
IWebResponseMessage response = null;
try
{
response = await WebUtils.WebClient.GetAsync(uri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
var pageText = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
return(GetSongsFromPageText(pageText, uri));
}
catch (WebClientException ex)
{
return(PageReadResult.FromWebClientException(ex, uri));
}
catch (Exception ex)
{
string message = $"Uncaught error getting page {uri?.ToString()}: {ex.Message}";
Logger?.Error(message);
return(new PageReadResult(uri, null, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.Unknown));
}
finally
{
response?.Dispose();
response = null;
}
}
public void NothingToReadMeansNoPagingRequestTest()
{
var offset = 0;
var take = 10;
var total = 99;
_changeSetController.SetAlignedValues(offset, offset, false);
var requestRequest = new PageReadRequest()
{
Offset = offset, Take = take
};
var result = default(PageReadResult <CollectionModel>);
var items = new List <CollectionModel>()
{
new CollectionModel(), new CollectionModel()
};
var res = new PageReadResult <CollectionModel>(offset, 5, items)
{
Total = total
};
var pcReturn = Observable.Return(res);
_pagingController.Setup(p => p.ReadPage(offset, take)).Returns(pcReturn);
var busyStates = new List <bool>();
_changeSetController.BusyObservable.Subscribe(b => busyStates.Add(b));
var readPage = _changeSetController.ReadPageObservable(requestRequest).Subscribe(r => result = r);
Assert.Equal(res.Offset, result.Offset);
Assert.Equal(0, result.AmountRead);
Assert.Null(result.Total); // we don't know the total since never read..
Assert.Equal(3, busyStates.Count);
Assert.False(busyStates[0]);
Assert.True(busyStates[1]);
Assert.False(busyStates[2]);
_pagingController.Verify(p => p.ReadPage(offset, take), Times.Never);
}
public void ExceptionPassedAndRetryCorrectlyOccursTest()
{
var exception = new InvalidOperationException();
var exceptionRaised = default(Exception);
var errorRequest = default(PageReadRequest);
var raised = false;
var result = new PageReadResult <CollectionModel>(1, null, null);
Func <PageReadRequest, Exception, IObservable <bool> > exc = (pr, e) =>
{
raised = true;
exceptionRaised = e;
errorRequest = pr;
return(Observable.Return(true));
};
Func <PageReadRequest, IObservable <PageReadResult <CollectionModel> > > func = (r) => this.GetData(r, exception, result);
_pagingController = new ObservablePagingController <CollectionModel>(func, exc);
var offset = 10;
var take = 10;
PageReadResult <CollectionModel> readValue = default(PageReadResult <CollectionModel>);
var read = _pagingController.ReadPage(offset, take).Subscribe(v => readValue = v);
Assert.True(raised);
Assert.Same(exception, exceptionRaised);
Assert.Equal(offset, errorRequest.Offset);
Assert.Equal(take, errorRequest.Take);
Assert.Equal(2, _invokeCount);
Assert.Same(result, readValue);
}
public void NormalReadFunctionsCorrectlyTest()
{
var offset = 0;
var take = 10;
var total = 99;
_changeSetController.SetAlignedValues(offset, offset + take, true);
var requestRequest = new PageReadRequest()
{
Offset = offset, Take = take
};
var result = default(PageReadResult <CollectionModel>);
var items = new List <CollectionModel>()
{
new CollectionModel(), new CollectionModel()
};
var res = new PageReadResult <CollectionModel>(offset, 5, items);
res.Total = total;
var pcReturn = Observable.Return(res);
_pagingController.Setup(p => p.ReadPage(offset, take)).Returns(pcReturn);
var readPage = _changeSetController.ReadPageObservable(requestRequest).Subscribe(r => result = r);
Assert.Equal(res.Offset, result.Offset);
Assert.Equal(res.Total, result.Total);
Assert.Equal(res.Items[0], result.Items[0]);
Assert.Equal(res.Items[1], result.Items[1]);
Assert.Equal(total, _changeSetController.Total);
_pagingController.Verify(p => p.ReadPage(offset, take), Times.Once);
}
public static async Task <PageReadResult> GetSongsFromPageAsync(Uri uri, CancellationToken cancellationToken)
{
if (uri == null)
{
throw new ArgumentNullException(nameof(uri), "uri cannot be null in BeatSaverReader.GetSongsFromPageAsync.");
}
string pageText = string.Empty;
var songs = new List <ScrapedSong>();
IWebResponseMessage response = null;
try
{
response = await WebUtils.GetBeatSaverAsync(uri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
pageText = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
}
catch (WebClientException ex)
{
return(PageReadResult.FromWebClientException(ex, uri));
}
catch (OperationCanceledException ex)
{
return(new PageReadResult(uri, null, ex, PageErrorType.Cancelled));
}
finally
{
response?.Dispose();
response = null;
}
foreach (var song in ParseSongsFromPage(pageText, uri))
{
songs.Add(song);
}
return(new PageReadResult(uri, songs));
}
public void ExceptionPassedToExceptionHandlerTest()
{
var exception = new InvalidOperationException();
var readResult = new Subject <PageReadResult <CollectionModel> >();
readResult.OnError(exception);
Func <PageReadRequest, IObservable <PageReadResult <CollectionModel> > > func = (r) => readResult;
var exceptionRaised = default(Exception);
var errorRequest = default(PageReadRequest);
var raised = false;
Func <PageReadRequest, Exception, IObservable <bool> > exc = (pr, e) =>
{
raised = true;
exceptionRaised = e;
errorRequest = pr;
return(Observable.Return(false));
};
_pagingController = new ObservablePagingController <CollectionModel>(func, exc);
var offset = 10;
var take = 10;
PageReadResult <CollectionModel> readValue = default(PageReadResult <CollectionModel>);
var read = _pagingController.ReadPage(offset, take).Subscribe(v => readValue = v);
Assert.True(raised);
Assert.Same(exception, exceptionRaised);
Assert.Equal(offset, errorRequest.Offset);
Assert.Equal(take, errorRequest.Take);
}
/// <summary>
///
/// </summary>
/// <param name="page"></param>
/// <param name="cancellationToken"></param>
/// <exception cref="InvalidFeedSettingsException">Thrown when the feed's settings aren't valid.</exception>
/// <returns></returns>
public async Task <PageReadResult> GetSongsAsync(Uri uri, CancellationToken cancellationToken)
{
string pageText = "";
Dictionary <string, ScrapedSong> songs = new Dictionary <string, ScrapedSong>();
Logger.Debug($"Getting songs from '{uri}'");
IWebResponseMessage?response = null;
try
{
response = await WebUtils.WebClient.GetAsync(uri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
pageText = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
}
catch (OperationCanceledException ex)
{
return(PageReadResult.CancelledResult(uri, ex));
}
catch (WebClientException ex)
{
string errorText = string.Empty;
int statusCode = ex?.Response?.StatusCode ?? 0;
if (statusCode != 0)
{
switch (statusCode)
{
case 404:
errorText = $"{uri.ToString()} was not found.";
break;
case 408:
errorText = $"Timeout getting first page in ScoreSaberReader: {uri}: {ex.Message}";
break;
default:
errorText = $"Site Error getting first page in ScoreSaberReader: {uri}: {ex.Message}";
break;
}
}
Logger?.Debug(errorText);
// No need for a stacktrace if it's one of these errors.
if (!(statusCode == 404 || statusCode == 408 || statusCode == 500))
{
Logger?.Debug($"{ex.Message}\n{ex.StackTrace}");
}
return(PageReadResult.FromWebClientException(ex, uri));
}
catch (Exception ex)
{
string message = $"Uncaught error getting the first page in ScoreSaberReader.GetSongsFromScoreSaberAsync(): {ex.Message}";
return(new PageReadResult(uri, new List <ScrapedSong>(), null, null, 0, new FeedReaderException(message, ex, FeedReaderFailureCode.SourceFailed), PageErrorType.Unknown));
}
finally
{
response?.Dispose();
response = null;
}
bool isLastPage;
ScrapedSong?firstSong = null;
ScrapedSong?lastSong = null;
int songsOnPage = 0;
try
{
List <ScrapedSong>?diffs = GetSongsFromPageText(pageText, uri, Settings.StoreRawData || StoreRawData);
firstSong = diffs?.FirstOrDefault();
lastSong = diffs?.LastOrDefault();
songsOnPage = diffs?.Count ?? 0;
isLastPage = (diffs?.Count ?? 0) < SongsPerPage;
foreach (ScrapedSong?diff in diffs)
{
if (!songs.ContainsKey(diff.Hash) && (Settings.Filter == null || Settings.Filter(diff)))
{
songs.Add(diff.Hash, diff);
}
if (Settings.StopWhenAny != null && Settings.StopWhenAny(diff))
{
isLastPage = true;
}
}
}
catch (JsonReaderException ex)
{
string message = "Unable to parse JSON from text";
Logger?.Debug($"{message}: {ex.Message}\n{ex.StackTrace}");
return(new PageReadResult(uri, null, firstSong, lastSong, songsOnPage, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.ParsingError));
}
catch (Exception ex)
{
string message = $"Unhandled exception from GetSongsFromPageText() while parsing {uri}";
Logger?.Debug($"{message}: {ex.Message}\n{ex.StackTrace}");
return(new PageReadResult(uri, null, firstSong, lastSong, songsOnPage, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.ParsingError));
}
return(new PageReadResult(uri, songs.Values.ToList(), firstSong, lastSong, songsOnPage, isLastPage));
}
/// <summary>
///
/// </summary>
/// <param name="page"></param>
/// <param name="cancellationToken"></param>
/// <exception cref="InvalidFeedSettingsException">Thrown when the feed's settings aren't valid.</exception>
/// <returns></returns>
public async Task <PageReadResult> GetSongsAsync(Uri uri, CancellationToken cancellationToken)
{
string pageText;
JObject result;
List <ScrapedSong> newSongs;
Logger.Debug($"Getting songs from '{uri}'");
//int? lastPage;
bool isLastPage = false;
IWebResponseMessage?response = null;
ScrapedSong? firstSong = null;
ScrapedSong? lastSong = null;
int songsOnPage = 0;
try
{
response = await WebUtils.GetBeatSaverAsync(uri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
#pragma warning disable CS8602 // Dereference of a possibly null reference.
pageText = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
#pragma warning restore CS8602 // Dereference of a possibly null reference.
result = JObject.Parse(pageText);
if (result?["docs"] == null)
{
Logger?.Warning($"Error checking Beat Saver's {Name} feed.");
return(new PageReadResult(uri, null, null, null, 0, new FeedReaderException($"Error getting page in BeatSaverFeed.GetSongsFromPageAsync()", null, FeedReaderFailureCode.PageFailed), PageErrorType.ParsingError));
}
//if (configuredLastPage > 0)
// isLastPage = Math.Min(configuredLastPage, lastPage.Value) <= page;
//else
// isLastPage = page >= lastPage.Value;
newSongs = new List <ScrapedSong>();
var scrapedSongs = BeatSaverReader.ParseSongsFromJson(result, uri, Settings.StoreRawData || StoreRawData);
firstSong = scrapedSongs.FirstOrDefault();
lastSong = scrapedSongs.LastOrDefault();
songsOnPage = scrapedSongs.Count;
foreach (var song in scrapedSongs)
{
if (Settings.Filter == null || Settings.Filter(song))
{
newSongs.Add(song);
}
if (Settings.StopWhenAny != null && Settings.StopWhenAny(song))
{
isLastPage = true;
break;
}
}
if (scrapedSongs.Count == 0)
{
isLastPage = true;
}
}
catch (WebClientException ex)
{
string errorText = string.Empty;
if (ex.Response != null)
{
switch (ex.Response.StatusCode)
{
case 408:
errorText = "Timeout";
break;
default:
errorText = "Site Error";
break;
}
}
string message = $"{errorText} getting a response from {uri}: {ex.Message}";
Logger?.Debug(message);
Logger?.Debug($"{ex.Message}\n{ex.StackTrace}");
return(PageReadResult.FromWebClientException(ex, uri));
}
catch (JsonReaderException ex)
{
string message = $"Unable to parse JSON from text on page {uri.ToString()}";
Logger?.Debug(message);
Logger?.Debug($"{ex.Message}\n{ex.StackTrace}");
return(new PageReadResult(uri, null, firstSong, lastSong, songsOnPage, new FeedReaderException(message, ex, FeedReaderFailureCode.SourceFailed), PageErrorType.ParsingError));
}
catch (OperationCanceledException ex)
{
return(PageReadResult.CancelledResult(uri, ex));
}
catch (Exception ex)
{
string message = $"Uncaught error getting page {uri} in BeatSaverFeed.GetSongsFromPageAsync(): {ex.Message}";
return(new PageReadResult(uri, null, firstSong, lastSong, songsOnPage, new FeedReaderException(message, ex, FeedReaderFailureCode.SourceFailed), PageErrorType.ParsingError));
}
finally
{
response?.Dispose();
response = null;
}
//if (lastPage.HasValue && !isLastPage)
// return new BeatSaverPageResult(pageUri, newSongs, page, lastPage.Value);
//else
return(new PageReadResult(uri, newSongs, firstSong, lastSong, songsOnPage, isLastPage));
}
// TODO: Abort early when bsaber.com is down (check if all items in block failed?)
// TODO: Make cancellationToken actually do something.
/// <summary>
/// Gets all songs from the feed defined by the provided settings.
/// </summary>
/// <param name="settings"></param>
/// <param name="cancellationToken"></param>
/// <exception cref="InvalidCastException">Thrown when the passed IFeedSettings isn't a BeastSaberFeedSettings.</exception>
/// <exception cref="ArgumentException">Thrown when trying to access a feed that requires a username and the username wasn't provided.</exception>
/// <exception cref="OperationCanceledException"></exception>
/// <returns></returns>
public async Task <FeedResult> GetSongsFromFeedAsync(IFeedSettings settings, CancellationToken cancellationToken)
{
if (cancellationToken.IsCancellationRequested)
{
return(FeedResult.CancelledResult);
}
if (settings == null)
{
throw new ArgumentNullException(nameof(settings), "settings cannot be null for BeastSaberReader.GetSongsFromFeedAsync.");
}
Dictionary <string, ScrapedSong> retDict = new Dictionary <string, ScrapedSong>();
if (!(settings is BeastSaberFeedSettings _settings))
{
throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE);
}
if (_settings.FeedIndex != 2 && string.IsNullOrEmpty(_username?.Trim()))
{
//Logger?.Error($"Can't access feed without a valid username in the config file");
throw new ArgumentException("Cannot access this feed without a valid username.");
}
int pageIndex = settings.StartingPage;
int maxPages = _settings.MaxPages;
bool useMaxSongs = _settings.MaxSongs != 0;
bool useMaxPages = maxPages != 0;
if (useMaxPages && pageIndex > 1)
{
maxPages = maxPages + pageIndex - 1;
}
var ProcessPageBlock = new TransformBlock <Uri, PageReadResult>(async feedUri =>
{
Stopwatch sw = new Stopwatch();
sw.Start();
//Logger?.Debug($"Checking URL: {feedUrl}");
string pageText = "";
ContentType contentType = ContentType.Unknown;
string contentTypeStr = string.Empty;
IWebResponseMessage response = null;
try
{
response = await WebUtils.WebClient.GetAsync(feedUri, cancellationToken).ConfigureAwait(false);
if ((response?.StatusCode ?? 500) == 500)
{
response?.Dispose();
response = null;
Logger?.Warning($"Internal server error on {feedUri}, retrying in 20 seconds");
await Task.Delay(20000).ConfigureAwait(false);
response = await WebUtils.WebClient.GetAsync(feedUri, cancellationToken).ConfigureAwait(false);
}
response.EnsureSuccessStatusCode();
contentTypeStr = response.Content.ContentType.ToLower();
if (ContentDictionary.ContainsKey(contentTypeStr))
{
contentType = ContentDictionary[contentTypeStr];
}
else
{
contentType = ContentType.Unknown;
}
pageText = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
}
catch (WebClientException ex)
{
return(PageReadResult.FromWebClientException(ex, feedUri));
}
catch (OperationCanceledException)
{
return(new PageReadResult(feedUri, null, new FeedReaderException("Page read was cancelled.", new OperationCanceledException(), FeedReaderFailureCode.Cancelled), PageErrorType.Cancelled));
}
catch (Exception ex)
{
string message = $"Error downloading {feedUri} in TransformBlock.";
Logger?.Debug(message);
Logger?.Debug($"{ex.Message}\n{ex.StackTrace}");
return(new PageReadResult(feedUri, null, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.Unknown));
}
finally
{
response?.Dispose();
response = null;
}
List <ScrapedSong> newSongs = null;
try
{
newSongs = GetSongsFromPageText(pageText, feedUri, contentType);
}
catch (JsonReaderException ex)
{
// TODO: Probably don't need a logger message here, caller can deal with it.
string message = $"Error parsing page text for {feedUri} in TransformBlock.";
Logger?.Debug(message);
Logger?.Debug($"{ex.Message}\n{ex.StackTrace}");
return(new PageReadResult(feedUri, null, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.ParsingError));
}
catch (XmlException ex)
{
// TODO: Probably don't need a logger message here, caller can deal with it.
string message = $"Error parsing page text for {feedUri} in TransformBlock.";
Logger?.Debug(message);
Logger?.Debug($"{ex.Message}\n{ex.StackTrace}");
return(new PageReadResult(feedUri, null, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.ParsingError));
}
catch (Exception ex)
{
// TODO: Probably don't need a logger message here, caller can deal with it.
string message = $"Uncaught error parsing page text for {feedUri} in TransformBlock.";
Logger?.Debug(message);
Logger?.Debug($"{ex.Message}\n{ex.StackTrace}");
return(new PageReadResult(feedUri, null, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.Unknown));
}
sw.Stop();
//Logger?.Debug($"Task for {feedUrl} completed in {sw.ElapsedMilliseconds}ms");
return(new PageReadResult(feedUri, newSongs));
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = MaxConcurrency,
BoundedCapacity = MaxConcurrency,
CancellationToken = cancellationToken
//#if NETSTANDARD
// , EnsureOrdered = true
//#endif
});
bool continueLooping = true;
int itemsInBlock = 0;
List <PageReadResult> pageResults = new List <PageReadResult>(maxPages + 2);
do
{
if (cancellationToken.IsCancellationRequested)
{
continueLooping = false;
}
while (continueLooping)
{
if (Utilities.IsPaused)
{
await Utilities.WaitUntil(() => !Utilities.IsPaused, 500, cancellationToken).ConfigureAwait(false);
}
if (cancellationToken.IsCancellationRequested)
{
continueLooping = false;
break;
}
var feedUrl = GetPageUri(Feeds[_settings.Feed].BaseUrl, pageIndex);
await ProcessPageBlock.SendAsync(feedUrl, cancellationToken).ConfigureAwait(false); // TODO: Need check with SongsPerPage
itemsInBlock++;
pageIndex++;
if ((pageIndex > maxPages && useMaxPages) || cancellationToken.IsCancellationRequested)
{
continueLooping = false;
}
// TODO: Better http error handling, what if only a single page is broken and returns 0 songs?
while (ProcessPageBlock.OutputCount > 0 || itemsInBlock == MaxConcurrency || !continueLooping)
{
if (cancellationToken.IsCancellationRequested)
{
continueLooping = false;
break;
}
if (itemsInBlock <= 0)
{
break;
}
await ProcessPageBlock.OutputAvailableAsync(cancellationToken).ConfigureAwait(false);
while (ProcessPageBlock.TryReceive(out PageReadResult pageResult))
{
if (pageResult != null)
{
pageResults.Add(pageResult);
}
if (Utilities.IsPaused)
{
await Utilities.WaitUntil(() => !Utilities.IsPaused, 500, cancellationToken).ConfigureAwait(false);
}
itemsInBlock--;
if (pageResult == null || pageResult.Count == 0) // TODO: This will trigger if a single page has an error.
{
Logger?.Debug("Received no new songs, last page reached.");
ProcessPageBlock.Complete();
itemsInBlock = 0;
continueLooping = false;
break;
}
if (pageResult.Count > 0)
{
Logger?.Debug($"Receiving {pageResult.Count} potential songs from {pageResult.Uri}");
}
else
{
Logger?.Debug($"Did not find any songs in {Name}.{settings.FeedName}.");
}
// TODO: Process PageReadResults for better error feedback.
foreach (var song in pageResult.Songs)
{
if (!retDict.ContainsKey(song.Hash))
{
if (retDict.Count < settings.MaxSongs || settings.MaxSongs == 0)
{
retDict.Add(song.Hash, song);
}
if (retDict.Count >= settings.MaxSongs && useMaxSongs)
{
continueLooping = false;
}
}
}
if (!useMaxPages || pageIndex <= maxPages)
{
if (retDict.Count < settings.MaxSongs)
{
continueLooping = true;
}
}
}
}
}
}while (continueLooping);
return(new FeedResult(retDict, pageResults));
}
/// <summary>
/// Test request observable used for generating exceptions and results
/// </summary>
/// <param name="request"></param>
/// <param name="exception"></param>
/// <param name="result"></param>
/// <returns></returns>
private IObservable <PageReadResult <CollectionModel> > GetData(PageReadRequest request, Exception exception, PageReadResult <CollectionModel> result)
{
_invokeCount = 0;
return(Observable.Create <PageReadResult <CollectionModel> >(o =>
{
++_invokeCount;
if (_invokeCount == 1)
{
o.OnError(exception);
}
else
{
o.OnNext(result);
o.OnCompleted();
}
return Disposable.Empty;
}));
}
/// <summary>
///
/// </summary>
/// <param name="page"></param>
/// <param name="cancellationToken"></param>
/// <exception cref="InvalidFeedSettingsException">Thrown when the feed's settings aren't valid.</exception>
/// <returns></returns>
public async Task <PageReadResult> GetSongsAsync(Uri pageUri, CancellationToken cancellationToken)
{
Stopwatch sw = new Stopwatch();
sw.Start();
string pageText = "";
bool isLastPage = false;
Logger.Debug($"Getting songs from '{pageUri}'");
ContentType contentType = ContentType.Unknown;
string? contentTypeStr;
IWebResponseMessage?response = null;
//PageReadResult result = null;
try
{
response = await WebUtils.WebClient.GetAsync(pageUri, cancellationToken).ConfigureAwait(false);
if ((response?.StatusCode ?? 500) == 500)
{
response?.Dispose();
response = null;
Logger?.Warning($"Internal server error on {pageUri}, retrying in 20 seconds");
await Task.Delay(20000).ConfigureAwait(false);
response = await WebUtils.WebClient.GetAsync(pageUri, cancellationToken).ConfigureAwait(false);
}
if (response == null)
{
throw new WebClientException($"Response was null for '{pageUri}'.");
}
response.EnsureSuccessStatusCode();
contentTypeStr = response.Content?.ContentType?.ToLower();
if (contentTypeStr != null && ContentDictionary.ContainsKey(contentTypeStr))
{
contentType = ContentDictionary[contentTypeStr];
}
else
{
contentType = ContentType.Unknown;
}
if (response.Content == null)
{
throw new WebClientException($"Response content was null for '{pageUri}'.");
}
pageText = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
}
catch (WebClientException ex)
{
return(PageReadResult.FromWebClientException(ex, pageUri));
}
catch (OperationCanceledException)
{
return(new PageReadResult(pageUri, null, null, null, 0, new FeedReaderException("Page read was cancelled.", new OperationCanceledException(), FeedReaderFailureCode.Cancelled), PageErrorType.Cancelled));
}
catch (Exception ex)
{
string message = $"Error downloading {pageUri} in TransformBlock.";
Logger?.Debug(message);
Logger?.Debug($"{ex.Message}\n{ex.StackTrace}");
return(new PageReadResult(pageUri, null, null, null, 0, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.Unknown));
}
finally
{
response?.Dispose();
response = null;
}
List <ScrapedSong> newSongs;
ScrapedSong?firstSong = null;
ScrapedSong?lastSong = null;
int songsOnPage = 0;
try
{
var scrapedSongs = GetSongsFromPageText(pageText, pageUri, contentType, Settings.StoreRawData || StoreRawData);
isLastPage = scrapedSongs.Count == 0;
firstSong = scrapedSongs.FirstOrDefault();
lastSong = scrapedSongs.LastOrDefault();
songsOnPage = scrapedSongs.Count;
newSongs = new List <ScrapedSong>();
foreach (var song in scrapedSongs)
{
if (Settings.Filter == null || Settings.Filter(song))
{
newSongs.Add(song);
}
if (Settings.StopWhenAny != null && Settings.StopWhenAny(song))
{
isLastPage = true;
}
}
}
catch (JsonReaderException ex)
{
// TODO: Probably don't need a logger message here, caller can deal with it.
string message = $"Error parsing page text for {pageUri} in TransformBlock.";
Logger?.Debug(message);
Logger?.Debug($"{ex.Message}\n{ex.StackTrace}");
return(new PageReadResult(pageUri, null, firstSong, lastSong, songsOnPage, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.ParsingError));
}
catch (XmlException ex)
{
// TODO: Probably don't need a logger message here, caller can deal with it.
string message = $"Error parsing page text for {pageUri} in TransformBlock.";
Logger?.Debug(message);
Logger?.Debug($"{ex.Message}\n{ex.StackTrace}");
return(new PageReadResult(pageUri, null, firstSong, lastSong, songsOnPage, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.ParsingError));
}
catch (Exception ex)
{
// TODO: Probably don't need a logger message here, caller can deal with it.
string message = $"Uncaught error parsing page text for {pageUri} in TransformBlock.";
Logger?.Debug(message);
Logger?.Debug($"{ex.Message}\n{ex.StackTrace}");
return(new PageReadResult(pageUri, null, firstSong, lastSong, songsOnPage, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.Unknown));
}
sw.Stop();
//Logger?.Debug($"Task for {feedUrl} completed in {sw.ElapsedMilliseconds}ms");
return(new PageReadResult(pageUri, newSongs, firstSong, lastSong, songsOnPage, isLastPage));
}
/// <summary>
///
/// </summary>
/// <param name="_settings"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
/// <exception cref="InvalidCastException">Throw when the provided settings object isn't a BeatSaverFeedSettings</exception>
/// <exception cref="ArgumentNullException">Thrown when <paramref name="_settings"/> is null.</exception>
public async Task <FeedResult> GetSongsFromScoreSaberAsync(ScoreSaberFeedSettings _settings, IProgress <ReaderProgress> progress, CancellationToken cancellationToken)
{
if (_settings == null)
{
throw new ArgumentNullException(nameof(_settings), "settings cannot be null for ScoreSaberReader.GetSongsFromScoreSaberAsync");
}
if (!(_settings is ScoreSaberFeedSettings settings))
{
throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE);
}
// "https://scoresaber.com/api.php?function=get-leaderboards&cat={CATKEY}&limit={LIMITKEY}&page={PAGENUMKEY}&ranked={RANKEDKEY}"
int songsPerPage = settings.SongsPerPage;
if (songsPerPage == 0)
{
songsPerPage = 100;
}
int pageNum = settings.StartingPage;
//int maxPages = (int)Math.Ceiling(settings.MaxSongs / ((float)songsPerPage));
int maxPages = settings.MaxPages;
int pagesChecked = 0;
if (pageNum > 1 && maxPages != 0)
{
maxPages = maxPages + pageNum - 1;
}
//if (settings.MaxPages > 0)
// maxPages = maxPages < settings.MaxPages ? maxPages : settings.MaxPages; // Take the lower limit.
var feed = new ScoreSaberFeed(settings);
try
{
feed.EnsureValidSettings();
}
catch (InvalidFeedSettingsException ex)
{
return(new FeedResult(null, null, ex, FeedResultError.Error));
}
Dictionary <string, ScrapedSong> songs = new Dictionary <string, ScrapedSong>();
var pageResults = new List <PageReadResult>();
Uri uri = feed.GetUriForPage(1);
PageReadResult result = await feed.GetSongsAsync(uri, cancellationToken).ConfigureAwait(false);
pageResults.Add(result);
foreach (var song in result.Songs)
{
if (!songs.ContainsKey(song.Hash) && (songs.Count < settings.MaxSongs || settings.MaxSongs == 0))
{
songs.Add(song.Hash, song);
}
}
pagesChecked++;
progress?.Report(new ReaderProgress(pagesChecked, songs.Count));
bool continueLooping = true;
do
{
pageNum++;
//int diffCount = 0;
if ((maxPages > 0 && pageNum > maxPages) || (settings.MaxSongs > 0 && songs.Count >= settings.MaxSongs))
{
break;
}
if (Utilities.IsPaused)
{
await Utilities.WaitUntil(() => !Utilities.IsPaused, 500).ConfigureAwait(false);
}
// TODO: Handle PageReadResult here
uri = feed.GetUriForPage(pageNum);
var pageResult = await feed.GetSongsAsync(uri, cancellationToken).ConfigureAwait(false);
pageResults.Add(pageResult);
if (pageResult.PageError == PageErrorType.Cancelled)
{
return(FeedResult.GetCancelledResult(songs, pageResults));
}
int uniqueSongCount = 0;
foreach (var song in pageResult.Songs)
{
//diffCount++;
if (!songs.ContainsKey(song.Hash) && (songs.Count < settings.MaxSongs || settings.MaxSongs == 0))
{
songs.Add(song.Hash, song);
uniqueSongCount++;
}
}
int prog = Interlocked.Increment(ref pagesChecked);
progress?.Report(new ReaderProgress(prog, uniqueSongCount));
if (uniqueSongCount > 0)
{
Logger?.Debug($"Receiving {uniqueSongCount} potential songs from {pageResult.Uri}");
}
else
{
Logger?.Debug($"Did not find any new songs on page '{pageResult.Uri}' of {Name}.{settings.FeedName}.");
}
if (pageResult.IsLastPage)
{
Logger?.Debug($"Last page reached.");
continueLooping = false;
}
if (!pageResult.Successful)
{
Logger?.Debug($"Page {pageResult.Uri.ToString()} failed, ending read.");
if (pageResult.Exception != null)
{
Logger?.Debug($"{pageResult.Exception.Message}\n{pageResult.Exception.StackTrace}");
}
continueLooping = false;
}
//pageReadTasks.Add(GetSongsFromPageAsync(url.ToString()));
if ((maxPages > 0 && pageNum >= maxPages) || (settings.MaxSongs > 0 && songs.Count >= settings.MaxSongs))
{
continueLooping = false;
}
} while (continueLooping);
if (pageResults.Any(r => r.PageError == PageErrorType.Cancelled))
{
return(FeedResult.GetCancelledResult(songs, pageResults));
}
return(new FeedResult(songs, pageResults));
}
