public static async Task <ScrapedSong> GetSongByKeyAsync(string key, CancellationToken cancellationToken)
{
var uri = new Uri(BEATSAVER_DETAILS_BASE_URL + key);
string pageText = "";
ScrapedSong song = null;
IWebResponseMessage response = null;
try
{
response = await WebUtils.GetBeatSaverAsync(uri, cancellationToken).ConfigureAwait(false);
if (response.IsSuccessStatusCode)
{
pageText = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
}
else
{
Logger?.Error($"Error getting song by key, {uri} responded with {response.StatusCode}:{response.ReasonPhrase}");
return(song);
}
if (string.IsNullOrEmpty(pageText))
{
Logger?.Warning($"Unable to get web page at {uri}");
return(null);
}
}
catch (WebClientException ex)
{
string errorText = string.Empty;
if (ex.Response != null)
{
switch (ex.Response.StatusCode)
{
case 408:
errorText = "Timeout";
break;
default:
errorText = "Site Error";
break;
}
}
Logger?.Error($"{errorText} while trying to populate fields for {key}");
return(null);
}
catch (AggregateException ae)
{
ae.WriteExceptions($"Exception while trying to get details for {key}");
}
catch (Exception ex)
{
Logger?.Exception($"Exception getting page {uri}", ex);
throw;
}
finally
{
response?.Dispose();
response = null;
}
song = ParseSongsFromPage(pageText, uri).FirstOrDefault();
return(song);
}
/// <summary>
/// Downloads a file from the specified URI to the specified path (path includes file name).
/// Creates the target directory if it doesn't exist. All exceptions are stored in the DownloadResult.
/// </summary>
/// <param name="uri"></param>
/// <param name="path"></param>
/// <exception cref="OperationCanceledException"></exception>
/// <returns></returns>
public static async Task <DownloadResult> DownloadFileAsync(Uri uri, string path, CancellationToken cancellationToken, bool overwrite = true)
{
string actualPath = path;
int statusCode = 0;
if (uri == null)
{
return(new DownloadResult(null, DownloadResultStatus.InvalidRequest, 0));
}
if (!overwrite && File.Exists(path))
{
return(new DownloadResult(null, DownloadResultStatus.IOFailed, 0));
}
try
{
using (IWebResponseMessage response = await SongFeedReaders.WebUtils.GetBeatSaverAsync(uri, cancellationToken, 30, 2).ConfigureAwait(false))
{
statusCode = response?.StatusCode ?? 0;
try
{
Directory.GetParent(path).Create();
actualPath = await response.Content.ReadAsFileAsync(path, overwrite, cancellationToken).ConfigureAwait(false);
}
catch (IOException ex)
{
// Also catches DirectoryNotFoundException
return(new DownloadResult(null, DownloadResultStatus.IOFailed, statusCode, response.ReasonPhrase, ex));
}
catch (InvalidOperationException ex)
{
// File already exists and overwrite is false.
return(new DownloadResult(null, DownloadResultStatus.IOFailed, statusCode, response.ReasonPhrase, ex));
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
return(new DownloadResult(null, DownloadResultStatus.Unknown, statusCode, response?.ReasonPhrase, ex));
}
}
}
catch (WebClientException ex)
{
int faultedCode = ex.Response?.StatusCode ?? 0;
DownloadResultStatus downloadResultStatus = DownloadResultStatus.NetFailed;
if (faultedCode == 404)
{
downloadResultStatus = DownloadResultStatus.NetNotFound;
}
return(new DownloadResult(null, downloadResultStatus, faultedCode, ex.Response?.ReasonPhrase, ex.Response?.Exception));
}
catch (OperationCanceledException ex)
{
return(new DownloadResult(null, DownloadResultStatus.Canceled, 0, ex?.Message, ex));
}
catch (Exception ex)
{
return(new DownloadResult(null, DownloadResultStatus.NetFailed, 0, ex?.Message, ex));
}
return(new DownloadResult(actualPath, DownloadResultStatus.Success, statusCode));
}
public static async Task <PageReadResult> GetSongByHashAsync(string hash, CancellationToken cancellationToken)
{
var uri = new Uri(BEATSAVER_GETBYHASH_BASE_URL + hash);
ScrapedSong song = null;
var pageError = PageErrorType.None;
Exception exception = null;
IWebResponseMessage response = null;
try
{
response = await WebUtils.GetBeatSaverAsync(uri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
var pageText = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
song = ParseSongsFromPage(pageText, uri).FirstOrDefault();
}
catch (WebClientException ex)
{
string errorText = string.Empty;
if (ex.Response != null)
{
switch (ex.Response.StatusCode)
{
case 408:
errorText = "Timeout";
break;
default:
errorText = "Site Error";
break;
}
}
string message = $"{errorText} while trying to populate fields for {hash}";
Logger?.Error(message);
exception = new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed);
}
catch (OperationCanceledException ex)
{
exception = new FeedReaderException(ex.Message, ex, FeedReaderFailureCode.Cancelled);
}
catch (AggregateException ae)
{
string message = $"Exception while trying to get details for {hash}";
ae.WriteExceptions(message);
exception = new FeedReaderException(message, ae, FeedReaderFailureCode.Generic);
}
catch (Exception ex)
{
string message = $"Exception getting page {uri.ToString()}";
Logger?.Exception(message, ex);
exception = new FeedReaderException(message, ex, FeedReaderFailureCode.Generic);
}
finally
{
response?.Dispose();
response = null;
}
List <ScrapedSong> retList = new List <ScrapedSong>();
if (song != null)
{
retList.Add(song);
}
return(new PageReadResult(uri, retList, exception, pageError));
}
/// <summary>
///
/// </summary>
/// <param name="authorName"></param>
/// <param name="cancellationToken"></param>
/// <exception cref="OperationCanceledException"></exception>
/// <returns></returns>
public static async Task <string> GetAuthorIDAsync(string authorName, CancellationToken cancellationToken)
{
if (string.IsNullOrEmpty(authorName))
{
return(string.Empty);
}
if (_authors.ContainsKey(authorName))
{
return(_authors[authorName]);
}
string mapperId = string.Empty;
int page = 0;
int? totalResults;
Uri sourceUri = null;
string pageText;
JObject result;
JToken matchingSong;
JToken[] songJSONAry;
var queryBuilder = new SearchQueryBuilder(BeatSaverSearchType.author, authorName);
do
{
Logger?.Debug($"Checking page {page + 1} for the author ID.");
sourceUri = new Uri(Feeds[BeatSaverFeed.Search].BaseUrl.Replace(SEARCHTYPEKEY, "advanced").Replace(SEARCHQUERY, queryBuilder.GetQueryString()).Replace(PAGEKEY, (page * SongsPerPage).ToString()));
result = new JObject();
IWebResponseMessage response = null;
try
{
response = await WebUtils.GetBeatSaverAsync(sourceUri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
pageText = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
result = JObject.Parse(pageText);
}
catch (WebClientException ex)
{
string errorText = string.Empty;
if (ex.Response != null)
{
switch (ex.Response.StatusCode)
{
case 408:
errorText = "Timeout";
break;
default:
errorText = "Site Error";
break;
}
}
Logger?.Error($"{errorText} getting UploaderID from author name, {sourceUri} responded with {ex.Response?.StatusCode}:{ex.Response?.ReasonPhrase}");
return(string.Empty);
}
catch (JsonReaderException ex)
{
// TODO: Should I break the loop here, or keep trying?
Logger?.Exception("Unable to parse JSON from text", ex);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
Logger?.Error($"Uncaught error getting UploaderID from author name {authorName}");
Logger?.Debug($"{ex}");
return(string.Empty);
}
finally
{
response?.Dispose();
response = null;
}
totalResults = result["totalDocs"]?.Value <int>(); // TODO: Check this
if (totalResults == null || totalResults == 0)
{
Logger?.Warning($"No songs by {authorName} found, is the name spelled correctly?");
return(string.Empty);
}
songJSONAry = result["docs"].ToArray();
matchingSong = (JObject)songJSONAry.FirstOrDefault(c => c["uploader"]?["username"]?.Value <string>()?.ToLower() == authorName.ToLower());
page++;
sourceUri = new Uri(Feeds[BeatSaverFeed.Search].BaseUrl.Replace(SEARCHQUERY, authorName).Replace(PAGEKEY, (page * SongsPerPage).ToString()));
} while ((matchingSong == null) && page * SongsPerPage < totalResults);
if (matchingSong == null)
{
Logger?.Warning($"No songs by {authorName} found, is the name spelled correctly?");
return(string.Empty);
}
mapperId = matchingSong["uploader"]["_id"].Value <string>();
_authors.TryAdd(authorName, mapperId);
return(mapperId);
}
/// <summary>
///
/// </summary>
/// <param name="settings"></param>
/// <returns></returns>
/// <exception cref="ArgumentNullException">Thrown when settings is null.</exception>
public static async Task <FeedResult> GetBeatSaverSongsAsync(BeatSaverFeedSettings settings, CancellationToken cancellationToken)
{
if (settings == null)
{
throw new ArgumentNullException(nameof(settings), "settings cannot be null for BeatSaverReader.GetBeatSaverSongsAsync");
}
if (cancellationToken.IsCancellationRequested)
{
return(FeedResult.CancelledResult);
}
// TODO: double checks the first page
int feedIndex = settings.FeedIndex;
bool useMaxPages = settings.MaxPages != 0;
bool useMaxSongs = settings.MaxSongs != 0;
Dictionary <string, ScrapedSong> songs = new Dictionary <string, ScrapedSong>();
string pageText = string.Empty;
JObject result = new JObject();
var pageUri = GetPageUrl(feedIndex);
IWebResponseMessage response = null;
try
{
response = await GetBeatSaverAsync(pageUri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
pageText = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
result = JObject.Parse(pageText);
}
catch (WebClientException ex)
{
string errorText = string.Empty;
if (ex.Response != null)
{
switch (ex.Response.StatusCode)
{
case 408:
errorText = "Timeout";
break;
default:
errorText = "Site Error";
break;
}
}
string message = $"{errorText} getting a response from {pageUri}: {ex.Message}";
Logger?.Debug(message);
Logger?.Debug($"{ex.Message}\n{ex.StackTrace}");
return(new FeedResult(null, null, new FeedReaderException(message, ex, FeedReaderFailureCode.SourceFailed), FeedResultError.Error));
}
catch (JsonReaderException ex)
{
string message = "Unable to parse JSON from text on first page in GetBeatSaverSongAsync()";
Logger?.Debug(message);
Logger?.Debug($"{ex.Message}\n{ex.StackTrace}");
return(new FeedResult(null, null, new FeedReaderException(message, ex, FeedReaderFailureCode.SourceFailed), FeedResultError.Error));
}
catch (OperationCanceledException)
{
return(FeedResult.CancelledResult);
}
catch (Exception ex)
{
string message = $"Uncaught error getting the first page in BeatSaverReader.GetBeatSaverSongsAsync(): {ex.Message}";
return(new FeedResult(null, null, new FeedReaderException(message, ex, FeedReaderFailureCode.SourceFailed), FeedResultError.Error));
}
finally
{
response?.Dispose();
response = null;
}
int?numSongs = result["totalDocs"]?.Value <int>();
int?lastPage = result["lastPage"]?.Value <int>();
if (numSongs == null || lastPage == null || numSongs == 0)
{
Logger?.Warning($"Error checking Beat Saver's {settings.FeedName} feed.");
return(new FeedResult(null, null, new FeedReaderException($"Error getting the first page in GetBeatSaverSongsAsync()", null, FeedReaderFailureCode.SourceFailed), FeedResultError.Error));
}
Logger?.Info($"Checking Beat Saver's {settings.FeedName} feed, {numSongs} songs available");
int maxPages = settings.MaxPages;
int pageNum = Math.Max(settings.StartingPage - 1, 0);
if (pageNum > 0 && useMaxPages)
{
maxPages += pageNum; // Add starting page to maxPages so we actually get songs if maxPages < starting page
}
List <Task <PageReadResult> > pageReadTasks = new List <Task <PageReadResult> >();
bool continueLooping = true;
do
{
pageUri = GetPageUrl(feedIndex, pageNum);
Logger?.Trace($"Creating task for {pageUri.ToString()}");
pageReadTasks.Add(GetSongsFromPageAsync(pageUri, cancellationToken));
pageNum++;
if ((pageNum > lastPage))
{
continueLooping = false;
}
if (useMaxPages && (pageNum >= maxPages))
{
continueLooping = false;
}
if (useMaxSongs && pageNum * SongsPerPage >= settings.MaxSongs)
{
continueLooping = false;
}
} while (continueLooping);
try
{
// TODO: Won't hit max songs if a song is added while reading the pages. (It'll bump all the songs down and we'll get a repeat)
await Task.WhenAll(pageReadTasks.ToArray()).ConfigureAwait(false);
}
#pragma warning disable CA1031 // Do not catch general exception types
catch (Exception ex)
#pragma warning restore CA1031 // Do not catch general exception types
{
string message = $"Error waiting for pageReadTasks: {ex.Message}";
// TODO: Probably don't need logging here.
Logger?.Debug(message);
Logger?.Debug(ex.StackTrace);
return(new FeedResult(null, null, new FeedReaderException(message, ex, FeedReaderFailureCode.SourceFailed), FeedResultError.Error));
}
var pageResults = new List <PageReadResult>();
foreach (var job in pageReadTasks)
{
var pageResult = await job.ConfigureAwait(false);
pageResults.Add(pageResult);
foreach (var song in pageResult.Songs)
{
if (songs.ContainsKey(song.Hash))
{
continue;
}
if (!useMaxSongs || songs.Count < settings.MaxSongs)
{
songs.Add(song.Hash, song);
}
}
}
return(new FeedResult(songs, pageResults));
}
public static void CompareGetAsync(IWebClient client1, IWebClient client2, Func <IWebClient, Task <IWebResponseMessage> > action)
{
IWebResponseMessage response1 = null;
IWebResponseMessage response2 = null;
Exception client1Exception = null;
Exception client2Exception = null;
try
{
response1 = action(client1).Result;
}
catch (AssertFailedException)
{
throw;
}
catch (AggregateException ex)
{
var assertFailedException = ex.InnerExceptions.Where(e => e is AssertFailedException).FirstOrDefault();
if (assertFailedException != null)
{
throw assertFailedException;
}
if (ex.InnerExceptions.Count == 1)
{
client1Exception = ex.InnerException;
}
else
{
client1Exception = ex;
}
}
catch (Exception ex)
{
client1Exception = ex;
}
try
{
response2 = action(client2).Result;
}
catch (AssertFailedException)
{
throw;
}
catch (AggregateException ex)
{
var assertFailedException = ex.InnerExceptions.Where(e => e is AssertFailedException).FirstOrDefault();
if (assertFailedException != null)
{
throw assertFailedException;
}
if (ex.InnerExceptions.Count == 1)
{
client2Exception = ex.InnerException;
}
else
{
client2Exception = ex;
}
}
catch (Exception ex)
{
client2Exception = ex;
}
if (client1Exception != null || client2Exception != null)
{
Console.WriteLine($"{client1?.GetType().Name}: {client1Exception?.GetType().Name}: {client1Exception?.Message}");
Console.WriteLine($"{client2?.GetType().Name}: {client2Exception?.GetType().Name}: {client2Exception?.Message}");
Assert.AreEqual(client1Exception?.GetType(), client2Exception?.GetType());
if (client1Exception is WebClientException webException1 && client2Exception is WebClientException webException2)
{
Assert.AreEqual(webException1.Uri?.ToString(), webException2.Uri?.ToString(), "WebClientExceptions' Uris do not match.");
CompareResponses(webException1.Response, webException2.Response);
}
else if (client1Exception is OperationCanceledException || client2Exception is OperationCanceledException)
{
if (client1Exception is OperationCanceledException && client2Exception is OperationCanceledException)
{
Console.WriteLine($"Operation canceled");
}
else
{
Assert.Fail($"One client was canceled, the other wasn't.");
}
}
else
{
Assert.Fail("Not a WebClientException");
}
return;
}
public async Task <FeedResult> GetSongsFromScoreSaberAsync(ScoreSaberFeedSettings settings, CancellationToken cancellationToken)
{
if (settings == null)
{
throw new ArgumentNullException(nameof(settings), "settings cannot be null for ScoreSaberReader.GetSongsFromScoreSaberAsync");
}
// "https://scoresaber.com/api.php?function=get-leaderboards&cat={CATKEY}&limit={LIMITKEY}&page={PAGENUMKEY}&ranked={RANKEDKEY}"
int songsPerPage = settings.SongsPerPage;
if (songsPerPage == 0)
{
songsPerPage = 100;
}
int pageNum = settings.StartingPage;
//int maxPages = (int)Math.Ceiling(settings.MaxSongs / ((float)songsPerPage));
int maxPages = settings.MaxPages;
if (pageNum > 1 && maxPages != 0)
{
maxPages = maxPages + pageNum - 1;
}
//if (settings.MaxPages > 0)
// maxPages = maxPages < settings.MaxPages ? maxPages : settings.MaxPages; // Take the lower limit.
Dictionary <string, ScrapedSong> songs = new Dictionary <string, ScrapedSong>();
StringBuilder url = new StringBuilder(Feeds[settings.Feed].BaseUrl);
Dictionary <string, string> urlReplacements = new Dictionary <string, string>()
{
{ LIMITKEY, songsPerPage.ToString() },
{ PAGENUMKEY, pageNum.ToString() },
{ RANKEDKEY, settings.RankedOnly ? "1" : "0" }
};
if (settings.Feed == ScoreSaberFeed.Search)
{
urlReplacements.Add(QUERYKEY, settings.SearchQuery);
}
GetPageUrl(ref url, urlReplacements);
var uri = new Uri(url.ToString());
string pageText = "";
var pageResults = new List <PageReadResult>();
IWebResponseMessage response = null;
try
{
response = await WebUtils.WebClient.GetAsync(uri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
pageText = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
}
catch (WebClientException ex)
{
string errorText = string.Empty;
int statusCode = ex?.Response?.StatusCode ?? 0;
if (statusCode != 0)
{
switch (statusCode)
{
case 408:
errorText = "Timeout";
break;
default:
errorText = "Site Error";
break;
}
}
string message = $"{errorText} getting first page in ScoreSaberReader: {uri}: {ex.Message}";
Logger?.Debug(message);
// No need for a stacktrace if it's one of these errors.
if (!(statusCode == 408 || statusCode == 500))
{
Logger?.Debug($"{ex.Message}\n{ex.StackTrace}");
}
return(new FeedResult(null, null, new FeedReaderException(message, ex, FeedReaderFailureCode.SourceFailed), FeedResultError.Error));
}
catch (Exception ex)
{
string message = $"Uncaught error getting the first page in ScoreSaberReader.GetSongsFromScoreSaberAsync(): {ex.Message}";
return(new FeedResult(null, null, new FeedReaderException(message, ex, FeedReaderFailureCode.SourceFailed), FeedResultError.Error));
}
finally
{
response?.Dispose();
response = null;
}
var result = GetSongsFromPageText(pageText, uri);
pageResults.Add(result);
foreach (var song in result.Songs)
{
if (!songs.ContainsKey(song.Hash) && (songs.Count < settings.MaxSongs || settings.MaxSongs == 0))
{
songs.Add(song.Hash, song);
}
}
bool continueLooping = true;
do
{
pageNum++;
int diffCount = 0;
if ((maxPages > 0 && pageNum > maxPages) || (settings.MaxSongs > 0 && songs.Count >= settings.MaxSongs))
{
break;
}
url.Clear();
url.Append(Feeds[settings.Feed].BaseUrl);
if (!urlReplacements.ContainsKey(PAGENUMKEY))
{
urlReplacements.Add(PAGENUMKEY, pageNum.ToString());
}
else
{
urlReplacements[PAGENUMKEY] = pageNum.ToString();
}
GetPageUrl(ref url, urlReplacements);
uri = new Uri(url.ToString());
if (Utilities.IsPaused)
{
await Utilities.WaitUntil(() => !Utilities.IsPaused, 500).ConfigureAwait(false);
}
// TODO: Handle PageReadResult here
var pageResult = await GetSongsFromPageAsync(uri, cancellationToken).ConfigureAwait(false);
pageResults.Add(pageResult);
foreach (var song in pageResult.Songs)
{
diffCount++;
if (!songs.ContainsKey(song.Hash) && (songs.Count < settings.MaxSongs || settings.MaxSongs == 0))
{
songs.Add(song.Hash, song);
}
}
if (diffCount == 0)
{
Logger?.Debug($"No diffs found on {uri.ToString()}, should be after last page.");
continueLooping = false;
}
//pageReadTasks.Add(GetSongsFromPageAsync(url.ToString()));
if ((maxPages > 0 && pageNum >= maxPages) || (settings.MaxSongs > 0 && songs.Count >= settings.MaxSongs))
{
continueLooping = false;
}
} while (continueLooping);
return(new FeedResult(songs, pageResults));
}
