/// <summary>
///
/// </summary>
/// <param name="_settings"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
/// <exception cref="InvalidCastException">Thrown when the provided IFeedSettings isn't a ScoreSaberFeedSettings.</exception>
/// <exception cref="ArgumentException">Thrown when the Search feed is selected and the query in settings isn't valid.</exception>
/// <exception cref="ArgumentOutOfRangeException">Thrown when the feed specified in the settings isn't valid.</exception>
/// <exception cref="ArgumentNullException">Thrown when settings is null.</exception>
public override Task <FeedResult> GetSongsFromFeedAsync(IFeedSettings _settings, IProgress <ReaderProgress> progress, CancellationToken cancellationToken)
{
PrepareReader();
if (_settings == null)
{
throw new ArgumentNullException(nameof(_settings), "settings cannot be null for ScoreSaberReader.GetSongsFromFeedAsync");
}
if (!(_settings is ScoreSaberFeedSettings settings))
{
throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE);
}
if (!((settings.FeedIndex >= 0 && settings.FeedIndex <= 3) || settings.FeedIndex == 99)) // Validate FeedIndex
{
throw new ArgumentOutOfRangeException(nameof(_settings), "_settings contains an invalid FeedIndex value for ScoreSaberReader");
}
Dictionary <string, ScrapedSong> retDict = new Dictionary <string, ScrapedSong>();
if (settings.Feed == ScoreSaberFeedName.TopRanked || settings.Feed == ScoreSaberFeedName.LatestRanked)
{
settings.RankedOnly = true;
}
if (settings.Feed == ScoreSaberFeedName.Search)
{
if (!IsValidSearchQuery(settings.SearchQuery))
{
throw new ArgumentException($"Search query '{settings.SearchQuery ?? "<nul>"}' is not a valid query.");
}
}
return(GetSongsFromScoreSaberAsync(settings, progress, cancellationToken));
}
public AtomFeedGenerator(IFeedDataClient feedDataClient,
IFeedService feedService,
IPublishService publishService,
IFeedSettings feedSettings)
: base(feedDataClient, feedService, publishService, feedSettings)
{
}
public RssFeedGenerator(IFeedDataClient feedDataClient,
IFeedService feedService,
IPublishService publishService,
IFeedSettings feedSettings)
: base(feedDataClient, feedService, publishService, feedSettings)
{
}
public override string GetFeedName(IFeedSettings settings)
{
if (!(settings is ScoreSaberFeedSettings ssSettings))
{
throw new ArgumentException("Settings is not ScoreSaberFeedSettings", nameof(settings));
}
return(ScoreSaberFeed.Feeds[ssSettings.Feed].DisplayName);
}
public string GetFeedName(IFeedSettings settings)
{
if (!(settings is BeastSaberFeedSettings ssSettings))
{
throw new ArgumentException("Settings is not BeastSaberFeedSettings", nameof(settings));
}
return(Feeds[ssSettings.Feed].DisplayName);
}
private static ArticleFeedGenerator CreateFeedGenerator(IFeedSettings feedSettings)
{
string feedFormat = feedSettings.FeedFormat;
switch (feedFormat.ToLower())
{
case "atom": return new AtomFeedGenerator(new RareburgClient(), new RareburgArticleFeedService(), new S3PublishService(), feedSettings);
case "rss": return new RssFeedGenerator(new RareburgClient(), new RareburgArticleFeedService(), new S3PublishService(), feedSettings);
default: throw new ArgumentException("Unknown feed format");
}
}
public ArticleFeedGenerator(IFeedDataClient feedDataClient,
IFeedService feedService,
IPublishService publishService,
IFeedSettings feedSettings)
{
_feedDataClient = feedDataClient;
_feedService = feedService;
_publishService = publishService;
_feedSettings = feedSettings;
}
public ArticleFeedGenerator(IFeedDataClient feedDataClient,
IFeedService feedService,
IPublishService publishService,
IFeedSettings feedSettings)
{
_feedDataClient = feedDataClient;
_feedService = feedService;
_publishService = publishService;
_feedSettings = feedSettings;
}
/// <summary>
///
/// </summary>
/// <param name="_settings"></param>
/// <exception cref="InvalidCastException">Thrown when the passed IFeedSettings isn't a BeatSaverFeedSettings</exception>
/// <returns></returns>
public Dictionary <string, SongInfo> GetSongsFromFeed(IFeedSettings _settings)
{
PrepareReader();
if (!(_settings is BeatSaverFeedSettings settings))
{
throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE);
}
List <SongInfo> songs = new List <SongInfo>();
switch (settings.FeedIndex)
{
// Author
case 0:
foreach (var author in settings.Authors)
{
songs.AddRange(GetSongsByAuthor(author));
}
break;
// Newest
case 1:
songs.AddRange(GetNewestSongs(settings));
break;
// Top
case 2:
break;
default:
break;
}
Dictionary <string, SongInfo> retDict = new Dictionary <string, SongInfo>();
foreach (var song in songs)
{
if (retDict.ContainsKey(song.id))
{
if (retDict[song.id].SongVersion < song.SongVersion)
{
Logger.Debug($"Song with ID {song.id} already exists, updating");
retDict[song.id] = song;
}
else
{
Logger.Debug($"Song with ID {song.id} is already the newest version");
}
}
else
{
retDict.Add(song.id, song);
}
}
return(retDict);
}
static void Main(string[] args)
{
IConfigurationFactory configFactory = new AppConfigConfigurationFactory();
// IConfigurationFactory configFactory = new DynamoDBConfigurationFactory();
IApiSettings apiSettings = configFactory.GetApiSettings();
IFeedSettings feedSettings = configFactory.GetFeedSettings();
IFeedServiceSettings feedServiceSettings = configFactory.GetFeedServiceSettings();
IS3PublisherSettings s3PublishSettings = configFactory.GetS3PublisherSettings();
IOfflineClientSettings offlineClientSettings = configFactory.GetOfflineClientSettings();
}
/// <summary>
/// Gets the DisplayName of the feed specified in settings.
/// </summary>
/// <param name="settings"></param>
/// <returns></returns>
/// <exception cref="ArgumentNullException">Thrown when settings is null</exception>
/// <exception cref="InvalidCastException">Thrown when settings is not a BeatSaverFeedSettings.</exception>
public override string GetFeedName(IFeedSettings settings)
{
if (settings == null)
{
throw new ArgumentNullException(nameof(settings), "settings cannot be null for BeatSaverReader.GetFeedName");
}
if (!(settings is BeatSaverFeedSettings ssSettings))
{
throw new InvalidCastException("Settings is not BeatSaverFeedSettings in BeatSaverReader.GetFeedName");
}
return(BeatSaverFeed.Feeds[ssSettings.Feed].DisplayName);
}
private static ArticleFeedGenerator CreateFeedGenerator(IFeedSettings feedSettings)
{
string feedFormat = feedSettings.FeedFormat;
switch (feedFormat.ToLower())
{
case "atom": return(new AtomFeedGenerator(new RareburgClient(), new RareburgArticleFeedService(), new S3PublishService(), feedSettings));
case "rss": return(new RssFeedGenerator(new RareburgClient(), new RareburgArticleFeedService(), new S3PublishService(), feedSettings));
default: throw new ArgumentException("Unknown feed format");
}
}
public async Task <Dictionary <string, ScrapedSong> > GetSongsFromFeedAsync(IFeedSettings _settings, CancellationToken cancellationToken)
{
PrepareReader();
if (!(_settings is BeatSaverFeedSettings settings))
{
throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE);
}
List <ScrapedSong> songs = null;
switch ((BeatSaverFeed)settings.FeedIndex)
{
// Author
case BeatSaverFeed.Author:
string songSource = string.Empty;
List <ScrapedSong> newSongs = null;
songs = new List <ScrapedSong>();
foreach (var author in settings.Authors)
{
newSongs = await GetSongsByAuthorAsync(author, CalcMaxPages(settings.MaxPages, settings.MaxSongs)).ConfigureAwait(false);
songSource = "Beat Saver";
songs.AddRange(newSongs.Take(settings.MaxSongs));
Logger.Info($"Found {newSongs.Count} songs uploaded by {author} from {songSource}");
}
break;
case BeatSaverFeed.Search:
songs = await SearchAsync(settings.SearchCriteria, settings.SearchType).ConfigureAwait(false);
break;
// Latest/Hot/Plays/Downloads
default:
songs = await GetBeatSaverSongsAsync(settings).ConfigureAwait(false);
break;
}
Dictionary <string, ScrapedSong> retDict = new Dictionary <string, ScrapedSong>();
foreach (var song in songs)
{
if (!retDict.ContainsKey(song.Hash))
{
retDict.Add(song.Hash, song);
}
}
return(retDict);
}
public FeedResult GetSongsFromFeed(IFeedSettings settings)
{
try
{
return(GetSongsFromFeedAsync(settings, CancellationToken.None).Result);
}
catch (AggregateException ex)
{
var flattened = ex.Flatten();
if (flattened.InnerExceptions.Count == 1)
{
throw flattened.InnerException;
}
throw ex;
}
}
public virtual FeedResult GetSongsFromFeed(IFeedSettings settings, CancellationToken cancellationToken)
{
try
{
return(GetSongsFromFeedAsync(settings, default(IProgress <ReaderProgress>), cancellationToken).Result);
}
catch (AggregateException ex)
{
var flattened = ex.Flatten();
if (flattened.InnerExceptions.Count == 1)
{
throw flattened.InnerException;
}
throw ex;
}
}
public Dictionary <string, SongInfo> GetSongsFromFeed(IFeedSettings _settings)
{
PrepareReader();
if (!(_settings is ScoreSaberFeedSettings settings))
{
throw new InvalidCastException(INVALID_FEED_SETTINGS_MESSAGE);
}
List <SongInfo> songs = new List <SongInfo>();
switch (settings.FeedIndex)
{
// Author
case 0:
songs.AddRange(GetTopPPSongs(settings));
break;
default:
break;
}
Dictionary <string, SongInfo> retDict = new Dictionary <string, SongInfo>();
foreach (var song in songs)
{
if (retDict.ContainsKey(song.id))
{
if (retDict[song.id].SongVersion < song.SongVersion)
{
Logger.Debug($"Song with ID {song.id} already exists, updating");
retDict[song.id] = song;
}
else if (retDict[song.id].SongVersion == song.SongVersion)
{
Logger.Debug($"Tried to add a song we already got");
}
else
{
Logger.Debug($"Song with ID {song.id} is already the newest version");
}
}
else
{
retDict.Add(song.id, song);
}
}
return(retDict);
}
public Dictionary <string, ScrapedSong> GetSongsFromFeed(IFeedSettings settings)
{
// Pointless to have these checks here?
PrepareReader();
if (!(settings is BeastSaberFeedSettings _settings))
{
throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE);
}
if (_settings.FeedIndex != 2 && string.IsNullOrEmpty(_username?.Trim()))
{
Logger.Error($"Can't access feed without a valid username in the config file");
throw new ArgumentException("Cannot access this feed without a valid username.");
}
var retDict = GetSongsFromFeedAsync(settings).Result;
return(retDict);
}
public void DownloadSongsFromFeed(string feedType, IFeedSettings _settings)
{
if (!FeedReaders.ContainsKey(feedType))
{
Logger.Error($"Invalid feed type passed to DownloadSongsFromFeed: {feedType}");
return;
}
var reader = FeedReaders[feedType];
DateTime startTime = DateTime.Now;
Dictionary <int, SongInfo> songs;
List <Playlist> playlists = new List <Playlist> {
_syncSaberSongs
};
playlists.AddRange(reader.PlaylistsForFeed(_settings.FeedIndex));
try
{
songs = reader.GetSongsFromFeed(_settings);
}
catch (InvalidCastException)
{
Logger.Error($"Wrong type of IFeedSettings given to {feedType}");
return;
}
int totalSongs = songs.Count;
Logger.Debug($"Finished checking pages, found {totalSongs} songs");
List <SongInfo> matchedSongs = DownloadSongs(songs, out (List <SongInfo> exists, List <SongInfo> history)skippedSongs, _settings.UseSongKeyAsOutputFolder);
_songDownloadHistory.AddRange(skippedSongs.exists.Select(s => s.key));
Logger.Debug("Jobs finished, Processing downloads...");
int downloadCount = SuccessfulDownloads.Count;
int failedCount = FailedDownloads.Count;
ProcessDownloads(playlists);
foreach (var p in playlists)
{
matchedSongs.ForEach(s => p.TryAdd(s.hash, s.key, s.songName));
p.WritePlaylist();
}
var timeElapsed = (DateTime.Now - startTime);
Logger.Info($"Downloaded {downloadCount} songs from {reader.Source}'s {_settings.FeedName} feed in {FormatTimeSpan(timeElapsed)}. " +
$"Skipped {skippedSongs.exists.Count} songs that already exist and {skippedSongs.history.Count} that are only in history{(failedCount > 0 ? $", failed to download {failedCount} songs." : "")}.");
}
static void Main(string[] args)
{
IConfigurationFactory configFactory = new AppConfigConfigurationFactory();
// IConfigurationFactory configFactory = new DynamoDBConfigurationFactory();
IApiSettings apiSettings = configFactory.GetApiSettings();
IFeedSettings feedSettings = configFactory.GetFeedSettings();
IFeedServiceSettings feedServiceSettings = configFactory.GetFeedServiceSettings();
IS3PublisherSettings s3PublishSettings = configFactory.GetS3PublisherSettings();
IOfflineClientSettings offlineClientSettings = configFactory.GetOfflineClientSettings();
var feedDataClient = new OfflineRareburgClient(offlineClientSettings);
var feedService = new RareburgArticleFeedService(feedServiceSettings);
var publishService = new S3PublishService(s3PublishSettings, feedSettings);
var feedGenerator = CreateFeedGenerator(feedDataClient, feedService, publishService, feedSettings);
feedGenerator.Run();
}
/// <summary>
///
/// </summary>
/// <param name="_settings"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
/// <exception cref="InvalidCastException">Throw when the provided settings object isn't a BeatSaverFeedSettings</exception>
/// <exception cref="ArgumentNullException">Thrown when <paramref name="_settings"/> is null.</exception>
public async Task <FeedResult> GetSongsFromFeedAsync(IFeedSettings _settings, CancellationToken cancellationToken)
{
PrepareReader();
if (_settings == null)
{
throw new ArgumentNullException(nameof(_settings), "Settings cannot be null for BeatSaverReader.GetSongsFromFeedAsync");
}
if (!(_settings is BeatSaverFeedSettings settings))
{
throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE);
}
FeedResult songs = null;
switch ((BeatSaverFeed)settings.FeedIndex)
{
// Author
case BeatSaverFeed.Author:
songs = await GetSongsByAuthorAsync(settings.Criteria, cancellationToken, CalcMaxSongs(settings.MaxPages, settings.MaxSongs)).ConfigureAwait(false);
break;
case BeatSaverFeed.Search:
songs = await SearchAsync(settings, cancellationToken).ConfigureAwait(false);
break;
// Latest/Hot/Plays/Downloads
default:
songs = await GetBeatSaverSongsAsync(settings, cancellationToken).ConfigureAwait(false);
break;
}
//Dictionary<string, ScrapedSong> retDict = new Dictionary<string, ScrapedSong>();
//foreach (var song in songs)
//{
// if (!retDict.ContainsKey(song.Hash))
// {
// retDict.Add(song.Hash, song);
// }
//}
return(songs);
}
public async Task <Dictionary <string, ScrapedSong> > GetSongsFromFeedAsync(IFeedSettings _settings, CancellationToken cancellationToken)
{
PrepareReader();
if (!(_settings is ScoreSaberFeedSettings settings))
{
throw new InvalidCastException(INVALID_FEED_SETTINGS_MESSAGE);
}
Dictionary <string, ScrapedSong> retDict = new Dictionary <string, ScrapedSong>();
int maxSongs = settings.MaxSongs > 0 ? settings.MaxSongs : settings.SongsPerPage * settings.SongsPerPage;
switch (settings.Feed)
{
case ScoreSaberFeed.Trending:
retDict = await GetSongsFromScoreSaberAsync(settings).ConfigureAwait(false);
break;
case ScoreSaberFeed.LatestRanked:
settings.RankedOnly = true;
retDict = await GetSongsFromScoreSaberAsync(settings).ConfigureAwait(false);
break;
case ScoreSaberFeed.TopPlayed:
retDict = await GetSongsFromScoreSaberAsync(settings).ConfigureAwait(false);
break;
case ScoreSaberFeed.TopRanked:
settings.RankedOnly = true;
retDict = await GetSongsFromScoreSaberAsync(settings).ConfigureAwait(false);
break;
default:
break;
}
return(retDict);
}
public EditionXmlFormatter(IFeedSettings feedInfo)
{
m_feedInfo = feedInfo;
}
public async Task <Dictionary <string, ScrapedSong> > GetSongsFromFeedAsync(IFeedSettings settings)
{
return(await GetSongsFromFeedAsync(settings, CancellationToken.None).ConfigureAwait(false));
}
public Dictionary <string, ScrapedSong> GetSongsFromFeed(IFeedSettings _settings)
{
return(GetSongsFromFeedAsync(_settings).Result);
}
public async Task <Dictionary <string, ScrapedSong> > ReadFeed(IFeedReader reader, IFeedSettings settings, Playlist feedPlaylist, PlaylistStyle playlistStyle)
{
//Logger.log?.Info($"Getting songs from {feedName} feed.");
var songs = await reader.GetSongsFromFeedAsync(settings).ConfigureAwait(false) ?? new Dictionary <string, ScrapedSong>();
if (songs.Count > 0 && playlistStyle == PlaylistStyle.Replace)
{
feedPlaylist.Clear();
}
var addDate = DateTime.Now;
var decrement = new TimeSpan(1);
foreach (var scrapedSong in songs)
{
if (HistoryManager.TryGetValue(scrapedSong.Value.Hash, out var historyEntry) &&
(historyEntry.Flag == HistoryFlag.NotFound ||
historyEntry.Flag == HistoryFlag.Deleted))
{
continue;
}
//if (string.IsNullOrEmpty(scrapedSong.Value.SongKey))
//{
// try
// {
// //Logger.log?.Info($"Grabbing key from BeatSaver: {scrapedSong.Value.SongName} by {scrapedSong.Value.MapperName}");
// // ScrapedSong doesn't have a Beat Saver key associated with it, probably scraped from ScoreSaber
// scrapedSong.Value.UpdateFrom(await BeatSaverReader.GetSongByHashAsync(scrapedSong.Key), false);
// }
// catch (ArgumentNullException)
// {
// Logger.log?.Warn($"Unable to find {scrapedSong.Value?.SongName} by {scrapedSong.Value?.MapperName} on Beat Saver ({scrapedSong.Key})");
// }
//}
var song = scrapedSong.Value.ToPlaylistSong();
song.DateAdded = addDate;
feedPlaylist?.TryAdd(song);
addDate = addDate - decrement;
}
feedPlaylist?.TryWriteFile();
return(songs);
}
public S3PublishService(IS3PublisherSettings s3PublisherSettings, IFeedSettings feedSettings)
{
_s3PublisherSettings = s3PublisherSettings;
_feedSettings = feedSettings;
}
/// <summary>
///
/// </summary>
/// <param name="_settings"></param>
/// <exception cref="InvalidCastException">Thrown when the passed IFeedSettings isn't a BeatSaverFeedSettings</exception>
/// <returns></returns>
public Dictionary <int, SongInfo> GetSongsFromFeed(IFeedSettings _settings)
{
PrepareReader();
if (!(_settings is BeatSaverFeedSettings settings))
{
throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE);
}
List <SongInfo> songs = new List <SongInfo>();
switch ((BeatSaverFeeds)settings.FeedIndex)
{
// Author
case BeatSaverFeeds.AUTHOR:
List <SongInfo> newSongs = null;
string songSource = string.Empty;
foreach (var author in settings.Authors)
{
if (!settings.searchOnline)
{
newSongs = ScrapedDataProvider.Songs.Values.Where(s => !string.IsNullOrEmpty(s.BeatSaverInfo?.uploader?.username) && s.BeatSaverInfo?.uploader.username.ToLower() == author.ToLower()).ToList();
songSource = "scraped data";
}
if (newSongs == null || newSongs.Count == 0)
{
newSongs = GetSongsByAuthor(author);
songSource = "Beat Saver";
}
songs.AddRange(newSongs);
Logger.Info($"Found {newSongs.Count} songs uploaded by {author} from {songSource}");
}
break;
// Newest
case BeatSaverFeeds.LATEST:
songs.AddRange(GetNewestSongs(settings));
break;
// Top
case BeatSaverFeeds.HOT:
break;
default:
break;
}
Dictionary <int, SongInfo> retDict = new Dictionary <int, SongInfo>();
foreach (var song in songs)
{
if (retDict.ContainsKey(song.keyAsInt))
{/*
* if (retDict[song.keyAsInt].SongVersion < song.SongVersion)
* {
* Logger.Debug($"Song with ID {song.keyAsInt} already exists, updating");
* retDict[song.keyAsInt] = song;
* }
* else
* {
* Logger.Debug($"Song with ID {song.keyAsInt} is already the newest version");
* }*/
}
else
{
retDict.Add(song.keyAsInt, song);
}
}
return(retDict);
}
public Dictionary <int, SongInfo> GetSongsFromFeed(IFeedSettings _settings)
{
PrepareReader();
if (!(_settings is ScoreSaberFeedSettings settings))
{
throw new InvalidCastException(INVALID_FEED_SETTINGS_MESSAGE);
}
List <SongInfo> songs = new List <SongInfo>();
int maxSongs = settings.MaxSongs > 0 ? settings.MaxSongs : settings.SongsPerPage * settings.SongsPerPage;
switch ((ScoreSaberFeeds)settings.FeedIndex)
{
case ScoreSaberFeeds.TRENDING:
if (!settings.searchOnline)
{
if (maxSongs <= 0)
{
throw new ArgumentException("You must specify a value greater than 0 for MaxPages or MaxSongs.");
}
songs.AddRange(ScrapedDataProvider.Songs.Values.Where(s => s.ScoreSaberInfo.Count > 0).OrderByDescending(s =>
s.ScoreSaberInfo.Values.Select(ss => ss.scores_day).Aggregate((a, b) => a + b)).Take(maxSongs));
}
break;
case ScoreSaberFeeds.LATEST_RANKED:
if (!settings.searchOnline)
{
if (maxSongs <= 0)
{
throw new ArgumentException("You must specify a value greater than 0 for MaxPages or MaxSongs.");
}
songs.AddRange(ScrapedDataProvider.Songs.Values.Where(s => s.RankedDifficulties.Count > 0).OrderByDescending(ss =>
ss.ScoreSaberInfo.Keys.Max()).Take(maxSongs));
}
break;
case ScoreSaberFeeds.TOP_PLAYED:
if (!settings.searchOnline)
{
if (maxSongs <= 0)
{
throw new ArgumentException("You must specify a value greater than 0 for MaxPages or MaxSongs.");
}
songs.AddRange(ScrapedDataProvider.Songs.Values.Where(s => s.ScoreSaberInfo.Count > 0).OrderByDescending(s =>
s.ScoreSaberInfo.Values.Select(ss => ss.scores).Aggregate((a, b) => a + b)).Take(maxSongs));
}
break;
case ScoreSaberFeeds.TOP_RANKED:
if (!settings.searchOnline)
{
if (maxSongs <= 0)
{
throw new ArgumentException("You must specify a value greater than 0 for MaxPages or MaxSongs.");
}
songs.AddRange(ScrapedDataProvider.Songs.Values.Where(s => s.RankedDifficulties.Count > 0).OrderByDescending(s => s.RankedDifficulties.Max(d => d.Value)).Take(maxSongs));
}
else
{
songs.AddRange(GetTopPPSongs(settings));
}
break;
default:
break;
}
Dictionary <int, SongInfo> retDict = new Dictionary <int, SongInfo>();
foreach (var song in songs)
{
if (!retDict.ContainsKey(song.keyAsInt))
{
retDict.Add(song.keyAsInt, song);
}
}
return(retDict);
}
public Task <FeedResult> GetSongsFromFeedAsync(IFeedSettings settings)
{
return(GetSongsFromFeedAsync(settings, CancellationToken.None));
}
// TODO: Abort early when bsaber.com is down (check if all items in block failed?)
// TODO: Make cancellationToken actually do something.
/// <summary>
/// Gets all songs from the feed defined by the provided settings.
/// </summary>
/// <param name="settings"></param>
/// <param name="cancellationToken"></param>
/// <exception cref="InvalidCastException">Thrown when the passed IFeedSettings isn't a BeastSaberFeedSettings.</exception>
/// <exception cref="ArgumentException">Thrown when trying to access a feed that requires a username and the username wasn't provided.</exception>
/// <exception cref="OperationCanceledException"></exception>
/// <returns></returns>
public async Task <FeedResult> GetSongsFromFeedAsync(IFeedSettings settings, CancellationToken cancellationToken)
{
if (cancellationToken.IsCancellationRequested)
{
return(FeedResult.CancelledResult);
}
if (settings == null)
{
throw new ArgumentNullException(nameof(settings), "settings cannot be null for BeastSaberReader.GetSongsFromFeedAsync.");
}
Dictionary <string, ScrapedSong> retDict = new Dictionary <string, ScrapedSong>();
if (!(settings is BeastSaberFeedSettings _settings))
{
throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE);
}
if (_settings.FeedIndex != 2 && string.IsNullOrEmpty(_username?.Trim()))
{
//Logger?.Error($"Can't access feed without a valid username in the config file");
throw new ArgumentException("Cannot access this feed without a valid username.");
}
int pageIndex = settings.StartingPage;
int maxPages = _settings.MaxPages;
bool useMaxSongs = _settings.MaxSongs != 0;
bool useMaxPages = maxPages != 0;
if (useMaxPages && pageIndex > 1)
{
maxPages = maxPages + pageIndex - 1;
}
var ProcessPageBlock = new TransformBlock <Uri, PageReadResult>(async feedUri =>
{
Stopwatch sw = new Stopwatch();
sw.Start();
//Logger?.Debug($"Checking URL: {feedUrl}");
string pageText = "";
ContentType contentType = ContentType.Unknown;
string contentTypeStr = string.Empty;
IWebResponseMessage response = null;
try
{
response = await WebUtils.WebClient.GetAsync(feedUri, cancellationToken).ConfigureAwait(false);
if ((response?.StatusCode ?? 500) == 500)
{
response?.Dispose();
response = null;
Logger?.Warning($"Internal server error on {feedUri}, retrying in 20 seconds");
await Task.Delay(20000).ConfigureAwait(false);
response = await WebUtils.WebClient.GetAsync(feedUri, cancellationToken).ConfigureAwait(false);
}
response.EnsureSuccessStatusCode();
contentTypeStr = response.Content.ContentType.ToLower();
if (ContentDictionary.ContainsKey(contentTypeStr))
{
contentType = ContentDictionary[contentTypeStr];
}
else
{
contentType = ContentType.Unknown;
}
pageText = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
}
catch (WebClientException ex)
{
return(PageReadResult.FromWebClientException(ex, feedUri));
}
catch (OperationCanceledException)
{
return(new PageReadResult(feedUri, null, new FeedReaderException("Page read was cancelled.", new OperationCanceledException(), FeedReaderFailureCode.Cancelled), PageErrorType.Cancelled));
}
catch (Exception ex)
{
string message = $"Error downloading {feedUri} in TransformBlock.";
Logger?.Debug(message);
Logger?.Debug($"{ex.Message}\n{ex.StackTrace}");
return(new PageReadResult(feedUri, null, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.Unknown));
}
finally
{
response?.Dispose();
response = null;
}
List <ScrapedSong> newSongs = null;
try
{
newSongs = GetSongsFromPageText(pageText, feedUri, contentType);
}
catch (JsonReaderException ex)
{
// TODO: Probably don't need a logger message here, caller can deal with it.
string message = $"Error parsing page text for {feedUri} in TransformBlock.";
Logger?.Debug(message);
Logger?.Debug($"{ex.Message}\n{ex.StackTrace}");
return(new PageReadResult(feedUri, null, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.ParsingError));
}
catch (XmlException ex)
{
// TODO: Probably don't need a logger message here, caller can deal with it.
string message = $"Error parsing page text for {feedUri} in TransformBlock.";
Logger?.Debug(message);
Logger?.Debug($"{ex.Message}\n{ex.StackTrace}");
return(new PageReadResult(feedUri, null, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.ParsingError));
}
catch (Exception ex)
{
// TODO: Probably don't need a logger message here, caller can deal with it.
string message = $"Uncaught error parsing page text for {feedUri} in TransformBlock.";
Logger?.Debug(message);
Logger?.Debug($"{ex.Message}\n{ex.StackTrace}");
return(new PageReadResult(feedUri, null, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.Unknown));
}
sw.Stop();
//Logger?.Debug($"Task for {feedUrl} completed in {sw.ElapsedMilliseconds}ms");
return(new PageReadResult(feedUri, newSongs));
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = MaxConcurrency,
BoundedCapacity = MaxConcurrency,
CancellationToken = cancellationToken
//#if NETSTANDARD
// , EnsureOrdered = true
//#endif
});
bool continueLooping = true;
int itemsInBlock = 0;
List <PageReadResult> pageResults = new List <PageReadResult>(maxPages + 2);
do
{
if (cancellationToken.IsCancellationRequested)
{
continueLooping = false;
}
while (continueLooping)
{
if (Utilities.IsPaused)
{
await Utilities.WaitUntil(() => !Utilities.IsPaused, 500, cancellationToken).ConfigureAwait(false);
}
if (cancellationToken.IsCancellationRequested)
{
continueLooping = false;
break;
}
var feedUrl = GetPageUri(Feeds[_settings.Feed].BaseUrl, pageIndex);
await ProcessPageBlock.SendAsync(feedUrl, cancellationToken).ConfigureAwait(false); // TODO: Need check with SongsPerPage
itemsInBlock++;
pageIndex++;
if ((pageIndex > maxPages && useMaxPages) || cancellationToken.IsCancellationRequested)
{
continueLooping = false;
}
// TODO: Better http error handling, what if only a single page is broken and returns 0 songs?
while (ProcessPageBlock.OutputCount > 0 || itemsInBlock == MaxConcurrency || !continueLooping)
{
if (cancellationToken.IsCancellationRequested)
{
continueLooping = false;
break;
}
if (itemsInBlock <= 0)
{
break;
}
await ProcessPageBlock.OutputAvailableAsync(cancellationToken).ConfigureAwait(false);
while (ProcessPageBlock.TryReceive(out PageReadResult pageResult))
{
if (pageResult != null)
{
pageResults.Add(pageResult);
}
if (Utilities.IsPaused)
{
await Utilities.WaitUntil(() => !Utilities.IsPaused, 500, cancellationToken).ConfigureAwait(false);
}
itemsInBlock--;
if (pageResult == null || pageResult.Count == 0) // TODO: This will trigger if a single page has an error.
{
Logger?.Debug("Received no new songs, last page reached.");
ProcessPageBlock.Complete();
itemsInBlock = 0;
continueLooping = false;
break;
}
if (pageResult.Count > 0)
{
Logger?.Debug($"Receiving {pageResult.Count} potential songs from {pageResult.Uri}");
}
else
{
Logger?.Debug($"Did not find any songs in {Name}.{settings.FeedName}.");
}
// TODO: Process PageReadResults for better error feedback.
foreach (var song in pageResult.Songs)
{
if (!retDict.ContainsKey(song.Hash))
{
if (retDict.Count < settings.MaxSongs || settings.MaxSongs == 0)
{
retDict.Add(song.Hash, song);
}
if (retDict.Count >= settings.MaxSongs && useMaxSongs)
{
continueLooping = false;
}
}
}
if (!useMaxPages || pageIndex <= maxPages)
{
if (retDict.Count < settings.MaxSongs)
{
continueLooping = true;
}
}
}
}
}
}while (continueLooping);
return(new FeedResult(retDict, pageResults));
}
public abstract Task <FeedResult> GetSongsFromFeedAsync(IFeedSettings settings, IProgress <ReaderProgress> progress, CancellationToken cancellationToken);
public virtual Task <FeedResult> GetSongsFromFeedAsync(IFeedSettings settings) => GetSongsFromFeedAsync(settings, default(IProgress <ReaderProgress>), CancellationToken.None);
// TODO: Abort early when bsaber.com is down (check if all items in block failed?)
// TODO: Make cancellationToken actually do something.
/// <summary>
/// Gets all songs from the feed defined by the provided settings.
/// </summary>
/// <param name="settings"></param>
/// <param name="cancellationToken"></param>
/// <exception cref="InvalidCastException">Thrown when the passed IFeedSettings isn't a BeastSaberFeedSettings.</exception>
/// <exception cref="ArgumentException">Thrown when trying to access a feed that requires a username and the username wasn't provided.</exception>
/// <returns></returns>
public async Task <Dictionary <string, ScrapedSong> > GetSongsFromFeedAsync(IFeedSettings settings, CancellationToken cancellationToken)
{
if (cancellationToken != CancellationToken.None)
{
Logger.Warning("CancellationToken in GetSongsFromFeedAsync isn't implemented.");
}
if (settings == null)
{
throw new ArgumentNullException(nameof(settings), "settings cannot be null for BeastSaberReader.GetSongsFromFeedAsync.");
}
Dictionary <string, ScrapedSong> retDict = new Dictionary <string, ScrapedSong>();
if (!(settings is BeastSaberFeedSettings _settings))
{
throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE);
}
if (_settings.FeedIndex != 2 && string.IsNullOrEmpty(_username?.Trim()))
{
Logger.Error($"Can't access feed without a valid username in the config file");
throw new ArgumentException("Cannot access this feed without a valid username.");
}
int pageIndex = settings.StartingPage;
int maxPages = _settings.MaxPages;
bool useMaxSongs = _settings.MaxSongs != 0;
bool useMaxPages = maxPages != 0;
if (useMaxPages && pageIndex > 1)
{
maxPages = maxPages + pageIndex - 1;
}
var ProcessPageBlock = new TransformBlock <Uri, List <ScrapedSong> >(async feedUrl =>
{
Stopwatch sw = new Stopwatch();
sw.Start();
//Logger.Debug($"Checking URL: {feedUrl}");
string pageText = "";
ContentType contentType;
string contentTypeStr = string.Empty;
try
{
using (var response = await WebUtils.WebClient.GetAsync(feedUrl).ConfigureAwait(false))
{
contentTypeStr = response.Content.ContentType.ToLower();
if (ContentDictionary.ContainsKey(contentTypeStr))
{
contentType = ContentDictionary[contentTypeStr];
}
else
{
contentType = ContentType.Unknown;
}
pageText = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
}
}
catch (HttpRequestException ex)
{
Logger.Exception($"Error downloading {feedUrl} in TransformBlock.", ex);
return(new List <ScrapedSong>());
}
var newSongs = GetSongsFromPageText(pageText, feedUrl, contentType);
sw.Stop();
//Logger.Debug($"Task for {feedUrl} completed in {sw.ElapsedMilliseconds}ms");
return(newSongs.Count > 0 ? newSongs : null);
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = MaxConcurrency,
BoundedCapacity = MaxConcurrency,
EnsureOrdered = true
});
bool continueLooping = true;
int itemsInBlock = 0;
do
{
while (continueLooping)
{
var feedUrl = GetPageUri(Feeds[_settings.Feed].BaseUrl, pageIndex);
await ProcessPageBlock.SendAsync(feedUrl).ConfigureAwait(false); // TODO: Need check with SongsPerPage
itemsInBlock++;
pageIndex++;
if (pageIndex > maxPages && useMaxPages)
{
continueLooping = false;
}
while (ProcessPageBlock.OutputCount > 0 || itemsInBlock == MaxConcurrency || !continueLooping)
{
if (itemsInBlock <= 0)
{
break;
}
await ProcessPageBlock.OutputAvailableAsync().ConfigureAwait(false);
while (ProcessPageBlock.TryReceive(out List <ScrapedSong> newSongs))
{
itemsInBlock--;
if (newSongs == null)
{
Logger.Debug("Received no new songs, last page reached.");
ProcessPageBlock.Complete();
itemsInBlock = 0;
continueLooping = false;
break;
}
Logger.Debug($"Receiving {newSongs.Count} potential songs from {newSongs.First().SourceUri}");
foreach (var song in newSongs)
{
if (retDict.ContainsKey(song.Hash))
{
Logger.Debug($"Song {song.Hash} already exists.");
}
else
{
if (retDict.Count < settings.MaxSongs || settings.MaxSongs == 0)
{
retDict.Add(song.Hash, song);
}
if (retDict.Count >= settings.MaxSongs && useMaxSongs)
{
continueLooping = false;
}
}
}
if (!useMaxPages || pageIndex <= maxPages)
{
if (retDict.Count < settings.MaxSongs)
{
continueLooping = true;
}
}
}
}
}
}while (continueLooping);
return(retDict);
}
public abstract string GetFeedName(IFeedSettings settings);
// TODO: Abort early when bsaber.com is down (check if all items in block failed?)
// TODO: Make cancellationToken actually do something.
/// <summary>
/// Gets all songs from the feed defined by the provided settings.
/// </summary>
/// <param name="settings"></param>
/// <param name="cancellationToken"></param>
/// <exception cref="ArgumentNullException">Thrown when <paramref name="_settings"/> is null.</exception>
/// <exception cref="InvalidCastException">Thrown when the passed IFeedSettings isn't a BeastSaberFeedSettings.</exception>
/// <exception cref="OperationCanceledException"></exception>
/// <returns></returns>
public async override Task <FeedResult> GetSongsFromFeedAsync(IFeedSettings settings, IProgress <ReaderProgress> progress, CancellationToken cancellationToken)
{
if (cancellationToken.IsCancellationRequested)
{
return(FeedResult.CancelledResult);
}
if (settings == null)
{
throw new ArgumentNullException(nameof(settings), "settings cannot be null for BeastSaberReader.GetSongsFromFeedAsync.");
}
Dictionary <string, ScrapedSong> retDict = new Dictionary <string, ScrapedSong>();
if (!(settings is BeastSaberFeedSettings _settings))
{
throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE);
}
if (_settings.Feed != BeastSaberFeedName.CuratorRecommended && string.IsNullOrEmpty(_settings.Username))
{
_settings.Username = Username;
}
BeastSaberFeed feed = new BeastSaberFeed(_settings)
{
StoreRawData = StoreRawData
};
try
{
feed.EnsureValidSettings();
}
catch (InvalidFeedSettingsException ex)
{
return(new FeedResult(null, null, ex, FeedResultError.Error));
}
int pageIndex = settings.StartingPage;
int maxPages = _settings.MaxPages;
int pagesChecked = 0;
bool useMaxSongs = _settings.MaxSongs != 0;
bool useMaxPages = maxPages != 0;
if (useMaxPages && pageIndex > 1)
{
maxPages = maxPages + pageIndex - 1;
}
var ProcessPageBlock = new TransformBlock <int, PageReadResult>(async pageNum =>
{
return(await feed.GetSongsFromPageAsync(pageNum, cancellationToken).ConfigureAwait(false));
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = MaxConcurrency,
BoundedCapacity = MaxConcurrency,
CancellationToken = cancellationToken
//#if NETSTANDARD
// , EnsureOrdered = true
//#endif
});
bool continueLooping = true;
int itemsInBlock = 0;
List <PageReadResult> pageResults = new List <PageReadResult>(maxPages + 2);
do
{
if (cancellationToken.IsCancellationRequested)
{
continueLooping = false;
}
while (continueLooping)
{
if (Utilities.IsPaused)
{
await Utilities.WaitUntil(() => !Utilities.IsPaused, 500, cancellationToken).ConfigureAwait(false);
}
if (cancellationToken.IsCancellationRequested)
{
continueLooping = false;
break;
}
await ProcessPageBlock.SendAsync(pageIndex, cancellationToken).ConfigureAwait(false); // TODO: Need check with SongsPerPage
itemsInBlock++;
pageIndex++;
if ((pageIndex > maxPages && useMaxPages) || cancellationToken.IsCancellationRequested)
{
continueLooping = false;
}
// TODO: Better http error handling, what if only a single page is broken and returns 0 songs?
while (ProcessPageBlock.OutputCount > 0 || itemsInBlock == MaxConcurrency || !continueLooping)
{
if (cancellationToken.IsCancellationRequested)
{
continueLooping = false;
break;
}
if (itemsInBlock <= 0)
{
break;
}
await ProcessPageBlock.OutputAvailableAsync(cancellationToken).ConfigureAwait(false);
while (ProcessPageBlock.TryReceive(out PageReadResult pageResult))
{
int songsAdded = 0;
if (pageResult != null)
{
pageResults.Add(pageResult);
}
if (Utilities.IsPaused)
{
await Utilities.WaitUntil(() => !Utilities.IsPaused, 500, cancellationToken).ConfigureAwait(false);
}
itemsInBlock--;
if (pageResult.IsLastPage || pageResult == null || pageResult.Count == 0) // TODO: This will trigger if a single page has an error.
{
Logger?.Debug("Received no new songs, last page reached.");
ProcessPageBlock.Complete();
itemsInBlock = 0;
continueLooping = false;
break;
}
if (pageResult.Count > 0)
{
Logger?.Debug($"Receiving {pageResult.Count} potential songs from {pageResult.Uri}");
}
else
{
Logger?.Debug($"Did not find any songs on page '{pageResult.Uri}' of {Name}.{settings.FeedName}.");
}
// TODO: Process PageReadResults for better error feedback.
foreach (var song in pageResult.Songs)
{
if (!retDict.ContainsKey(song.Hash))
{
if (retDict.Count < settings.MaxSongs || settings.MaxSongs == 0)
{
retDict.Add(song.Hash, song);
songsAdded++;
}
if (retDict.Count >= settings.MaxSongs && useMaxSongs)
{
continueLooping = false;
}
}
}
int prog = Interlocked.Increment(ref pagesChecked);
progress?.Report(new ReaderProgress(prog, songsAdded));
if (!useMaxPages || pageIndex <= maxPages)
{
if (retDict.Count < settings.MaxSongs)
{
continueLooping = true;
}
}
}
}
}
}while (continueLooping);
if (pageResults.Any(r => r.PageError == PageErrorType.Cancelled))
{
return(FeedResult.GetCancelledResult(retDict, pageResults));
}
return(new FeedResult(retDict, pageResults));
}