/// <summary> /// /// </summary> /// <param name="_settings"></param> /// <param name="cancellationToken"></param> /// <returns></returns> /// <exception cref="InvalidCastException">Thrown when the provided IFeedSettings isn't a ScoreSaberFeedSettings.</exception> /// <exception cref="ArgumentException">Thrown when the Search feed is selected and the query in settings isn't valid.</exception> /// <exception cref="ArgumentOutOfRangeException">Thrown when the feed specified in the settings isn't valid.</exception> /// <exception cref="ArgumentNullException">Thrown when settings is null.</exception> public override Task <FeedResult> GetSongsFromFeedAsync(IFeedSettings _settings, IProgress <ReaderProgress> progress, CancellationToken cancellationToken) { PrepareReader(); if (_settings == null) { throw new ArgumentNullException(nameof(_settings), "settings cannot be null for ScoreSaberReader.GetSongsFromFeedAsync"); } if (!(_settings is ScoreSaberFeedSettings settings)) { throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE); } if (!((settings.FeedIndex >= 0 && settings.FeedIndex <= 3) || settings.FeedIndex == 99)) // Validate FeedIndex { throw new ArgumentOutOfRangeException(nameof(_settings), "_settings contains an invalid FeedIndex value for ScoreSaberReader"); } Dictionary <string, ScrapedSong> retDict = new Dictionary <string, ScrapedSong>(); if (settings.Feed == ScoreSaberFeedName.TopRanked || settings.Feed == ScoreSaberFeedName.LatestRanked) { settings.RankedOnly = true; } if (settings.Feed == ScoreSaberFeedName.Search) { if (!IsValidSearchQuery(settings.SearchQuery)) { throw new ArgumentException($"Search query '{settings.SearchQuery ?? "<nul>"}' is not a valid query."); } } return(GetSongsFromScoreSaberAsync(settings, progress, cancellationToken)); }
public AtomFeedGenerator(IFeedDataClient feedDataClient, IFeedService feedService, IPublishService publishService, IFeedSettings feedSettings) : base(feedDataClient, feedService, publishService, feedSettings) { }
public RssFeedGenerator(IFeedDataClient feedDataClient, IFeedService feedService, IPublishService publishService, IFeedSettings feedSettings) : base(feedDataClient, feedService, publishService, feedSettings) { }
public override string GetFeedName(IFeedSettings settings) { if (!(settings is ScoreSaberFeedSettings ssSettings)) { throw new ArgumentException("Settings is not ScoreSaberFeedSettings", nameof(settings)); } return(ScoreSaberFeed.Feeds[ssSettings.Feed].DisplayName); }
public string GetFeedName(IFeedSettings settings) { if (!(settings is BeastSaberFeedSettings ssSettings)) { throw new ArgumentException("Settings is not BeastSaberFeedSettings", nameof(settings)); } return(Feeds[ssSettings.Feed].DisplayName); }
private static ArticleFeedGenerator CreateFeedGenerator(IFeedSettings feedSettings) { string feedFormat = feedSettings.FeedFormat; switch (feedFormat.ToLower()) { case "atom": return new AtomFeedGenerator(new RareburgClient(), new RareburgArticleFeedService(), new S3PublishService(), feedSettings); case "rss": return new RssFeedGenerator(new RareburgClient(), new RareburgArticleFeedService(), new S3PublishService(), feedSettings); default: throw new ArgumentException("Unknown feed format"); } }
public ArticleFeedGenerator(IFeedDataClient feedDataClient, IFeedService feedService, IPublishService publishService, IFeedSettings feedSettings) { _feedDataClient = feedDataClient; _feedService = feedService; _publishService = publishService; _feedSettings = feedSettings; }
public ArticleFeedGenerator(IFeedDataClient feedDataClient, IFeedService feedService, IPublishService publishService, IFeedSettings feedSettings) { _feedDataClient = feedDataClient; _feedService = feedService; _publishService = publishService; _feedSettings = feedSettings; }
/// <summary> /// /// </summary> /// <param name="_settings"></param> /// <exception cref="InvalidCastException">Thrown when the passed IFeedSettings isn't a BeatSaverFeedSettings</exception> /// <returns></returns> public Dictionary <string, SongInfo> GetSongsFromFeed(IFeedSettings _settings) { PrepareReader(); if (!(_settings is BeatSaverFeedSettings settings)) { throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE); } List <SongInfo> songs = new List <SongInfo>(); switch (settings.FeedIndex) { // Author case 0: foreach (var author in settings.Authors) { songs.AddRange(GetSongsByAuthor(author)); } break; // Newest case 1: songs.AddRange(GetNewestSongs(settings)); break; // Top case 2: break; default: break; } Dictionary <string, SongInfo> retDict = new Dictionary <string, SongInfo>(); foreach (var song in songs) { if (retDict.ContainsKey(song.id)) { if (retDict[song.id].SongVersion < song.SongVersion) { Logger.Debug($"Song with ID {song.id} already exists, updating"); retDict[song.id] = song; } else { Logger.Debug($"Song with ID {song.id} is already the newest version"); } } else { retDict.Add(song.id, song); } } return(retDict); }
static void Main(string[] args) { IConfigurationFactory configFactory = new AppConfigConfigurationFactory(); // IConfigurationFactory configFactory = new DynamoDBConfigurationFactory(); IApiSettings apiSettings = configFactory.GetApiSettings(); IFeedSettings feedSettings = configFactory.GetFeedSettings(); IFeedServiceSettings feedServiceSettings = configFactory.GetFeedServiceSettings(); IS3PublisherSettings s3PublishSettings = configFactory.GetS3PublisherSettings(); IOfflineClientSettings offlineClientSettings = configFactory.GetOfflineClientSettings(); }
/// <summary> /// Gets the DisplayName of the feed specified in settings. /// </summary> /// <param name="settings"></param> /// <returns></returns> /// <exception cref="ArgumentNullException">Thrown when settings is null</exception> /// <exception cref="InvalidCastException">Thrown when settings is not a BeatSaverFeedSettings.</exception> public override string GetFeedName(IFeedSettings settings) { if (settings == null) { throw new ArgumentNullException(nameof(settings), "settings cannot be null for BeatSaverReader.GetFeedName"); } if (!(settings is BeatSaverFeedSettings ssSettings)) { throw new InvalidCastException("Settings is not BeatSaverFeedSettings in BeatSaverReader.GetFeedName"); } return(BeatSaverFeed.Feeds[ssSettings.Feed].DisplayName); }
private static ArticleFeedGenerator CreateFeedGenerator(IFeedSettings feedSettings) { string feedFormat = feedSettings.FeedFormat; switch (feedFormat.ToLower()) { case "atom": return(new AtomFeedGenerator(new RareburgClient(), new RareburgArticleFeedService(), new S3PublishService(), feedSettings)); case "rss": return(new RssFeedGenerator(new RareburgClient(), new RareburgArticleFeedService(), new S3PublishService(), feedSettings)); default: throw new ArgumentException("Unknown feed format"); } }
public async Task <Dictionary <string, ScrapedSong> > GetSongsFromFeedAsync(IFeedSettings _settings, CancellationToken cancellationToken) { PrepareReader(); if (!(_settings is BeatSaverFeedSettings settings)) { throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE); } List <ScrapedSong> songs = null; switch ((BeatSaverFeed)settings.FeedIndex) { // Author case BeatSaverFeed.Author: string songSource = string.Empty; List <ScrapedSong> newSongs = null; songs = new List <ScrapedSong>(); foreach (var author in settings.Authors) { newSongs = await GetSongsByAuthorAsync(author, CalcMaxPages(settings.MaxPages, settings.MaxSongs)).ConfigureAwait(false); songSource = "Beat Saver"; songs.AddRange(newSongs.Take(settings.MaxSongs)); Logger.Info($"Found {newSongs.Count} songs uploaded by {author} from {songSource}"); } break; case BeatSaverFeed.Search: songs = await SearchAsync(settings.SearchCriteria, settings.SearchType).ConfigureAwait(false); break; // Latest/Hot/Plays/Downloads default: songs = await GetBeatSaverSongsAsync(settings).ConfigureAwait(false); break; } Dictionary <string, ScrapedSong> retDict = new Dictionary <string, ScrapedSong>(); foreach (var song in songs) { if (!retDict.ContainsKey(song.Hash)) { retDict.Add(song.Hash, song); } } return(retDict); }
public FeedResult GetSongsFromFeed(IFeedSettings settings) { try { return(GetSongsFromFeedAsync(settings, CancellationToken.None).Result); } catch (AggregateException ex) { var flattened = ex.Flatten(); if (flattened.InnerExceptions.Count == 1) { throw flattened.InnerException; } throw ex; } }
public virtual FeedResult GetSongsFromFeed(IFeedSettings settings, CancellationToken cancellationToken) { try { return(GetSongsFromFeedAsync(settings, default(IProgress <ReaderProgress>), cancellationToken).Result); } catch (AggregateException ex) { var flattened = ex.Flatten(); if (flattened.InnerExceptions.Count == 1) { throw flattened.InnerException; } throw ex; } }
public Dictionary <string, SongInfo> GetSongsFromFeed(IFeedSettings _settings) { PrepareReader(); if (!(_settings is ScoreSaberFeedSettings settings)) { throw new InvalidCastException(INVALID_FEED_SETTINGS_MESSAGE); } List <SongInfo> songs = new List <SongInfo>(); switch (settings.FeedIndex) { // Author case 0: songs.AddRange(GetTopPPSongs(settings)); break; default: break; } Dictionary <string, SongInfo> retDict = new Dictionary <string, SongInfo>(); foreach (var song in songs) { if (retDict.ContainsKey(song.id)) { if (retDict[song.id].SongVersion < song.SongVersion) { Logger.Debug($"Song with ID {song.id} already exists, updating"); retDict[song.id] = song; } else if (retDict[song.id].SongVersion == song.SongVersion) { Logger.Debug($"Tried to add a song we already got"); } else { Logger.Debug($"Song with ID {song.id} is already the newest version"); } } else { retDict.Add(song.id, song); } } return(retDict); }
public Dictionary <string, ScrapedSong> GetSongsFromFeed(IFeedSettings settings) { // Pointless to have these checks here? PrepareReader(); if (!(settings is BeastSaberFeedSettings _settings)) { throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE); } if (_settings.FeedIndex != 2 && string.IsNullOrEmpty(_username?.Trim())) { Logger.Error($"Can't access feed without a valid username in the config file"); throw new ArgumentException("Cannot access this feed without a valid username."); } var retDict = GetSongsFromFeedAsync(settings).Result; return(retDict); }
public void DownloadSongsFromFeed(string feedType, IFeedSettings _settings) { if (!FeedReaders.ContainsKey(feedType)) { Logger.Error($"Invalid feed type passed to DownloadSongsFromFeed: {feedType}"); return; } var reader = FeedReaders[feedType]; DateTime startTime = DateTime.Now; Dictionary <int, SongInfo> songs; List <Playlist> playlists = new List <Playlist> { _syncSaberSongs }; playlists.AddRange(reader.PlaylistsForFeed(_settings.FeedIndex)); try { songs = reader.GetSongsFromFeed(_settings); } catch (InvalidCastException) { Logger.Error($"Wrong type of IFeedSettings given to {feedType}"); return; } int totalSongs = songs.Count; Logger.Debug($"Finished checking pages, found {totalSongs} songs"); List <SongInfo> matchedSongs = DownloadSongs(songs, out (List <SongInfo> exists, List <SongInfo> history)skippedSongs, _settings.UseSongKeyAsOutputFolder); _songDownloadHistory.AddRange(skippedSongs.exists.Select(s => s.key)); Logger.Debug("Jobs finished, Processing downloads..."); int downloadCount = SuccessfulDownloads.Count; int failedCount = FailedDownloads.Count; ProcessDownloads(playlists); foreach (var p in playlists) { matchedSongs.ForEach(s => p.TryAdd(s.hash, s.key, s.songName)); p.WritePlaylist(); } var timeElapsed = (DateTime.Now - startTime); Logger.Info($"Downloaded {downloadCount} songs from {reader.Source}'s {_settings.FeedName} feed in {FormatTimeSpan(timeElapsed)}. " + $"Skipped {skippedSongs.exists.Count} songs that already exist and {skippedSongs.history.Count} that are only in history{(failedCount > 0 ? $", failed to download {failedCount} songs." : "")}."); }
static void Main(string[] args) { IConfigurationFactory configFactory = new AppConfigConfigurationFactory(); // IConfigurationFactory configFactory = new DynamoDBConfigurationFactory(); IApiSettings apiSettings = configFactory.GetApiSettings(); IFeedSettings feedSettings = configFactory.GetFeedSettings(); IFeedServiceSettings feedServiceSettings = configFactory.GetFeedServiceSettings(); IS3PublisherSettings s3PublishSettings = configFactory.GetS3PublisherSettings(); IOfflineClientSettings offlineClientSettings = configFactory.GetOfflineClientSettings(); var feedDataClient = new OfflineRareburgClient(offlineClientSettings); var feedService = new RareburgArticleFeedService(feedServiceSettings); var publishService = new S3PublishService(s3PublishSettings, feedSettings); var feedGenerator = CreateFeedGenerator(feedDataClient, feedService, publishService, feedSettings); feedGenerator.Run(); }
/// <summary> /// /// </summary> /// <param name="_settings"></param> /// <param name="cancellationToken"></param> /// <returns></returns> /// <exception cref="InvalidCastException">Throw when the provided settings object isn't a BeatSaverFeedSettings</exception> /// <exception cref="ArgumentNullException">Thrown when <paramref name="_settings"/> is null.</exception> public async Task <FeedResult> GetSongsFromFeedAsync(IFeedSettings _settings, CancellationToken cancellationToken) { PrepareReader(); if (_settings == null) { throw new ArgumentNullException(nameof(_settings), "Settings cannot be null for BeatSaverReader.GetSongsFromFeedAsync"); } if (!(_settings is BeatSaverFeedSettings settings)) { throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE); } FeedResult songs = null; switch ((BeatSaverFeed)settings.FeedIndex) { // Author case BeatSaverFeed.Author: songs = await GetSongsByAuthorAsync(settings.Criteria, cancellationToken, CalcMaxSongs(settings.MaxPages, settings.MaxSongs)).ConfigureAwait(false); break; case BeatSaverFeed.Search: songs = await SearchAsync(settings, cancellationToken).ConfigureAwait(false); break; // Latest/Hot/Plays/Downloads default: songs = await GetBeatSaverSongsAsync(settings, cancellationToken).ConfigureAwait(false); break; } //Dictionary<string, ScrapedSong> retDict = new Dictionary<string, ScrapedSong>(); //foreach (var song in songs) //{ // if (!retDict.ContainsKey(song.Hash)) // { // retDict.Add(song.Hash, song); // } //} return(songs); }
public async Task <Dictionary <string, ScrapedSong> > GetSongsFromFeedAsync(IFeedSettings _settings, CancellationToken cancellationToken) { PrepareReader(); if (!(_settings is ScoreSaberFeedSettings settings)) { throw new InvalidCastException(INVALID_FEED_SETTINGS_MESSAGE); } Dictionary <string, ScrapedSong> retDict = new Dictionary <string, ScrapedSong>(); int maxSongs = settings.MaxSongs > 0 ? settings.MaxSongs : settings.SongsPerPage * settings.SongsPerPage; switch (settings.Feed) { case ScoreSaberFeed.Trending: retDict = await GetSongsFromScoreSaberAsync(settings).ConfigureAwait(false); break; case ScoreSaberFeed.LatestRanked: settings.RankedOnly = true; retDict = await GetSongsFromScoreSaberAsync(settings).ConfigureAwait(false); break; case ScoreSaberFeed.TopPlayed: retDict = await GetSongsFromScoreSaberAsync(settings).ConfigureAwait(false); break; case ScoreSaberFeed.TopRanked: settings.RankedOnly = true; retDict = await GetSongsFromScoreSaberAsync(settings).ConfigureAwait(false); break; default: break; } return(retDict); }
public EditionXmlFormatter(IFeedSettings feedInfo) { m_feedInfo = feedInfo; }
public async Task <Dictionary <string, ScrapedSong> > GetSongsFromFeedAsync(IFeedSettings settings) { return(await GetSongsFromFeedAsync(settings, CancellationToken.None).ConfigureAwait(false)); }
public Dictionary <string, ScrapedSong> GetSongsFromFeed(IFeedSettings _settings) { return(GetSongsFromFeedAsync(_settings).Result); }
public async Task <Dictionary <string, ScrapedSong> > ReadFeed(IFeedReader reader, IFeedSettings settings, Playlist feedPlaylist, PlaylistStyle playlistStyle) { //Logger.log?.Info($"Getting songs from {feedName} feed."); var songs = await reader.GetSongsFromFeedAsync(settings).ConfigureAwait(false) ?? new Dictionary <string, ScrapedSong>(); if (songs.Count > 0 && playlistStyle == PlaylistStyle.Replace) { feedPlaylist.Clear(); } var addDate = DateTime.Now; var decrement = new TimeSpan(1); foreach (var scrapedSong in songs) { if (HistoryManager.TryGetValue(scrapedSong.Value.Hash, out var historyEntry) && (historyEntry.Flag == HistoryFlag.NotFound || historyEntry.Flag == HistoryFlag.Deleted)) { continue; } //if (string.IsNullOrEmpty(scrapedSong.Value.SongKey)) //{ // try // { // //Logger.log?.Info($"Grabbing key from BeatSaver: {scrapedSong.Value.SongName} by {scrapedSong.Value.MapperName}"); // // ScrapedSong doesn't have a Beat Saver key associated with it, probably scraped from ScoreSaber // scrapedSong.Value.UpdateFrom(await BeatSaverReader.GetSongByHashAsync(scrapedSong.Key), false); // } // catch (ArgumentNullException) // { // Logger.log?.Warn($"Unable to find {scrapedSong.Value?.SongName} by {scrapedSong.Value?.MapperName} on Beat Saver ({scrapedSong.Key})"); // } //} var song = scrapedSong.Value.ToPlaylistSong(); song.DateAdded = addDate; feedPlaylist?.TryAdd(song); addDate = addDate - decrement; } feedPlaylist?.TryWriteFile(); return(songs); }
public S3PublishService(IS3PublisherSettings s3PublisherSettings, IFeedSettings feedSettings) { _s3PublisherSettings = s3PublisherSettings; _feedSettings = feedSettings; }
/// <summary> /// /// </summary> /// <param name="_settings"></param> /// <exception cref="InvalidCastException">Thrown when the passed IFeedSettings isn't a BeatSaverFeedSettings</exception> /// <returns></returns> public Dictionary <int, SongInfo> GetSongsFromFeed(IFeedSettings _settings) { PrepareReader(); if (!(_settings is BeatSaverFeedSettings settings)) { throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE); } List <SongInfo> songs = new List <SongInfo>(); switch ((BeatSaverFeeds)settings.FeedIndex) { // Author case BeatSaverFeeds.AUTHOR: List <SongInfo> newSongs = null; string songSource = string.Empty; foreach (var author in settings.Authors) { if (!settings.searchOnline) { newSongs = ScrapedDataProvider.Songs.Values.Where(s => !string.IsNullOrEmpty(s.BeatSaverInfo?.uploader?.username) && s.BeatSaverInfo?.uploader.username.ToLower() == author.ToLower()).ToList(); songSource = "scraped data"; } if (newSongs == null || newSongs.Count == 0) { newSongs = GetSongsByAuthor(author); songSource = "Beat Saver"; } songs.AddRange(newSongs); Logger.Info($"Found {newSongs.Count} songs uploaded by {author} from {songSource}"); } break; // Newest case BeatSaverFeeds.LATEST: songs.AddRange(GetNewestSongs(settings)); break; // Top case BeatSaverFeeds.HOT: break; default: break; } Dictionary <int, SongInfo> retDict = new Dictionary <int, SongInfo>(); foreach (var song in songs) { if (retDict.ContainsKey(song.keyAsInt)) {/* * if (retDict[song.keyAsInt].SongVersion < song.SongVersion) * { * Logger.Debug($"Song with ID {song.keyAsInt} already exists, updating"); * retDict[song.keyAsInt] = song; * } * else * { * Logger.Debug($"Song with ID {song.keyAsInt} is already the newest version"); * }*/ } else { retDict.Add(song.keyAsInt, song); } } return(retDict); }
public Dictionary <int, SongInfo> GetSongsFromFeed(IFeedSettings _settings) { PrepareReader(); if (!(_settings is ScoreSaberFeedSettings settings)) { throw new InvalidCastException(INVALID_FEED_SETTINGS_MESSAGE); } List <SongInfo> songs = new List <SongInfo>(); int maxSongs = settings.MaxSongs > 0 ? settings.MaxSongs : settings.SongsPerPage * settings.SongsPerPage; switch ((ScoreSaberFeeds)settings.FeedIndex) { case ScoreSaberFeeds.TRENDING: if (!settings.searchOnline) { if (maxSongs <= 0) { throw new ArgumentException("You must specify a value greater than 0 for MaxPages or MaxSongs."); } songs.AddRange(ScrapedDataProvider.Songs.Values.Where(s => s.ScoreSaberInfo.Count > 0).OrderByDescending(s => s.ScoreSaberInfo.Values.Select(ss => ss.scores_day).Aggregate((a, b) => a + b)).Take(maxSongs)); } break; case ScoreSaberFeeds.LATEST_RANKED: if (!settings.searchOnline) { if (maxSongs <= 0) { throw new ArgumentException("You must specify a value greater than 0 for MaxPages or MaxSongs."); } songs.AddRange(ScrapedDataProvider.Songs.Values.Where(s => s.RankedDifficulties.Count > 0).OrderByDescending(ss => ss.ScoreSaberInfo.Keys.Max()).Take(maxSongs)); } break; case ScoreSaberFeeds.TOP_PLAYED: if (!settings.searchOnline) { if (maxSongs <= 0) { throw new ArgumentException("You must specify a value greater than 0 for MaxPages or MaxSongs."); } songs.AddRange(ScrapedDataProvider.Songs.Values.Where(s => s.ScoreSaberInfo.Count > 0).OrderByDescending(s => s.ScoreSaberInfo.Values.Select(ss => ss.scores).Aggregate((a, b) => a + b)).Take(maxSongs)); } break; case ScoreSaberFeeds.TOP_RANKED: if (!settings.searchOnline) { if (maxSongs <= 0) { throw new ArgumentException("You must specify a value greater than 0 for MaxPages or MaxSongs."); } songs.AddRange(ScrapedDataProvider.Songs.Values.Where(s => s.RankedDifficulties.Count > 0).OrderByDescending(s => s.RankedDifficulties.Max(d => d.Value)).Take(maxSongs)); } else { songs.AddRange(GetTopPPSongs(settings)); } break; default: break; } Dictionary <int, SongInfo> retDict = new Dictionary <int, SongInfo>(); foreach (var song in songs) { if (!retDict.ContainsKey(song.keyAsInt)) { retDict.Add(song.keyAsInt, song); } } return(retDict); }
public Task <FeedResult> GetSongsFromFeedAsync(IFeedSettings settings) { return(GetSongsFromFeedAsync(settings, CancellationToken.None)); }
// TODO: Abort early when bsaber.com is down (check if all items in block failed?) // TODO: Make cancellationToken actually do something. /// <summary> /// Gets all songs from the feed defined by the provided settings. /// </summary> /// <param name="settings"></param> /// <param name="cancellationToken"></param> /// <exception cref="InvalidCastException">Thrown when the passed IFeedSettings isn't a BeastSaberFeedSettings.</exception> /// <exception cref="ArgumentException">Thrown when trying to access a feed that requires a username and the username wasn't provided.</exception> /// <exception cref="OperationCanceledException"></exception> /// <returns></returns> public async Task <FeedResult> GetSongsFromFeedAsync(IFeedSettings settings, CancellationToken cancellationToken) { if (cancellationToken.IsCancellationRequested) { return(FeedResult.CancelledResult); } if (settings == null) { throw new ArgumentNullException(nameof(settings), "settings cannot be null for BeastSaberReader.GetSongsFromFeedAsync."); } Dictionary <string, ScrapedSong> retDict = new Dictionary <string, ScrapedSong>(); if (!(settings is BeastSaberFeedSettings _settings)) { throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE); } if (_settings.FeedIndex != 2 && string.IsNullOrEmpty(_username?.Trim())) { //Logger?.Error($"Can't access feed without a valid username in the config file"); throw new ArgumentException("Cannot access this feed without a valid username."); } int pageIndex = settings.StartingPage; int maxPages = _settings.MaxPages; bool useMaxSongs = _settings.MaxSongs != 0; bool useMaxPages = maxPages != 0; if (useMaxPages && pageIndex > 1) { maxPages = maxPages + pageIndex - 1; } var ProcessPageBlock = new TransformBlock <Uri, PageReadResult>(async feedUri => { Stopwatch sw = new Stopwatch(); sw.Start(); //Logger?.Debug($"Checking URL: {feedUrl}"); string pageText = ""; ContentType contentType = ContentType.Unknown; string contentTypeStr = string.Empty; IWebResponseMessage response = null; try { response = await WebUtils.WebClient.GetAsync(feedUri, cancellationToken).ConfigureAwait(false); if ((response?.StatusCode ?? 500) == 500) { response?.Dispose(); response = null; Logger?.Warning($"Internal server error on {feedUri}, retrying in 20 seconds"); await Task.Delay(20000).ConfigureAwait(false); response = await WebUtils.WebClient.GetAsync(feedUri, cancellationToken).ConfigureAwait(false); } response.EnsureSuccessStatusCode(); contentTypeStr = response.Content.ContentType.ToLower(); if (ContentDictionary.ContainsKey(contentTypeStr)) { contentType = ContentDictionary[contentTypeStr]; } else { contentType = ContentType.Unknown; } pageText = await response.Content.ReadAsStringAsync().ConfigureAwait(false); } catch (WebClientException ex) { return(PageReadResult.FromWebClientException(ex, feedUri)); } catch (OperationCanceledException) { return(new PageReadResult(feedUri, null, new FeedReaderException("Page read was cancelled.", new OperationCanceledException(), FeedReaderFailureCode.Cancelled), PageErrorType.Cancelled)); } catch (Exception ex) { string message = $"Error downloading {feedUri} in TransformBlock."; Logger?.Debug(message); Logger?.Debug($"{ex.Message}\n{ex.StackTrace}"); return(new PageReadResult(feedUri, null, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.Unknown)); } finally { response?.Dispose(); response = null; } List <ScrapedSong> newSongs = null; try { newSongs = GetSongsFromPageText(pageText, feedUri, contentType); } catch (JsonReaderException ex) { // TODO: Probably don't need a logger message here, caller can deal with it. string message = $"Error parsing page text for {feedUri} in TransformBlock."; Logger?.Debug(message); Logger?.Debug($"{ex.Message}\n{ex.StackTrace}"); return(new PageReadResult(feedUri, null, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.ParsingError)); } catch (XmlException ex) { // TODO: Probably don't need a logger message here, caller can deal with it. string message = $"Error parsing page text for {feedUri} in TransformBlock."; Logger?.Debug(message); Logger?.Debug($"{ex.Message}\n{ex.StackTrace}"); return(new PageReadResult(feedUri, null, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.ParsingError)); } catch (Exception ex) { // TODO: Probably don't need a logger message here, caller can deal with it. string message = $"Uncaught error parsing page text for {feedUri} in TransformBlock."; Logger?.Debug(message); Logger?.Debug($"{ex.Message}\n{ex.StackTrace}"); return(new PageReadResult(feedUri, null, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.Unknown)); } sw.Stop(); //Logger?.Debug($"Task for {feedUrl} completed in {sw.ElapsedMilliseconds}ms"); return(new PageReadResult(feedUri, newSongs)); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = MaxConcurrency, BoundedCapacity = MaxConcurrency, CancellationToken = cancellationToken //#if NETSTANDARD // , EnsureOrdered = true //#endif }); bool continueLooping = true; int itemsInBlock = 0; List <PageReadResult> pageResults = new List <PageReadResult>(maxPages + 2); do { if (cancellationToken.IsCancellationRequested) { continueLooping = false; } while (continueLooping) { if (Utilities.IsPaused) { await Utilities.WaitUntil(() => !Utilities.IsPaused, 500, cancellationToken).ConfigureAwait(false); } if (cancellationToken.IsCancellationRequested) { continueLooping = false; break; } var feedUrl = GetPageUri(Feeds[_settings.Feed].BaseUrl, pageIndex); await ProcessPageBlock.SendAsync(feedUrl, cancellationToken).ConfigureAwait(false); // TODO: Need check with SongsPerPage itemsInBlock++; pageIndex++; if ((pageIndex > maxPages && useMaxPages) || cancellationToken.IsCancellationRequested) { continueLooping = false; } // TODO: Better http error handling, what if only a single page is broken and returns 0 songs? while (ProcessPageBlock.OutputCount > 0 || itemsInBlock == MaxConcurrency || !continueLooping) { if (cancellationToken.IsCancellationRequested) { continueLooping = false; break; } if (itemsInBlock <= 0) { break; } await ProcessPageBlock.OutputAvailableAsync(cancellationToken).ConfigureAwait(false); while (ProcessPageBlock.TryReceive(out PageReadResult pageResult)) { if (pageResult != null) { pageResults.Add(pageResult); } if (Utilities.IsPaused) { await Utilities.WaitUntil(() => !Utilities.IsPaused, 500, cancellationToken).ConfigureAwait(false); } itemsInBlock--; if (pageResult == null || pageResult.Count == 0) // TODO: This will trigger if a single page has an error. { Logger?.Debug("Received no new songs, last page reached."); ProcessPageBlock.Complete(); itemsInBlock = 0; continueLooping = false; break; } if (pageResult.Count > 0) { Logger?.Debug($"Receiving {pageResult.Count} potential songs from {pageResult.Uri}"); } else { Logger?.Debug($"Did not find any songs in {Name}.{settings.FeedName}."); } // TODO: Process PageReadResults for better error feedback. foreach (var song in pageResult.Songs) { if (!retDict.ContainsKey(song.Hash)) { if (retDict.Count < settings.MaxSongs || settings.MaxSongs == 0) { retDict.Add(song.Hash, song); } if (retDict.Count >= settings.MaxSongs && useMaxSongs) { continueLooping = false; } } } if (!useMaxPages || pageIndex <= maxPages) { if (retDict.Count < settings.MaxSongs) { continueLooping = true; } } } } } }while (continueLooping); return(new FeedResult(retDict, pageResults)); }
public abstract Task <FeedResult> GetSongsFromFeedAsync(IFeedSettings settings, IProgress <ReaderProgress> progress, CancellationToken cancellationToken);
public virtual Task <FeedResult> GetSongsFromFeedAsync(IFeedSettings settings) => GetSongsFromFeedAsync(settings, default(IProgress <ReaderProgress>), CancellationToken.None);
// TODO: Abort early when bsaber.com is down (check if all items in block failed?) // TODO: Make cancellationToken actually do something. /// <summary> /// Gets all songs from the feed defined by the provided settings. /// </summary> /// <param name="settings"></param> /// <param name="cancellationToken"></param> /// <exception cref="InvalidCastException">Thrown when the passed IFeedSettings isn't a BeastSaberFeedSettings.</exception> /// <exception cref="ArgumentException">Thrown when trying to access a feed that requires a username and the username wasn't provided.</exception> /// <returns></returns> public async Task <Dictionary <string, ScrapedSong> > GetSongsFromFeedAsync(IFeedSettings settings, CancellationToken cancellationToken) { if (cancellationToken != CancellationToken.None) { Logger.Warning("CancellationToken in GetSongsFromFeedAsync isn't implemented."); } if (settings == null) { throw new ArgumentNullException(nameof(settings), "settings cannot be null for BeastSaberReader.GetSongsFromFeedAsync."); } Dictionary <string, ScrapedSong> retDict = new Dictionary <string, ScrapedSong>(); if (!(settings is BeastSaberFeedSettings _settings)) { throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE); } if (_settings.FeedIndex != 2 && string.IsNullOrEmpty(_username?.Trim())) { Logger.Error($"Can't access feed without a valid username in the config file"); throw new ArgumentException("Cannot access this feed without a valid username."); } int pageIndex = settings.StartingPage; int maxPages = _settings.MaxPages; bool useMaxSongs = _settings.MaxSongs != 0; bool useMaxPages = maxPages != 0; if (useMaxPages && pageIndex > 1) { maxPages = maxPages + pageIndex - 1; } var ProcessPageBlock = new TransformBlock <Uri, List <ScrapedSong> >(async feedUrl => { Stopwatch sw = new Stopwatch(); sw.Start(); //Logger.Debug($"Checking URL: {feedUrl}"); string pageText = ""; ContentType contentType; string contentTypeStr = string.Empty; try { using (var response = await WebUtils.WebClient.GetAsync(feedUrl).ConfigureAwait(false)) { contentTypeStr = response.Content.ContentType.ToLower(); if (ContentDictionary.ContainsKey(contentTypeStr)) { contentType = ContentDictionary[contentTypeStr]; } else { contentType = ContentType.Unknown; } pageText = await response.Content.ReadAsStringAsync().ConfigureAwait(false); } } catch (HttpRequestException ex) { Logger.Exception($"Error downloading {feedUrl} in TransformBlock.", ex); return(new List <ScrapedSong>()); } var newSongs = GetSongsFromPageText(pageText, feedUrl, contentType); sw.Stop(); //Logger.Debug($"Task for {feedUrl} completed in {sw.ElapsedMilliseconds}ms"); return(newSongs.Count > 0 ? newSongs : null); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = MaxConcurrency, BoundedCapacity = MaxConcurrency, EnsureOrdered = true }); bool continueLooping = true; int itemsInBlock = 0; do { while (continueLooping) { var feedUrl = GetPageUri(Feeds[_settings.Feed].BaseUrl, pageIndex); await ProcessPageBlock.SendAsync(feedUrl).ConfigureAwait(false); // TODO: Need check with SongsPerPage itemsInBlock++; pageIndex++; if (pageIndex > maxPages && useMaxPages) { continueLooping = false; } while (ProcessPageBlock.OutputCount > 0 || itemsInBlock == MaxConcurrency || !continueLooping) { if (itemsInBlock <= 0) { break; } await ProcessPageBlock.OutputAvailableAsync().ConfigureAwait(false); while (ProcessPageBlock.TryReceive(out List <ScrapedSong> newSongs)) { itemsInBlock--; if (newSongs == null) { Logger.Debug("Received no new songs, last page reached."); ProcessPageBlock.Complete(); itemsInBlock = 0; continueLooping = false; break; } Logger.Debug($"Receiving {newSongs.Count} potential songs from {newSongs.First().SourceUri}"); foreach (var song in newSongs) { if (retDict.ContainsKey(song.Hash)) { Logger.Debug($"Song {song.Hash} already exists."); } else { if (retDict.Count < settings.MaxSongs || settings.MaxSongs == 0) { retDict.Add(song.Hash, song); } if (retDict.Count >= settings.MaxSongs && useMaxSongs) { continueLooping = false; } } } if (!useMaxPages || pageIndex <= maxPages) { if (retDict.Count < settings.MaxSongs) { continueLooping = true; } } } } } }while (continueLooping); return(retDict); }
public abstract string GetFeedName(IFeedSettings settings);
// TODO: Abort early when bsaber.com is down (check if all items in block failed?) // TODO: Make cancellationToken actually do something. /// <summary> /// Gets all songs from the feed defined by the provided settings. /// </summary> /// <param name="settings"></param> /// <param name="cancellationToken"></param> /// <exception cref="ArgumentNullException">Thrown when <paramref name="_settings"/> is null.</exception> /// <exception cref="InvalidCastException">Thrown when the passed IFeedSettings isn't a BeastSaberFeedSettings.</exception> /// <exception cref="OperationCanceledException"></exception> /// <returns></returns> public async override Task <FeedResult> GetSongsFromFeedAsync(IFeedSettings settings, IProgress <ReaderProgress> progress, CancellationToken cancellationToken) { if (cancellationToken.IsCancellationRequested) { return(FeedResult.CancelledResult); } if (settings == null) { throw new ArgumentNullException(nameof(settings), "settings cannot be null for BeastSaberReader.GetSongsFromFeedAsync."); } Dictionary <string, ScrapedSong> retDict = new Dictionary <string, ScrapedSong>(); if (!(settings is BeastSaberFeedSettings _settings)) { throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE); } if (_settings.Feed != BeastSaberFeedName.CuratorRecommended && string.IsNullOrEmpty(_settings.Username)) { _settings.Username = Username; } BeastSaberFeed feed = new BeastSaberFeed(_settings) { StoreRawData = StoreRawData }; try { feed.EnsureValidSettings(); } catch (InvalidFeedSettingsException ex) { return(new FeedResult(null, null, ex, FeedResultError.Error)); } int pageIndex = settings.StartingPage; int maxPages = _settings.MaxPages; int pagesChecked = 0; bool useMaxSongs = _settings.MaxSongs != 0; bool useMaxPages = maxPages != 0; if (useMaxPages && pageIndex > 1) { maxPages = maxPages + pageIndex - 1; } var ProcessPageBlock = new TransformBlock <int, PageReadResult>(async pageNum => { return(await feed.GetSongsFromPageAsync(pageNum, cancellationToken).ConfigureAwait(false)); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = MaxConcurrency, BoundedCapacity = MaxConcurrency, CancellationToken = cancellationToken //#if NETSTANDARD // , EnsureOrdered = true //#endif }); bool continueLooping = true; int itemsInBlock = 0; List <PageReadResult> pageResults = new List <PageReadResult>(maxPages + 2); do { if (cancellationToken.IsCancellationRequested) { continueLooping = false; } while (continueLooping) { if (Utilities.IsPaused) { await Utilities.WaitUntil(() => !Utilities.IsPaused, 500, cancellationToken).ConfigureAwait(false); } if (cancellationToken.IsCancellationRequested) { continueLooping = false; break; } await ProcessPageBlock.SendAsync(pageIndex, cancellationToken).ConfigureAwait(false); // TODO: Need check with SongsPerPage itemsInBlock++; pageIndex++; if ((pageIndex > maxPages && useMaxPages) || cancellationToken.IsCancellationRequested) { continueLooping = false; } // TODO: Better http error handling, what if only a single page is broken and returns 0 songs? while (ProcessPageBlock.OutputCount > 0 || itemsInBlock == MaxConcurrency || !continueLooping) { if (cancellationToken.IsCancellationRequested) { continueLooping = false; break; } if (itemsInBlock <= 0) { break; } await ProcessPageBlock.OutputAvailableAsync(cancellationToken).ConfigureAwait(false); while (ProcessPageBlock.TryReceive(out PageReadResult pageResult)) { int songsAdded = 0; if (pageResult != null) { pageResults.Add(pageResult); } if (Utilities.IsPaused) { await Utilities.WaitUntil(() => !Utilities.IsPaused, 500, cancellationToken).ConfigureAwait(false); } itemsInBlock--; if (pageResult.IsLastPage || pageResult == null || pageResult.Count == 0) // TODO: This will trigger if a single page has an error. { Logger?.Debug("Received no new songs, last page reached."); ProcessPageBlock.Complete(); itemsInBlock = 0; continueLooping = false; break; } if (pageResult.Count > 0) { Logger?.Debug($"Receiving {pageResult.Count} potential songs from {pageResult.Uri}"); } else { Logger?.Debug($"Did not find any songs on page '{pageResult.Uri}' of {Name}.{settings.FeedName}."); } // TODO: Process PageReadResults for better error feedback. foreach (var song in pageResult.Songs) { if (!retDict.ContainsKey(song.Hash)) { if (retDict.Count < settings.MaxSongs || settings.MaxSongs == 0) { retDict.Add(song.Hash, song); songsAdded++; } if (retDict.Count >= settings.MaxSongs && useMaxSongs) { continueLooping = false; } } } int prog = Interlocked.Increment(ref pagesChecked); progress?.Report(new ReaderProgress(prog, songsAdded)); if (!useMaxPages || pageIndex <= maxPages) { if (retDict.Count < settings.MaxSongs) { continueLooping = true; } } } } } }while (continueLooping); if (pageResults.Any(r => r.PageError == PageErrorType.Cancelled)) { return(FeedResult.GetCancelledResult(retDict, pageResults)); } return(new FeedResult(retDict, pageResults)); }