public void CuratorRecommended_LimitedSongs() { var reader = new BeastSaberReader("Zingabopp", DefaultMaxConcurrency) { StoreRawData = true }; int maxSongs = 60; var settings = new BeastSaberFeedSettings((int)BeastSaberFeedName.CuratorRecommended) { MaxSongs = maxSongs }; var result = reader.GetSongsFromFeedAsync(settings).Result; Assert.IsTrue(result.Count == maxSongs); Assert.IsFalse(result.Songs.Any(s => string.IsNullOrEmpty(s.Key))); Assert.IsFalse(result.Songs.Any(s => s.Value.DownloadUri == null)); try { Assert.AreEqual(2, result.PagesChecked); } catch (AssertFailedException) { for(int i = 0; i < result.PageResults.Count(); i++) { var page = result.PageResults[i]; Console.WriteLine($"Page {i + 1}: {page.Count} songs. {page.Uri}"); } Assert.AreEqual(3, result.PagesChecked); } var firstSong = result.Songs.First().Value; Assert.IsFalse(string.IsNullOrEmpty(firstSong.RawData)); var firstRawData = JToken.Parse(firstSong.RawData); Assert.IsTrue(firstRawData["hash"]?.Value<string>().ToUpper() == firstSong.Hash); }
public void Bookmarks_UnlimitedSongs() { var reader = new BeastSaberReader("Zingabopp", DefaultMaxConcurrency); int maxSongs = 0; var settings = new BeastSaberFeedSettings((int)BeastSaberFeedName.Bookmarks) { MaxSongs = maxSongs }; var songList = reader.GetSongsFromFeedAsync(settings).Result; Assert.IsTrue(songList.Count > 0); Assert.IsFalse(songList.Songs.Any(s => string.IsNullOrEmpty(s.Key))); }
public void Followings_LimitedSongs() { var reader = new BeastSaberReader("Zingabopp", DefaultMaxConcurrency) { StoreRawData = true }; int maxSongs = 60; var settings = new BeastSaberFeedSettings((int)BeastSaberFeedName.Following) { MaxSongs = maxSongs }; var result = reader.GetSongsFromFeedAsync(settings).Result; Assert.IsTrue(result.Count == maxSongs); //Assert.IsFalse(songList.Count > maxSongs); Assert.IsFalse(result.Songs.Any(s => string.IsNullOrEmpty(s.Key))); }
protected async Task <JobStats> GetBeastSaberAsync(BeatSyncConfig config, IJobBuilder jobBuilder, JobManager jobManager, CancellationToken cancellationToken) { BeastSaberConfig sourceConfig = config.BeastSaber; JobStats sourceStats = new JobStats(); if (!sourceConfig.Enabled) { return(sourceStats); } BeastSaberReader reader = new BeastSaberReader(sourceConfig.Username, sourceConfig.MaxConcurrentPageChecks); FeedConfigBase[] feedConfigs = new FeedConfigBase[] { sourceConfig.Bookmarks, sourceConfig.Follows, sourceConfig.CuratorRecommended }; if (!feedConfigs.Any(f => f.Enabled)) { Logger.log?.Info($"No feeds enabled for {reader.Name}"); return(sourceStats); } SourceStarted?.Invoke(this, "BeastSaber"); foreach (FeedConfigBase?feedConfig in feedConfigs.Where(c => c.Enabled)) { //if (string.IsNullOrEmpty(sourceConfig.Username) && feedConfig.GetType() != typeof(BeastSaberCuratorRecommended)) //{ // Logger.log?.Warn($" {feedConfig.GetType().Name} feed not available without a valid username."); // continue; //} Logger.log?.Info($" Starting {feedConfig.GetType().Name} feed..."); FeedResult results = await reader.GetSongsFromFeedAsync(feedConfig.ToFeedSettings()).ConfigureAwait(false); if (results.Successful) { IEnumerable <IJob>?jobs = CreateJobs(results, jobBuilder, jobManager, cancellationToken); JobResult[] jobResults = await Task.WhenAll(jobs.Select(j => j.JobTask).ToArray()); JobStats feedStats = new JobStats(jobResults); ProcessFinishedJobs(jobs, jobBuilder.SongTargets, config, feedConfig); Logger.log?.Info($" Finished {feedConfig.GetType().Name} feed: ({feedStats})."); sourceStats += feedStats; } else { if (results.Exception != null) { Logger.log?.Error($" Error getting results from {feedConfig.GetType().Name}: {results.Exception.Message}"); Logger.log?.Debug(results.Exception); } else { Logger.log?.Error($" Error getting results from {feedConfig.GetType().Name}: Unknown error."); } } } Logger.log?.Info($" Finished BeastSaber reading: ({sourceStats})."); return(sourceStats); }
private static void Tests() { //ScrapedDataProvider.Initialize(); WebUtils.Initialize(5); ScrapedDataProvider.BeatSaverSongs.WriteFile(); var bsReader = new BeatSaverReader(); //var bsSongs = bsReader.GetSongsFromFeed(new BeatSaverFeedSettings((int)BeatSaverFeeds.LATEST) { MaxPages = 10, searchOnline = true }); var authorSongs = BeatSaverReader.GetSongsByAuthor("ruckasdfus"); var song = ScrapedDataProvider.Songs.Values.Where(s => s.ScoreSaberInfo.Any(d => d.Value.uid == 155732)).FirstOrDefault(); var found = ScrapedDataProvider.TryGetSongByKey("3f57", out SongInfo badSong, false); var CustomSongsPath = Path.Combine(OldConfig.BeatSaberPath, "Beat Saber_Data", "CustomLevels"); var tempFolder = new DirectoryInfo(Path.Combine(Path.GetTempPath(), badSong.key + ".zip")); var outputFolder = new DirectoryInfo(Path.Combine(CustomSongsPath, $"{badSong.key} ({Utilities.MakeSafeFilename(badSong.songName)} - {Utilities.MakeSafeFilename(badSong.authorName)})")); var job = new DownloadJob(badSong, tempFolder.FullName, outputFolder.FullName); job.RunJobAsync().Wait(); var br = new BeastSaberReader("Zingabopp", 3); var text = WebUtils.GetPageText("https://bsaber.com/wp-json/bsaber-api/songs/?bookmarked_by=Zingabopp&page=1"); var bSongs = br.GetSongsFromPage(text); //ScrapedDataProvider.Initialize(); //ScrapedDataProvider.Initialize(); var bsScrape = ScrapedDataProvider.BeatSaverSongs; var ssScrape = ScrapedDataProvider.ScoreSaberSongs; ScrapedDataProvider.TryGetSongByHash("501f6b1bddb2af72abda0f1e6b7b89cb1eb3db67", out SongInfo deletedSong); //var job = new DownloadJob(deletedSong, "test.zip", @"ScrapedData\test"); //var jobTask = job.RunJobAsync(); //jobTask.Wait(); bsScrape.AddOrUpdate(null); var resp = WebUtils.HttpClient.GetAsync("https://beatsaver.com/api/maps/detail/b"); Task.WaitAll(resp); var rateHeaders = resp.Result.Headers.Where(h => h.Key.StartsWith("Rate-Limit")).ToDictionary(x => x.Key, x => x.Value.FirstOrDefault()); var remoteTime = resp.Result.Headers.Date; var rateInfo = WebUtils.ParseRateLimit(rateHeaders); Console.WriteLine("Reset Timespan: " + rateInfo.TimeToReset.ToString()); foreach (var item in resp.Result.Headers) { Console.WriteLine($"{item.Key}: {string.Join("|", item.Value)}"); } var trending = ScrapedDataProvider.Songs.Values.Where(s => s.ScoreSaberInfo.Count > 0).OrderByDescending(s => s.ScoreSaberInfo.Values.Select(ss => ss.scores).Aggregate((a, b) => a + b)).Take(100); var detTrending = trending.Select(s => (s.ScoreSaberInfo.Values.Select(ss => ss.scores).Aggregate((a, b) => a + b), s)).ToList(); }
public void GetSongsFromFeed_Bookmarks() { var reader = new BeastSaberReader("Zingabopp", DefaultMaxConcurrency); int maxSongs = 60; var settings = new BeastSaberFeedSettings((int)BeastSaberFeed.Bookmarks) { MaxSongs = maxSongs }; var songList = reader.GetSongsFromFeed(settings); Assert.IsTrue(songList.Count > 0); Assert.IsFalse(songList.Count > maxSongs); Assert.IsFalse(songList.Any(s => string.IsNullOrEmpty(s.Key))); }
public void CuratorRecommended_UnlimitedSongs() { var reader = new BeastSaberReader("Zingabopp", DefaultMaxConcurrency) { StoreRawData = true }; int maxSongs = 0; var settings = new BeastSaberFeedSettings((int)BeastSaberFeedName.CuratorRecommended) { MaxSongs = maxSongs }; var result = reader.GetSongsFromFeedAsync(settings).Result; Assert.IsTrue(result.Count != 0); Assert.IsFalse(result.Songs.Any(s => string.IsNullOrEmpty(s.Key))); Assert.IsFalse(result.Songs.Any(s => s.Value.DownloadUri == null)); Assert.IsTrue(result.PagesChecked >= 26); var firstSong = result.Songs.First().Value; var firstRawData = JToken.Parse(firstSong.RawData); Assert.IsTrue(firstRawData["hash"]?.Value<string>().ToUpper() == firstSong.Hash); }
public void CancelledInProgress() { var cts = new CancellationTokenSource(500); IFeedReader reader = new BeastSaberReader("Zingabopp", 1); int maxSongs = 300; var settings = new BeastSaberFeedSettings((int)BeastSaberFeedName.CuratorRecommended) { MaxSongs = maxSongs }; var result = reader.GetSongsFromFeed(settings, cts.Token); Assert.IsFalse(result.Successful); Assert.AreEqual(FeedResultError.Cancelled, result.ErrorCode); cts.Dispose(); }
public void GetSongsFromFeed_CuratorRecommended_LimitedPages() { var reader = new BeastSaberReader("Zingabopp", DefaultMaxConcurrency); int maxSongs = 30; int maxPages = 2; var settings = new BeastSaberFeedSettings((int)BeastSaberFeed.CuratorRecommended) { MaxPages = maxPages, MaxSongs = maxSongs }; var songList = reader.GetSongsFromFeed(settings); Assert.IsTrue(songList.Count == maxSongs); Assert.IsFalse(songList.Any(s => string.IsNullOrEmpty(s.Key))); Assert.IsFalse(songList.Any(s => s.Value.DownloadUri == null)); }
public void GetSongsFromFeed_Followings_SinglePage() { var reader = new BeastSaberReader("Zingabopp", DefaultMaxConcurrency); int maxSongs = 150; int maxPages = 1; var settings = new BeastSaberFeedSettings((int)BeastSaberFeed.Following) { MaxSongs = maxSongs, MaxPages = maxPages }; var songList = reader.GetSongsFromFeed(settings); Assert.IsTrue(songList.Count > 0); Assert.IsTrue(songList.Count <= 100); //Assert.IsFalse(songList.Count > maxSongs); Assert.IsFalse(songList.Any(s => string.IsNullOrEmpty(s.Key))); }
public void Success_JSON() { var reader = new BeastSaberReader("Zingabopp", DefaultMaxConcurrency); var text = File.ReadAllText("Data\\BeastSaberJsonPage.json"); Uri uri = null; var songList = reader.GetSongsFromPageText(text, uri, BeastSaberReader.ContentType.JSON); Assert.IsTrue(songList.Count == 20); var firstHash = "a3bbbe2d6f64dfe8324c7098d5c35281d21fd20f".ToUpper(); var firstUrl = "https://beatsaver.com/api/download/key/5679"; Assert.IsTrue(songList.First().Hash == firstHash); Assert.IsTrue(songList.First().DownloadUri.ToString() == firstUrl); var lastHash = "20b9326bd71db4454aba08df06b035ea536322a9".ToUpper(); var lastUrl = "https://beatsaver.com/api/download/key/55d1"; Assert.IsTrue(songList.Last().Hash == lastHash); Assert.IsTrue(songList.Last().DownloadUri.ToString() == lastUrl); Assert.IsFalse(songList.Any(s => string.IsNullOrEmpty(s.Hash))); Assert.IsFalse(songList.Any(s => s.DownloadUri == null)); }
public void Success_XML() { var reader = new BeastSaberReader("Zingabopp", DefaultMaxConcurrency) { StoreRawData = true }; var text = File.ReadAllText("Data\\BeastSaberXMLPage.xml"); Uri uri = null; var songList = reader.GetSongsFromPageText(text, uri, BeastSaberReader.ContentType.XML); Assert.IsTrue(songList.Count == 50); var firstHash = "74575254ae759f3f836eb521b4b80093ca52cd3d".ToUpper(); var firstKey = "56ff"; var firstLevelAuthor = "Rustic"; var firstTitle = "Xilent – Code Blood"; var firstDownloadUrl = "https://beatsaver.com/api/download/key/56ff"; var firstUrl = "https://beatsaver.com/api/download/key/56ff"; var firstSong = songList.First(); Assert.IsTrue(firstSong.Hash == firstHash); Assert.IsTrue(firstSong.DownloadUri.ToString() == firstUrl); // Raw Data test JToken firstRawData = JToken.Parse(firstSong.RawData); Assert.IsTrue(firstRawData["Hash"]?.Value <string>().ToUpper() == firstHash); Assert.IsTrue(firstRawData["SongKey"]?.Value <string>() == firstKey); Assert.IsTrue(firstRawData["LevelAuthorName"]?.Value <string>() == firstLevelAuthor); Assert.IsTrue(firstRawData["SongTitle"]?.Value <string>() == firstTitle); Assert.IsTrue(firstRawData["DownloadURL"]?.Value <string>() == firstDownloadUrl); var lastHash = "e3487474b70d969927e459a1590e93b7ad25a436".ToUpper(); var lastUrl = "https://beatsaver.com/api/download/key/5585"; Assert.IsTrue(songList.Last().Hash == lastHash); Assert.IsTrue(songList.Last().DownloadUri.ToString() == lastUrl); Assert.IsFalse(songList.Any(s => string.IsNullOrEmpty(s.Hash))); Assert.IsFalse(songList.Any(s => s.DownloadUri == null)); }
public void GetSongsFromFeed_Followings_SinglePage_LimitedSongs() { var reader = new BeastSaberReader("Zingabopp", DefaultMaxConcurrency) { StoreRawData = true }; int maxSongs = 20; int maxPages = 1; var settings = new BeastSaberFeedSettings((int)BeastSaberFeed.Following) { MaxSongs = maxSongs, MaxPages = maxPages }; var songList = reader.GetSongsFromFeed(settings); var pagesChecked = songList.Values.GroupBy(s => s.SourceUri); Assert.IsTrue(pagesChecked.Count() == 1); Assert.IsTrue(songList.Count > 0); Assert.IsTrue(songList.Count <= 20); //Assert.IsFalse(songList.Count > maxSongs); Assert.IsFalse(songList.Any(s => string.IsNullOrEmpty(s.Key))); }
public void BeastSaberBookmarks() { CancellationTokenSource cts = new CancellationTokenSource(); var config = DefaultConfig; var sourceConfig = DefaultConfig.BeastSaber; sourceConfig.MaxConcurrentPageChecks = 5; var feedConfig = sourceConfig.Bookmarks; feedConfig.MaxSongs = 60; var songDownloader = new SongDownloader(config, null, null, SONGSPATH); var reader = new BeastSaberReader(sourceConfig.Username, sourceConfig.MaxConcurrentPageChecks); var settings = feedConfig.ToFeedSettings(); var result = songDownloader.ReadFeed(reader, settings, 0, null, PlaylistStyle.Append, cts.Token).Result; if (!result.Successful) { Console.WriteLine(result.FaultedResults.First().Exception); } Assert.IsNotNull(result); Assert.IsTrue(result.Count > 0); }
public async Task <Dictionary <string, ScrapedSong> > ReadBeastSaber() { if (BeatSync.Paused) { await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false); } Stopwatch sw = new Stopwatch(); sw.Start(); Logger.log?.Info("Starting BeastSaber reading"); var config = Config.BeastSaber; BeastSaberReader reader = null; try { reader = new BeastSaberReader(config.Username, config.MaxConcurrentPageChecks); } catch (Exception ex) { Logger.log?.Error(ex); return(null); } var readerSongs = new Dictionary <string, ScrapedSong>(); if (config.Bookmarks.Enabled && !string.IsNullOrEmpty(config.Username)) { try { var feedSettings = config.Bookmarks.ToFeedSettings(); var feedPlaylist = config.Bookmarks.CreatePlaylist ? PlaylistManager.GetPlaylist(config.Bookmarks.FeedPlaylist) : null; var playlistStyle = config.Bookmarks.PlaylistStyle; if (BeatSync.Paused) { await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false); } var songs = await ReadFeed(reader, feedSettings, feedPlaylist, playlistStyle).ConfigureAwait(false); var pages = songs.Values.Select(s => s.SourceUri.ToString()).Distinct().Count(); var feedName = reader.GetFeedName(feedSettings); Logger.log?.Info($"{reader.Name}.{feedName} Feed: Found {songs.Count} songs from {pages} {(pages == 1 ? "page" : "pages")}."); readerSongs.Merge(songs); } catch (ArgumentException ex) { Logger.log?.Critical("Exception in BeastSaber Bookmarks: " + ex.Message); } catch (Exception ex) { Logger.log?.Error("Exception in ReadBeastSaber, Bookmarks."); Logger.log?.Error(ex); } } else if (string.IsNullOrEmpty(config.Username)) { Logger.log?.Warn("BeastSaber Bookmarks feed is enabled, but a username has not been provided."); } if (config.Follows.Enabled && !string.IsNullOrEmpty(config.Username)) { try { var feedSettings = config.Follows.ToFeedSettings(); var feedPlaylist = config.Follows.CreatePlaylist ? PlaylistManager.GetPlaylist(config.Follows.FeedPlaylist) : null; var playlistStyle = config.Follows.PlaylistStyle; if (BeatSync.Paused) { await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false); } var songs = await ReadFeed(reader, feedSettings, feedPlaylist, playlistStyle).ConfigureAwait(false); var pages = songs.Values.Select(s => s.SourceUri.ToString()).Distinct().Count(); var feedName = reader.GetFeedName(feedSettings); Logger.log?.Info($"{reader.Name}.{feedName} Feed: Found {songs.Count} songs from {pages} {(pages == 1 ? "page" : "pages")}."); readerSongs.Merge(songs); } catch (ArgumentException ex) { Logger.log?.Critical("Exception in BeastSaber Follows: " + ex.Message); } catch (Exception ex) { Logger.log?.Error("Exception in ReadBeastSaber, Follows."); Logger.log?.Error(ex); } } else if (string.IsNullOrEmpty(config.Username)) { Logger.log?.Warn("BeastSaber Follows feed is enabled, but a username has not been provided."); } if (config.CuratorRecommended.Enabled) { try { var feedSettings = config.CuratorRecommended.ToFeedSettings(); var feedPlaylist = config.CuratorRecommended.CreatePlaylist ? PlaylistManager.GetPlaylist(config.CuratorRecommended.FeedPlaylist) : null; var playlistStyle = config.CuratorRecommended.PlaylistStyle; if (BeatSync.Paused) { await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false); } var songs = await ReadFeed(reader, feedSettings, feedPlaylist, playlistStyle).ConfigureAwait(false); var pages = songs.Values.Select(s => s.SourceUri.ToString()).Distinct().Count(); var feedName = reader.GetFeedName(feedSettings); Logger.log?.Info($"{reader.Name}.{feedName} Feed: Found {songs.Count} songs from {pages} {(pages == 1 ? "page" : "pages")}."); readerSongs.Merge(songs); } catch (Exception ex) { Logger.log?.Error("Exception in ReadBeastSaber, Curator Recommended."); Logger.log?.Error(ex); } } if (BeatSync.Paused) { await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false); } sw.Stop(); var totalPages = readerSongs.Values.Select(s => s.SourceUri.ToString()).Distinct().Count(); Logger.log?.Info($"{reader.Name}: Found {readerSongs.Count} songs on {totalPages} {(totalPages == 1 ? "page" : "pages")} in {sw.Elapsed.ToString()}"); return(readerSongs); }