예제 #1
0
        public void Success_Downloads()
        {
            var reader = new BeatSaverReader()
            {
                StoreRawData = true
            };
            int maxSongs = 50;
            var settings = new BeatSaverFeedSettings((int)BeatSaverFeed.Downloads)
            {
                MaxSongs = maxSongs
            };
            var result = reader.GetSongsFromFeed(settings);

            Assert.IsTrue(result.Count == settings.MaxSongs);
            int expectedPages = ExpectedPagesForSongs(result.Count);

            Assert.AreEqual(expectedPages, result.PagesChecked);
            foreach (var song in result.Songs.Values)
            {
                Console.WriteLine($"{song.SongName} by {song.MapperName}, {song.Hash}");
            }
        }
예제 #2
0
        public void ScrapeNewSongs()
        {
            int      lastBeatSaverCount  = 0;
            int      lastScoreSaberCount = 0;
            DateTime lastBSScrape        = DateTime.MinValue;
            DateTime lastSSScrape        = DateTime.MinValue;

            if (ScrapedDataProvider.BeatSaverSongs.HasData)
            {
                lastBeatSaverCount = ScrapedDataProvider.BeatSaverSongs.Data.Count;
                lastBSScrape       = ScrapedDataProvider.BeatSaverSongs.Data.Max(s => s.ScrapedAt);
            }
            if (ScrapedDataProvider.ScoreSaberSongs.HasData)
            {
                lastScoreSaberCount = ScrapedDataProvider.ScoreSaberSongs.Data.Count;
                lastSSScrape        = ScrapedDataProvider.ScoreSaberSongs.Data.Max(s => s.ScrapedAt);
            }
            Logger.Info($"Scraping new songs. Last Beat Saver scrape was at {lastBSScrape.ToString()}.");
            var bsReader = FeedReaders[BeatSaverReader.NameKey] as BeatSaverReader;

            BeatSaverReader.ScrapeBeatSaver(200, true);
            ScrapedDataProvider.BeatSaverSongs.WriteFile();
            int newBeatSaverSongs = ScrapedDataProvider.BeatSaverSongs.Data.Count - lastBeatSaverCount;

            Logger.Info($"Scraped {(newBeatSaverSongs).ToString()} new song{(newBeatSaverSongs == 1 ? "" : "s")} from Beat Saver.");

            if ((DateTime.Now - lastSSScrape).TotalHours > 3)
            {
                Logger.Info($"Scraping new ScoreSaber difficulties. Last ScoreSaber scrape was at {lastSSScrape.ToString()}.");
                int newScoreSaberSongs = ScrapedDataProvider.ScoreSaberSongs.Data.Count - lastScoreSaberCount;
                ScoreSaberReader.ScrapeScoreSaber(1000, 500, true, 2);
                Logger.Info($"Scraped {(newScoreSaberSongs).ToString()} new difficult{(newScoreSaberSongs == 1 ? "y" : "ies")} from ScoreSaber.");
                ScrapedDataProvider.ScoreSaberSongs.WriteFile();
            }
            else
            {
                Logger.Info($"Last ScoreSaber scrape was at {lastSSScrape.ToString()}, skipping.");
            }
        }
예제 #3
0
        public void BeatSaverHot_Cancelled()
        {
            var config       = DefaultConfig;
            var sourceConfig = DefaultConfig.BeatSaver;

            sourceConfig.MaxConcurrentPageChecks = 5;
            var feedConfig = sourceConfig.Hot;

            feedConfig.MaxSongs = 60;
            var songDownloader = new SongDownloader(config, null, null, SONGSPATH);
            var reader         = new BeatSaverReader();
            var settings       = feedConfig.ToFeedSettings();

            CancellationTokenSource cts = new CancellationTokenSource(150);
            var resultTask = songDownloader.ReadFeed(reader, settings, 0, null, PlaylistStyle.Append, cts.Token);


            var result = resultTask.Result;

            Assert.IsNotNull(result);
            Assert.IsFalse(result.Successful);
            Assert.AreEqual(FeedResultError.Cancelled, result.ErrorCode);
        }
예제 #4
0
        public void Search_Default_LimitedSongs()
        {
            var reader = new BeatSaverReader()
            {
                StoreRawData = true
            };
            int maxSongs = 10;
            var settings = new BeatSaverFeedSettings((int)BeatSaverFeed.Search)
            {
                MaxSongs = maxSongs, Criteria = "Believer"
            };
            var result = reader.GetSongsFromFeed(settings);

            Assert.IsTrue(result.Count > 0);
            Assert.IsTrue(result.Count <= 10);
            int expectedPages = ExpectedPagesForSongs(result.Count);

            Assert.AreEqual(expectedPages, result.PagesChecked);
            foreach (var song in result.Songs.Values)
            {
                Console.WriteLine($"{song.SongName} by {song.MapperName}, {song.Hash}");
            }
        }
        public void ParseSongsFromPage_Test()
        {
            string pageText = File.ReadAllText(@"Data\BeatSaverListPage.json");
            Uri    uri      = null;
            var    songs    = BeatSaverReader.ParseSongsFromPage(pageText, uri);

            Assert.IsTrue(songs.Count == 10);
            foreach (var song in songs)
            {
                Assert.IsFalse(song.DownloadUri == null);
                Assert.IsFalse(string.IsNullOrEmpty(song.Hash));
                Assert.IsFalse(string.IsNullOrEmpty(song.MapperName));
                Assert.IsFalse(string.IsNullOrEmpty(song.RawData));
                Assert.IsFalse(string.IsNullOrEmpty(song.SongName));
            }
            var    firstSong = JObject.Parse(songs.First().RawData);
            string firstHash = firstSong["hash"]?.Value <string>();

            Assert.IsTrue(firstHash == "27639680f92a9588b7cce843fc7aaa0f5dc720f8");
            string firstUploader = firstSong["uploader"]?["username"]?.Value <string>();

            Assert.IsTrue(firstUploader == "latte");
        }
        public void GetSongsFromFeed_Authors_Test()
        {
            var reader = new BeatSaverReader()
            {
                StoreRawData = true
            };
            var authorList = new string[] { "BlackBlazon", "greatyazer" };
            var settings   = new BeatSaverFeedSettings((int)BeatSaverFeed.Author)
            {
                Authors = authorList, MaxSongs = 59
            };
            var songsByAuthor   = reader.GetSongsFromFeed(settings);
            var detectedAuthors = songsByAuthor.Values.Select(s => s.MapperName.ToLower()).Distinct();

            foreach (var song in songsByAuthor)
            {
                Assert.IsTrue(song.Value.DownloadUri != null);
                Assert.IsTrue(authorList.Any(a => a.ToLower() == song.Value.MapperName.ToLower()));
            }
            foreach (var author in authorList)
            {
                Assert.IsTrue(songsByAuthor.Any(s => s.Value.MapperName.ToLower() == author.ToLower()));
            }

            // BlackBlazon check
            var blazonHash = "58de2d709a45b68fdb1dbbfefb187f59f629bfc5".ToUpper();
            var blazonSong = songsByAuthor[blazonHash];

            Assert.IsTrue(blazonSong != null);
            Assert.IsTrue(blazonSong.DownloadUri != null);
            // GreatYazer check
            var songHash  = "bf8c016dc6b9832ece3030f05277bbbe67db790d".ToUpper();
            var yazerSong = songsByAuthor[songHash];

            Assert.IsTrue(yazerSong != null);
            Assert.IsTrue(yazerSong.DownloadUri != null);
        }
예제 #7
0
        public void Search_User()
        {
            var reader = new BeatSaverReader()
            {
                StoreRawData = true
            };
            int    maxSongs   = 10;
            string criteria   = "19F2879D11A91B51A5C090D63471C3E8D9B7AEE3";
            var    searchType = BeatSaverSearchType.hash;
            var    settings   = new BeatSaverFeedSettings((int)BeatSaverFeed.Search)
            {
                MaxSongs = maxSongs, Criteria = criteria, SearchType = searchType
            };
            var result = reader.GetSongsFromFeed(settings);

            Assert.AreEqual(1, result.Count);
            int expectedPages = ExpectedPagesForSongs(result.Count);

            Assert.AreEqual(expectedPages, result.PagesChecked);
            foreach (var song in result.Songs.Values)
            {
                Console.WriteLine($"{song.SongName} by {song.MapperName}, {song.Hash}");
            }
        }
예제 #8
0
        public void Success()
        {
            string path     = Path.Combine("Data", "NewBeatSaver", "List", "Latest", "Latest_0.json");
            string pageText = File.ReadAllText(path);
            Uri    uri      = null;
            var    songs    = BeatSaverReader.ParseSongsFromPage(pageText, uri, true);

            Assert.AreEqual(20, songs.Count);
            foreach (var song in songs)
            {
                Assert.IsFalse(song.DownloadUri == null);
                Assert.IsFalse(string.IsNullOrEmpty(song.Hash));
                Assert.IsFalse(string.IsNullOrEmpty(song.LevelAuthorName));
                Assert.IsFalse(string.IsNullOrEmpty(song.RawData));
                Assert.IsFalse(string.IsNullOrEmpty(song.Name));
            }
            var    firstSong = JObject.Parse(songs.First().RawData);
            string firstHash = firstSong["versions"].First()["hash"]?.Value <string>();

            Assert.AreEqual("e2512e6fbf85059d9fd9b429f62b2e618dd4d7e9", firstHash);
            string firstUploader = firstSong["uploader"]?["name"]?.Value <string>();

            Assert.AreEqual("itzrimuru", firstUploader);
        }
예제 #9
0
        protected async Task <JobStats> GetBeatSaverAsync(BeatSyncConfig config, IJobBuilder jobBuilder, JobManager jobManager, CancellationToken cancellationToken)
        {
            BeatSaverConfig sourceConfig = config.BeatSaver;
            JobStats        sourceStats  = new JobStats();

            if (!sourceConfig.Enabled)
            {
                return(sourceStats);
            }

            BeatSaverReader reader = new BeatSaverReader();

            FeedConfigBase[] feedConfigs = new FeedConfigBase[] { sourceConfig.Hot, sourceConfig.Downloads, sourceConfig.Latest };
            if (!(feedConfigs.Any(f => f.Enabled) || sourceConfig.FavoriteMappers.Enabled))
            {
                Logger.log?.Info($"No feeds enabled for {reader.Name}");
                return(sourceStats);
            }

            SourceStarted?.Invoke(this, "BeatSaver");
            foreach (FeedConfigBase?feedConfig in feedConfigs.Where(c => c.Enabled))
            {
                Logger.log?.Info($"  Starting {feedConfig.GetType().Name} feed...");
                FeedResult results = await reader.GetSongsFromFeedAsync(feedConfig.ToFeedSettings(), cancellationToken).ConfigureAwait(false);

                if (results.Successful)
                {
                    IEnumerable <IJob>?jobs       = CreateJobs(results, jobBuilder, jobManager, cancellationToken);
                    JobResult[]        jobResults = await Task.WhenAll(jobs.Select(j => j.JobTask).ToArray());

                    JobStats feedStats = new JobStats(jobResults);
                    ProcessFinishedJobs(jobs, jobBuilder.SongTargets, config, feedConfig);
                    Logger.log?.Info($"  Finished {feedConfig.GetType().Name} feed: ({feedStats}).");
                    sourceStats += feedStats;
                }
                else
                {
                    if (results.Exception != null)
                    {
                        Logger.log?.Error($"  Error getting results from {feedConfig.GetType().Name}{results.Exception.Message}");
                        Logger.log?.Debug($"{results.Exception}");
                    }
                    else
                    {
                        Logger.log?.Error($"  Error getting results from {feedConfig.GetType().Name}: Unknown error.");
                    }
                }
            }

            string[] mappers = sourceConfig.FavoriteMappers.Mappers ?? Array.Empty <string>();
            if (sourceConfig.FavoriteMappers.Enabled)
            {
                FeedConfigBase feedConfig = sourceConfig.FavoriteMappers;
                if (mappers.Length > 0)
                {
                    Logger.log?.Info("  Starting FavoriteMappers feed...");
                    List <IPlaylist> playlists       = new List <IPlaylist>();
                    List <IPlaylist> feedPlaylists   = new List <IPlaylist>();
                    List <IPlaylist> recentPlaylists = new List <IPlaylist>();
                    JobStats         feedStats       = new JobStats();
                    foreach (string?mapper in mappers)
                    {
                        Logger.log?.Info($"  Getting songs by {mapper}...");
                        playlists.Clear();

                        FeedResult results = await reader.GetSongsFromFeedAsync(sourceConfig.FavoriteMappers.ToFeedSettings(mapper)).ConfigureAwait(false);

                        if (results.Successful)
                        {
                            foreach (ITargetWithPlaylists?targetWithPlaylist in jobBuilder.SongTargets.Where(t => t is ITargetWithPlaylists).Select(t => (ITargetWithPlaylists)t))
                            {
                                PlaylistManager?playlistManager = targetWithPlaylist.PlaylistManager;
                                if (playlistManager != null)
                                {
                                    if (config.RecentPlaylistDays > 0)
                                    {
                                        recentPlaylists.Add(playlistManager.GetOrAddPlaylist(BuiltInPlaylist.BeatSyncRecent));
                                    }
                                    if (config.AllBeatSyncSongsPlaylist)
                                    {
                                        playlists.Add(playlistManager.GetOrAddPlaylist(BuiltInPlaylist.BeatSyncAll));
                                    }
                                    if (sourceConfig.FavoriteMappers.CreatePlaylist)
                                    {
                                        IPlaylist feedPlaylist;
                                        try
                                        {
                                            if (sourceConfig.FavoriteMappers.SeparateMapperPlaylists)
                                            {
                                                feedPlaylist = playlistManager.GetOrCreateAuthorPlaylist(mapper);
                                            }
                                            else
                                            {
                                                feedPlaylist = playlistManager.GetOrAddPlaylist(BuiltInPlaylist.BeatSaverFavoriteMappers);
                                            }
                                            feedPlaylists.Add(feedPlaylist);
                                            playlists.Add(feedPlaylist);
                                        }
                                        catch (ArgumentException ex)
                                        {
                                            Logger.log?.Error($"Error getting playlist for FavoriteMappers: {ex.Message}");
                                            Logger.log?.Debug(ex);
                                        }
                                    }
                                }
                            }
                            IEnumerable <IJob> jobs       = CreateJobs(results, jobBuilder, jobManager, cancellationToken) ?? Array.Empty <IJob>();
                            JobResult[]        jobResults = await Task.WhenAll(jobs.Select(j => j.JobTask).ToArray());

                            JobStats mapperStats = new JobStats(jobResults);
                            feedStats += mapperStats;
                            if (jobs.Any(j => j.Result?.Successful ?? false) && feedConfig.PlaylistStyle == PlaylistStyle.Replace)
                            {
                                // TODO: This should only apply to successful targets.
                                foreach (IPlaylist?feedPlaylist in feedPlaylists)
                                {
                                    feedPlaylist.Clear();
                                    feedPlaylist.RaisePlaylistChanged();
                                }
                            }
                            ProcessFinishedJobs(jobs, playlists, recentPlaylists);

                            Logger.log?.Info($"  Finished getting songs by {mapper}: ({mapperStats}).");
                        }
                        else
                        {
                            if (results.Exception != null)
                            {
                                Logger.log?.Error($"Error getting songs by {mapper}: {results.Exception.Message}");
                                Logger.log?.Debug(results.Exception);
                            }
                            else
                            {
                                Logger.log?.Error($"Error getting songs by {mapper}");
                            }
                        }
                    }
                    sourceStats += feedStats;
                    Logger.log?.Info($"  Finished {feedConfig.GetType().Name} feed: ({feedStats}).");
                }
                else
                {
                    Logger.log?.Warn("  No FavoriteMappers found, skipping...");
                }
            }

            Logger.log?.Info($"  Finished BeatSaver reading: ({sourceStats}).");
            return(sourceStats);
        }
예제 #10
0
        public async Task <Dictionary <string, ScrapedSong> > ReadBeatSaver(Playlist allPlaylist = null)
        {
            if (BeatSync.Paused)
            {
                await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false);
            }
            Stopwatch sw = new Stopwatch();

            sw.Start();
            Logger.log?.Info("Starting BeatSaver reading");

            var             config = Config.BeatSaver;
            BeatSaverReader reader = null;

            try
            {
                reader = new BeatSaverReader();
            }
            catch (Exception ex)
            {
                Logger.log?.Error("Exception creating BeatSaverReader in ReadBeatSaver.");
                Logger.log?.Error(ex);
                return(null);
            }
            var readerSongs = new Dictionary <string, ScrapedSong>();

            if (config.FavoriteMappers.Enabled && (FavoriteMappers.Mappers?.Count() ?? 0) > 0)
            {
                try
                {
                    var      feedSettings = config.FavoriteMappers.ToFeedSettings() as BeatSaverFeedSettings;
                    Playlist feedPlaylist = null;
                    if (!config.FavoriteMappers.SeparateMapperPlaylists)
                    {
                        feedPlaylist = config.FavoriteMappers.CreatePlaylist
                            ? PlaylistManager.GetPlaylist(config.FavoriteMappers.FeedPlaylist)
                            : null;
                    }

                    var playlistStyle = config.FavoriteMappers.PlaylistStyle;
                    var songs         = new Dictionary <string, ScrapedSong>();
                    foreach (var author in FavoriteMappers.Mappers)
                    {
                        feedSettings.Criteria = author;
                        if (BeatSync.Paused)
                        {
                            await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false);
                        }
                        var authorSongs = await ReadFeed(reader, feedSettings, feedPlaylist, playlistStyle).ConfigureAwait(false);

                        Logger.log?.Info($"   FavoriteMappers: Found {authorSongs.Count} songs by {author}");
                        if (config.FavoriteMappers.CreatePlaylist && config.FavoriteMappers.SeparateMapperPlaylists)
                        {
                            var playlistFileName = $"{author}.bplist";
                            var mapperPlaylist   = PlaylistManager.GetOrAdd(playlistFileName, () => new Playlist(playlistFileName, author, "BeatSync", "1"));
                            if (mapperPlaylist != null)
                            {
                                if (playlistStyle == PlaylistStyle.Replace)
                                {
                                    mapperPlaylist.Clear();
                                }
                                foreach (var song in authorSongs.Values)
                                {
                                    mapperPlaylist.TryAdd(song.ToPlaylistSong());
                                }
                            }
                        }

                        songs.Merge(authorSongs);
                    }
                    if (feedPlaylist != null)
                    {
                        foreach (var song in songs.Values)
                        {
                            feedPlaylist.TryAdd(song.ToPlaylistSong());
                        }
                    }
                    var pages    = songs.Values.Select(s => s.SourceUri.ToString()).Distinct().Count();
                    var feedName = reader.GetFeedName(feedSettings);
                    Logger.log?.Info($"{reader.Name}.{feedName} Feed: Found {songs.Count} songs from {pages} {(pages == 1 ? "page" : "pages")}.");
                    readerSongs.Merge(songs);
                }
                catch (InvalidCastException ex)
                {
                    Logger.log?.Error($"This should never happen in ReadBeatSaver.\n{ex.Message}");
                }
                catch (ArgumentNullException ex)
                {
                    Logger.log?.Critical("Exception in ReadBeatSaver, FavoriteMappers: " + ex.Message);
                }
                catch (Exception ex)
                {
                    Logger.log?.Error("Exception in ReadBeatSaver, FavoriteMappers.");
                    Logger.log?.Error(ex);
                }
            }
            else if (config.FavoriteMappers.Enabled)
            {
                Logger.log?.Warn("BeatSaver's FavoriteMappers feed is enabled, but no mappers could be found in UserData\\FavoriteMappers.ini");
            }
            if (config.Hot.Enabled)
            {
                try
                {
                    var feedSettings = config.Hot.ToFeedSettings();
                    var feedPlaylist = config.Hot.CreatePlaylist
                        ? PlaylistManager.GetPlaylist(config.Hot.FeedPlaylist)
                        : null;
                    var playlistStyle = config.Hot.PlaylistStyle;
                    if (BeatSync.Paused)
                    {
                        await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false);
                    }
                    var songs = await ReadFeed(reader, feedSettings, feedPlaylist, playlistStyle).ConfigureAwait(false);

                    var pages    = songs.Values.Select(s => s.SourceUri.ToString()).Distinct().Count();
                    var feedName = reader.GetFeedName(feedSettings);
                    Logger.log?.Info($"{reader.Name}.{feedName} Feed: Found {songs.Count} songs from {pages} {(pages == 1 ? "page" : "pages")}.");
                    readerSongs.Merge(songs);
                }
                catch (InvalidCastException ex)
                {
                    Logger.log?.Error($"This should never happen in ReadBeatSaver.\n{ex.Message}");
                }
                catch (ArgumentNullException ex)
                {
                    Logger.log?.Critical("Exception in ReadBeatSaver, Hot: " + ex.Message);
                }
                catch (Exception ex)
                {
                    Logger.log?.Error("Exception in ReadBeatSaver, Hot.");
                    Logger.log?.Error(ex);
                }
            }
            if (config.Downloads.Enabled)
            {
                try
                {
                    var feedSettings = config.Downloads.ToFeedSettings();
                    var feedPlaylist = config.Downloads.CreatePlaylist
                        ? PlaylistManager.GetPlaylist(config.Downloads.FeedPlaylist)
                        : null;
                    var playlistStyle = config.Downloads.PlaylistStyle;
                    if (BeatSync.Paused)
                    {
                        await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false);
                    }
                    var songs = await ReadFeed(reader, feedSettings, feedPlaylist, playlistStyle).ConfigureAwait(false);

                    var pages    = songs.Values.Select(s => s.SourceUri.ToString()).Distinct().Count();
                    var feedName = reader.GetFeedName(feedSettings);
                    Logger.log?.Info($"{reader.Name}.{feedName} Feed: Found {songs.Count} songs from {pages} {(pages == 1 ? "page" : "pages")}.");
                    readerSongs.Merge(songs);
                }
                catch (InvalidCastException ex)
                {
                    Logger.log?.Error($"This should never happen in ReadBeatSaver.\n{ex.Message}");
                }
                catch (ArgumentNullException ex)
                {
                    Logger.log?.Critical("Exception in ReadBeatSaver, Downloads: " + ex.Message);
                }
                catch (Exception ex)
                {
                    Logger.log?.Error("Exception in ReadBeatSaver, Downloads.");
                    Logger.log?.Error(ex);
                }
            }
            sw.Stop();
            if (BeatSync.Paused)
            {
                await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false);
            }
            var totalPages = readerSongs.Values.Select(s => s.SourceUri.ToString()).Distinct().Count();

            Logger.log?.Info($"{reader.Name}: Found {readerSongs.Count} songs on {totalPages} {(totalPages == 1 ? "page" : "pages")} in {sw.Elapsed.ToString()}");
            return(readerSongs);
        }