public void Success_Downloads()
{
var reader = new BeatSaverReader()
{
StoreRawData = true
};
int maxSongs = 50;
var settings = new BeatSaverFeedSettings((int)BeatSaverFeed.Downloads)
{
MaxSongs = maxSongs
};
var result = reader.GetSongsFromFeed(settings);
Assert.IsTrue(result.Count == settings.MaxSongs);
int expectedPages = ExpectedPagesForSongs(result.Count);
Assert.AreEqual(expectedPages, result.PagesChecked);
foreach (var song in result.Songs.Values)
{
Console.WriteLine($"{song.SongName} by {song.MapperName}, {song.Hash}");
}
}
public void ScrapeNewSongs()
{
int lastBeatSaverCount = 0;
int lastScoreSaberCount = 0;
DateTime lastBSScrape = DateTime.MinValue;
DateTime lastSSScrape = DateTime.MinValue;
if (ScrapedDataProvider.BeatSaverSongs.HasData)
{
lastBeatSaverCount = ScrapedDataProvider.BeatSaverSongs.Data.Count;
lastBSScrape = ScrapedDataProvider.BeatSaverSongs.Data.Max(s => s.ScrapedAt);
}
if (ScrapedDataProvider.ScoreSaberSongs.HasData)
{
lastScoreSaberCount = ScrapedDataProvider.ScoreSaberSongs.Data.Count;
lastSSScrape = ScrapedDataProvider.ScoreSaberSongs.Data.Max(s => s.ScrapedAt);
}
Logger.Info($"Scraping new songs. Last Beat Saver scrape was at {lastBSScrape.ToString()}.");
var bsReader = FeedReaders[BeatSaverReader.NameKey] as BeatSaverReader;
BeatSaverReader.ScrapeBeatSaver(200, true);
ScrapedDataProvider.BeatSaverSongs.WriteFile();
int newBeatSaverSongs = ScrapedDataProvider.BeatSaverSongs.Data.Count - lastBeatSaverCount;
Logger.Info($"Scraped {(newBeatSaverSongs).ToString()} new song{(newBeatSaverSongs == 1 ? "" : "s")} from Beat Saver.");
if ((DateTime.Now - lastSSScrape).TotalHours > 3)
{
Logger.Info($"Scraping new ScoreSaber difficulties. Last ScoreSaber scrape was at {lastSSScrape.ToString()}.");
int newScoreSaberSongs = ScrapedDataProvider.ScoreSaberSongs.Data.Count - lastScoreSaberCount;
ScoreSaberReader.ScrapeScoreSaber(1000, 500, true, 2);
Logger.Info($"Scraped {(newScoreSaberSongs).ToString()} new difficult{(newScoreSaberSongs == 1 ? "y" : "ies")} from ScoreSaber.");
ScrapedDataProvider.ScoreSaberSongs.WriteFile();
}
else
{
Logger.Info($"Last ScoreSaber scrape was at {lastSSScrape.ToString()}, skipping.");
}
}
public void BeatSaverHot_Cancelled()
{
var config = DefaultConfig;
var sourceConfig = DefaultConfig.BeatSaver;
sourceConfig.MaxConcurrentPageChecks = 5;
var feedConfig = sourceConfig.Hot;
feedConfig.MaxSongs = 60;
var songDownloader = new SongDownloader(config, null, null, SONGSPATH);
var reader = new BeatSaverReader();
var settings = feedConfig.ToFeedSettings();
CancellationTokenSource cts = new CancellationTokenSource(150);
var resultTask = songDownloader.ReadFeed(reader, settings, 0, null, PlaylistStyle.Append, cts.Token);
var result = resultTask.Result;
Assert.IsNotNull(result);
Assert.IsFalse(result.Successful);
Assert.AreEqual(FeedResultError.Cancelled, result.ErrorCode);
}
public void Search_Default_LimitedSongs()
{
var reader = new BeatSaverReader()
{
StoreRawData = true
};
int maxSongs = 10;
var settings = new BeatSaverFeedSettings((int)BeatSaverFeed.Search)
{
MaxSongs = maxSongs, Criteria = "Believer"
};
var result = reader.GetSongsFromFeed(settings);
Assert.IsTrue(result.Count > 0);
Assert.IsTrue(result.Count <= 10);
int expectedPages = ExpectedPagesForSongs(result.Count);
Assert.AreEqual(expectedPages, result.PagesChecked);
foreach (var song in result.Songs.Values)
{
Console.WriteLine($"{song.SongName} by {song.MapperName}, {song.Hash}");
}
}
public void ParseSongsFromPage_Test()
{
string pageText = File.ReadAllText(@"Data\BeatSaverListPage.json");
Uri uri = null;
var songs = BeatSaverReader.ParseSongsFromPage(pageText, uri);
Assert.IsTrue(songs.Count == 10);
foreach (var song in songs)
{
Assert.IsFalse(song.DownloadUri == null);
Assert.IsFalse(string.IsNullOrEmpty(song.Hash));
Assert.IsFalse(string.IsNullOrEmpty(song.MapperName));
Assert.IsFalse(string.IsNullOrEmpty(song.RawData));
Assert.IsFalse(string.IsNullOrEmpty(song.SongName));
}
var firstSong = JObject.Parse(songs.First().RawData);
string firstHash = firstSong["hash"]?.Value <string>();
Assert.IsTrue(firstHash == "27639680f92a9588b7cce843fc7aaa0f5dc720f8");
string firstUploader = firstSong["uploader"]?["username"]?.Value <string>();
Assert.IsTrue(firstUploader == "latte");
}
public void GetSongsFromFeed_Authors_Test()
{
var reader = new BeatSaverReader()
{
StoreRawData = true
};
var authorList = new string[] { "BlackBlazon", "greatyazer" };
var settings = new BeatSaverFeedSettings((int)BeatSaverFeed.Author)
{
Authors = authorList, MaxSongs = 59
};
var songsByAuthor = reader.GetSongsFromFeed(settings);
var detectedAuthors = songsByAuthor.Values.Select(s => s.MapperName.ToLower()).Distinct();
foreach (var song in songsByAuthor)
{
Assert.IsTrue(song.Value.DownloadUri != null);
Assert.IsTrue(authorList.Any(a => a.ToLower() == song.Value.MapperName.ToLower()));
}
foreach (var author in authorList)
{
Assert.IsTrue(songsByAuthor.Any(s => s.Value.MapperName.ToLower() == author.ToLower()));
}
// BlackBlazon check
var blazonHash = "58de2d709a45b68fdb1dbbfefb187f59f629bfc5".ToUpper();
var blazonSong = songsByAuthor[blazonHash];
Assert.IsTrue(blazonSong != null);
Assert.IsTrue(blazonSong.DownloadUri != null);
// GreatYazer check
var songHash = "bf8c016dc6b9832ece3030f05277bbbe67db790d".ToUpper();
var yazerSong = songsByAuthor[songHash];
Assert.IsTrue(yazerSong != null);
Assert.IsTrue(yazerSong.DownloadUri != null);
}
public void Search_User()
{
var reader = new BeatSaverReader()
{
StoreRawData = true
};
int maxSongs = 10;
string criteria = "19F2879D11A91B51A5C090D63471C3E8D9B7AEE3";
var searchType = BeatSaverSearchType.hash;
var settings = new BeatSaverFeedSettings((int)BeatSaverFeed.Search)
{
MaxSongs = maxSongs, Criteria = criteria, SearchType = searchType
};
var result = reader.GetSongsFromFeed(settings);
Assert.AreEqual(1, result.Count);
int expectedPages = ExpectedPagesForSongs(result.Count);
Assert.AreEqual(expectedPages, result.PagesChecked);
foreach (var song in result.Songs.Values)
{
Console.WriteLine($"{song.SongName} by {song.MapperName}, {song.Hash}");
}
}
public void Success()
{
string path = Path.Combine("Data", "NewBeatSaver", "List", "Latest", "Latest_0.json");
string pageText = File.ReadAllText(path);
Uri uri = null;
var songs = BeatSaverReader.ParseSongsFromPage(pageText, uri, true);
Assert.AreEqual(20, songs.Count);
foreach (var song in songs)
{
Assert.IsFalse(song.DownloadUri == null);
Assert.IsFalse(string.IsNullOrEmpty(song.Hash));
Assert.IsFalse(string.IsNullOrEmpty(song.LevelAuthorName));
Assert.IsFalse(string.IsNullOrEmpty(song.RawData));
Assert.IsFalse(string.IsNullOrEmpty(song.Name));
}
var firstSong = JObject.Parse(songs.First().RawData);
string firstHash = firstSong["versions"].First()["hash"]?.Value <string>();
Assert.AreEqual("e2512e6fbf85059d9fd9b429f62b2e618dd4d7e9", firstHash);
string firstUploader = firstSong["uploader"]?["name"]?.Value <string>();
Assert.AreEqual("itzrimuru", firstUploader);
}
protected async Task <JobStats> GetBeatSaverAsync(BeatSyncConfig config, IJobBuilder jobBuilder, JobManager jobManager, CancellationToken cancellationToken)
{
BeatSaverConfig sourceConfig = config.BeatSaver;
JobStats sourceStats = new JobStats();
if (!sourceConfig.Enabled)
{
return(sourceStats);
}
BeatSaverReader reader = new BeatSaverReader();
FeedConfigBase[] feedConfigs = new FeedConfigBase[] { sourceConfig.Hot, sourceConfig.Downloads, sourceConfig.Latest };
if (!(feedConfigs.Any(f => f.Enabled) || sourceConfig.FavoriteMappers.Enabled))
{
Logger.log?.Info($"No feeds enabled for {reader.Name}");
return(sourceStats);
}
SourceStarted?.Invoke(this, "BeatSaver");
foreach (FeedConfigBase?feedConfig in feedConfigs.Where(c => c.Enabled))
{
Logger.log?.Info($" Starting {feedConfig.GetType().Name} feed...");
FeedResult results = await reader.GetSongsFromFeedAsync(feedConfig.ToFeedSettings(), cancellationToken).ConfigureAwait(false);
if (results.Successful)
{
IEnumerable <IJob>?jobs = CreateJobs(results, jobBuilder, jobManager, cancellationToken);
JobResult[] jobResults = await Task.WhenAll(jobs.Select(j => j.JobTask).ToArray());
JobStats feedStats = new JobStats(jobResults);
ProcessFinishedJobs(jobs, jobBuilder.SongTargets, config, feedConfig);
Logger.log?.Info($" Finished {feedConfig.GetType().Name} feed: ({feedStats}).");
sourceStats += feedStats;
}
else
{
if (results.Exception != null)
{
Logger.log?.Error($" Error getting results from {feedConfig.GetType().Name}{results.Exception.Message}");
Logger.log?.Debug($"{results.Exception}");
}
else
{
Logger.log?.Error($" Error getting results from {feedConfig.GetType().Name}: Unknown error.");
}
}
}
string[] mappers = sourceConfig.FavoriteMappers.Mappers ?? Array.Empty <string>();
if (sourceConfig.FavoriteMappers.Enabled)
{
FeedConfigBase feedConfig = sourceConfig.FavoriteMappers;
if (mappers.Length > 0)
{
Logger.log?.Info(" Starting FavoriteMappers feed...");
List <IPlaylist> playlists = new List <IPlaylist>();
List <IPlaylist> feedPlaylists = new List <IPlaylist>();
List <IPlaylist> recentPlaylists = new List <IPlaylist>();
JobStats feedStats = new JobStats();
foreach (string?mapper in mappers)
{
Logger.log?.Info($" Getting songs by {mapper}...");
playlists.Clear();
FeedResult results = await reader.GetSongsFromFeedAsync(sourceConfig.FavoriteMappers.ToFeedSettings(mapper)).ConfigureAwait(false);
if (results.Successful)
{
foreach (ITargetWithPlaylists?targetWithPlaylist in jobBuilder.SongTargets.Where(t => t is ITargetWithPlaylists).Select(t => (ITargetWithPlaylists)t))
{
PlaylistManager?playlistManager = targetWithPlaylist.PlaylistManager;
if (playlistManager != null)
{
if (config.RecentPlaylistDays > 0)
{
recentPlaylists.Add(playlistManager.GetOrAddPlaylist(BuiltInPlaylist.BeatSyncRecent));
}
if (config.AllBeatSyncSongsPlaylist)
{
playlists.Add(playlistManager.GetOrAddPlaylist(BuiltInPlaylist.BeatSyncAll));
}
if (sourceConfig.FavoriteMappers.CreatePlaylist)
{
IPlaylist feedPlaylist;
try
{
if (sourceConfig.FavoriteMappers.SeparateMapperPlaylists)
{
feedPlaylist = playlistManager.GetOrCreateAuthorPlaylist(mapper);
}
else
{
feedPlaylist = playlistManager.GetOrAddPlaylist(BuiltInPlaylist.BeatSaverFavoriteMappers);
}
feedPlaylists.Add(feedPlaylist);
playlists.Add(feedPlaylist);
}
catch (ArgumentException ex)
{
Logger.log?.Error($"Error getting playlist for FavoriteMappers: {ex.Message}");
Logger.log?.Debug(ex);
}
}
}
}
IEnumerable <IJob> jobs = CreateJobs(results, jobBuilder, jobManager, cancellationToken) ?? Array.Empty <IJob>();
JobResult[] jobResults = await Task.WhenAll(jobs.Select(j => j.JobTask).ToArray());
JobStats mapperStats = new JobStats(jobResults);
feedStats += mapperStats;
if (jobs.Any(j => j.Result?.Successful ?? false) && feedConfig.PlaylistStyle == PlaylistStyle.Replace)
{
// TODO: This should only apply to successful targets.
foreach (IPlaylist?feedPlaylist in feedPlaylists)
{
feedPlaylist.Clear();
feedPlaylist.RaisePlaylistChanged();
}
}
ProcessFinishedJobs(jobs, playlists, recentPlaylists);
Logger.log?.Info($" Finished getting songs by {mapper}: ({mapperStats}).");
}
else
{
if (results.Exception != null)
{
Logger.log?.Error($"Error getting songs by {mapper}: {results.Exception.Message}");
Logger.log?.Debug(results.Exception);
}
else
{
Logger.log?.Error($"Error getting songs by {mapper}");
}
}
}
sourceStats += feedStats;
Logger.log?.Info($" Finished {feedConfig.GetType().Name} feed: ({feedStats}).");
}
else
{
Logger.log?.Warn(" No FavoriteMappers found, skipping...");
}
}
Logger.log?.Info($" Finished BeatSaver reading: ({sourceStats}).");
return(sourceStats);
}
public async Task <Dictionary <string, ScrapedSong> > ReadBeatSaver(Playlist allPlaylist = null)
{
if (BeatSync.Paused)
{
await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false);
}
Stopwatch sw = new Stopwatch();
sw.Start();
Logger.log?.Info("Starting BeatSaver reading");
var config = Config.BeatSaver;
BeatSaverReader reader = null;
try
{
reader = new BeatSaverReader();
}
catch (Exception ex)
{
Logger.log?.Error("Exception creating BeatSaverReader in ReadBeatSaver.");
Logger.log?.Error(ex);
return(null);
}
var readerSongs = new Dictionary <string, ScrapedSong>();
if (config.FavoriteMappers.Enabled && (FavoriteMappers.Mappers?.Count() ?? 0) > 0)
{
try
{
var feedSettings = config.FavoriteMappers.ToFeedSettings() as BeatSaverFeedSettings;
Playlist feedPlaylist = null;
if (!config.FavoriteMappers.SeparateMapperPlaylists)
{
feedPlaylist = config.FavoriteMappers.CreatePlaylist
? PlaylistManager.GetPlaylist(config.FavoriteMappers.FeedPlaylist)
: null;
}
var playlistStyle = config.FavoriteMappers.PlaylistStyle;
var songs = new Dictionary <string, ScrapedSong>();
foreach (var author in FavoriteMappers.Mappers)
{
feedSettings.Criteria = author;
if (BeatSync.Paused)
{
await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false);
}
var authorSongs = await ReadFeed(reader, feedSettings, feedPlaylist, playlistStyle).ConfigureAwait(false);
Logger.log?.Info($" FavoriteMappers: Found {authorSongs.Count} songs by {author}");
if (config.FavoriteMappers.CreatePlaylist && config.FavoriteMappers.SeparateMapperPlaylists)
{
var playlistFileName = $"{author}.bplist";
var mapperPlaylist = PlaylistManager.GetOrAdd(playlistFileName, () => new Playlist(playlistFileName, author, "BeatSync", "1"));
if (mapperPlaylist != null)
{
if (playlistStyle == PlaylistStyle.Replace)
{
mapperPlaylist.Clear();
}
foreach (var song in authorSongs.Values)
{
mapperPlaylist.TryAdd(song.ToPlaylistSong());
}
}
}
songs.Merge(authorSongs);
}
if (feedPlaylist != null)
{
foreach (var song in songs.Values)
{
feedPlaylist.TryAdd(song.ToPlaylistSong());
}
}
var pages = songs.Values.Select(s => s.SourceUri.ToString()).Distinct().Count();
var feedName = reader.GetFeedName(feedSettings);
Logger.log?.Info($"{reader.Name}.{feedName} Feed: Found {songs.Count} songs from {pages} {(pages == 1 ? "page" : "pages")}.");
readerSongs.Merge(songs);
}
catch (InvalidCastException ex)
{
Logger.log?.Error($"This should never happen in ReadBeatSaver.\n{ex.Message}");
}
catch (ArgumentNullException ex)
{
Logger.log?.Critical("Exception in ReadBeatSaver, FavoriteMappers: " + ex.Message);
}
catch (Exception ex)
{
Logger.log?.Error("Exception in ReadBeatSaver, FavoriteMappers.");
Logger.log?.Error(ex);
}
}
else if (config.FavoriteMappers.Enabled)
{
Logger.log?.Warn("BeatSaver's FavoriteMappers feed is enabled, but no mappers could be found in UserData\\FavoriteMappers.ini");
}
if (config.Hot.Enabled)
{
try
{
var feedSettings = config.Hot.ToFeedSettings();
var feedPlaylist = config.Hot.CreatePlaylist
? PlaylistManager.GetPlaylist(config.Hot.FeedPlaylist)
: null;
var playlistStyle = config.Hot.PlaylistStyle;
if (BeatSync.Paused)
{
await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false);
}
var songs = await ReadFeed(reader, feedSettings, feedPlaylist, playlistStyle).ConfigureAwait(false);
var pages = songs.Values.Select(s => s.SourceUri.ToString()).Distinct().Count();
var feedName = reader.GetFeedName(feedSettings);
Logger.log?.Info($"{reader.Name}.{feedName} Feed: Found {songs.Count} songs from {pages} {(pages == 1 ? "page" : "pages")}.");
readerSongs.Merge(songs);
}
catch (InvalidCastException ex)
{
Logger.log?.Error($"This should never happen in ReadBeatSaver.\n{ex.Message}");
}
catch (ArgumentNullException ex)
{
Logger.log?.Critical("Exception in ReadBeatSaver, Hot: " + ex.Message);
}
catch (Exception ex)
{
Logger.log?.Error("Exception in ReadBeatSaver, Hot.");
Logger.log?.Error(ex);
}
}
if (config.Downloads.Enabled)
{
try
{
var feedSettings = config.Downloads.ToFeedSettings();
var feedPlaylist = config.Downloads.CreatePlaylist
? PlaylistManager.GetPlaylist(config.Downloads.FeedPlaylist)
: null;
var playlistStyle = config.Downloads.PlaylistStyle;
if (BeatSync.Paused)
{
await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false);
}
var songs = await ReadFeed(reader, feedSettings, feedPlaylist, playlistStyle).ConfigureAwait(false);
var pages = songs.Values.Select(s => s.SourceUri.ToString()).Distinct().Count();
var feedName = reader.GetFeedName(feedSettings);
Logger.log?.Info($"{reader.Name}.{feedName} Feed: Found {songs.Count} songs from {pages} {(pages == 1 ? "page" : "pages")}.");
readerSongs.Merge(songs);
}
catch (InvalidCastException ex)
{
Logger.log?.Error($"This should never happen in ReadBeatSaver.\n{ex.Message}");
}
catch (ArgumentNullException ex)
{
Logger.log?.Critical("Exception in ReadBeatSaver, Downloads: " + ex.Message);
}
catch (Exception ex)
{
Logger.log?.Error("Exception in ReadBeatSaver, Downloads.");
Logger.log?.Error(ex);
}
}
sw.Stop();
if (BeatSync.Paused)
{
await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false);
}
var totalPages = readerSongs.Values.Select(s => s.SourceUri.ToString()).Distinct().Count();
Logger.log?.Info($"{reader.Name}: Found {readerSongs.Count} songs on {totalPages} {(totalPages == 1 ? "page" : "pages")} in {sw.Elapsed.ToString()}");
return(readerSongs);
}
