public void CuratorRecommended_LimitedSongs()
{
var reader = new BeastSaberReader("Zingabopp", DefaultMaxConcurrency) { StoreRawData = true };
int maxSongs = 60;
var settings = new BeastSaberFeedSettings((int)BeastSaberFeedName.CuratorRecommended) { MaxSongs = maxSongs };
var result = reader.GetSongsFromFeedAsync(settings).Result;
Assert.IsTrue(result.Count == maxSongs);
Assert.IsFalse(result.Songs.Any(s => string.IsNullOrEmpty(s.Key)));
Assert.IsFalse(result.Songs.Any(s => s.Value.DownloadUri == null));
try
{
Assert.AreEqual(2, result.PagesChecked);
}
catch (AssertFailedException)
{
for(int i = 0; i < result.PageResults.Count(); i++)
{
var page = result.PageResults[i];
Console.WriteLine($"Page {i + 1}: {page.Count} songs. {page.Uri}");
}
Assert.AreEqual(3, result.PagesChecked);
}
var firstSong = result.Songs.First().Value;
Assert.IsFalse(string.IsNullOrEmpty(firstSong.RawData));
var firstRawData = JToken.Parse(firstSong.RawData);
Assert.IsTrue(firstRawData["hash"]?.Value<string>().ToUpper() == firstSong.Hash);
}
public void Bookmarks_UnlimitedSongs()
{
var reader = new BeastSaberReader("Zingabopp", DefaultMaxConcurrency);
int maxSongs = 0;
var settings = new BeastSaberFeedSettings((int)BeastSaberFeedName.Bookmarks) { MaxSongs = maxSongs };
var songList = reader.GetSongsFromFeedAsync(settings).Result;
Assert.IsTrue(songList.Count > 0);
Assert.IsFalse(songList.Songs.Any(s => string.IsNullOrEmpty(s.Key)));
}
public void Followings_LimitedSongs()
{
var reader = new BeastSaberReader("Zingabopp", DefaultMaxConcurrency) { StoreRawData = true };
int maxSongs = 60;
var settings = new BeastSaberFeedSettings((int)BeastSaberFeedName.Following) { MaxSongs = maxSongs };
var result = reader.GetSongsFromFeedAsync(settings).Result;
Assert.IsTrue(result.Count == maxSongs);
//Assert.IsFalse(songList.Count > maxSongs);
Assert.IsFalse(result.Songs.Any(s => string.IsNullOrEmpty(s.Key)));
}
protected async Task <JobStats> GetBeastSaberAsync(BeatSyncConfig config, IJobBuilder jobBuilder, JobManager jobManager, CancellationToken cancellationToken)
{
BeastSaberConfig sourceConfig = config.BeastSaber;
JobStats sourceStats = new JobStats();
if (!sourceConfig.Enabled)
{
return(sourceStats);
}
BeastSaberReader reader = new BeastSaberReader(sourceConfig.Username, sourceConfig.MaxConcurrentPageChecks);
FeedConfigBase[] feedConfigs = new FeedConfigBase[] { sourceConfig.Bookmarks, sourceConfig.Follows, sourceConfig.CuratorRecommended };
if (!feedConfigs.Any(f => f.Enabled))
{
Logger.log?.Info($"No feeds enabled for {reader.Name}");
return(sourceStats);
}
SourceStarted?.Invoke(this, "BeastSaber");
foreach (FeedConfigBase?feedConfig in feedConfigs.Where(c => c.Enabled))
{
//if (string.IsNullOrEmpty(sourceConfig.Username) && feedConfig.GetType() != typeof(BeastSaberCuratorRecommended))
//{
// Logger.log?.Warn($" {feedConfig.GetType().Name} feed not available without a valid username.");
// continue;
//}
Logger.log?.Info($" Starting {feedConfig.GetType().Name} feed...");
FeedResult results = await reader.GetSongsFromFeedAsync(feedConfig.ToFeedSettings()).ConfigureAwait(false);
if (results.Successful)
{
IEnumerable <IJob>?jobs = CreateJobs(results, jobBuilder, jobManager, cancellationToken);
JobResult[] jobResults = await Task.WhenAll(jobs.Select(j => j.JobTask).ToArray());
JobStats feedStats = new JobStats(jobResults);
ProcessFinishedJobs(jobs, jobBuilder.SongTargets, config, feedConfig);
Logger.log?.Info($" Finished {feedConfig.GetType().Name} feed: ({feedStats}).");
sourceStats += feedStats;
}
else
{
if (results.Exception != null)
{
Logger.log?.Error($" Error getting results from {feedConfig.GetType().Name}: {results.Exception.Message}");
Logger.log?.Debug(results.Exception);
}
else
{
Logger.log?.Error($" Error getting results from {feedConfig.GetType().Name}: Unknown error.");
}
}
}
Logger.log?.Info($" Finished BeastSaber reading: ({sourceStats}).");
return(sourceStats);
}
private static void Tests()
{
//ScrapedDataProvider.Initialize();
WebUtils.Initialize(5);
ScrapedDataProvider.BeatSaverSongs.WriteFile();
var bsReader = new BeatSaverReader();
//var bsSongs = bsReader.GetSongsFromFeed(new BeatSaverFeedSettings((int)BeatSaverFeeds.LATEST) { MaxPages = 10, searchOnline = true });
var authorSongs = BeatSaverReader.GetSongsByAuthor("ruckasdfus");
var song = ScrapedDataProvider.Songs.Values.Where(s => s.ScoreSaberInfo.Any(d => d.Value.uid == 155732)).FirstOrDefault();
var found = ScrapedDataProvider.TryGetSongByKey("3f57", out SongInfo badSong, false);
var CustomSongsPath = Path.Combine(OldConfig.BeatSaberPath, "Beat Saber_Data", "CustomLevels");
var tempFolder = new DirectoryInfo(Path.Combine(Path.GetTempPath(), badSong.key + ".zip"));
var outputFolder = new DirectoryInfo(Path.Combine(CustomSongsPath, $"{badSong.key} ({Utilities.MakeSafeFilename(badSong.songName)} - {Utilities.MakeSafeFilename(badSong.authorName)})"));
var job = new DownloadJob(badSong, tempFolder.FullName, outputFolder.FullName);
job.RunJobAsync().Wait();
var br = new BeastSaberReader("Zingabopp", 3);
var text = WebUtils.GetPageText("https://bsaber.com/wp-json/bsaber-api/songs/?bookmarked_by=Zingabopp&page=1");
var bSongs = br.GetSongsFromPage(text);
//ScrapedDataProvider.Initialize();
//ScrapedDataProvider.Initialize();
var bsScrape = ScrapedDataProvider.BeatSaverSongs;
var ssScrape = ScrapedDataProvider.ScoreSaberSongs;
ScrapedDataProvider.TryGetSongByHash("501f6b1bddb2af72abda0f1e6b7b89cb1eb3db67", out SongInfo deletedSong);
//var job = new DownloadJob(deletedSong, "test.zip", @"ScrapedData\test");
//var jobTask = job.RunJobAsync();
//jobTask.Wait();
bsScrape.AddOrUpdate(null);
var resp = WebUtils.HttpClient.GetAsync("https://beatsaver.com/api/maps/detail/b");
Task.WaitAll(resp);
var rateHeaders = resp.Result.Headers.Where(h => h.Key.StartsWith("Rate-Limit")).ToDictionary(x => x.Key, x => x.Value.FirstOrDefault());
var remoteTime = resp.Result.Headers.Date;
var rateInfo = WebUtils.ParseRateLimit(rateHeaders);
Console.WriteLine("Reset Timespan: " + rateInfo.TimeToReset.ToString());
foreach (var item in resp.Result.Headers)
{
Console.WriteLine($"{item.Key}: {string.Join("|", item.Value)}");
}
var trending = ScrapedDataProvider.Songs.Values.Where(s => s.ScoreSaberInfo.Count > 0).OrderByDescending(s => s.ScoreSaberInfo.Values.Select(ss => ss.scores).Aggregate((a, b) => a + b)).Take(100);
var detTrending = trending.Select(s => (s.ScoreSaberInfo.Values.Select(ss => ss.scores).Aggregate((a, b) => a + b), s)).ToList();
}
public void GetSongsFromFeed_Bookmarks()
{
var reader = new BeastSaberReader("Zingabopp", DefaultMaxConcurrency);
int maxSongs = 60;
var settings = new BeastSaberFeedSettings((int)BeastSaberFeed.Bookmarks)
{
MaxSongs = maxSongs
};
var songList = reader.GetSongsFromFeed(settings);
Assert.IsTrue(songList.Count > 0);
Assert.IsFalse(songList.Count > maxSongs);
Assert.IsFalse(songList.Any(s => string.IsNullOrEmpty(s.Key)));
}
public void CuratorRecommended_UnlimitedSongs()
{
var reader = new BeastSaberReader("Zingabopp", DefaultMaxConcurrency) { StoreRawData = true };
int maxSongs = 0;
var settings = new BeastSaberFeedSettings((int)BeastSaberFeedName.CuratorRecommended) { MaxSongs = maxSongs };
var result = reader.GetSongsFromFeedAsync(settings).Result;
Assert.IsTrue(result.Count != 0);
Assert.IsFalse(result.Songs.Any(s => string.IsNullOrEmpty(s.Key)));
Assert.IsFalse(result.Songs.Any(s => s.Value.DownloadUri == null));
Assert.IsTrue(result.PagesChecked >= 26);
var firstSong = result.Songs.First().Value;
var firstRawData = JToken.Parse(firstSong.RawData);
Assert.IsTrue(firstRawData["hash"]?.Value<string>().ToUpper() == firstSong.Hash);
}
public void CancelledInProgress()
{
var cts = new CancellationTokenSource(500);
IFeedReader reader = new BeastSaberReader("Zingabopp", 1);
int maxSongs = 300;
var settings = new BeastSaberFeedSettings((int)BeastSaberFeedName.CuratorRecommended)
{
MaxSongs = maxSongs
};
var result = reader.GetSongsFromFeed(settings, cts.Token);
Assert.IsFalse(result.Successful);
Assert.AreEqual(FeedResultError.Cancelled, result.ErrorCode);
cts.Dispose();
}
public void GetSongsFromFeed_CuratorRecommended_LimitedPages()
{
var reader = new BeastSaberReader("Zingabopp", DefaultMaxConcurrency);
int maxSongs = 30;
int maxPages = 2;
var settings = new BeastSaberFeedSettings((int)BeastSaberFeed.CuratorRecommended)
{
MaxPages = maxPages, MaxSongs = maxSongs
};
var songList = reader.GetSongsFromFeed(settings);
Assert.IsTrue(songList.Count == maxSongs);
Assert.IsFalse(songList.Any(s => string.IsNullOrEmpty(s.Key)));
Assert.IsFalse(songList.Any(s => s.Value.DownloadUri == null));
}
public void GetSongsFromFeed_Followings_SinglePage()
{
var reader = new BeastSaberReader("Zingabopp", DefaultMaxConcurrency);
int maxSongs = 150;
int maxPages = 1;
var settings = new BeastSaberFeedSettings((int)BeastSaberFeed.Following)
{
MaxSongs = maxSongs,
MaxPages = maxPages
};
var songList = reader.GetSongsFromFeed(settings);
Assert.IsTrue(songList.Count > 0);
Assert.IsTrue(songList.Count <= 100);
//Assert.IsFalse(songList.Count > maxSongs);
Assert.IsFalse(songList.Any(s => string.IsNullOrEmpty(s.Key)));
}
public void Success_JSON()
{
var reader = new BeastSaberReader("Zingabopp", DefaultMaxConcurrency);
var text = File.ReadAllText("Data\\BeastSaberJsonPage.json");
Uri uri = null;
var songList = reader.GetSongsFromPageText(text, uri, BeastSaberReader.ContentType.JSON);
Assert.IsTrue(songList.Count == 20);
var firstHash = "a3bbbe2d6f64dfe8324c7098d5c35281d21fd20f".ToUpper();
var firstUrl = "https://beatsaver.com/api/download/key/5679";
Assert.IsTrue(songList.First().Hash == firstHash);
Assert.IsTrue(songList.First().DownloadUri.ToString() == firstUrl);
var lastHash = "20b9326bd71db4454aba08df06b035ea536322a9".ToUpper();
var lastUrl = "https://beatsaver.com/api/download/key/55d1";
Assert.IsTrue(songList.Last().Hash == lastHash);
Assert.IsTrue(songList.Last().DownloadUri.ToString() == lastUrl);
Assert.IsFalse(songList.Any(s => string.IsNullOrEmpty(s.Hash)));
Assert.IsFalse(songList.Any(s => s.DownloadUri == null));
}
public void Success_XML()
{
var reader = new BeastSaberReader("Zingabopp", DefaultMaxConcurrency)
{
StoreRawData = true
};
var text = File.ReadAllText("Data\\BeastSaberXMLPage.xml");
Uri uri = null;
var songList = reader.GetSongsFromPageText(text, uri, BeastSaberReader.ContentType.XML);
Assert.IsTrue(songList.Count == 50);
var firstHash = "74575254ae759f3f836eb521b4b80093ca52cd3d".ToUpper();
var firstKey = "56ff";
var firstLevelAuthor = "Rustic";
var firstTitle = "Xilent – Code Blood";
var firstDownloadUrl = "https://beatsaver.com/api/download/key/56ff";
var firstUrl = "https://beatsaver.com/api/download/key/56ff";
var firstSong = songList.First();
Assert.IsTrue(firstSong.Hash == firstHash);
Assert.IsTrue(firstSong.DownloadUri.ToString() == firstUrl);
// Raw Data test
JToken firstRawData = JToken.Parse(firstSong.RawData);
Assert.IsTrue(firstRawData["Hash"]?.Value <string>().ToUpper() == firstHash);
Assert.IsTrue(firstRawData["SongKey"]?.Value <string>() == firstKey);
Assert.IsTrue(firstRawData["LevelAuthorName"]?.Value <string>() == firstLevelAuthor);
Assert.IsTrue(firstRawData["SongTitle"]?.Value <string>() == firstTitle);
Assert.IsTrue(firstRawData["DownloadURL"]?.Value <string>() == firstDownloadUrl);
var lastHash = "e3487474b70d969927e459a1590e93b7ad25a436".ToUpper();
var lastUrl = "https://beatsaver.com/api/download/key/5585";
Assert.IsTrue(songList.Last().Hash == lastHash);
Assert.IsTrue(songList.Last().DownloadUri.ToString() == lastUrl);
Assert.IsFalse(songList.Any(s => string.IsNullOrEmpty(s.Hash)));
Assert.IsFalse(songList.Any(s => s.DownloadUri == null));
}
public void GetSongsFromFeed_Followings_SinglePage_LimitedSongs()
{
var reader = new BeastSaberReader("Zingabopp", DefaultMaxConcurrency)
{
StoreRawData = true
};
int maxSongs = 20;
int maxPages = 1;
var settings = new BeastSaberFeedSettings((int)BeastSaberFeed.Following)
{
MaxSongs = maxSongs,
MaxPages = maxPages
};
var songList = reader.GetSongsFromFeed(settings);
var pagesChecked = songList.Values.GroupBy(s => s.SourceUri);
Assert.IsTrue(pagesChecked.Count() == 1);
Assert.IsTrue(songList.Count > 0);
Assert.IsTrue(songList.Count <= 20);
//Assert.IsFalse(songList.Count > maxSongs);
Assert.IsFalse(songList.Any(s => string.IsNullOrEmpty(s.Key)));
}
public void BeastSaberBookmarks()
{
CancellationTokenSource cts = new CancellationTokenSource();
var config = DefaultConfig;
var sourceConfig = DefaultConfig.BeastSaber;
sourceConfig.MaxConcurrentPageChecks = 5;
var feedConfig = sourceConfig.Bookmarks;
feedConfig.MaxSongs = 60;
var songDownloader = new SongDownloader(config, null, null, SONGSPATH);
var reader = new BeastSaberReader(sourceConfig.Username, sourceConfig.MaxConcurrentPageChecks);
var settings = feedConfig.ToFeedSettings();
var result = songDownloader.ReadFeed(reader, settings, 0, null, PlaylistStyle.Append, cts.Token).Result;
if (!result.Successful)
{
Console.WriteLine(result.FaultedResults.First().Exception);
}
Assert.IsNotNull(result);
Assert.IsTrue(result.Count > 0);
}
public async Task <Dictionary <string, ScrapedSong> > ReadBeastSaber()
{
if (BeatSync.Paused)
{
await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false);
}
Stopwatch sw = new Stopwatch();
sw.Start();
Logger.log?.Info("Starting BeastSaber reading");
var config = Config.BeastSaber;
BeastSaberReader reader = null;
try
{
reader = new BeastSaberReader(config.Username, config.MaxConcurrentPageChecks);
}
catch (Exception ex)
{
Logger.log?.Error(ex);
return(null);
}
var readerSongs = new Dictionary <string, ScrapedSong>();
if (config.Bookmarks.Enabled && !string.IsNullOrEmpty(config.Username))
{
try
{
var feedSettings = config.Bookmarks.ToFeedSettings();
var feedPlaylist = config.Bookmarks.CreatePlaylist
? PlaylistManager.GetPlaylist(config.Bookmarks.FeedPlaylist)
: null;
var playlistStyle = config.Bookmarks.PlaylistStyle;
if (BeatSync.Paused)
{
await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false);
}
var songs = await ReadFeed(reader, feedSettings, feedPlaylist, playlistStyle).ConfigureAwait(false);
var pages = songs.Values.Select(s => s.SourceUri.ToString()).Distinct().Count();
var feedName = reader.GetFeedName(feedSettings);
Logger.log?.Info($"{reader.Name}.{feedName} Feed: Found {songs.Count} songs from {pages} {(pages == 1 ? "page" : "pages")}.");
readerSongs.Merge(songs);
}
catch (ArgumentException ex)
{
Logger.log?.Critical("Exception in BeastSaber Bookmarks: " + ex.Message);
}
catch (Exception ex)
{
Logger.log?.Error("Exception in ReadBeastSaber, Bookmarks.");
Logger.log?.Error(ex);
}
}
else if (string.IsNullOrEmpty(config.Username))
{
Logger.log?.Warn("BeastSaber Bookmarks feed is enabled, but a username has not been provided.");
}
if (config.Follows.Enabled && !string.IsNullOrEmpty(config.Username))
{
try
{
var feedSettings = config.Follows.ToFeedSettings();
var feedPlaylist = config.Follows.CreatePlaylist
? PlaylistManager.GetPlaylist(config.Follows.FeedPlaylist)
: null;
var playlistStyle = config.Follows.PlaylistStyle;
if (BeatSync.Paused)
{
await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false);
}
var songs = await ReadFeed(reader, feedSettings, feedPlaylist, playlistStyle).ConfigureAwait(false);
var pages = songs.Values.Select(s => s.SourceUri.ToString()).Distinct().Count();
var feedName = reader.GetFeedName(feedSettings);
Logger.log?.Info($"{reader.Name}.{feedName} Feed: Found {songs.Count} songs from {pages} {(pages == 1 ? "page" : "pages")}.");
readerSongs.Merge(songs);
}
catch (ArgumentException ex)
{
Logger.log?.Critical("Exception in BeastSaber Follows: " + ex.Message);
}
catch (Exception ex)
{
Logger.log?.Error("Exception in ReadBeastSaber, Follows.");
Logger.log?.Error(ex);
}
}
else if (string.IsNullOrEmpty(config.Username))
{
Logger.log?.Warn("BeastSaber Follows feed is enabled, but a username has not been provided.");
}
if (config.CuratorRecommended.Enabled)
{
try
{
var feedSettings = config.CuratorRecommended.ToFeedSettings();
var feedPlaylist = config.CuratorRecommended.CreatePlaylist
? PlaylistManager.GetPlaylist(config.CuratorRecommended.FeedPlaylist)
: null;
var playlistStyle = config.CuratorRecommended.PlaylistStyle;
if (BeatSync.Paused)
{
await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false);
}
var songs = await ReadFeed(reader, feedSettings, feedPlaylist, playlistStyle).ConfigureAwait(false);
var pages = songs.Values.Select(s => s.SourceUri.ToString()).Distinct().Count();
var feedName = reader.GetFeedName(feedSettings);
Logger.log?.Info($"{reader.Name}.{feedName} Feed: Found {songs.Count} songs from {pages} {(pages == 1 ? "page" : "pages")}.");
readerSongs.Merge(songs);
}
catch (Exception ex)
{
Logger.log?.Error("Exception in ReadBeastSaber, Curator Recommended.");
Logger.log?.Error(ex);
}
}
if (BeatSync.Paused)
{
await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false);
}
sw.Stop();
var totalPages = readerSongs.Values.Select(s => s.SourceUri.ToString()).Distinct().Count();
Logger.log?.Info($"{reader.Name}: Found {readerSongs.Count} songs on {totalPages} {(totalPages == 1 ? "page" : "pages")} in {sw.Elapsed.ToString()}");
return(readerSongs);
}
