private static void DoFullScrape()
{
BeatSaverReader.ScrapeBeatSaver(400, false);
ScrapedDataProvider.BeatSaverSongs.WriteFile();
ScoreSaberReader.ScrapeScoreSaber(5000, 13000, false);
ScrapedDataProvider.ScoreSaberSongs.WriteFile();
}
public void GetSongsFromFeed_Trending()
{
var reader = new ScoreSaberReader();
int maxSongs = 100;
var settings = new ScoreSaberFeedSettings((int)ScoreSaberFeedName.Trending)
{
MaxSongs = maxSongs, SongsPerPage = 40, RankedOnly = true
};
var songList = reader.GetSongsFromFeed(settings);
Assert.IsTrue(songList.Count == maxSongs);
Assert.IsFalse(songList.Songs.Keys.Any(k => string.IsNullOrEmpty(k)));
}
public void GetSongsFromFeed_Search()
{
var reader = new ScoreSaberReader();
int maxSongs = 40;
var settings = new ScoreSaberFeedSettings((int)ScoreSaberFeedName.Search)
{
MaxSongs = maxSongs, SongsPerPage = 40, RankedOnly = true, SearchQuery = "Believer"
};
var songList = reader.GetSongsFromFeed(settings);
Assert.IsTrue(songList.Songs.Values.Any(s => s.Name.ToLower().Contains("believer")));
Assert.IsFalse(songList.Songs.Keys.Any(k => string.IsNullOrEmpty(k)));
}
protected async Task <JobStats> GetScoreSaberAsync(BeatSyncConfig config, IJobBuilder jobBuilder, JobManager jobManager, CancellationToken cancellationToken)
{
ScoreSaberConfig sourceConfig = config.ScoreSaber;
JobStats sourceStats = new JobStats();
if (!sourceConfig.Enabled)
{
return(sourceStats);
}
ScoreSaberReader reader = new ScoreSaberReader();
FeedConfigBase[] feedConfigs = new FeedConfigBase[] { sourceConfig.TopRanked, sourceConfig.LatestRanked, sourceConfig.Trending, sourceConfig.TopPlayed };
if (!feedConfigs.Any(f => f.Enabled))
{
Logger.log?.Info($"No feeds enabled for {reader.Name}");
return(sourceStats);
}
SourceStarted?.Invoke(this, "ScoreSaber");
foreach (FeedConfigBase?feedConfig in feedConfigs.Where(c => c.Enabled))
{
Logger.log?.Info($" Starting {feedConfig.GetType().Name} feed...");
FeedResult results = await reader.GetSongsFromFeedAsync(feedConfig.ToFeedSettings()).ConfigureAwait(false);
if (results.Successful)
{
IEnumerable <IJob>?jobs = CreateJobs(results, jobBuilder, jobManager, cancellationToken);
JobResult[] jobResults = await Task.WhenAll(jobs.Select(j => j.JobTask).ToArray());
JobStats feedStats = new JobStats(jobResults);
ProcessFinishedJobs(jobs, jobBuilder.SongTargets, config, feedConfig);
Logger.log?.Info($" Finished {feedConfig.GetType().Name} feed: ({feedStats}).");
sourceStats += feedStats;
}
else
{
if (results.Exception != null)
{
Logger.log?.Error($" Error getting results from {feedConfig.GetType().Name}: {results.Exception.Message}");
Logger.log?.Debug(results.Exception);
}
else
{
Logger.log?.Error($" Error getting results from {feedConfig.GetType().Name}: Unknown error.");
}
}
}
Logger.log?.Info($" Finished ScoreSaber reading: ({sourceStats}).");
return(sourceStats);
}
public void GetSongsFromFeed_TopRanked()
{
var reader = new ScoreSaberReader();
int maxSongs = 0;
var settings = new ScoreSaberFeedSettings((int)ScoreSaberFeedName.TopRanked)
{
MaxSongs = maxSongs, SongsPerPage = 40, RankedOnly = true
};
var songList = reader.GetSongsFromFeed(settings);
Console.WriteLine($"{songList.Count} songs.");
Assert.IsTrue(songList.Count >= 367);
Assert.IsFalse(songList.Songs.Keys.Any(k => string.IsNullOrEmpty(k)));
}
public void GetSongsFromFeed_LatestRanked()
{
var reader = new ScoreSaberReader();
int maxSongs = 50;
var settings = new ScoreSaberFeedSettings((int)ScoreSaberFeedName.LatestRanked)
{
MaxSongs = maxSongs, SongsPerPage = 40, RankedOnly = true
};
var result = reader.GetSongsFromFeed(settings);
Assert.IsTrue(result.Count == maxSongs);
Assert.IsFalse(result.Songs.Keys.Any(k => string.IsNullOrEmpty(k)));
Assert.IsTrue(1 <= result.PagesChecked);
}
public void CancelledInProgress()
{
var reader = new ScoreSaberReader();
var cts = new CancellationTokenSource(500);
int maxSongs = 50;
var settings = new ScoreSaberFeedSettings((int)ScoreSaberFeedName.LatestRanked)
{
MaxSongs = maxSongs, SongsPerPage = 10, RankedOnly = true
};
var result = reader.GetSongsFromFeed(settings, cts.Token);
Assert.IsFalse(result.Successful);
Assert.AreEqual(FeedResultError.Cancelled, result.ErrorCode);
cts.Dispose();
}
public void GetSongsFromPageText_Url()
{
var reader = new ScoreSaberReader()
{
StoreRawData = true
};
var pageText = File.ReadAllText("Data\\ScoreSaberPage.json");
string url = Path.GetFullPath("Data\\ScoreSaberPage.json");
var songList = ScoreSaberFeed.GetSongsFromPageText(pageText, new Uri(url), true);
Assert.IsTrue(songList.Count == 50);
var firstHash = "0597F8F7D8E396EBFEF511DC9EC98B69635CE532";
Assert.IsTrue(songList.First().Hash == firstHash);
var firstRawData = JToken.Parse(songList.First().RawData);
Assert.IsTrue(firstRawData["uid"]?.Value <int>() == 143199);
var lastHash = "F369747C6B54914DEAA163AAE85816BA5A8C1845";
Assert.IsTrue(songList.Last().Hash == lastHash);
}
public void GetSongsFromPageText()
{
var reader = new ScoreSaberReader()
{
StoreRawData = true
};
var pageText = File.ReadAllText("Data\\ScoreSaberPage.json");
Uri sourceUri = null;
var songList = reader.GetSongsFromPageText(pageText, sourceUri);
Assert.IsTrue(songList.Count == 50);
var firstHash = "0597F8F7D8E396EBFEF511DC9EC98B69635CE532";
Assert.IsTrue(songList.Songs.First().Hash == firstHash);
var firstRawData = JToken.Parse(songList.Songs.First().RawData);
Assert.IsTrue(firstRawData["uid"]?.Value <int>() == 143199);
var lastHash = "F369747C6B54914DEAA163AAE85816BA5A8C1845";
Assert.IsTrue(songList.Songs.Last().Hash == lastHash);
}
private static void Tests()
{
var thing = new ScrappedSong();
Web.HttpClientWrapper.Initialize(5);
List <ScrappedSong> scrapedDict;
using (StreamReader file = File.OpenText(@"C:\Users\Jared\source\repos\SyncSaberService\SyncSaberService\bin\Debug\ScrapedData\combinedScrappedData.json"))
{
JsonSerializer serializer = new JsonSerializer();
scrapedDict = (List <ScrappedSong>)serializer.Deserialize(file, typeof(List <ScrappedSong>));
}
DownloadJob testJob = new DownloadJob(new SongInfo("111-111", "testName", "", "testAuthor"), "temp", "CustomSongs");
var testTask = testJob.RunJobAsync();
testTask.Wait();
var searchTest = BeatSaverReader.Search("6A097D39A5FA94F3B736E6EEF5A519A2", BeatSaverReader.SearchType.hash);
var testReader = new ScoreSaberReader();
var sssongs = testReader.GetSSSongsFromPage(HttpClientWrapper.GetPageText("https://scoresaber.com/api.php?function=get-leaderboards&cat=3&limit=5&page=39&ranked=1"));
foreach (var sssong in sssongs)
{
sssong.PopulateFields();
}
var songs = testReader.GetSongsFromFeed(new ScoreSaberFeedSettings(0)
{
MaxPages = 10
});
SongInfo song = new SongInfo("18750-20381", "test", "testUrl", "testAuthor");
song.PopulateFields();
var test = song["key"];
var test2 = song["id"];
var test3 = song["uploaderId"];
}
public void ScrapeNewSongs()
{
int lastBeatSaverCount = 0;
int lastScoreSaberCount = 0;
DateTime lastBSScrape = DateTime.MinValue;
DateTime lastSSScrape = DateTime.MinValue;
if (ScrapedDataProvider.BeatSaverSongs.HasData)
{
lastBeatSaverCount = ScrapedDataProvider.BeatSaverSongs.Data.Count;
lastBSScrape = ScrapedDataProvider.BeatSaverSongs.Data.Max(s => s.ScrapedAt);
}
if (ScrapedDataProvider.ScoreSaberSongs.HasData)
{
lastScoreSaberCount = ScrapedDataProvider.ScoreSaberSongs.Data.Count;
lastSSScrape = ScrapedDataProvider.ScoreSaberSongs.Data.Max(s => s.ScrapedAt);
}
Logger.Info($"Scraping new songs. Last Beat Saver scrape was at {lastBSScrape.ToString()}.");
var bsReader = FeedReaders[BeatSaverReader.NameKey] as BeatSaverReader;
BeatSaverReader.ScrapeBeatSaver(200, true);
ScrapedDataProvider.BeatSaverSongs.WriteFile();
int newBeatSaverSongs = ScrapedDataProvider.BeatSaverSongs.Data.Count - lastBeatSaverCount;
Logger.Info($"Scraped {(newBeatSaverSongs).ToString()} new song{(newBeatSaverSongs == 1 ? "" : "s")} from Beat Saver.");
if ((DateTime.Now - lastSSScrape).TotalHours > 3)
{
Logger.Info($"Scraping new ScoreSaber difficulties. Last ScoreSaber scrape was at {lastSSScrape.ToString()}.");
int newScoreSaberSongs = ScrapedDataProvider.ScoreSaberSongs.Data.Count - lastScoreSaberCount;
ScoreSaberReader.ScrapeScoreSaber(1000, 500, true, 2);
Logger.Info($"Scraped {(newScoreSaberSongs).ToString()} new difficult{(newScoreSaberSongs == 1 ? "y" : "ies")} from ScoreSaber.");
ScrapedDataProvider.ScoreSaberSongs.WriteFile();
}
else
{
Logger.Info($"Last ScoreSaber scrape was at {lastSSScrape.ToString()}, skipping.");
}
}
public async Task <Dictionary <string, ScrapedSong> > ReadScoreSaber(Playlist allPlaylist = null)
{
if (BeatSync.Paused)
{
await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false);
}
Stopwatch sw = new Stopwatch();
sw.Start();
Logger.log?.Info("Starting ScoreSaber reading");
var config = Config.ScoreSaber;
ScoreSaberReader reader = null;
try
{
reader = new ScoreSaberReader();
}
catch (Exception ex)
{
Logger.log?.Error(ex);
return(null);
}
var readerSongs = new Dictionary <string, ScrapedSong>();
if (config.TopRanked.Enabled)
{
try
{
var feedSettings = config.TopRanked.ToFeedSettings();
var feedPlaylist = config.TopRanked.CreatePlaylist
? PlaylistManager.GetPlaylist(config.TopRanked.FeedPlaylist)
: null;
var playlistStyle = config.TopRanked.PlaylistStyle;
if (BeatSync.Paused)
{
await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false);
}
var songs = await ReadFeed(reader, feedSettings, feedPlaylist, playlistStyle).ConfigureAwait(false);
var pages = songs.Values.Select(s => s.SourceUri.ToString()).Distinct().Count();
var feedName = reader.GetFeedName(feedSettings);
Logger.log?.Info($"{reader.Name}.{feedName} Feed: Found {songs.Count} songs from {pages} {(pages == 1 ? "page" : "pages")}.");
readerSongs.Merge(songs);
}
catch (Exception ex)
{
Logger.log?.Error("Exception in ReadScoreSaber, Top Ranked.");
Logger.log?.Error(ex);
}
}
if (config.Trending.Enabled)
{
try
{
var feedSettings = config.Trending.ToFeedSettings();
var feedPlaylist = config.Trending.CreatePlaylist
? PlaylistManager.GetPlaylist(config.Trending.FeedPlaylist)
: null;
var playlistStyle = config.Trending.PlaylistStyle;
if (BeatSync.Paused)
{
await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false);
}
var songs = await ReadFeed(reader, feedSettings, feedPlaylist, playlistStyle).ConfigureAwait(false);
var pages = songs.Values.Select(s => s.SourceUri.ToString()).Distinct().Count();
var feedName = reader.GetFeedName(feedSettings);
Logger.log?.Info($"{reader.Name}.{feedName} Feed: Found {songs.Count} songs from {pages} {(pages == 1 ? "page" : "pages")}.");
readerSongs.Merge(songs);
}
catch (Exception ex)
{
Logger.log?.Error("Exception in ReadScoreSaber, Trending.");
Logger.log?.Error(ex);
}
}
if (config.TopPlayed.Enabled)
{
try
{
var feedSettings = config.TopPlayed.ToFeedSettings();
var feedPlaylist = config.TopPlayed.CreatePlaylist
? PlaylistManager.GetPlaylist(config.TopPlayed.FeedPlaylist)
: null;
var playlistStyle = config.TopPlayed.PlaylistStyle;
if (BeatSync.Paused)
{
await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false);
}
var songs = await ReadFeed(reader, feedSettings, feedPlaylist, playlistStyle).ConfigureAwait(false);
var pages = songs.Values.Select(s => s.SourceUri.ToString()).Distinct().Count();
var feedName = reader.GetFeedName(feedSettings);
Logger.log?.Info($"{reader.Name}.{feedName} Feed: Found {songs.Count} songs from {pages} {(pages == 1 ? "page" : "pages")}.");
readerSongs.Merge(songs);
}
catch (Exception ex)
{
Logger.log?.Error("Exception in ReadScoreSaber, Top Played.");
Logger.log?.Error(ex);
}
}
if (config.LatestRanked.Enabled)
{
try
{
var feedSettings = config.LatestRanked.ToFeedSettings();
var feedPlaylist = config.LatestRanked.CreatePlaylist
? PlaylistManager.GetPlaylist(config.LatestRanked.FeedPlaylist)
: null;
var playlistStyle = config.LatestRanked.PlaylistStyle;
if (BeatSync.Paused)
{
await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false);
}
var songs = await ReadFeed(reader, feedSettings, feedPlaylist, playlistStyle).ConfigureAwait(false);
var pages = songs.Values.Select(s => s.SourceUri.ToString()).Distinct().Count();
var feedName = reader.GetFeedName(feedSettings);
Logger.log?.Info($"{reader.Name}.{feedName} Feed: Found {songs.Count} songs from {pages} {(pages == 1 ? "page" : "pages")}.");
readerSongs.Merge(songs);
}
catch (Exception ex)
{
Logger.log?.Error("Exception in ReadScoreSaber, Latest Ranked.");
Logger.log?.Error(ex);
}
}
sw.Stop();
if (BeatSync.Paused)
{
await SongFeedReaders.Utilities.WaitUntil(() => !BeatSync.Paused, 500).ConfigureAwait(false);
}
var totalPages = readerSongs.Values.Select(s => s.SourceUri.ToString()).Distinct().Count();
Logger.log?.Info($"{reader.Name}: Found {readerSongs.Count} songs on {totalPages} {(totalPages == 1 ? "page" : "pages")} in {sw.Elapsed.ToString()}");
return(readerSongs);
}