public static async Task <List <Song> > GetSongsFromPageAsync(string url, bool useDateLimit = false)
{
string pageText = string.Empty;
List <Song> songs = new List <Song>();;
try
{
using (var response = await WebClient.GetAsync(url).ConfigureAwait(false))
{
pageText = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
}
Logger.Debug($"Successful got pageText from {url}");
foreach (var song in ParseSongsFromPage(pageText))
{
songs.Add(ScrapedDataProvider.GetOrCreateSong(song));
}
}
catch (Exception)
{
Logger.Error($"Error getting page text from {url}");
}
return(songs);
}
public static SongInfo GetSongByHash(string hash)
{
string url = BEATSAVER_GETBYHASH_BASE_URL + hash.ToLowerInvariant();
string pageText = "";
BeatSaverSong song;
try
{
var pageTask = WebUtils.TryGetStringAsync(url);
pageTask.Wait();
pageText = pageTask.Result;
if (string.IsNullOrEmpty(pageText))
{
Logger.Warning($"Unable to get web page at {url}");
return(null);
}
}
catch (HttpRequestException)
{
Logger.Error($"HttpRequestException while trying to populate fields for {hash}");
return(null);
}
catch (AggregateException ae)
{
ae.WriteExceptions($"Exception while trying to get details for {hash}");
}
catch (Exception ex)
{
Logger.Exception("Exception getting page", ex);
}
song = ParseSongsFromPage(pageText).FirstOrDefault();
song.ScrapedAt = DateTime.Now;
return(ScrapedDataProvider.GetOrCreateSong(song));
}
/// <summary>
/// Creates a SongInfo from a JObject. Sets the ScrapedAt time for the song.
/// </summary>
/// <param name="song"></param>
/// <returns></returns>
public static BeatSaverSong ParseSongFromJson(JObject song)
{
//JSONObject song = (JSONObject) aKeyValue;
string songIndex = song["key"]?.Value <string>();
string songName = song["name"]?.Value <string>();
string author = song["uploader"]?["username"]?.Value <string>();
string songUrl = "https://beatsaver.com/download/" + songIndex;
if (BeatSaverSong.TryParseBeatSaver(song, out BeatSaverSong newSong))
{
newSong.ScrapedAt = DateTime.Now;
SongInfo songInfo = ScrapedDataProvider.GetOrCreateSong(newSong);
songInfo.BeatSaverInfo = newSong;
return(newSong);
}
else
{
if (!(string.IsNullOrEmpty(songIndex)))
{
// TODO: look at this
Logger.Warning($"Couldn't parse song {songIndex}, skipping.");// using sparse definition.");
//return new SongInfo(songIndex, songName, songUrl, author);
}
else
{
Logger.Error("Unable to identify song, skipping");
}
}
return(null);
}
public static async Task <Song> GetSongByHashAsync(string hash, CancellationToken cancellationToken)
{
Uri uri = new Uri(BEATSAVER_GETBYHASH_BASE_URL + hash);
string pageText = "";
try
{
using (var response = await WebUtils.GetBeatSaverAsync(uri, cancellationToken).ConfigureAwait(false))
{
pageText = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
}
if (string.IsNullOrEmpty(pageText))
{
Logger.Warning($"Unable to get web page at {uri}");
return(null);
}
}
catch (HttpRequestException)
{
Logger.Error($"HttpRequestException while trying to populate fields for {hash}");
return(null);
}
catch (AggregateException ae)
{
ae.WriteExceptions($"Exception while trying to get details for {hash}");
}
catch (Exception ex)
{
Logger.Exception("Exception getting page", ex);
}
Song song = ParseSongsFromPage(pageText).FirstOrDefault();
song.ScrapedAt = DateTime.Now;
return(ScrapedDataProvider.GetOrCreateSong(song));
}
public static ScoreSaberSong GetScoreSaberSongFromJson(JToken song)
{
//JSONObject song = (JSONObject) aKeyValue;
//string songIndex = song["key"]?.Value<string>();
string songHash = song["id"]?.Value <string>();
string songName = song["name"]?.Value <string>();
string author = song["author"]?.Value <string>();
//string songUrl = "https://beatsaver.com/download/" + songIndex;
ScoreSaberSong newSong = null;
if (ScoreSaberSong.TryParseScoreSaberSong(song, ref newSong))
{
//newSong.Feed = "followings";
newSong.ScrapedAt = DateTime.Now;
ScrapedDataProvider.GetOrCreateSong(newSong, true);
return(newSong);
}
else
{
if (!(string.IsNullOrEmpty(songName)))
{
Logger.Warning($"Couldn't parse song {songName}, using sparse definition.");
//songs.Add(new ScoreSaberSong("", songName, "", author));
}
else
{
Logger.Error("Unable to identify song, skipping");
}
}
return(null);
}
public static List <SongInfo> GetSongsFromPage(string pageText)
{
var sssongs = GetSSSongsFromPage(pageText);
List <SongInfo> songs = new List <SongInfo>();
SongInfo tempSong;
//sssongs.AsParallel().WithDegreeOfParallelism(Config.MaxConcurrentPageChecks).ForAll(s => s.PopulateFields());
//Parallel.ForEach(sssongs, new ParallelOptions { MaxDegreeOfParallelism = Config.MaxConcurrentPageChecks }, s => s.PopulateFields());
foreach (var song in sssongs)
{
tempSong = ScrapedDataProvider.GetSong(song, false); // Don't search online because creating the ScoreSaberSong already tried
if (tempSong != null && !string.IsNullOrEmpty(tempSong.key))
{
//tempSong.ScoreSaberInfo.AddOrUpdate(song.uid, song);
songs.Add(tempSong);
}
else
{
Logger.Warning($"Could not find song {song.name} with hash {song.hash} on Beat Saver, skipping...");
}
}
return(songs);
}
//[TestMethod]
public void BuildDatabaseFromJsonTest()
{
var bsScrape = @"TestData\BeatSaverScrape.json";
var ssScrape = @"TestData\ScoreSaberScrape.json";
var dbPath = "songsTest.db";
ScrapedDataProvider.BuildDatabaseFromJson(dbPath, bsScrape, ssScrape);
}
private static void Tests()
{
//ScrapedDataProvider.Initialize();
WebUtils.Initialize(5);
ScrapedDataProvider.BeatSaverSongs.WriteFile();
var bsReader = new BeatSaverReader();
//var bsSongs = bsReader.GetSongsFromFeed(new BeatSaverFeedSettings((int)BeatSaverFeeds.LATEST) { MaxPages = 10, searchOnline = true });
var authorSongs = BeatSaverReader.GetSongsByAuthor("ruckasdfus");
var song = ScrapedDataProvider.Songs.Values.Where(s => s.ScoreSaberInfo.Any(d => d.Value.uid == 155732)).FirstOrDefault();
var found = ScrapedDataProvider.TryGetSongByKey("3f57", out SongInfo badSong, false);
var CustomSongsPath = Path.Combine(OldConfig.BeatSaberPath, "Beat Saber_Data", "CustomLevels");
var tempFolder = new DirectoryInfo(Path.Combine(Path.GetTempPath(), badSong.key + ".zip"));
var outputFolder = new DirectoryInfo(Path.Combine(CustomSongsPath, $"{badSong.key} ({Utilities.MakeSafeFilename(badSong.songName)} - {Utilities.MakeSafeFilename(badSong.authorName)})"));
var job = new DownloadJob(badSong, tempFolder.FullName, outputFolder.FullName);
job.RunJobAsync().Wait();
var br = new BeastSaberReader("Zingabopp", 3);
var text = WebUtils.GetPageText("https://bsaber.com/wp-json/bsaber-api/songs/?bookmarked_by=Zingabopp&page=1");
var bSongs = br.GetSongsFromPage(text);
//ScrapedDataProvider.Initialize();
//ScrapedDataProvider.Initialize();
var bsScrape = ScrapedDataProvider.BeatSaverSongs;
var ssScrape = ScrapedDataProvider.ScoreSaberSongs;
ScrapedDataProvider.TryGetSongByHash("501f6b1bddb2af72abda0f1e6b7b89cb1eb3db67", out SongInfo deletedSong);
//var job = new DownloadJob(deletedSong, "test.zip", @"ScrapedData\test");
//var jobTask = job.RunJobAsync();
//jobTask.Wait();
bsScrape.AddOrUpdate(null);
var resp = WebUtils.HttpClient.GetAsync("https://beatsaver.com/api/maps/detail/b");
Task.WaitAll(resp);
var rateHeaders = resp.Result.Headers.Where(h => h.Key.StartsWith("Rate-Limit")).ToDictionary(x => x.Key, x => x.Value.FirstOrDefault());
var remoteTime = resp.Result.Headers.Date;
var rateInfo = WebUtils.ParseRateLimit(rateHeaders);
Console.WriteLine("Reset Timespan: " + rateInfo.TimeToReset.ToString());
foreach (var item in resp.Result.Headers)
{
Console.WriteLine($"{item.Key}: {string.Join("|", item.Value)}");
}
var trending = ScrapedDataProvider.Songs.Values.Where(s => s.ScoreSaberInfo.Count > 0).OrderByDescending(s => s.ScoreSaberInfo.Values.Select(ss => ss.scores).Aggregate((a, b) => a + b)).Take(100);
var detTrending = trending.Select(s => (s.ScoreSaberInfo.Values.Select(ss => ss.scores).Aggregate((a, b) => a + b), s)).ToList();
}
public static List <SongInfo> GetSongsFromPage(string url)
{
string pageText = GetPageText(url);
var songs = new List <SongInfo>();
foreach (var song in ParseSongsFromPage(pageText))
{
songs.Add(ScrapedDataProvider.GetOrCreateSong(song));
}
return(songs);
}
public static async Task <List <SongInfo> > GetSongsFromPageAsync(string url, bool useDateLimit = false)
{
string pageText = string.Empty;
List <SongInfo> songs = new List <SongInfo>();;
try
{
pageText = await GetPageTextAsync(url).ConfigureAwait(false);
Logger.Debug($"Successful got pageText from {url}");
foreach (var song in ParseSongsFromPage(pageText))
{
songs.Add(ScrapedDataProvider.GetOrCreateSong(song));
}
}
catch (Exception ex)
{
Logger.Error($"Error getting page text from {url}");
}
return(songs);
}
public static List <SongInfo> Search(string criteria, SearchType type)
{
if (type == SearchType.key)
{
return(new List <SongInfo>()
{
GetSongByKey(criteria)
});
}
if (type == SearchType.user)
{
return(GetSongsByUploaderId(criteria));
}
if (type == SearchType.hash)
{
return(new List <SongInfo>()
{
GetSongByHash(criteria)
});
}
StringBuilder url;
url = new StringBuilder(Feeds[BeatSaverFeeds.SEARCH].BaseUrl);
url.Replace(SEARCHTYPEKEY, type.ToString());
url.Replace(SEARCHKEY, criteria);
string pageText = GetPageText(url.ToString());
var songs = new List <SongInfo>();
foreach (var song in ParseSongsFromPage(pageText))
{
songs.Add(ScrapedDataProvider.GetOrCreateSong(song));
}
return(songs);
}
static void Main(string[] args)
{
try
{
Logger.ShortenSourceName = true;
try
{
OldConfig.Initialize();
Logger.LogLevel = OldConfig.StrToLogLevel(OldConfig.LoggingLevel);
if (Logger.LogLevel < LogLevel.Info)
{
Logger.ShortenSourceName = false;
}
}
catch (FileNotFoundException ex)
{
Logger.Exception("Error initializing Config", ex);
}
Logger.Info($"Using Beat Saber directory: {OldConfig.BeatSaberPath}");
ScrapedDataProvider.Initialize();
Logger.Info($"Scrapes loaded, {ScrapedDataProvider.BeatSaverSongs.Data.Count} BeatSaverSongs and {ScrapedDataProvider.ScoreSaberSongs.Data.Count} ScoreSaber difficulties loaded");
//DoFullScrape();
//var scoreSaberSongs = ScrapedDataProvider.ScoreSaberSongs.Data.Select(ss => ss.hash).Distinct().Count();
//var activeSongs = ScrapedDataProvider.Songs.Values.Where(s => s.ScoreSaberInfo?.Values.Count > 0).Count();
//Tests();
try
{
if (args.Length > 0)
{
var bsDir = new DirectoryInfo(args[0]);
if (bsDir.Exists)
{
if (bsDir.GetFiles("Beat Saber.exe").Length > 0)
{
Logger.Info("Found Beat Saber.exe");
OldConfig.BeatSaberPath = bsDir.FullName;
Logger.Info($"Updated Beat Saber directory path to {OldConfig.BeatSaberPath}");
Console.WriteLine("Press any key to continue...");
Console.ReadKey();
}
else
{
Logger.Warning($"Provided directory does not appear to be Beat Saber's root folder, ignoring it");
}
}
}
}
catch (Exception ex)
{
Logger.Exception($"Error parsing command line arguments", ex);
}
if (!OldConfig.CriticalError)
{
WebUtils.Initialize(OldConfig.MaxConcurrentPageChecks);
Stopwatch sw = new Stopwatch();
sw.Start();
SyncSaber ss = new SyncSaber();
ss.ScrapeNewSongs();
Console.WriteLine();
if (OldConfig.SyncFavoriteMappersFeed && OldConfig.FavoriteMappers.Count > 0)
{
Logger.Info($"Downloading songs from FavoriteMappers.ini...");
try
{
ss.DownloadSongsFromFeed(BeatSaverReader.NameKey, new BeatSaverFeedSettings(0)
{
Authors = OldConfig.FavoriteMappers.ToArray()
});
}
catch (AggregateException ae)
{
ae.WriteExceptions($"Exceptions downloading songs from FavoriteMappers.ini.");
}
catch (Exception ex)
{
Logger.Exception("Exception downloading songs from FavoriteMappers.ini.", ex);
}
}
else
{
if (OldConfig.SyncFavoriteMappersFeed)
{
Logger.Warning($"Skipping FavoriteMappers.ini feed, no authors found in {Path.Combine(OldConfig.BeatSaberPath, "UserData", "FavoriteMappers.ini")}");
}
}
if (OldConfig.SyncFollowingsFeed)
{
// Followings
Console.WriteLine();
Logger.Info($"Downloading songs from {BeastSaberReader.Feeds[BeastSaberFeeds.FOLLOWING].Name} feed...");
try
{
//ss.DownloadBeastSaberFeed(0, Web.BeastSaberReader.GetMaxBeastSaberPages(0));
ss.DownloadSongsFromFeed(BeastSaberReader.NameKey, new BeastSaberFeedSettings(0)
{
MaxPages = OldConfig.MaxFollowingsPages
});
}
catch (AggregateException ae)
{
ae.WriteExceptions($"Exceptions downloading songs from BeastSaberFeed: Following.");
}
catch (Exception ex)
{
Logger.Exception($"Exception downloading BeastSaberFeed: Following", ex);
}
}
// Bookmarks
if (OldConfig.SyncBookmarksFeed)
{
Console.WriteLine();
Logger.Info($"Downloading songs from {BeastSaberReader.Feeds[BeastSaberFeeds.BOOKMARKS].Name} feed...");
try
{
//ss.DownloadBeastSaberFeed(1, Web.BeastSaberReader.GetMaxBeastSaberPages(1));
ss.DownloadSongsFromFeed(BeastSaberReader.NameKey, new BeastSaberFeedSettings(1)
{
MaxPages = OldConfig.MaxBookmarksPages
});
}
catch (AggregateException ae)
{
ae.WriteExceptions($"Exceptions downloading songs from BeastSaberFeed: Bookmarks.");
}
catch (Exception ex)
{
Logger.Exception($"Exception downloading BeastSaberFeed: Bookmarks", ex);
}
}
if (OldConfig.SyncCuratorRecommendedFeed)
{
// Curator Recommended
Console.WriteLine();
Logger.Info($"Downloading songs from {BeastSaberReader.Feeds[BeastSaberFeeds.CURATOR_RECOMMENDED].Name} feed...");
try
{
//ss.DownloadBeastSaberFeed(2, Web.BeastSaberReader.GetMaxBeastSaberPages(2));
ss.DownloadSongsFromFeed(BeastSaberReader.NameKey, new BeastSaberFeedSettings(2)
{
MaxPages = OldConfig.MaxCuratorRecommendedPages
});
}
catch (AggregateException ae)
{
ae.WriteExceptions($"Exceptions downloading songs from BeastSaberFeed: Curator Recommended.");
}
catch (Exception ex)
{
Logger.Exception($"Exception downloading BeastSaberFeed: Curator Recommended", ex);
}
}
if (OldConfig.SyncTopPPFeed)
{
// ScoreSaber Top PP
Console.WriteLine();
Logger.Info($"Downloading songs from {ScoreSaberReader.Feeds[ScoreSaberFeeds.TOP_RANKED].Name} feed...");
try
{
//ss.DownloadBeastSaberFeed(2, Web.BeastSaberReader.GetMaxBeastSaberPages(2));
ss.DownloadSongsFromFeed(ScoreSaberReader.NameKey, new ScoreSaberFeedSettings((int)ScoreSaberFeeds.TOP_RANKED)
{
MaxSongs = 1000,//Config.MaxScoreSaberSongs,
SongsPerPage = 10,
searchOnline = true
});
}
catch (AggregateException ae)
{
ae.WriteExceptions($"Exceptions downloading songs from ScoreSaberFeed: Top Ranked.");
}
catch (Exception ex)
{
Logger.Exception($"Exception downloading ScoreSaberFeed: Top Ranked.", ex);
}
}
/*
* Console.WriteLine();
* Logger.Info($"Downloading newest songs on Beat Saver...");
* try
* {
* ss.DownloadSongsFromFeed(BeatSaverReader.NameKey, new BeatSaverFeedSettings(1) {
* MaxPages = Config.MaxBeatSaverPages
* });
* }
*
* catch (Exception ex)
* {
* Logger.Exception("Exception downloading BeatSaver newest feed.", ex);
* }
*/
sw.Stop();
var processingTime = new TimeSpan(sw.ElapsedTicks);
Console.WriteLine();
Logger.Info($"Finished downloading songs in {(int)processingTime.TotalMinutes} min {processingTime.Seconds} sec");
}
else
{
foreach (string e in OldConfig.Errors)
{
Logger.Error($"Invalid setting: {e} = {OldConfig.Setting[e]}");
}
}
ScrapedDataProvider.BeatSaverSongs.WriteFile();
ScrapedDataProvider.ScoreSaberSongs.WriteFile();
}
catch (OutOfDateException ex)
{
Logger.Error(ex.Message);
}
catch (AggregateException ae)
{
ae.WriteExceptions($"Uncaught exceptions in Main()");
}
catch (Exception ex)
{
Logger.Exception("Uncaught exception in Main()", ex);
}
Console.WriteLine("Press any key to continue...");
Console.ReadKey();
}
/// <summary>
/// Parses the page text and returns all the songs it can find.
/// </summary>
/// <param name="pageText"></param>
/// <exception cref="XmlException">Invalid XML in pageText</exception>
/// <returns></returns>
public List <SongInfo> GetSongsFromPage(string pageText)
{
List <SongInfo> songsOnPage = new List <SongInfo>();
List <BSaberSong> bSongs = new List <BSaberSong>();
int totalSongsForPage = 0;
if (pageText.ToLower().StartsWith(@"<?xml"))
{
bool retry = false;
XmlDocument xmlDocument = new XmlDocument();
do
{
try
{
xmlDocument.LoadXml(pageText);
retry = false;
}
catch (XmlException ex)
{
if (retry == true)
{
Logger.Exception("Exception parsing XML.", ex);
retry = false;
}
else
{
Logger.Warning("Invalid XML formatting detected, attempting to fix...");
pageText = pageText.Replace(" & ", " & ");
retry = true;
}
//File.WriteAllText("ErrorText.xml", pageText);
}
} while (retry == true);
List <Task> populateTasks = new List <Task>();
XmlNodeList xmlNodeList = xmlDocument.DocumentElement.SelectNodes("/rss/channel/item");
foreach (object obj in xmlNodeList)
{
XmlNode node = (XmlNode)obj;
if (node["DownloadURL"] == null || node["SongTitle"] == null)
{
Logger.Debug("Not a song! Skipping!");
}
else
{
// TODO: Not really using any of this except the hash.
string songName = node["SongTitle"].InnerText;
string downloadUrl = node["DownloadURL"]?.InnerText;
string hash = node["Hash"]?.InnerText?.ToUpper();
string authorName = node["LevelAuthorName"]?.InnerText;
string songKey = node["SongKey"]?.InnerText;
if (downloadUrl.Contains("dl.php"))
{
Logger.Warning("Skipping BeastSaber download with old url format!");
totalSongsForPage++;
}
else
{
string songIndex = !string.IsNullOrEmpty(songKey) ? songKey : downloadUrl.Substring(downloadUrl.LastIndexOf('/') + 1);
//string mapper = !string.IsNullOrEmpty(authorName) ? authorName : GetMapperFromBsaber(node.InnerText);
//string songUrl = !string.IsNullOrEmpty(downloadUrl) ? downloadUrl : BeatSaverDownloadURL_Base + songIndex;
if (ScrapedDataProvider.TryGetSongByKey(songIndex, out SongInfo currentSong))
{
songsOnPage.Add(currentSong);
}
}
}
}
}
else // Page is JSON (hopefully)
{
JObject result = new JObject();
try
{
result = JObject.Parse(pageText);
}
catch (Exception ex)
{
Logger.Exception("Unable to parse JSON from text", ex);
}
var songs = result["songs"];
foreach (var bSong in songs)
{
// Try to get the song hash from BeastSaber
string songHash = bSong["hash"]?.Value <string>();
if (!string.IsNullOrEmpty(songHash))
{
if (ScrapedDataProvider.TryGetSongByHash(songHash, out SongInfo currentSong))
{
songsOnPage.Add(currentSong);
}
}
else
{
// Unable to get song hash, try getting song_key from BeastSaber
string songKey = bSong["song_key"]?.Value <string>();
if (!string.IsNullOrEmpty(songKey))
{
if (ScrapedDataProvider.TryGetSongByKey(songKey, out SongInfo currentSong))
{
songsOnPage.Add(currentSong);
}
else
{
Logger.Debug($"ScrapedDataProvider could not find song: {bSong.Value<string>()}");
}
}
else
{
Logger.Debug($"Not a song, skipping: {bSong.ToString()}");
}
}
}
}
//Task.WaitAll(populateTasks.ToArray());
Logger.Debug($"{songsOnPage.Count} songs on the page");
return(songsOnPage);
}
