public static void BuildDatabaseFromJson(string databasePath, string beatSaverScrapePath, string scoreSaberPath)
{
var dbFile = new FileInfo(databasePath);
var bsScrapeFile = new FileInfo(beatSaverScrapePath);
var ssScrapeFile = new FileInfo(scoreSaberPath);
dbFile.Directory.Create();
using (var context = new SongDataContext(dbFile.FullName))
{
context.Database.EnsureDeleted();
context.Database.EnsureCreated();
JToken songList = null;
var serializer = new JsonSerializer();
var listSongs = new List <Song>();
int count = 0;
Stopwatch sw = new Stopwatch();
List <ScoreSaberDifficulty> ssDiffs = null;
using (var fs = new FileStream(ssScrapeFile.FullName, FileMode.Open))
using (var sr = new StreamReader(fs))
using (var jsonTextReader = new JsonTextReader(sr))
{
//songList = JToken.Parse(sr);
ssDiffs = serializer.Deserialize <List <ScoreSaberDifficulty> >(jsonTextReader);
}
if (count == 0)
{
using (var fs = new FileStream(bsScrapeFile.FullName, FileMode.Open))
using (var sr = new StreamReader(fs))
using (var jsonTextReader = new JsonTextReader(sr))
{
//songList = JToken.Parse(sr);
songList = serializer.Deserialize <JToken>(jsonTextReader);
}
sw.Start();
foreach (var jSong in songList.Children())
{
var newSong = Song.CreateFromJson(jSong);
newSong.ScoreSaberDifficulties = new List <ScoreSaberDifficulty>();
var matchedSSdiffs = ssDiffs.Where(d => d.SongHash == newSong.Hash);
foreach (var diff in matchedSSdiffs)
{
newSong.ScoreSaberDifficulties.Add(diff);
}
//listSongs.Add(newSong);
context.Add(newSong);
//context.SaveChanges();
//var newSong = Song.CreateFromJson(jSong);
//context.Add(Song.CreateFromJson(jSong));
//context.SaveChanges();
count++;
}
}
context.SaveChanges();
}
}
static void Main(string[] args)
{
Console.WriteLine("Testing...");
//ScrapedDataProvider.Initialize();
//SongHashDataProvider songHashes = new SongHashDataProvider();
//songHashes.Initialize();
//PlayerDataProvider playerData = new PlayerDataProvider();
//playerData.Initialize();
//ScrapedDataProvider.ScoreSaberSongs.Initialize("ScoreSaberScrape.json");
//var songs = ScrapedDataProvider.Songs.SelectMany(s => s.Value.ScoreSaberDifficulties).GroupBy(d => d.DifficultyName);
//var topSongs = ScrapedDataProvider.Songs.Values
// .OrderByDescending(s => s.ScoreSaberDifficulties.Aggregate(0, (a, b) => a + b.ScoresPerDay))
// .Select(s => $"{s.KeyAsInt} {s.SongName} - Uploaded: {s.Uploaded.ToShortDateString()}, Plays in past 24 hours: {s.ScoreSaberDifficulties.Aggregate(0, (a, b) => a + b.ScoresPerDay)}");
//foreach (var item in ScrapedDataProvider.BeatSaverSongs.Data)
//{
// var test = new Song(item, ScrapedDataProvider.ScoreSaberSongs.Data.Where(d => d.SongHash == item.hash));
// //Console.WriteLine($"{item.Key}: {item.Count()}");
//}
//var rankedMaul = ssScrape.Where(d => d.Ranked && d.DifficultyName.ToLower().Contains("dm")).ToList();
SongDataContext context = new SongDataContext()
{
EnableSensitiveDataLogging = false, UseLoggerFactory = false
};
context.Database.EnsureDeleted();
context.Database.EnsureCreated();
context.LoadQuery(context.Songs.Where(s => true)).Load();
//string fileRead = File.ReadAllText("BeatSaverTestSongs.json");
//string fileRead = File.ReadAllText("BeatSaverTestSongsUpdate.json");
//var songList = JToken.Parse(fileRead)["docs"];
//string fileRead = File.ReadAllText(@"ScrapedData\BeatSaverScrape.json");
JToken songList = null;
var serializer = new JsonSerializer();
var listSongs = new List <Song>();
int count = 0;
Stopwatch sw = new Stopwatch();
List <ScoreSaberDifficulty> ssDiffs = null;
using (var fs = new FileStream(@"ScrapedData\ScoreSaberScrape.json", FileMode.Open))
using (var sr = new StreamReader(fs))
using (var jsonTextReader = new JsonTextReader(sr))
{
//songList = JToken.Parse(sr);
ssDiffs = serializer.Deserialize <List <ScoreSaberDifficulty> >(jsonTextReader);
}
if (count == 0)
{
using (var fs = new FileStream(@"ScrapedData\BeatSaverScrape.json", FileMode.Open))
using (var sr = new StreamReader(fs))
using (var jsonTextReader = new JsonTextReader(sr))
{
//songList = JToken.Parse(sr);
songList = serializer.Deserialize <JToken>(jsonTextReader);
}
sw.Start();
foreach (var jSong in songList.Children())
{
var newSong = Song.CreateFromJson(jSong);
newSong.ScoreSaberDifficulties = new List <ScoreSaberDifficulty>();
var matchedSSdiffs = ssDiffs.Where(d => d.SongHash == newSong.Hash);
foreach (var diff in matchedSSdiffs)
{
newSong.ScoreSaberDifficulties.Add(diff);
}
//listSongs.Add(newSong);
context.Add(newSong);
//context.SaveChanges();
//var newSong = Song.CreateFromJson(jSong);
//context.Add(Song.CreateFromJson(jSong));
//context.SaveChanges();
count++;
}
Console.WriteLine($"-----Song processing took {sw.Elapsed.TotalSeconds}");
sw.Restart();
//context.AddRange(listSongs);
//context.SaveChanges();
Console.WriteLine($"-----Database save took {sw.Elapsed.TotalSeconds}");
}
count = 0;
sw.Restart();
//foreach (var jSong in songList.Children())
//{
// var newSong = Song.CreateFromJson(jSong);
// //listSongs.Add(newSong);
// context.Add(newSong);
// //context.SaveChanges();
// //var newSong = Song.CreateFromJson(jSong);
// //context.Add(Song.CreateFromJson(jSong));
// //context.SaveChanges();
// count++;
//}
//context.AddRange(ssDiffs);
//Console.WriteLine($"-----Song processing took {sw.Elapsed.TotalSeconds}");
//sw.Restart();
//context.AddRange(listSongs);
context.SaveChanges();
Console.WriteLine($"-----Database save took {sw.Elapsed.TotalSeconds}");
return;
//var songs = ScrapedDataProvider.Songs;
//context.Songs.UpdateRange(listSongs);
//context.SaveChanges();
//ScrapedDataProvider.TryGetSongByHash("2FDDB136BDA7F9E29B4CB6621D6D8E0F8A43B126", out Song song);
//ScrapedDataProvider.TryGetSongByKey("b", out Song believer);
//string hash = believer.Hash.ToLower();
//ScrapedDataProvider.BeatSaverSongs.WriteFile();
}