public void AddingSongs_Test()
{
using (SongDataContext context = new SongDataContext("songAddTest.db"))
{
context.Database.EnsureDeleted();
context.Database.EnsureCreated();
string json = File.ReadAllText(@"TestData\BeatSaverTestSongs.json");
var songList = JToken.Parse(json);
foreach (var jSong in songList["docs"].Children())
{
var newSong = Song.CreateFromJson(jSong);
context.AddOrUpdate(newSong);
}
context.SaveChanges();
}
using (SongDataContext context = new SongDataContext("songAddTest.db"))
{
context.LoadQuery(context.Songs.Where(s => true)).Load();
var song = context.Songs.Find("5d2662f993d6020006079bf8");
Assert.IsTrue(song != null);
Assert.IsTrue(song.Uploader.UploaderName == "snail0fd3ath");
Assert.IsTrue(song.Key == "5658");
Assert.IsTrue(song.Downloads == 3);
Assert.IsTrue(song.BeatmapCharacteristics.Count == 2);
var songChar = song.BeatmapCharacteristics.Where(bc => bc.CharacteristicName == "Standard").Single();
Assert.IsTrue(songChar != null);
Assert.IsTrue(songChar.CharacteristicDifficulties.Count == 3);
var charDiff = songChar.CharacteristicDifficulties.Where(d => d.DifficultyLevel == 3).Single();
Assert.IsTrue(charDiff != null);
Assert.IsTrue(charDiff.Difficulty == "Hard");
Assert.IsTrue(charDiff.Bombs == 15);
}
}
public void TestExpressions()
{
var minDate = new DateTime(2019, 7, 1);
var maxDate = new DateTime(2019, 7, 3);
var parameter = Expression.Parameter(typeof(Song), "s");
var member = Expression.Property(parameter, "Uploaded");
var minConstant = Expression.Constant(minDate);
var maxConstant = Expression.Constant(maxDate);
var body = Expression.AndAlso(
Expression.GreaterThan(member, minConstant),
Expression.LessThan(member, maxConstant));
var finalExpression = Expression.Lambda <Func <Song, bool> >(body, parameter);
var orderByPropertyExp = Expression.Lambda <Func <Song, DateTime> >(
member,
parameter);
var context = new SongDataContext();
context.Database.EnsureCreated();
//context.Songs.Where(finalExpression).OrderByDescending(s => s.Uploaded).Load();
context.Songs.Where(finalExpression).OrderByDescending(orderByPropertyExp).Load();
var test = context.Songs.Local.Select(s => new { s.SongName, s.Uploaded }).ToList();
Assert.IsTrue(test.All(s => s.Uploaded > minDate && s.Uploaded < maxDate));
//var test = context.Songs.Where(lam).OrderBy(s => s.Uploaded).Select(s => new { s.SongName, s.Uploaded }).ToList();
}
public static void AddOrUpdate <TEntity>(this SongDataContext context, TEntity updatedEntity, List <object> refChain = null)
where TEntity : DatabaseDataType
{
if (refChain == null)
{
refChain = new List <object>();
}
if (refChain.Contains(updatedEntity))
{
return;
}
refChain.Add(updatedEntity);
if (context == null)
{
context = new SongDataContext();
}
var entity = context.Find(updatedEntity.GetType(), updatedEntity.PrimaryKey);
if (entity == null)
{
entity = AddIfMissing(context, updatedEntity);
}
var existingEntity = context.Entry(entity);
// Update properties.
foreach (var oldProp in existingEntity.Properties)
{
if (Attribute.IsDefined(oldProp.Metadata.PropertyInfo, typeof(Updatable)))
{
//Console.WriteLine($"Checking property: {oldProp.Metadata.Name}");
object newVal = updatedEntity[$"{oldProp.Metadata.Name}"];
if (newVal != null && !newVal.Equals(oldProp.CurrentValue))
{
oldProp.CurrentValue = newVal;
Console.WriteLine($"Updating ({string.Join(", ", updatedEntity.PrimaryKey)})'s {oldProp.Metadata.Name} property from {oldProp.OriginalValue?.ToString() ?? "null"} to {newVal?.ToString() ?? "null"}");
}
}
}
/* Probably don't need to worry about this, if these change it'll be because the song was updated.
* // Update references
* foreach (var item in existingEntity.References)
* {
* if(item.CurrentValue is IDatabaseDataType refEntity)
* {
* if (refChain == null)
* refChain = new List<object[]>() { entity.PrimaryKey };
* if (refChain.Any(i => i.SequenceEqual(refEntity.PrimaryKey)))
* continue; // This object was already navigated to
*
* }
* }
* // Update Collections
*/
//return songEntry.Entity;
}
public static void BuildDatabaseFromJson(string databasePath, string beatSaverScrapePath, string scoreSaberPath)
{
var dbFile = new FileInfo(databasePath);
var bsScrapeFile = new FileInfo(beatSaverScrapePath);
var ssScrapeFile = new FileInfo(scoreSaberPath);
dbFile.Directory.Create();
using (var context = new SongDataContext(dbFile.FullName))
{
context.Database.EnsureDeleted();
context.Database.EnsureCreated();
JToken songList = null;
var serializer = new JsonSerializer();
var listSongs = new List <Song>();
int count = 0;
Stopwatch sw = new Stopwatch();
List <ScoreSaberDifficulty> ssDiffs = null;
using (var fs = new FileStream(ssScrapeFile.FullName, FileMode.Open))
using (var sr = new StreamReader(fs))
using (var jsonTextReader = new JsonTextReader(sr))
{
//songList = JToken.Parse(sr);
ssDiffs = serializer.Deserialize <List <ScoreSaberDifficulty> >(jsonTextReader);
}
if (count == 0)
{
using (var fs = new FileStream(bsScrapeFile.FullName, FileMode.Open))
using (var sr = new StreamReader(fs))
using (var jsonTextReader = new JsonTextReader(sr))
{
//songList = JToken.Parse(sr);
songList = serializer.Deserialize <JToken>(jsonTextReader);
}
sw.Start();
foreach (var jSong in songList.Children())
{
var newSong = Song.CreateFromJson(jSong);
newSong.ScoreSaberDifficulties = new List <ScoreSaberDifficulty>();
var matchedSSdiffs = ssDiffs.Where(d => d.SongHash == newSong.Hash);
foreach (var diff in matchedSSdiffs)
{
newSong.ScoreSaberDifficulties.Add(diff);
}
//listSongs.Add(newSong);
context.Add(newSong);
//context.SaveChanges();
//var newSong = Song.CreateFromJson(jSong);
//context.Add(Song.CreateFromJson(jSong));
//context.SaveChanges();
count++;
}
}
context.SaveChanges();
}
}
public static DatabaseDataType AddIfMissing(this SongDataContext context, object entityToAdd, List <object> refChain = null, Func <DatabaseDataType, bool> searchPredicate = null)
{
if (!typeof(DatabaseDataType).IsAssignableFrom(entityToAdd.GetType()))
{
throw new ArgumentException("EntityToAdd is not a DatabaseDataType");
}
DatabaseDataType retVal = entityToAdd as DatabaseDataType;
if (refChain == null)
{
refChain = new List <object>();
}
if (refChain.Contains(entityToAdd))
{
return(retVal);
}
if (entityToAdd is Song song)
{
retVal = context.AddIfMissing <Song>(song, refChain);
}
else if (entityToAdd is ScoreSaberDifficulty ssDiff)
{
retVal = context.AddIfMissing <ScoreSaberDifficulty>(ssDiff, refChain);
}
//else if (entityToAdd is Characteristic characteristic)
// retVal = context.AddIfMissing<Characteristic>(characteristic, refChain);
else if (entityToAdd is BeatmapCharacteristic bmChar)
{
retVal = context.AddIfMissing <BeatmapCharacteristic>(bmChar, refChain);//, bc => (bc.CharacteristicName == bmChar.CharacteristicName && bc.SongId == bmChar.SongId));
}
else if (entityToAdd is CharacteristicDifficulty charDiff)
{
retVal = context.AddIfMissing <CharacteristicDifficulty>(charDiff, refChain);//, cd => (cd.Difficulty == charDiff.Difficulty && cd.BeatmapCharacteristic.SongId == charDiff.BeatmapCharacteristic.SongId));
}
//else if (entityToAdd is SongDifficulty songDiff)
// retVal = context.AddIfMissing<SongDifficulty>(songDiff, refChain);
//else if (entityToAdd is Difficulty difficulty)
// retVal = context.AddIfMissing<Difficulty>(difficulty, refChain);
else if (entityToAdd is Uploader uploader)
{
retVal = context.AddIfMissing <Uploader>(uploader, refChain);
}
return(retVal);
}
public void UpdatingSongs_Test()
{
SongDataContext context = new SongDataContext("songUpdateTest.db");
context.Database.EnsureDeleted();
context.Database.EnsureCreated();
string json = File.ReadAllText(@"TestData\BeatSaverListWithConverted.json");
var songList = JToken.Parse(json);
foreach (var jSong in songList["docs"].Children())
{
var newSong = Song.CreateFromJson(jSong);
context.AddOrUpdate(newSong);
}
context.SaveChanges();
var convertedGroup = context.Songs.Local.GroupBy(s => s.Converted);
}
public static TEntity AddIfMissing <TEntity>(this SongDataContext context, TEntity entityToAdd, List <object> refChain = null, Func <TEntity, bool> searchPredicate = null)
where TEntity : DatabaseDataType
{
if (refChain == null)
{
refChain = new List <object>();
}
if (refChain.Contains(entityToAdd))
{
return(entityToAdd);
}
if (context == null)
{
context = new SongDataContext();
}
TEntity entity = null;
if (searchPredicate == null)
{
entity = context.Find <TEntity>(entityToAdd.PrimaryKey);
//entity = context.Set<TEntity>().Where(e => e.PrimaryKey == entityToAdd.PrimaryKey).Take(1).FirstOrDefault();
}
else
{
entity = context.Set <TEntity>().Where(searchPredicate).Take(1).FirstOrDefault();
}
EntityEntry newEntity;
if (entity == null)
{
newEntity = context.Add(entityToAdd);
}
else
{
newEntity = context.Entry(entity);
}
if (entity == null)
{
newEntity.State = EntityState.Added;
}
refChain.Add(newEntity.Entity);
foreach (var nav in newEntity.Navigations)
{
if (nav.CurrentValue == null)
{
continue;
}
if (nav.CurrentValue is IEnumerable <DatabaseDataType> collection)
{
foreach (var item in collection)
{
context.AddIfMissing(((object)item), refChain);
}
}
else
{
context.AddIfMissing(nav.CurrentValue, refChain);
}
}
return((TEntity)newEntity.Entity);
}
static void Main(string[] args)
{
Console.WriteLine("Testing...");
//ScrapedDataProvider.Initialize();
//SongHashDataProvider songHashes = new SongHashDataProvider();
//songHashes.Initialize();
//PlayerDataProvider playerData = new PlayerDataProvider();
//playerData.Initialize();
//ScrapedDataProvider.ScoreSaberSongs.Initialize("ScoreSaberScrape.json");
//var songs = ScrapedDataProvider.Songs.SelectMany(s => s.Value.ScoreSaberDifficulties).GroupBy(d => d.DifficultyName);
//var topSongs = ScrapedDataProvider.Songs.Values
// .OrderByDescending(s => s.ScoreSaberDifficulties.Aggregate(0, (a, b) => a + b.ScoresPerDay))
// .Select(s => $"{s.KeyAsInt} {s.SongName} - Uploaded: {s.Uploaded.ToShortDateString()}, Plays in past 24 hours: {s.ScoreSaberDifficulties.Aggregate(0, (a, b) => a + b.ScoresPerDay)}");
//foreach (var item in ScrapedDataProvider.BeatSaverSongs.Data)
//{
// var test = new Song(item, ScrapedDataProvider.ScoreSaberSongs.Data.Where(d => d.SongHash == item.hash));
// //Console.WriteLine($"{item.Key}: {item.Count()}");
//}
//var rankedMaul = ssScrape.Where(d => d.Ranked && d.DifficultyName.ToLower().Contains("dm")).ToList();
SongDataContext context = new SongDataContext()
{
EnableSensitiveDataLogging = false, UseLoggerFactory = false
};
context.Database.EnsureDeleted();
context.Database.EnsureCreated();
context.LoadQuery(context.Songs.Where(s => true)).Load();
//string fileRead = File.ReadAllText("BeatSaverTestSongs.json");
//string fileRead = File.ReadAllText("BeatSaverTestSongsUpdate.json");
//var songList = JToken.Parse(fileRead)["docs"];
//string fileRead = File.ReadAllText(@"ScrapedData\BeatSaverScrape.json");
JToken songList = null;
var serializer = new JsonSerializer();
var listSongs = new List <Song>();
int count = 0;
Stopwatch sw = new Stopwatch();
List <ScoreSaberDifficulty> ssDiffs = null;
using (var fs = new FileStream(@"ScrapedData\ScoreSaberScrape.json", FileMode.Open))
using (var sr = new StreamReader(fs))
using (var jsonTextReader = new JsonTextReader(sr))
{
//songList = JToken.Parse(sr);
ssDiffs = serializer.Deserialize <List <ScoreSaberDifficulty> >(jsonTextReader);
}
if (count == 0)
{
using (var fs = new FileStream(@"ScrapedData\BeatSaverScrape.json", FileMode.Open))
using (var sr = new StreamReader(fs))
using (var jsonTextReader = new JsonTextReader(sr))
{
//songList = JToken.Parse(sr);
songList = serializer.Deserialize <JToken>(jsonTextReader);
}
sw.Start();
foreach (var jSong in songList.Children())
{
var newSong = Song.CreateFromJson(jSong);
newSong.ScoreSaberDifficulties = new List <ScoreSaberDifficulty>();
var matchedSSdiffs = ssDiffs.Where(d => d.SongHash == newSong.Hash);
foreach (var diff in matchedSSdiffs)
{
newSong.ScoreSaberDifficulties.Add(diff);
}
//listSongs.Add(newSong);
context.Add(newSong);
//context.SaveChanges();
//var newSong = Song.CreateFromJson(jSong);
//context.Add(Song.CreateFromJson(jSong));
//context.SaveChanges();
count++;
}
Console.WriteLine($"-----Song processing took {sw.Elapsed.TotalSeconds}");
sw.Restart();
//context.AddRange(listSongs);
//context.SaveChanges();
Console.WriteLine($"-----Database save took {sw.Elapsed.TotalSeconds}");
}
count = 0;
sw.Restart();
//foreach (var jSong in songList.Children())
//{
// var newSong = Song.CreateFromJson(jSong);
// //listSongs.Add(newSong);
// context.Add(newSong);
// //context.SaveChanges();
// //var newSong = Song.CreateFromJson(jSong);
// //context.Add(Song.CreateFromJson(jSong));
// //context.SaveChanges();
// count++;
//}
//context.AddRange(ssDiffs);
//Console.WriteLine($"-----Song processing took {sw.Elapsed.TotalSeconds}");
//sw.Restart();
//context.AddRange(listSongs);
context.SaveChanges();
Console.WriteLine($"-----Database save took {sw.Elapsed.TotalSeconds}");
return;
//var songs = ScrapedDataProvider.Songs;
//context.Songs.UpdateRange(listSongs);
//context.SaveChanges();
//ScrapedDataProvider.TryGetSongByHash("2FDDB136BDA7F9E29B4CB6621D6D8E0F8A43B126", out Song song);
//ScrapedDataProvider.TryGetSongByKey("b", out Song believer);
//string hash = believer.Hash.ToLower();
//ScrapedDataProvider.BeatSaverSongs.WriteFile();
}