public static RismDbContext CreateContext()
{
var context = new RismDbContext(new DbContextOptionsBuilder().UseMySql("server=localhost;database=manufaktura-rism;uid=admin;pwd=123123").Options);
context.ChangeTracker.AutoDetectChangesEnabled = false;
context.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking;
return(context);
}
public void Migrate(int maxRecords)
{
//var path = @"C:\Databases\rismAllMARCXMLexample\rism_130616_example.xml";
var path = @"C:\Databases\rismAllMARCXML\rism_170316.xml";
var counter = 0;
using (var fs = new FileStream(path, FileMode.Open, FileAccess.Read))
{
using (var reader = XmlReader.Create(fs, new XmlReaderSettings {
IgnoreWhitespace = true
}))
{
while (reader.Read() && reader.Name != "record")
{
reader.MoveToContent();
}
while (counter < maxRecords)
{
var record = reader.ReadOuterXml();
if (string.IsNullOrWhiteSpace(record))
{
break;
}
var recordElement = XElement.Parse(record);
var dbRecord = ParseRecord(recordElement);
counter++;
Console.WriteLine($"Record {counter} ({dbRecord.Title} - {dbRecord.ComposerName}) added.");
if (counter % 100 == 0)
{
dbContext.SaveChanges();
}
if (counter % 600 == 0)
{
dbContext = Dependencies.CreateContext(); //Recreate
}
}
dbContext.SaveChanges();
}
}
}
public SearchController([FromServices] RismDbContext context, PlaineAndEasieService plaineAndEasieService, ScoreRendererService scoreRendererService)
{
this.context = context;
this.scoreRendererService = scoreRendererService;
this.plaineAndEasieService = plaineAndEasieService;
}
public void GenerateHashes()
{
var pageSize = 2000;
var processedincipits = 0;
while (true)
{
var sw = new Stopwatch();
sw.Start();
if (processedincipits % 10000 == 0)
{
dbContext = Dependencies.CreateContext(); //Recreate
}
Console.WriteLine($"Searching for incipits...");
var incipits = GetIncipitsBatch(pageSize);
var numberOfPlanes = 40;
Console.WriteLine($"Processing hashes for incipits...");
foreach (var incipit in incipits)
{
for (var numberOfDimensions = 1; numberOfDimensions <= Constants.MaxNumberOfDimensionsForLsh; numberOfDimensions++)
{
var position = GetIncipitVector(new[] {
incipit.Interval1,
incipit.Interval2,
incipit.Interval3,
incipit.Interval4,
incipit.Interval5,
incipit.Interval6,
incipit.Interval7,
incipit.Interval8,
incipit.Interval9,
incipit.Interval10,
incipit.Interval11,
incipit.Interval12
}.Take(numberOfDimensions).ToArray());
var lshAlgorithm = GetPlaneGroup(1, numberOfPlanes, numberOfDimensions);
var hash = lshAlgorithm.ComputeHash(position);
if (numberOfDimensions == 1)
{
incipit.Hash1d = hash;
}
if (numberOfDimensions == 2)
{
incipit.Hash2d = hash;
}
if (numberOfDimensions == 3)
{
incipit.Hash3d = hash;
}
if (numberOfDimensions == 4)
{
incipit.Hash4d = hash;
}
if (numberOfDimensions == 5)
{
incipit.Hash5d = hash;
}
if (numberOfDimensions == 6)
{
incipit.Hash6d = hash;
}
}
dbContext.Entry(incipit).State = EntityState.Modified;
}
dbContext.SaveChanges();
processedincipits += incipits.Length;
sw.Stop();
var timeRemaining = TimeSpan.FromSeconds(((1775885 - processedincipits) / pageSize) * sw.Elapsed.TotalSeconds);
Console.WriteLine($"Completed {processedincipits} incipits. Time remaining: {timeRemaining}.");
if (incipits.Length < pageSize)
{
break;
}
}
}
public HashGenerationServiceInIncipitTable(RismDbContext dbContext)
{
this.dbContext = dbContext;
}
public HashGenerationServiceForDistinctIncipits(RismDbContext dbContext, PlaineAndEasieService plaineAndEasieService)
{
this.dbContext = dbContext;
this.plaineAndEasieService = plaineAndEasieService;
}
public void GenerateHashes(int numberOfGroups)
{
var incipitCache = new List <Incipit>();
for (var numberOfDimensions = 1; numberOfDimensions <= Constants.MaxNumberOfDimensionsForLsh; numberOfDimensions++)
{
var hashCache = new List <SpatialHash>();
dbContext = Dependencies.CreateContext(); //Recreate
Console.WriteLine($"Searching distinct melodies for {numberOfDimensions} dimensions...");
var distinctMelodies = GetDistinctMelodies(numberOfDimensions);
Console.WriteLine($"Found {distinctMelodies.Length} distinct melodies for {numberOfDimensions} dimensions.");
var patternNumber = 1;
foreach (var distinctIntervalPattern in distinctMelodies)
{
Console.WriteLine($"Generating {numberOfDimensions}-dimensional hashes for pattern {string.Join(" ", distinctIntervalPattern)} ({patternNumber++} of {distinctMelodies.Length}).");
var position = GetIncipitVector(distinctIntervalPattern);
var numberOfPlanes = 15; //There's no use to increase number of planes according to number of dimensions. Planes always divide space in two.
Console.WriteLine($"Searching incipit ids for pattern {string.Join(" ", distinctIntervalPattern)}...");
var allIncipitsForThisPattern = numberOfDimensions == 1 ? GetIncipitsByIntervalPatternFromDb(distinctIntervalPattern, incipitCache)
: GetIncipitsByIntervalPatternFromCache(distinctIntervalPattern, incipitCache);
foreach (var groupNumber in Enumerable.Range(1, numberOfGroups))
{
var lshAlgorithm = GetPlaneGroup(groupNumber, numberOfPlanes, numberOfDimensions);
var hash = lshAlgorithm.ComputeHash(position);
var spatialHash = new SpatialHash {
Id = $"{numberOfDimensions}-{groupNumber}-{hash}"
};
var existingHash = hashCache.FirstOrDefault(h => h.Id == spatialHash.Id);
if (existingHash == null)
{
dbContext.SpatialHashes.Add(spatialHash);
dbContext.SaveChanges();
hashCache.Add(spatialHash);
}
else
{
spatialHash = existingHash;
}
var skip = 0;
var pageSize = 300;
while (true)
{
Console.WriteLine($"Adding incipit ids for pattern {string.Join(" ", distinctIntervalPattern)} - group {groupNumber} ({skip}/{allIncipitsForThisPattern.Length})...");
var page = allIncipitsForThisPattern.Skip(skip).Take(pageSize).ToArray();
foreach (var incipit in page)
{
dbContext.SpatialHashIncipits.Add(new SpatialHashIncipit {
SpatialHashId = spatialHash.Id, IncipitId = incipit.Id
});
}
dbContext.SaveChanges();
if (page.Length < pageSize)
{
break;
}
skip += page.Length;
}
}
dbContext.SaveChanges();
dbContext = Dependencies.CreateContext(); //Recreate
}
dbContext.SaveChanges();
}
}
public MigrationService(RismDbContext dbContext, PlaineAndEasieService plaineAndEasieService)
{
this.dbContext = dbContext;
this.plaineAndEasieService = plaineAndEasieService;
}
