public async Task CalculateSimilarity(DumpMetainfo dumpA, bool force, DateTime timeFrom) { try { var swTotal = new Stopwatch(); swTotal.Start(); var resultA = await dumpRepo.GetResult(dumpA.BundleId, dumpA.DumpId); if (resultA == null) { return; // no results found. do nothing. } var allDumps = dumpRepo.GetAll().Where(x => x.Created >= timeFrom).OrderBy(x => x.Created); Console.WriteLine($"starting CalculateSimilarity for {allDumps.Count()} dumps; {dumpA} (TID:{Thread.CurrentThread.ManagedThreadId})"); int i = allDumps.Count(); var sw = new Stopwatch(); foreach (var dumpB in allDumps) { i--; sw.Start(); if (!force) { if (await relationShipRepo.GetRelationShip(dumpA.Id, dumpB.Id) != 0) { continue; // relationship already exists. skip! } } if (!PreSelectOnMetadata(dumpA, dumpB)) { continue; } var resultB = await dumpRepo.GetResult(dumpB.BundleId, dumpB.DumpId); if (resultB == null) { continue; } if (!PreSelectOnResults(resultA, resultB)) { continue; } CrashSimilarity crashSimilarity = CrashSimilarity.Calculate(resultA, resultB); // only store value if above a certain threshold to avoid unnecessary disk writes if (crashSimilarity.OverallSimilarity > 0.2) { await relationShipRepo.UpdateSimilarity(dumpA.Id, dumpB.Id, crashSimilarity); } sw.Stop(); //Console.WriteLine($"CalculateSimilarity.Finished for {dumpA}/{dumpB} ({i} to go...); (elapsed: {sw.Elapsed}) (TID:{Thread.CurrentThread.ManagedThreadId})"); sw.Reset(); } swTotal.Stop(); Console.WriteLine($"CalculateSimilarity.Finished for all {allDumps.Count()} dumps (total elapsed: {swTotal.Elapsed}); {dumpA} (TID:{Thread.CurrentThread.ManagedThreadId})"); } catch (Exception e) { Console.WriteLine(e.Message); } }
public async Task CalculateSimilarityAsync(DumpMetainfo dumpA, bool force, DateTime timeFrom) { try { var swTotal = new Stopwatch(); swTotal.Start(); if (!dumpRepo.IsPopulated) { Console.WriteLine($"CalculateSimilarity for {dumpA.Id} is blocked because dumpRepo is not yet fully populated..."); await Utility.BlockUntil(() => dumpRepo.IsPopulated); Console.WriteLine($"...continuing CalculateSimilarity for {dumpA.Id}."); } var resultA = await GetOrCreateMiniInfo(dumpA.Id); var allDumps = dumpRepo.GetAll().Where(x => x.Created >= timeFrom).OrderBy(x => x.Created); Console.WriteLine($"starting CalculateSimilarity for {allDumps.Count()} dumps; {dumpA} (TID:{Thread.CurrentThread.ManagedThreadId})"); foreach (var dumpB in allDumps) { if (!force) { var existingSimilarity = await relationShipRepo.GetRelationShip(dumpA.Id, dumpB.Id); if (existingSimilarity != 0) { // relationship already exists. skip! // but make sure the relationship is stored bi-directional await relationShipRepo.UpdateSimilarity(dumpA.Id, dumpB.Id, existingSimilarity); continue; } } if (!PreSelectOnMetadata(dumpA, dumpB)) { continue; } var resultB = await GetOrCreateMiniInfo(dumpB.Id); if (!PreSelectOnResults(resultA, resultB)) { continue; } CrashSimilarity crashSimilarity = CrashSimilarity.Calculate(resultA, resultB); // only store value if above a certain threshold to avoid unnecessary disk writes if (crashSimilarity.OverallSimilarity > 0.6) { await relationShipRepo.UpdateSimilarity(dumpA.Id, dumpB.Id, crashSimilarity.OverallSimilarity); } //Console.WriteLine($"CalculateSimilarity.Finished for {dumpA}/{dumpB} ({i} to go...); (elapsed: {sw.Elapsed}) (TID:{Thread.CurrentThread.ManagedThreadId})"); } await relationShipRepo.FlushDirtyRelationships(); swTotal.Stop(); Console.WriteLine($"CalculateSimilarity.Finished for all {allDumps.Count()} dumps (total elapsed: {swTotal.Elapsed}); {dumpA} (TID:{Thread.CurrentThread.ManagedThreadId})"); } catch (Exception e) { Console.Error.WriteLine($"CalculateSimilarity failed: {e}"); } }