public async Task CalculateSimilarity(DumpMetainfo dumpA, bool force, DateTime timeFrom)
{
try {
var swTotal = new Stopwatch();
swTotal.Start();
var resultA = await dumpRepo.GetResult(dumpA.BundleId, dumpA.DumpId);
if (resultA == null)
{
return; // no results found. do nothing.
}
var allDumps = dumpRepo.GetAll().Where(x => x.Created >= timeFrom).OrderBy(x => x.Created);
Console.WriteLine($"starting CalculateSimilarity for {allDumps.Count()} dumps; {dumpA} (TID:{Thread.CurrentThread.ManagedThreadId})");
int i = allDumps.Count();
var sw = new Stopwatch();
foreach (var dumpB in allDumps)
{
i--;
sw.Start();
if (!force)
{
if (await relationShipRepo.GetRelationShip(dumpA.Id, dumpB.Id) != 0)
{
continue; // relationship already exists. skip!
}
}
if (!PreSelectOnMetadata(dumpA, dumpB))
{
continue;
}
var resultB = await dumpRepo.GetResult(dumpB.BundleId, dumpB.DumpId);
if (resultB == null)
{
continue;
}
if (!PreSelectOnResults(resultA, resultB))
{
continue;
}
CrashSimilarity crashSimilarity = CrashSimilarity.Calculate(resultA, resultB);
// only store value if above a certain threshold to avoid unnecessary disk writes
if (crashSimilarity.OverallSimilarity > 0.2)
{
await relationShipRepo.UpdateSimilarity(dumpA.Id, dumpB.Id, crashSimilarity);
}
sw.Stop();
//Console.WriteLine($"CalculateSimilarity.Finished for {dumpA}/{dumpB} ({i} to go...); (elapsed: {sw.Elapsed}) (TID:{Thread.CurrentThread.ManagedThreadId})");
sw.Reset();
}
swTotal.Stop();
Console.WriteLine($"CalculateSimilarity.Finished for all {allDumps.Count()} dumps (total elapsed: {swTotal.Elapsed}); {dumpA} (TID:{Thread.CurrentThread.ManagedThreadId})");
} catch (Exception e) {
Console.WriteLine(e.Message);
}
}
public async Task <DumpMiniInfo> GetOrCreateMiniInfo(DumpIdentifier id)
{
if (dumpRepo.MiniInfoExists(id))
{
try {
var loadedMiniInfo = await dumpRepo.GetMiniInfo(id);
if (loadedMiniInfo.DumpSimilarityInfoVersion == CrashSimilarity.MiniInfoVersion)
{
return(loadedMiniInfo);
}
} catch {
Console.WriteLine($"could not load miniinfo for {id}. will re-create it.");
}
}
// no mini-info exists yet, or version is outdated. re-create.
var result = await dumpRepo.GetResult(id);
var miniInfo =
(result == null)
? new DumpMiniInfo() // just store an empty mini-info if result is null
: CrashSimilarity.SDResultToMiniInfo(result);
await dumpRepo.StoreMiniInfo(id, miniInfo);
return(miniInfo);
}
public async Task PublishFaultReport(DumpMetainfo dumpInfo)
{
var result = await dumpRepository.GetResult(dumpInfo.Id);
var faultReport = FaultReportCreator.CreateFaultReport(result);
var bundleInfo = bundleRepository.Get(dumpInfo.BundleId);
if (bundleInfo.CustomProperties.ContainsKey("sourceId"))
{
faultReport.SourceId = bundleRepository.Get(dumpInfo.BundleId).CustomProperties["sourceId"];
}
await faultReportSender.SendFaultReport(dumpInfo, faultReport);
}
public async Task PushAllResultsAsync(bool clean)
{
if (elasticClient == null)
{
throw new InvalidOperationException("ElasticSearch has not been initialized! Please verify that the settings specify a correct elastic search host.");
}
if (clean)
{
DeleteIndex();
CreateIndex();
}
IEnumerable <string> documentIds = GetAllDocumentIds();
int nErrorsLogged = 0;
var bundles = bundleRepo.GetAll();
if (bundles == null)
{
throw new InvalidOperationException("Bundle repository must be populated before pushing data into ES.");
}
// Note that this ES push can be improved significantly by using bulk operations to create the documents
foreach (BundleMetainfo bundle in bundles)
{
var dumps = dumpRepo.Get(bundle.BundleId);
if (dumps == null)
{
continue;
}
foreach (DumpMetainfo dump in dumps)
{
if (documentIds.Contains(bundle.BundleId + "/" + dump.DumpId))
{
continue;
}
SDResult result = await dumpRepo.GetResult(bundle.BundleId, dump.DumpId);
if (result != null)
{
bool success = await PushResultAsync(result, bundle, dump);
if (!success && nErrorsLogged < 20)
{
Console.WriteLine($"Failed to create document for {dump.BundleId}/{dump.DumpId}");
nErrorsLogged++;
}
}
}
}
}
public async Task Analyze(DumpMetainfo dumpInfo, string dumpFilePath, string analysisWorkingDir)
{
try {
dumpRepo.SetDumpStatus(dumpInfo.BundleId, dumpInfo.DumpId, DumpStatus.Analyzing);
if (dumpInfo.DumpType == DumpType.WindowsDump)
{
await AnalyzeWindows(dumpInfo, new DirectoryInfo(analysisWorkingDir), dumpFilePath);
}
else if (dumpInfo.DumpType == DumpType.LinuxCoreDump)
{
await LinuxAnalyzationAsync(dumpInfo, new DirectoryInfo(analysisWorkingDir), dumpFilePath);
}
else
{
throw new Exception("unknown dumptype. here be dragons");
}
dumpRepo.SetDumpStatus(dumpInfo.BundleId, dumpInfo.DumpId, DumpStatus.Finished);
// Re-fetch dump info as it was updated
dumpInfo = dumpRepo.Get(dumpInfo.BundleId, dumpInfo.DumpId);
SDResult result = await dumpRepo.GetResult(dumpInfo.BundleId, dumpInfo.DumpId);
if (result != null)
{
var bundle = bundleRepo.Get(dumpInfo.BundleId);
await elasticSearch.PushResultAsync(result, bundle, dumpInfo);
}
} catch (Exception e) {
Console.WriteLine(e.Message);
dumpRepo.SetDumpStatus(dumpInfo.BundleId, dumpInfo.DumpId, DumpStatus.Failed, e.ToString());
} finally {
if (settings.Value.DeleteDumpAfterAnalysis)
{
dumpStorage.DeleteDumpFile(dumpInfo.BundleId, dumpInfo.DumpId);
}
await notifications.NotifyDumpAnalysisFinished(dumpInfo);
similarityService.ScheduleSimilarityAnalysis(dumpInfo, false, DateTime.Now - TimeSpan.FromDays(90)); // last 90 days.
}
}
public async Task Analyze(DumpMetainfo dumpInfo, string dumpFilePath, string analysisWorkingDir)
{
try {
dumpRepo.SetDumpStatus(dumpInfo.BundleId, dumpInfo.DumpId, DumpStatus.Analyzing);
if (dumpInfo.DumpType == DumpType.WindowsDump)
{
await AnalyzeWindows(dumpInfo, new DirectoryInfo(analysisWorkingDir), dumpFilePath);
}
else if (dumpInfo.DumpType == DumpType.LinuxCoreDump)
{
await LinuxAnalyzationAsync(dumpInfo, new DirectoryInfo(analysisWorkingDir), dumpFilePath);
}
else
{
throw new Exception("unknown dumptype. here be dragons");
}
dumpRepo.SetDumpStatus(dumpInfo.BundleId, dumpInfo.DumpId, DumpStatus.Finished);
SDResult result = dumpRepo.GetResult(dumpInfo.BundleId, dumpInfo.DumpId, out string err);
if (result != null)
{
var bundle = bundleRepo.Get(dumpInfo.BundleId);
await elasticSearch.PushResultAsync(result, bundle, dumpInfo);
}
} catch (Exception e) {
Console.WriteLine(e.Message);
dumpRepo.SetDumpStatus(dumpInfo.BundleId, dumpInfo.DumpId, DumpStatus.Failed, e.ToString());
} finally {
if (settings.Value.DeleteDumpAfterAnalysis)
{
dumpStorage.DeleteDumpFile(dumpInfo.BundleId, dumpInfo.DumpId);
}
await notifications.NotifyDumpAnalysisFinished(dumpInfo);
}
}
public SDResult GetResult(string bundleId, string dumpId, out string error)
{
return(dumpRepo.GetResult(bundleId, dumpId, out error));
}
public async Task <string> GetMessage(DumpMetainfo dumpInfo)
{
var model = new SlackMessageViewModel();
var res = dumpRepo.GetResult(dumpInfo.BundleId, dumpInfo.DumpId, out string error);
var engine = new EngineFactory().ForEmbeddedResources(typeof(SlackMessageViewModel));
model.TopProperties.Add(dumpInfo.DumpType == DumpType.WindowsDump ? "Windows" : "Linux");
model.DumpFilename = Path.GetFileName(dumpInfo.DumpFileName);
model.Url = $"{superDumpUrl}/Home/Report?bundleId={dumpInfo.BundleId}&dumpId={dumpInfo.DumpId}";
if (res != null)
{
model.TopProperties.Add(res.SystemContext.ProcessArchitecture);
if (res.IsManagedProcess)
{
model.TopProperties.Add(".NET");
}
if (res.SystemContext.Modules.Any(x => x.FileName.Contains("jvm.dll")))
{
model.TopProperties.Add("Java");
}
if (res.SystemContext.Modules.Any(x => x.FileName.Contains("jvm.so")))
{
model.TopProperties.Add("Java");
}
if (res.SystemContext.Modules.Any(x => x.FileName.Contains("iiscore.dll")))
{
model.TopProperties.Add("IIS");
}
if (res.SystemContext.Modules.Any(x => x.FileName.Contains("nginx.so")))
{
model.TopProperties.Add("NGINX");
}
if (res.SystemContext.Modules.Any(x => x.FileName.Contains("httpd/modules")))
{
model.TopProperties.Add("Apache");
}
if (res.SystemContext.Modules.Any(x => x.FileName.Contains("node.exe")))
{
model.TopProperties.Add("Node.js");
}
var agentModules = res.SystemContext.Modules.Where(x => x.Tags.Any(t => t.Equals(SDTag.DynatraceAgentTag))).Select(m => m.ToString());
model.AgentModules = agentModules.ToList();
model.NumManagedExceptions = res.ThreadInformation.Count(x => x.Value.Tags.Any(t => t.Equals(SDTag.ManagedExceptionTag)));
model.NumNativeExceptions = res.ThreadInformation.Count(x => x.Value.Tags.Any(t => t.Equals(SDTag.NativeExceptionTag)));
model.NumAssertErrors = res.ThreadInformation.Count(x => x.Value.Tags.Any(t => t.Equals(SDTag.AssertionErrorTag)));
SDThread managedExceptionThread = res.ThreadInformation.Values.FirstOrDefault(x => x.Tags.Any(t => t.Equals(SDTag.ManagedExceptionTag)));
SDClrException clrException = managedExceptionThread?.LastException;
if (clrException != null)
{
model.TopException = clrException.Type;
model.Stacktrace = clrException.StackTrace.ToString();
}
if (res.LastEvent != null && !res.LastEvent.Description.Contains("Break instruction")) // break instruction events are useless
{
model.LastEvent = $"{res.LastEvent.Type}: {res.LastEvent.Description}";
}
}
return(await engine.CompileRenderAsync("SlackMessage", model));
}
public SDResult GetResult(string bundleId, string dumpId)
{
return(dumpRepo.GetResult(bundleId, dumpId));
}
public async Task <SDResult> GetResult(DumpIdentifier id)
{
return(await dumpRepo.GetResult(id));
}
public async Task <SDResult> GetResult(string bundleId, string dumpId)
{
return(await dumpRepo.GetResult(bundleId, dumpId));
}
public async Task PushAllResultsAsync(bool clean)
{
if (elasticClient == null)
{
throw new InvalidOperationException("ElasticSearch has not been initialized! Please verify that the settings specify a correct elastic search host.");
}
await BlockIfBundleRepoNotReady("ElasticSearchService.PushAllResultsAsync");
if (clean)
{
DeleteIndex();
CreateIndex();
// since we are clean, we can do everything in one bulk
var dumps = dumpRepo.GetAll().OrderByDescending(x => x.Created);
foreach (var dumpsBatch in dumps.Batch(100))
{
var tasks = dumpsBatch.Select(x => Task.Run(async() => new { res = await dumpRepo.GetResult(x.Id), bundleInfo = bundleRepo.Get(x.BundleId), dumpInfo = x }));
var results = (await Task.WhenAll(tasks)).Where(x => x.res != null);
Console.WriteLine($"pushing {results.Count()} results into elasticsearch");
var sdResults = results.Select(x => ElasticSDResult.FromResultOrDefault(x.res, x.bundleInfo, x.dumpInfo, pathHelper)).Where(x => x != null);
await PushBulk(sdResults);
}
return;
}
IEnumerable <string> documentIds = GetAllDocumentIds();
int nErrorsLogged = 0;
var bundles = bundleRepo.GetAll();
if (bundles == null)
{
throw new InvalidOperationException("Bundle repository must be populated before pushing data into ES.");
}
// In order to check if a dump has already been added, we go through them all and add one at the time
// There is potential to optimize this and still do a bulk add.
foreach (BundleMetainfo bundle in bundles)
{
var dumps = dumpRepo.Get(bundle.BundleId);
if (dumps == null)
{
continue;
}
foreach (DumpMetainfo dump in dumps)
{
if (documentIds.Contains(bundle.BundleId + "/" + dump.DumpId))
{
continue;
}
SDResult result = await dumpRepo.GetResult(dump.Id);
if (result != null)
{
bool success = await PushResultAsync(result, bundle, dump);
if (!success && nErrorsLogged < 20)
{
Console.WriteLine($"Failed to create document for {dump.BundleId}/{dump.DumpId}");
nErrorsLogged++;
}
}
}
}
}
