public DumpMetainfo GetDump(DumpIdentifier id) { if (!string.IsNullOrEmpty(id.BundleId) && !string.IsNullOrEmpty(id.DumpId)) { return(dumpRepo.Get(id)); } else { return(null); } }
public DumpMetainfo GetDump(string bundleId, string dumpId) { if (!string.IsNullOrEmpty(bundleId) && !string.IsNullOrEmpty(dumpId)) { return(dumpRepo.Get(bundleId, dumpId)); } else { return(null); } }
public async Task AnalyzeAsync(DumpMetainfo dumpInfo, string analysisWorkingDir) { await bundleRepo.BlockIfBundleRepoNotReady($"AnalysisService.Analyze for {dumpInfo.Id}"); dumpRepo.SetDumpStatus(dumpInfo.Id, DumpStatus.Analyzing, string.Empty); AnalyzerState state = AnalyzerState.Initialized; foreach (AnalyzerJob analyzerJob in analyzerPipeline.Analyzers) { if (state == AnalyzerState.Cancelled) { break; } state = await analyzerJob.AnalyzeDump(dumpInfo, analysisWorkingDir, state); } if (state == AnalyzerState.Failed || state == AnalyzerState.Initialized) { dumpRepo.SetDumpStatus(dumpInfo.Id, DumpStatus.Failed); } else { dumpRepo.SetDumpStatus(dumpInfo.Id, DumpStatus.Finished); dumpInfo = dumpRepo.Get(dumpInfo.Id); foreach (PostAnalysisJob job in analyzerPipeline.PostAnalysisJobs) { await job.AnalyzeDump(dumpInfo); } } await notifications.NotifyDumpAnalysisFinished(dumpInfo); }
public void ScheduleSimilarityAnalysis(DumpIdentifier dumpId, bool force, DateTime timeFrom) { if (!settings.Value.SimilarityDetectionEnabled) { return; } ScheduleSimilarityAnalysis(dumpRepo.Get(dumpId), force, timeFrom); }
public async Task AnalyzeAsync(DumpMetainfo dumpInfo, string dumpFilePath, string analysisWorkingDir) { await BlockIfBundleRepoNotReady($"AnalysisService.Analyze for {dumpInfo.Id}"); try { dumpRepo.SetDumpStatus(dumpInfo.Id, DumpStatus.Analyzing); if (new FileInfo(dumpFilePath).Length == 0) { dumpRepo.SetDumpStatus(dumpInfo.Id, DumpStatus.Failed, "The primary dump file is empty!"); return; } if (dumpInfo.DumpType == DumpType.WindowsDump) { await AnalyzeWindows(dumpInfo, new DirectoryInfo(analysisWorkingDir), dumpFilePath); } else if (dumpInfo.DumpType == DumpType.LinuxCoreDump) { await LinuxAnalyzationAsync(dumpInfo, new DirectoryInfo(analysisWorkingDir), dumpFilePath); } else { throw new Exception("unknown dumptype. here be dragons"); } // Re-fetch dump info as it was updated dumpInfo = dumpRepo.Get(dumpInfo.Id); SDResult result = await dumpRepo.GetResultAndThrow(dumpInfo.Id); if (result != null) { dumpRepo.WriteResult(dumpInfo.Id, result); dumpRepo.SetDumpStatus(dumpInfo.Id, DumpStatus.Finished); var bundle = bundleRepo.Get(dumpInfo.BundleId); await elasticSearch.PushResultAsync(result, bundle, dumpInfo); } } catch (Exception e) { Console.WriteLine(e.Message); dumpRepo.SetDumpStatus(dumpInfo.Id, DumpStatus.Failed, e.ToString()); } finally { if (settings.Value.DeleteDumpAfterAnalysis) { dumpStorage.DeleteDumpFile(dumpInfo.Id); } await notifications.NotifyDumpAnalysisFinished(dumpInfo); similarityService.ScheduleSimilarityAnalysis(dumpInfo, false, DateTime.Now - TimeSpan.FromDays(settings.Value.SimilarityDetectionMaxDays)); } }
public void RemoveOldDumps() { foreach (var bundle in bundleRepo.GetAll()) { foreach (var dump in dumpRepo.Get(bundle.BundleId)) { if (dump.Created < DateTime.Now.Subtract(TimeSpan.FromDays(settings.DumpRetentionDays))) { RemoveOldDumps(dump); } } } }
public async Task PushAllResultsAsync(bool clean) { if (elasticClient == null) { throw new InvalidOperationException("ElasticSearch has not been initialized! Please verify that the settings specify a correct elastic search host."); } if (clean) { DeleteIndex(); CreateIndex(); } IEnumerable <string> documentIds = GetAllDocumentIds(); int nErrorsLogged = 0; var bundles = bundleRepo.GetAll(); if (bundles == null) { throw new InvalidOperationException("Bundle repository must be populated before pushing data into ES."); } // Note that this ES push can be improved significantly by using bulk operations to create the documents foreach (BundleMetainfo bundle in bundles) { var dumps = dumpRepo.Get(bundle.BundleId); if (dumps == null) { continue; } foreach (DumpMetainfo dump in dumps) { if (documentIds.Contains(bundle.BundleId + "/" + dump.DumpId)) { continue; } SDResult result = await dumpRepo.GetResult(bundle.BundleId, dump.DumpId); if (result != null) { bool success = await PushResultAsync(result, bundle, dump); if (!success && nErrorsLogged < 20) { Console.WriteLine($"Failed to create document for {dump.BundleId}/{dump.DumpId}"); nErrorsLogged++; } } } } }
public async Task Analyze(DumpMetainfo dumpInfo, string dumpFilePath, string analysisWorkingDir) { try { dumpRepo.SetDumpStatus(dumpInfo.BundleId, dumpInfo.DumpId, DumpStatus.Analyzing); if (dumpInfo.DumpType == DumpType.WindowsDump) { await AnalyzeWindows(dumpInfo, new DirectoryInfo(analysisWorkingDir), dumpFilePath); } else if (dumpInfo.DumpType == DumpType.LinuxCoreDump) { await LinuxAnalyzationAsync(dumpInfo, new DirectoryInfo(analysisWorkingDir), dumpFilePath); } else { throw new Exception("unknown dumptype. here be dragons"); } dumpRepo.SetDumpStatus(dumpInfo.BundleId, dumpInfo.DumpId, DumpStatus.Finished); // Re-fetch dump info as it was updated dumpInfo = dumpRepo.Get(dumpInfo.BundleId, dumpInfo.DumpId); SDResult result = await dumpRepo.GetResult(dumpInfo.BundleId, dumpInfo.DumpId); if (result != null) { var bundle = bundleRepo.Get(dumpInfo.BundleId); await elasticSearch.PushResultAsync(result, bundle, dumpInfo); } } catch (Exception e) { Console.WriteLine(e.Message); dumpRepo.SetDumpStatus(dumpInfo.BundleId, dumpInfo.DumpId, DumpStatus.Failed, e.ToString()); } finally { if (settings.Value.DeleteDumpAfterAnalysis) { dumpStorage.DeleteDumpFile(dumpInfo.BundleId, dumpInfo.DumpId); } await notifications.NotifyDumpAnalysisFinished(dumpInfo); similarityService.ScheduleSimilarityAnalysis(dumpInfo, false, DateTime.Now - TimeSpan.FromDays(90)); // last 90 days. } }
public void RemoveOldDumps() { // Check if the jiraIssueRepository is populated to avoid deleting dumps with open issues if (settings.UseJiraIntegration && !jiraIssueRepository.IsPopulated) { return; } var jiraExtensionTime = settings.UseJiraIntegration ? TimeSpan.FromDays(settings.JiraIntegrationSettings.JiraDumpRetentionTimeExtensionDays) : TimeSpan.Zero; foreach (var bundle in bundleRepo.GetAll()) { if (bundle == null) { continue; } foreach (var dump in dumpRepo.Get(bundle.BundleId)) { if (dump == null) { continue; } if (settings.UseJiraIntegration && jiraIssueRepository.HasBundleOpenIssues(bundle.BundleId)) { // do not set the dump deletion date if it would shorten the current retention time if (jiraExtensionTime > dump.PlannedDeletionDate - DateTime.Now) { dumpRepo.SetPlannedDeletionDate(dump.Id, DateTime.Now + jiraExtensionTime, JiraRetentionExtensionReason); } } else if (dump.PlannedDeletionDate < DateTime.Now) { RemoveOldDumps(dump); } } } }
private static async Task <DumpViewModel> ToDumpViewModel(DumpIdentifier id, DumpRepository dumpRepo, BundleRepository bundleRepo, SimilarityService similarityService = null) { return(await ToDumpViewModel(dumpRepo.Get(id), dumpRepo, bundleRepo, similarityService)); }
public async Task PushAllResultsAsync(bool clean) { if (elasticClient == null) { throw new InvalidOperationException("ElasticSearch has not been initialized! Please verify that the settings specify a correct elastic search host."); } await BlockIfBundleRepoNotReady("ElasticSearchService.PushAllResultsAsync"); if (clean) { DeleteIndex(); CreateIndex(); // since we are clean, we can do everything in one bulk var dumps = dumpRepo.GetAll().OrderByDescending(x => x.Created); foreach (var dumpsBatch in dumps.Batch(100)) { var tasks = dumpsBatch.Select(x => Task.Run(async() => new { res = await dumpRepo.GetResult(x.Id), bundleInfo = bundleRepo.Get(x.BundleId), dumpInfo = x })); var results = (await Task.WhenAll(tasks)).Where(x => x.res != null); Console.WriteLine($"pushing {results.Count()} results into elasticsearch"); var sdResults = results.Select(x => ElasticSDResult.FromResultOrDefault(x.res, x.bundleInfo, x.dumpInfo, pathHelper)).Where(x => x != null); await PushBulk(sdResults); } return; } IEnumerable <string> documentIds = GetAllDocumentIds(); int nErrorsLogged = 0; var bundles = bundleRepo.GetAll(); if (bundles == null) { throw new InvalidOperationException("Bundle repository must be populated before pushing data into ES."); } // In order to check if a dump has already been added, we go through them all and add one at the time // There is potential to optimize this and still do a bulk add. foreach (BundleMetainfo bundle in bundles) { var dumps = dumpRepo.Get(bundle.BundleId); if (dumps == null) { continue; } foreach (DumpMetainfo dump in dumps) { if (documentIds.Contains(bundle.BundleId + "/" + dump.DumpId)) { continue; } SDResult result = await dumpRepo.GetResult(dump.Id); if (result != null) { bool success = await PushResultAsync(result, bundle, dump); if (!success && nErrorsLogged < 20) { Console.WriteLine($"Failed to create document for {dump.BundleId}/{dump.DumpId}"); nErrorsLogged++; } } } } }
private async Task <DumpViewModel> ToDumpViewModel(DumpIdentifier id, bool includeSimilarities = false) { return(await ToDumpViewModel(dumpRepo.Get(id), includeSimilarities)); }
public void ScheduleSimilarityAnalysis(DumpIdentifier dumpId, bool force, DateTime timeFrom) { ScheduleSimilarityAnalysis(dumpRepo.Get(dumpId), force, timeFrom); }