public DumpMetainfo GetDump(DumpIdentifier id)
{
if (!string.IsNullOrEmpty(id.BundleId) && !string.IsNullOrEmpty(id.DumpId))
{
return(dumpRepo.Get(id));
}
else
{
return(null);
}
}
public DumpMetainfo GetDump(string bundleId, string dumpId)
{
if (!string.IsNullOrEmpty(bundleId) && !string.IsNullOrEmpty(dumpId))
{
return(dumpRepo.Get(bundleId, dumpId));
}
else
{
return(null);
}
}
public async Task AnalyzeAsync(DumpMetainfo dumpInfo, string analysisWorkingDir)
{
await bundleRepo.BlockIfBundleRepoNotReady($"AnalysisService.Analyze for {dumpInfo.Id}");
dumpRepo.SetDumpStatus(dumpInfo.Id, DumpStatus.Analyzing, string.Empty);
AnalyzerState state = AnalyzerState.Initialized;
foreach (AnalyzerJob analyzerJob in analyzerPipeline.Analyzers)
{
if (state == AnalyzerState.Cancelled)
{
break;
}
state = await analyzerJob.AnalyzeDump(dumpInfo, analysisWorkingDir, state);
}
if (state == AnalyzerState.Failed || state == AnalyzerState.Initialized)
{
dumpRepo.SetDumpStatus(dumpInfo.Id, DumpStatus.Failed);
}
else
{
dumpRepo.SetDumpStatus(dumpInfo.Id, DumpStatus.Finished);
dumpInfo = dumpRepo.Get(dumpInfo.Id);
foreach (PostAnalysisJob job in analyzerPipeline.PostAnalysisJobs)
{
await job.AnalyzeDump(dumpInfo);
}
}
await notifications.NotifyDumpAnalysisFinished(dumpInfo);
}
public void ScheduleSimilarityAnalysis(DumpIdentifier dumpId, bool force, DateTime timeFrom)
{
if (!settings.Value.SimilarityDetectionEnabled)
{
return;
}
ScheduleSimilarityAnalysis(dumpRepo.Get(dumpId), force, timeFrom);
}
public async Task AnalyzeAsync(DumpMetainfo dumpInfo, string dumpFilePath, string analysisWorkingDir)
{
await BlockIfBundleRepoNotReady($"AnalysisService.Analyze for {dumpInfo.Id}");
try {
dumpRepo.SetDumpStatus(dumpInfo.Id, DumpStatus.Analyzing);
if (new FileInfo(dumpFilePath).Length == 0)
{
dumpRepo.SetDumpStatus(dumpInfo.Id, DumpStatus.Failed, "The primary dump file is empty!");
return;
}
if (dumpInfo.DumpType == DumpType.WindowsDump)
{
await AnalyzeWindows(dumpInfo, new DirectoryInfo(analysisWorkingDir), dumpFilePath);
}
else if (dumpInfo.DumpType == DumpType.LinuxCoreDump)
{
await LinuxAnalyzationAsync(dumpInfo, new DirectoryInfo(analysisWorkingDir), dumpFilePath);
}
else
{
throw new Exception("unknown dumptype. here be dragons");
}
// Re-fetch dump info as it was updated
dumpInfo = dumpRepo.Get(dumpInfo.Id);
SDResult result = await dumpRepo.GetResultAndThrow(dumpInfo.Id);
if (result != null)
{
dumpRepo.WriteResult(dumpInfo.Id, result);
dumpRepo.SetDumpStatus(dumpInfo.Id, DumpStatus.Finished);
var bundle = bundleRepo.Get(dumpInfo.BundleId);
await elasticSearch.PushResultAsync(result, bundle, dumpInfo);
}
} catch (Exception e) {
Console.WriteLine(e.Message);
dumpRepo.SetDumpStatus(dumpInfo.Id, DumpStatus.Failed, e.ToString());
} finally {
if (settings.Value.DeleteDumpAfterAnalysis)
{
dumpStorage.DeleteDumpFile(dumpInfo.Id);
}
await notifications.NotifyDumpAnalysisFinished(dumpInfo);
similarityService.ScheduleSimilarityAnalysis(dumpInfo, false, DateTime.Now - TimeSpan.FromDays(settings.Value.SimilarityDetectionMaxDays));
}
}
public void RemoveOldDumps()
{
foreach (var bundle in bundleRepo.GetAll())
{
foreach (var dump in dumpRepo.Get(bundle.BundleId))
{
if (dump.Created < DateTime.Now.Subtract(TimeSpan.FromDays(settings.DumpRetentionDays)))
{
RemoveOldDumps(dump);
}
}
}
}
public async Task PushAllResultsAsync(bool clean)
{
if (elasticClient == null)
{
throw new InvalidOperationException("ElasticSearch has not been initialized! Please verify that the settings specify a correct elastic search host.");
}
if (clean)
{
DeleteIndex();
CreateIndex();
}
IEnumerable <string> documentIds = GetAllDocumentIds();
int nErrorsLogged = 0;
var bundles = bundleRepo.GetAll();
if (bundles == null)
{
throw new InvalidOperationException("Bundle repository must be populated before pushing data into ES.");
}
// Note that this ES push can be improved significantly by using bulk operations to create the documents
foreach (BundleMetainfo bundle in bundles)
{
var dumps = dumpRepo.Get(bundle.BundleId);
if (dumps == null)
{
continue;
}
foreach (DumpMetainfo dump in dumps)
{
if (documentIds.Contains(bundle.BundleId + "/" + dump.DumpId))
{
continue;
}
SDResult result = await dumpRepo.GetResult(bundle.BundleId, dump.DumpId);
if (result != null)
{
bool success = await PushResultAsync(result, bundle, dump);
if (!success && nErrorsLogged < 20)
{
Console.WriteLine($"Failed to create document for {dump.BundleId}/{dump.DumpId}");
nErrorsLogged++;
}
}
}
}
}
public async Task Analyze(DumpMetainfo dumpInfo, string dumpFilePath, string analysisWorkingDir)
{
try {
dumpRepo.SetDumpStatus(dumpInfo.BundleId, dumpInfo.DumpId, DumpStatus.Analyzing);
if (dumpInfo.DumpType == DumpType.WindowsDump)
{
await AnalyzeWindows(dumpInfo, new DirectoryInfo(analysisWorkingDir), dumpFilePath);
}
else if (dumpInfo.DumpType == DumpType.LinuxCoreDump)
{
await LinuxAnalyzationAsync(dumpInfo, new DirectoryInfo(analysisWorkingDir), dumpFilePath);
}
else
{
throw new Exception("unknown dumptype. here be dragons");
}
dumpRepo.SetDumpStatus(dumpInfo.BundleId, dumpInfo.DumpId, DumpStatus.Finished);
// Re-fetch dump info as it was updated
dumpInfo = dumpRepo.Get(dumpInfo.BundleId, dumpInfo.DumpId);
SDResult result = await dumpRepo.GetResult(dumpInfo.BundleId, dumpInfo.DumpId);
if (result != null)
{
var bundle = bundleRepo.Get(dumpInfo.BundleId);
await elasticSearch.PushResultAsync(result, bundle, dumpInfo);
}
} catch (Exception e) {
Console.WriteLine(e.Message);
dumpRepo.SetDumpStatus(dumpInfo.BundleId, dumpInfo.DumpId, DumpStatus.Failed, e.ToString());
} finally {
if (settings.Value.DeleteDumpAfterAnalysis)
{
dumpStorage.DeleteDumpFile(dumpInfo.BundleId, dumpInfo.DumpId);
}
await notifications.NotifyDumpAnalysisFinished(dumpInfo);
similarityService.ScheduleSimilarityAnalysis(dumpInfo, false, DateTime.Now - TimeSpan.FromDays(90)); // last 90 days.
}
}
public void RemoveOldDumps()
{
// Check if the jiraIssueRepository is populated to avoid deleting dumps with open issues
if (settings.UseJiraIntegration && !jiraIssueRepository.IsPopulated)
{
return;
}
var jiraExtensionTime = settings.UseJiraIntegration ? TimeSpan.FromDays(settings.JiraIntegrationSettings.JiraDumpRetentionTimeExtensionDays) : TimeSpan.Zero;
foreach (var bundle in bundleRepo.GetAll())
{
if (bundle == null)
{
continue;
}
foreach (var dump in dumpRepo.Get(bundle.BundleId))
{
if (dump == null)
{
continue;
}
if (settings.UseJiraIntegration && jiraIssueRepository.HasBundleOpenIssues(bundle.BundleId))
{
// do not set the dump deletion date if it would shorten the current retention time
if (jiraExtensionTime > dump.PlannedDeletionDate - DateTime.Now)
{
dumpRepo.SetPlannedDeletionDate(dump.Id, DateTime.Now + jiraExtensionTime, JiraRetentionExtensionReason);
}
}
else if (dump.PlannedDeletionDate < DateTime.Now)
{
RemoveOldDumps(dump);
}
}
}
}
private static async Task <DumpViewModel> ToDumpViewModel(DumpIdentifier id, DumpRepository dumpRepo, BundleRepository bundleRepo, SimilarityService similarityService = null)
{
return(await ToDumpViewModel(dumpRepo.Get(id), dumpRepo, bundleRepo, similarityService));
}
public async Task PushAllResultsAsync(bool clean)
{
if (elasticClient == null)
{
throw new InvalidOperationException("ElasticSearch has not been initialized! Please verify that the settings specify a correct elastic search host.");
}
await BlockIfBundleRepoNotReady("ElasticSearchService.PushAllResultsAsync");
if (clean)
{
DeleteIndex();
CreateIndex();
// since we are clean, we can do everything in one bulk
var dumps = dumpRepo.GetAll().OrderByDescending(x => x.Created);
foreach (var dumpsBatch in dumps.Batch(100))
{
var tasks = dumpsBatch.Select(x => Task.Run(async() => new { res = await dumpRepo.GetResult(x.Id), bundleInfo = bundleRepo.Get(x.BundleId), dumpInfo = x }));
var results = (await Task.WhenAll(tasks)).Where(x => x.res != null);
Console.WriteLine($"pushing {results.Count()} results into elasticsearch");
var sdResults = results.Select(x => ElasticSDResult.FromResultOrDefault(x.res, x.bundleInfo, x.dumpInfo, pathHelper)).Where(x => x != null);
await PushBulk(sdResults);
}
return;
}
IEnumerable <string> documentIds = GetAllDocumentIds();
int nErrorsLogged = 0;
var bundles = bundleRepo.GetAll();
if (bundles == null)
{
throw new InvalidOperationException("Bundle repository must be populated before pushing data into ES.");
}
// In order to check if a dump has already been added, we go through them all and add one at the time
// There is potential to optimize this and still do a bulk add.
foreach (BundleMetainfo bundle in bundles)
{
var dumps = dumpRepo.Get(bundle.BundleId);
if (dumps == null)
{
continue;
}
foreach (DumpMetainfo dump in dumps)
{
if (documentIds.Contains(bundle.BundleId + "/" + dump.DumpId))
{
continue;
}
SDResult result = await dumpRepo.GetResult(dump.Id);
if (result != null)
{
bool success = await PushResultAsync(result, bundle, dump);
if (!success && nErrorsLogged < 20)
{
Console.WriteLine($"Failed to create document for {dump.BundleId}/{dump.DumpId}");
nErrorsLogged++;
}
}
}
}
}
private async Task <DumpViewModel> ToDumpViewModel(DumpIdentifier id, bool includeSimilarities = false)
{
return(await ToDumpViewModel(dumpRepo.Get(id), includeSimilarities));
}
public void ScheduleSimilarityAnalysis(DumpIdentifier dumpId, bool force, DateTime timeFrom)
{
ScheduleSimilarityAnalysis(dumpRepo.Get(dumpId), force, timeFrom);
}