public static ElasticSDResult FromResult(SDResult result, BundleMetainfo bundleInfo, DumpMetainfo dumpInfo)
{
ElasticSDResult eResult = new ElasticSDResult()
{
BundleId = dumpInfo.BundleId,
DumpId = dumpInfo.DumpId,
Timestamp = dumpInfo.Created,
Type = dumpInfo.DumpType.ToString(),
Executable = (result.SystemContext as SDCDSystemContext)?.FileName ?? "",
IsManaged = result.IsManagedProcess,
ProcessArchitecture = result.SystemContext.ProcessArchitecture,
SystemArchitecture = result.SystemContext.SystemArchitecture,
NrThreads = result.ThreadInformation.Count,
LastEventDescription = result.LastEvent?.Description,
LoadedModules = result.SystemContext.DistinctModules().Select(m => m.FileName).Aggregate("", (m1, m2) => m1 + " " + m2),
LoadedModulesVersioned = result.SystemContext.DistinctModules().Select(m => $"{m.FileName}:{m.Version ?? "-"}").Aggregate("", (m1, m2) => m1 + " " + m2),
DynatraceLoadedModulesVersioned = result.SystemContext.DistinctModules().Where(m => m.Tags.Contains(SDTag.DynatraceAgentTag)).Select(m => $"{m.FileName}:{m.Version ?? "-"}").Aggregate("", (m1, m2) => m1 + " " + m2),
ExitType = result.LastEvent?.Type ?? ""
};
bundleInfo.CustomProperties.TryGetValue("ref", out string reference);
eResult.Reference = reference;
if (dumpInfo.Finished != null && dumpInfo.Started != null)
{
int durationSecs = (int)dumpInfo.Finished.Subtract(dumpInfo.Started).TotalSeconds;
if (durationSecs > 0)
{
// Only if a valid duration is present
eResult.AnalyzationDuration = durationSecs;
}
}
return(eResult);
}
internal IEnumerable <ElasticSDResult> SearchDumpsByJson(string jsonQuery)
{
var result = elasticClient.LowLevel.Search <SearchResponse <dynamic> >(jsonQuery);
if (!result.IsValid)
{
throw new Exception($"elastic search query failed: {result.DebugInformation}");
}
foreach (var res in result.Documents)
{
ElasticSDResult deserialized = null;
try {
string str = Convert.ToString(res);
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(str))) {
deserialized = elasticClient.RequestResponseSerializer.Deserialize <ElasticSDResult>(stream);
}
} catch (Exception e) {
Console.WriteLine("error deserializing elasticsearch result");
}
if (deserialized != null)
{
yield return(deserialized);
}
}
//SearchRequest searchRequest;
//using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(jsonQuery))) {
// searchRequest = elasticClient.RequestResponseSerializer.Deserialize<SearchRequest>(stream);
//}
//var result = elasticClient.Search<ElasticSDResult>(searchRequest);
//return Enumerable.Empty<ElasticSDResult>();
}
public async Task <bool> PushResultAsync(SDResult result, BundleMetainfo bundleInfo, DumpMetainfo dumpInfo)
{
if (elasticClient != null)
{
var response = await elasticClient.CreateAsync(ElasticSDResult.FromResult(result, bundleInfo, dumpInfo, pathHelper));
if (!response.Created)
{
return(false);
}
}
return(true);
}
public static ElasticSDResult FromResult(SDResult result, BundleMetainfo bundleInfo, DumpMetainfo dumpInfo, PathHelper pathHelper)
{
if (result == null)
{
throw new ArgumentNullException("result");
}
var eResult = new ElasticSDResult()
{
BundleId = dumpInfo.BundleId,
DumpId = dumpInfo.DumpId,
Timestamp = dumpInfo.Created,
Type = dumpInfo.DumpType.ToString(),
Executable = (result.SystemContext as SDCDSystemContext)?.FileName ?? "",
IsManaged = result.IsManagedProcess,
ProcessArchitecture = result.SystemContext?.ProcessArchitecture,
SystemArchitecture = result.SystemContext?.SystemArchitecture,
NrThreads = result.ThreadInformation != null ? result.ThreadInformation.Count : 0,
LastEventDescription = result.LastEvent?.Description,
LoadedModules = result.SystemContext?.DistinctModules().Select(m => m.FileName).Aggregate("", (m1, m2) => m1 + " " + m2),
LoadedModulesVersioned = result.SystemContext?.DistinctModules().Select(m => $"{m.FileName}:{m.Version ?? "-"}").Aggregate("", (m1, m2) => m1 + " " + m2),
DynatraceLoadedModulesVersioned = result.SystemContext?.DistinctModules().Where(m => m.Tags.Contains(SDTag.DynatraceAgentTag)).Select(m => $"{m.FileName}:{m.Version ?? "-"}").Aggregate("", (m1, m2) => m1 + " " + m2),
ExitType = result.LastEvent?.Type ?? ""
};
bundleInfo.CustomProperties.TryGetValue("ref", out string reference);
eResult.Reference = reference;
bundleInfo.CustomProperties.TryGetValue("tenantId", out string tenantId);
eResult.TenantId = tenantId;
eResult.FaultingStacktrace = result.GetErrorOrLastExecutingThread()?.ToText();
eResult.Stacktraces = ""; // TODO
if (dumpInfo.Finished != null && dumpInfo.Started != null)
{
int durationSecs = (int)dumpInfo.Finished.Subtract(dumpInfo.Started).TotalSeconds;
if (durationSecs > 0)
{
// Only if a valid duration is present
eResult.AnalyzationDuration = durationSecs;
}
}
long dumpSize = ComputeDumpFileSizeKb(bundleInfo, dumpInfo, pathHelper);
if (dumpSize >= 0)
{
eResult.DumpSizeKb = dumpSize;
}
return(eResult);
}
private static async Task <DumpViewModel> ToDumpViewModel(ElasticSDResult elasticSDResult, DumpRepository dumpRepo, BundleRepository bundleRepo, SimilarityService similarityService = null)
{
return(await ToDumpViewModel(elasticSDResult.DumpIdentifier, dumpRepo, bundleRepo, similarityService));
}
public async Task <bool> PushResultAsync(SDResult result, BundleMetainfo bundleInfo, DumpMetainfo dumpInfo)
{
if (elasticClient != null)
{
new Nest.CreateRequest <ElasticSDResult>(RESULT_IDX);
var response = await elasticClient.CreateAsync(new CreateDescriptor <ElasticSDResult>(ElasticSDResult.FromResult(result, bundleInfo, dumpInfo, pathHelper)));
if (response.Result != Result.Created)
{
return(false);
}
}
return(true);
}
public async Task PushAllResultsAsync(bool clean)
{
if (elasticClient == null)
{
throw new InvalidOperationException("ElasticSearch has not been initialized! Please verify that the settings specify a correct elastic search host.");
}
await BlockIfBundleRepoNotReady("ElasticSearchService.PushAllResultsAsync");
if (clean)
{
DeleteIndex();
CreateIndex();
// since we are clean, we can do everything in one bulk
var dumps = dumpRepo.GetAll().OrderByDescending(x => x.Created);
foreach (var dumpsBatch in dumps.Batch(100))
{
var tasks = dumpsBatch.Select(x => Task.Run(async() => new { res = await dumpRepo.GetResult(x.Id), bundleInfo = bundleRepo.Get(x.BundleId), dumpInfo = x }));
var results = (await Task.WhenAll(tasks)).Where(x => x.res != null);
Console.WriteLine($"pushing {results.Count()} results into elasticsearch");
var sdResults = results.Select(x => ElasticSDResult.FromResultOrDefault(x.res, x.bundleInfo, x.dumpInfo, pathHelper)).Where(x => x != null);
await PushBulk(sdResults);
}
return;
}
IEnumerable <string> documentIds = GetAllDocumentIds();
int nErrorsLogged = 0;
var bundles = bundleRepo.GetAll();
if (bundles == null)
{
throw new InvalidOperationException("Bundle repository must be populated before pushing data into ES.");
}
// In order to check if a dump has already been added, we go through them all and add one at the time
// There is potential to optimize this and still do a bulk add.
foreach (BundleMetainfo bundle in bundles)
{
var dumps = dumpRepo.Get(bundle.BundleId);
if (dumps == null)
{
continue;
}
foreach (DumpMetainfo dump in dumps)
{
if (documentIds.Contains(bundle.BundleId + "/" + dump.DumpId))
{
continue;
}
SDResult result = await dumpRepo.GetResult(dump.Id);
if (result != null)
{
bool success = await PushResultAsync(result, bundle, dump);
if (!success && nErrorsLogged < 20)
{
Console.WriteLine($"Failed to create document for {dump.BundleId}/{dump.DumpId}");
nErrorsLogged++;
}
}
}
}
}
public async Task <bool> PushResultAsync(SDResult result, BundleMetainfo bundleInfo, DumpMetainfo dumpInfo)
{
try {
if (elasticClient != null && result != null)
{
new Nest.CreateRequest <ElasticSDResult>(RESULT_IDX);
var response = await elasticClient.CreateAsync(new CreateDescriptor <ElasticSDResult>(ElasticSDResult.FromResult(result, bundleInfo, dumpInfo, pathHelper)));
if (response.Result != Result.Created)
{
Console.WriteLine($"PushResultAsync failed for {dumpInfo.Id}. DebugInformation: {response.DebugInformation}");
return(false);
}
}
return(true);
} catch (Exception e) {
Console.WriteLine($"PushResultAsync failed for {dumpInfo.Id} with exception: {e}");
return(false);
}
}
private async Task <DumpViewModel> ToDumpViewModel(ElasticSDResult elasticSDResult, bool includeSimilarities = false)
{
return(await ToDumpViewModel(elasticSDResult.DumpIdentifier, includeSimilarities));
}