private TestResourceSnapshot GetTestResourceSnapshot(TestStage testStage, IAssemblyInfo assemblyInfo)
{
var timeStamp = DateTime.UtcNow;
var assemblyName = GetAssemblyShortName(assemblyInfo);
var cpuUsage = _metricCacher.GetValue(MetricCacher.Keys.Server.CpuUsage, _cpuUsageCalculator.Calculate);
var memoryInfo = _metricCacher.GetValue <MemoryInfoResult>(MetricCacher.Keys.Server.MemoryInfoExtended);
var snapshot = new TestResourceSnapshot
{
TotalScratchAllocatedMemory = new Size(MemoryInformation.GetTotalScratchAllocatedMemory(), SizeUnit.Bytes).GetValue(SizeUnit.Megabytes),
TotalDirtyMemory = new Size(MemoryInformation.GetDirtyMemoryState().TotalDirtyInBytes, SizeUnit.Bytes).GetValue(SizeUnit.Megabytes),
IsHighDirty = MemoryInformation.GetDirtyMemoryState().IsHighDirty,
TestStage = testStage,
Timestamp = timeStamp.ToString("o"),
AssemblyName = assemblyName,
MachineCpuUsage = (long)cpuUsage.MachineCpuUsage,
ProcessCpuUsage = (long)cpuUsage.ProcessCpuUsage,
ProcessMemoryUsageInMb = memoryInfo.WorkingSet.GetValue(SizeUnit.Megabytes),
TotalMemoryInMb = memoryInfo.TotalPhysicalMemory.GetValue(SizeUnit.Megabytes),
TotalCommittableMemoryInMb = memoryInfo.TotalCommittableMemory.GetValue(SizeUnit.Megabytes),
AvailableMemoryInMb = memoryInfo.AvailableMemory.GetValue(SizeUnit.Megabytes),
CurrentCommitChargeInMb = memoryInfo.CurrentCommitCharge.GetValue(SizeUnit.Megabytes),
SharedCleanMemoryInMb = memoryInfo.SharedCleanMemory.GetValue(SizeUnit.Megabytes),
TotalScratchDirtyMemory = memoryInfo.TotalScratchDirtyMemory.GetValue(SizeUnit.Megabytes)
};
return(snapshot);
}
private MemoryMetrics GetMemoryMetrics()
{
var result = new MemoryMetrics();
var memoryInfoResult = Server.MetricCacher.GetValue <MemoryInfoResult>(MetricCacher.Keys.Server.MemoryInfoExtended);
result.InstalledMemoryInMb = memoryInfoResult.InstalledMemory.GetValue(SizeUnit.Megabytes);
result.PhysicalMemoryInMb = memoryInfoResult.TotalPhysicalMemory.GetValue(SizeUnit.Megabytes);
result.AllocatedMemoryInMb = memoryInfoResult.WorkingSet.GetValue(SizeUnit.Megabytes);
result.LowMemorySeverity = LowMemoryNotification.Instance.IsLowMemory(memoryInfoResult,
new LowMemoryMonitor(), out _);
result.TotalSwapSizeInMb = memoryInfoResult.TotalSwapSize.GetValue(SizeUnit.Megabytes);
result.TotalSwapUsageInMb = memoryInfoResult.TotalSwapUsage.GetValue(SizeUnit.Megabytes);
result.WorkingSetSwapUsageInMb = memoryInfoResult.WorkingSetSwapUsage.GetValue(SizeUnit.Megabytes);
var totalDirtyInBytes = MemoryInformation.GetDirtyMemoryState().TotalDirtyInBytes;
result.TotalDirtyInMb = new Size(totalDirtyInBytes, SizeUnit.Bytes).GetValue(SizeUnit.Megabytes);
return(result);
}
private static DynamicJsonValue MemoryStatsInternal()
{
var memInfo = MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader();
var memoryUsageRecords = MemoryInformation.GetMemoryUsageRecords();
long totalMapping = 0;
var fileMappingByDir = new Dictionary <string, Dictionary <string, ConcurrentDictionary <IntPtr, long> > >();
var fileMappingSizesByDir = new Dictionary <string, long>();
foreach (var mapping in NativeMemory.FileMapping)
{
var dir = Path.GetDirectoryName(mapping.Key);
if (fileMappingByDir.TryGetValue(dir, out Dictionary <string, ConcurrentDictionary <IntPtr, long> > value) == false)
{
value = new Dictionary <string, ConcurrentDictionary <IntPtr, long> >();
fileMappingByDir[dir] = value;
}
value[mapping.Key] = mapping.Value.Value.Info;
foreach (var singleMapping in mapping.Value.Value.Info)
{
fileMappingSizesByDir.TryGetValue(dir, out long prevSize);
fileMappingSizesByDir[dir] = prevSize + singleMapping.Value;
totalMapping += singleMapping.Value;
}
}
var prefixLength = LongestCommonPrefixLength(new List <string>(fileMappingSizesByDir.Keys));
var fileMappings = new DynamicJsonArray();
foreach (var sizes in fileMappingSizesByDir.OrderByDescending(x => x.Value))
{
if (fileMappingByDir.TryGetValue(sizes.Key, out Dictionary <string, ConcurrentDictionary <IntPtr, long> > value))
{
var details = new DynamicJsonValue();
var dir = new DynamicJsonValue
{
[nameof(MemoryInfoMappingItem.Directory)] = sizes.Key.Substring(prefixLength),
[nameof(MemoryInfoMappingItem.TotalDirectorySize)] = sizes.Value,
[nameof(MemoryInfoMappingItem.HumaneTotalDirectorySize)] = Size.Humane(sizes.Value),
[nameof(MemoryInfoMappingItem.Details)] = details
};
foreach (var file in value.OrderBy(x => x.Key))
{
long totalMapped = 0;
var dja = new DynamicJsonArray();
var dic = new Dictionary <long, long>();
foreach (var mapping in file.Value)
{
totalMapped += mapping.Value;
dic.TryGetValue(mapping.Value, out long prev);
dic[mapping.Value] = prev + 1;
}
foreach (var maps in dic)
{
dja.Add(new DynamicJsonValue
{
[nameof(MemoryInfoMappingDetails.Size)] = maps.Key,
[nameof(MemoryInfoMappingDetails.Count)] = maps.Value
});
}
var fileSize = GetFileSize(file.Key);
details[Path.GetFileName(file.Key)] = new DynamicJsonValue
{
[nameof(MemoryInfoMappingFileInfo.FileSize)] = fileSize,
[nameof(MemoryInfoMappingFileInfo.HumaneFileSize)] = Size.Humane(fileSize),
[nameof(MemoryInfoMappingFileInfo.TotalMapped)] = totalMapped,
[nameof(MemoryInfoMappingFileInfo.HumaneTotalMapped)] = Size.Humane(totalMapped),
[nameof(MemoryInfoMappingFileInfo.Mappings)] = dja
};
}
fileMappings.Add(dir);
}
}
long totalUnmanagedAllocations = NativeMemory.TotalAllocatedMemory;
var threads = new DynamicJsonArray();
foreach (var stats in NativeMemory.AllThreadStats
.Where(x => x.IsThreadAlive())
.GroupBy(x => x.Name)
.OrderByDescending(x => x.Sum(y => y.TotalAllocated)))
{
var unmanagedAllocations = stats.Sum(x => x.TotalAllocated);
var ids = new DynamicJsonArray(stats.OrderByDescending(x => x.TotalAllocated).Select(x => new DynamicJsonValue
{
["Id"] = x.UnmanagedThreadId,
["ManagedThreadId"] = x.ManagedThreadId,
["Allocations"] = x.TotalAllocated,
["HumaneAllocations"] = Size.Humane(x.TotalAllocated)
}));
var groupStats = new DynamicJsonValue
{
["Name"] = stats.Key,
["Allocations"] = unmanagedAllocations,
["HumaneAllocations"] = Size.Humane(unmanagedAllocations)
};
if (ids.Count == 1)
{
var threadStats = stats.First();
groupStats["Id"] = threadStats.UnmanagedThreadId;
groupStats["ManagedThreadId"] = threadStats.ManagedThreadId;
}
else
{
groupStats["Ids"] = ids;
}
threads.Add(groupStats);
}
long managedMemoryInBytes = AbstractLowMemoryMonitor.GetManagedMemoryInBytes();
long workingSetInBytes = memInfo.WorkingSet.GetValue(SizeUnit.Bytes);
var dirtyMemoryState = MemoryInformation.GetDirtyMemoryState();
var encryptionBuffers = EncryptionBuffersPool.Instance.GetStats();
var djv = new DynamicJsonValue
{
[nameof(MemoryInfo.WorkingSet)] = workingSetInBytes,
[nameof(MemoryInfo.TotalUnmanagedAllocations)] = totalUnmanagedAllocations,
[nameof(MemoryInfo.ManagedAllocations)] = managedMemoryInBytes,
[nameof(MemoryInfo.TotalMemoryMapped)] = totalMapping,
[nameof(MemoryInfo.TotalScratchDirtyMemory)] = Size.Humane(memInfo.TotalScratchDirtyMemory.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.PhysicalMem)] = Size.Humane(memInfo.TotalPhysicalMemory.GetValue(SizeUnit.Bytes)),
[nameof(DirtyMemoryState.IsHighDirty)] = dirtyMemoryState.IsHighDirty,
["DirtyMemory"] = Size.Humane(dirtyMemoryState.TotalDirtyInBytes),
[nameof(MemoryInfo.AvailableMemory)] = Size.Humane(memInfo.AvailableMemory.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.AvailableMemoryForProcessing)] = Size.Humane(memInfo.AvailableMemoryForProcessing.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.HighMemLastOneMinute)] = Size.Humane(memoryUsageRecords.High.LastOneMinute.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.LowMemLastOneMinute)] = Size.Humane(memoryUsageRecords.Low.LastOneMinute.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.HighMemLastFiveMinute)] = Size.Humane(memoryUsageRecords.High.LastFiveMinutes.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.LowMemLastFiveMinute)] = Size.Humane(memoryUsageRecords.Low.LastFiveMinutes.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.HighMemSinceStartup)] = Size.Humane(memoryUsageRecords.High.SinceStartup.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.LowMemSinceStartup)] = Size.Humane(memoryUsageRecords.Low.SinceStartup.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.Humane)] = new DynamicJsonValue
{
[nameof(MemoryInfoHumane.WorkingSet)] = Size.Humane(workingSetInBytes),
[nameof(MemoryInfoHumane.TotalUnmanagedAllocations)] = Size.Humane(totalUnmanagedAllocations),
["EncryptionBuffers"] = Size.Humane(encryptionBuffers.TotalSize),
[nameof(MemoryInfoHumane.ManagedAllocations)] = Size.Humane(managedMemoryInBytes),
[nameof(MemoryInfoHumane.TotalMemoryMapped)] = Size.Humane(totalMapping)
},
["Threads"] = threads,
[nameof(MemoryInfo.Mappings)] = fileMappings
};
return(djv);
}
public override DirtyMemoryState GetDirtyMemoryState()
{
return(MemoryInformation.GetDirtyMemoryState());
}
private static void WriteMemoryStats(BlittableJsonTextWriter writer, JsonOperationContext context, bool includeThreads, bool includeMappings)
{
writer.WriteStartObject();
var memInfo = MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader();
long managedMemoryInBytes = AbstractLowMemoryMonitor.GetManagedMemoryInBytes();
long totalUnmanagedAllocations = NativeMemory.TotalAllocatedMemory;
var encryptionBuffers = EncryptionBuffersPool.Instance.GetStats();
var dirtyMemoryState = MemoryInformation.GetDirtyMemoryState();
var memoryUsageRecords = MemoryInformation.GetMemoryUsageRecords();
long totalMapping = 0;
var fileMappingByDir = new Dictionary <string, Dictionary <string, ConcurrentDictionary <IntPtr, long> > >();
var fileMappingSizesByDir = new Dictionary <string, long>();
foreach (var mapping in NativeMemory.FileMapping)
{
var dir = Path.GetDirectoryName(mapping.Key);
if (fileMappingByDir.TryGetValue(dir, out Dictionary <string, ConcurrentDictionary <IntPtr, long> > value) == false)
{
value = new Dictionary <string, ConcurrentDictionary <IntPtr, long> >();
fileMappingByDir[dir] = value;
}
value[mapping.Key] = mapping.Value.Value.Info;
foreach (var singleMapping in mapping.Value.Value.Info)
{
fileMappingSizesByDir.TryGetValue(dir, out long prevSize);
fileMappingSizesByDir[dir] = prevSize + singleMapping.Value;
totalMapping += singleMapping.Value;
}
}
var djv = new DynamicJsonValue
{
[nameof(MemoryInfo.PhysicalMemory)] = memInfo.TotalPhysicalMemory.ToString(),
[nameof(MemoryInfo.WorkingSet)] = memInfo.WorkingSet.ToString(),
[nameof(MemoryInfo.ManagedAllocations)] = Size.Humane(managedMemoryInBytes),
[nameof(MemoryInfo.UnmanagedAllocations)] = Size.Humane(totalUnmanagedAllocations),
[nameof(MemoryInfo.EncryptionBuffers)] = Size.Humane(encryptionBuffers.TotalSize),
[nameof(MemoryInfo.MemoryMapped)] = Size.Humane(totalMapping),
[nameof(MemoryInfo.ScratchDirtyMemory)] = memInfo.TotalScratchDirtyMemory.ToString(),
[nameof(MemoryInfo.IsHighDirty)] = dirtyMemoryState.IsHighDirty,
[nameof(MemoryInfo.DirtyMemory)] = Size.Humane(dirtyMemoryState.TotalDirtyInBytes),
[nameof(MemoryInfo.AvailableMemory)] = Size.Humane(memInfo.AvailableMemory.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.AvailableMemoryForProcessing)] = memInfo.AvailableMemoryForProcessing.ToString(),
[nameof(MemoryInfo.HighMemLastOneMinute)] = memoryUsageRecords.High.LastOneMinute.ToString(),
[nameof(MemoryInfo.LowMemLastOneMinute)] = memoryUsageRecords.Low.LastOneMinute.ToString(),
[nameof(MemoryInfo.HighMemLastFiveMinute)] = memoryUsageRecords.High.LastFiveMinutes.ToString(),
[nameof(MemoryInfo.LowMemLastFiveMinute)] = memoryUsageRecords.Low.LastFiveMinutes.ToString(),
[nameof(MemoryInfo.HighMemSinceStartup)] = memoryUsageRecords.High.SinceStartup.ToString(),
[nameof(MemoryInfo.LowMemSinceStartup)] = memoryUsageRecords.Low.SinceStartup.ToString(),
};
writer.WritePropertyName(nameof(MemoryInformation));
context.Write(writer, djv);
writer.WriteComma();
writer.WritePropertyName("Threads");
writer.WriteStartArray();
WriteThreads(includeThreads, writer, context);
writer.WriteEndArray();
writer.WriteComma();
writer.WritePropertyName(nameof(MemoryInfo.Mappings));
writer.WriteStartArray();
WriteMappings(includeMappings, writer, context, fileMappingSizesByDir, fileMappingByDir);
writer.WriteEndArray();
writer.WriteEndObject();
}
protected override Gauge32 GetData()
{
var totalDirtyInBytes = MemoryInformation.GetDirtyMemoryState().TotalDirtyInBytes;
return(new Gauge32(new Size(totalDirtyInBytes, SizeUnit.Bytes).GetValue(SizeUnit.Megabytes)));
}