public static bool CanIncreaseMemoryUsageForThread()
{
var memoryInfo = MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader();
var memoryAssumedFreeOrCheapToFree = memoryInfo.AvailableMemoryForProcessing;
var allocatedForProcessing = GetTotalCurrentlyAllocatedForProcessing();
return(memoryAssumedFreeOrCheapToFree >= allocatedForProcessing);
}
private static MessageDetails OutOfMemoryDetails(Exception exception)
{
var memoryInfo = MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader();
return(new MessageDetails
{
Message = $"Managed memory: {new Size(AbstractLowMemoryMonitor.GetManagedMemoryInBytes(), SizeUnit.Bytes)}, " +
$"Unmanaged allocations: {new Size(AbstractLowMemoryMonitor.GetUnmanagedAllocationsInBytes(), SizeUnit.Bytes)}, " +
$"Shared clean: {memoryInfo.SharedCleanMemory}, " +
$"Working set: {memoryInfo.WorkingSet}, " +
$"Available memory: {memoryInfo.AvailableMemory}, " +
$"Calculated Available memory: {memoryInfo.AvailableWithoutTotalCleanMemory}, " +
$"Total memory: {memoryInfo.TotalPhysicalMemory} {Environment.NewLine}" +
$"Error: {exception}"
});
}
private static DynamicJsonValue MemoryStatsInternal()
{
var memInfo = MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader();
var memoryUsageRecords = MemoryInformation.GetMemoryUsageRecords();
long totalMapping = 0;
var fileMappingByDir = new Dictionary <string, Dictionary <string, ConcurrentDictionary <IntPtr, long> > >();
var fileMappingSizesByDir = new Dictionary <string, long>();
foreach (var mapping in NativeMemory.FileMapping)
{
var dir = Path.GetDirectoryName(mapping.Key);
if (fileMappingByDir.TryGetValue(dir, out Dictionary <string, ConcurrentDictionary <IntPtr, long> > value) == false)
{
value = new Dictionary <string, ConcurrentDictionary <IntPtr, long> >();
fileMappingByDir[dir] = value;
}
value[mapping.Key] = mapping.Value.Value.Info;
foreach (var singleMapping in mapping.Value.Value.Info)
{
fileMappingSizesByDir.TryGetValue(dir, out long prevSize);
fileMappingSizesByDir[dir] = prevSize + singleMapping.Value;
totalMapping += singleMapping.Value;
}
}
var prefixLength = LongestCommonPrefixLength(new List <string>(fileMappingSizesByDir.Keys));
var fileMappings = new DynamicJsonArray();
foreach (var sizes in fileMappingSizesByDir.OrderByDescending(x => x.Value))
{
if (fileMappingByDir.TryGetValue(sizes.Key, out Dictionary <string, ConcurrentDictionary <IntPtr, long> > value))
{
var details = new DynamicJsonValue();
var dir = new DynamicJsonValue
{
[nameof(MemoryInfoMappingItem.Directory)] = sizes.Key.Substring(prefixLength),
[nameof(MemoryInfoMappingItem.TotalDirectorySize)] = sizes.Value,
[nameof(MemoryInfoMappingItem.HumaneTotalDirectorySize)] = Size.Humane(sizes.Value),
[nameof(MemoryInfoMappingItem.Details)] = details
};
foreach (var file in value.OrderBy(x => x.Key))
{
long totalMapped = 0;
var dja = new DynamicJsonArray();
var dic = new Dictionary <long, long>();
foreach (var mapping in file.Value)
{
totalMapped += mapping.Value;
dic.TryGetValue(mapping.Value, out long prev);
dic[mapping.Value] = prev + 1;
}
foreach (var maps in dic)
{
dja.Add(new DynamicJsonValue
{
[nameof(MemoryInfoMappingDetails.Size)] = maps.Key,
[nameof(MemoryInfoMappingDetails.Count)] = maps.Value
});
}
var fileSize = GetFileSize(file.Key);
details[Path.GetFileName(file.Key)] = new DynamicJsonValue
{
[nameof(MemoryInfoMappingFileInfo.FileSize)] = fileSize,
[nameof(MemoryInfoMappingFileInfo.HumaneFileSize)] = Size.Humane(fileSize),
[nameof(MemoryInfoMappingFileInfo.TotalMapped)] = totalMapped,
[nameof(MemoryInfoMappingFileInfo.HumaneTotalMapped)] = Size.Humane(totalMapped),
[nameof(MemoryInfoMappingFileInfo.Mappings)] = dja
};
}
fileMappings.Add(dir);
}
}
long totalUnmanagedAllocations = NativeMemory.TotalAllocatedMemory;
var threads = new DynamicJsonArray();
foreach (var stats in NativeMemory.AllThreadStats
.Where(x => x.IsThreadAlive())
.GroupBy(x => x.Name)
.OrderByDescending(x => x.Sum(y => y.TotalAllocated)))
{
var unmanagedAllocations = stats.Sum(x => x.TotalAllocated);
var ids = new DynamicJsonArray(stats.OrderByDescending(x => x.TotalAllocated).Select(x => new DynamicJsonValue
{
["Id"] = x.UnmanagedThreadId,
["ManagedThreadId"] = x.ManagedThreadId,
["Allocations"] = x.TotalAllocated,
["HumaneAllocations"] = Size.Humane(x.TotalAllocated)
}));
var groupStats = new DynamicJsonValue
{
["Name"] = stats.Key,
["Allocations"] = unmanagedAllocations,
["HumaneAllocations"] = Size.Humane(unmanagedAllocations)
};
if (ids.Count == 1)
{
var threadStats = stats.First();
groupStats["Id"] = threadStats.UnmanagedThreadId;
groupStats["ManagedThreadId"] = threadStats.ManagedThreadId;
}
else
{
groupStats["Ids"] = ids;
}
threads.Add(groupStats);
}
long managedMemoryInBytes = AbstractLowMemoryMonitor.GetManagedMemoryInBytes();
long workingSetInBytes = memInfo.WorkingSet.GetValue(SizeUnit.Bytes);
var dirtyMemoryState = MemoryInformation.GetDirtyMemoryState();
var encryptionBuffers = EncryptionBuffersPool.Instance.GetStats();
var djv = new DynamicJsonValue
{
[nameof(MemoryInfo.WorkingSet)] = workingSetInBytes,
[nameof(MemoryInfo.TotalUnmanagedAllocations)] = totalUnmanagedAllocations,
[nameof(MemoryInfo.ManagedAllocations)] = managedMemoryInBytes,
[nameof(MemoryInfo.TotalMemoryMapped)] = totalMapping,
[nameof(MemoryInfo.TotalScratchDirtyMemory)] = Size.Humane(memInfo.TotalScratchDirtyMemory.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.PhysicalMem)] = Size.Humane(memInfo.TotalPhysicalMemory.GetValue(SizeUnit.Bytes)),
[nameof(DirtyMemoryState.IsHighDirty)] = dirtyMemoryState.IsHighDirty,
["DirtyMemory"] = Size.Humane(dirtyMemoryState.TotalDirtyInBytes),
[nameof(MemoryInfo.AvailableMemory)] = Size.Humane(memInfo.AvailableMemory.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.AvailableMemoryForProcessing)] = Size.Humane(memInfo.AvailableMemoryForProcessing.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.HighMemLastOneMinute)] = Size.Humane(memoryUsageRecords.High.LastOneMinute.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.LowMemLastOneMinute)] = Size.Humane(memoryUsageRecords.Low.LastOneMinute.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.HighMemLastFiveMinute)] = Size.Humane(memoryUsageRecords.High.LastFiveMinutes.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.LowMemLastFiveMinute)] = Size.Humane(memoryUsageRecords.Low.LastFiveMinutes.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.HighMemSinceStartup)] = Size.Humane(memoryUsageRecords.High.SinceStartup.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.LowMemSinceStartup)] = Size.Humane(memoryUsageRecords.Low.SinceStartup.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.Humane)] = new DynamicJsonValue
{
[nameof(MemoryInfoHumane.WorkingSet)] = Size.Humane(workingSetInBytes),
[nameof(MemoryInfoHumane.TotalUnmanagedAllocations)] = Size.Humane(totalUnmanagedAllocations),
["EncryptionBuffers"] = Size.Humane(encryptionBuffers.TotalSize),
[nameof(MemoryInfoHumane.ManagedAllocations)] = Size.Humane(managedMemoryInBytes),
[nameof(MemoryInfoHumane.TotalMemoryMapped)] = Size.Humane(totalMapping)
},
["Threads"] = threads,
[nameof(MemoryInfo.Mappings)] = fileMappings
};
return(djv);
}
public static bool TryIncreasingMemoryUsageForThread(NativeMemory.ThreadStats threadStats,
ref Size currentMaximumAllowedMemory,
Size currentlyInUse,
bool isRunningOn32Bits,
Logger logger,
out ProcessMemoryUsage currentUsage)
{
if (isRunningOn32Bits)
{
currentUsage = null;
return(false);
}
// we run out our memory quota, so we need to see if we can increase it or break
var memoryInfo = MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader();
currentUsage = GetProcessMemoryUsage(memoryInfo);
var memoryAssumedFreeOrCheapToFree = memoryInfo.AvailableMemoryForProcessing;
// there isn't enough available memory to try, we want to leave some out for other things
if (memoryAssumedFreeOrCheapToFree <
Size.Min(memoryInfo.TotalPhysicalMemory / 50, new Size(1, SizeUnit.Gigabytes)))
{
if (logger.IsInfoEnabled)
{
logger.Info(
$"{threadStats.Name} which is already using {currentlyInUse}/{currentMaximumAllowedMemory} and the system has " +
$"{memoryInfo.AvailableMemoryForProcessing}/{memoryInfo.TotalPhysicalMemory} free RAM. Also have ~{memoryInfo.SharedCleanMemory} in mmap " +
"files that can be cleanly released, not enough to proceed in batch.");
}
return(false);
}
// If there isn't enough here to double our current allocation, we won't allocate any more
// we do this check in this way to prevent multiple indexes of hitting this at the
// same time and each thinking that they have enough space
if (memoryAssumedFreeOrCheapToFree < currentMaximumAllowedMemory)
{
if (logger.IsInfoEnabled)
{
logger.Info(
$"{threadStats} which is already using {currentlyInUse}/{currentMaximumAllowedMemory} and the system has" +
$"{memoryInfo.AvailableMemoryForProcessing}/{memoryInfo.TotalPhysicalMemory} free RAM. Also have ~{memoryInfo.SharedCleanMemory} in mmap " +
"files that can be cleanly released, not enough to proceed in batch.");
}
return(false);
}
var allocatedForProcessing = GetTotalCurrentlyAllocatedForProcessing();
if (memoryAssumedFreeOrCheapToFree < allocatedForProcessing)
{
if (logger.IsInfoEnabled)
{
logger.Info(
$"Total allocated memory for processing: {allocatedForProcessing}, free memory: {memoryAssumedFreeOrCheapToFree}, " +
$"total memory: {memoryInfo.TotalPhysicalMemory}.");
}
return(false);
}
// even though we have twice as much memory as we have current allocated, we will
// only increment by 16MB to avoid over allocation by multiple indexes. This way,
// we'll check often as we go along this
var oldBudget = currentMaximumAllowedMemory;
currentMaximumAllowedMemory = currentlyInUse + new Size(16, SizeUnit.Megabytes);
if (logger.IsInfoEnabled)
{
logger.Info(
$"Increasing memory budget for {threadStats.Name} which is using {currentlyInUse}/{oldBudget} and the system has" +
$"{memoryAssumedFreeOrCheapToFree}/{memoryInfo.TotalPhysicalMemory} free RAM with {memoryInfo.SharedCleanMemory} in mmap " +
$"files that can be cleanly released. Budget increased to {currentMaximumAllowedMemory}");
}
return(true);
}
public override MemoryInfoResult GetMemoryInfoOnce()
{
return(MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader());
}
private static void WriteMemoryStats(BlittableJsonTextWriter writer, JsonOperationContext context, bool includeThreads, bool includeMappings)
{
writer.WriteStartObject();
var memInfo = MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader();
long managedMemoryInBytes = AbstractLowMemoryMonitor.GetManagedMemoryInBytes();
long totalUnmanagedAllocations = NativeMemory.TotalAllocatedMemory;
var encryptionBuffers = EncryptionBuffersPool.Instance.GetStats();
var dirtyMemoryState = MemoryInformation.GetDirtyMemoryState();
var memoryUsageRecords = MemoryInformation.GetMemoryUsageRecords();
long totalMapping = 0;
var fileMappingByDir = new Dictionary <string, Dictionary <string, ConcurrentDictionary <IntPtr, long> > >();
var fileMappingSizesByDir = new Dictionary <string, long>();
foreach (var mapping in NativeMemory.FileMapping)
{
var dir = Path.GetDirectoryName(mapping.Key);
if (fileMappingByDir.TryGetValue(dir, out Dictionary <string, ConcurrentDictionary <IntPtr, long> > value) == false)
{
value = new Dictionary <string, ConcurrentDictionary <IntPtr, long> >();
fileMappingByDir[dir] = value;
}
value[mapping.Key] = mapping.Value.Value.Info;
foreach (var singleMapping in mapping.Value.Value.Info)
{
fileMappingSizesByDir.TryGetValue(dir, out long prevSize);
fileMappingSizesByDir[dir] = prevSize + singleMapping.Value;
totalMapping += singleMapping.Value;
}
}
var djv = new DynamicJsonValue
{
[nameof(MemoryInfo.PhysicalMemory)] = memInfo.TotalPhysicalMemory.ToString(),
[nameof(MemoryInfo.WorkingSet)] = memInfo.WorkingSet.ToString(),
[nameof(MemoryInfo.ManagedAllocations)] = Size.Humane(managedMemoryInBytes),
[nameof(MemoryInfo.UnmanagedAllocations)] = Size.Humane(totalUnmanagedAllocations),
[nameof(MemoryInfo.EncryptionBuffers)] = Size.Humane(encryptionBuffers.TotalSize),
[nameof(MemoryInfo.MemoryMapped)] = Size.Humane(totalMapping),
[nameof(MemoryInfo.ScratchDirtyMemory)] = memInfo.TotalScratchDirtyMemory.ToString(),
[nameof(MemoryInfo.IsHighDirty)] = dirtyMemoryState.IsHighDirty,
[nameof(MemoryInfo.DirtyMemory)] = Size.Humane(dirtyMemoryState.TotalDirtyInBytes),
[nameof(MemoryInfo.AvailableMemory)] = Size.Humane(memInfo.AvailableMemory.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.AvailableMemoryForProcessing)] = memInfo.AvailableMemoryForProcessing.ToString(),
[nameof(MemoryInfo.HighMemLastOneMinute)] = memoryUsageRecords.High.LastOneMinute.ToString(),
[nameof(MemoryInfo.LowMemLastOneMinute)] = memoryUsageRecords.Low.LastOneMinute.ToString(),
[nameof(MemoryInfo.HighMemLastFiveMinute)] = memoryUsageRecords.High.LastFiveMinutes.ToString(),
[nameof(MemoryInfo.LowMemLastFiveMinute)] = memoryUsageRecords.Low.LastFiveMinutes.ToString(),
[nameof(MemoryInfo.HighMemSinceStartup)] = memoryUsageRecords.High.SinceStartup.ToString(),
[nameof(MemoryInfo.LowMemSinceStartup)] = memoryUsageRecords.Low.SinceStartup.ToString(),
};
writer.WritePropertyName(nameof(MemoryInformation));
context.Write(writer, djv);
writer.WriteComma();
writer.WritePropertyName("Threads");
writer.WriteStartArray();
WriteThreads(includeThreads, writer, context);
writer.WriteEndArray();
writer.WriteComma();
writer.WritePropertyName(nameof(MemoryInfo.Mappings));
writer.WriteStartArray();
WriteMappings(includeMappings, writer, context, fileMappingSizesByDir, fileMappingByDir);
writer.WriteEndArray();
writer.WriteEndObject();
}