public static bool CanIncreaseMemoryUsageForThread() { var memoryInfo = MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader(); var memoryAssumedFreeOrCheapToFree = memoryInfo.AvailableMemoryForProcessing; var allocatedForProcessing = GetTotalCurrentlyAllocatedForProcessing(); return(memoryAssumedFreeOrCheapToFree >= allocatedForProcessing); }
private static MessageDetails OutOfMemoryDetails(Exception exception) { var memoryInfo = MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader(); return(new MessageDetails { Message = $"Managed memory: {new Size(AbstractLowMemoryMonitor.GetManagedMemoryInBytes(), SizeUnit.Bytes)}, " + $"Unmanaged allocations: {new Size(AbstractLowMemoryMonitor.GetUnmanagedAllocationsInBytes(), SizeUnit.Bytes)}, " + $"Shared clean: {memoryInfo.SharedCleanMemory}, " + $"Working set: {memoryInfo.WorkingSet}, " + $"Available memory: {memoryInfo.AvailableMemory}, " + $"Calculated Available memory: {memoryInfo.AvailableWithoutTotalCleanMemory}, " + $"Total memory: {memoryInfo.TotalPhysicalMemory} {Environment.NewLine}" + $"Error: {exception}" }); }
private static DynamicJsonValue MemoryStatsInternal() { var memInfo = MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader(); var memoryUsageRecords = MemoryInformation.GetMemoryUsageRecords(); long totalMapping = 0; var fileMappingByDir = new Dictionary <string, Dictionary <string, ConcurrentDictionary <IntPtr, long> > >(); var fileMappingSizesByDir = new Dictionary <string, long>(); foreach (var mapping in NativeMemory.FileMapping) { var dir = Path.GetDirectoryName(mapping.Key); if (fileMappingByDir.TryGetValue(dir, out Dictionary <string, ConcurrentDictionary <IntPtr, long> > value) == false) { value = new Dictionary <string, ConcurrentDictionary <IntPtr, long> >(); fileMappingByDir[dir] = value; } value[mapping.Key] = mapping.Value.Value.Info; foreach (var singleMapping in mapping.Value.Value.Info) { fileMappingSizesByDir.TryGetValue(dir, out long prevSize); fileMappingSizesByDir[dir] = prevSize + singleMapping.Value; totalMapping += singleMapping.Value; } } var prefixLength = LongestCommonPrefixLength(new List <string>(fileMappingSizesByDir.Keys)); var fileMappings = new DynamicJsonArray(); foreach (var sizes in fileMappingSizesByDir.OrderByDescending(x => x.Value)) { if (fileMappingByDir.TryGetValue(sizes.Key, out Dictionary <string, ConcurrentDictionary <IntPtr, long> > value)) { var details = new DynamicJsonValue(); var dir = new DynamicJsonValue { [nameof(MemoryInfoMappingItem.Directory)] = sizes.Key.Substring(prefixLength), [nameof(MemoryInfoMappingItem.TotalDirectorySize)] = sizes.Value, [nameof(MemoryInfoMappingItem.HumaneTotalDirectorySize)] = Size.Humane(sizes.Value), [nameof(MemoryInfoMappingItem.Details)] = details }; foreach (var file in value.OrderBy(x => x.Key)) { long totalMapped = 0; var dja = new DynamicJsonArray(); var dic = new Dictionary <long, long>(); foreach (var mapping in file.Value) { totalMapped += mapping.Value; dic.TryGetValue(mapping.Value, out long prev); dic[mapping.Value] = prev + 1; } foreach (var maps in dic) { dja.Add(new DynamicJsonValue { [nameof(MemoryInfoMappingDetails.Size)] = maps.Key, [nameof(MemoryInfoMappingDetails.Count)] = maps.Value }); } var fileSize = GetFileSize(file.Key); details[Path.GetFileName(file.Key)] = new DynamicJsonValue { [nameof(MemoryInfoMappingFileInfo.FileSize)] = fileSize, [nameof(MemoryInfoMappingFileInfo.HumaneFileSize)] = Size.Humane(fileSize), [nameof(MemoryInfoMappingFileInfo.TotalMapped)] = totalMapped, [nameof(MemoryInfoMappingFileInfo.HumaneTotalMapped)] = Size.Humane(totalMapped), [nameof(MemoryInfoMappingFileInfo.Mappings)] = dja }; } fileMappings.Add(dir); } } long totalUnmanagedAllocations = NativeMemory.TotalAllocatedMemory; var threads = new DynamicJsonArray(); foreach (var stats in NativeMemory.AllThreadStats .Where(x => x.IsThreadAlive()) .GroupBy(x => x.Name) .OrderByDescending(x => x.Sum(y => y.TotalAllocated))) { var unmanagedAllocations = stats.Sum(x => x.TotalAllocated); var ids = new DynamicJsonArray(stats.OrderByDescending(x => x.TotalAllocated).Select(x => new DynamicJsonValue { ["Id"] = x.UnmanagedThreadId, ["ManagedThreadId"] = x.ManagedThreadId, ["Allocations"] = x.TotalAllocated, ["HumaneAllocations"] = Size.Humane(x.TotalAllocated) })); var groupStats = new DynamicJsonValue { ["Name"] = stats.Key, ["Allocations"] = unmanagedAllocations, ["HumaneAllocations"] = Size.Humane(unmanagedAllocations) }; if (ids.Count == 1) { var threadStats = stats.First(); groupStats["Id"] = threadStats.UnmanagedThreadId; groupStats["ManagedThreadId"] = threadStats.ManagedThreadId; } else { groupStats["Ids"] = ids; } threads.Add(groupStats); } long managedMemoryInBytes = AbstractLowMemoryMonitor.GetManagedMemoryInBytes(); long workingSetInBytes = memInfo.WorkingSet.GetValue(SizeUnit.Bytes); var dirtyMemoryState = MemoryInformation.GetDirtyMemoryState(); var encryptionBuffers = EncryptionBuffersPool.Instance.GetStats(); var djv = new DynamicJsonValue { [nameof(MemoryInfo.WorkingSet)] = workingSetInBytes, [nameof(MemoryInfo.TotalUnmanagedAllocations)] = totalUnmanagedAllocations, [nameof(MemoryInfo.ManagedAllocations)] = managedMemoryInBytes, [nameof(MemoryInfo.TotalMemoryMapped)] = totalMapping, [nameof(MemoryInfo.TotalScratchDirtyMemory)] = Size.Humane(memInfo.TotalScratchDirtyMemory.GetValue(SizeUnit.Bytes)), [nameof(MemoryInfo.PhysicalMem)] = Size.Humane(memInfo.TotalPhysicalMemory.GetValue(SizeUnit.Bytes)), [nameof(DirtyMemoryState.IsHighDirty)] = dirtyMemoryState.IsHighDirty, ["DirtyMemory"] = Size.Humane(dirtyMemoryState.TotalDirtyInBytes), [nameof(MemoryInfo.AvailableMemory)] = Size.Humane(memInfo.AvailableMemory.GetValue(SizeUnit.Bytes)), [nameof(MemoryInfo.AvailableMemoryForProcessing)] = Size.Humane(memInfo.AvailableMemoryForProcessing.GetValue(SizeUnit.Bytes)), [nameof(MemoryInfo.HighMemLastOneMinute)] = Size.Humane(memoryUsageRecords.High.LastOneMinute.GetValue(SizeUnit.Bytes)), [nameof(MemoryInfo.LowMemLastOneMinute)] = Size.Humane(memoryUsageRecords.Low.LastOneMinute.GetValue(SizeUnit.Bytes)), [nameof(MemoryInfo.HighMemLastFiveMinute)] = Size.Humane(memoryUsageRecords.High.LastFiveMinutes.GetValue(SizeUnit.Bytes)), [nameof(MemoryInfo.LowMemLastFiveMinute)] = Size.Humane(memoryUsageRecords.Low.LastFiveMinutes.GetValue(SizeUnit.Bytes)), [nameof(MemoryInfo.HighMemSinceStartup)] = Size.Humane(memoryUsageRecords.High.SinceStartup.GetValue(SizeUnit.Bytes)), [nameof(MemoryInfo.LowMemSinceStartup)] = Size.Humane(memoryUsageRecords.Low.SinceStartup.GetValue(SizeUnit.Bytes)), [nameof(MemoryInfo.Humane)] = new DynamicJsonValue { [nameof(MemoryInfoHumane.WorkingSet)] = Size.Humane(workingSetInBytes), [nameof(MemoryInfoHumane.TotalUnmanagedAllocations)] = Size.Humane(totalUnmanagedAllocations), ["EncryptionBuffers"] = Size.Humane(encryptionBuffers.TotalSize), [nameof(MemoryInfoHumane.ManagedAllocations)] = Size.Humane(managedMemoryInBytes), [nameof(MemoryInfoHumane.TotalMemoryMapped)] = Size.Humane(totalMapping) }, ["Threads"] = threads, [nameof(MemoryInfo.Mappings)] = fileMappings }; return(djv); }
public static bool TryIncreasingMemoryUsageForThread(NativeMemory.ThreadStats threadStats, ref Size currentMaximumAllowedMemory, Size currentlyInUse, bool isRunningOn32Bits, Logger logger, out ProcessMemoryUsage currentUsage) { if (isRunningOn32Bits) { currentUsage = null; return(false); } // we run out our memory quota, so we need to see if we can increase it or break var memoryInfo = MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader(); currentUsage = GetProcessMemoryUsage(memoryInfo); var memoryAssumedFreeOrCheapToFree = memoryInfo.AvailableMemoryForProcessing; // there isn't enough available memory to try, we want to leave some out for other things if (memoryAssumedFreeOrCheapToFree < Size.Min(memoryInfo.TotalPhysicalMemory / 50, new Size(1, SizeUnit.Gigabytes))) { if (logger.IsInfoEnabled) { logger.Info( $"{threadStats.Name} which is already using {currentlyInUse}/{currentMaximumAllowedMemory} and the system has " + $"{memoryInfo.AvailableMemoryForProcessing}/{memoryInfo.TotalPhysicalMemory} free RAM. Also have ~{memoryInfo.SharedCleanMemory} in mmap " + "files that can be cleanly released, not enough to proceed in batch."); } return(false); } // If there isn't enough here to double our current allocation, we won't allocate any more // we do this check in this way to prevent multiple indexes of hitting this at the // same time and each thinking that they have enough space if (memoryAssumedFreeOrCheapToFree < currentMaximumAllowedMemory) { if (logger.IsInfoEnabled) { logger.Info( $"{threadStats} which is already using {currentlyInUse}/{currentMaximumAllowedMemory} and the system has" + $"{memoryInfo.AvailableMemoryForProcessing}/{memoryInfo.TotalPhysicalMemory} free RAM. Also have ~{memoryInfo.SharedCleanMemory} in mmap " + "files that can be cleanly released, not enough to proceed in batch."); } return(false); } var allocatedForProcessing = GetTotalCurrentlyAllocatedForProcessing(); if (memoryAssumedFreeOrCheapToFree < allocatedForProcessing) { if (logger.IsInfoEnabled) { logger.Info( $"Total allocated memory for processing: {allocatedForProcessing}, free memory: {memoryAssumedFreeOrCheapToFree}, " + $"total memory: {memoryInfo.TotalPhysicalMemory}."); } return(false); } // even though we have twice as much memory as we have current allocated, we will // only increment by 16MB to avoid over allocation by multiple indexes. This way, // we'll check often as we go along this var oldBudget = currentMaximumAllowedMemory; currentMaximumAllowedMemory = currentlyInUse + new Size(16, SizeUnit.Megabytes); if (logger.IsInfoEnabled) { logger.Info( $"Increasing memory budget for {threadStats.Name} which is using {currentlyInUse}/{oldBudget} and the system has" + $"{memoryAssumedFreeOrCheapToFree}/{memoryInfo.TotalPhysicalMemory} free RAM with {memoryInfo.SharedCleanMemory} in mmap " + $"files that can be cleanly released. Budget increased to {currentMaximumAllowedMemory}"); } return(true); }
public override MemoryInfoResult GetMemoryInfoOnce() { return(MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader()); }
private static void WriteMemoryStats(BlittableJsonTextWriter writer, JsonOperationContext context, bool includeThreads, bool includeMappings) { writer.WriteStartObject(); var memInfo = MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader(); long managedMemoryInBytes = AbstractLowMemoryMonitor.GetManagedMemoryInBytes(); long totalUnmanagedAllocations = NativeMemory.TotalAllocatedMemory; var encryptionBuffers = EncryptionBuffersPool.Instance.GetStats(); var dirtyMemoryState = MemoryInformation.GetDirtyMemoryState(); var memoryUsageRecords = MemoryInformation.GetMemoryUsageRecords(); long totalMapping = 0; var fileMappingByDir = new Dictionary <string, Dictionary <string, ConcurrentDictionary <IntPtr, long> > >(); var fileMappingSizesByDir = new Dictionary <string, long>(); foreach (var mapping in NativeMemory.FileMapping) { var dir = Path.GetDirectoryName(mapping.Key); if (fileMappingByDir.TryGetValue(dir, out Dictionary <string, ConcurrentDictionary <IntPtr, long> > value) == false) { value = new Dictionary <string, ConcurrentDictionary <IntPtr, long> >(); fileMappingByDir[dir] = value; } value[mapping.Key] = mapping.Value.Value.Info; foreach (var singleMapping in mapping.Value.Value.Info) { fileMappingSizesByDir.TryGetValue(dir, out long prevSize); fileMappingSizesByDir[dir] = prevSize + singleMapping.Value; totalMapping += singleMapping.Value; } } var djv = new DynamicJsonValue { [nameof(MemoryInfo.PhysicalMemory)] = memInfo.TotalPhysicalMemory.ToString(), [nameof(MemoryInfo.WorkingSet)] = memInfo.WorkingSet.ToString(), [nameof(MemoryInfo.ManagedAllocations)] = Size.Humane(managedMemoryInBytes), [nameof(MemoryInfo.UnmanagedAllocations)] = Size.Humane(totalUnmanagedAllocations), [nameof(MemoryInfo.EncryptionBuffers)] = Size.Humane(encryptionBuffers.TotalSize), [nameof(MemoryInfo.MemoryMapped)] = Size.Humane(totalMapping), [nameof(MemoryInfo.ScratchDirtyMemory)] = memInfo.TotalScratchDirtyMemory.ToString(), [nameof(MemoryInfo.IsHighDirty)] = dirtyMemoryState.IsHighDirty, [nameof(MemoryInfo.DirtyMemory)] = Size.Humane(dirtyMemoryState.TotalDirtyInBytes), [nameof(MemoryInfo.AvailableMemory)] = Size.Humane(memInfo.AvailableMemory.GetValue(SizeUnit.Bytes)), [nameof(MemoryInfo.AvailableMemoryForProcessing)] = memInfo.AvailableMemoryForProcessing.ToString(), [nameof(MemoryInfo.HighMemLastOneMinute)] = memoryUsageRecords.High.LastOneMinute.ToString(), [nameof(MemoryInfo.LowMemLastOneMinute)] = memoryUsageRecords.Low.LastOneMinute.ToString(), [nameof(MemoryInfo.HighMemLastFiveMinute)] = memoryUsageRecords.High.LastFiveMinutes.ToString(), [nameof(MemoryInfo.LowMemLastFiveMinute)] = memoryUsageRecords.Low.LastFiveMinutes.ToString(), [nameof(MemoryInfo.HighMemSinceStartup)] = memoryUsageRecords.High.SinceStartup.ToString(), [nameof(MemoryInfo.LowMemSinceStartup)] = memoryUsageRecords.Low.SinceStartup.ToString(), }; writer.WritePropertyName(nameof(MemoryInformation)); context.Write(writer, djv); writer.WriteComma(); writer.WritePropertyName("Threads"); writer.WriteStartArray(); WriteThreads(includeThreads, writer, context); writer.WriteEndArray(); writer.WriteComma(); writer.WritePropertyName(nameof(MemoryInfo.Mappings)); writer.WriteStartArray(); WriteMappings(includeMappings, writer, context, fileMappingSizesByDir, fileMappingByDir); writer.WriteEndArray(); writer.WriteEndObject(); }