Example #1
0
        public static bool CanIncreaseMemoryUsageForThread()
        {
            var memoryInfo = MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader();
            var memoryAssumedFreeOrCheapToFree = memoryInfo.AvailableMemoryForProcessing;
            var allocatedForProcessing         = GetTotalCurrentlyAllocatedForProcessing();

            return(memoryAssumedFreeOrCheapToFree >= allocatedForProcessing);
        }
Example #2
0
        private static MessageDetails OutOfMemoryDetails(Exception exception)
        {
            var memoryInfo = MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader();

            return(new MessageDetails
            {
                Message = $"Managed memory: {new Size(AbstractLowMemoryMonitor.GetManagedMemoryInBytes(), SizeUnit.Bytes)}, " +
                          $"Unmanaged allocations: {new Size(AbstractLowMemoryMonitor.GetUnmanagedAllocationsInBytes(), SizeUnit.Bytes)}, " +
                          $"Shared clean: {memoryInfo.SharedCleanMemory}, " +
                          $"Working set: {memoryInfo.WorkingSet}, " +
                          $"Available memory: {memoryInfo.AvailableMemory}, " +
                          $"Calculated Available memory: {memoryInfo.AvailableWithoutTotalCleanMemory}, " +
                          $"Total memory: {memoryInfo.TotalPhysicalMemory} {Environment.NewLine}" +
                          $"Error: {exception}"
            });
        }
Example #3
0
        private static DynamicJsonValue MemoryStatsInternal()
        {
            var memInfo            = MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader();
            var memoryUsageRecords = MemoryInformation.GetMemoryUsageRecords();

            long totalMapping          = 0;
            var  fileMappingByDir      = new Dictionary <string, Dictionary <string, ConcurrentDictionary <IntPtr, long> > >();
            var  fileMappingSizesByDir = new Dictionary <string, long>();

            foreach (var mapping in NativeMemory.FileMapping)
            {
                var dir = Path.GetDirectoryName(mapping.Key);

                if (fileMappingByDir.TryGetValue(dir, out Dictionary <string, ConcurrentDictionary <IntPtr, long> > value) == false)
                {
                    value = new Dictionary <string, ConcurrentDictionary <IntPtr, long> >();
                    fileMappingByDir[dir] = value;
                }

                value[mapping.Key] = mapping.Value.Value.Info;
                foreach (var singleMapping in mapping.Value.Value.Info)
                {
                    fileMappingSizesByDir.TryGetValue(dir, out long prevSize);
                    fileMappingSizesByDir[dir] = prevSize + singleMapping.Value;
                    totalMapping += singleMapping.Value;
                }
            }

            var prefixLength = LongestCommonPrefixLength(new List <string>(fileMappingSizesByDir.Keys));

            var fileMappings = new DynamicJsonArray();

            foreach (var sizes in fileMappingSizesByDir.OrderByDescending(x => x.Value))
            {
                if (fileMappingByDir.TryGetValue(sizes.Key, out Dictionary <string, ConcurrentDictionary <IntPtr, long> > value))
                {
                    var details = new DynamicJsonValue();

                    var dir = new DynamicJsonValue
                    {
                        [nameof(MemoryInfoMappingItem.Directory)]                = sizes.Key.Substring(prefixLength),
                        [nameof(MemoryInfoMappingItem.TotalDirectorySize)]       = sizes.Value,
                        [nameof(MemoryInfoMappingItem.HumaneTotalDirectorySize)] = Size.Humane(sizes.Value),
                        [nameof(MemoryInfoMappingItem.Details)] = details
                    };
                    foreach (var file in value.OrderBy(x => x.Key))
                    {
                        long totalMapped = 0;
                        var  dja         = new DynamicJsonArray();
                        var  dic         = new Dictionary <long, long>();
                        foreach (var mapping in file.Value)
                        {
                            totalMapped += mapping.Value;
                            dic.TryGetValue(mapping.Value, out long prev);
                            dic[mapping.Value] = prev + 1;
                        }

                        foreach (var maps in dic)
                        {
                            dja.Add(new DynamicJsonValue
                            {
                                [nameof(MemoryInfoMappingDetails.Size)]  = maps.Key,
                                [nameof(MemoryInfoMappingDetails.Count)] = maps.Value
                            });
                        }

                        var fileSize = GetFileSize(file.Key);
                        details[Path.GetFileName(file.Key)] = new DynamicJsonValue
                        {
                            [nameof(MemoryInfoMappingFileInfo.FileSize)]          = fileSize,
                            [nameof(MemoryInfoMappingFileInfo.HumaneFileSize)]    = Size.Humane(fileSize),
                            [nameof(MemoryInfoMappingFileInfo.TotalMapped)]       = totalMapped,
                            [nameof(MemoryInfoMappingFileInfo.HumaneTotalMapped)] = Size.Humane(totalMapped),
                            [nameof(MemoryInfoMappingFileInfo.Mappings)]          = dja
                        };
                    }

                    fileMappings.Add(dir);
                }
            }

            long totalUnmanagedAllocations = NativeMemory.TotalAllocatedMemory;
            var  threads = new DynamicJsonArray();

            foreach (var stats in NativeMemory.AllThreadStats
                     .Where(x => x.IsThreadAlive())
                     .GroupBy(x => x.Name)
                     .OrderByDescending(x => x.Sum(y => y.TotalAllocated)))
            {
                var unmanagedAllocations = stats.Sum(x => x.TotalAllocated);
                var ids = new DynamicJsonArray(stats.OrderByDescending(x => x.TotalAllocated).Select(x => new DynamicJsonValue
                {
                    ["Id"] = x.UnmanagedThreadId,
                    ["ManagedThreadId"]   = x.ManagedThreadId,
                    ["Allocations"]       = x.TotalAllocated,
                    ["HumaneAllocations"] = Size.Humane(x.TotalAllocated)
                }));
                var groupStats = new DynamicJsonValue
                {
                    ["Name"]              = stats.Key,
                    ["Allocations"]       = unmanagedAllocations,
                    ["HumaneAllocations"] = Size.Humane(unmanagedAllocations)
                };
                if (ids.Count == 1)
                {
                    var threadStats = stats.First();
                    groupStats["Id"] = threadStats.UnmanagedThreadId;
                    groupStats["ManagedThreadId"] = threadStats.ManagedThreadId;
                }
                else
                {
                    groupStats["Ids"] = ids;
                }

                threads.Add(groupStats);
            }

            long managedMemoryInBytes = AbstractLowMemoryMonitor.GetManagedMemoryInBytes();
            long workingSetInBytes    = memInfo.WorkingSet.GetValue(SizeUnit.Bytes);
            var  dirtyMemoryState     = MemoryInformation.GetDirtyMemoryState();
            var  encryptionBuffers    = EncryptionBuffersPool.Instance.GetStats();

            var djv = new DynamicJsonValue
            {
                [nameof(MemoryInfo.WorkingSet)] = workingSetInBytes,
                [nameof(MemoryInfo.TotalUnmanagedAllocations)] = totalUnmanagedAllocations,
                [nameof(MemoryInfo.ManagedAllocations)]        = managedMemoryInBytes,
                [nameof(MemoryInfo.TotalMemoryMapped)]         = totalMapping,
                [nameof(MemoryInfo.TotalScratchDirtyMemory)]   = Size.Humane(memInfo.TotalScratchDirtyMemory.GetValue(SizeUnit.Bytes)),
                [nameof(MemoryInfo.PhysicalMem)]       = Size.Humane(memInfo.TotalPhysicalMemory.GetValue(SizeUnit.Bytes)),
                [nameof(DirtyMemoryState.IsHighDirty)] = dirtyMemoryState.IsHighDirty,
                ["DirtyMemory"] = Size.Humane(dirtyMemoryState.TotalDirtyInBytes),
                [nameof(MemoryInfo.AvailableMemory)] = Size.Humane(memInfo.AvailableMemory.GetValue(SizeUnit.Bytes)),
                [nameof(MemoryInfo.AvailableMemoryForProcessing)] = Size.Humane(memInfo.AvailableMemoryForProcessing.GetValue(SizeUnit.Bytes)),
                [nameof(MemoryInfo.HighMemLastOneMinute)]         = Size.Humane(memoryUsageRecords.High.LastOneMinute.GetValue(SizeUnit.Bytes)),
                [nameof(MemoryInfo.LowMemLastOneMinute)]          = Size.Humane(memoryUsageRecords.Low.LastOneMinute.GetValue(SizeUnit.Bytes)),
                [nameof(MemoryInfo.HighMemLastFiveMinute)]        = Size.Humane(memoryUsageRecords.High.LastFiveMinutes.GetValue(SizeUnit.Bytes)),
                [nameof(MemoryInfo.LowMemLastFiveMinute)]         = Size.Humane(memoryUsageRecords.Low.LastFiveMinutes.GetValue(SizeUnit.Bytes)),
                [nameof(MemoryInfo.HighMemSinceStartup)]          = Size.Humane(memoryUsageRecords.High.SinceStartup.GetValue(SizeUnit.Bytes)),
                [nameof(MemoryInfo.LowMemSinceStartup)]           = Size.Humane(memoryUsageRecords.Low.SinceStartup.GetValue(SizeUnit.Bytes)),

                [nameof(MemoryInfo.Humane)] = new DynamicJsonValue
                {
                    [nameof(MemoryInfoHumane.WorkingSet)] = Size.Humane(workingSetInBytes),
                    [nameof(MemoryInfoHumane.TotalUnmanagedAllocations)] = Size.Humane(totalUnmanagedAllocations),
                    ["EncryptionBuffers"] = Size.Humane(encryptionBuffers.TotalSize),
                    [nameof(MemoryInfoHumane.ManagedAllocations)] = Size.Humane(managedMemoryInBytes),
                    [nameof(MemoryInfoHumane.TotalMemoryMapped)]  = Size.Humane(totalMapping)
                },

                ["Threads"] = threads,

                [nameof(MemoryInfo.Mappings)] = fileMappings
            };

            return(djv);
        }
Example #4
0
        public static bool TryIncreasingMemoryUsageForThread(NativeMemory.ThreadStats threadStats,
                                                             ref Size currentMaximumAllowedMemory,
                                                             Size currentlyInUse,
                                                             bool isRunningOn32Bits,
                                                             Logger logger,
                                                             out ProcessMemoryUsage currentUsage)
        {
            if (isRunningOn32Bits)
            {
                currentUsage = null;
                return(false);
            }

            // we run out our memory quota, so we need to see if we can increase it or break
            var memoryInfo = MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader();

            currentUsage = GetProcessMemoryUsage(memoryInfo);

            var memoryAssumedFreeOrCheapToFree = memoryInfo.AvailableMemoryForProcessing;

            // there isn't enough available memory to try, we want to leave some out for other things
            if (memoryAssumedFreeOrCheapToFree <
                Size.Min(memoryInfo.TotalPhysicalMemory / 50, new Size(1, SizeUnit.Gigabytes)))
            {
                if (logger.IsInfoEnabled)
                {
                    logger.Info(
                        $"{threadStats.Name} which is already using {currentlyInUse}/{currentMaximumAllowedMemory} and the system has " +
                        $"{memoryInfo.AvailableMemoryForProcessing}/{memoryInfo.TotalPhysicalMemory} free RAM. Also have ~{memoryInfo.SharedCleanMemory} in mmap " +
                        "files that can be cleanly released, not enough to proceed in batch.");
                }

                return(false);
            }

            // If there isn't enough here to double our current allocation, we won't allocate any more
            // we do this check in this way to prevent multiple indexes of hitting this at the
            // same time and each thinking that they have enough space
            if (memoryAssumedFreeOrCheapToFree < currentMaximumAllowedMemory)
            {
                if (logger.IsInfoEnabled)
                {
                    logger.Info(
                        $"{threadStats} which is already using {currentlyInUse}/{currentMaximumAllowedMemory} and the system has" +
                        $"{memoryInfo.AvailableMemoryForProcessing}/{memoryInfo.TotalPhysicalMemory} free RAM. Also have ~{memoryInfo.SharedCleanMemory} in mmap " +
                        "files that can be cleanly released, not enough to proceed in batch.");
                }

                return(false);
            }

            var allocatedForProcessing = GetTotalCurrentlyAllocatedForProcessing();

            if (memoryAssumedFreeOrCheapToFree < allocatedForProcessing)
            {
                if (logger.IsInfoEnabled)
                {
                    logger.Info(
                        $"Total allocated memory for processing: {allocatedForProcessing}, free memory: {memoryAssumedFreeOrCheapToFree}, " +
                        $"total memory: {memoryInfo.TotalPhysicalMemory}.");
                }

                return(false);
            }

            // even though we have twice as much memory as we have current allocated, we will
            // only increment by 16MB to avoid over allocation by multiple indexes. This way,
            // we'll check often as we go along this
            var oldBudget = currentMaximumAllowedMemory;

            currentMaximumAllowedMemory = currentlyInUse + new Size(16, SizeUnit.Megabytes);

            if (logger.IsInfoEnabled)
            {
                logger.Info(
                    $"Increasing memory budget for {threadStats.Name} which is using  {currentlyInUse}/{oldBudget} and the system has" +
                    $"{memoryAssumedFreeOrCheapToFree}/{memoryInfo.TotalPhysicalMemory} free RAM with {memoryInfo.SharedCleanMemory} in mmap " +
                    $"files that can be cleanly released. Budget increased to {currentMaximumAllowedMemory}");
            }

            return(true);
        }
Example #5
0
 public override MemoryInfoResult GetMemoryInfoOnce()
 {
     return(MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader());
 }
Example #6
0
        private static void WriteMemoryStats(BlittableJsonTextWriter writer, JsonOperationContext context, bool includeThreads, bool includeMappings)
        {
            writer.WriteStartObject();

            var  memInfo = MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader();
            long managedMemoryInBytes      = AbstractLowMemoryMonitor.GetManagedMemoryInBytes();
            long totalUnmanagedAllocations = NativeMemory.TotalAllocatedMemory;
            var  encryptionBuffers         = EncryptionBuffersPool.Instance.GetStats();
            var  dirtyMemoryState          = MemoryInformation.GetDirtyMemoryState();
            var  memoryUsageRecords        = MemoryInformation.GetMemoryUsageRecords();

            long totalMapping          = 0;
            var  fileMappingByDir      = new Dictionary <string, Dictionary <string, ConcurrentDictionary <IntPtr, long> > >();
            var  fileMappingSizesByDir = new Dictionary <string, long>();

            foreach (var mapping in NativeMemory.FileMapping)
            {
                var dir = Path.GetDirectoryName(mapping.Key);

                if (fileMappingByDir.TryGetValue(dir, out Dictionary <string, ConcurrentDictionary <IntPtr, long> > value) == false)
                {
                    value = new Dictionary <string, ConcurrentDictionary <IntPtr, long> >();
                    fileMappingByDir[dir] = value;
                }

                value[mapping.Key] = mapping.Value.Value.Info;
                foreach (var singleMapping in mapping.Value.Value.Info)
                {
                    fileMappingSizesByDir.TryGetValue(dir, out long prevSize);
                    fileMappingSizesByDir[dir] = prevSize + singleMapping.Value;
                    totalMapping += singleMapping.Value;
                }
            }

            var djv = new DynamicJsonValue
            {
                [nameof(MemoryInfo.PhysicalMemory)]               = memInfo.TotalPhysicalMemory.ToString(),
                [nameof(MemoryInfo.WorkingSet)]                   = memInfo.WorkingSet.ToString(),
                [nameof(MemoryInfo.ManagedAllocations)]           = Size.Humane(managedMemoryInBytes),
                [nameof(MemoryInfo.UnmanagedAllocations)]         = Size.Humane(totalUnmanagedAllocations),
                [nameof(MemoryInfo.EncryptionBuffers)]            = Size.Humane(encryptionBuffers.TotalSize),
                [nameof(MemoryInfo.MemoryMapped)]                 = Size.Humane(totalMapping),
                [nameof(MemoryInfo.ScratchDirtyMemory)]           = memInfo.TotalScratchDirtyMemory.ToString(),
                [nameof(MemoryInfo.IsHighDirty)]                  = dirtyMemoryState.IsHighDirty,
                [nameof(MemoryInfo.DirtyMemory)]                  = Size.Humane(dirtyMemoryState.TotalDirtyInBytes),
                [nameof(MemoryInfo.AvailableMemory)]              = Size.Humane(memInfo.AvailableMemory.GetValue(SizeUnit.Bytes)),
                [nameof(MemoryInfo.AvailableMemoryForProcessing)] = memInfo.AvailableMemoryForProcessing.ToString(),
                [nameof(MemoryInfo.HighMemLastOneMinute)]         = memoryUsageRecords.High.LastOneMinute.ToString(),
                [nameof(MemoryInfo.LowMemLastOneMinute)]          = memoryUsageRecords.Low.LastOneMinute.ToString(),
                [nameof(MemoryInfo.HighMemLastFiveMinute)]        = memoryUsageRecords.High.LastFiveMinutes.ToString(),
                [nameof(MemoryInfo.LowMemLastFiveMinute)]         = memoryUsageRecords.Low.LastFiveMinutes.ToString(),
                [nameof(MemoryInfo.HighMemSinceStartup)]          = memoryUsageRecords.High.SinceStartup.ToString(),
                [nameof(MemoryInfo.LowMemSinceStartup)]           = memoryUsageRecords.Low.SinceStartup.ToString(),
            };

            writer.WritePropertyName(nameof(MemoryInformation));
            context.Write(writer, djv);

            writer.WriteComma();
            writer.WritePropertyName("Threads");
            writer.WriteStartArray();
            WriteThreads(includeThreads, writer, context);
            writer.WriteEndArray();

            writer.WriteComma();
            writer.WritePropertyName(nameof(MemoryInfo.Mappings));
            writer.WriteStartArray();
            WriteMappings(includeMappings, writer, context, fileMappingSizesByDir, fileMappingByDir);
            writer.WriteEndArray();

            writer.WriteEndObject();
        }