Exemplo n.º 1
0
        private static ProcessMemoryUsage GetProcessMemoryUsage(MemoryInfoResult memoryInfo)
        {
            var workingSetInBytes = memoryInfo.WorkingSet.GetValue(SizeUnit.Bytes);
            var privateMemory     = MemoryInformation.GetManagedMemoryInBytes() + MemoryInformation.GetUnManagedAllocationsInBytes();

            return(new ProcessMemoryUsage(workingSetInBytes, privateMemory));
        }
Exemplo n.º 2
0
        private static byte *ThrowFailedToAllocate(long size, ThreadStats thread, OutOfMemoryException e)
        {
            long allocated = 0;

            foreach (var threadAllocationsValue in AllThreadStats)
            {
                allocated += threadAllocationsValue.TotalAllocated;
            }
            var managed         = MemoryInformation.GetManagedMemoryInBytes();
            var unmanagedMemory = MemoryInformation.GetUnManagedAllocationsInBytes();

            throw new OutOfMemoryException($"Failed to allocate additional {new Size(size, SizeUnit.Bytes)} " +
                                           $"to already allocated {new Size(thread.TotalAllocated, SizeUnit.Bytes)} by this thread. " +
                                           $"Total allocated by all threads: {new Size(allocated, SizeUnit.Bytes)}, " +
                                           $"Managed memory: {new Size(managed, SizeUnit.Bytes)}, " +
                                           $"Un-managed memory: {new Size(unmanagedMemory, SizeUnit.Bytes)}", e);
        }
Exemplo n.º 3
0
        public bool Execute(DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation,
                            IndexingStatsScope stats, CancellationToken token)
        {
            if (_mapReduceContext.StoreByReduceKeyHash.Count == 0)
            {
                WriteLastEtags(indexContext); // we need to write etags here, because if we filtered everything during map then we will loose last indexed etag information and this will cause an endless indexing loop
                return(false);
            }

            ReduceResultsSchema.Create(indexContext.Transaction.InnerTransaction, PageNumberToReduceResultTableName, 32);
            var table = indexContext.Transaction.InnerTransaction.OpenTable(ReduceResultsSchema, PageNumberToReduceResultTableName);

            var lowLevelTransaction = indexContext.Transaction.InnerTransaction.LowLevelTransaction;

            var writer = writeOperation.Value;

            var treeScopeStats         = stats.For(IndexingOperation.Reduce.TreeScope, start: false);
            var nestedValuesScopeStats = stats.For(IndexingOperation.Reduce.NestedValuesScope, start: false);

            foreach (var store in _mapReduceContext.StoreByReduceKeyHash)
            {
                token.ThrowIfCancellationRequested();

                using (var reduceKeyHash = indexContext.GetLazyString(store.Key.ToString(CultureInfo.InvariantCulture)))
                    using (store.Value)
                        using (_aggregationBatch)
                        {
                            var modifiedStore = store.Value;

                            switch (modifiedStore.Type)
                            {
                            case MapResultsStorageType.Tree:
                                using (treeScopeStats.Start())
                                {
                                    HandleTreeReduction(indexContext, treeScopeStats, modifiedStore, lowLevelTransaction, writer, reduceKeyHash, table, token);
                                }
                                break;

                            case MapResultsStorageType.Nested:
                                using (nestedValuesScopeStats.Start())
                                {
                                    HandleNestedValuesReduction(indexContext, nestedValuesScopeStats, modifiedStore, writer, reduceKeyHash, token);
                                }
                                break;

                            default:
                                throw new ArgumentOutOfRangeException(modifiedStore.Type.ToString());
                            }
                        }

                if (_mapReduceContext.FreedPages.Count > 0)
                {
                    long tmp = 0;
                    using (treeScopeStats.Start())
                        using (Slice.External(indexContext.Allocator, (byte *)&tmp, sizeof(long), out Slice pageNumberSlice))
                        {
                            foreach (var freedPage in _mapReduceContext.FreedPages)
                            {
                                tmp = Bits.SwapBytes(freedPage);
                                table.DeleteByKey(pageNumberSlice);
                            }
                        }
                }
            }

            if (stats.Duration >= MinReduceDurationToCalculateProcessMemoryUsage)
            {
                var workingSet    = MemoryInformation.GetWorkingSetInBytes();
                var privateMemory = MemoryInformation.GetManagedMemoryInBytes() + MemoryInformation.GetUnManagedAllocationsInBytes();
                stats.RecordReduceMemoryStats(workingSet, privateMemory);
            }

            WriteLastEtags(indexContext);
            _mapReduceContext.StoreNextMapResultId();

            return(false);
        }
Exemplo n.º 4
0
        private static DynamicJsonValue MemoryStatsInternal()
        {
            var memInfo            = MemoryInformation.GetMemInfoUsingOneTimeSmapsReader();
            var memoryUsageRecords = MemoryInformation.GetMemoryUsageRecords();

            long totalMapping          = 0;
            var  fileMappingByDir      = new Dictionary <string, Dictionary <string, ConcurrentDictionary <IntPtr, long> > >();
            var  fileMappingSizesByDir = new Dictionary <string, long>();

            foreach (var mapping in NativeMemory.FileMapping)
            {
                var dir = Path.GetDirectoryName(mapping.Key);

                if (fileMappingByDir.TryGetValue(dir, out Dictionary <string, ConcurrentDictionary <IntPtr, long> > value) == false)
                {
                    value = new Dictionary <string, ConcurrentDictionary <IntPtr, long> >();
                    fileMappingByDir[dir] = value;
                }

                value[mapping.Key] = mapping.Value.Value.Info;
                foreach (var singleMapping in mapping.Value.Value.Info)
                {
                    fileMappingSizesByDir.TryGetValue(dir, out long prevSize);
                    fileMappingSizesByDir[dir] = prevSize + singleMapping.Value;
                    totalMapping += singleMapping.Value;
                }
            }

            var prefixLength = LongestCommonPrefixLength(new List <string>(fileMappingSizesByDir.Keys));

            var fileMappings = new DynamicJsonArray();

            foreach (var sizes in fileMappingSizesByDir.OrderByDescending(x => x.Value))
            {
                if (fileMappingByDir.TryGetValue(sizes.Key, out Dictionary <string, ConcurrentDictionary <IntPtr, long> > value))
                {
                    var details = new DynamicJsonValue();

                    var dir = new DynamicJsonValue
                    {
                        [nameof(MemoryInfoMappingItem.Directory)]                = sizes.Key.Substring(prefixLength),
                        [nameof(MemoryInfoMappingItem.TotalDirectorySize)]       = sizes.Value,
                        [nameof(MemoryInfoMappingItem.HumaneTotalDirectorySize)] = Size.Humane(sizes.Value),
                        [nameof(MemoryInfoMappingItem.Details)] = details
                    };
                    foreach (var file in value.OrderBy(x => x.Key))
                    {
                        long totalMapped = 0;
                        var  dja         = new DynamicJsonArray();
                        var  dic         = new Dictionary <long, long>();
                        foreach (var mapping in file.Value)
                        {
                            totalMapped += mapping.Value;
                            dic.TryGetValue(mapping.Value, out long prev);
                            dic[mapping.Value] = prev + 1;
                        }

                        foreach (var maps in dic)
                        {
                            dja.Add(new DynamicJsonValue
                            {
                                [nameof(MemoryInfoMappingDetails.Size)]  = maps.Key,
                                [nameof(MemoryInfoMappingDetails.Count)] = maps.Value
                            });
                        }

                        var fileSize = GetFileSize(file.Key);
                        details[Path.GetFileName(file.Key)] = new DynamicJsonValue
                        {
                            [nameof(MemoryInfoMappingFileInfo.FileSize)]          = fileSize,
                            [nameof(MemoryInfoMappingFileInfo.HumaneFileSize)]    = Size.Humane(fileSize),
                            [nameof(MemoryInfoMappingFileInfo.TotalMapped)]       = totalMapped,
                            [nameof(MemoryInfoMappingFileInfo.HumaneTotalMapped)] = Size.Humane(totalMapped),
                            [nameof(MemoryInfoMappingFileInfo.Mappings)]          = dja
                        };
                    }

                    fileMappings.Add(dir);
                }
            }

            long totalUnmanagedAllocations = 0;
            var  threads = new DynamicJsonArray();

            foreach (var stats in NativeMemory.AllThreadStats
                     .Where(x => x.IsThreadAlive())
                     .GroupBy(x => x.Name)
                     .OrderByDescending(x => x.Sum(y => y.TotalAllocated)))
            {
                var unmanagedAllocations = stats.Sum(x => x.TotalAllocated);
                totalUnmanagedAllocations += unmanagedAllocations;
                var ids = new DynamicJsonArray(stats.OrderByDescending(x => x.TotalAllocated).Select(x => new DynamicJsonValue
                {
                    ["Id"] = x.UnmanagedThreadId,
                    ["ManagedThreadId"]   = x.Id,
                    ["Allocations"]       = x.TotalAllocated,
                    ["HumaneAllocations"] = Size.Humane(x.TotalAllocated)
                }));
                var groupStats = new DynamicJsonValue
                {
                    ["Name"]              = stats.Key,
                    ["Allocations"]       = unmanagedAllocations,
                    ["HumaneAllocations"] = Size.Humane(unmanagedAllocations)
                };
                if (ids.Count == 1)
                {
                    var threadStats = stats.First();
                    groupStats["Id"] = threadStats.UnmanagedThreadId;
                    groupStats["ManagedThreadId"] = threadStats.Id;
                }
                else
                {
                    groupStats["Ids"] = ids;
                }

                threads.Add(groupStats);
            }

            long managedMemoryInBytes = MemoryInformation.GetManagedMemoryInBytes();
            long workingSetInBytes    = memInfo.WorkingSet.GetValue(SizeUnit.Bytes);
            var  djv = new DynamicJsonValue
            {
                [nameof(MemoryInfo.WorkingSet)] = workingSetInBytes,
                [nameof(MemoryInfo.TotalUnmanagedAllocations)] = totalUnmanagedAllocations,
                [nameof(MemoryInfo.ManagedAllocations)]        = managedMemoryInBytes,
                [nameof(MemoryInfo.TotalMemoryMapped)]         = totalMapping,
                [nameof(MemoryInfo.PhysicalMem)]           = Size.Humane(memInfo.TotalPhysicalMemory.GetValue(SizeUnit.Bytes)),
                [nameof(MemoryInfo.FreeMem)]               = Size.Humane(memInfo.AvailableWithoutTotalCleanMemory.GetValue(SizeUnit.Bytes)),
                [nameof(MemoryInfo.HighMemLastOneMinute)]  = Size.Humane(memoryUsageRecords.High.LastOneMinute.GetValue(SizeUnit.Bytes)),
                [nameof(MemoryInfo.LowMemLastOneMinute)]   = Size.Humane(memoryUsageRecords.Low.LastOneMinute.GetValue(SizeUnit.Bytes)),
                [nameof(MemoryInfo.HighMemLastFiveMinute)] = Size.Humane(memoryUsageRecords.High.LastFiveMinutes.GetValue(SizeUnit.Bytes)),
                [nameof(MemoryInfo.LowMemLastFiveMinute)]  = Size.Humane(memoryUsageRecords.Low.LastFiveMinutes.GetValue(SizeUnit.Bytes)),
                [nameof(MemoryInfo.HighMemSinceStartup)]   = Size.Humane(memoryUsageRecords.High.SinceStartup.GetValue(SizeUnit.Bytes)),
                [nameof(MemoryInfo.LowMemSinceStartup)]    = Size.Humane(memoryUsageRecords.Low.SinceStartup.GetValue(SizeUnit.Bytes)),

                [nameof(MemoryInfo.Humane)] = new DynamicJsonValue
                {
                    [nameof(MemoryInfoHumane.WorkingSet)] = Size.Humane(workingSetInBytes),
                    [nameof(MemoryInfoHumane.TotalUnmanagedAllocations)] = Size.Humane(totalUnmanagedAllocations),
                    [nameof(MemoryInfoHumane.ManagedAllocations)]        = Size.Humane(managedMemoryInBytes),
                    [nameof(MemoryInfoHumane.TotalMemoryMapped)]         = Size.Humane(totalMapping)
                },

                ["Threads"] = threads,

                [nameof(MemoryInfo.Mappings)] = fileMappings
            };

            return(djv);
        }