private static ProcessMemoryUsage GetProcessMemoryUsage(MemoryInfoResult memoryInfo) { var workingSetInBytes = memoryInfo.WorkingSet.GetValue(SizeUnit.Bytes); var privateMemory = MemoryInformation.GetManagedMemoryInBytes() + MemoryInformation.GetUnManagedAllocationsInBytes(); return(new ProcessMemoryUsage(workingSetInBytes, privateMemory)); }
private static byte *ThrowFailedToAllocate(long size, ThreadStats thread, OutOfMemoryException e) { long allocated = 0; foreach (var threadAllocationsValue in AllThreadStats) { allocated += threadAllocationsValue.TotalAllocated; } var managed = MemoryInformation.GetManagedMemoryInBytes(); var unmanagedMemory = MemoryInformation.GetUnManagedAllocationsInBytes(); throw new OutOfMemoryException($"Failed to allocate additional {new Size(size, SizeUnit.Bytes)} " + $"to already allocated {new Size(thread.TotalAllocated, SizeUnit.Bytes)} by this thread. " + $"Total allocated by all threads: {new Size(allocated, SizeUnit.Bytes)}, " + $"Managed memory: {new Size(managed, SizeUnit.Bytes)}, " + $"Un-managed memory: {new Size(unmanagedMemory, SizeUnit.Bytes)}", e); }
public bool Execute(DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, CancellationToken token) { if (_mapReduceContext.StoreByReduceKeyHash.Count == 0) { WriteLastEtags(indexContext); // we need to write etags here, because if we filtered everything during map then we will loose last indexed etag information and this will cause an endless indexing loop return(false); } ReduceResultsSchema.Create(indexContext.Transaction.InnerTransaction, PageNumberToReduceResultTableName, 32); var table = indexContext.Transaction.InnerTransaction.OpenTable(ReduceResultsSchema, PageNumberToReduceResultTableName); var lowLevelTransaction = indexContext.Transaction.InnerTransaction.LowLevelTransaction; var writer = writeOperation.Value; var treeScopeStats = stats.For(IndexingOperation.Reduce.TreeScope, start: false); var nestedValuesScopeStats = stats.For(IndexingOperation.Reduce.NestedValuesScope, start: false); foreach (var store in _mapReduceContext.StoreByReduceKeyHash) { token.ThrowIfCancellationRequested(); using (var reduceKeyHash = indexContext.GetLazyString(store.Key.ToString(CultureInfo.InvariantCulture))) using (store.Value) using (_aggregationBatch) { var modifiedStore = store.Value; switch (modifiedStore.Type) { case MapResultsStorageType.Tree: using (treeScopeStats.Start()) { HandleTreeReduction(indexContext, treeScopeStats, modifiedStore, lowLevelTransaction, writer, reduceKeyHash, table, token); } break; case MapResultsStorageType.Nested: using (nestedValuesScopeStats.Start()) { HandleNestedValuesReduction(indexContext, nestedValuesScopeStats, modifiedStore, writer, reduceKeyHash, token); } break; default: throw new ArgumentOutOfRangeException(modifiedStore.Type.ToString()); } } if (_mapReduceContext.FreedPages.Count > 0) { long tmp = 0; using (treeScopeStats.Start()) using (Slice.External(indexContext.Allocator, (byte *)&tmp, sizeof(long), out Slice pageNumberSlice)) { foreach (var freedPage in _mapReduceContext.FreedPages) { tmp = Bits.SwapBytes(freedPage); table.DeleteByKey(pageNumberSlice); } } } } if (stats.Duration >= MinReduceDurationToCalculateProcessMemoryUsage) { var workingSet = MemoryInformation.GetWorkingSetInBytes(); var privateMemory = MemoryInformation.GetManagedMemoryInBytes() + MemoryInformation.GetUnManagedAllocationsInBytes(); stats.RecordReduceMemoryStats(workingSet, privateMemory); } WriteLastEtags(indexContext); _mapReduceContext.StoreNextMapResultId(); return(false); }
private static DynamicJsonValue MemoryStatsInternal() { var memInfo = MemoryInformation.GetMemInfoUsingOneTimeSmapsReader(); var memoryUsageRecords = MemoryInformation.GetMemoryUsageRecords(); long totalMapping = 0; var fileMappingByDir = new Dictionary <string, Dictionary <string, ConcurrentDictionary <IntPtr, long> > >(); var fileMappingSizesByDir = new Dictionary <string, long>(); foreach (var mapping in NativeMemory.FileMapping) { var dir = Path.GetDirectoryName(mapping.Key); if (fileMappingByDir.TryGetValue(dir, out Dictionary <string, ConcurrentDictionary <IntPtr, long> > value) == false) { value = new Dictionary <string, ConcurrentDictionary <IntPtr, long> >(); fileMappingByDir[dir] = value; } value[mapping.Key] = mapping.Value.Value.Info; foreach (var singleMapping in mapping.Value.Value.Info) { fileMappingSizesByDir.TryGetValue(dir, out long prevSize); fileMappingSizesByDir[dir] = prevSize + singleMapping.Value; totalMapping += singleMapping.Value; } } var prefixLength = LongestCommonPrefixLength(new List <string>(fileMappingSizesByDir.Keys)); var fileMappings = new DynamicJsonArray(); foreach (var sizes in fileMappingSizesByDir.OrderByDescending(x => x.Value)) { if (fileMappingByDir.TryGetValue(sizes.Key, out Dictionary <string, ConcurrentDictionary <IntPtr, long> > value)) { var details = new DynamicJsonValue(); var dir = new DynamicJsonValue { [nameof(MemoryInfoMappingItem.Directory)] = sizes.Key.Substring(prefixLength), [nameof(MemoryInfoMappingItem.TotalDirectorySize)] = sizes.Value, [nameof(MemoryInfoMappingItem.HumaneTotalDirectorySize)] = Size.Humane(sizes.Value), [nameof(MemoryInfoMappingItem.Details)] = details }; foreach (var file in value.OrderBy(x => x.Key)) { long totalMapped = 0; var dja = new DynamicJsonArray(); var dic = new Dictionary <long, long>(); foreach (var mapping in file.Value) { totalMapped += mapping.Value; dic.TryGetValue(mapping.Value, out long prev); dic[mapping.Value] = prev + 1; } foreach (var maps in dic) { dja.Add(new DynamicJsonValue { [nameof(MemoryInfoMappingDetails.Size)] = maps.Key, [nameof(MemoryInfoMappingDetails.Count)] = maps.Value }); } var fileSize = GetFileSize(file.Key); details[Path.GetFileName(file.Key)] = new DynamicJsonValue { [nameof(MemoryInfoMappingFileInfo.FileSize)] = fileSize, [nameof(MemoryInfoMappingFileInfo.HumaneFileSize)] = Size.Humane(fileSize), [nameof(MemoryInfoMappingFileInfo.TotalMapped)] = totalMapped, [nameof(MemoryInfoMappingFileInfo.HumaneTotalMapped)] = Size.Humane(totalMapped), [nameof(MemoryInfoMappingFileInfo.Mappings)] = dja }; } fileMappings.Add(dir); } } long totalUnmanagedAllocations = 0; var threads = new DynamicJsonArray(); foreach (var stats in NativeMemory.AllThreadStats .Where(x => x.IsThreadAlive()) .GroupBy(x => x.Name) .OrderByDescending(x => x.Sum(y => y.TotalAllocated))) { var unmanagedAllocations = stats.Sum(x => x.TotalAllocated); totalUnmanagedAllocations += unmanagedAllocations; var ids = new DynamicJsonArray(stats.OrderByDescending(x => x.TotalAllocated).Select(x => new DynamicJsonValue { ["Id"] = x.UnmanagedThreadId, ["ManagedThreadId"] = x.Id, ["Allocations"] = x.TotalAllocated, ["HumaneAllocations"] = Size.Humane(x.TotalAllocated) })); var groupStats = new DynamicJsonValue { ["Name"] = stats.Key, ["Allocations"] = unmanagedAllocations, ["HumaneAllocations"] = Size.Humane(unmanagedAllocations) }; if (ids.Count == 1) { var threadStats = stats.First(); groupStats["Id"] = threadStats.UnmanagedThreadId; groupStats["ManagedThreadId"] = threadStats.Id; } else { groupStats["Ids"] = ids; } threads.Add(groupStats); } long managedMemoryInBytes = MemoryInformation.GetManagedMemoryInBytes(); long workingSetInBytes = memInfo.WorkingSet.GetValue(SizeUnit.Bytes); var djv = new DynamicJsonValue { [nameof(MemoryInfo.WorkingSet)] = workingSetInBytes, [nameof(MemoryInfo.TotalUnmanagedAllocations)] = totalUnmanagedAllocations, [nameof(MemoryInfo.ManagedAllocations)] = managedMemoryInBytes, [nameof(MemoryInfo.TotalMemoryMapped)] = totalMapping, [nameof(MemoryInfo.PhysicalMem)] = Size.Humane(memInfo.TotalPhysicalMemory.GetValue(SizeUnit.Bytes)), [nameof(MemoryInfo.FreeMem)] = Size.Humane(memInfo.AvailableWithoutTotalCleanMemory.GetValue(SizeUnit.Bytes)), [nameof(MemoryInfo.HighMemLastOneMinute)] = Size.Humane(memoryUsageRecords.High.LastOneMinute.GetValue(SizeUnit.Bytes)), [nameof(MemoryInfo.LowMemLastOneMinute)] = Size.Humane(memoryUsageRecords.Low.LastOneMinute.GetValue(SizeUnit.Bytes)), [nameof(MemoryInfo.HighMemLastFiveMinute)] = Size.Humane(memoryUsageRecords.High.LastFiveMinutes.GetValue(SizeUnit.Bytes)), [nameof(MemoryInfo.LowMemLastFiveMinute)] = Size.Humane(memoryUsageRecords.Low.LastFiveMinutes.GetValue(SizeUnit.Bytes)), [nameof(MemoryInfo.HighMemSinceStartup)] = Size.Humane(memoryUsageRecords.High.SinceStartup.GetValue(SizeUnit.Bytes)), [nameof(MemoryInfo.LowMemSinceStartup)] = Size.Humane(memoryUsageRecords.Low.SinceStartup.GetValue(SizeUnit.Bytes)), [nameof(MemoryInfo.Humane)] = new DynamicJsonValue { [nameof(MemoryInfoHumane.WorkingSet)] = Size.Humane(workingSetInBytes), [nameof(MemoryInfoHumane.TotalUnmanagedAllocations)] = Size.Humane(totalUnmanagedAllocations), [nameof(MemoryInfoHumane.ManagedAllocations)] = Size.Humane(managedMemoryInBytes), [nameof(MemoryInfoHumane.TotalMemoryMapped)] = Size.Humane(totalMapping) }, ["Threads"] = threads, [nameof(MemoryInfo.Mappings)] = fileMappings }; return(djv); }