private static ProcessMemoryUsage GetProcessMemoryUsage(MemoryInfoResult memoryInfo)
{
var workingSetInBytes = memoryInfo.WorkingSet.GetValue(SizeUnit.Bytes);
var privateMemory = AbstractLowMemoryMonitor.GetManagedMemoryInBytes() + AbstractLowMemoryMonitor.GetUnmanagedAllocationsInBytes();
return(new ProcessMemoryUsage(workingSetInBytes, privateMemory));
}
private static MessageDetails OutOfMemoryDetails(Exception exception)
{
var memoryInfo = MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader();
return(new MessageDetails
{
Message = $"Managed memory: {new Size(AbstractLowMemoryMonitor.GetManagedMemoryInBytes(), SizeUnit.Bytes)}, " +
$"Unmanaged allocations: {new Size(AbstractLowMemoryMonitor.GetUnmanagedAllocationsInBytes(), SizeUnit.Bytes)}, " +
$"Shared clean: {memoryInfo.SharedCleanMemory}, " +
$"Working set: {memoryInfo.WorkingSet}, " +
$"Available memory: {memoryInfo.AvailableMemory}, " +
$"Calculated Available memory: {memoryInfo.AvailableWithoutTotalCleanMemory}, " +
$"Total memory: {memoryInfo.TotalPhysicalMemory} {Environment.NewLine}" +
$"Error: {exception}"
});
}
private static byte *ThrowFailedToAllocate(long size, ThreadStats thread, OutOfMemoryException e)
{
long allocated = 0;
foreach (var threadAllocationsValue in AllThreadStats)
{
allocated += threadAllocationsValue.TotalAllocated;
}
var managed = AbstractLowMemoryMonitor.GetManagedMemoryInBytes();
var unmanagedMemory = AbstractLowMemoryMonitor.GetUnmanagedAllocationsInBytes();
throw new OutOfMemoryException($"Failed to allocate additional {new Size(size, SizeUnit.Bytes)} " +
$"to already allocated {new Size(thread.TotalAllocated, SizeUnit.Bytes)} by this thread. " +
$"Total allocated by all threads: {new Size(allocated, SizeUnit.Bytes)}, " +
$"Managed memory: {new Size(managed, SizeUnit.Bytes)}, " +
$"Un-managed memory: {new Size(unmanagedMemory, SizeUnit.Bytes)}", e);
}
public bool Execute(DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation,
IndexingStatsScope stats, CancellationToken token)
{
if (_mapReduceContext.StoreByReduceKeyHash.Count == 0)
{
WriteLastEtags(indexContext); // we need to write etags here, because if we filtered everything during map then we will loose last indexed etag information and this will cause an endless indexing loop
return(false);
}
ReduceResultsSchema.Create(indexContext.Transaction.InnerTransaction, PageNumberToReduceResultTableName, 32);
var table = indexContext.Transaction.InnerTransaction.OpenTable(ReduceResultsSchema, PageNumberToReduceResultTableName);
var lowLevelTransaction = indexContext.Transaction.InnerTransaction.LowLevelTransaction;
var writer = writeOperation.Value;
var treeScopeStats = stats.For(IndexingOperation.Reduce.TreeScope, start: false);
var nestedValuesScopeStats = stats.For(IndexingOperation.Reduce.NestedValuesScope, start: false);
foreach (var store in _mapReduceContext.StoreByReduceKeyHash)
{
token.ThrowIfCancellationRequested();
using (var reduceKeyHash = indexContext.GetLazyString(store.Key.ToString(CultureInfo.InvariantCulture)))
using (store.Value)
using (_aggregationBatch)
{
var modifiedStore = store.Value;
switch (modifiedStore.Type)
{
case MapResultsStorageType.Tree:
using (treeScopeStats.Start())
{
HandleTreeReduction(indexContext, treeScopeStats, modifiedStore, lowLevelTransaction, writer, reduceKeyHash, table, token);
}
break;
case MapResultsStorageType.Nested:
using (nestedValuesScopeStats.Start())
{
HandleNestedValuesReduction(indexContext, nestedValuesScopeStats, modifiedStore, writer, reduceKeyHash, token);
}
break;
default:
throw new ArgumentOutOfRangeException(modifiedStore.Type.ToString());
}
}
if (_mapReduceContext.FreedPages.Count > 0)
{
long tmp = 0;
using (treeScopeStats.Start())
using (Slice.External(indexContext.Allocator, (byte *)&tmp, sizeof(long), out Slice pageNumberSlice))
{
foreach (var freedPage in _mapReduceContext.FreedPages)
{
tmp = Bits.SwapBytes(freedPage);
table.DeleteByKey(pageNumberSlice);
}
}
}
}
if (stats.Duration >= MinReduceDurationToCalculateProcessMemoryUsage)
{
var workingSet = MemoryInformation.GetWorkingSetInBytes();
var privateMemory = AbstractLowMemoryMonitor.GetManagedMemoryInBytes() + AbstractLowMemoryMonitor.GetUnmanagedAllocationsInBytes();
stats.RecordReduceMemoryStats(workingSet, privateMemory);
}
WriteLastEtags(indexContext);
_mapReduceContext.StoreNextMapResultId();
return(false);
}
private static DynamicJsonValue MemoryStatsInternal()
{
var memInfo = MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader();
var memoryUsageRecords = MemoryInformation.GetMemoryUsageRecords();
long totalMapping = 0;
var fileMappingByDir = new Dictionary <string, Dictionary <string, ConcurrentDictionary <IntPtr, long> > >();
var fileMappingSizesByDir = new Dictionary <string, long>();
foreach (var mapping in NativeMemory.FileMapping)
{
var dir = Path.GetDirectoryName(mapping.Key);
if (fileMappingByDir.TryGetValue(dir, out Dictionary <string, ConcurrentDictionary <IntPtr, long> > value) == false)
{
value = new Dictionary <string, ConcurrentDictionary <IntPtr, long> >();
fileMappingByDir[dir] = value;
}
value[mapping.Key] = mapping.Value.Value.Info;
foreach (var singleMapping in mapping.Value.Value.Info)
{
fileMappingSizesByDir.TryGetValue(dir, out long prevSize);
fileMappingSizesByDir[dir] = prevSize + singleMapping.Value;
totalMapping += singleMapping.Value;
}
}
var prefixLength = LongestCommonPrefixLength(new List <string>(fileMappingSizesByDir.Keys));
var fileMappings = new DynamicJsonArray();
foreach (var sizes in fileMappingSizesByDir.OrderByDescending(x => x.Value))
{
if (fileMappingByDir.TryGetValue(sizes.Key, out Dictionary <string, ConcurrentDictionary <IntPtr, long> > value))
{
var details = new DynamicJsonValue();
var dir = new DynamicJsonValue
{
[nameof(MemoryInfoMappingItem.Directory)] = sizes.Key.Substring(prefixLength),
[nameof(MemoryInfoMappingItem.TotalDirectorySize)] = sizes.Value,
[nameof(MemoryInfoMappingItem.HumaneTotalDirectorySize)] = Size.Humane(sizes.Value),
[nameof(MemoryInfoMappingItem.Details)] = details
};
foreach (var file in value.OrderBy(x => x.Key))
{
long totalMapped = 0;
var dja = new DynamicJsonArray();
var dic = new Dictionary <long, long>();
foreach (var mapping in file.Value)
{
totalMapped += mapping.Value;
dic.TryGetValue(mapping.Value, out long prev);
dic[mapping.Value] = prev + 1;
}
foreach (var maps in dic)
{
dja.Add(new DynamicJsonValue
{
[nameof(MemoryInfoMappingDetails.Size)] = maps.Key,
[nameof(MemoryInfoMappingDetails.Count)] = maps.Value
});
}
var fileSize = GetFileSize(file.Key);
details[Path.GetFileName(file.Key)] = new DynamicJsonValue
{
[nameof(MemoryInfoMappingFileInfo.FileSize)] = fileSize,
[nameof(MemoryInfoMappingFileInfo.HumaneFileSize)] = Size.Humane(fileSize),
[nameof(MemoryInfoMappingFileInfo.TotalMapped)] = totalMapped,
[nameof(MemoryInfoMappingFileInfo.HumaneTotalMapped)] = Size.Humane(totalMapped),
[nameof(MemoryInfoMappingFileInfo.Mappings)] = dja
};
}
fileMappings.Add(dir);
}
}
long totalUnmanagedAllocations = NativeMemory.TotalAllocatedMemory;
var threads = new DynamicJsonArray();
foreach (var stats in NativeMemory.AllThreadStats
.Where(x => x.IsThreadAlive())
.GroupBy(x => x.Name)
.OrderByDescending(x => x.Sum(y => y.TotalAllocated)))
{
var unmanagedAllocations = stats.Sum(x => x.TotalAllocated);
var ids = new DynamicJsonArray(stats.OrderByDescending(x => x.TotalAllocated).Select(x => new DynamicJsonValue
{
["Id"] = x.UnmanagedThreadId,
["ManagedThreadId"] = x.ManagedThreadId,
["Allocations"] = x.TotalAllocated,
["HumaneAllocations"] = Size.Humane(x.TotalAllocated)
}));
var groupStats = new DynamicJsonValue
{
["Name"] = stats.Key,
["Allocations"] = unmanagedAllocations,
["HumaneAllocations"] = Size.Humane(unmanagedAllocations)
};
if (ids.Count == 1)
{
var threadStats = stats.First();
groupStats["Id"] = threadStats.UnmanagedThreadId;
groupStats["ManagedThreadId"] = threadStats.ManagedThreadId;
}
else
{
groupStats["Ids"] = ids;
}
threads.Add(groupStats);
}
long managedMemoryInBytes = AbstractLowMemoryMonitor.GetManagedMemoryInBytes();
long workingSetInBytes = memInfo.WorkingSet.GetValue(SizeUnit.Bytes);
var dirtyMemoryState = MemoryInformation.GetDirtyMemoryState();
var encryptionBuffers = EncryptionBuffersPool.Instance.GetStats();
var djv = new DynamicJsonValue
{
[nameof(MemoryInfo.WorkingSet)] = workingSetInBytes,
[nameof(MemoryInfo.TotalUnmanagedAllocations)] = totalUnmanagedAllocations,
[nameof(MemoryInfo.ManagedAllocations)] = managedMemoryInBytes,
[nameof(MemoryInfo.TotalMemoryMapped)] = totalMapping,
[nameof(MemoryInfo.TotalScratchDirtyMemory)] = Size.Humane(memInfo.TotalScratchDirtyMemory.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.PhysicalMem)] = Size.Humane(memInfo.TotalPhysicalMemory.GetValue(SizeUnit.Bytes)),
[nameof(DirtyMemoryState.IsHighDirty)] = dirtyMemoryState.IsHighDirty,
["DirtyMemory"] = Size.Humane(dirtyMemoryState.TotalDirtyInBytes),
[nameof(MemoryInfo.AvailableMemory)] = Size.Humane(memInfo.AvailableMemory.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.AvailableMemoryForProcessing)] = Size.Humane(memInfo.AvailableMemoryForProcessing.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.HighMemLastOneMinute)] = Size.Humane(memoryUsageRecords.High.LastOneMinute.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.LowMemLastOneMinute)] = Size.Humane(memoryUsageRecords.Low.LastOneMinute.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.HighMemLastFiveMinute)] = Size.Humane(memoryUsageRecords.High.LastFiveMinutes.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.LowMemLastFiveMinute)] = Size.Humane(memoryUsageRecords.Low.LastFiveMinutes.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.HighMemSinceStartup)] = Size.Humane(memoryUsageRecords.High.SinceStartup.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.LowMemSinceStartup)] = Size.Humane(memoryUsageRecords.Low.SinceStartup.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.Humane)] = new DynamicJsonValue
{
[nameof(MemoryInfoHumane.WorkingSet)] = Size.Humane(workingSetInBytes),
[nameof(MemoryInfoHumane.TotalUnmanagedAllocations)] = Size.Humane(totalUnmanagedAllocations),
["EncryptionBuffers"] = Size.Humane(encryptionBuffers.TotalSize),
[nameof(MemoryInfoHumane.ManagedAllocations)] = Size.Humane(managedMemoryInBytes),
[nameof(MemoryInfoHumane.TotalMemoryMapped)] = Size.Humane(totalMapping)
},
["Threads"] = threads,
[nameof(MemoryInfo.Mappings)] = fileMappings
};
return(djv);
}
protected override Gauge32 GetData()
{
var managedMemoryInBytes = AbstractLowMemoryMonitor.GetManagedMemoryInBytes();
return(new Gauge32(new Size(managedMemoryInBytes, SizeUnit.Bytes).GetValue(SizeUnit.Megabytes)));
}
private static void WriteMemoryStats(BlittableJsonTextWriter writer, JsonOperationContext context, bool includeThreads, bool includeMappings)
{
writer.WriteStartObject();
var memInfo = MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader();
long managedMemoryInBytes = AbstractLowMemoryMonitor.GetManagedMemoryInBytes();
long totalUnmanagedAllocations = NativeMemory.TotalAllocatedMemory;
var encryptionBuffers = EncryptionBuffersPool.Instance.GetStats();
var dirtyMemoryState = MemoryInformation.GetDirtyMemoryState();
var memoryUsageRecords = MemoryInformation.GetMemoryUsageRecords();
long totalMapping = 0;
var fileMappingByDir = new Dictionary <string, Dictionary <string, ConcurrentDictionary <IntPtr, long> > >();
var fileMappingSizesByDir = new Dictionary <string, long>();
foreach (var mapping in NativeMemory.FileMapping)
{
var dir = Path.GetDirectoryName(mapping.Key);
if (fileMappingByDir.TryGetValue(dir, out Dictionary <string, ConcurrentDictionary <IntPtr, long> > value) == false)
{
value = new Dictionary <string, ConcurrentDictionary <IntPtr, long> >();
fileMappingByDir[dir] = value;
}
value[mapping.Key] = mapping.Value.Value.Info;
foreach (var singleMapping in mapping.Value.Value.Info)
{
fileMappingSizesByDir.TryGetValue(dir, out long prevSize);
fileMappingSizesByDir[dir] = prevSize + singleMapping.Value;
totalMapping += singleMapping.Value;
}
}
var djv = new DynamicJsonValue
{
[nameof(MemoryInfo.PhysicalMemory)] = memInfo.TotalPhysicalMemory.ToString(),
[nameof(MemoryInfo.WorkingSet)] = memInfo.WorkingSet.ToString(),
[nameof(MemoryInfo.ManagedAllocations)] = Size.Humane(managedMemoryInBytes),
[nameof(MemoryInfo.UnmanagedAllocations)] = Size.Humane(totalUnmanagedAllocations),
[nameof(MemoryInfo.EncryptionBuffers)] = Size.Humane(encryptionBuffers.TotalSize),
[nameof(MemoryInfo.MemoryMapped)] = Size.Humane(totalMapping),
[nameof(MemoryInfo.ScratchDirtyMemory)] = memInfo.TotalScratchDirtyMemory.ToString(),
[nameof(MemoryInfo.IsHighDirty)] = dirtyMemoryState.IsHighDirty,
[nameof(MemoryInfo.DirtyMemory)] = Size.Humane(dirtyMemoryState.TotalDirtyInBytes),
[nameof(MemoryInfo.AvailableMemory)] = Size.Humane(memInfo.AvailableMemory.GetValue(SizeUnit.Bytes)),
[nameof(MemoryInfo.AvailableMemoryForProcessing)] = memInfo.AvailableMemoryForProcessing.ToString(),
[nameof(MemoryInfo.HighMemLastOneMinute)] = memoryUsageRecords.High.LastOneMinute.ToString(),
[nameof(MemoryInfo.LowMemLastOneMinute)] = memoryUsageRecords.Low.LastOneMinute.ToString(),
[nameof(MemoryInfo.HighMemLastFiveMinute)] = memoryUsageRecords.High.LastFiveMinutes.ToString(),
[nameof(MemoryInfo.LowMemLastFiveMinute)] = memoryUsageRecords.Low.LastFiveMinutes.ToString(),
[nameof(MemoryInfo.HighMemSinceStartup)] = memoryUsageRecords.High.SinceStartup.ToString(),
[nameof(MemoryInfo.LowMemSinceStartup)] = memoryUsageRecords.Low.SinceStartup.ToString(),
};
writer.WritePropertyName(nameof(MemoryInformation));
context.Write(writer, djv);
writer.WriteComma();
writer.WritePropertyName("Threads");
writer.WriteStartArray();
WriteThreads(includeThreads, writer, context);
writer.WriteEndArray();
writer.WriteComma();
writer.WritePropertyName(nameof(MemoryInfo.Mappings));
writer.WriteStartArray();
WriteMappings(includeMappings, writer, context, fileMappingSizesByDir, fileMappingByDir);
writer.WriteEndArray();
writer.WriteEndObject();
}