private static MessageDetails OutOfMemoryDetails(Exception exception)
{
var memoryInfo = MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader();
return(new MessageDetails
{
Message = $"Managed memory: {new Size(AbstractLowMemoryMonitor.GetManagedMemoryInBytes(), SizeUnit.Bytes)}, " +
$"Unmanaged allocations: {new Size(AbstractLowMemoryMonitor.GetUnmanagedAllocationsInBytes(), SizeUnit.Bytes)}, " +
$"Shared clean: {memoryInfo.SharedCleanMemory}, " +
$"Working set: {memoryInfo.WorkingSet}, " +
$"Available memory: {memoryInfo.AvailableMemory}, " +
$"Calculated Available memory: {memoryInfo.AvailableWithoutTotalCleanMemory}, " +
$"Total memory: {memoryInfo.TotalPhysicalMemory} {Environment.NewLine}" +
$"Error: {exception}"
});
}
private static byte *ThrowFailedToAllocate(long size, ThreadStats thread, OutOfMemoryException e)
{
long allocated = 0;
foreach (var threadAllocationsValue in AllThreadStats)
{
allocated += threadAllocationsValue.TotalAllocated;
}
var managed = AbstractLowMemoryMonitor.GetManagedMemoryInBytes();
var unmanagedMemory = AbstractLowMemoryMonitor.GetUnmanagedAllocationsInBytes();
throw new OutOfMemoryException($"Failed to allocate additional {new Size(size, SizeUnit.Bytes)} " +
$"to already allocated {new Size(thread.TotalAllocated, SizeUnit.Bytes)} by this thread. " +
$"Total allocated by all threads: {new Size(allocated, SizeUnit.Bytes)}, " +
$"Managed memory: {new Size(managed, SizeUnit.Bytes)}, " +
$"Un-managed memory: {new Size(unmanagedMemory, SizeUnit.Bytes)}", e);
}
public bool Execute(DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation,
IndexingStatsScope stats, CancellationToken token)
{
if (_mapReduceContext.StoreByReduceKeyHash.Count == 0)
{
WriteLastEtags(indexContext); // we need to write etags here, because if we filtered everything during map then we will loose last indexed etag information and this will cause an endless indexing loop
return(false);
}
ReduceResultsSchema.Create(indexContext.Transaction.InnerTransaction, PageNumberToReduceResultTableName, 32);
var table = indexContext.Transaction.InnerTransaction.OpenTable(ReduceResultsSchema, PageNumberToReduceResultTableName);
var lowLevelTransaction = indexContext.Transaction.InnerTransaction.LowLevelTransaction;
var writer = writeOperation.Value;
var treeScopeStats = stats.For(IndexingOperation.Reduce.TreeScope, start: false);
var nestedValuesScopeStats = stats.For(IndexingOperation.Reduce.NestedValuesScope, start: false);
foreach (var store in _mapReduceContext.StoreByReduceKeyHash)
{
token.ThrowIfCancellationRequested();
using (var reduceKeyHash = indexContext.GetLazyString(store.Key.ToString(CultureInfo.InvariantCulture)))
using (store.Value)
using (_aggregationBatch)
{
var modifiedStore = store.Value;
switch (modifiedStore.Type)
{
case MapResultsStorageType.Tree:
using (treeScopeStats.Start())
{
HandleTreeReduction(indexContext, treeScopeStats, modifiedStore, lowLevelTransaction, writer, reduceKeyHash, table, token);
}
break;
case MapResultsStorageType.Nested:
using (nestedValuesScopeStats.Start())
{
HandleNestedValuesReduction(indexContext, nestedValuesScopeStats, modifiedStore, writer, reduceKeyHash, token);
}
break;
default:
throw new ArgumentOutOfRangeException(modifiedStore.Type.ToString());
}
}
if (_mapReduceContext.FreedPages.Count > 0)
{
long tmp = 0;
using (treeScopeStats.Start())
using (Slice.External(indexContext.Allocator, (byte *)&tmp, sizeof(long), out Slice pageNumberSlice))
{
foreach (var freedPage in _mapReduceContext.FreedPages)
{
tmp = Bits.SwapBytes(freedPage);
table.DeleteByKey(pageNumberSlice);
}
}
}
}
if (stats.Duration >= MinReduceDurationToCalculateProcessMemoryUsage)
{
var workingSet = MemoryInformation.GetWorkingSetInBytes();
var privateMemory = AbstractLowMemoryMonitor.GetManagedMemoryInBytes() + AbstractLowMemoryMonitor.GetUnmanagedAllocationsInBytes();
stats.RecordReduceMemoryStats(workingSet, privateMemory);
}
WriteLastEtags(indexContext);
_mapReduceContext.StoreNextMapResultId();
return(false);
}
private static ProcessMemoryUsage GetProcessMemoryUsage(MemoryInfoResult memoryInfo)
{
var workingSetInBytes = memoryInfo.WorkingSet.GetValue(SizeUnit.Bytes);
var privateMemory = AbstractLowMemoryMonitor.GetManagedMemoryInBytes() + AbstractLowMemoryMonitor.GetUnmanagedAllocationsInBytes();
return(new ProcessMemoryUsage(workingSetInBytes, privateMemory));
}
protected override Gauge32 GetData()
{
var unmanagedMemoryInBytes = AbstractLowMemoryMonitor.GetUnmanagedAllocationsInBytes();
return(new Gauge32(new Size(unmanagedMemoryInBytes, SizeUnit.Bytes).GetValue(SizeUnit.Megabytes)));
}
