public static MemoryInfoResult GetMemoryInfo(SmapsReader smapsReader = null)
{
if (_failedToGetAvailablePhysicalMemory)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Because of a previous error in getting available memory, we are now lying and saying we have 256MB free");
}
return(FailedResult);
}
try
{
if (PlatformDetails.RunningOnPosix == false)
{
return(GetMemoryInfoWindows());
}
if (PlatformDetails.RunningOnMacOsx)
{
return(GetMemoryInfoMacOs());
}
return(GetMemoryInfoLinux(smapsReader));
}
catch (Exception e)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Error while trying to get available memory, will stop trying and report that there is 256MB free only from now on", e);
}
_failedToGetAvailablePhysicalMemory = true;
return(FailedResult);
}
}
public static MachineResources GetMachineResources(SmapsReader smapsReader)
{
using (var currentProcess = Process.GetCurrentProcess())
{
var memInfo = MemoryInformation.GetMemoryInfo(smapsReader);
var isLowMemory = LowMemoryNotification.Instance.IsLowMemory(memInfo);
var workingSet = PlatformDetails.RunningOnLinux
? MemoryInformation.GetRssMemoryUsage(currentProcess.Id) - memInfo.AvailableWithoutTotalCleanMemory.GetValue(SizeUnit.Bytes)
: currentProcess.WorkingSet64;
var cpuInfo = CpuUsage.Calculate();
var machineResources = new MachineResources
{
TotalMemory = memInfo.TotalPhysicalMemory.GetValue(SizeUnit.Bytes),
AvailableMemory = memInfo.AvailableMemory.GetValue(SizeUnit.Bytes),
AvailableWithoutTotalCleanMemory = memInfo.AvailableWithoutTotalCleanMemory.GetValue(SizeUnit.Bytes),
SystemCommitLimit = memInfo.TotalCommittableMemory.GetValue(SizeUnit.Bytes),
CommittedMemory = memInfo.CurrentCommitCharge.GetValue(SizeUnit.Bytes),
ProcessMemoryUsage = workingSet,
IsWindows = PlatformDetails.RunningOnPosix == false,
IsLowMemory = isLowMemory,
LowMemoryThreshold = LowMemoryNotification.Instance.LowMemoryThreshold.GetValue(SizeUnit.Bytes),
CommitChargeThreshold = LowMemoryNotification.Instance.GetCommitChargeThreshold(memInfo).GetValue(SizeUnit.Bytes),
MachineCpuUsage = cpuInfo.MachineCpuUsage,
ProcessCpuUsage = Math.Min(cpuInfo.MachineCpuUsage, cpuInfo.ProcessCpuUsage) // min as sometimes +-1% due to time sampling
};
return(machineResources);
}
}
public static MemoryInfoResult GetMemoryInformationUsingOneTimeSmapsReader()
{
SmapsReader smapsReader = null;
byte[][] buffers = null;
try
{
if (PlatformDetails.RunningOnLinux)
{
var buffer1 = ArrayPool <byte> .Shared.Rent(SmapsReader.BufferSize);
var buffer2 = ArrayPool <byte> .Shared.Rent(SmapsReader.BufferSize);
buffers = new[] { buffer1, buffer2 };
smapsReader = new SmapsReader(new[] { buffer1, buffer2 });
}
return(GetMemoryInfo(smapsReader, extended: true));
}
finally
{
if (buffers != null)
{
ArrayPool <byte> .Shared.Return(buffers[0]);
ArrayPool <byte> .Shared.Return(buffers[1]);
}
}
}
public void ParsesSmapsProperly()
{
var assembly = typeof(SmapsReaderTests).Assembly;
var smapsReader = new SmapsReader(new[]
{
new byte[SmapsReader.BufferSize], new byte[SmapsReader.BufferSize]
});
SmapsReader.SmapsReadResult <SmapsTestResult> result;
using (var fs =
assembly.GetManifestResourceStream("SlowTests.Data.RavenDB_15159.12119.smaps.gz"))
using (var deflated = new GZipStream(fs, CompressionMode.Decompress))
using (var smapsStream = new FakeProcSmapsEntriesStream(deflated))
{
result = smapsReader
.CalculateMemUsageFromSmaps <SmapsTestResult>(smapsStream, pid: 1234);
}
// 385 .buffers
// 181 .voron
Assert.Equal(385 + 181, result.SmapsResults.Entries.Count);
// cat 12119.smaps | grep -e "rw-s" -A 3 | awk '$1 ~ /Rss/ {sum += $2} END {print sum}'
Assert.Equal(722136L * 1024, result.Rss);
// cat 12119.smaps | grep -e "rw-s" -A 16 | awk '$1 ~ /Swap/ {sum += $2} END {print sum}'
Assert.Equal(1348L * 1024, result.Swap);
}
private static MemoryInfoResult GetMemoryInfoLinux(SmapsReader smapsReader)
{
var fromProcMemInfo = GetFromProcMemInfo(smapsReader);
var totalPhysicalMemoryInBytes = fromProcMemInfo.TotalMemory.GetValue(SizeUnit.Bytes);
var cgroupMemoryLimit = KernelVirtualFileSystemUtils.ReadNumberFromCgroupFile(CgroupMemoryLimit);
var cgroupMaxMemoryUsage = KernelVirtualFileSystemUtils.ReadNumberFromCgroupFile(CgroupMaxMemoryUsage);
// here we need to deal with _soft_ limit, so we'll take the largest of these values
var maxMemoryUsage = Math.Max(cgroupMemoryLimit ?? 0, cgroupMaxMemoryUsage ?? 0);
if (maxMemoryUsage != 0 && maxMemoryUsage <= totalPhysicalMemoryInBytes)
{
// running in a limited cgroup
var commitedMemoryInBytes = 0L;
var cgroupMemoryUsage = KernelVirtualFileSystemUtils.ReadNumberFromCgroupFile(CgroupMemoryUsage);
if (cgroupMemoryUsage != null)
{
commitedMemoryInBytes = cgroupMemoryUsage.Value;
fromProcMemInfo.Commited.Set(commitedMemoryInBytes, SizeUnit.Bytes);
fromProcMemInfo.MemAvailable.Set(maxMemoryUsage - cgroupMemoryUsage.Value, SizeUnit.Bytes);
}
fromProcMemInfo.TotalMemory.Set(maxMemoryUsage, SizeUnit.Bytes);
fromProcMemInfo.CommitLimit.Set(Math.Max(maxMemoryUsage, commitedMemoryInBytes), SizeUnit.Bytes);
}
return(BuildPosixMemoryInfoResult(
fromProcMemInfo.MemAvailable,
fromProcMemInfo.TotalMemory,
fromProcMemInfo.Commited,
fromProcMemInfo.CommitLimit,
fromProcMemInfo.AvailableWithoutTotalCleanMemory,
fromProcMemInfo.SharedCleanMemory
));
}
public ServerMetricCacher(RavenServer server)
{
_server = server;
if (PlatformDetails.RunningOnLinux)
{
_smapsReader = new SmapsReader(new[] { new byte[SmapsReader.BufferSize], new byte[SmapsReader.BufferSize] });
}
}
public Task MemorySmaps()
{
if (PlatformDetails.RunningOnLinux == false)
{
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
{
var rc = Win32MemoryQueryMethods.GetMaps();
var djv = new DynamicJsonValue
{
["Totals"] = new DynamicJsonValue
{
["WorkingSet"] = rc.WorkingSet,
["SharedClean"] = "N/A",
["PrivateClean"] = "N/A",
["TotalClean"] = rc.ProcessClean,
["RssHumanly"] = Sizes.Humane(rc.WorkingSet),
["SharedCleanHumanly"] = "N/A",
["PrivateCleanHumanly"] = "N/A",
["TotalCleanHumanly"] = Sizes.Humane(rc.ProcessClean)
},
["Details"] = rc.Json
};
using (var write = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(write, djv);
}
return(Task.CompletedTask);
}
}
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
{
var result = new SmapsReader().CalculateMemUsageFromSmaps();
var djv = new DynamicJsonValue
{
["Totals"] = new DynamicJsonValue
{
["WorkingSet"] = result.Rss,
["SharedClean"] = result.SharedClean,
["PrivateClean"] = result.PrivateClean,
["TotalClean"] = result.SharedClean + result.PrivateClean,
["RssHumanly"] = Sizes.Humane(result.Rss),
["SharedCleanHumanly"] = Sizes.Humane(result.SharedClean),
["PrivateCleanHumanly"] = Sizes.Humane(result.PrivateClean),
["TotalCleanHumanly"] = Sizes.Humane(result.SharedClean + result.PrivateClean)
},
["Details"] = result.Json
};
using (var write = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(write, djv);
}
return(Task.CompletedTask);
}
}
private static MemoryInfoResult GetMemoryInfoLinux(SmapsReader smapsReader, bool extended)
{
var fromProcMemInfo = new ProcMemInfoResults();
GetFromProcMemInfo(smapsReader, ref fromProcMemInfo);
var totalPhysicalMemoryInBytes = fromProcMemInfo.TotalMemory.GetValue(SizeUnit.Bytes);
var cgroupMemoryLimit = KernelVirtualFileSystemUtils.ReadNumberFromCgroupFile(CgroupMemoryLimit);
var cgroupMaxMemoryUsage = KernelVirtualFileSystemUtils.ReadNumberFromCgroupFile(CgroupMaxMemoryUsage);
// here we need to deal with _soft_ limit, so we'll take the largest of these values
var maxMemoryUsage = Math.Max(cgroupMemoryLimit ?? 0, cgroupMaxMemoryUsage ?? 0);
if (maxMemoryUsage != 0 && maxMemoryUsage <= totalPhysicalMemoryInBytes)
{
// running in a limited cgroup
var commitedMemoryInBytes = 0L;
var cgroupMemoryUsage = LowMemoryNotification.Instance.UseTotalDirtyMemInsteadOfMemUsage // RDBS-45
? fromProcMemInfo.TotalDirty.GetValue(SizeUnit.Bytes)
: KernelVirtualFileSystemUtils.ReadNumberFromCgroupFile(CgroupMemoryUsage);
if (cgroupMemoryUsage != null)
{
commitedMemoryInBytes = cgroupMemoryUsage.Value;
fromProcMemInfo.Commited.Set(commitedMemoryInBytes, SizeUnit.Bytes);
fromProcMemInfo.MemAvailable.Set(maxMemoryUsage - cgroupMemoryUsage.Value, SizeUnit.Bytes);
var realAvailable = maxMemoryUsage - cgroupMemoryUsage.Value + fromProcMemInfo.SharedCleanMemory.GetValue(SizeUnit.Bytes);
if (realAvailable < 0)
{
realAvailable = 0;
}
fromProcMemInfo.AvailableWithoutTotalCleanMemory.Set(realAvailable, SizeUnit.Bytes);
}
fromProcMemInfo.TotalMemory.Set(maxMemoryUsage, SizeUnit.Bytes);
fromProcMemInfo.CommitLimit.Set(Math.Max(maxMemoryUsage, commitedMemoryInBytes), SizeUnit.Bytes);
}
var workingSet = new Size(0, SizeUnit.Bytes);
if (smapsReader != null)
{
// extended info is needed
workingSet.Set(GetRssMemoryUsage(), SizeUnit.Bytes);
}
return(BuildPosixMemoryInfoResult(
fromProcMemInfo.MemAvailable,
fromProcMemInfo.TotalMemory,
fromProcMemInfo.Commited,
fromProcMemInfo.CommitLimit,
fromProcMemInfo.AvailableWithoutTotalCleanMemory,
fromProcMemInfo.SharedCleanMemory,
workingSet,
extended));
}
public async Task Get()
{
using (var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync())
{
using (var writer = new NotificationCenterWebSocketWriter(webSocket, ServerStore.ServerDashboardNotifications, ServerStore.ContextPool,
ServerStore.ServerShutdown))
{
var isValidFor = GetDatabaseAccessValidationFunc();
byte[][] buffers = null;
try
{
SmapsReader smapsReader = null;
if (PlatformDetails.RunningOnLinux)
{
var buffer1 = ArrayPool <byte> .Shared.Rent(SmapsReader.BufferSize);
var buffer2 = ArrayPool <byte> .Shared.Rent(SmapsReader.BufferSize);
buffers = new[] { buffer1, buffer2 };
smapsReader = new SmapsReader(new[] { buffer1, buffer2 });
}
var machineResources = MachineResourcesNotificationSender.GetMachineResources(smapsReader);
await writer.WriteToWebSocket(machineResources.ToJson());
using (var cts = CancellationTokenSource.CreateLinkedTokenSource(ServerStore.ServerShutdown))
{
var databasesInfo = DatabasesInfoNotificationSender.FetchDatabasesInfo(ServerStore, isValidFor, cts);
foreach (var info in databasesInfo)
{
await writer.WriteToWebSocket(info.ToJson());
}
}
}
catch (Exception e)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Failed to send the initial server dashboard data", e);
}
}
finally
{
if (buffers != null)
{
ArrayPool <byte> .Shared.Return(buffers[0]);
ArrayPool <byte> .Shared.Return(buffers[1]);
}
}
await writer.WriteNotifications(isValidFor);
}
}
}
internal static bool GetFromProcMemInfo(SmapsReader smapsReader, ref ProcMemInfoResults procMemInfoResults)
{
const string path = "/proc/meminfo";
// this is different then sysinfo freeram+buffered (and the closest to the real free memory)
// MemFree is really different then MemAvailable (while free is usually lower then the real free,
// and available is only estimated free which sometimes higher then the real free memory)
// for some distros we have only MemFree
try
{
using (var bufferedReader = new KernelVirtualFileSystemUtils.BufferedPosixKeyValueOutputValueReader(path))
{
bufferedReader.ReadFileIntoBuffer();
var memAvailableInKb = bufferedReader.ExtractNumericValueFromKeyValuePairsFormattedFile(MemAvailable);
var memFreeInKb = bufferedReader.ExtractNumericValueFromKeyValuePairsFormattedFile(MemFree);
var totalMemInKb = bufferedReader.ExtractNumericValueFromKeyValuePairsFormattedFile(MemTotal);
var swapTotalInKb = bufferedReader.ExtractNumericValueFromKeyValuePairsFormattedFile(SwapTotal);
var commitedInKb = bufferedReader.ExtractNumericValueFromKeyValuePairsFormattedFile(Committed_AS);
var totalClean = new Size(memAvailableInKb, SizeUnit.Kilobytes);
var totalDirty = new Size(0, SizeUnit.Bytes);
var sharedCleanMemory = new Size(0, SizeUnit.Bytes);
if (smapsReader != null)
{
var result = smapsReader.CalculateMemUsageFromSmaps <SmapsReaderNoAllocResults>();
totalClean.Add(result.SharedClean, SizeUnit.Bytes);
totalClean.Add(result.PrivateClean, SizeUnit.Bytes);
sharedCleanMemory.Set(result.SharedClean, SizeUnit.Bytes);
totalDirty.Add(result.TotalDirty, SizeUnit.Bytes);
}
procMemInfoResults.MemAvailable = new Size(Math.Max(memAvailableInKb, memFreeInKb), SizeUnit.Kilobytes);
procMemInfoResults.TotalMemory = new Size(totalMemInKb, SizeUnit.Kilobytes);
procMemInfoResults.Commited = new Size(commitedInKb, SizeUnit.Kilobytes);
// on Linux, we use the swap + ram as the commit limit, because the actual limit
// is dependent on many different factors
procMemInfoResults.CommitLimit = new Size(totalMemInKb + swapTotalInKb, SizeUnit.Kilobytes);
procMemInfoResults.AvailableWithoutTotalCleanMemory = totalClean;
procMemInfoResults.SharedCleanMemory = sharedCleanMemory;
procMemInfoResults.TotalDirty = totalDirty;
}
}
catch (Exception ex)
{
if (Logger.IsInfoEnabled)
{
Logger.Info($"Failed to read value from {path}", ex);
}
return(false);
}
return(true);
}
public TestResourcesAnalyzerMetricCacher(ICpuUsageCalculator cpuUsageCalculator)
{
if (PlatformDetails.RunningOnLinux)
{
_smapsReader = new SmapsReader(new[] { new byte[SmapsReader.BufferSize], new byte[SmapsReader.BufferSize] });
}
Register(Keys.Server.CpuUsage, _cacheRefreshRate, cpuUsageCalculator.Calculate);
Register(Keys.Server.MemoryInfo, _cacheRefreshRate, CalculateMemoryInfo);
Register(Keys.Server.MemoryInfoExtended, _cacheRefreshRate, CalculateMemoryInfoExtended);
}
public TestResourcesAnalyzerMetricCacher(ICpuUsageCalculator cpuUsageCalculator)
{
if (PlatformDetails.RunningOnLinux)
{
_smapsReader = new SmapsReader(new[] { new byte[SmapsReader.BufferSize], new byte[SmapsReader.BufferSize] });
}
Register(MetricCacher.Keys.Server.CpuUsage, TimeSpan.FromSeconds(1), cpuUsageCalculator.Calculate);
Register(MetricCacher.Keys.Server.MemoryInfo, TimeSpan.FromSeconds(1), CalculateMemoryInfo);
Register(MetricCacher.Keys.Server.MemoryInfoExtended, TimeSpan.FromSeconds(1), CalculateMemoryInfoExtended);
}
public LowMemoryMonitor()
{
if (PlatformDetails.RunningOnLinux)
{
var buffer1 = ArrayPool <byte> .Shared.Rent(SmapsReader.BufferSize);
var buffer2 = ArrayPool <byte> .Shared.Rent(SmapsReader.BufferSize);
_buffers = new[] { buffer1, buffer2 };
_smapsReader = new SmapsReader(_buffers);
}
}
protected override async Task DoWork()
{
var now = DateTime.UtcNow;
var timeSpan = now - _lastSentNotification;
if (timeSpan < _notificationsThrottle)
{
await WaitOrThrowOperationCanceled(_notificationsThrottle - timeSpan);
}
byte[][] buffers = null;
try
{
if (CancellationToken.IsCancellationRequested)
{
return;
}
if (_watchers.Count == 0)
{
return;
}
SmapsReader smapsReader = null;
if (PlatformDetails.RunningOnLinux)
{
var buffer1 = ArrayPool <byte> .Shared.Rent(SmapsReader.BufferSize);
var buffer2 = ArrayPool <byte> .Shared.Rent(SmapsReader.BufferSize);
buffers = new [] { buffer1, buffer2 };
smapsReader = new SmapsReader(new[] { buffer1, buffer2 });
}
var machineResources = GetMachineResources(smapsReader);
foreach (var watcher in _watchers)
{
// serialize to avoid race conditions
// please notice we call ToJson inside a loop since DynamicJsonValue is not thread-safe
watcher.NotificationsQueue.Enqueue(machineResources.ToJson());
}
}
finally
{
_lastSentNotification = DateTime.UtcNow;
if (buffers != null)
{
ArrayPool <byte> .Shared.Return(buffers[0]);
ArrayPool <byte> .Shared.Return(buffers[1]);
}
}
}
internal static MemoryInfoResult GetMemoryInfo(SmapsReader smapsReader = null, bool extended = false)
{
if (_failedToGetAvailablePhysicalMemory)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Because of a previous error in getting available memory, we are now lying and saying we have 256MB free");
}
return(FailedResult);
}
try
{
extended &= PlatformDetails.RunningOnLinux == false;
MemoryInfoResult result;
using (var process = extended ? Process.GetCurrentProcess() : null)
{
if (PlatformDetails.RunningOnPosix == false)
{
result = GetMemoryInfoWindows(process, extended);
}
else if (PlatformDetails.RunningOnMacOsx)
{
result = GetMemoryInfoMacOs(process, extended);
}
else
{
result = GetMemoryInfoLinux(smapsReader, extended);
}
}
var totalScratchAllocated = GetTotalScratchAllocatedMemory();
result.AvailableMemory.Add(-totalScratchAllocated, SizeUnit.Bytes);
return(result);
}
catch (Exception e)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Error while trying to get available memory, will stop trying and report that there is 256MB free only from now on", e);
}
_failedToGetAvailablePhysicalMemory = true;
return(FailedResult);
}
}
public MachineResourcesNotificationSender(string resourceName,
ConcurrentSet <ConnectedWatcher> watchers, TimeSpan notificationsThrottle, CancellationToken shutdown)
: base(resourceName, shutdown)
{
_watchers = watchers;
_notificationsThrottle = notificationsThrottle;
if (PlatformDetails.RunningOnLinux)
{
var buffer1 = ArrayPool <byte> .Shared.Rent(SmapsReader.BufferSize);
var buffer2 = ArrayPool <byte> .Shared.Rent(SmapsReader.BufferSize);
_buffers = new[] { buffer1, buffer2 };
_smapsReader = new SmapsReader(new[] { buffer1, buffer2 });
}
}
public static MachineResources GetMachineResources(SmapsReader smapsReader, ICpuUsageCalculator cpuUsageCalculator)
{
var memInfo = MemoryInformation.GetMemoryInfo(smapsReader, extendedInfo: true);
var cpuInfo = cpuUsageCalculator.Calculate();
var machineResources = new MachineResources
{
TotalMemory = memInfo.TotalPhysicalMemory.GetValue(SizeUnit.Bytes),
AvailableMemory = memInfo.AvailableMemory.GetValue(SizeUnit.Bytes),
AvailableWithoutTotalCleanMemory = memInfo.AvailableWithoutTotalCleanMemory.GetValue(SizeUnit.Bytes),
SystemCommitLimit = memInfo.TotalCommittableMemory.GetValue(SizeUnit.Bytes),
CommittedMemory = memInfo.CurrentCommitCharge.GetValue(SizeUnit.Bytes),
ProcessMemoryUsage = memInfo.WorkingSet.GetValue(SizeUnit.Bytes),
IsWindows = PlatformDetails.RunningOnPosix == false,
IsLowMemory = LowMemoryNotification.Instance.IsLowMemory(memInfo, smapsReader),
LowMemoryThreshold = LowMemoryNotification.Instance.LowMemoryThreshold.GetValue(SizeUnit.Bytes),
CommitChargeThreshold = LowMemoryNotification.Instance.GetCommitChargeThreshold(memInfo).GetValue(SizeUnit.Bytes),
MachineCpuUsage = cpuInfo.MachineCpuUsage,
ProcessCpuUsage = cpuInfo.ProcessCpuUsage
};
return(machineResources);
}
public MachineResources GetMachineResources(SmapsReader smapsReader)
{
return(GetMachineResources(smapsReader, _server.CpuUsageCalculator));
}
public Task MemorySmaps()
{
if (PlatformDetails.RunningOnLinux == false)
{
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
using (var process = Process.GetCurrentProcess())
{
var sharedClean = MemoryInformation.GetSharedCleanInBytes(process);
var rc = Win32MemoryQueryMethods.GetMaps();
var djv = new DynamicJsonValue
{
["Totals"] = new DynamicJsonValue
{
["WorkingSet"] = process.WorkingSet64,
["SharedClean"] = Sizes.Humane(sharedClean),
["PrivateClean"] = "N/A",
["TotalClean"] = rc.ProcessClean,
["RssHumanly"] = Sizes.Humane(process.WorkingSet64),
["SharedCleanHumanly"] = Sizes.Humane(sharedClean),
["PrivateCleanHumanly"] = "N/A",
["TotalCleanHumanly"] = Sizes.Humane(rc.ProcessClean)
},
["Details"] = rc.Json
};
using (var write = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(write, djv);
}
return(Task.CompletedTask);
}
}
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
{
var buffers = new[]
{
ArrayPool <byte> .Shared.Rent(SmapsReader.BufferSize),
ArrayPool <byte> .Shared.Rent(SmapsReader.BufferSize)
};
try
{
var result = new SmapsReader(buffers).CalculateMemUsageFromSmaps <SmapsReaderJsonResults>();
var procStatus = MemoryInformation.GetMemoryUsageFromProcStatus();
var djv = new DynamicJsonValue
{
["Totals"] = new DynamicJsonValue
{
["WorkingSet"] = result.Rss,
["SharedClean"] = result.SharedClean,
["PrivateClean"] = result.PrivateClean,
["TotalClean"] = result.SharedClean + result.PrivateClean,
["TotalDirty"] = result.TotalDirty, // This includes not only r-ws buffer and voron files, but also dotnet's and heap dirty memory
["WorkingSetSwap"] = result.Swap, // Swap values sum for r-ws entries only
["Swap"] = procStatus.Swap,
["RssHumanly"] = Sizes.Humane(result.Rss),
["SwapHumanly"] = Sizes.Humane(procStatus.Swap),
["WorkingSetSwapHumanly"] = Sizes.Humane(result.Swap),
["SharedCleanHumanly"] = Sizes.Humane(result.SharedClean),
["PrivateCleanHumanly"] = Sizes.Humane(result.PrivateClean),
["TotalCleanHumanly"] = Sizes.Humane(result.SharedClean + result.PrivateClean)
},
["Details"] = result.SmapsResults.ReturnResults()
};
using (var write = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(write, djv);
}
return(Task.CompletedTask);
}
finally
{
ArrayPool <byte> .Shared.Return(buffers[0]);
ArrayPool <byte> .Shared.Return(buffers[1]);
}
}
}
private static MemoryInfoResult GetMemoryInfoLinux(SmapsReader smapsReader, bool extended)
{
var fromProcMemInfo = new ProcMemInfoResults();
GetFromProcMemInfo(smapsReader, ref fromProcMemInfo);
var totalPhysicalMemoryInBytes = fromProcMemInfo.TotalMemory.GetValue(SizeUnit.Bytes);
var cgroupMemoryLimit = KernelVirtualFileSystemUtils.ReadNumberFromCgroupFile(CgroupMemoryLimit);
var cgroupMaxMemoryUsage = KernelVirtualFileSystemUtils.ReadNumberFromCgroupFile(CgroupMaxMemoryUsage);
// here we need to deal with _soft_ limit, so we'll take the largest of these values
var maxMemoryUsage = Math.Max(cgroupMemoryLimit ?? 0, cgroupMaxMemoryUsage ?? 0);
if (maxMemoryUsage != 0 && maxMemoryUsage <= totalPhysicalMemoryInBytes)
{
// running in a limited cgroup
var commitedMemoryInBytes = 0L;
var cgroupMemoryUsage = LowMemoryNotification.Instance.UseTotalDirtyMemInsteadOfMemUsage // RDBS-45
? fromProcMemInfo.TotalDirty.GetValue(SizeUnit.Bytes)
: KernelVirtualFileSystemUtils.ReadNumberFromCgroupFile(CgroupMemoryUsage);
if (cgroupMemoryUsage != null)
{
commitedMemoryInBytes = cgroupMemoryUsage.Value;
fromProcMemInfo.Commited.Set(commitedMemoryInBytes, SizeUnit.Bytes);
fromProcMemInfo.AvailableMemory.Set(maxMemoryUsage - cgroupMemoryUsage.Value, SizeUnit.Bytes);
var realAvailable = maxMemoryUsage - cgroupMemoryUsage.Value + fromProcMemInfo.SharedCleanMemory.GetValue(SizeUnit.Bytes);
if (realAvailable < 0)
{
realAvailable = 0;
}
fromProcMemInfo.AvailableMemoryForProcessing.Set(realAvailable, SizeUnit.Bytes);
}
fromProcMemInfo.TotalMemory.Set(maxMemoryUsage, SizeUnit.Bytes);
fromProcMemInfo.CommitLimit.Set(Math.Max(maxMemoryUsage, commitedMemoryInBytes), SizeUnit.Bytes);
}
var workingSet = new Size(0, SizeUnit.Bytes);
var swapUsage = new Size(0, SizeUnit.Bytes);
if (smapsReader != null)
{
// extended info is needed
var procStatus = GetMemoryUsageFromProcStatus();
workingSet.Set(procStatus.Rss, SizeUnit.Bytes);
swapUsage.Set(procStatus.Swap, SizeUnit.Bytes);
}
SetMemoryRecords(fromProcMemInfo.AvailableMemoryForProcessing.GetValue(SizeUnit.Bytes));
return(new MemoryInfoResult
{
TotalCommittableMemory = fromProcMemInfo.CommitLimit,
CurrentCommitCharge = fromProcMemInfo.Commited,
AvailableMemory = fromProcMemInfo.AvailableMemory,
AvailableMemoryForProcessing = fromProcMemInfo.AvailableMemoryForProcessing,
SharedCleanMemory = fromProcMemInfo.SharedCleanMemory,
TotalPhysicalMemory = fromProcMemInfo.TotalMemory,
InstalledMemory = fromProcMemInfo.TotalMemory,
WorkingSet = workingSet,
TotalSwapSize = fromProcMemInfo.TotalSwap,
TotalSwapUsage = swapUsage,
WorkingSetSwapUsage = fromProcMemInfo.WorkingSetSwap,
IsExtended = extended
});
}
internal int CheckMemoryStatus(SmapsReader smapsReader)
{
int timeout;
bool isLowMemory;
long totalUnmanagedAllocations;
(Size AvailableMemory, Size TotalPhysicalMemory, Size CurrentCommitCharge)stats;
try
{
isLowMemory = GetLowMemory(out totalUnmanagedAllocations, out stats, smapsReader);
}
catch (OutOfMemoryException)
{
isLowMemory = true;
stats = (new Size(), new Size(), new Size());
totalUnmanagedAllocations = -1;
}
if (isLowMemory)
{
if (LowMemoryState == false)
{
try
{
if (_logger.IsInfoEnabled)
{
_logger.Info("Low memory detected, will try to reduce memory usage...");
}
AddLowMemEvent(LowMemReason.LowMemOnTimeoutChk,
stats.AvailableMemory.GetValue(SizeUnit.Bytes),
totalUnmanagedAllocations,
stats.TotalPhysicalMemory.GetValue(SizeUnit.Bytes),
stats.CurrentCommitCharge.GetValue(SizeUnit.Bytes));
}
catch (OutOfMemoryException)
{
// nothing we can do, we'll wait and try again
}
}
LowMemoryState = true;
_clearInactiveHandlersCounter = 0;
RunLowMemoryHandlers(true);
timeout = 500;
}
else
{
if (LowMemoryState)
{
if (_logger.IsInfoEnabled)
{
_logger.Info("Back to normal memory usage detected");
}
AddLowMemEvent(LowMemReason.BackToNormal,
stats.AvailableMemory.GetValue(SizeUnit.Bytes),
totalUnmanagedAllocations,
stats.TotalPhysicalMemory.GetValue(SizeUnit.Bytes),
stats.CurrentCommitCharge.GetValue(SizeUnit.Bytes));
}
LowMemoryState = false;
RunLowMemoryHandlers(false);
timeout = stats.AvailableMemory < _lowMemoryThreshold * 2 ? 1000 : 5000;
}
return(timeout);
}
private void MonitorMemoryUsage()
{
SmapsReader smapsReader = PlatformDetails.RunningOnLinux ? new SmapsReader(new[] { new byte[SmapsReader.BufferSize], new byte[SmapsReader.BufferSize] }) : null;
NativeMemory.EnsureRegistered();
var memoryAvailableHandles = new WaitHandle[] { _simulatedLowMemory, _shutdownRequested };
var timeout = 5 * 1000;
try
{
while (true)
{
try
{
var result = WaitHandle.WaitAny(memoryAvailableHandles, timeout);
switch (result)
{
case WaitHandle.WaitTimeout:
timeout = CheckMemoryStatus(smapsReader);
break;
case 0:
SimulateLowMemory();
break;
case 1: // shutdown requested
return;
default:
return;
}
}
catch (OutOfMemoryException)
{
try
{
if (_logger.IsInfoEnabled)
{
_logger.Info("Out of memory error in the low memory notification thread, will wait 5 seconds before trying to check memory status again. The system is likely running out of memory");
}
}
catch
{
// can't even log, nothing we can do here
}
if (_shutdownRequested.WaitOne(5000))
{
return; // shutdown requested
}
}
}
}
catch (Exception e)
{
if (_logger.IsOperationsEnabled)
{
_logger.Operations("Catastrophic failure in low memory notification", e);
}
}
}
public void Can_check_memory_status()
{
SmapsReader smapsReader = PlatformDetails.RunningOnLinux ? new SmapsReader(new[] { new byte[SmapsReader.BufferSize], new byte[SmapsReader.BufferSize] }) : null;
LowMemoryNotification.Instance.CheckMemoryStatus(smapsReader);
}
private LowMemoryMonitor()
{
_smapsReader = PlatformDetails.RunningOnLinux ? new SmapsReader(new[] { new byte[SmapsReader.BufferSize], new byte[SmapsReader.BufferSize] }) : null;
}
private static bool GetFromProcMemInfo(SmapsReader smapsReader, ref ProcMemInfoResults procMemInfoResults)
{
const string path = "/proc/meminfo";
// this is different then sysinfo freeram+buffered (and the closest to the real free memory)
// MemFree is really different then MemAvailable (while free is usually lower then the real free,
// and available is only estimated free which sometimes higher then the real free memory)
// for some distros we have only MemFree
try
{
using (var bufferedReader = new KernelVirtualFileSystemUtils.BufferedPosixKeyValueOutputValueReader(path))
{
bufferedReader.ReadFileIntoBuffer();
var memAvailableInKb = bufferedReader.ExtractNumericValueFromKeyValuePairsFormattedFile(MemAvailable);
var memFreeInKb = bufferedReader.ExtractNumericValueFromKeyValuePairsFormattedFile(MemFree);
var totalMemInKb = bufferedReader.ExtractNumericValueFromKeyValuePairsFormattedFile(MemTotal);
var swapTotalInKb = bufferedReader.ExtractNumericValueFromKeyValuePairsFormattedFile(SwapTotal);
var commitedInKb = bufferedReader.ExtractNumericValueFromKeyValuePairsFormattedFile(Committed_AS);
var totalClean = new Size(0, SizeUnit.Kilobytes);
var totalDirty = new Size(0, SizeUnit.Bytes);
var sharedCleanMemory = new Size(0, SizeUnit.Bytes);
var swapWorkingSet = new Size(0, SizeUnit.Bytes);
if (smapsReader != null)
{
var result = smapsReader.CalculateMemUsageFromSmaps <SmapsReaderNoAllocResults>();
totalClean.Add(result.SharedClean, SizeUnit.Bytes);
totalClean.Add(result.PrivateClean, SizeUnit.Bytes);
sharedCleanMemory.Set(result.SharedClean, SizeUnit.Bytes);
totalDirty.Add(result.TotalDirty, SizeUnit.Bytes);
swapWorkingSet.Add(result.Swap, SizeUnit.Bytes);
}
procMemInfoResults.AvailableMemory = new Size(memFreeInKb, SizeUnit.Kilobytes);
procMemInfoResults.TotalMemory = new Size(totalMemInKb, SizeUnit.Kilobytes);
procMemInfoResults.Commited = new Size(commitedInKb, SizeUnit.Kilobytes);
// on Linux, we use the swap + ram as the commit limit, because the actual limit
// is dependent on many different factors
procMemInfoResults.CommitLimit = new Size(totalMemInKb + swapTotalInKb, SizeUnit.Kilobytes);
// AvailableMemoryForProcessing: AvailableMemory actually does add reclaimable memory (divided by 2), so if AvailableMemory is equal or lower then the _real_ available memory
// If it is lower the the real value because of RavenDB's Clean memory - then we use 'totalClean' as reference
// Otherwise - either it is correct value, or it is lower because of (clean or dirty memory of) another process
var availableMemoryForProcessing = new Size(Math.Max(memAvailableInKb, memFreeInKb), SizeUnit.Kilobytes);
procMemInfoResults.AvailableMemoryForProcessing = Size.Max(availableMemoryForProcessing, totalClean);
procMemInfoResults.SharedCleanMemory = sharedCleanMemory;
procMemInfoResults.TotalDirty = totalDirty;
procMemInfoResults.TotalSwap = new Size(swapTotalInKb, SizeUnit.Kilobytes);
procMemInfoResults.WorkingSetSwap = swapWorkingSet;
}
}
catch (Exception ex)
{
if (Logger.IsInfoEnabled)
{
Logger.Info($"Failed to read value from {path}", ex);
}
return(false);
}
return(true);
}
