private static void MemoryStatisticsForPosix()
{
int clearInactiveHandlersCounter = 0;
new Thread(() =>
{
RavenConfiguration configuration = new RavenConfiguration();
while (true)
{
int waitRC = // poll each 5 seconds
WaitHandle.WaitAny(new[] { StopPosixLowMemThreadEvent, LowPosixMemorySimulationEvent }, TimeSpan.FromSeconds(5));
switch (waitRC)
{
case 0: // stopLowMemThreadEvent
if (Log.IsDebugEnabled)
{
Log.Debug("MemoryStatisticsForPosix : stopLowMemThreadEvent triggered");
}
return;
case 1: // lowMemorySimulationEvent
LowPosixMemorySimulationEvent.Reset();
Log.Warn("Low memory simulation, will try to reduce memory usage...");
RunLowMemoryHandlers("System detected low memory");
break;
case WaitHandle.WaitTimeout: // poll available mem
LowPosixMemorySimulationEvent.Reset();
if (++clearInactiveHandlersCounter > 60) // 5 minutes == WaitAny 5 Secs * 60
{
clearInactiveHandlersCounter = 0;
ClearInactiveHandlers();
break;
}
sysinfo_t info = new sysinfo_t();
if (Syscall.sysinfo(ref info) != 0)
{
Log.Warn("Failure when trying to wait for low memory notification. No low memory notifications will be raised.");
}
else
{
ulong availableMem = info.AvailableRam / (1024L * 1024);
if (availableMem < (ulong)configuration.LowMemoryForLinuxDetectionInMB)
{
clearInactiveHandlersCounter = 0;
Log.Warn("Low memory detected, will try to reduce memory usage...");
RunLowMemoryHandlers("System detected low memory manually");
Thread.Sleep(TimeSpan.FromSeconds(60)); // prevent triggering the event to frequent when the low memory notification object is in the signaled state
}
}
break;
}
}
})
{
IsBackground = true,
Name = "Low Posix memory notification thread"
}.Start();
}
public static unsafe MemoryInfoResult GetMemoryInfo()
{
if (_failedToGetAvailablePhysicalMemory)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Because of a previous error in getting available memory, we are now lying and saying we have 256MB free");
}
return(FailedResult);
}
try
{
if (PlatformDetails.RunningOnPosix == false)
{
// Windows
var memoryStatus = new MemoryStatusEx
{
dwLength = (uint)sizeof(MemoryStatusEx)
};
var result = GlobalMemoryStatusEx(&memoryStatus);
if (result == false)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Failure when trying to read memory info from Windows, error code is: " + Marshal.GetLastWin32Error());
}
return(FailedResult);
}
return(new MemoryInfoResult
{
AvailableMemory = new Size((long)memoryStatus.ullAvailPhys, SizeUnit.Bytes),
TotalPhysicalMemory = new Size((long)memoryStatus.ullTotalPhys, SizeUnit.Bytes),
});
}
// read both cgroup and sysinfo memory stats, and use the lowest if applicable
var cgroupMemoryLimit = KernelVirtualFileSystemUtils.ReadNumberFromCgroupFile("/sys/fs/cgroup/memory/memory.limit_in_bytes");
var cgroupMemoryUsage = KernelVirtualFileSystemUtils.ReadNumberFromCgroupFile("/sys/fs/cgroup/memory/memory.usage_in_bytes");
ulong availableRamInBytes;
ulong totalPhysicalMemoryInBytes;
if (PlatformDetails.RunningOnMacOsx == false)
{
// Linux
var info = new sysinfo_t();
if (Syscall.sysinfo(ref info) != 0)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Failure when trying to read memory info from posix, error code was: " + Marshal.GetLastWin32Error());
}
return(FailedResult);
}
availableRamInBytes = info.AvailableRam;
totalPhysicalMemoryInBytes = info.TotalRam;
}
else
{
// MacOS
var mib = new[] { (int)TopLevelIdentifiersMacOs.CTL_HW, (int)CtkHwIdentifiersMacOs.HW_MEMSIZE };
ulong physicalMemory = 0;
var len = sizeof(ulong);
if (Syscall.sysctl(mib, 2, &physicalMemory, &len, null, UIntPtr.Zero) != 0)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Failure when trying to read physical memory info from MacOS, error code was: " + Marshal.GetLastWin32Error());
}
return(FailedResult);
}
totalPhysicalMemoryInBytes = physicalMemory;
uint pageSize;
var vmStats = new vm_statistics64();
var machPort = Syscall.mach_host_self();
var count = sizeof(vm_statistics64) / sizeof(uint);
if (Syscall.host_page_size(machPort, &pageSize) != 0 ||
Syscall.host_statistics64(machPort, (int)FlavorMacOs.HOST_VM_INFO64, &vmStats, &count) != 0)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Failure when trying to get vm_stats from MacOS, error code was: " + Marshal.GetLastWin32Error());
}
return(FailedResult);
}
availableRamInBytes = vmStats.FreePagesCount * (ulong)pageSize;
}
Size availableRam, totalPhysicalMemory;
if (cgroupMemoryLimit < (long)totalPhysicalMemoryInBytes)
{
availableRam = new Size(cgroupMemoryLimit - cgroupMemoryUsage, SizeUnit.Bytes);
totalPhysicalMemory = new Size(cgroupMemoryLimit, SizeUnit.Bytes);
}
else
{
availableRam = new Size((long)availableRamInBytes, SizeUnit.Bytes);
totalPhysicalMemory = new Size((long)totalPhysicalMemoryInBytes, SizeUnit.Bytes);
}
return(new MemoryInfoResult
{
AvailableMemory = availableRam,
TotalPhysicalMemory = totalPhysicalMemory
});
}
catch (Exception e)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Error while trying to get available memory, will stop trying and report that there is 256MB free only from now on", e);
}
_failedToGetAvailablePhysicalMemory = true;
return(FailedResult);
}
}
public static unsafe MemoryInfoResult GetMemoryInfo()
{
if (failedToGetAvailablePhysicalMemory)
{
if (_logger.IsInfoEnabled)
{
_logger.Info("Because of a previous error in getting available memory, we are now lying and saying we have 256MB free");
}
return(failedResult);
}
try
{
if (StorageEnvironmentOptions.RunningOnPosix)
{
sysinfo_t info = new sysinfo_t();
if (Syscall.sysinfo(ref info) != 0)
{
if (_logger.IsInfoEnabled)
{
_logger.Info("Failure when trying to read memory info from posix, error code was: " + Marshal.GetLastWin32Error());
}
return(failedResult);
}
return(new MemoryInfoResult
{
AvailableMemory = new Size((long)info.AvailableRam, SizeUnit.Bytes),
TotalPhysicalMemory = new Size((long)info.TotalRam, SizeUnit.Bytes),
});
}
var memoryStatus = new MemoryStatusEx
{
dwLength = (uint)sizeof(MemoryStatusEx)
};
var result = GlobalMemoryStatusEx(&memoryStatus);
if (result == false)
{
if (_logger.IsInfoEnabled)
{
_logger.Info("Failure when trying to read memory info from Windows, error code is: " + Marshal.GetLastWin32Error());
}
return(failedResult);
}
return(new MemoryInfoResult
{
AvailableMemory = new Size((long)memoryStatus.ullAvailPhys, SizeUnit.Bytes),
TotalPhysicalMemory = new Size((long)memoryStatus.ullTotalPhys, SizeUnit.Bytes),
});
}
catch (Exception e)
{
if (_logger.IsInfoEnabled)
{
_logger.Info("Error while trying to get available memory, will stop trying and report that there is 256MB free only from now on", e);
}
failedToGetAvailablePhysicalMemory = true;
return(failedResult);
}
}
public static extern int sysinfo(ref sysinfo_t info);
public static unsafe MemoryInfoResult GetMemoryInfo()
{
if (_failedToGetAvailablePhysicalMemory)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Because of a previous error in getting available memory, we are now lying and saying we have 256MB free");
}
return(FailedResult);
}
try
{
if (PlatformDetails.RunningOnPosix == false)
{
// windows
var memoryStatus = new MemoryStatusEx
{
dwLength = (uint)sizeof(MemoryStatusEx)
};
if (GlobalMemoryStatusEx(&memoryStatus) == false)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Failure when trying to read memory info from Windows, error code is: " + Marshal.GetLastWin32Error());
}
return(FailedResult);
}
// The amount of physical memory retrieved by the GetPhysicallyInstalledSystemMemory function
// must be equal to or greater than the amount reported by the GlobalMemoryStatusEx function
// if it is less, the SMBIOS data is malformed and the function fails with ERROR_INVALID_DATA.
// Malformed SMBIOS data may indicate a problem with the user's computer.
var fetchedInstalledMemory = GetPhysicallyInstalledSystemMemory(out var installedMemoryInKb);
SetMemoryRecords((long)memoryStatus.ullAvailPhys);
return(new MemoryInfoResult
{
TotalCommittableMemory = new Size((long)memoryStatus.ullTotalPageFile, SizeUnit.Bytes),
CurrentCommitCharge = new Size((long)(memoryStatus.ullTotalPageFile - memoryStatus.ullAvailPageFile), SizeUnit.Bytes),
AvailableMemory = new Size((long)memoryStatus.ullAvailPhys, SizeUnit.Bytes),
TotalPhysicalMemory = new Size((long)memoryStatus.ullTotalPhys, SizeUnit.Bytes),
InstalledMemory = fetchedInstalledMemory ?
new Size(installedMemoryInKb, SizeUnit.Kilobytes) :
new Size((long)memoryStatus.ullTotalPhys, SizeUnit.Bytes),
MemoryUsageRecords = new MemoryInfoResult.MemoryUsageLowHigh
{
High = new MemoryInfoResult.MemoryUsageIntervals
{
LastOneMinute = new Size(HighLastOneMinute, SizeUnit.Bytes),
LastFiveMinutes = new Size(HighLastFiveMinutes, SizeUnit.Bytes),
SinceStartup = new Size(HighSinceStartup, SizeUnit.Bytes)
},
Low = new MemoryInfoResult.MemoryUsageIntervals
{
LastOneMinute = new Size(LowLastOneMinute, SizeUnit.Bytes),
LastFiveMinutes = new Size(LowLastFiveMinutes, SizeUnit.Bytes),
SinceStartup = new Size(LowSinceStartup, SizeUnit.Bytes)
}
}
});
}
// read both cgroup and sysinfo memory stats, and use the lowest if applicable
var cgroupMemoryLimit = KernelVirtualFileSystemUtils.ReadNumberFromCgroupFile("/sys/fs/cgroup/memory/memory.limit_in_bytes");
var cgroupMemoryUsage = KernelVirtualFileSystemUtils.ReadNumberFromCgroupFile("/sys/fs/cgroup/memory/memory.usage_in_bytes");
ulong availableRamInBytes;
ulong totalPhysicalMemoryInBytes;
if (PlatformDetails.RunningOnMacOsx == false)
{
// linux
int rc = 0;
ulong totalram = 0;
if (PlatformDetails.Is32Bits == false)
{
var info = new sysinfo_t();
rc = Syscall.sysinfo(ref info);
totalram = info.TotalRam;
}
else
{
var info = new sysinfo_t_32bit();
rc = Syscall.sysinfo(ref info);
totalram = info.TotalRam;
}
if (rc != 0)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Failure when trying to read memory info from posix, error code was: " + Marshal.GetLastWin32Error());
}
return(FailedResult);
}
var availableAndFree = GetAvailableAndFreeMemoryFromProcMemInfo();
availableRamInBytes = (ulong)availableAndFree.MemAvailable;
totalPhysicalMemoryInBytes = totalram;
}
else
{
// macOS
var mib = new[] { (int)TopLevelIdentifiersMacOs.CTL_HW, (int)CtkHwIdentifiersMacOs.HW_MEMSIZE };
ulong physicalMemory = 0;
var len = sizeof(ulong);
if (Syscall.sysctl(mib, 2, &physicalMemory, &len, null, UIntPtr.Zero) != 0)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Failure when trying to read physical memory info from MacOS, error code was: " + Marshal.GetLastWin32Error());
}
return(FailedResult);
}
totalPhysicalMemoryInBytes = physicalMemory;
uint pageSize;
var vmStats = new vm_statistics64();
var machPort = Syscall.mach_host_self();
var count = sizeof(vm_statistics64) / sizeof(uint);
if (Syscall.host_page_size(machPort, &pageSize) != 0 ||
Syscall.host_statistics64(machPort, (int)FlavorMacOs.HOST_VM_INFO64, &vmStats, &count) != 0)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Failure when trying to get vm_stats from MacOS, error code was: " + Marshal.GetLastWin32Error());
}
return(FailedResult);
}
availableRamInBytes = (vmStats.FreePagesCount + vmStats.InactivePagesCount) * (ulong)pageSize;
}
Size availableRam, totalPhysicalMemory;
if (cgroupMemoryLimit < (long)totalPhysicalMemoryInBytes)
{
availableRam = new Size(cgroupMemoryLimit - cgroupMemoryUsage, SizeUnit.Bytes);
totalPhysicalMemory = new Size(cgroupMemoryLimit, SizeUnit.Bytes);
}
else
{
availableRam = new Size((long)availableRamInBytes, SizeUnit.Bytes);
totalPhysicalMemory = new Size((long)totalPhysicalMemoryInBytes, SizeUnit.Bytes);
}
SetMemoryRecords((long)availableRamInBytes);
var commitAsAndCommitLimit = GetCommitAsAndCommitLimitFromProcMemInfo();
return(new MemoryInfoResult
{
TotalCommittableMemory = new Size(commitAsAndCommitLimit.CommitLimit, SizeUnit.Kilobytes),
CurrentCommitCharge = new Size(commitAsAndCommitLimit.Committed_AS, SizeUnit.Kilobytes),
AvailableMemory = availableRam,
TotalPhysicalMemory = totalPhysicalMemory,
InstalledMemory = totalPhysicalMemory,
MemoryUsageRecords = new MemoryInfoResult.MemoryUsageLowHigh
{
High = new MemoryInfoResult.MemoryUsageIntervals
{
LastOneMinute = new Size(HighLastOneMinute, SizeUnit.Bytes),
LastFiveMinutes = new Size(HighLastFiveMinutes, SizeUnit.Bytes),
SinceStartup = new Size(HighSinceStartup, SizeUnit.Bytes)
},
Low = new MemoryInfoResult.MemoryUsageIntervals
{
LastOneMinute = new Size(LowLastOneMinute, SizeUnit.Bytes),
LastFiveMinutes = new Size(LowLastFiveMinutes, SizeUnit.Bytes),
SinceStartup = new Size(LowSinceStartup, SizeUnit.Bytes)
}
}
});
}
catch (Exception e)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Error while trying to get available memory, will stop trying and report that there is 256MB free only from now on", e);
}
_failedToGetAvailablePhysicalMemory = true;
return(FailedResult);
}
}
public static unsafe MemoryInfoResult GetMemoryInfo()
{
if (_failedToGetAvailablePhysicalMemory)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Because of a previous error in getting available memory, we are now lying and saying we have 256MB free");
}
return(FailedResult);
}
try
{
if (PlatformDetails.RunningOnPosix == false)
{
// windows
var memoryStatus = new MemoryStatusEx
{
dwLength = (uint)sizeof(MemoryStatusEx)
};
if (GlobalMemoryStatusEx(&memoryStatus) == false)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Failure when trying to read memory info from Windows, error code is: " + Marshal.GetLastWin32Error());
}
return(FailedResult);
}
if (GetPhysicallyInstalledSystemMemory(out var installedMemoryInKb) == false)
{
installedMemoryInKb = (long)memoryStatus.ullTotalPhys;
}
SetMemoryRecords((long)memoryStatus.ullAvailPhys);
return(new MemoryInfoResult
{
AvailableMemory = new Size((long)memoryStatus.ullAvailPhys, SizeUnit.Bytes),
TotalPhysicalMemory = new Size((long)memoryStatus.ullTotalPhys, SizeUnit.Bytes),
InstalledMemory = new Size(installedMemoryInKb, SizeUnit.Kilobytes),
MemoryUsageRecords = new MemoryInfoResult.MemoryUsageLowHigh
{
High = new MemoryInfoResult.MemoryUsageIntervals
{
LastOneMinute = new Size(HighLastOneMinute, SizeUnit.Bytes),
LastFiveMinutes = new Size(HighLastFiveMinutes, SizeUnit.Bytes),
SinceStartup = new Size(HighSinceStartup, SizeUnit.Bytes)
},
Low = new MemoryInfoResult.MemoryUsageIntervals
{
LastOneMinute = new Size(LowLastOneMinute, SizeUnit.Bytes),
LastFiveMinutes = new Size(LowLastFiveMinutes, SizeUnit.Bytes),
SinceStartup = new Size(LowSinceStartup, SizeUnit.Bytes)
}
}
});
}
// read both cgroup and sysinfo memory stats, and use the lowest if applicable
var cgroupMemoryLimit = KernelVirtualFileSystemUtils.ReadNumberFromCgroupFile("/sys/fs/cgroup/memory/memory.limit_in_bytes");
var cgroupMemoryUsage = KernelVirtualFileSystemUtils.ReadNumberFromCgroupFile("/sys/fs/cgroup/memory/memory.usage_in_bytes");
ulong availableRamInBytes;
ulong totalPhysicalMemoryInBytes;
if (PlatformDetails.RunningOnMacOsx == false)
{
// linux
var info = new sysinfo_t();
if (Syscall.sysinfo(ref info) != 0)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Failure when trying to read memory info from posix, error code was: " + Marshal.GetLastWin32Error());
}
return(FailedResult);
}
availableRamInBytes = (ulong)GetAvailableMemoryFromProcMemInfo();
totalPhysicalMemoryInBytes = info.TotalRam;
}
else
{
// macOS
var mib = new[] { (int)TopLevelIdentifiersMacOs.CTL_HW, (int)CtkHwIdentifiersMacOs.HW_MEMSIZE };
ulong physicalMemory = 0;
var len = sizeof(ulong);
if (Syscall.sysctl(mib, 2, &physicalMemory, &len, null, UIntPtr.Zero) != 0)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Failure when trying to read physical memory info from MacOS, error code was: " + Marshal.GetLastWin32Error());
}
return(FailedResult);
}
totalPhysicalMemoryInBytes = physicalMemory;
uint pageSize;
var vmStats = new vm_statistics64();
var machPort = Syscall.mach_host_self();
var count = sizeof(vm_statistics64) / sizeof(uint);
if (Syscall.host_page_size(machPort, &pageSize) != 0 ||
Syscall.host_statistics64(machPort, (int)FlavorMacOs.HOST_VM_INFO64, &vmStats, &count) != 0)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Failure when trying to get vm_stats from MacOS, error code was: " + Marshal.GetLastWin32Error());
}
return(FailedResult);
}
availableRamInBytes = (vmStats.FreePagesCount + vmStats.InactivePagesCount) * (ulong)pageSize;
}
Size availableRam, totalPhysicalMemory;
if (cgroupMemoryLimit < (long)totalPhysicalMemoryInBytes)
{
availableRam = new Size(cgroupMemoryLimit - cgroupMemoryUsage, SizeUnit.Bytes);
totalPhysicalMemory = new Size(cgroupMemoryLimit, SizeUnit.Bytes);
}
else
{
availableRam = new Size((long)availableRamInBytes, SizeUnit.Bytes);
totalPhysicalMemory = new Size((long)totalPhysicalMemoryInBytes, SizeUnit.Bytes);
}
SetMemoryRecords((long)availableRamInBytes);
return(new MemoryInfoResult
{
AvailableMemory = availableRam,
TotalPhysicalMemory = totalPhysicalMemory,
//TODO: http://issues.hibernatingrhinos.com/issue/RavenDB-8468
InstalledMemory = totalPhysicalMemory,
MemoryUsageRecords = new MemoryInfoResult.MemoryUsageLowHigh
{
High = new MemoryInfoResult.MemoryUsageIntervals
{
LastOneMinute = new Size(HighLastOneMinute, SizeUnit.Bytes),
LastFiveMinutes = new Size(HighLastFiveMinutes, SizeUnit.Bytes),
SinceStartup = new Size(HighSinceStartup, SizeUnit.Bytes)
},
Low = new MemoryInfoResult.MemoryUsageIntervals
{
LastOneMinute = new Size(LowLastOneMinute, SizeUnit.Bytes),
LastFiveMinutes = new Size(LowLastFiveMinutes, SizeUnit.Bytes),
SinceStartup = new Size(LowSinceStartup, SizeUnit.Bytes)
}
}
});
}
catch (Exception e)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Error while trying to get available memory, will stop trying and report that there is 256MB free only from now on", e);
}
_failedToGetAvailablePhysicalMemory = true;
return(FailedResult);
}
}