private PagerState CreatePagerState()
{
var fileSize = GetFileSize();
var startingBaseAddressPtr = Syscall.mmap(IntPtr.Zero, (ulong)fileSize,
MmapProts.PROT_READ | MmapProts.PROT_WRITE,
MmapFlags.MAP_SHARED, _fd, 0);
if (startingBaseAddressPtr.ToInt64() == -1) //system didn't succeed in mapping the address where we wanted
{
PosixHelper.ThrowLastError(Marshal.GetLastWin32Error());
}
var allocationInfo = new PagerState.AllocationInfo
{
BaseAddress = (byte *)startingBaseAddressPtr.ToPointer(),
Size = fileSize,
MappedFile = null
};
var newPager = new PagerState(this)
{
Files = null, // unused
MapBase = allocationInfo.BaseAddress,
AllocationInfos = new[] { allocationInfo }
};
newPager.AddRef(); // one for the pager
return(newPager);
}
private PagerState CreatePagerState()
{
var fileSize = GetFileSize();
var mmflags = _copyOnWriteMode ? MmapFlags.MAP_PRIVATE : MmapFlags.MAP_SHARED;
var startingBaseAddressPtr = Syscall.mmap64(IntPtr.Zero, (UIntPtr)fileSize,
MmapProts.PROT_READ | MmapProts.PROT_WRITE,
mmflags, _fd, 0L);
if (startingBaseAddressPtr.ToInt64() == -1) //system didn't succeed in mapping the address where we wanted
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, "mmap on " + FileName);
}
NativeMemory.RegisterFileMapping(FileName.FullPath, startingBaseAddressPtr, fileSize, GetAllocatedInBytes);
var allocationInfo = new PagerState.AllocationInfo
{
BaseAddress = (byte *)startingBaseAddressPtr.ToPointer(),
Size = fileSize,
MappedFile = null
};
return(new PagerState(this, Options.PrefetchSegmentSize, Options.PrefetchResetThreshold, allocationInfo));
}
private PagerState CreatePagerState()
{
var startingBaseAddressPtr = Syscall.mmap(IntPtr.Zero, (UIntPtr)_totalAllocationSize,
MmapProts.PROT_READ | MmapProts.PROT_WRITE,
MmapFlags.MAP_SHARED, _fd, IntPtr.Zero);
if (startingBaseAddressPtr.ToInt64() == -1) //system didn't succeed in mapping the address where we wanted
{
var err = Marshal.GetLastWin32Error();
PosixHelper.ThrowLastError(err, "mmap on " + FileName);
}
NativeMemory.RegisterFileMapping(FileName, startingBaseAddressPtr, _totalAllocationSize);
var allocationInfo = new PagerState.AllocationInfo
{
BaseAddress = (byte *)startingBaseAddressPtr.ToPointer(),
Size = _totalAllocationSize,
MappedFile = null
};
var newPager = new PagerState(this)
{
Files = null, // unused
MapBase = allocationInfo.BaseAddress,
AllocationInfos = new[] { allocationInfo }
};
return(newPager);
}
private PagerState CreatePagerState()
{
var mmf = MemoryMappedFile.CreateFromFile(_fileStream, null, _fileStream.Length,
_memoryMappedFileAccess,
HandleInheritability.None, true);
var fileMappingHandle = mmf.SafeMemoryMappedFileHandle.DangerousGetHandle();
Win32MemoryMapNativeMethods.NativeFileMapAccessType mmFileAccessType;
if (_copyOnWriteMode)
{
mmFileAccessType = Win32MemoryMapNativeMethods.NativeFileMapAccessType.Copy;
}
else
{
mmFileAccessType = _access == Win32NativeFileAccess.GenericRead
? Win32MemoryMapNativeMethods.NativeFileMapAccessType.Read
: Win32MemoryMapNativeMethods.NativeFileMapAccessType.Read |
Win32MemoryMapNativeMethods.NativeFileMapAccessType.Write;
}
var startingBaseAddressPtr = Win32MemoryMapNativeMethods.MapViewOfFileEx(fileMappingHandle,
mmFileAccessType,
0, 0,
UIntPtr.Zero, //map all what was "reserved" in CreateFileMapping on previous row
null);
if (startingBaseAddressPtr == (byte *)0) //system didn't succeed in mapping the address where we wanted
{
var innerException = new Win32Exception(Marshal.GetLastWin32Error(), "Failed to MapView of file " + FileName);
var errorMessage = string.Format(
"Unable to allocate more pages - unsuccessfully tried to allocate continuous block of virtual memory with size = {0:##,###;;0} bytes",
(_fileStream.Length));
throw new OutOfMemoryException(errorMessage, innerException);
}
NativeMemory.RegisterFileMapping(_fileInfo.FullName, new IntPtr(startingBaseAddressPtr), _fileStream.Length);
// If we are working on memory validation mode, then protect the pages by default.
ProtectPageRange(startingBaseAddressPtr, (ulong)_fileStream.Length);
var allocationInfo = new PagerState.AllocationInfo
{
BaseAddress = startingBaseAddressPtr,
Size = _fileStream.Length,
MappedFile = mmf
};
var newPager = new PagerState(this)
{
Files = new[] { mmf },
MapBase = startingBaseAddressPtr,
AllocationInfos = new[] { allocationInfo }
};
return(newPager);
}
private PagerState CreatePagerState()
{
var mmf = MemoryMappedFile.CreateFromFile(_fileStream, null, _fileStream.Length,
_memoryMappedFileAccess,
HandleInheritability.None, true);
var fileMappingHandle = mmf.SafeMemoryMappedFileHandle.DangerousGetHandle();
var mmFileAccessType = _access == Win32NativeFileAccess.GenericRead
? Win32MemoryMapNativeMethods.NativeFileMapAccessType.Read
: Win32MemoryMapNativeMethods.NativeFileMapAccessType.Read | Win32MemoryMapNativeMethods.NativeFileMapAccessType.Write;
var startingBaseAddressPtr = Win32MemoryMapNativeMethods.MapViewOfFileEx(fileMappingHandle,
mmFileAccessType,
0, 0,
UIntPtr.Zero, //map all what was "reserved" in CreateFileMapping on previous row
null);
if (startingBaseAddressPtr == (byte *)0) //system didn't succeed in mapping the address where we wanted
{
var innerException = new Win32Exception();
var errorMessage = string.Format(
"Unable to allocate more pages - unsuccessfully tried to allocate continuous block of virtual memory with size = {0:##,###;;0} bytes",
(_fileStream.Length));
throw new OutOfMemoryException(errorMessage, innerException);
}
var allocationInfo = new PagerState.AllocationInfo
{
BaseAddress = startingBaseAddressPtr,
Size = _fileStream.Length,
MappedFile = mmf
};
var newPager = new PagerState(this)
{
Files = new[] { mmf },
MapBase = startingBaseAddressPtr,
AllocationInfos = new[] { allocationInfo }
};
newPager.AddRef(); // one for the pager
return(newPager);
}
private PagerState CreatePagerState()
{
_fileStreamLength = _fileStream.Length;
var mmf = MemoryMappedFile.CreateFromFile(_fileStream, null, _fileStreamLength,
_memoryMappedFileAccess,
HandleInheritability.None, true);
var allocation = new PagerState.AllocationInfo
{
MappedFile = mmf,
BaseAddress = null,
Size = 0
};
var newPager = new PagerState(this, Options.PrefetchSegmentSize, Options.PrefetchResetThreshold, allocation);
_hFileMappingObject = mmf.SafeMemoryMappedFileHandle.DangerousGetHandle();
return(newPager);
}
private PagerState CreatePagerState()
{
var mmf = MemoryMappedFile.CreateFromFile(_fileStream, null, _fileStream.Length,
_memoryMappedFileAccess,
null, HandleInheritability.None, true);
var fileMappingHandle = mmf.SafeMemoryMappedFileHandle.DangerousGetHandle();
var mmFileAccessType = _access == NativeFileAccess.GenericRead
? MemoryMapNativeMethods.NativeFileMapAccessType.Read
: MemoryMapNativeMethods.NativeFileMapAccessType.Read | MemoryMapNativeMethods.NativeFileMapAccessType.Write;
var startingBaseAddressPtr = MemoryMapNativeMethods.MapViewOfFileEx(fileMappingHandle,
mmFileAccessType,
0, 0,
UIntPtr.Zero, //map all what was "reserved" in CreateFileMapping on previous row
null);
if (startingBaseAddressPtr == (byte *)0) //system didn't succeed in mapping the address where we wanted
{
throw new Win32Exception();
}
var allocationInfo = new PagerState.AllocationInfo
{
BaseAddress = startingBaseAddressPtr,
Size = _fileStream.Length,
MappedFile = mmf
};
var newPager = new PagerState(this)
{
Files = new[] { mmf },
Accessor = null, //not available since MapViewOfFileEx is used (instead of MapViewOfFile - which is used in managed wrapper)
MapBase = startingBaseAddressPtr,
AllocationInfos = new[] { allocationInfo }
};
newPager.AddRef(); // one for the pager
return(newPager);
}
protected internal override PagerState AllocateMorePages(long newLength)
{
if (DisposeOnceRunner.Disposed)
{
ThrowAlreadyDisposedException();
}
var newLengthAfterAdjustment = NearestSizeToPageSize(newLength);
if (newLengthAfterAdjustment <= _totalAllocationSize)
{
return(null);
}
var rc = rvn_allocate_more_space(newLengthAfterAdjustment, _handle, out var newAddress, out var errorCode);
if (rc != FailCodes.Success)
{
PalHelper.ThrowLastError(rc, errorCode, $"can't allocate more pages (rc={rc}) for '{FileName.FullPath}'. Requested {newLength} (adjusted to {newLengthAfterAdjustment})");
}
// TODO : Get rid of allocation info
var allocationInfo = new PagerState.AllocationInfo
{
BaseAddress = (byte *)newAddress,
Size = newLengthAfterAdjustment,
MappedFile = null
};
var newPagerState = new PagerState(this, Options.PrefetchSegmentSize, Options.PrefetchResetThreshold, allocationInfo);
newPagerState.DebugVerify(newLengthAfterAdjustment);
newPagerState.CopyPrefetchState(_pagerState);
SetPagerState(newPagerState);
_totalAllocationSize = newLengthAfterAdjustment;
NumberOfAllocatedPages = _totalAllocationSize / Constants.Storage.PageSize;
return(newPagerState);
}
private PagerState CreateInitialPagerState(long size, byte *requestedBaseAddress)
{
var allocationSize = NearestSizeToAllocationGranularity(size);
var mmf = MemoryMappedFile.CreateNew(null, allocationSize, MemoryMappedFileAccess.ReadWrite);
var fileMappingHandle = mmf.SafeMemoryMappedFileHandle.DangerousGetHandle();
var startingBaseAddressPtr = MemoryMapNativeMethods.MapViewOfFileEx(fileMappingHandle,
MemoryMapNativeMethods.NativeFileMapAccessType.Read | MemoryMapNativeMethods.NativeFileMapAccessType.Write,
0, 0,
UIntPtr.Zero, //map all what was "reserved" in CreateFileMapping on previous row
requestedBaseAddress);
if (startingBaseAddressPtr == (byte *)0) //system didn't succeed in mapping the address where we wanted
{
throw new Win32Exception();
}
var allocationInfo = new PagerState.AllocationInfo
{
BaseAddress = startingBaseAddressPtr,
Size = allocationSize,
MappedFile = mmf
};
var newPager = new PagerState(this)
{
Files = new[] { mmf },
Accessor = null, //not available since MapViewOfFileEx is used (instead of MapViewOfFile - which is used in managed wrapper)
MapBase = startingBaseAddressPtr,
AllocationInfos = new[] { allocationInfo }
};
newPager.AddRef();
return(newPager);
}
private PagerState CreateInitialPagerState(long size, byte* requestedBaseAddress)
{
var allocationSize = NearestSizeToAllocationGranularity(size);
var mmf = MemoryMappedFile.CreateNew(null, allocationSize, MemoryMappedFileAccess.ReadWrite);
var fileMappingHandle = mmf.SafeMemoryMappedFileHandle.DangerousGetHandle();
var startingBaseAddressPtr = MemoryMapNativeMethods.MapViewOfFileEx(fileMappingHandle,
MemoryMapNativeMethods.NativeFileMapAccessType.Read | MemoryMapNativeMethods.NativeFileMapAccessType.Write,
0, 0,
UIntPtr.Zero, //map all what was "reserved" in CreateFileMapping on previous row
requestedBaseAddress);
if (startingBaseAddressPtr == (byte*)0) //system didn't succeed in mapping the address where we wanted
throw new Win32Exception();
var allocationInfo = new PagerState.AllocationInfo
{
BaseAddress = startingBaseAddressPtr,
Size = allocationSize,
MappedFile = mmf
};
var newPager = new PagerState(this)
{
Files = new[] { mmf },
Accessor = null, //not available since MapViewOfFileEx is used (instead of MapViewOfFile - which is used in managed wrapper)
MapBase = startingBaseAddressPtr,
AllocationInfos = new[] { allocationInfo }
};
newPager.AddRef();
return newPager;
}
private PagerState CreatePagerState()
{
var mmf = MemoryMappedFile.CreateFromFile(_fileStream, null, _fileStream.Length,
_memoryMappedFileAccess,
null, HandleInheritability.None, true);
var fileMappingHandle = mmf.SafeMemoryMappedFileHandle.DangerousGetHandle();
var mmFileAccessType = _access == NativeFileAccess.GenericRead
? MemoryMapNativeMethods.NativeFileMapAccessType.Read
: MemoryMapNativeMethods.NativeFileMapAccessType.Read | MemoryMapNativeMethods.NativeFileMapAccessType.Write;
var startingBaseAddressPtr = MemoryMapNativeMethods.MapViewOfFileEx(fileMappingHandle,
mmFileAccessType,
0, 0,
UIntPtr.Zero, //map all what was "reserved" in CreateFileMapping on previous row
null);
if (startingBaseAddressPtr == (byte*)0) //system didn't succeed in mapping the address where we wanted
throw new Win32Exception();
var allocationInfo = new PagerState.AllocationInfo
{
BaseAddress = startingBaseAddressPtr,
Size = _fileStream.Length,
MappedFile = mmf
};
var newPager = new PagerState(this)
{
Files = new[] { mmf },
Accessor = null, //not available since MapViewOfFileEx is used (instead of MapViewOfFile - which is used in managed wrapper)
MapBase = startingBaseAddressPtr,
AllocationInfos = new[] { allocationInfo }
};
newPager.AddRef(); // one for the pager
return newPager;
}
private bool TryHandleFailureToLockMemory(PagerState newState, PagerState.AllocationInfo info)
{
var currentProcess = Process.GetCurrentProcess();
var sum = Bits.NextPowerOf2(newState.AllocationInfos.Sum(x => x.Size) * 2);
if (PlatformDetails.RunningOnPosix == false)
{
// From: https://msdn.microsoft.com/en-us/library/windows/desktop/ms686234(v=vs.85).aspx
// "The maximum number of pages that a process can lock is equal to the number of pages in its minimum working set minus a small overhead"
// let's increase the max size of memory we can lock by increasing the MinWorkingSet. On Windows, that is available for all users
var nextSize = Bits.NextPowerOf2(currentProcess.MinWorkingSet.ToInt64() + sum);
if (nextSize > int.MaxValue && IntPtr.Size == sizeof(int))
{
nextSize = int.MaxValue;
}
// Minimum working set size must be less than or equal to the maximum working set size.
// Let's increase the max as well.
if (nextSize > (long)currentProcess.MaxWorkingSet)
{
try
{
currentProcess.MaxWorkingSet = new IntPtr(nextSize);
}
catch (Exception e)
{
throw new InsufficientMemoryException($"Need to increase the min working set size from {(long)currentProcess.MinWorkingSet:#,#;;0} to {nextSize:#,#;;0} but the max working set size was too small: {(long)currentProcess.MaxWorkingSet:#,#;;0}. " +
$"Failed to increase the max working set size so we can lock {info.Size:#,#;;0} for {FileName}. With encrypted " +
"databases we lock some memory in order to avoid leaking secrets to disk. Treating this as a catastrophic error " +
"and aborting the current operation.", e);
}
}
try
{
currentProcess.MinWorkingSet = new IntPtr(nextSize);
}
catch (Exception e)
{
throw new InsufficientMemoryException($"Failed to increase the min working set size so we can lock {info.Size:#,#;;0} for {FileName}. With encrypted " +
"databases we lock some memory in order to avoid leaking secrets to disk. Treating this as a catastrophic error " +
"and aborting the current operation.", e);
}
// now we can try again, after we raised the limit, we only do so once, though
if (Sodium.sodium_mlock(info.BaseAddress, (UIntPtr)info.Size) == 0)
{
return(false);
}
}
var msg =
$"Unable to lock memory for {FileName} with size {info.Size:#,#;;0}), with encrypted databases we lock some memory in order to avoid leaking secrets to disk. Treating this as a catastrophic error and aborting the current operation.{Environment.NewLine}";
if (PlatformDetails.RunningOnPosix)
{
msg +=
$"The admin may configure higher limits using: 'sudo prlimit --pid {currentProcess.Id} --memlock={sum}' to increase the limit. (It's recommended to do that as part of the startup script){Environment.NewLine}";
}
else
{
msg +=
$"Already tried to raise the the process min working set to {currentProcess.MinWorkingSet.ToInt64():#,#;;0} but still got a failure.{Environment.NewLine}";
}
msg += "This behavior is controlled by the 'Security.DoNotConsiderMemoryLockFailureAsCatastrophicError' setting (expert only, modifications of this setting is not recommended).";
throw new InsufficientMemoryException(msg);
}
public RvnMemoryMapPager(StorageEnvironmentOptions options, VoronPathSetting file, long?initialFileSize = null, bool canPrefetchAhead = true, bool usePageProtection = false, bool deleteOnClose = false)
: base(options, canPrefetchAhead, usePageProtection)
{
DeleteOnClose = deleteOnClose;
FileName = file;
var copyOnWriteMode = options.CopyOnWriteMode && FileName.FullPath.EndsWith(Constants.DatabaseFilename);
_logger = LoggingSource.Instance.GetLogger <StorageEnvironment>($"Pager-{file}");
if (initialFileSize.HasValue == false || initialFileSize.Value == 0)
{
initialFileSize = Math.Max(SysInfo.PageSize * 16, 64 * 1024);
}
if (initialFileSize % SysInfo.PageSize != 0)
{
initialFileSize += SysInfo.PageSize - initialFileSize % SysInfo.PageSize;
}
Debug.Assert(file != null);
var mmapOptions = copyOnWriteMode ? MmapOptions.CopyOnWrite : MmapOptions.None;
if (DeleteOnClose)
{
mmapOptions |= MmapOptions.DeleteOnClose;
}
var rc = rvn_create_and_mmap64_file(
file.FullPath,
initialFileSize.Value,
mmapOptions,
out _handle,
out var baseAddress,
out _totalAllocationSize,
out var errorCode);
if (rc != FailCodes.Success)
{
try
{
PalHelper.ThrowLastError(rc, errorCode, $"rvn_create_and_mmap64_file failed on {rc} for '{file.FullPath}'");
}
catch (DiskFullException dfEx)
{
var diskSpaceResult = DiskSpaceChecker.GetDiskSpaceInfo(file.FullPath);
throw new DiskFullException(file.FullPath, initialFileSize.Value, diskSpaceResult?.TotalFreeSpace.GetValue(SizeUnit.Bytes), dfEx.Message);
}
}
NumberOfAllocatedPages = _totalAllocationSize / Constants.Storage.PageSize;
NativeMemory.RegisterFileMapping(FileName.FullPath, new IntPtr(baseAddress), _totalAllocationSize, GetAllocatedInBytes);
var allocationInfo = new PagerState.AllocationInfo
{
BaseAddress = (byte *)baseAddress,
Size = _totalAllocationSize,
MappedFile = null
};
var pager = new PagerState(this, Options.PrefetchSegmentSize, Options.PrefetchResetThreshold, allocationInfo);
SetPagerState(pager);
}