protected AbstractPager()
{
_increaseSize = MinIncreaseSize;
MaxNodeSize = 1024;
Debug.Assert((PageSize - Constants.PageHeaderSize) / Constants.MinKeysInPage >= 1024);
PageMinSpace = (int)(PageMaxSpace * 0.33);
PagerState = new PagerState(this);
PagerState.AddRef();
}
public unsafe override byte* AcquirePagePointer(long pageNumber, PagerState pagerState = null)
{
long page = 0;
foreach (var buffer in _buffers)
{
if (page + buffer.SizeInPages > pageNumber)
return buffer.Pointer + ((pageNumber - page)*PageSize);
page += buffer.SizeInPages;
}
throw new InvalidOperationException("Could not find a matching page number: " + pageNumber);
}
public Page Read(long pageNumber, PagerState pagerState = null)
{
ThrowObjectDisposedIfNeeded();
if (pageNumber + 1 > NumberOfAllocatedPages)
{
throw new InvalidOperationException("Cannot get page number " + pageNumber +
" because number of allocated pages is " + NumberOfAllocatedPages);
}
return new Page(AcquirePagePointer(pageNumber, pagerState), _source, PageSize);
}
protected AbstractPager()
{
_increaseSize = MinIncreaseSize;
// MaxNodeSize is usually persisted as an unsigned short. Therefore, we must ensure it is not possible to have an overflow.
Debug.Assert(NodeMaxSize < ushort.MaxValue);
Debug.Assert((PageSize - Constants.PageHeaderSize) / Constants.MinKeysInPage >= 1024);
PageMinSpace = (int)(PageMaxSpace * 0.33);
PagerState = new PagerState(this);
PagerState.AddRef();
}
public PureMemoryPager()
{
var ptr = Marshal.AllocHGlobal(MinIncreaseSize);
var buffer = new Buffer
{
Handle = ptr,
Base = (byte*)ptr.ToPointer(),
Size = MinIncreaseSize
};
_buffers.Append(buffer);
NumberOfAllocatedPages = 0;
PagerState.Release();
PagerState = new PagerState(this);
PagerState.AddRef();
}
public PureMemoryPager(byte[] data)
{
var ptr = Marshal.AllocHGlobal(data.Length);
var buffer = new Buffer
{
Handle = ptr,
Base = (byte*)ptr.ToPointer(),
Size = data.Length
};
_buffers = _buffers.Append(buffer);
NumberOfAllocatedPages = data.Length / PageSize;
PagerState.Release();
PagerState = new PagerState(this);
PagerState.AddRef();
fixed (byte* origin = data)
{
NativeMethods.memcpy(buffer.Base, origin, data.Length);
}
}
public abstract byte *AcquirePagePointer(long pageNumber, PagerState pagerState = null);
private PagerState AllocateMorePagesAndRemapContinuously(long allocationSize)
{
var retryCount = 0;
while (retryCount++ < MaxAllocationRetries)
{
byte *newBaseAddress;
if (TryFindContinuousMemory((ulong)(_totalAllocationSize + allocationSize), out newBaseAddress) == false)
{
var message =
string.Format(
"Unable to allocate more pages - unsuccessfully tried to allocate continuous block of size = {0} bytes\r\n" +
"It is likely that we are suffering from virtual memory exhaustion or memory fragmentation.\r\n" +
"64 bits process: {1}\r\n" +
"If you are running in 32 bits, this is expected, and you need to run in 64 bits to resume normal operations.\r\n" +
"If you are running in 64 bits, this is likely an error and should be reported."
, (_totalAllocationSize + allocationSize), Environment.Is64BitProcess);
throw new OutOfMemoryException(message);
}
bool failedToAllocate = false;
long offset = 0;
var allocationInfoAfterReallocation = new List <PagerState.AllocationInfo>();
foreach (var allocationInfo in PagerState.AllocationInfos)
{
var newAlloctedBaseAddress = Win32MemoryMapNativeMethods.MapViewOfFileEx(allocationInfo.MappedFile.SafeMemoryMappedFileHandle.DangerousGetHandle(),
Win32MemoryMapNativeMethods.NativeFileMapAccessType.Read |
Win32MemoryMapNativeMethods.NativeFileMapAccessType.Write,
0, 0,
UIntPtr.Zero,
newBaseAddress + offset);
if (newAlloctedBaseAddress == null || newAlloctedBaseAddress == (byte *)0)
{
Debug.WriteLine("Failed to remap file continuously. Unmapping already mapped files and re-trying");
UndoMappings(allocationInfoAfterReallocation);
failedToAllocate = true;
break;
}
offset += allocationInfo.Size;
allocationInfoAfterReallocation.Add(new PagerState.AllocationInfo
{
BaseAddress = newAlloctedBaseAddress,
MappedFile = allocationInfo.MappedFile,
Size = allocationInfo.Size
});
}
if (failedToAllocate)
{
continue;
}
var newAllocationInfo = TryCreateNewFileMappingAtAddress(allocationSize, newBaseAddress + _totalAllocationSize);
if (newAllocationInfo == null)
{
UndoMappings(allocationInfoAfterReallocation);
continue;
}
var newPagerState = new PagerState(this)
{
Files = PagerState.Files.Concat(newAllocationInfo.MappedFile),
AllocationInfos = allocationInfoAfterReallocation.Concat(newAllocationInfo),
MapBase = newBaseAddress
};
return(newPagerState);
}
throw new InvalidOperationException(
string.Format(
"Something bad has happened, after {0} tries, could not find any spot in virtual memory to remap continuous virtual memory for {1:##,###;;0} bytes",
MaxAllocationRetries, allocationSize));
}
public override byte *AcquirePagePointerForNewPage(IPagerLevelTransactionState tx, long pageNumber, int numberOfPages, PagerState pagerState = null)
{
return(AcquirePagePointer(tx, pageNumber, pagerState));
}
private bool TryHandleFailureToLockMemory(PagerState newState, PagerState.AllocationInfo info)
{
var currentProcess = Process.GetCurrentProcess();
var sum = Bits.NextPowerOf2(newState.AllocationInfos.Sum(x => x.Size) * 2);
if (PlatformDetails.RunningOnPosix == false)
{
// From: https://msdn.microsoft.com/en-us/library/windows/desktop/ms686234(v=vs.85).aspx
// "The maximum number of pages that a process can lock is equal to the number of pages in its minimum working set minus a small overhead"
// let's increase the max size of memory we can lock by increasing the MinWorkingSet. On Windows, that is available for all users
var nextSize = Bits.NextPowerOf2(currentProcess.MinWorkingSet.ToInt64() + sum);
if (nextSize > int.MaxValue && IntPtr.Size == sizeof(int))
{
nextSize = int.MaxValue;
}
// Minimum working set size must be less than or equal to the maximum working set size.
// Let's increase the max as well.
if (nextSize > (long)currentProcess.MaxWorkingSet)
{
try
{
currentProcess.MaxWorkingSet = new IntPtr(nextSize);
}
catch (Exception e)
{
throw new InsufficientMemoryException($"Need to increase the min working set size from {(long)currentProcess.MinWorkingSet:#,#;;0} to {nextSize:#,#;;0} but the max working set size was too small: {(long)currentProcess.MaxWorkingSet:#,#;;0}. " +
$"Failed to increase the max working set size so we can lock {info.Size:#,#;;0} for {FileName}. With encrypted " +
"databases we lock some memory in order to avoid leaking secrets to disk. Treating this as a catastrophic error " +
"and aborting the current operation.", e);
}
}
try
{
currentProcess.MinWorkingSet = new IntPtr(nextSize);
}
catch (Exception e)
{
throw new InsufficientMemoryException($"Failed to increase the min working set size so we can lock {info.Size:#,#;;0} for {FileName}. With encrypted " +
"databases we lock some memory in order to avoid leaking secrets to disk. Treating this as a catastrophic error " +
"and aborting the current operation.", e);
}
// now we can try again, after we raised the limit, we only do so once, though
if (Sodium.sodium_mlock(info.BaseAddress, (UIntPtr)info.Size) == 0)
{
return(false);
}
}
var msg =
$"Unable to lock memory for {FileName} with size {info.Size:#,#;;0}), with encrypted databases we lock some memory in order to avoid leaking secrets to disk. Treating this as a catastrophic error and aborting the current operation.{Environment.NewLine}";
if (PlatformDetails.RunningOnPosix)
{
msg +=
$"The admin may configure higher limits using: 'sudo prlimit --pid {currentProcess.Id} --memlock={sum}' to increase the limit. (It's recommended to do that as part of the startup script){Environment.NewLine}";
}
else
{
msg +=
$"Already tried to raise the the process min working set to {currentProcess.MinWorkingSet.ToInt64():#,#;;0} but still got a failure.{Environment.NewLine}";
}
msg += "This behavior is controlled by the 'Security.DoNotConsiderMemoryLockFailureAsCatastrophicError' setting (expert only, modifications of this setting is not recommended).";
throw new InsufficientMemoryException(msg);
}
private PagerState AllocateMorePagesAndRemapContinuously(long allocationSize)
{
var retryCount = 0;
while (retryCount++ < MaxAllocationRetries)
{
byte* newBaseAddress;
if (TryFindContinuousMemory((ulong)(_totalAllocationSize + allocationSize), out newBaseAddress) == false)
{
var message =
string.Format(
"Unable to allocate more pages - unsuccessfully tried to allocate continuous block of size = {0} bytes\r\n" +
"It is likely that we are suffering from virtual memory exhaustion or memory fragmentation.\r\n" +
"64 bits process: {1}\r\n" +
"If you are running in 32 bits, this is expected, and you need to run in 64 bits to resume normal operations.\r\n" +
"If you are running in 64 bits, this is likely an error and should be reported."
, (_totalAllocationSize + allocationSize), Environment.Is64BitProcess);
throw new OutOfMemoryException(message);
}
bool failedToAllocate = false;
long offset = 0;
var allocationInfoAfterReallocation = new List<PagerState.AllocationInfo>();
foreach (var allocationInfo in PagerState.AllocationInfos)
{
var newAlloctedBaseAddress = MemoryMapNativeMethods.MapViewOfFileEx(allocationInfo.MappedFile.SafeMemoryMappedFileHandle.DangerousGetHandle(),
MemoryMapNativeMethods.NativeFileMapAccessType.Read | MemoryMapNativeMethods.NativeFileMapAccessType.Write,
0, 0,
UIntPtr.Zero,
newBaseAddress + offset);
if (newAlloctedBaseAddress == null || newAlloctedBaseAddress == (byte*)0)
{
Debug.WriteLine("Failed to remap file continuously. Unmapping already mapped files and re-trying");
UndoMappings(allocationInfoAfterReallocation);
failedToAllocate = true;
break;
}
offset += allocationInfo.Size;
allocationInfoAfterReallocation.Add(new PagerState.AllocationInfo
{
BaseAddress = newAlloctedBaseAddress,
MappedFile = allocationInfo.MappedFile,
Size = allocationInfo.Size
});
}
if (failedToAllocate)
continue;
var newAllocationInfo = TryCreateNewFileMappingAtAddress(allocationSize, newBaseAddress + _totalAllocationSize);
if (newAllocationInfo == null)
{
UndoMappings(allocationInfoAfterReallocation);
continue;
}
var newPagerState = new PagerState(this)
{
Files = PagerState.Files.Concat(newAllocationInfo.MappedFile),
AllocationInfos = allocationInfoAfterReallocation.Concat(newAllocationInfo),
MapBase = newBaseAddress
};
return newPagerState;
}
throw new InvalidOperationException(
string.Format(
"Something bad has happened, after {0} tries, could not find any spot in virtual memory to remap continuous virtual memory for {1:##,###;;0} bytes",
MaxAllocationRetries, allocationSize));
}
public Page ReadPage(long p, PagerState pagerState = null)
{
return _scratchPager.Read(p, pagerState);
}
public Page ReadPage(int scratchNumber, long p, PagerState pagerState = null)
{
return(_scratchBuffers[scratchNumber].ReadPage(p, pagerState));
}
public static byte *AcquirePagePointerWithOverflowHandling <T>(this AbstractPager pager, T tx, long pageNumber, PagerState pagerState) where T : IPagerLevelTransactionState
{
// Case 1: Page is not overflow ==> no problem, returning a pointer to existing mapping
var pageHeader = (PageHeader *)pager.AcquirePagePointer(tx, pageNumber, pagerState);
if ((pageHeader->Flags & PageFlags.Overflow) != PageFlags.Overflow)
{
return((byte *)pageHeader);
}
// Case 2: Page is overflow and already mapped large enough ==> no problem, returning a pointer to existing mapping
if (pager.EnsureMapped(tx, pageNumber, GetNumberOfOverflowPages(pageHeader->OverflowSize)) == false)
{
return((byte *)pageHeader);
}
// Case 3: Page is overflow and was ensuredMapped above, view was re-mapped so we need to acquire a pointer to the new mapping.
return(pager.AcquirePagePointer(tx, pageNumber, pagerState));
}
public override unsafe byte *AcquirePagePointer(Transaction tx, long pageNumber, PagerState pagerState = null)
{
long page = 0;
foreach (var buffer in _buffers)
{
if (page + buffer.SizeInPages > pageNumber)
{
return(buffer.Pointer + ((pageNumber - page) * PageSize));
}
page += buffer.SizeInPages;
}
throw new InvalidOperationException("Could not find a matching page number: " + pageNumber);
}
public void EnsurePagerStateReference(ref PagerState state)
{
}
public override unsafe byte *AcquirePagePointer(IPagerLevelTransactionState tx, long pageNumber, PagerState pagerState = null)
{
// We need to decide what pager we are going to use right now or risk inconsistencies when performing prefetches from disk.
var state = pagerState ?? _pagerState;
if (PlatformDetails.CanPrefetch)
{
if (this._pagerState.ShouldPrefetchSegment(pageNumber, out void *virtualAddress, out long bytes))
{
Syscall.madvise(new IntPtr(virtualAddress), (UIntPtr)bytes, MAdvFlags.MADV_WILLNEED);
}
}
return(base.AcquirePagePointer(tx, pageNumber, state));
}
public RvnMemoryMapPager(StorageEnvironmentOptions options, VoronPathSetting file, long?initialFileSize = null, bool canPrefetchAhead = true, bool usePageProtection = false, bool deleteOnClose = false)
: base(options, canPrefetchAhead, usePageProtection)
{
DeleteOnClose = deleteOnClose;
FileName = file;
var copyOnWriteMode = options.CopyOnWriteMode && FileName.FullPath.EndsWith(Constants.DatabaseFilename);
_logger = LoggingSource.Instance.GetLogger <StorageEnvironment>($"Pager-{file}");
if (initialFileSize.HasValue == false || initialFileSize.Value == 0)
{
initialFileSize = Math.Max(SysInfo.PageSize * 16, 64 * 1024);
}
if (initialFileSize % SysInfo.PageSize != 0)
{
initialFileSize += SysInfo.PageSize - initialFileSize % SysInfo.PageSize;
}
Debug.Assert(file != null);
var mmapOptions = copyOnWriteMode ? MmapOptions.CopyOnWrite : MmapOptions.None;
if (DeleteOnClose)
{
mmapOptions |= MmapOptions.DeleteOnClose;
}
var rc = rvn_create_and_mmap64_file(
file.FullPath,
initialFileSize.Value,
mmapOptions,
out _handle,
out var baseAddress,
out _totalAllocationSize,
out var errorCode);
if (rc != FailCodes.Success)
{
try
{
PalHelper.ThrowLastError(rc, errorCode, $"rvn_create_and_mmap64_file failed on {rc} for '{file.FullPath}'");
}
catch (DiskFullException dfEx)
{
var diskSpaceResult = DiskSpaceChecker.GetDiskSpaceInfo(file.FullPath);
throw new DiskFullException(file.FullPath, initialFileSize.Value, diskSpaceResult?.TotalFreeSpace.GetValue(SizeUnit.Bytes), dfEx.Message);
}
}
NumberOfAllocatedPages = _totalAllocationSize / Constants.Storage.PageSize;
NativeMemory.RegisterFileMapping(FileName.FullPath, new IntPtr(baseAddress), _totalAllocationSize, GetAllocatedInBytes);
var allocationInfo = new PagerState.AllocationInfo
{
BaseAddress = (byte *)baseAddress,
Size = _totalAllocationSize,
MappedFile = null
};
var pager = new PagerState(this, Options.PrefetchSegmentSize, Options.PrefetchResetThreshold, allocationInfo);
SetPagerState(pager);
}
public override byte *AcquirePagePointer(IPagerLevelTransactionState tx, long pageNumber, PagerState pagerState = null)
{
var state = pagerState ?? _pagerState;
if (CanPrefetch.Value && state != null)
{
if (state.ShouldPrefetchSegment(pageNumber, out var virtualAddress, out var bytes))
{
rvn_prefetch_virtual_memory(virtualAddress, bytes, out _); // ignore if unsuccessful
}
}
return(base.AcquirePagePointer(tx, pageNumber, state));
}
public override byte *AcquirePagePointerForNewPage(IPagerLevelTransactionState tx, long pageNumber, int numberOfPages, PagerState pagerState = null)
{
// New page -> no need to read page, just allocate a new buffer
var state = GetTransactionState(tx);
var size = numberOfPages * Constants.Storage.PageSize;
var buffer = GetBufferAndAddToTxState(pageNumber, state, size);
return(buffer.Pointer);
}
public override byte *AcquirePagePointer(IPagerLevelTransactionState tx, long pageNumber, PagerState pagerState = null)
{
var state = GetTransactionState(tx);
if (state.LoadedBuffers.TryGetValue(pageNumber, out var buffer))
{
return(buffer.Pointer);
}
var pagePointer = Inner.AcquirePagePointer(tx, pageNumber, pagerState);
var pageHeader = (PageHeader *)pagePointer;
var size = GetNumberOfPages(pageHeader) * Constants.Storage.PageSize;
buffer = GetBufferAndAddToTxState(pageNumber, state, size);
Memory.Copy(buffer.Pointer, pagePointer, buffer.Size);
DecryptPage((PageHeader *)buffer.Pointer);
if (Sodium.crypto_generichash(buffer.Hash, EncryptionBuffer.HashSize, buffer.Pointer, (ulong)buffer.Size, null, UIntPtr.Zero) != 0)
{
ThrowInvalidHash();
}
return(buffer.Pointer);
}
public override byte *AcquirePagePointer(IPagerLevelTransactionState tx, long pageNumber, PagerState pagerState = null)
{
var state = GetTransactionState(tx);
if (state.LoadedBuffers.TryGetValue(pageNumber, out var buffer))
{
return(buffer.Pointer);
}
var pagePointer = Inner.AcquirePagePointer(tx, pageNumber, pagerState);
var pageHeader = (PageHeader *)pagePointer;
var size = GetNumberOfPages(pageHeader) * Constants.Storage.PageSize;
buffer = GetBufferAndAddToTxState(pageNumber, state, size);
Memory.Copy(buffer.Pointer, pagePointer, buffer.Size);
DecryptPage((PageHeader *)buffer.Pointer);
buffer.Checksum = Hashing.XXHash64.Calculate(buffer.Pointer, (ulong)buffer.Size);
return(buffer.Pointer);
}
public virtual byte* AcquirePagePointer(Transaction tx, long pageNumber, PagerState pagerState = null)
{
ThrowObjectDisposedIfNeeded();
var state = pagerState ?? PagerState;
if (tx != null)
tx.EnsurePagerStateReference(state);
return state.MapBase + pageNumber * PageSize;
}
public override byte *AcquireRawPagePointer(IPagerLevelTransactionState tx, long pageNumber, PagerState pagerState = null)
{
return(Inner.AcquireRawPagePointer(tx, pageNumber, pagerState));
}
public override byte* AcquirePagePointer(long pageNumber, PagerState pagerState = null)
{
return (pagerState ?? PagerState).MapBase + (pageNumber * PageSize);
}
void IPagerLevelTransactionState.EnsurePagerStateReference(PagerState state)
{
//nothing to do
}
public Simple4KbBatchWrites(AbstractPager abstractPager)
{
_abstractPager = abstractPager;
_pagerState = _abstractPager.GetPagerStateAndAddRefAtomically();
}
public override byte *AcquirePagePointer(IPagerLevelTransactionState tx, long pageNumber, PagerState pagerState = null)
{
// We need to decide what pager we are going to use right now or risk inconsistencies when performing prefetches from disk.
var state = pagerState ?? _pagerState;
if (CanPrefetch.Value && state != null)
{
if (state.ShouldPrefetchSegment(pageNumber, out void *virtualAddress, out long bytes))
{
var command = new PalDefinitions.PrefetchRanges(virtualAddress, bytes);
GlobalPrefetchingBehavior.GlobalPrefetcher.Value.CommandQueue.TryAdd(command, 0);
}
}
return(base.AcquirePagePointer(tx, pageNumber, state));
}
public override int CopyPage(I4KbBatchWrites destI4KbBatchWrites, long pageNumber, PagerState pagerState)
{
long sizeToMap = AllocationGranularity;
var distanceFromStart = (pageNumber % NumberOfPagesInAllocationGranularity);
var allocationStartPosition = pageNumber - distanceFromStart;
var offset = allocationStartPosition * Constants.Storage.PageSize;
var mmflags = _copyOnWriteMode ? MmapFlags.MAP_PRIVATE : MmapFlags.MAP_SHARED;
var result = Syscall.mmap64(IntPtr.Zero, (UIntPtr)sizeToMap,
MmapProts.PROT_READ | MmapProts.PROT_WRITE,
mmflags, _fd, offset);
try
{
if (result.ToInt64() == -1) //system didn't succeed in mapping the address where we wanted
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err,
$"Unable to map (default view size) {sizeToMap / Constants.Size.Kilobyte:#,#0} kb for page {pageNumber} starting at {allocationStartPosition} on {FileName}");
}
var pageHeader = (PageHeader *)(result.ToInt64() + distanceFromStart * Constants.Storage.PageSize);
int numberOfPages = 1;
if ((pageHeader->Flags & PageFlags.Overflow) == PageFlags.Overflow)
{
numberOfPages = VirtualPagerLegacyExtensions.GetNumberOfOverflowPages(pageHeader->OverflowSize);
}
if (numberOfPages + distanceFromStart > NumberOfPagesInAllocationGranularity)
{
Syscall.munmap(result, (UIntPtr)sizeToMap);
result = new IntPtr(-1);
sizeToMap = NearestSizeToAllocationGranularity((numberOfPages + distanceFromStart) *
Constants.Storage.PageSize);
result = Syscall.mmap64(IntPtr.Zero, (UIntPtr)sizeToMap, MmapProts.PROT_READ | MmapProts.PROT_WRITE,
mmflags, _fd, offset);
if (result.ToInt64() == -1)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err,
$"Unable to map {sizeToMap / Constants.Size.Kilobyte:#,#0} kb for page {pageNumber} starting at {allocationStartPosition} on {FileName}");
}
pageHeader = (PageHeader *)(result.ToInt64() + (distanceFromStart * Constants.Storage.PageSize));
}
const int adjustPageSize = (Constants.Storage.PageSize) / (4 * Constants.Size.Kilobyte);
destI4KbBatchWrites.Write(pageHeader->PageNumber * adjustPageSize, numberOfPages * adjustPageSize,
(byte *)pageHeader);
return(numberOfPages);
}
finally
{
if (result.ToInt64() != -1)
{
Syscall.munmap(result, (UIntPtr)sizeToMap);
}
}
}
public virtual int CopyPage(IPagerBatchWrites destPagerBatchWrites, int scratchNumber, long p, PagerState pagerState)
{
var item = GetScratchBufferFile(scratchNumber);
ScratchBufferFile bufferFile = item.File;
return(bufferFile.CopyPage(destPagerBatchWrites, p, pagerState));
}
public override byte *AcquirePagePointer(IPagerLevelTransactionState tx, long pageNumber, PagerState pagerState = null)
{
if (DisposeOnceRunner.Disposed)
{
ThrowAlreadyDisposedException();
}
if (pageNumber > NumberOfAllocatedPages || pageNumber < 0)
{
ThrowOnInvalidPageNumber(pageNumber);
}
var state = GetTransactionState(tx);
var distanceFromStart = (pageNumber % NumberOfPagesInAllocationGranularity);
var allocationStartPosition = pageNumber - distanceFromStart;
if (state.LoadedPages.TryGetValue(allocationStartPosition, out var page))
{
return(page.Pointer + (distanceFromStart * Constants.Storage.PageSize));
}
page = MapPages(state, allocationStartPosition, AllocationGranularity);
return(page.Pointer + (distanceFromStart * Constants.Storage.PageSize));
}
private PagerState CreateNewPagerState()
{
MemoryMappedFileAccess memoryMappedFileAccess = _access == NativeFileAccess.GenericRead
? MemoryMappedFileAccess.Read
: MemoryMappedFileAccess.ReadWrite;
MemoryMappedFile mmf = MemoryMappedFile.CreateFromFile(_fileStream, null, _fileStream.Length,
memoryMappedFileAccess,
null, HandleInheritability.None, true);
MemoryMappedViewAccessor accessor = mmf.CreateViewAccessor(0, _fileStream.Length, memoryMappedFileAccess);
byte* p = null;
accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref p);
var newPager = new PagerState(this)
{
Accessor = accessor,
Files = new[] { mmf },
MapBase = p
};
newPager.AddRef(); // one for the pager
return newPager;
}
public override byte *AcquirePagePointer(long pageNumber, PagerState pagerState = null)
{
return((pagerState ?? PagerState).MapBase + (pageNumber * PageSize));
}
public override byte* AcquirePagePointer(long pageNumber, PagerState pagerState = null)
{
ThrowObjectDisposedIfNeeded();
return (pagerState ?? PagerState).MapBase + (pageNumber*PageSize);
}
private PagerState CreateNewPagerState()
{
var mmf = MemoryMappedFile.CreateFromFile(_fileStream, Guid.NewGuid().ToString(), _fileStream.Length,
MemoryMappedFileAccess.Read, null, HandleInheritability.None, true);
MemoryMappedViewAccessor accessor;
try
{
accessor = mmf.CreateViewAccessor(0, _fileStream.Length, MemoryMappedFileAccess.Read);
}
catch (Exception)
{
mmf.Dispose();
throw;
}
byte* p = null;
accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref p);
var newPager = new PagerState(this)
{
Accessor = accessor,
Files = new[] { mmf },
MapBase = p
};
newPager.AddRef(); // one for the pager
return newPager;
}
public override byte* AcquirePagePointer(long pageNumber, PagerState pagerState = null)
{
if (pageNumber >= NumberOfAllocatedPages)
throw new InvalidOperationException("Tried to read a page that wasn't committed");
return _baseAddress + (pageNumber * PageSize);
}
public override int CopyPage(I4KbBatchWrites destwI4KbBatchWrites, long p, PagerState pagerState)
{
return(CopyPageImpl(destwI4KbBatchWrites, p, pagerState));
}
public Page ReadPage(Transaction tx, long p, PagerState pagerState = null)
{
return _scratchPager.Read(tx, p, pagerState);
}
private PagerState CreatePagerState()
{
var mmf = MemoryMappedFile.CreateFromFile(_fileStream, null, _fileStream.Length,
_memoryMappedFileAccess,
null, HandleInheritability.None, true);
var fileMappingHandle = mmf.SafeMemoryMappedFileHandle.DangerousGetHandle();
var mmFileAccessType = _access == NativeFileAccess.GenericRead
? MemoryMapNativeMethods.NativeFileMapAccessType.Read
: MemoryMapNativeMethods.NativeFileMapAccessType.Read | MemoryMapNativeMethods.NativeFileMapAccessType.Write;
var startingBaseAddressPtr = MemoryMapNativeMethods.MapViewOfFileEx(fileMappingHandle,
mmFileAccessType,
0, 0,
UIntPtr.Zero, //map all what was "reserved" in CreateFileMapping on previous row
null);
if (startingBaseAddressPtr == (byte*)0) //system didn't succeed in mapping the address where we wanted
throw new Win32Exception();
var allocationInfo = new PagerState.AllocationInfo
{
BaseAddress = startingBaseAddressPtr,
Size = _fileStream.Length,
MappedFile = mmf
};
var newPager = new PagerState(this)
{
Files = new[] { mmf },
Accessor = null, //not available since MapViewOfFileEx is used (instead of MapViewOfFile - which is used in managed wrapper)
MapBase = startingBaseAddressPtr,
AllocationInfos = new[] { allocationInfo }
};
newPager.AddRef(); // one for the pager
return newPager;
}
public override int CopyPage(I4KbBatchWrites destI4KbBatchWrites, long pageNumber, PagerState pagerState)
{
var distanceFromStart = (pageNumber % NumberOfPagesInAllocationGranularity);
var allocationStartPosition = pageNumber - distanceFromStart;
var offset = new WindowsMemoryMapPager.SplitValue {
Value = (ulong)allocationStartPosition * (ulong)Constants.Storage.PageSize
};
var result = MapViewOfFileEx(_hFileMappingObject, _mmFileAccessType, offset.High,
offset.Low,
(UIntPtr)AllocationGranularity, null);
try
{
if (result == null)
{
var lastWin32Error = Marshal.GetLastWin32Error();
throw new Win32Exception($"Unable to map (default view size) {AllocationGranularity / Constants.Size.Kilobyte:#,#0} kb for page {pageNumber} starting at {allocationStartPosition} on {FileName}",
new Win32Exception(lastWin32Error));
}
var pageHeader = (PageHeader *)(result + distanceFromStart * Constants.Storage.PageSize);
int numberOfPages = 1;
if ((pageHeader->Flags & PageFlags.Overflow) == PageFlags.Overflow)
{
numberOfPages = VirtualPagerLegacyExtensions.GetNumberOfOverflowPages(pageHeader->OverflowSize);
}
if (numberOfPages + distanceFromStart > NumberOfPagesInAllocationGranularity)
{
UnmapViewOfFile(result);
result = null;
var newSize = NearestSizeToAllocationGranularity((numberOfPages + distanceFromStart) * Constants.Storage.PageSize);
result = MapViewOfFileEx(_hFileMappingObject, _mmFileAccessType, offset.High,
offset.Low,
(UIntPtr)newSize, null);
if (result == null)
{
var lastWin32Error = Marshal.GetLastWin32Error();
throw new Win32Exception($"Unable to map {newSize / Constants.Size.Kilobyte:#,#0} kb for page {pageNumber} starting at {allocationStartPosition} on {FileName}",
new Win32Exception(lastWin32Error));
}
pageHeader = (PageHeader *)(result + (distanceFromStart * Constants.Storage.PageSize));
}
const int adjustPageSize = (Constants.Storage.PageSize) / (4 * Constants.Size.Kilobyte);
destI4KbBatchWrites.Write(pageHeader->PageNumber * adjustPageSize, numberOfPages * adjustPageSize, (byte *)pageHeader);
return(numberOfPages);
}
finally
{
if (result != null)
{
UnmapViewOfFile(result);
}
}
}
public Page ReadPage(Transaction tx, long pageNumber, PagerState scratchPagerState)
{
// read transactions have to read from journal snapshots
if (tx.Flags == TransactionFlags.Read)
{
// read log snapshots from the back to get the most recent version of a page
for (var i = tx.JournalSnapshots.Count - 1; i >= 0; i--)
{
JournalFile.PagePosition value;
if (tx.JournalSnapshots[i].PageTranslationTable.TryGetValue(tx, pageNumber, out value))
{
var page = _env.ScratchBufferPool.ReadPage(value.ScratchPos, scratchPagerState);
Debug.Assert(page.PageNumber == pageNumber);
return page;
}
}
return null;
}
// write transactions can read directly from journals
var files = _files;
for (var i = files.Count - 1; i >= 0; i--)
{
JournalFile.PagePosition value;
if (files[i].PageTranslationTable.TryGetValue(tx, pageNumber, out value))
{
var page = _env.ScratchBufferPool.ReadPage(value.ScratchPos, scratchPagerState);
Debug.Assert(page.PageNumber == pageNumber);
return page;
}
}
return null;
}
public override byte *AcquirePagePointer(long pageNumber, PagerState pagerState = null)
{
ThrowObjectDisposedIfNeeded();
return((pagerState ?? PagerState).MapBase + (pageNumber * PageSize));
}
public Page ReadPage(int scratchNumber, long p, PagerState pagerState = null)
{
ScratchBufferFile bufferFile;
ScratchBufferCacheItem item = lastScratchBuffer;
if ( item.Number == scratchNumber )
{
bufferFile = item.File;
}
else
{
bufferFile = _scratchBuffers[scratchNumber];
lastScratchBuffer = new ScratchBufferCacheItem( scratchNumber, bufferFile );
}
return bufferFile.ReadPage(p, pagerState);
}
public int CopyPage(I4KbBatchWrites destI4KbBatchWrites, long p, PagerState pagerState)
{
return(_scratchPager.CopyPage(destI4KbBatchWrites, p, pagerState));
}
public Page ReadPage(LowLevelTransaction tx, long p, PagerState pagerState = null)
{
return(new Page(_scratchPager.AcquirePagePointerWithOverflowHandling(tx, p, pagerState)));
}
private byte *AcquirePagePointerInternal(IPagerLevelTransactionState tx, long pageNumber, PagerState pagerState)
{
if (DisposeOnceRunner.Disposed)
{
goto AlreadyDisposed;
}
if (pageNumber > NumberOfAllocatedPages || pageNumber < 0)
{
goto InvalidPageNumber;
}
var state = pagerState ?? _pagerState;
tx?.EnsurePagerStateReference(state);
return(state.MapBase + pageNumber * Constants.Storage.PageSize);
AlreadyDisposed : ThrowAlreadyDisposedException();
InvalidPageNumber : ThrowOnInvalidPageNumber(pageNumber);
return(null); // Will never happen.
}
public abstract byte* AcquirePagePointer(long pageNumber, PagerState pagerState = null);
public abstract int CopyPage(I4KbBatchWrites destwI4KbBatchWrites, long p, PagerState pagerState);
private PagerState CreateInitialPagerState(long size, byte* requestedBaseAddress)
{
var allocationSize = NearestSizeToAllocationGranularity(size);
var mmf = MemoryMappedFile.CreateNew(null, allocationSize, MemoryMappedFileAccess.ReadWrite);
var fileMappingHandle = mmf.SafeMemoryMappedFileHandle.DangerousGetHandle();
var startingBaseAddressPtr = MemoryMapNativeMethods.MapViewOfFileEx(fileMappingHandle,
MemoryMapNativeMethods.NativeFileMapAccessType.Read | MemoryMapNativeMethods.NativeFileMapAccessType.Write,
0, 0,
UIntPtr.Zero, //map all what was "reserved" in CreateFileMapping on previous row
requestedBaseAddress);
if (startingBaseAddressPtr == (byte*)0) //system didn't succeed in mapping the address where we wanted
throw new Win32Exception();
var allocationInfo = new PagerState.AllocationInfo
{
BaseAddress = startingBaseAddressPtr,
Size = allocationSize,
MappedFile = mmf
};
var newPager = new PagerState(this)
{
Files = new[] { mmf },
Accessor = null, //not available since MapViewOfFileEx is used (instead of MapViewOfFile - which is used in managed wrapper)
MapBase = startingBaseAddressPtr,
AllocationInfos = new[] { allocationInfo }
};
newPager.AddRef();
return newPager;
}
public virtual byte *AcquireRawPagePointer(IPagerLevelTransactionState tx, long pageNumber, PagerState pagerState = null)
{
return(AcquirePagePointer(tx, pageNumber, pagerState));
}
public Page ReadPage(int scratchNumber, long p, PagerState pagerState = null)
{
return _scratchBuffers[scratchNumber].ReadPage(p, pagerState);
}
public override void AllocateMorePages(Transaction tx, long newLength)
{
var oldSize = NumberOfAllocatedPages * PageSize;
if (newLength < oldSize)
throw new ArgumentException("Cannot set the legnth to less than the current length");
if (newLength == oldSize)
return; // nothing to do
var increaseSize = (newLength - oldSize);
NumberOfAllocatedPages += increaseSize / PageSize;
var newPtr = Marshal.AllocHGlobal(new IntPtr(increaseSize));
var buffer = new Buffer
{
Handle = newPtr,
Base = (byte*) newPtr.ToPointer(),
Size = increaseSize
};
_buffers = _buffers.Append(buffer);
var newPager = new PagerState(this);
newPager.AddRef(); // one for the pager
if (tx != null) // we only pass null during startup, and we don't need it there
{
newPager.AddRef(); // one for the current transaction
tx.AddPagerState(newPager);
}
PagerState.Release();
PagerState = newPager;
}
