internal override void UnprotectPageRange(byte *start, ulong size, bool force = false)
{
if (size == 0)
{
return;
}
if (UsePageProtection || force)
{
Win32MemoryProtectMethods.MEMORY_BASIC_INFORMATION memoryInfo1 = new Win32MemoryProtectMethods.MEMORY_BASIC_INFORMATION();
int vQueryFirstOutput = Win32MemoryProtectMethods.VirtualQuery(start, &memoryInfo1, new UIntPtr(size));
int vQueryFirstError = Marshal.GetLastWin32Error();
Win32MemoryProtectMethods.MemoryProtection oldProtection;
bool status = Win32MemoryProtectMethods.VirtualProtect(start, new UIntPtr(size), Win32MemoryProtectMethods.MemoryProtection.READWRITE, out oldProtection);
if (!status)
{
int vProtectError = Marshal.GetLastWin32Error();
Win32MemoryProtectMethods.MEMORY_BASIC_INFORMATION memoryInfo2 = new Win32MemoryProtectMethods.MEMORY_BASIC_INFORMATION();
int vQuerySecondOutput = Win32MemoryProtectMethods.VirtualQuery(start, &memoryInfo2, new UIntPtr(size));
int vQuerySecondError = Marshal.GetLastWin32Error();
Debugger.Break();
}
}
}
public static byte *Allocate4KbAlignedMemory(long size, out ThreadStats thread)
{
Debug.Assert(size >= 0);
thread = ThreadAllocations.Value;
thread.Allocations += size;
if (PlatformDetails.RunningOnPosix)
{
byte *ptr;
var rc = Syscall.posix_memalign(&ptr, (IntPtr)4096, (IntPtr)size);
if (rc != 0)
{
Syscall.ThrowLastError(rc, "Could not allocate memory");
}
return(ptr);
}
var allocate4KbAllignedMemory = Win32MemoryProtectMethods.VirtualAlloc(null, (UIntPtr)size, Win32MemoryProtectMethods.AllocationType.COMMIT,
Win32MemoryProtectMethods.MemoryProtection.READWRITE);
if (allocate4KbAllignedMemory == null)
{
ThrowFailedToAllocate();
}
return(allocate4KbAllignedMemory);
}
public static void Free4KbAlignedMemory(byte *ptr, int size, ThreadStats stats)
{
Debug.Assert(ptr != null);
var currentThreadValue = ThreadAllocations.Value;
if (currentThreadValue == stats)
{
currentThreadValue.Allocations -= size;
FixupReleasesFromOtherThreads(currentThreadValue);
}
else
{
Interlocked.Add(ref stats.ReleasesFromOtherThreads, size);
}
var p = new IntPtr(ptr);
if (PlatformDetails.RunningOnPosix)
{
Syscall.free(p);
return;
}
if (Win32MemoryProtectMethods.VirtualFree(ptr, UIntPtr.Zero, Win32MemoryProtectMethods.FreeType.MEM_RELEASE) == false)
{
ThrowFailedToFree();
}
}
public static void Free(byte *p)
{
var remaining = (int)((long)p % 4096);
var firstWritablePage = p - remaining;
for (int i = 0; i < remaining; i++)
{
if (firstWritablePage[i] != 0xED)
{
throw new InvalidOperationException("Invalid memory usage, you killed Ed!");
}
}
Win32MemoryProtectMethods.MemoryProtection protect;
var address = firstWritablePage - 4096;
// this will access the memory, which will error if this was already freed
if (Win32MemoryProtectMethods.VirtualProtect(address, (UIntPtr)4096, Win32MemoryProtectMethods.MemoryProtection.READWRITE, out protect) ==
false)
{
throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to VirtualProtect (ElectricFence) at address=" + new IntPtr(address));
}
var dwSize = *(int *)address;
// decommit, not release, they are not available for reuse again, any
// future access will throw
if (Win32MemoryProtectMethods.VirtualFree(address, (UIntPtr)dwSize, Win32MemoryProtectMethods.FreeType.MEM_DECOMMIT) == false)
{
throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to VirtualFree (ElectricFence) at address=" + new IntPtr(address));
}
}
public static void Free4KbAlignedMemory(byte *ptr, long size, Sparrow.Utils.NativeMemory.ThreadStats stats)
{
Debug.Assert(ptr != null);
if (stats != null)
{
Sparrow.Utils.NativeMemory.UpdateMemoryStatsForThread(stats, size);
}
Interlocked.Add(ref Sparrow.Utils.NativeMemory._totalAllocatedMemory, -size);
if (PlatformDetails.RunningOnPosix)
{
var result = Syscall.munmap((IntPtr)ptr, (UIntPtr)(uint)size);
if (result == -1)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, "Failed to munmap ");
}
return;
}
if (Win32MemoryProtectMethods.VirtualFree(ptr, UIntPtr.Zero, Win32MemoryProtectMethods.FreeType.MEM_RELEASE) == false)
{
ThrowFailedToFree();
}
}
static void FreeMemoryAtEndOfPager(byte *adjacentBlockAddress)
{
if (adjacentBlockAddress == null || adjacentBlockAddress == (byte *)0)
{
return;
}
if (StorageEnvironmentOptions.RunningOnPosix)
{
Syscall.munmap(new IntPtr(adjacentBlockAddress), (UIntPtr)16);
return;
}
Win32MemoryProtectMethods.VirtualFree(adjacentBlockAddress, UIntPtr.Zero, Win32MemoryProtectMethods.FreeType.MEM_RELEASE);
}
byte *AllocateMemoryAtEndOfPager(long totalAllocationSize)
{
if (StorageEnvironmentOptions.RunningOnPosix)
{
var p = Syscall.mmap64(new IntPtr(Env.Options.DataPager.PagerState.MapBase + totalAllocationSize), (UIntPtr)16,
MmapProts.PROT_READ | MmapProts.PROT_WRITE, MmapFlags.MAP_ANONYMOUS, -1, 0L);
if (p.ToInt64() == -1)
{
return(null);
}
return((byte *)p.ToPointer());
}
return(Win32MemoryProtectMethods.VirtualAlloc(Env.Options.DataPager.PagerState.MapBase + totalAllocationSize, new UIntPtr(16),
Win32MemoryProtectMethods.AllocationType.RESERVE, Win32MemoryProtectMethods.MemoryProtection.EXECUTE_READWRITE));
}
public static byte *Allocate(int size)
{
var remaining = size % 4096;
var sizeInPages = (size / 4096) + (remaining == 0 ? 0 : 1);
var allocatedSize = ((sizeInPages + 2) * 4096);
var virtualAlloc = Win32MemoryProtectMethods.VirtualAlloc(null, (UIntPtr)allocatedSize, Win32MemoryProtectMethods.AllocationType.COMMIT,
Win32MemoryProtectMethods.MemoryProtection.READWRITE);
if (virtualAlloc == null)
{
throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to VirtualAlloc (ElectricFence) size=" + size);
}
*(int *)virtualAlloc = allocatedSize;
Win32MemoryProtectMethods.MemoryProtection protect;
if (Win32MemoryProtectMethods.VirtualProtect(virtualAlloc, (UIntPtr)(4096), Win32MemoryProtectMethods.MemoryProtection.NOACCESS,
out protect) == false)
{
throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to VirtualProtect (ElectricFence) at address=" + new IntPtr(virtualAlloc));
}
if (Win32MemoryProtectMethods.VirtualProtect(virtualAlloc + (sizeInPages + 1) * 4096, (UIntPtr)(4096), Win32MemoryProtectMethods.MemoryProtection.NOACCESS,
out protect) == false)
{
throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to VirtualProtect (ElectricFence) at address=" + new IntPtr(virtualAlloc + (sizeInPages + 1) * 4096));
}
var firstWritablePage = virtualAlloc + 4096;
Win32UnmanagedMemory.Set(firstWritablePage, 0xED, 4096 * sizeInPages); // don't assume zero'ed mem
if (remaining == 0)
{
return(firstWritablePage);
}
// give the memory out so its end would be at the 2nd guard page
return(firstWritablePage + (4096 - remaining));
}
public static byte *Allocate4KbAlignedMemory(long size, out Sparrow.Utils.NativeMemory.ThreadStats thread)
{
Debug.Assert(size >= 0);
thread = Sparrow.Utils.NativeMemory.ThreadAllocations.Value;
thread.Allocations += size;
Interlocked.Add(ref Sparrow.Utils.NativeMemory._totalAllocatedMemory, size);
if (PlatformDetails.RunningOnPosix)
{
// we pass NULL (IntPtr.Zero) as the first parameter (address / start) so the kernel chooses the(page-aligned) address at which to create the mapping
var pageAlignedMemory = Syscall.mmap64(IntPtr.Zero, (UIntPtr)size, MmapProts.PROT_READ | MmapProts.PROT_WRITE,
MmapFlags.MAP_PRIVATE | MmapFlags.MAP_ANONYMOUS, -1, 0L);
if (pageAlignedMemory.ToInt64() == -1)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err,
$"Could not allocate memory (allocation size: {size / Constants.Size.Kilobyte:#,#0} kb)");
}
return((byte *)pageAlignedMemory);
}
var allocate4KbAlignedMemory = Win32MemoryProtectMethods.VirtualAlloc(null, (UIntPtr)size, Win32MemoryProtectMethods.AllocationType.COMMIT,
Win32MemoryProtectMethods.MemoryProtection.READWRITE);
if (allocate4KbAlignedMemory == null)
{
ThrowFailedToAllocate();
}
return(allocate4KbAlignedMemory);
}
public static void Free4KbAlignedMemory(byte *ptr, long size, Sparrow.Utils.NativeMemory.ThreadStats stats)
{
Debug.Assert(ptr != null);
if (stats != null)
{
Sparrow.Utils.NativeMemory.UpdateMemoryStatsForThread(stats, size);
}
Interlocked.Add(ref Sparrow.Utils.NativeMemory._totalAllocatedMemory, -size);
var p = new IntPtr(ptr);
if (PlatformDetails.RunningOnPosix)
{
Syscall.free(p);
return;
}
if (Win32MemoryProtectMethods.VirtualFree(ptr, UIntPtr.Zero, Win32MemoryProtectMethods.FreeType.MEM_RELEASE) == false)
{
ThrowFailedToFree();
}
}
