public unsafe void LZ4Test(int size)
{
byte *encodeInput = NativeMemory.AllocateMemory(size);
int compressedSize;
byte *encodeOutput;
var originStr = string.Join("", Enumerable.Repeat(1, size).Select(x => "sample"));
var bytes = Encoding.UTF8.GetBytes(originStr);
var maximumOutputLength = LZ4.MaximumOutputLength(bytes.Length);
fixed(byte *pb = bytes)
{
encodeOutput = NativeMemory.AllocateMemory((int)maximumOutputLength);
compressedSize = LZ4.Encode64(pb, encodeOutput, bytes.Length, (int)maximumOutputLength);
}
Array.Clear(bytes, 0, bytes.Length);
fixed(byte *pb = bytes)
{
LZ4.Decode64(encodeOutput, compressedSize, pb, bytes.Length, true);
}
var actual = Encoding.UTF8.GetString(bytes);
Assert.Equal(originStr, actual);
NativeMemory.Free(encodeInput, size);
NativeMemory.Free(encodeOutput, maximumOutputLength);
}
public void Setup()
{
source = NativeMemory.AllocateMemory(size, out var _);
modified = NativeMemory.AllocateMemory(size, out var _);
destination = NativeMemory.AllocateMemory(size, out var _);
var r = new Random();
for (int i = 0; i < size; i++)
{
int b = r.Next();
source[i] = (byte)b;
modified[i] = (byte)b;
}
for (int i = 0; i < 100; i++)
{
int start = r.Next(size - 1000);
int end = start + 256 + r.Next(512);
for (; start < end; start++)
{
source[i] = 0;
}
}
}
public ArenaMemoryAllocator(SharedMultipleUseFlag lowMemoryFlag, int initialSize = 1024 * 1024)
{
_initialSize = initialSize;
_ptrStart = _ptrCurrent = NativeMemory.AllocateMemory(initialSize, out _allocatingThread);
_allocated = initialSize;
_used = 0;
TotalUsed = 0;
_lowMemoryFlag = lowMemoryFlag;
}
public Win32FileJournalWriter(StorageEnvironmentOptions options, VoronPathSetting filename, long journalSize,
Win32NativeFileAccess access = Win32NativeFileAccess.GenericWrite,
Win32NativeFileShare shareMode = Win32NativeFileShare.Read)
{
try
{
_options = options;
_filename = filename;
_handle = Win32NativeFileMethods.CreateFile(filename.FullPath,
access, shareMode, IntPtr.Zero,
Win32NativeFileCreationDisposition.OpenAlways,
options.WinOpenFlags, IntPtr.Zero);
if (_handle.IsInvalid)
{
throw new IOException("When opening file " + filename, new Win32Exception(Marshal.GetLastWin32Error()));
}
var length = new FileInfo(filename.FullPath).Length;
if (length < journalSize)
{
try
{
Win32NativeFileMethods.SetFileLength(_handle, journalSize);
}
catch (Exception)
{
try
{
_handle?.Dispose();
_handle = null;
File.Delete(_filename.FullPath);
}
catch (Exception)
{
// there's nothing we can do about it
}
throw;
}
length = journalSize;
}
NumberOfAllocated4Kb = (int)(length / (4 * Constants.Size.Kilobyte));
_nativeOverlapped = (NativeOverlapped *)NativeMemory.AllocateMemory(sizeof(NativeOverlapped));
_nativeOverlapped->InternalLow = IntPtr.Zero;
_nativeOverlapped->InternalHigh = IntPtr.Zero;
}
catch
{
Dispose();
throw;
}
}
public void RenewArena()
{
if (_ptrStart != null)
{
return;
}
_ptrStart = _ptrCurrent = NativeMemory.AllocateMemory(_allocated, out _allocatingThread);
_used = 0;
}
public bool Read(byte *buffer, long numOfBytes, long offsetInFile)
{
if (_readHandle == null)
{
var handle = Win32NativeFileMethods.CreateFile(_filename.FullPath,
Win32NativeFileAccess.GenericRead,
Win32NativeFileShare.Write | Win32NativeFileShare.Read | Win32NativeFileShare.Delete,
IntPtr.Zero,
Win32NativeFileCreationDisposition.OpenExisting,
Win32NativeFileAttributes.Normal,
IntPtr.Zero);
if (handle.IsInvalid)
{
throw new IOException("When opening file " + _filename, new Win32Exception(Marshal.GetLastWin32Error()));
}
_readHandle = handle;
}
var nativeOverlapped = (NativeOverlapped *)NativeMemory.AllocateMemory(sizeof(NativeOverlapped));
try
{
nativeOverlapped->OffsetLow = (int)(offsetInFile & 0xffffffff);
nativeOverlapped->OffsetHigh = (int)(offsetInFile >> 32);
nativeOverlapped->EventHandle = IntPtr.Zero;
while (numOfBytes > 0)
{
if (Win32NativeFileMethods.ReadFile(_readHandle, buffer, (int)Math.Min(numOfBytes, int.MaxValue), out int read, nativeOverlapped) == false)
{
int lastWin32Error = Marshal.GetLastWin32Error();
if (lastWin32Error == Win32NativeFileMethods.ErrorHandleEof)
{
return(false);
}
if (lastWin32Error == Win32NativeFileMethods.ErrorInvalidHandle)
{
_readHandle = null;
}
throw new Win32Exception(lastWin32Error, $"Unable to read from {_filename}, error code: {lastWin32Error}");
}
numOfBytes -= read;
buffer += read;
offsetInFile += read;
nativeOverlapped->OffsetLow = (int)(offsetInFile & 0xffffffff);
nativeOverlapped->OffsetHigh = (int)(offsetInFile >> 32);
}
return(true);
}
finally
{
NativeMemory.Free((byte *)nativeOverlapped, sizeof(NativeOverlapped));
}
}
public ArenaMemoryAllocator(int initialSize = 1024 * 1024)
{
_ptrStart = _ptrCurrent = NativeMemory.AllocateMemory(initialSize, out _allocatingThread);
_allocated = initialSize;
_used = 0;
if (Logger.IsInfoEnabled)
{
Logger.Info($"ArenaMemoryAllocator was created with initial capacity of {initialSize:#,#;;0} bytes");
}
}
public override unsafe void WriteHeader(string filename, FileHeader *header)
{
if (Disposed)
{
throw new ObjectDisposedException("PureMemoryStorageEnvironmentOptions");
}
IntPtr ptr;
if (_headers.TryGetValue(filename, out ptr) == false)
{
ptr = (IntPtr)NativeMemory.AllocateMemory(sizeof(FileHeader));
_headers[filename] = ptr;
}
Memory.Copy((byte *)ptr, (byte *)header, sizeof(FileHeader));
}
public SmallStringCompression(string[] termsTable)
{
_termsTable = termsTable;
if (termsTable.Length + 8 > byte.MaxValue)
{
throw new InvalidOperationException("Too many terms defined");
}
_termsTableBytes = new byte *[termsTable.Length];
_maxVerbatimLen = Math.Min(byte.MaxValue - termsTable.Length, 48);
_hashTable = new byte *[byte.MaxValue][];
for (int i = 0; i < termsTable.Length; i++)
{
var byteCount = Encodings.Utf8.GetByteCount(termsTable[i]);
if (byteCount > byte.MaxValue)
{
throw new InvalidOperationException("Term " + termsTable[i] + " is too big");
}
var ptr = (byte *)NativeMemory.AllocateMemory(byteCount + 2);
_termsTableBytes[i] = ptr;
ptr[0] = (byte)byteCount;
fixed(char *pChars = termsTable[i])
{
var bytes = Encodings.Utf8.GetBytes(pChars, termsTable[i].Length, ptr + 1, byteCount);
if (bytes != byteCount)
{
throw new InvalidOperationException("Bug, UTF8 encoding mismatch for GetByteCount and GetBytes for " + termsTable[i]);
}
}
ptr[byteCount + 1] = (byte)i;
_maxTermSize = Math.Max(_maxTermSize, byteCount);
AddToHash(ptr, byteCount);
}
var empty = new byte *[0];
for (int i = 0; i < _hashTable.Length; i++)
{
if (_hashTable[i] == null)
{
_hashTable[i] = empty;
}
}
}
public bool Read(long pageNumber, byte *buffer, int count)
{
if (_readHandle == null)
{
_readHandle = Win32NativeFileMethods.CreateFile(_filename,
Win32NativeFileAccess.GenericRead,
Win32NativeFileShare.Write | Win32NativeFileShare.Read | Win32NativeFileShare.Delete,
IntPtr.Zero,
Win32NativeFileCreationDisposition.OpenExisting,
Win32NativeFileAttributes.Normal,
IntPtr.Zero);
}
long position = pageNumber * _options.PageSize;
NativeOverlapped *nativeOverlapped = (NativeOverlapped *)NativeMemory.AllocateMemory(sizeof(NativeOverlapped));
try
{
nativeOverlapped->OffsetLow = (int)(position & 0xffffffff);
nativeOverlapped->OffsetHigh = (int)(position >> 32);
nativeOverlapped->EventHandle = IntPtr.Zero;
while (count > 0)
{
int read;
if (Win32NativeFileMethods.ReadFile(_readHandle, buffer, count, out read, nativeOverlapped) == false)
{
int lastWin32Error = Marshal.GetLastWin32Error();
if (lastWin32Error == Win32NativeFileMethods.ErrorHandleEof)
{
return(false);
}
throw new Win32Exception(lastWin32Error);
}
count -= read;
buffer += read;
position += read;
nativeOverlapped->OffsetLow = (int)(position & 0xffffffff);
nativeOverlapped->OffsetHigh = (int)(position >> 32);
}
return(true);
}
finally
{
NativeMemory.Free((byte *)nativeOverlapped, sizeof(NativeOverlapped));
}
}
private void GrowArena(int requestedSize)
{
if (requestedSize >= MaxArenaSize)
{
throw new ArgumentOutOfRangeException(nameof(requestedSize));
}
// we need the next allocation to cover at least the next expansion (also doubling)
// so we'll allocate 3 times as much as was requested, or twice as much as we already have
// the idea is that a single allocation can server for multiple (increasing in size) calls
long newSize = Math.Max(Bits.NextPowerOf2(requestedSize) * 3, _allocated);
if (newSize > MaxArenaSize)
{
newSize = MaxArenaSize;
}
if (Logger.IsInfoEnabled)
{
if (newSize > 512 * 1024 * 1024)
{
Logger.Info(
$"Arena main buffer reached size of {newSize:#,#;0} bytes (previously {_allocated:#,#;0} bytes), check if you forgot to reset the context. From now on we grow this arena in 1GB chunks.");
}
Logger.Info(
$"Increased size of buffer from {_allocated:#,#;0} to {newSize:#,#;0} because we need {requestedSize:#,#;0}. _used={_used:#,#;0}");
}
NativeMemory.ThreadStats thread;
var newBuffer = NativeMemory.AllocateMemory(newSize, out thread);
// Save the old buffer pointer to be released when the arena is reset
if (_olderBuffers == null)
{
_olderBuffers = new List <Tuple <IntPtr, long, NativeMemory.ThreadStats> >();
}
_olderBuffers.Add(Tuple.Create(new IntPtr(_ptrStart), _allocated, _allocatingThread));
_allocatingThread = thread;
_allocated = newSize;
_ptrStart = newBuffer;
_ptrCurrent = _ptrStart;
_used = 0;
}
private void GrowArena(int requestedSize)
{
if (requestedSize >= MaxArenaSize)
{
throw new ArgumentOutOfRangeException(nameof(requestedSize), requestedSize,
$"Requested arena resize to {requestedSize} while current size is {_allocated} and maximum size is {MaxArenaSize}");
}
long newSize = GetPreferredSize(requestedSize);
if (newSize > MaxArenaSize)
{
newSize = MaxArenaSize;
}
byte *newBuffer;
NativeMemory.ThreadStats thread;
try
{
newBuffer = NativeMemory.AllocateMemory(newSize, out thread);
}
catch (OutOfMemoryException oom)
when(oom.Data?.Contains("Recoverable") != true) // this can be raised if the commit charge is low
{
// we were too eager with memory allocations?
newBuffer = NativeMemory.AllocateMemory(requestedSize, out thread);
newSize = requestedSize;
}
// Save the old buffer pointer to be released when the arena is reset
if (_olderBuffers == null)
{
_olderBuffers = new List <Tuple <IntPtr, long, NativeMemory.ThreadStats> >();
}
_olderBuffers.Add(Tuple.Create(new IntPtr(_ptrStart), _allocated, _allocatingThread));
_allocatingThread = thread;
_allocated = newSize;
_ptrStart = newBuffer;
_ptrCurrent = _ptrStart;
_used = 0;
}
public unsafe void Scratch_file_is_aware_of_potentialy_active_readers()
{
var numberOfPages = 20;
var handle = NativeMemory.AllocateMemory(numberOfPages * Constants.Storage.PageSize);
var buffer = new PureMemoryJournalWriter.Buffer
{
Pointer = handle,
SizeInPages = numberOfPages
};
try
{
using (var env = StorageEnvironmentOptions.CreateMemoryOnly())
using (var pager = new FragmentedPureMemoryPager(env, ImmutableAppendOnlyList <PureMemoryJournalWriter.Buffer> .Empty.Append(buffer)))
using (var file = new ScratchBufferFile(pager, 0))
{
Assert.False(file.HasActivelyUsedBytes(2));
file.Allocate(null, 1, 1);
file.Allocate(null, 1, 1);
file.Allocate(null, 1, 1);
file.Allocate(null, 1, 1);
file.Allocate(null, 1, 1);
file.Free(0, 1);
file.Free(1, 3);
file.Free(2, 4);
file.Free(3, 7);
file.Free(4, 9);
for (int i = 0; i <= 9; i++)
{
Assert.True(file.HasActivelyUsedBytes(i));
}
Assert.False(file.HasActivelyUsedBytes(10));
Assert.False(file.HasActivelyUsedBytes(20));
}
}
finally
{
NativeMemory.Free(handle, buffer.SizeInPages);
}
}
private void GrowArena(int requestedSize)
{
if (_lowMemoryFlag)
{
throw new LowMemoryException($"Request to grow the arena by {requestedSize} because we are under memory pressure");
}
if (requestedSize >= MaxArenaSize)
{
throw new ArgumentOutOfRangeException(nameof(requestedSize));
}
LowMemoryNotification.NotifyAllocationPending();
// we need the next allocation to cover at least the next expansion (also doubling)
// so we'll allocate 3 times as much as was requested, or as much as we already have
// the idea is that a single allocation can server for multiple (increasing in size) calls
long newSize = Math.Max(Bits.NextPowerOf2(requestedSize) * 3, _initialSize);
if (newSize > MaxArenaSize)
{
newSize = MaxArenaSize;
}
NativeMemory.ThreadStats thread;
var newBuffer = NativeMemory.AllocateMemory(newSize, out thread);
// Save the old buffer pointer to be released when the arena is reset
if (_olderBuffers == null)
{
_olderBuffers = new List <Tuple <IntPtr, long, NativeMemory.ThreadStats> >();
}
_olderBuffers.Add(Tuple.Create(new IntPtr(_ptrStart), _allocated, _allocatingThread));
_allocatingThread = thread;
_allocated = newSize;
_ptrStart = newBuffer;
_ptrCurrent = _ptrStart;
_used = 0;
}
public AllocatedMemoryData Allocate(int size)
{
#if MEM_GUARD
return(new AllocatedMemoryData
{
SizeInBytes = size,
Address = ElectricFencedMemory.Allocate(size),
});
#else
var actualSize = Bits.PowerOf2(size);
var index = GetIndexFromSize(actualSize);
NativeMemory.ThreadStats stats;
if (index == -1)
{
return(new AllocatedMemoryData()
{
SizeInBytes = size,
Address = NativeMemory.AllocateMemory(size, out stats),
AllocatingThread = stats
});
}
actualSize = GetIndexSize(ref index, actualSize); // when we request 7 bytes, we want to get 16 bytes
if (_freeSegments[index].TryPop(out AllocatedMemoryData list))
{
return(list);
}
return(new AllocatedMemoryData()
{
SizeInBytes = actualSize,
Address = NativeMemory.AllocateMemory(actualSize, out stats),
AllocatingThread = stats
});
#endif
}
static LazyStringParser()
{
// PERF: The following code will build tables like this that will guarantee that the addition is outside of the
// proper range to detect failures.
// shift1 = new[] {
// OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 0-9
// OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 10
// OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 20
// OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 30
// OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, 0, 1, // 40
// 2, 3, 4, 5, 6, 7, 8, 9, OOR, OOR, // 50
// OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 60
// OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 70
// OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 80
// OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 90
// OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 100
// OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 110
// OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 120
// OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 130
// OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 140
// OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 150
// OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 160
// OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 170
// OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 180
// OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 190
// OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 200
// OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 210
// OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 220
// OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 230
// OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 240
// OOR, OOR, OOR, OOR, OOR, OOR // 250
//};
// PERF: will use native memory to avoid the overhead of the bound check which cannot be evicted otherwise.
shift1 = (int *)NativeMemory.AllocateMemory(256 * sizeof(int));
for (int i = 0; i < 256; i++)
{
if ((char)i >= '0' && (char)i <= '9')
{
shift1[i] = (int)(i - '0');
}
else
{
shift1[i] = OOR;
}
}
shift10 = (int *)NativeMemory.AllocateMemory(256 * sizeof(int));
for (int i = 0; i < 256; i++)
{
if ((char)i >= '0' && (char)i <= '9')
{
shift10[i] = 10 * (int)(i - '0');
}
else
{
shift10[i] = OOR;
}
}
shift100 = (int *)NativeMemory.AllocateMemory(256 * sizeof(int));
for (int i = 0; i < 256; i++)
{
if ((char)i >= '0' && (char)i <= '9')
{
shift100[i] = 100 * (int)(i - '0');
}
else
{
shift100[i] = OOR;
}
}
shift1000 = (int *)NativeMemory.AllocateMemory(256 * sizeof(int));
for (int i = 0; i < 256; i++)
{
if ((char)i >= '0' && (char)i <= '9')
{
shift1000[i] = 1000 * (int)(i - '0');
}
else
{
shift1000[i] = OOR;
}
}
}