public void Return(SectorAlignedMemory page)
{
#if CHECK_FOR_LEAKS
Interlocked.Increment(ref totalReturns);
#endif
Debug.Assert(queue[page.level] != null);
page.available_bytes = 0;
page.required_bytes = 0;
page.valid_offset = 0;
Array.Clear(page.buffer, 0, page.buffer.Length);
if (!Disabled)
{
if (UnpinOnReturn)
{
page.handle.Free();
page.handle = default;
}
queue[page.level].Enqueue(page);
}
else
{
#if NET5_0_OR_GREATER
if (UnpinOnReturn)
{
page.handle.Free();
}
#else
page.handle.Free();
#endif
page.buffer = null;
}
}
public unsafe SectorAlignedMemory Get(int numRecords)
{
int requiredSize = sectorSize + (((numRecords) * recordSize + (sectorSize - 1)) & ~(sectorSize - 1));
int index = Position(requiredSize / sectorSize);
if (queue[index] == null)
{
var localPool = new ConcurrentQueue <SectorAlignedMemory>();
Interlocked.CompareExchange(ref queue[index], localPool, null);
}
if (!Disabled && queue[index].TryDequeue(out SectorAlignedMemory page))
{
return(page);
}
page = new SectorAlignedMemory
{
level = index,
buffer = new byte[sectorSize * (1 << index)]
};
page.handle = GCHandle.Alloc(page.buffer, GCHandleType.Pinned);
page.aligned_pointer = (byte *)(((long)page.handle.AddrOfPinnedObject() + (sectorSize - 1)) & ~(sectorSize - 1));
page.offset = (int)((long)page.aligned_pointer - (long)page.handle.AddrOfPinnedObject());
page.pool = this;
return(page);
}
public WriteCallbackWrapper(Stream logHandle, IOCompletionCallback callback, IAsyncResult asyncResult, SectorAlignedMemory memory)
{
this.callback = callback;
this.asyncResult = asyncResult;
this.memory = memory;
this.logHandle = logHandle;
}
/// <summary>
///
/// </summary>
/// <param name="segmentId"></param>
/// <param name="sourceAddress"></param>
/// <param name="destinationAddress"></param>
/// <param name="readLength"></param>
/// <param name="callback"></param>
/// <param name="asyncResult"></param>
public override unsafe void ReadAsync(int segmentId, ulong sourceAddress,
IntPtr destinationAddress,
uint readLength,
IOCompletionCallback callback,
IAsyncResult asyncResult)
{
ReadCallbackWrapper wrapper = null;
SectorAlignedMemory memory = null;
Stream logHandle = null;
try
{
memory = pool.Get((int)readLength);
logHandle = GetOrAddHandle(segmentId);
wrapper = new ReadCallbackWrapper(logHandle, callback, asyncResult, memory, destinationAddress, readLength, 0);
logHandle.Seek((long)sourceAddress, SeekOrigin.Begin);
logHandle.BeginRead(memory.buffer, 0, (int)readLength, wrapper.Callback, null);
}
catch (IOException e)
{
wrapper = new ReadCallbackWrapper(logHandle, callback, asyncResult, memory, destinationAddress, readLength, (uint)(e.HResult & 0x0000FFFF));
wrapper.Callback(asyncResult);
}
catch
{
wrapper = new ReadCallbackWrapper(logHandle, callback, asyncResult, memory, destinationAddress, readLength, uint.MaxValue);
wrapper.Callback(asyncResult);
}
}
public unsafe VarLenHeapContainer(ref T obj, IVariableLengthStruct <T> varLenStruct, SectorAlignedBufferPool pool)
{
var len = varLenStruct.GetLength(ref obj);
mem = pool.Get(len);
Buffer.MemoryCopy(Unsafe.AsPointer(ref obj), mem.GetValidPointer(), len, len);
}
public unsafe void Return(SectorAlignedMemory page)
{
Debug.Assert(queue[page.level] != null);
page.available_bytes = 0;
page.required_bytes = 0;
page.valid_offset = 0;
queue[page.level].Enqueue(page);
}
public ReadCallbackWrapper(IOCompletionCallback callback, IAsyncResult asyncResult, SectorAlignedMemory memory, IntPtr destinationAddress, uint readLength)
{
this.callback = callback;
this.asyncResult = asyncResult;
this.memory = memory;
this.destinationAddress = destinationAddress;
this.readLength = readLength;
}
public unsafe void Return(SectorAlignedMemory page)
{
Debug.Assert(queue[page.level] != null);
page.available_bytes = 0;
page.required_bytes = 0;
page.valid_offset = 0;
Array.Clear(page.buffer, 0, page.buffer.Length);
queue[page.level].Enqueue(page);
}
public ReadCallbackWrapper(Stream logHandle, IOCompletionCallback callback, IAsyncResult asyncResult, SectorAlignedMemory memory, IntPtr destinationAddress, uint readLength, uint errorCode)
{
this.logHandle = logHandle;
this.callback = callback;
this.asyncResult = asyncResult;
this.memory = memory;
this.destinationAddress = destinationAddress;
this.readLength = readLength;
this.errorCode = errorCode;
}
/// <summary>
/// Free
/// </summary>
public void Free()
{
if (freeBuffer1 != null)
{
freeBuffer1.Return();
freeBuffer1 = null;
}
if (freeBuffer2 != null)
{
freeBuffer2.Return();
freeBuffer2 = null;
}
}
/// <summary>
/// Free
/// </summary>
public void Free()
{
if (freeBuffer1 != null)
{
freeBuffer1.Return();
freeBuffer1 = null;
}
if (freeBuffer2 != null)
{
freeBuffer2.Return();
freeBuffer2 = null;
}
flushCompletionTracker?.CompleteFlush();
}
/// <summary>
/// Free
/// </summary>
public void Free()
{
if (freeBuffer1 != null)
{
freeBuffer1.Return();
freeBuffer1 = null;
}
if (freeBuffer2 != null)
{
freeBuffer2.Return();
freeBuffer2 = null;
}
completedSemaphore?.Release();
}
public void Return(SectorAlignedMemory page)
{
Debug.Assert(queue[page.level] != null);
page.available_bytes = 0;
page.required_bytes = 0;
page.valid_offset = 0;
Array.Clear(page.buffer, 0, page.buffer.Length);
if (!Disabled)
{
queue[page.level].Enqueue(page);
}
else
{
page.handle.Free();
page.buffer = null;
}
}
/// <summary>
/// Free
/// </summary>
public void Free()
{
if (freeBuffer1 != null)
{
freeBuffer1.Return();
freeBuffer1 = null;
}
if (freeBuffer2 != null)
{
freeBuffer2.Return();
freeBuffer2 = null;
}
if (handle != null)
{
handle.Signal();
}
}
public unsafe SectorAlignedMemory Get(int numRecords)
{
#if CHECK_FOR_LEAKS
Interlocked.Increment(ref totalGets);
#endif
int requiredSize = sectorSize + (((numRecords) * recordSize + (sectorSize - 1)) & ~(sectorSize - 1));
int index = Position(requiredSize / sectorSize);
if (queue[index] == null)
{
var localPool = new ConcurrentQueue <SectorAlignedMemory>();
Interlocked.CompareExchange(ref queue[index], localPool, null);
}
if (!Disabled && queue[index].TryDequeue(out SectorAlignedMemory page))
{
if (UnpinOnReturn)
{
page.handle = GCHandle.Alloc(page.buffer, GCHandleType.Pinned);
page.aligned_pointer = (byte *)(((long)page.handle.AddrOfPinnedObject() + (sectorSize - 1)) & ~((long)sectorSize - 1));
page.offset = (int)((long)page.aligned_pointer - (long)page.handle.AddrOfPinnedObject());
}
return(page);
}
page = new SectorAlignedMemory {
level = index
};
#if NET5_0_OR_GREATER
page.buffer = GC.AllocateArray <byte>(sectorSize * (1 << index), !UnpinOnReturn);
if (UnpinOnReturn)
{
page.handle = GCHandle.Alloc(page.buffer, GCHandleType.Pinned);
}
#else
page.buffer = new byte[sectorSize * (1 << index)];
page.handle = GCHandle.Alloc(page.buffer, GCHandleType.Pinned);
#endif
long pageAddr = (long)Unsafe.AsPointer(ref page.buffer[0]);
page.aligned_pointer = (byte *)((pageAddr + (sectorSize - 1)) & ~((long)sectorSize - 1));
page.offset = (int)((long)page.aligned_pointer - pageAddr);
page.pool = this;
return(page);
}
private void AsyncGetFromDisk(long fromLogical,
int numRecords,
IOCompletionCallback callback,
AsyncIOContext context,
SectorAlignedMemory result = default(SectorAlignedMemory))
{
while (numPendingReads > 120)
{
Thread.SpinWait(100);
// Do not protect if we are not already protected
// E.g., we are in an IO thread
if (epoch.IsProtected())
epoch.ProtectAndDrain();
}
Interlocked.Increment(ref numPendingReads);
hlog.AsyncReadRecordToMemory(fromLogical, numRecords, callback, context, result);
}
/// <summary>
///
/// </summary>
/// <param name="sourceAddress"></param>
/// <param name="segmentId"></param>
/// <param name="destinationAddress"></param>
/// <param name="numBytesToWrite"></param>
/// <param name="callback"></param>
/// <param name="asyncResult"></param>
public override unsafe void WriteAsync(IntPtr sourceAddress,
int segmentId,
ulong destinationAddress,
uint numBytesToWrite,
IOCompletionCallback callback,
IAsyncResult asyncResult)
{
WriteCallbackWrapper wrapper = null;
SectorAlignedMemory memory = null;
Stream logHandle = null;
try
{
memory = pool.Get((int)numBytesToWrite);
logHandle = GetOrAddHandle(segmentId);
wrapper = new WriteCallbackWrapper(logHandle, callback, asyncResult, memory, 0);
fixed(void *destination = memory.buffer)
{
Buffer.MemoryCopy((void *)sourceAddress, destination, numBytesToWrite, numBytesToWrite);
}
logHandle.Seek((long)destinationAddress, SeekOrigin.Begin);
logHandle.BeginWrite(memory.buffer, 0, (int)numBytesToWrite, wrapper.Callback, null);
}
catch (IOException e)
{
wrapper = new WriteCallbackWrapper(logHandle, callback, asyncResult, memory, (uint)(e.HResult & 0x0000FFFF));
wrapper.Callback(asyncResult);
}
catch
{
wrapper = new WriteCallbackWrapper(logHandle, callback, asyncResult, memory, uint.MaxValue);
wrapper.Callback(asyncResult);
}
}
/// <summary>
/// Get next record in iterator
/// </summary>
/// <param name="recordInfo"></param>
/// <returns></returns>
public unsafe bool GetNext(out RecordInfo recordInfo)
{
recordInfo = default;
while (true)
{
currentAddress = nextAddress;
// Check for boundary conditions
if (currentAddress >= endAddress)
{
return(false);
}
epoch?.Resume();
var headAddress = hlog.HeadAddress;
if (currentAddress < hlog.BeginAddress && !forceInMemory)
{
epoch?.Suspend();
throw new FasterException("Iterator address is less than log BeginAddress " + hlog.BeginAddress);
}
if (frameSize == 0 && currentAddress < headAddress && !forceInMemory)
{
epoch?.Suspend();
throw new FasterException("Iterator address is less than log HeadAddress in memory-scan mode");
}
var currentPage = currentAddress >> hlog.LogPageSizeBits;
var offset = currentAddress & hlog.PageSizeMask;
if (currentAddress < headAddress && !forceInMemory)
{
BufferAndLoad(currentAddress, currentPage, currentPage % frameSize, headAddress, endAddress);
}
long physicalAddress;
if (currentAddress >= headAddress || forceInMemory)
{
physicalAddress = hlog.GetPhysicalAddress(currentAddress);
}
else
{
physicalAddress = frame.GetPhysicalAddress(currentPage % frameSize, offset);
}
// Check if record fits on page, if not skip to next page
var recordSize = hlog.GetRecordSize(physicalAddress).Item2;
if ((currentAddress & hlog.PageSizeMask) + recordSize > hlog.PageSize)
{
nextAddress = (1 + (currentAddress >> hlog.LogPageSizeBits)) << hlog.LogPageSizeBits;
epoch?.Suspend();
continue;
}
nextAddress = currentAddress + recordSize;
ref var info = ref hlog.GetInfo(physicalAddress);
if (info.SkipOnScan || info.IsNull())
{
epoch?.Suspend();
continue;
}
currentPhysicalAddress = physicalAddress;
recordInfo = info;
if (currentAddress >= headAddress || forceInMemory)
{
memory?.Return();
memory = hlog.bufferPool.Get(recordSize);
Buffer.MemoryCopy((byte *)currentPhysicalAddress, memory.aligned_pointer, recordSize, recordSize);
currentPhysicalAddress = (long)memory.aligned_pointer;
}
epoch?.Suspend();
return(true);
}
public void AsyncReadRecordToMemory(long fromLogical, int numRecords, IOCompletionCallback callback, AsyncIOContext context, SectorAlignedMemory result = default(SectorAlignedMemory))
{
ulong fileOffset = (ulong)(AlignedPageSizeBytes * (fromLogical >> LogPageSizeBits) + (fromLogical & PageSizeMask));
ulong alignedFileOffset = (ulong)(((long)fileOffset / sectorSize) * sectorSize);
uint alignedReadLength = (uint)((long)fileOffset + numRecords - (long)alignedFileOffset);
alignedReadLength = (uint)((alignedReadLength + (sectorSize - 1)) & ~(sectorSize - 1));
var record = readBufferPool.Get((int)alignedReadLength);
record.valid_offset = (int)(fileOffset - alignedFileOffset);
record.available_bytes = (int)(alignedReadLength - (fileOffset - alignedFileOffset));
record.required_bytes = numRecords;
var asyncResult = default(AsyncGetFromDiskResult <AsyncIOContext>);
asyncResult.context = context;
if (result.buffer == null)
{
asyncResult.context.record = record;
device.ReadAsync(alignedFileOffset,
(IntPtr)asyncResult.context.record.aligned_pointer,
alignedReadLength,
callback,
asyncResult);
}
else
{
asyncResult.context.record = result;
asyncResult.context.objBuffer = record;
objlogDevice.ReadAsync(
(int)(context.logicalAddress >> LogSegmentSizeBits),
alignedFileOffset,
(IntPtr)asyncResult.context.objBuffer.aligned_pointer,
alignedReadLength,
callback,
asyncResult);
}
}
public unsafe void Return(SectorAlignedMemory page)
{
Debug.Assert(queue[page.level] != null);
queue[page.level].Enqueue(page);
}
/// <summary>
/// Invoked by users to obtain a record from disk. It uses sector aligned memory to read
/// the record efficiently into memory.
/// </summary>
/// <param name="fromLogical"></param>
/// <param name="numBytes"></param>
/// <param name="callback"></param>
/// <param name="context"></param>
/// <param name="result"></param>
protected override void AsyncReadRecordObjectsToMemory(long fromLogical, int numBytes, IOCompletionCallback callback, AsyncIOContext <Key, Value> context, SectorAlignedMemory result = default(SectorAlignedMemory))
{
throw new InvalidOperationException("AsyncReadRecordObjectsToMemory invalid for BlittableAllocator");
}
public WriteCallbackWrapper(IOCompletionCallback callback, IAsyncResult asyncResult, SectorAlignedMemory memory)
{
this.callback = callback;
this.asyncResult = asyncResult;
this.memory = memory;
}