/// <summary>
/// Flush pages from startPage (inclusive) to endPage (exclusive)
/// to specified log device and obj device
/// </summary>
/// <param name="startPage"></param>
/// <param name="endPage"></param>
/// <param name="device"></param>
public void AsyncFlushPagesToDevice(long startPage, long endPage, IDevice device, out CountdownEvent completed)
{
var asyncResult = new PageAsyncFlushResult();
int numPages = (int)(endPage - startPage);
ISegmentedDevice objlogDevice = null;
if (Key.HasObjectsToSerialize() || Value.HasObjectsToSerialize())
{
numPages = numPages * 2;
objlogDevice = CreateObjectLogDevice(device);
}
completed = new CountdownEvent(numPages);
asyncResult.handle = completed;
for (long flushPage = startPage; flushPage < endPage; flushPage++)
{
long pageStartAddress = flushPage << LogPageSizeBits;
long pageEndAddress = (flushPage + 1) << LogPageSizeBits;
WriteAsync(pointers[flushPage % BufferSize],
(ulong)(AlignedPageSizeBytes * (flushPage - startPage)),
(uint)(PageSize * PrivateRecordSize),
AsyncFlushPageToDeviceCallback,
asyncResult, device, objlogDevice);
}
}
public void Add(PageAsyncFlushResult <Empty> t)
{
lock (list)
{
list.AddFirst(t);
}
}
private void WriteAsync <TContext>(IntPtr alignedSourceAddress, ulong alignedDestinationAddress, uint numBytesToWrite,
IOCompletionCallback callback, PageAsyncFlushResult <TContext> asyncResult,
IDevice device)
{
device.WriteAsync(alignedSourceAddress, alignedDestinationAddress,
numBytesToWrite, callback, asyncResult);
}
public void AsyncReadPagesFromDevice(int numPages, long destinationStartPage, IDevice device, out CountdownEvent completed)
{
var asyncResult = new PageAsyncFlushResult();
ISegmentedDevice objlogDevice = null;
if (Key.HasObjectsToSerialize() || Value.HasObjectsToSerialize())
{
objlogDevice = CreateObjectLogDevice(device);
throw new Exception("Reading pages with object log not yet supported");
}
completed = new CountdownEvent(numPages);
asyncResult.handle = completed;
asyncResult.objlogDevice = objlogDevice;
for (long flushPage = destinationStartPage; flushPage < destinationStartPage + numPages; flushPage++)
{
long pageStartAddress = flushPage << LogPageSizeBits;
long pageEndAddress = (flushPage + 1) << LogPageSizeBits;
device.ReadAsync(
(ulong)(AlignedPageSizeBytes * (flushPage - destinationStartPage)),
pointers[flushPage % BufferSize],
PageSize,
AsyncReadPageFromDeviceCallback,
asyncResult);
}
}
/// <summary>
/// Flush pages from startPage (inclusive) to endPage (exclusive)
/// to specified log device and obj device
/// </summary>
/// <param name="startPage"></param>
/// <param name="endPage"></param>
/// <param name="device"></param>
/// <param name="objectLogDevice"></param>
/// <param name="completed"></param>
public void AsyncFlushPagesToDevice(long startPage, long endPage, IDevice device, IDevice objectLogDevice, out CountdownEvent completed)
{
int totalNumPages = (int)(endPage - startPage);
completed = new CountdownEvent(totalNumPages);
// We are writing to separate device, so use fresh segment offsets
var _segmentOffsets = new long[SegmentBufferSize];
for (long flushPage = startPage; flushPage < endPage; flushPage++)
{
var asyncResult = new PageAsyncFlushResult <Empty>
{
handle = completed,
count = 1
};
long pageStartAddress = flushPage << LogPageSizeBits;
long pageEndAddress = (flushPage + 1) << LogPageSizeBits;
// Intended destination is flushPage
WriteAsync((IntPtr)pointers[flushPage % BufferSize],
(ulong)(AlignedPageSizeBytes * (flushPage - startPage)),
PageSize,
AsyncFlushPageToDeviceCallback,
asyncResult, device, objectLogDevice, flushPage, _segmentOffsets);
}
}
/// <summary>
/// Flush pages from startPage (inclusive) to endPage (exclusive)
/// to specified log device and obj device
/// </summary>
/// <param name="startPage"></param>
/// <param name="endPage"></param>
/// <param name="device"></param>
/// <param name="completed"></param>
public void AsyncFlushPagesToDevice(long startPage, long endPage, IDevice device, out CountdownEvent completed)
{
ISegmentedDevice objlogDevice = null;
if (Key.HasObjectsToSerialize() || Value.HasObjectsToSerialize())
{
objlogDevice = CreateObjectLogDevice(device);
}
int totalNumPages = (int)(endPage - startPage);
completed = new CountdownEvent(totalNumPages);
for (long flushPage = startPage; flushPage < endPage; flushPage++)
{
var asyncResult = new PageAsyncFlushResult <Empty>
{
handle = completed,
count = 1
};
long pageStartAddress = flushPage << LogPageSizeBits;
long pageEndAddress = (flushPage + 1) << LogPageSizeBits;
WriteAsync(pointers[flushPage % BufferSize],
(ulong)(AlignedPageSizeBytes * (flushPage - startPage)),
PageSize,
AsyncFlushPageToDeviceCallback,
asyncResult, device, objlogDevice);
}
}
protected override void WriteAsyncToDevice <TContext>
(long startPage, long flushPage, int pageSize, IOCompletionCallback callback,
PageAsyncFlushResult <TContext> asyncResult, IDevice device, IDevice objectLogDevice)
{
var alignedPageSize = (pageSize + (sectorSize - 1)) & ~(sectorSize - 1);
WriteAsync((IntPtr)pointers[flushPage % BufferSize],
(ulong)(AlignedPageSizeBytes * (flushPage - startPage)),
(uint)alignedPageSize, callback, asyncResult,
device);
}
public bool Add(PageAsyncFlushResult <Empty> t)
{
int retries = 0;
do
{
for (int i = 0; i < maxSize; i++)
{
if (list[i] == default)
{
if (Interlocked.CompareExchange(ref list[i], t, default) == default)
/// <summary>
/// IOCompletion callback for page flush
/// </summary>
/// <param name="errorCode"></param>
/// <param name="numBytes"></param>
/// <param name="overlap"></param>
private void AsyncFlushPageCallback(uint errorCode, uint numBytes, NativeOverlapped *overlap)
{
if (errorCode != 0)
{
System.Diagnostics.Trace.TraceError("OverlappedStream GetQueuedCompletionStatus error: {0}", errorCode);
}
//Set the page status to flushed
PageAsyncFlushResult result = (PageAsyncFlushResult)Overlapped.Unpack(overlap).AsyncResult;
if (Interlocked.Decrement(ref result.count) == 0)
{
PageStatusIndicator[result.page % BufferSize].LastFlushedUntilAddress = result.untilAddress;
if (!result.partial)
{
while (true)
{
var oldStatus = PageStatusIndicator[result.page % BufferSize].PageFlushCloseStatus;
if (oldStatus.PageCloseStatus == CloseStatus.Closed)
{
ClearPage((int)(result.page % BufferSize), result.page == 0);
}
var newStatus = oldStatus;
newStatus.PageFlushStatus = FlushStatus.Flushed;
if (oldStatus.value == Interlocked.CompareExchange(ref PageStatusIndicator[result.page % BufferSize].PageFlushCloseStatus.value, newStatus.value, oldStatus.value))
{
break;
}
}
}
ShiftFlushedUntilAddress();
Interlocked.MemoryBarrier();
if (result.freeBuffer1.buffer != null)
{
result.freeBuffer1.Return();
}
if (result.freeBuffer2.buffer != null)
{
result.freeBuffer2.Return();
}
if (result.handle != null)
{
result.handle.Signal();
}
}
Overlapped.Free(overlap);
}
/// <summary>
/// Flush page range to disk
/// Called when all threads have agreed that a page range is sealed.
/// </summary>
/// <param name="startPage"></param>
/// <param name="untilAddress"></param>
private void AsyncFlushPages(long startPage, long untilAddress)
{
long endPage = (untilAddress >> LogPageSizeBits);
int numPages = (int)(endPage - startPage);
long offsetInEndPage = GetOffsetInPage(untilAddress);
if (offsetInEndPage > 0)
{
numPages++;
}
/* Request asynchronous writes to the device. If waitForPendingFlushComplete
* is set, then a CountDownEvent is set in the callback handle.
*/
for (long flushPage = startPage; flushPage < (startPage + numPages); flushPage++)
{
long pageStartAddress = flushPage << LogPageSizeBits;
long pageEndAddress = (flushPage + 1) << LogPageSizeBits;
var asyncResult = new PageAsyncFlushResult <Empty>
{
page = flushPage,
count = 1
};
if (pageEndAddress > untilAddress)
{
asyncResult.partial = true;
asyncResult.untilAddress = untilAddress;
}
else
{
asyncResult.partial = false;
asyncResult.untilAddress = pageEndAddress;
// Set status to in-progress
PageStatusIndicator[flushPage % BufferSize].PageFlushCloseStatus
= new FlushCloseStatus {
PageFlushStatus = PMMFlushStatus.InProgress, PageCloseStatus = PMMCloseStatus.Open
};
}
PageStatusIndicator[flushPage % BufferSize].LastFlushedUntilAddress = -1;
WriteAsync((IntPtr)pointers[flushPage % BufferSize],
(ulong)(AlignedPageSizeBytes * flushPage),
PageSize,
AsyncFlushPageCallback,
asyncResult, device, objectLogDevice);
}
}
/// <summary>
/// IOCompletion callback for page flush
/// </summary>
/// <param name="errorCode"></param>
/// <param name="numBytes"></param>
/// <param name="overlap"></param>
private void AsyncFlushPartialObjectLogCallback <TContext>(uint errorCode, uint numBytes, NativeOverlapped *overlap)
{
if (errorCode != 0)
{
Trace.TraceError("OverlappedStream GetQueuedCompletionStatus error: {0}", errorCode);
}
// Set the page status to flushed
PageAsyncFlushResult <TContext> result = (PageAsyncFlushResult <TContext>)Overlapped.Unpack(overlap).AsyncResult;
result.done.Set();
Overlapped.Free(overlap);
}
/// <summary>
/// IOCompletion callback for page flush
/// </summary>
/// <param name="errorCode"></param>
/// <param name="numBytes"></param>
/// <param name="overlap"></param>
private void AsyncFlushPageToDeviceCallback(uint errorCode, uint numBytes, NativeOverlapped *overlap)
{
if (errorCode != 0)
{
Trace.TraceError("OverlappedStream GetQueuedCompletionStatus error: {0}", errorCode);
}
PageAsyncFlushResult <Empty> result = (PageAsyncFlushResult <Empty>)Overlapped.Unpack(overlap).AsyncResult;
if (Interlocked.Decrement(ref result.count) == 0)
{
result.Free();
}
Overlapped.Free(overlap);
}
/// <summary>
/// Remove item from flush list with until-address equal to the specified address
/// </summary>
public bool RemovePreviousAdjacent(long address, out PageAsyncFlushResult <Empty> request)
{
lock (list)
{
for (var it = list.First; it != null;)
{
request = it.Value;
if (request.untilAddress == address)
{
list.Remove(it);
return(true);
}
it = it.Next;
}
}
request = null;
return(false);
}
/// <summary>
/// IOCompletion callback for page read
/// </summary>
/// <param name="errorCode"></param>
/// <param name="numBytes"></param>
/// <param name="overlap"></param>
private void AsyncReadPageFromDeviceCallback(uint errorCode, uint numBytes, NativeOverlapped *overlap)
{
if (errorCode != 0)
{
System.Diagnostics.Trace.TraceError("OverlappedStream GetQueuedCompletionStatus error: {0}", errorCode);
}
PageAsyncFlushResult result = (PageAsyncFlushResult)Overlapped.Unpack(overlap).AsyncResult;
if (result.handle != null)
{
result.handle.Signal();
}
Overlapped.Free(overlap);
if (Key.HasObjectsToSerialize() || Value.HasObjectsToSerialize())
{
throw new NotImplementedException("Recovery of object log not yet supported");
}
}
public void AsyncFlushPageToDiskRecovery <TContext>(
long flushPageStart,
int numPages,
IOCompletionCallback callback,
TContext context)
{
for (long flushPage = flushPageStart; flushPage < (flushPageStart + numPages); flushPage++)
{
int pageIndex = GetPageIndexForPage(flushPage);
var asyncResult = new PageAsyncFlushResult <TContext>()
{
page = flushPage,
context = context
};
device.WriteAsync(pointers[flushPage % BufferSize],
(ulong)(AlignedPageSizeBytes * flushPage),
(uint)(PageSize * PrivateRecordSize),
callback,
asyncResult);
}
}
private void WriteAsync <TContext>(IntPtr alignedSourceAddress, ulong alignedDestinationAddress, uint numBytesToWrite,
IOCompletionCallback callback, PageAsyncFlushResult <TContext> asyncResult,
IDevice device)
{
if (asyncResult.partial)
{
// Write only required bytes within the page
int aligned_start = (int)((asyncResult.fromAddress - (asyncResult.page << LogPageSizeBits)));
aligned_start = (aligned_start / sectorSize) * sectorSize;
int aligned_end = (int)((asyncResult.untilAddress - (asyncResult.page << LogPageSizeBits)));
aligned_end = ((aligned_end + (sectorSize - 1)) & ~(sectorSize - 1));
numBytesToWrite = (uint)(aligned_end - aligned_start);
device.WriteAsync(alignedSourceAddress + aligned_start, alignedDestinationAddress + (ulong)aligned_start, numBytesToWrite, callback, asyncResult);
}
else
{
device.WriteAsync(alignedSourceAddress, alignedDestinationAddress,
numBytesToWrite, callback, asyncResult);
}
}
/// <summary>
/// Flush pages from startPage (inclusive) to endPage (exclusive)
/// to specified log device and obj device
/// </summary>
/// <param name="startPage"></param>
/// <param name="endPage"></param>
/// <param name="device"></param>
/// <param name="objectLogDevice"></param>
/// <param name="completed"></param>
public void AsyncFlushPagesToDevice(long startPage, long endPage, IDevice device, IDevice objectLogDevice, out CountdownEvent completed)
{
int totalNumPages = (int)(endPage - startPage);
completed = new CountdownEvent(totalNumPages);
for (long flushPage = startPage; flushPage < endPage; flushPage++)
{
var asyncResult = new PageAsyncFlushResult<Empty>
{
handle = completed,
count = 1
};
long pageStartAddress = flushPage << LogPageSizeBits;
long pageEndAddress = (flushPage + 1) << LogPageSizeBits;
WriteAsync((IntPtr)pointers[flushPage % BufferSize],
(ulong)(AlignedPageSizeBytes * (flushPage - startPage)),
PageSize,
AsyncFlushPageToDeviceCallback,
asyncResult, device, objectLogDevice);
}
}
public void AsyncFlushPages <TContext>(
long flushPageStart,
int numPages,
IOCompletionCallback callback,
TContext context)
{
for (long flushPage = flushPageStart; flushPage < (flushPageStart + numPages); flushPage++)
{
int pageIndex = GetPageIndexForPage(flushPage);
var asyncResult = new PageAsyncFlushResult <TContext>()
{
page = flushPage,
context = context,
count = 1,
partial = false,
untilAddress = (flushPage + 1) << LogPageSizeBits
};
WriteAsync(pointers[flushPage % BufferSize],
(ulong)(AlignedPageSizeBytes * flushPage),
PageSize, callback, asyncResult, device, objlogDevice);
}
}
/// <summary>
/// IOCompletion callback for page flush
/// </summary>
/// <param name="errorCode"></param>
/// <param name="numBytes"></param>
/// <param name="overlap"></param>
private void AsyncFlushPageCallback(uint errorCode, uint numBytes, NativeOverlapped *overlap)
{
if (errorCode != 0)
{
Trace.TraceError("OverlappedStream GetQueuedCompletionStatus error: {0}", errorCode);
}
// Set the page status to flushed
PageAsyncFlushResult <Empty> result = (PageAsyncFlushResult <Empty>)Overlapped.Unpack(overlap).AsyncResult;
if (Interlocked.Decrement(ref result.count) == 0)
{
PageStatusIndicator[result.page % BufferSize].LastFlushedUntilAddress = result.untilAddress;
if (!result.partial)
{
while (true)
{
var oldStatus = PageStatusIndicator[result.page % BufferSize].PageFlushCloseStatus;
if (oldStatus.PageCloseStatus == PMMCloseStatus.Closed)
{
ClearPage((int)(result.page % BufferSize), result.page == 0);
}
var newStatus = oldStatus;
newStatus.PageFlushStatus = PMMFlushStatus.Flushed;
if (oldStatus.value == Interlocked.CompareExchange(ref PageStatusIndicator[result.page % BufferSize].PageFlushCloseStatus.value, newStatus.value, oldStatus.value))
{
break;
}
}
}
ShiftFlushedUntilAddress();
result.Free();
}
Overlapped.Free(overlap);
}
/// <summary>
/// IOCompletion callback for page flush
/// </summary>
/// <param name="errorCode"></param>
/// <param name="numBytes"></param>
/// <param name="overlap"></param>
private void AsyncFlushPageToDeviceCallback(uint errorCode, uint numBytes, NativeOverlapped *overlap)
{
if (errorCode != 0)
{
System.Diagnostics.Trace.TraceError("OverlappedStream GetQueuedCompletionStatus error: {0}", errorCode);
}
PageAsyncFlushResult result = (PageAsyncFlushResult)Overlapped.Unpack(overlap).AsyncResult;
if (result.freeBuffer1.buffer != null)
{
result.freeBuffer1.Return();
}
if (result.freeBuffer2.buffer != null)
{
result.freeBuffer2.Return();
}
if (result.handle != null)
{
result.handle.Signal();
}
Overlapped.Free(overlap);
}
private void WriteAsync <TContext>(IntPtr alignedSourceAddress, ulong alignedDestinationAddress, uint numBytesToWrite,
IOCompletionCallback callback, PageAsyncFlushResult <TContext> asyncResult,
IDevice device, IDevice objlogDevice, long intendedDestinationPage = -1, long[] localSegmentOffsets = null)
{
if (!pageHandlers.HasObjects())
{
device.WriteAsync(alignedSourceAddress, alignedDestinationAddress,
numBytesToWrite, callback, asyncResult);
return;
}
// Check if user did not override with special segment offsets
if (localSegmentOffsets == null)
{
localSegmentOffsets = segmentOffsets;
}
// need to write both page and object cache
asyncResult.count++;
var buffer = ioBufferPool.Get(PageSize);
Buffer.MemoryCopy((void *)alignedSourceAddress, buffer.aligned_pointer, numBytesToWrite, numBytesToWrite);
long ptr = (long)buffer.aligned_pointer;
List <long> addr = new List <long>();
asyncResult.freeBuffer1 = buffer;
// Correct for page 0 of HLOG
if (intendedDestinationPage < 0)
{
// By default, when we are not writing to a separate device, the intended
// destination page (logical) is the same as actual
intendedDestinationPage = (long)(alignedDestinationAddress >> LogPageSizeBits);
}
if (intendedDestinationPage == 0)
{
ptr += Constants.kFirstValidAddress;
}
var untilptr = (long)buffer.aligned_pointer + numBytesToWrite;
while (ptr < untilptr)
{
MemoryStream ms = new MemoryStream();
pageHandlers.Serialize(ref ptr, untilptr, ms, kObjectBlockSize, out List <long> addresses);
var _s = ms.ToArray();
ms.Close();
var _objBuffer = ioBufferPool.Get(_s.Length);
asyncResult.done = new AutoResetEvent(false);
var _alignedLength = (_s.Length + (sectorSize - 1)) & ~(sectorSize - 1);
var _objAddr = Interlocked.Add(ref localSegmentOffsets[(long)(alignedDestinationAddress >> LogSegmentSizeBits) % SegmentBufferSize], _alignedLength) - _alignedLength;
fixed(void *src = _s)
Buffer.MemoryCopy(src, _objBuffer.aligned_pointer, _s.Length, _s.Length);
foreach (var address in addresses)
{
*((long *)address) += _objAddr;
}
if (ptr < untilptr)
{
objlogDevice.WriteAsync(
(IntPtr)_objBuffer.aligned_pointer,
(int)(alignedDestinationAddress >> LogSegmentSizeBits),
(ulong)_objAddr, (uint)_alignedLength, AsyncFlushPartialObjectLogCallback <TContext>, asyncResult);
// Wait for write to complete before resuming next write
asyncResult.done.WaitOne();
_objBuffer.Return();
}
else
{
asyncResult.freeBuffer2 = _objBuffer;
objlogDevice.WriteAsync(
(IntPtr)_objBuffer.aligned_pointer,
(int)(alignedDestinationAddress >> LogSegmentSizeBits),
(ulong)_objAddr, (uint)_alignedLength, callback, asyncResult);
}
}
// Finally write the hlog page
device.WriteAsync((IntPtr)buffer.aligned_pointer, alignedDestinationAddress,
numBytesToWrite, callback, asyncResult);
}
protected override void WriteAsync <TContext>(long flushPage, IOCompletionCallback callback, PageAsyncFlushResult <TContext> asyncResult)
{
WriteAsync((IntPtr)pointers[flushPage % BufferSize],
(ulong)(AlignedPageSizeBytes * flushPage),
(uint)AlignedPageSizeBytes,
callback,
asyncResult, device);
}
private void WriteAsync <TContext>(IntPtr alignedSourceAddress, ulong alignedDestinationAddress, uint numBytesToWrite,
IOCompletionCallback callback, PageAsyncFlushResult <TContext> asyncResult,
IDevice device, ISegmentedDevice objlogDevice)
{
if (!Key.HasObjectsToSerialize() && !Value.HasObjectsToSerialize())
{
device.WriteAsync(alignedSourceAddress, alignedDestinationAddress,
numBytesToWrite, callback, asyncResult);
return;
}
// need to write both page and object cache
asyncResult.count++;
MemoryStream ms = new MemoryStream();
var buffer = ioBufferPool.Get(PageSize);
Buffer.MemoryCopy((void *)alignedSourceAddress, buffer.aligned_pointer, numBytesToWrite, numBytesToWrite);
long ptr = (long)buffer.aligned_pointer;
List <long> addr = new List <long>();
asyncResult.freeBuffer1 = buffer;
// Correct for page 0 of HLOG
if (alignedDestinationAddress >> LogPageSizeBits == 0)
{
ptr += Constants.kFirstValidAddress;
}
while (ptr < (long)buffer.aligned_pointer + numBytesToWrite)
{
if (!Layout.GetInfo(ptr)->Invalid)
{
long pos = ms.Position;
Key *key = Layout.GetKey(ptr);
Key.Serialize(key, ms);
((AddressInfo *)key)->IsDiskAddress = true;
((AddressInfo *)key)->Address = pos;
((AddressInfo *)key)->Size = (int)(ms.Position - pos);
addr.Add((long)key);
pos = ms.Position;
Value *value = Layout.GetValue(ptr);
Value.Serialize(value, ms);
((AddressInfo *)value)->IsDiskAddress = true;
((AddressInfo *)value)->Address = pos;
((AddressInfo *)value)->Size = (int)(ms.Position - pos);
addr.Add((long)value);
}
ptr += Layout.GetPhysicalSize(ptr);
}
var s = ms.ToArray();
var objBuffer = ioBufferPool.Get(s.Length);
asyncResult.freeBuffer2 = objBuffer;
var alignedLength = (s.Length + (sectorSize - 1)) & ~(sectorSize - 1);
var objAddr = Interlocked.Add(ref segmentOffsets[(alignedDestinationAddress >> LogSegmentSizeBits) % SegmentBufferSize], alignedLength) - alignedLength;
fixed(void *src = s)
Buffer.MemoryCopy(src, objBuffer.aligned_pointer, s.Length, s.Length);
foreach (var address in addr)
{
*((long *)address) += objAddr;
}
objlogDevice.WriteAsync(
(IntPtr)objBuffer.aligned_pointer,
(int)(alignedDestinationAddress >> LogSegmentSizeBits),
(ulong)objAddr, (uint)alignedLength, callback, asyncResult);
device.WriteAsync((IntPtr)buffer.aligned_pointer, alignedDestinationAddress,
numBytesToWrite, callback, asyncResult);
}
