private bool RetrievedObjects(byte *record, AsyncIOContext ctx)
{
if (!Key.HasObjectsToSerialize() && !Value.HasObjectsToSerialize())
{
return(true);
}
if (ctx.objBuffer.buffer == null)
{
// Issue IO for objects
long startAddress = -1;
int numBytes = 0;
if (Key.HasObjectsToSerialize())
{
var x = (AddressInfo *)Layout.GetKey((long)record);
if (x->IsDiskAddress)
{
numBytes += x->Size;
startAddress = x->Address;
}
}
if (Value.HasObjectsToSerialize())
{
var x = (AddressInfo *)Layout.GetValue((long)record);
if (x->IsDiskAddress)
{
numBytes += x->Size;
if (startAddress == -1)
{
startAddress = x->Address;
}
}
}
AsyncGetFromDisk(startAddress, numBytes,
AsyncGetFromDiskCallback, ctx, ctx.record);
return(false);
}
// Parse the key and value objects
MemoryStream ms = new MemoryStream(ctx.objBuffer.buffer);
ms.Seek(ctx.objBuffer.offset + ctx.objBuffer.valid_offset, SeekOrigin.Begin);
Key.Deserialize(Layout.GetKey((long)record), ms);
Value.Deserialize(Layout.GetValue((long)record), ms);
ctx.objBuffer.Return();
return(true);
}
public void AsyncReadPagesFromDevice <TContext>(
long readPageStart,
int numPages,
IOCompletionCallback callback,
TContext context,
out CountdownEvent completed,
long devicePageOffset = 0,
IDevice device = null)
{
var usedDevice = (device != null) ? device : this.device;
ISegmentedDevice usedObjlogDevice = null;
if (Key.HasObjectsToSerialize() || Value.HasObjectsToSerialize())
{
usedObjlogDevice = (device != null) ? CreateObjectLogDevice(device) : objlogDevice;
}
completed = new CountdownEvent(numPages);
for (long readPage = readPageStart; readPage < (readPageStart + numPages); readPage++)
{
int pageIndex = (int)(readPage % BufferSize);
if (values[pageIndex] == null)
{
// Allocate a new page
AllocatePage(pageIndex);
}
else
{
ClearPage(pageIndex, readPage == 0);
}
var asyncResult = new PageAsyncReadResult <TContext>()
{
page = readPage,
context = context,
handle = completed,
count = 1
};
ulong offsetInFile = (ulong)(AlignedPageSizeBytes * readPage);
if (device != null)
{
offsetInFile = (ulong)(AlignedPageSizeBytes * (readPage - devicePageOffset));
}
ReadAsync(offsetInFile, pointers[pageIndex], PageSize, callback, asyncResult, usedDevice, usedObjlogDevice);
}
}
private void ReadAsync <TContext>(
ulong alignedSourceAddress, IntPtr alignedDestinationAddress, uint aligned_read_length,
IOCompletionCallback callback, PageAsyncReadResult <TContext> asyncResult, IDevice device, ISegmentedDevice objlogDevice)
{
if (!Key.HasObjectsToSerialize() && !Value.HasObjectsToSerialize())
{
device.ReadAsync(alignedSourceAddress, alignedDestinationAddress,
aligned_read_length, callback, asyncResult);
return;
}
asyncResult.callback = callback;
asyncResult.count++;
asyncResult.objlogDevice = objlogDevice;
device.ReadAsync(alignedSourceAddress, alignedDestinationAddress,
aligned_read_length, AsyncReadPageCallback <TContext>, asyncResult);
}
/// <summary>
/// Deseialize part of page from stream
/// </summary>
/// <param name="ptr">From pointer</param>
/// <param name="untilptr">Until pointer</param>
/// <param name="stream">Stream</param>
public void Deserialize(long ptr, long untilptr, Stream stream)
{
while (ptr < untilptr)
{
if (!Layout.GetInfo(ptr)->Invalid)
{
if (Key.HasObjectsToSerialize())
{
Key.Deserialize(Layout.GetKey(ptr), stream);
}
if (Value.HasObjectsToSerialize())
{
Value.Deserialize(Layout.GetValue(ptr), stream);
}
}
ptr += Layout.GetPhysicalSize(ptr);
}
}
/// <summary>
/// IOCompletion callback for page read
/// </summary>
/// <param name="errorCode"></param>
/// <param name="numBytes"></param>
/// <param name="overlap"></param>
private void AsyncReadPageFromDeviceCallback(uint errorCode, uint numBytes, NativeOverlapped *overlap)
{
if (errorCode != 0)
{
System.Diagnostics.Trace.TraceError("OverlappedStream GetQueuedCompletionStatus error: {0}", errorCode);
}
PageAsyncFlushResult result = (PageAsyncFlushResult)Overlapped.Unpack(overlap).AsyncResult;
if (result.handle != null)
{
result.handle.Signal();
}
Overlapped.Free(overlap);
if (Key.HasObjectsToSerialize() || Value.HasObjectsToSerialize())
{
throw new NotImplementedException("Recovery of object log not yet supported");
}
}
/// <summary>
/// Clear page
/// </summary>
/// <param name="ptr">From pointer</param>
/// <param name="endptr">Until pointer</param>
public void ClearPage(long ptr, long endptr)
{
while (ptr < endptr)
{
if (!Layout.GetInfo(ptr)->Invalid)
{
if (Key.HasObjectsToSerialize())
{
Key *key = Layout.GetKey(ptr);
Key.Free(key);
}
if (Value.HasObjectsToSerialize())
{
Value *value = Layout.GetValue(ptr);
Value.Free(value);
}
}
ptr += Layout.GetPhysicalSize(ptr);
}
}
/// <summary>
/// Serialize part of page to stream
/// </summary>
/// <param name="ptr">From pointer</param>
/// <param name="untilptr">Until pointer</param>
/// <param name="stream">Stream</param>
/// <param name="objectBlockSize">Size of blocks to serialize in chunks of</param>
/// <param name="addr">List of addresses that need to be updated with offsets</param>
public void Serialize(ref long ptr, long untilptr, Stream stream, int objectBlockSize, out List <long> addr)
{
addr = new List <long>();
while (ptr < untilptr)
{
if (!Layout.GetInfo(ptr)->Invalid)
{
long pos = stream.Position;
if (Key.HasObjectsToSerialize())
{
Key *key = Layout.GetKey(ptr);
Key.Serialize(key, stream);
((AddressInfo *)key)->Address = pos;
((AddressInfo *)key)->Size = (int)(stream.Position - pos);
addr.Add((long)key);
}
if (Value.HasObjectsToSerialize())
{
pos = stream.Position;
Value *value = Layout.GetValue(ptr);
Value.Serialize(value, stream);
((AddressInfo *)value)->Address = pos;
((AddressInfo *)value)->Size = (int)(stream.Position - pos);
addr.Add((long)value);
}
}
ptr += Layout.GetPhysicalSize(ptr);
if (stream.Position > objectBlockSize)
{
return;
}
}
}
private void AsyncGetFromDiskCallback(
uint errorCode,
uint numBytes,
NativeOverlapped *overlap)
{
if (errorCode != 0)
{
Trace.TraceError("OverlappedStream GetQueuedCompletionStatus error: {0}", errorCode);
}
var result = (AsyncGetFromDiskResult <AsyncIOContext>)Overlapped.Unpack(overlap).AsyncResult;
Interlocked.Decrement(ref numPendingReads);
var ctx = result.context;
var record = ctx.record.GetValidPointer();
if (Layout.HasTotalRecord(record, ctx.record.available_bytes, out int requiredBytes))
{
//We have the complete record.
if (RetrievedObjects(record, ctx))
{
if (Key.Equals((Key *)ctx.key, Layout.GetKey((long)record)))
{
//The keys are same, so I/O is complete
// ctx.record = result.record;
ctx.callbackQueue.Add(ctx);
}
else
{
var oldAddress = ctx.logicalAddress;
//keys are not same. I/O is not complete
ctx.logicalAddress = ((RecordInfo *)record)->PreviousAddress;
if (ctx.logicalAddress != Constants.kInvalidAddress)
{
// Delete key, value, record
if (Key.HasObjectsToSerialize())
{
var physicalAddress = (long)ctx.record.GetValidPointer();
Key.Free(Layout.GetKey(physicalAddress));
}
if (Value.HasObjectsToSerialize())
{
var physicalAddress = (long)ctx.record.GetValidPointer();
Value.Free(Layout.GetValue(physicalAddress));
}
ctx.record.Return();
ctx.record = ctx.objBuffer = default(SectorAlignedMemory);
AsyncGetFromDisk(ctx.logicalAddress, requiredBytes, AsyncGetFromDiskCallback, ctx);
}
else
{
ctx.callbackQueue.Add(ctx);
}
}
}
}
else
{
ctx.record.Return();
AsyncGetFromDisk(ctx.logicalAddress, requiredBytes, AsyncGetFromDiskCallback, ctx);
}
Overlapped.Free(overlap);
}
internal void InternalContinuePendingRequestAndCallback(
ExecutionContext ctx,
AsyncIOContext request)
{
var handleLatches = false;
if ((ctx.version < threadCtx.version) // Thread has already shifted to (v+1)
||
(threadCtx.phase == Phase.PREPARE)) // Thread still in version v, but acquired shared-latch
{
handleLatches = true;
}
if (ctx.ioPendingRequests.TryGetValue(request.id, out PendingContext pendingContext))
{
var status = default(Status);
var internalStatus = default(OperationStatus);
// Remove from pending dictionary
ctx.ioPendingRequests.Remove(request.id);
// Issue the continue command
if (pendingContext.type == OperationType.READ)
{
internalStatus = InternalContinuePendingRead(ctx, request, ref pendingContext);
}
else
{
internalStatus = InternalContinuePendingRMW(ctx, request, ref pendingContext);;
}
// Delete key, value, record
if (Key.HasObjectsToSerialize())
{
var physicalAddress = (long)request.record.GetValidPointer();
Key.Free(Layout.GetKey(physicalAddress));
}
if (Value.HasObjectsToSerialize())
{
var physicalAddress = (long)request.record.GetValidPointer();
Value.Free(Layout.GetValue(physicalAddress));
}
request.record.Return();
// Handle operation status
if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND)
{
status = (Status)internalStatus;
}
else
{
status = HandleOperationStatus(ctx, pendingContext, internalStatus);
}
// If done, callback user code
if (status == Status.OK || status == Status.NOTFOUND)
{
if (handleLatches)
{
ReleaseSharedLatch(pendingContext.key);
}
if (pendingContext.type == OperationType.READ)
{
Functions.ReadCompletionCallback(pendingContext.key,
pendingContext.input,
pendingContext.output,
pendingContext.userContext,
status);
}
else
{
Functions.RMWCompletionCallback(pendingContext.key,
pendingContext.input,
pendingContext.userContext,
status);
}
}
}
}
private void AsyncReadPageCallback <TContext>(uint errorCode, uint numBytes, NativeOverlapped *overlap)
{
if (errorCode != 0)
{
Trace.TraceError("OverlappedStream GetQueuedCompletionStatus error: {0}", errorCode);
}
PageAsyncReadResult <TContext> result = (PageAsyncReadResult <TContext>)Overlapped.Unpack(overlap).AsyncResult;
if (Interlocked.Decrement(ref result.count) == 1)
{
// We will be issuing another I/O, so free this overlap
Overlapped.Free(overlap);
long ptr = (long)pointers[result.page % BufferSize];
// Correct for page 0 of HLOG
if (result.page == 0)
{
ptr += Constants.kFirstValidAddress;
}
long minObjAddress = long.MaxValue;
long maxObjAddress = long.MinValue;
while (ptr < (long)pointers[result.page % BufferSize] + PageSize)
{
if (!Layout.GetInfo(ptr)->Invalid)
{
if (Key.HasObjectsToSerialize())
{
Key *key = Layout.GetKey(ptr);
var addr = ((AddressInfo *)key)->Address;
if (addr < minObjAddress)
{
minObjAddress = addr;
}
addr += ((AddressInfo *)key)->Size;
if (addr > maxObjAddress)
{
maxObjAddress = addr;
}
}
if (Value.HasObjectsToSerialize())
{
Value *value = Layout.GetValue(ptr);
var addr = ((AddressInfo *)value)->Address;
if (addr < minObjAddress)
{
minObjAddress = addr;
}
addr += ((AddressInfo *)value)->Size;
if (addr > maxObjAddress)
{
maxObjAddress = addr;
}
}
}
ptr += Layout.GetPhysicalSize(ptr);
}
// Object log fragment should be aligned by construction
Debug.Assert(minObjAddress % sectorSize == 0);
var to_read = (int)(maxObjAddress - minObjAddress);
var objBuffer = ioBufferPool.Get(to_read);
result.freeBuffer1 = objBuffer;
var alignedLength = (to_read + (sectorSize - 1)) & ~(sectorSize - 1);
// Request objects from objlog
result.objlogDevice.ReadAsync(
(int)(result.page >> (LogSegmentSizeBits - LogPageSizeBits)),
(ulong)minObjAddress,
(IntPtr)objBuffer.aligned_pointer, (uint)alignedLength, AsyncReadPageCallback <TContext>, result);
}
else
{
// Load objects from buffer into memory
long ptr = (long)pointers[result.page % BufferSize];
// Correct for page 0 of HLOG
if (result.page == 0)
{
ptr += Constants.kFirstValidAddress;
}
MemoryStream ms = new MemoryStream(result.freeBuffer1.buffer);
ms.Seek(result.freeBuffer1.offset + result.freeBuffer1.valid_offset, SeekOrigin.Begin);
while (ptr < (long)pointers[result.page % BufferSize] + PageSize)
{
if (!Layout.GetInfo(ptr)->Invalid)
{
if (Key.HasObjectsToSerialize())
{
Key.Deserialize(Layout.GetKey(ptr), ms);
}
if (Value.HasObjectsToSerialize())
{
Value.Deserialize(Layout.GetValue(ptr), ms);
}
}
ptr += Layout.GetPhysicalSize(ptr);
}
ms.Dispose();
result.Free();
// Call the "real" page read callback
result.callback(errorCode, numBytes, overlap);
}
}
private void WriteAsync <TContext>(IntPtr alignedSourceAddress, ulong alignedDestinationAddress, uint numBytesToWrite,
IOCompletionCallback callback, PageAsyncFlushResult <TContext> asyncResult,
IDevice device, ISegmentedDevice objlogDevice)
{
if (!Key.HasObjectsToSerialize() && !Value.HasObjectsToSerialize())
{
device.WriteAsync(alignedSourceAddress, alignedDestinationAddress,
numBytesToWrite, callback, asyncResult);
return;
}
// need to write both page and object cache
asyncResult.count++;
MemoryStream ms = new MemoryStream();
var buffer = ioBufferPool.Get(PageSize);
Buffer.MemoryCopy((void *)alignedSourceAddress, buffer.aligned_pointer, numBytesToWrite, numBytesToWrite);
long ptr = (long)buffer.aligned_pointer;
List <long> addr = new List <long>();
asyncResult.freeBuffer1 = buffer;
// Correct for page 0 of HLOG
if (alignedDestinationAddress >> LogPageSizeBits == 0)
{
ptr += Constants.kFirstValidAddress;
}
while (ptr < (long)buffer.aligned_pointer + numBytesToWrite)
{
if (!Layout.GetInfo(ptr)->Invalid)
{
long pos = ms.Position;
Key *key = Layout.GetKey(ptr);
Key.Serialize(key, ms);
((AddressInfo *)key)->IsDiskAddress = true;
((AddressInfo *)key)->Address = pos;
((AddressInfo *)key)->Size = (int)(ms.Position - pos);
addr.Add((long)key);
pos = ms.Position;
Value *value = Layout.GetValue(ptr);
Value.Serialize(value, ms);
((AddressInfo *)value)->IsDiskAddress = true;
((AddressInfo *)value)->Address = pos;
((AddressInfo *)value)->Size = (int)(ms.Position - pos);
addr.Add((long)value);
}
ptr += Layout.GetPhysicalSize(ptr);
}
var s = ms.ToArray();
var objBuffer = ioBufferPool.Get(s.Length);
asyncResult.freeBuffer2 = objBuffer;
var alignedLength = (s.Length + (sectorSize - 1)) & ~(sectorSize - 1);
var objAddr = Interlocked.Add(ref segmentOffsets[(alignedDestinationAddress >> LogSegmentSizeBits) % SegmentBufferSize], alignedLength) - alignedLength;
fixed(void *src = s)
Buffer.MemoryCopy(src, objBuffer.aligned_pointer, s.Length, s.Length);
foreach (var address in addr)
{
*((long *)address) += objAddr;
}
objlogDevice.WriteAsync(
(IntPtr)objBuffer.aligned_pointer,
(int)(alignedDestinationAddress >> LogSegmentSizeBits),
(ulong)objAddr, (uint)alignedLength, callback, asyncResult);
device.WriteAsync((IntPtr)buffer.aligned_pointer, alignedDestinationAddress,
numBytesToWrite, callback, asyncResult);
}
protected void AsyncGetFromDiskCallback(
uint errorCode,
uint numBytes,
NativeOverlapped *overlap)
{
//Debugger.Break();
var result = (AsyncGetFromDiskResult <AsyncIOContext>)Overlapped.Unpack(overlap).AsyncResult;
try
{
if (errorCode != 0)
{
Trace.TraceError("OverlappedStream GetQueuedCompletionStatus error: {0}", errorCode);
}
}
catch (Exception ex)
{
Trace.TraceError("Completion Callback error, {0}", ex.Message);
}
finally
{
Interlocked.Decrement(ref numPendingReads);
var ctx = result.context;
var record = ctx.record.GetValidPointer();
if (Layout.HasTotalRecord(record, ctx.record.available_bytes, out int requiredBytes))
{
//We have the complete record.
if (RetrievedObjects(record, ctx))
{
if (Key.Equals(ctx.key, Layout.GetKey((long)record)))
{
//The keys are same, so I/O is complete
// ctx.record = result.record;
ctx.callbackQueue.Add(ctx);
}
else
{
var oldAddress = ctx.logicalAddress;
//keys are not same. I/O is not complete
ctx.logicalAddress = ((RecordInfo *)record)->PreviousAddress;
if (ctx.logicalAddress != Constants.kInvalidAddress)
{
// Delete key, value, record
if (Key.HasObjectsToSerialize())
{
var physicalAddress = (long)ctx.record.GetValidPointer();
Key.Free(Layout.GetKey(physicalAddress));
}
if (Value.HasObjectsToSerialize())
{
var physicalAddress = (long)ctx.record.GetValidPointer();
Value.Free(Layout.GetValue(physicalAddress));
}
ctx.record.Return();
AsyncGetFromDisk(ctx.logicalAddress, requiredBytes, AsyncGetFromDiskCallback, ctx);
}
else
{
//Console.WriteLine("Lookup Address = " + oldAddress);
//Console.WriteLine("RecordInfo: " + RecordInfo.ToString((RecordInfo*)record));
// Console.WriteLine("Record not found. Looking for: " + ctx.key->value + " found: " + Layout.GetKey((long)record)->value + " req bytes: " + requiredBytes);
ctx.callbackQueue.Add(ctx);
}
}
}
}
else
{
ctx.record.Return();
AsyncGetFromDisk(ctx.logicalAddress, requiredBytes, AsyncGetFromDiskCallback, ctx);
}
Overlapped.Free(overlap);
}
}
/// <summary>
/// Whether KVS has objects to serialize/deserialize
/// </summary>
/// <returns></returns>
public bool HasObjects()
{
return(Key.HasObjectsToSerialize() || Value.HasObjectsToSerialize());
}
/// <summary>
/// Get location and range of object log addresses for specified log page
/// </summary>
/// <param name="ptr"></param>
/// <param name="untilptr"></param>
/// <param name="objectBlockSize"></param>
/// <param name="startptr"></param>
/// <param name="size"></param>
public void GetObjectInfo(ref long ptr, long untilptr, int objectBlockSize, out long startptr, out long size)
{
long minObjAddress = long.MaxValue;
long maxObjAddress = long.MinValue;
while (ptr < untilptr)
{
if (!Layout.GetInfo(ptr)->Invalid)
{
if (Key.HasObjectsToSerialize())
{
Key *key = Layout.GetKey(ptr);
var addr = ((AddressInfo *)key)->Address;
// If object pointer is greater than kObjectSize from starting object pointer
if (minObjAddress != long.MaxValue && (addr - minObjAddress > objectBlockSize))
{
break;
}
if (addr < minObjAddress)
{
minObjAddress = addr;
}
addr += ((AddressInfo *)key)->Size;
if (addr > maxObjAddress)
{
maxObjAddress = addr;
}
}
if (Value.HasObjectsToSerialize())
{
Value *value = Layout.GetValue(ptr);
var addr = ((AddressInfo *)value)->Address;
// If object pointer is greater than kObjectSize from starting object pointer
if (minObjAddress != long.MaxValue && (addr - minObjAddress > objectBlockSize))
{
break;
}
if (addr < minObjAddress)
{
minObjAddress = addr;
}
addr += ((AddressInfo *)value)->Size;
if (addr > maxObjAddress)
{
maxObjAddress = addr;
}
}
}
ptr += Layout.GetPhysicalSize(ptr);
}
// Handle the case where no objects are to be written
if (minObjAddress == long.MaxValue && maxObjAddress == long.MinValue)
{
minObjAddress = 0;
maxObjAddress = 0;
}
startptr = minObjAddress;
size = maxObjAddress - minObjAddress;
}
