private void LoadSyntheticData() { Console.WriteLine("Loading synthetic data (uniform distribution)"); init_keys_ = new Key[kInitCount]; long val = 0; for (int idx = 0; idx < kInitCount; idx++) { init_keys_[idx] = new Key { value = val++ }; } Console.WriteLine("loaded " + kInitCount + " keys."); RandomGenerator generator = new RandomGenerator(); txn_keys_ = new Key[kTxnCount]; GCHandle handle2 = GCHandle.Alloc(txn_keys_, GCHandleType.Pinned); txn_keys_ptr = (Key *)handle2.AddrOfPinnedObject(); for (int idx = 0; idx < kTxnCount; idx++) { txn_keys_[idx] = new Key { value = (long)generator.Generate64(kInitCount) }; } Console.WriteLine("loaded " + kTxnCount + " txns."); }
public static int GetInitialPhysicalSize(Key *key, Input *input) { return (RecordInfo.GetLength() + Key.GetLength(default(Key *)) + Functions.InitialValueLength(key, input)); }
private void ClearPage(int page, bool pageZero) { if (Key.HasObjectsToSerialize() || Value.HasObjectsToSerialize()) { long ptr = (long)pointers[page]; int numBytes = PageSize; long endptr = ptr + numBytes; if (pageZero) { ptr += Constants.kFirstValidAddress; } List <long> addr = new List <long>(); while (ptr < endptr) { if (!Layout.GetInfo(ptr)->Invalid) { if (Key.HasObjectsToSerialize()) { Key *key = Layout.GetKey(ptr); Key.Free(key); } if (Value.HasObjectsToSerialize()) { Value *value = Layout.GetValue(ptr); Value.Free(value); } } ptr += Layout.GetPhysicalSize(ptr); } } Array.Clear(values[page], 0, values[page].Length); }
public static USlice GetData(Key *self) { if (self == null) { return(default(USlice)); } Contract.Assert((self->Header & Entry.FLAGS_DISPOSED) == 0, "Attempt to read a key that was disposed"); return(new USlice(&(self->Data), self->Size)); }
/// <summary>Copy an existing value to this page, and return the pointer to the copy</summary> /// <param name="value">Value that must be copied to this page</param> /// <returns>Pointer to the copy in this page</returns> public Key *TryAppend(Key *value) { Contract.Requires(value != null && Entry.GetObjectType(value) == EntryType.Value); uint rawSize = Key.SizeOf + value->Size; var entry = (Key *)TryAllocate(rawSize); if (entry == null) { return(null); // this page is full } UnmanagedHelpers.CopyUnsafe((byte *)entry, (byte *)value, rawSize); return(entry); }
public Status RMW(Key *key, Input *input, Context *userContext, long monotonicSerialNum) { var context = default(PendingContext); var internalStatus = InternalRMW(key, input, userContext, ref context); var status = default(Status); if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND) { status = (Status)internalStatus; } else { status = HandleOperationStatus(threadCtx, context, internalStatus); } threadCtx.serialNum = monotonicSerialNum; return(status); }
public unsafe void Add(ulong sequence, USlice userKey, USlice userValue) { // allocate the key var tmp = MemoryDatabaseHandler.PackUserKey(m_scratch, userKey); Key *key = m_keys.Append(tmp); Contract.Assert(key != null, "key == null"); // allocate the value uint size = userValue.Count; Value *value = m_values.Allocate(size, sequence, null, key); Contract.Assert(value != null, "value == null"); UnmanagedHelpers.CopyUnsafe(&(value->Data), userValue); key->Values = value; m_list.Add(new IntPtr(key)); }
/// <summary> /// Clear page /// </summary> /// <param name="ptr">From pointer</param> /// <param name="endptr">Until pointer</param> public void ClearPage(long ptr, long endptr) { while (ptr < endptr) { if (!Layout.GetInfo(ptr)->Invalid) { if (Key.HasObjectsToSerialize()) { Key *key = Layout.GetKey(ptr); Key.Free(key); } if (Value.HasObjectsToSerialize()) { Value *value = Layout.GetValue(ptr); Value.Free(value); } } ptr += Layout.GetPhysicalSize(ptr); } }
public override void Debug_Dump(bool detailed) { Contract.Requires(m_start != null && m_current != null); Key *current = (Key *)m_start; Key *end = (Key *)m_current; Trace.WriteLine(" # KeyPage: count=" + m_count.ToString("N0") + ", used=" + this.MemoryUsage.ToString("N0") + ", capacity=" + m_capacity.ToString("N0") + ", start=0x" + new IntPtr(m_start).ToString("X8") + ", end=0x" + new IntPtr(m_current).ToString("X8")); if (detailed) { while (current < end) { Trace.WriteLine(" - [" + Entry.GetObjectType(current).ToString() + "] 0x" + new IntPtr(current).ToString("X8") + " : " + current->Header.ToString("X8") + ", size=" + current->Size + ", h=0x" + current->HashCode.ToString("X4") + " : " + FdbKey.Dump(Key.GetData(current).ToSlice())); var value = current->Values; while (value != null) { Trace.WriteLine(" -> [" + Entry.GetObjectType(value) + "] 0x" + new IntPtr(value).ToString("X8") + " @ " + value->Sequence + " : " + Value.GetData(value).ToSlice().ToAsciiOrHexaString()); value = value->Previous; } current = Key.WalkNext(current); } } }
internal unsafe KeyboardState(IKeyboard keyboard) { Name = keyboard.Name; Index = keyboard.Index; IsConnected = keyboard.IsConnected; var keys = keyboard.SupportedKeys; _keyCount = _pressedKeyCount = keys.Count; _keys = (Key *)Marshal.AllocHGlobal(_keyCount * sizeof(Key)); _pressedKeys = (Key *)Marshal.AllocHGlobal(_keyCount * sizeof(Key)); _pressedKeyCount = 0; for (var i = 0; i < _keyCount; i++) { _keys[i] = keys[i]; if (keyboard.IsKeyPressed(keys[i])) { _pressedKeys[_pressedKeyCount++] = keys[i]; } } _keys = (Key *)Marshal.ReAllocHGlobal((IntPtr)_pressedKeys, (IntPtr)(_pressedKeyCount * sizeof(Key))); }
public Status RMW(Key *key, Input *input, Context *userContext, long monotonicSerialNum) { /* * if (((RecordInfo*)hlog.GetPhysicalAddress(0))->Invalid != true) * { * Debugger.Break(); * }*/ var context = default(PendingContext); var internalStatus = InternalRMW(key, input, userContext, ref context); var status = default(Status); if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND) { status = (Status)internalStatus; } else { status = HandleOperationStatus(threadCtx, context, internalStatus); } threadCtx.serialNum = monotonicSerialNum; return(status); }
public Status Upsert(Key *key, Value *desiredValue, Context *userContext, long monotonicSerialNum) { /* * Console.WriteLine(hlog.GetPhysicalAddress(0)); * if (((RecordInfo*)hlog.GetPhysicalAddress(0))->Invalid != true) * { * Debugger.Break(); * }*/ var context = default(PendingContext); var internalStatus = InternalUpsert(key, desiredValue, userContext, ref context); var status = default(Status); if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND) { status = (Status)internalStatus; } else { status = HandleOperationStatus(threadCtx, context, internalStatus); } threadCtx.serialNum = monotonicSerialNum; return(status); }
/// <summary> /// Serialize part of page to stream /// </summary> /// <param name="ptr">From pointer</param> /// <param name="untilptr">Until pointer</param> /// <param name="stream">Stream</param> /// <param name="objectBlockSize">Size of blocks to serialize in chunks of</param> /// <param name="addr">List of addresses that need to be updated with offsets</param> public void Serialize(ref long ptr, long untilptr, Stream stream, int objectBlockSize, out List <long> addr) { addr = new List <long>(); while (ptr < untilptr) { if (!Layout.GetInfo(ptr)->Invalid) { long pos = stream.Position; if (Key.HasObjectsToSerialize()) { Key *key = Layout.GetKey(ptr); Key.Serialize(key, stream); ((AddressInfo *)key)->Address = pos; ((AddressInfo *)key)->Size = (int)(stream.Position - pos); addr.Add((long)key); } if (Value.HasObjectsToSerialize()) { pos = stream.Position; Value *value = Layout.GetValue(ptr); Value.Serialize(value, stream); ((AddressInfo *)value)->Address = pos; ((AddressInfo *)value)->Size = (int)(stream.Position - pos); addr.Add((long)value); } } ptr += Layout.GetPhysicalSize(ptr); if (stream.Position > objectBlockSize) { return; } } }
internal bool GetLogicalAddress(Key *key, out long logicalAddress) { var bucket = default(HashBucket *); var slot = default(int); logicalAddress = Constants.kInvalidAddress; var physicalAddress = default(long); var info = default(RecordInfo *); var hash = Key.GetHashCode(key); var tag = (ushort)((ulong)hash >> Constants.kHashTagShift); var entry = default(HashBucketEntry); var tagExists = FindTag(hash, tag, ref bucket, ref slot, ref entry); if (tagExists) { logicalAddress = entry.word & Constants.kAddressMask; Debug.Assert(logicalAddress != 0); if (logicalAddress >= hlog.HeadAddress) { physicalAddress = hlog.GetPhysicalAddress(logicalAddress); if (!Key.Equals(key, Layout.GetKey(physicalAddress))) { logicalAddress = Layout.GetInfo(physicalAddress)->PreviousAddress; TraceBackForKeyMatch(key, logicalAddress, hlog.HeadAddress, out logicalAddress, out physicalAddress); } } } if (logicalAddress < hlog.HeadAddress && logicalAddress != Constants.kInvalidAddress) { return(false); } return(true); }
public static bool StillAlive(Key *self, ulong sequence) { if (self == null) { return(false); } if ((self->Header & Entry.FLAGS_UNREACHABLE) != 0) { // we have been marked as dead var value = self->Values; if (value == null) { return(false); } // check if the last value is a deletion? if (value->Sequence <= sequence && (value->Header & Value.FLAGS_DELETION) != 0) { // it is deleted return(false); } } return(true); }
public static void InitialUpdater(Key *key, Input *input, Value *value) { value->value = ((Value *)input)->value; }
public static int GetPhysicalSize(Key *key) { return(kSizeInBytes); }
public static void Copy(Key *src, Key *dst) { Utility.Copy((byte *)src, (byte *)dst, kSizeInBytes); }
public static long GetHash(Key *key) { return(Utility.HashBytes((byte *)key, kSizeInBytes)); }
public static bool IsEqual(Key *k1, Key *k2) { return(Utility.IsEqual((byte *)k1, (byte *)k2, kSizeInBytes)); }
public static int InitialValueLength(Key *key, Input *input) { return(Value.GetLength(default(Value *))); }
public static void InPlaceUpdater(Key *key, Input *input, Value *addr) { Interlocked.Add(ref addr->value, ((Value *)input)->value); }
public static void CopyUpdater(Key *key, Input *input, Value *oldValue, Value *newValue) { newValue->value = oldValue->value + ((Value *)input)->value; }
public static bool IsDisposed(Key *self) { return((self->Header & Entry.FLAGS_DISPOSED) != 0); }
private void AsyncReadPageCallback <TContext>(uint errorCode, uint numBytes, NativeOverlapped *overlap) { if (errorCode != 0) { Trace.TraceError("OverlappedStream GetQueuedCompletionStatus error: {0}", errorCode); } PageAsyncReadResult <TContext> result = (PageAsyncReadResult <TContext>)Overlapped.Unpack(overlap).AsyncResult; if (Interlocked.Decrement(ref result.count) == 1) { // We will be issuing another I/O, so free this overlap Overlapped.Free(overlap); long ptr = (long)pointers[result.page % BufferSize]; // Correct for page 0 of HLOG if (result.page == 0) { ptr += Constants.kFirstValidAddress; } long minObjAddress = long.MaxValue; long maxObjAddress = long.MinValue; while (ptr < (long)pointers[result.page % BufferSize] + PageSize) { if (!Layout.GetInfo(ptr)->Invalid) { if (Key.HasObjectsToSerialize()) { Key *key = Layout.GetKey(ptr); var addr = ((AddressInfo *)key)->Address; if (addr < minObjAddress) { minObjAddress = addr; } addr += ((AddressInfo *)key)->Size; if (addr > maxObjAddress) { maxObjAddress = addr; } } if (Value.HasObjectsToSerialize()) { Value *value = Layout.GetValue(ptr); var addr = ((AddressInfo *)value)->Address; if (addr < minObjAddress) { minObjAddress = addr; } addr += ((AddressInfo *)value)->Size; if (addr > maxObjAddress) { maxObjAddress = addr; } } } ptr += Layout.GetPhysicalSize(ptr); } // Object log fragment should be aligned by construction Debug.Assert(minObjAddress % sectorSize == 0); var to_read = (int)(maxObjAddress - minObjAddress); var objBuffer = ioBufferPool.Get(to_read); result.freeBuffer1 = objBuffer; var alignedLength = (to_read + (sectorSize - 1)) & ~(sectorSize - 1); // Request objects from objlog result.objlogDevice.ReadAsync( (int)(result.page >> (LogSegmentSizeBits - LogPageSizeBits)), (ulong)minObjAddress, (IntPtr)objBuffer.aligned_pointer, (uint)alignedLength, AsyncReadPageCallback <TContext>, result); } else { // Load objects from buffer into memory long ptr = (long)pointers[result.page % BufferSize]; // Correct for page 0 of HLOG if (result.page == 0) { ptr += Constants.kFirstValidAddress; } MemoryStream ms = new MemoryStream(result.freeBuffer1.buffer); ms.Seek(result.freeBuffer1.offset + result.freeBuffer1.valid_offset, SeekOrigin.Begin); while (ptr < (long)pointers[result.page % BufferSize] + PageSize) { if (!Layout.GetInfo(ptr)->Invalid) { if (Key.HasObjectsToSerialize()) { Key.Deserialize(Layout.GetKey(ptr), ms); } if (Value.HasObjectsToSerialize()) { Value.Deserialize(Layout.GetValue(ptr), ms); } } ptr += Layout.GetPhysicalSize(ptr); } ms.Dispose(); result.Free(); // Call the "real" page read callback result.callback(errorCode, numBytes, overlap); } }
private void WriteAsync <TContext>(IntPtr alignedSourceAddress, ulong alignedDestinationAddress, uint numBytesToWrite, IOCompletionCallback callback, PageAsyncFlushResult <TContext> asyncResult, IDevice device, ISegmentedDevice objlogDevice) { if (!Key.HasObjectsToSerialize() && !Value.HasObjectsToSerialize()) { device.WriteAsync(alignedSourceAddress, alignedDestinationAddress, numBytesToWrite, callback, asyncResult); return; } // need to write both page and object cache asyncResult.count++; MemoryStream ms = new MemoryStream(); var buffer = ioBufferPool.Get(PageSize); Buffer.MemoryCopy((void *)alignedSourceAddress, buffer.aligned_pointer, numBytesToWrite, numBytesToWrite); long ptr = (long)buffer.aligned_pointer; List <long> addr = new List <long>(); asyncResult.freeBuffer1 = buffer; // Correct for page 0 of HLOG if (alignedDestinationAddress >> LogPageSizeBits == 0) { ptr += Constants.kFirstValidAddress; } while (ptr < (long)buffer.aligned_pointer + numBytesToWrite) { if (!Layout.GetInfo(ptr)->Invalid) { long pos = ms.Position; Key *key = Layout.GetKey(ptr); Key.Serialize(key, ms); ((AddressInfo *)key)->IsDiskAddress = true; ((AddressInfo *)key)->Address = pos; ((AddressInfo *)key)->Size = (int)(ms.Position - pos); addr.Add((long)key); pos = ms.Position; Value *value = Layout.GetValue(ptr); Value.Serialize(value, ms); ((AddressInfo *)value)->IsDiskAddress = true; ((AddressInfo *)value)->Address = pos; ((AddressInfo *)value)->Size = (int)(ms.Position - pos); addr.Add((long)value); } ptr += Layout.GetPhysicalSize(ptr); } var s = ms.ToArray(); var objBuffer = ioBufferPool.Get(s.Length); asyncResult.freeBuffer2 = objBuffer; var alignedLength = (s.Length + (sectorSize - 1)) & ~(sectorSize - 1); var objAddr = Interlocked.Add(ref segmentOffsets[(alignedDestinationAddress >> LogSegmentSizeBits) % SegmentBufferSize], alignedLength) - alignedLength; fixed(void *src = s) Buffer.MemoryCopy(src, objBuffer.aligned_pointer, s.Length, s.Length); foreach (var address in addr) { *((long *)address) += objAddr; } objlogDevice.WriteAsync( (IntPtr)objBuffer.aligned_pointer, (int)(alignedDestinationAddress >> LogSegmentSizeBits), (ulong)objAddr, (uint)alignedLength, callback, asyncResult); device.WriteAsync((IntPtr)buffer.aligned_pointer, alignedDestinationAddress, numBytesToWrite, callback, asyncResult); }
private void LoadDataFromFile(string filePath) { string init_filename = filePath + "\\load_" + distribution + "_250M_raw.dat"; string txn_filename = filePath + "\\run_" + distribution + "_250M_1000M_raw.dat"; long count = 0; using (FileStream stream = File.Open(init_filename, FileMode.Open, FileAccess.Read, FileShare.Read)) { Console.WriteLine("loading keys from " + init_filename + " into memory..."); init_keys_ = new Key[kInitCount]; byte[] chunk = new byte[kFileChunkSize]; GCHandle chunk_handle = GCHandle.Alloc(chunk, GCHandleType.Pinned); byte * chunk_ptr = (byte *)chunk_handle.AddrOfPinnedObject(); long offset = 0; while (true) { stream.Position = offset; int size = stream.Read(chunk, 0, kFileChunkSize); for (int idx = 0; idx < size; idx += Key.kSizeInBytes) { init_keys_[count] = *((Key *)(chunk_ptr + idx)); ++count; } if (size == kFileChunkSize) { offset += kFileChunkSize; } else { break; } if (count == kInitCount) { break; } } if (count != kInitCount) { throw new InvalidDataException("Init file load fail!"); } } Console.WriteLine("loaded " + kInitCount + " keys."); using (FileStream stream = File.Open(txn_filename, FileMode.Open, FileAccess.Read, FileShare.Read)) { byte[] chunk = new byte[kFileChunkSize]; GCHandle chunk_handle = GCHandle.Alloc(chunk, GCHandleType.Pinned); byte * chunk_ptr = (byte *)chunk_handle.AddrOfPinnedObject(); Console.WriteLine("loading txns from " + txn_filename + " into memory..."); txn_keys_ = new Key[kTxnCount]; GCHandle handle2 = GCHandle.Alloc(txn_keys_, GCHandleType.Pinned); txn_keys_ptr = (Key *)handle2.AddrOfPinnedObject(); count = 0; long offset = 0; while (true) { stream.Position = offset; int size = stream.Read(chunk, 0, kFileChunkSize); for (int idx = 0; idx < size; idx += Key.kSizeInBytes) { txn_keys_[count] = *((Key *)(chunk_ptr + idx)); ++count; } if (size == kFileChunkSize) { offset += kFileChunkSize; } else { break; } if (count == kTxnCount) { break; } } if (count != kTxnCount) { throw new InvalidDataException("Txn file load fail!" + count + ":" + kTxnCount); } } Console.WriteLine("loaded " + kTxnCount + " txns."); }
public Status Delete(Key *key, Context *userContext, long monotonicSerialNum) { throw new NotImplementedException(); }
public static void SingleReader(Key *key, Input *input, Value *value, Output *dst) { Value.Copy(value, (Value *)dst); }
/// <summary>Return the address of the following value in the heap</summary> internal static Key *WalkNext(Key *self) { Contract.Requires(self != null && Entry.GetObjectType(self) == EntryType.Key); return((Key *)Entry.Align((byte *)self + Key.SizeOf + self->Size)); }