public VariableLengthBlittableAllocator(LogSettings settings, VariableLengthStructSettings <Key, Value> vlSettings, IFasterEqualityComparer <Key> comparer, Action <long, long> evictCallback = null, LightEpoch epoch = null, Action <CommitInfo> flushCallback = null) : base(settings, comparer, evictCallback, epoch, flushCallback) { values = new byte[BufferSize][]; handles = new GCHandle[BufferSize]; pointers = new long[BufferSize]; ptrHandle = GCHandle.Alloc(pointers, GCHandleType.Pinned); nativePointers = (long *)ptrHandle.AddrOfPinnedObject(); KeyLength = vlSettings.keyLength; ValueLength = vlSettings.valueLength; if (KeyLength == null) { fixedSizeKey = true; KeyLength = new FixedLengthStruct <Key>(); } if (ValueLength == null) { fixedSizeValue = true; ValueLength = new FixedLengthStruct <Value>(); } }
public long Compact(long untilAddress, bool shiftBeginAddress) { if (allocator is VariableLengthBlittableAllocator <Key, Value> varLen) { if (typeof(Key).IsGenericType && (typeof(Key).GetGenericTypeDefinition() == typeof(ReadOnlyMemory <>)) && Utility.IsBlittableType(typeof(Key).GetGenericArguments()[0]) && typeof(Value).IsGenericType && (typeof(Value).GetGenericTypeDefinition() == typeof(Memory <>)) && Utility.IsBlittableType(typeof(Value).GetGenericArguments()[0])) { MethodInfo method = GetType().GetMethod("CompactReadOnly", BindingFlags.NonPublic | BindingFlags.Instance); MethodInfo generic = method.MakeGenericMethod(typeof(Key).GetGenericArguments()[0]); return((long)generic.Invoke(this, new object[] { untilAddress, shiftBeginAddress })); } else if (typeof(Key).IsGenericType && (typeof(Key).GetGenericTypeDefinition() == typeof(Memory <>)) && Utility.IsBlittableType(typeof(Key).GetGenericArguments()[0]) && typeof(Value).IsGenericType && (typeof(Value).GetGenericTypeDefinition() == typeof(Memory <>)) && Utility.IsBlittableType(typeof(Value).GetGenericArguments()[0])) { MethodInfo method = GetType().GetMethod("CompactMemory", BindingFlags.NonPublic | BindingFlags.Instance); MethodInfo generic = method.MakeGenericMethod(typeof(Key).GetGenericArguments()[0]); return((long)generic.Invoke(this, new object[] { untilAddress, shiftBeginAddress })); } else { var functions = new LogVariableCompactFunctions <Key, Value, DefaultVariableCompactionFunctions <Key, Value> >(varLen, default); var variableLengthStructSettings = new VariableLengthStructSettings <Key, Value> { keyLength = varLen.KeyLength, valueLength = varLen.ValueLength, }; return(Compact(functions, default(DefaultVariableCompactionFunctions <Key, Value>), untilAddress, variableLengthStructSettings, shiftBeginAddress)); } } else { return(Compact(new LogCompactFunctions <Key, Value, DefaultCompactionFunctions <Key, Value> >(default), default(DefaultCompactionFunctions <Key, Value>), untilAddress, null, shiftBeginAddress));
private void Compact <T>(T functions, long untilAddress, VariableLengthStructSettings <Key, Value> variableLengthStructSettings) where T : IFunctions <Key, Value, Input, Output, Context> { var originalUntilAddress = untilAddress; var tempKv = new FasterKV <Key, Value, Input, Output, Context, T> (fht.IndexSize, functions, new LogSettings(), comparer: fht.Comparer, variableLengthStructSettings: variableLengthStructSettings); tempKv.StartSession(); int cnt = 0; using (var iter1 = fht.Log.Scan(fht.Log.BeginAddress, untilAddress)) { while (iter1.GetNext(out RecordInfo recordInfo)) { ref var key = ref iter1.GetKey(); ref var value = ref iter1.GetValue(); if (recordInfo.Tombstone) { tempKv.Delete(ref key, default(Context), 0); } else { tempKv.Upsert(ref key, ref value, default(Context), 0); } if (++cnt % 1000 == 0) { fht.Refresh(); tempKv.Refresh(); } }
/// <inheritdoc /> public FasterKV(long size, Functions functions, LogSettings logSettings, CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null, IFasterEqualityComparer <Key> comparer = null, VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null) { _functions = functions; _fasterKV = new FasterKV <Key, Value>(size, logSettings, checkpointSettings, serializerSettings, comparer, variableLengthStructSettings); }
/// <inheritdoc /> public FasterKV(long size, Functions functions, LogSettings logSettings, CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null, IFasterEqualityComparer <Key> comparer = null, VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null, IVariableLengthStruct <Value, Input> variableLengthStructForInput = null) { _functions = functions; _fasterKV = new FasterKV <Key, Value>(size, logSettings, checkpointSettings, serializerSettings, comparer, variableLengthStructSettings); _variableLengthStructForInput = variableLengthStructForInput; if (_fasterKV.hlog is VariableLengthBlittableAllocator <Key, Value> allocator && _variableLengthStructForInput == default) { _variableLengthStructForInput = new DefaultVariableLengthStruct <Value, Input>(allocator.ValueLength); } }
/// <summary> /// Compact the log until specified address, moving active /// records to the tail of the log. /// Uses default compaction functions that only deletes explicitly deleted records, copying is implemeted by shallow copying values from source to destination. /// </summary> /// <param name="untilAddress"></param> public void Compact(long untilAddress) { if (allocator is VariableLengthBlittableAllocator <Key, Value> varLen) { var functions = new LogVariableCompactFunctions <Key, Value, DefaultVariableCompactionFunctions <Key, Value> >(varLen, default); var variableLengthStructSettings = new VariableLengthStructSettings <Key, Value> { keyLength = varLen.KeyLength, valueLength = varLen.ValueLength, }; Compact(functions, default(DefaultVariableCompactionFunctions <Key, Value>), untilAddress, variableLengthStructSettings); } else { Compact(new LogCompactFunctions <Key, Value, DefaultCompactionFunctions <Key, Value> >(default), default(DefaultCompactionFunctions <Key, Value>), untilAddress, null);
/// <summary> /// Iterator for all (distinct) live key-values stored in FASTER /// </summary> /// <param name="untilAddress">Report records until this address (tail by default)</param> /// <returns>FASTER iterator</returns> public IFasterScanIterator <Key, Value> Iterate(long untilAddress = -1) { if (untilAddress == -1) { untilAddress = Log.TailAddress; } if (hlog is VariableLengthBlittableAllocator <Key, Value> varLen) { var functions = new LogVariableCompactFunctions <Key, Value, DefaultVariableCompactionFunctions <Key, Value> >(varLen, default); var variableLengthStructSettings = new VariableLengthStructSettings <Key, Value> { keyLength = varLen.KeyLength, valueLength = varLen.ValueLength, }; return(new FasterKVIterator <Key, Value, LogVariableCompactFunctions <Key, Value, DefaultVariableCompactionFunctions <Key, Value> >, DefaultVariableCompactionFunctions <Key, Value> > (this, functions, default, untilAddress, variableLengthStructSettings));
public VariableLengthBlittableAllocator(LogSettings settings, VariableLengthStructSettings <Key, Value> vlSettings, IFasterEqualityComparer <Key> comparer, Action <long, long> evictCallback = null, LightEpoch epoch = null, Action <CommitInfo> flushCallback = null) : base(settings, comparer, evictCallback, epoch, flushCallback) { overflowPagePool = new OverflowPool <PageUnit>(4, p => #if NET5_0_OR_GREATER { } #else p.handle.Free() #endif ); if (BufferSize > 0) { values = new byte[BufferSize][]; #if NET5_0_OR_GREATER pointers = GC.AllocateArray <long>(BufferSize, true); nativePointers = (long *)Unsafe.AsPointer(ref pointers[0]); #else pointers = new long[BufferSize]; handles = new GCHandle[BufferSize]; ptrHandle = GCHandle.Alloc(pointers, GCHandleType.Pinned); nativePointers = (long *)ptrHandle.AddrOfPinnedObject(); #endif } KeyLength = vlSettings.keyLength; ValueLength = vlSettings.valueLength; if (KeyLength == null) { fixedSizeKey = true; KeyLength = new FixedLengthStruct <Key>(); } if (ValueLength == null) { fixedSizeValue = true; ValueLength = new FixedLengthStruct <Value>(); } }
/// <summary> /// Compact the log until specified address using current session, moving active records to the tail of the log. /// </summary> /// <param name="untilAddress">Compact log until this address</param> /// <param name="shiftBeginAddress">Whether to shift begin address to untilAddress after compaction. To avoid /// <param name="compactionFunctions">User provided compaction functions (see <see cref="ICompactionFunctions{Key, Value}"/>).</param> /// data loss on failure, set this to false, and shift begin address only after taking a checkpoint. This /// ensures that records written to the tail during compaction are first made stable.</param> /// <returns>Address until which compaction was done</returns> public long Compact <CompactionFunctions>(long untilAddress, bool shiftBeginAddress, CompactionFunctions compactionFunctions) where CompactionFunctions : ICompactionFunctions <Key, Value> { if (!SupportAsync) { throw new FasterException("Do not perform compaction using a threadAffinitized session"); } VariableLengthStructSettings <Key, Value> vl = null; if (fht.hlog is VariableLengthBlittableAllocator <Key, Value> varLen) { vl = new VariableLengthStructSettings <Key, Value> { keyLength = varLen.KeyLength, valueLength = varLen.ValueLength, }; } return(fht.Log.Compact(this, functions, compactionFunctions, untilAddress, vl, shiftBeginAddress)); }
private void Compact <T>(T functions, long untilAddress, VariableLengthStructSettings <Key, Value> variableLengthStructSettings) where T : IFunctions <Key, Value, Input, Output, Context> { var fhtSession = fht.NewSession(); var originalUntilAddress = untilAddress; var tempKv = new FasterKV <Key, Value, Input, Output, Context, T> (fht.IndexSize, functions, new LogSettings(), comparer: fht.Comparer, variableLengthStructSettings: variableLengthStructSettings); var tempKvSession = tempKv.NewSession(); using (var iter1 = fht.Log.Scan(fht.Log.BeginAddress, untilAddress)) { while (iter1.GetNext(out RecordInfo recordInfo)) { ref var key = ref iter1.GetKey(); ref var value = ref iter1.GetValue(); if (recordInfo.Tombstone) { tempKvSession.Delete(ref key, default, 0);
/// <summary> /// Compact the log until specified address, moving active records to the tail of the log. /// </summary> /// <param name="functions">Functions used to manage key-values during compaction</param> /// <param name="cf">User provided compaction functions (see <see cref="ICompactionFunctions{Key, Value}"/>).</param> /// <param name="untilAddress">Compact log until this address</param> /// <param name="shiftBeginAddress">Whether to shift begin address to untilAddress after compaction. To avoid /// data loss on failure, set this to false, and shift begin address only after taking a checkpoint. This /// ensures that records written to the tail during compaction are first made stable.</param> /// <returns>Address until which compaction was done</returns> public long Compact <Input, Output, Context, Functions, CompactionFunctions>(Functions functions, CompactionFunctions cf, long untilAddress, bool shiftBeginAddress) where Functions : IFunctions <Key, Value, Input, Output, Context> where CompactionFunctions : ICompactionFunctions <Key, Value> { if (untilAddress > fht.Log.SafeReadOnlyAddress) { throw new FasterException("Can compact only until Log.SafeReadOnlyAddress"); } var originalUntilAddress = untilAddress; var lf = new LogCompactionFunctions <Key, Value, Input, Output, Context, Functions>(functions); using var fhtSession = fht.For(lf).NewSession <LogCompactionFunctions <Key, Value, Input, Output, Context, Functions> >(); VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null; if (allocator is VariableLengthBlittableAllocator <Key, Value> varLen) { variableLengthStructSettings = new VariableLengthStructSettings <Key, Value> { keyLength = varLen.KeyLength, valueLength = varLen.ValueLength, }; } using (var tempKv = new FasterKV <Key, Value>(fht.IndexSize, new LogSettings { LogDevice = new NullDevice(), ObjectLogDevice = new NullDevice() }, comparer: fht.Comparer, variableLengthStructSettings: variableLengthStructSettings)) using (var tempKvSession = tempKv.NewSession <Input, Output, Context, Functions>(functions)) { using (var iter1 = fht.Log.Scan(fht.Log.BeginAddress, untilAddress)) { while (iter1.GetNext(out var recordInfo)) { ref var key = ref iter1.GetKey(); ref var value = ref iter1.GetValue(); if (recordInfo.Tombstone || cf.IsDeleted(key, value)) { tempKvSession.Delete(ref key, default, 0);
public FasterKVIterator(FasterKV <Key, Value> fht, Functions functions, long untilAddress) { this.fht = fht; enumerationPhase = 0; VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null; if (fht.hlog is VariableLengthBlittableAllocator <Key, Value> varLen) { variableLengthStructSettings = new VariableLengthStructSettings <Key, Value> { keyLength = varLen.KeyLength, valueLength = varLen.ValueLength, }; } tempKv = new FasterKV <Key, Value>(fht.IndexSize, new LogSettings { LogDevice = new NullDevice(), ObjectLogDevice = new NullDevice(), MutableFraction = 1 }, comparer: fht.Comparer, variableLengthStructSettings: variableLengthStructSettings); tempKvSession = tempKv.NewSession <Input, Output, Context, Functions>(functions); iter1 = fht.Log.Scan(fht.Log.BeginAddress, untilAddress); }
/// <summary> /// Create FASTER instance /// </summary> /// <param name="size">Size of core index (#cache lines)</param> /// <param name="logSettings">Log settings</param> /// <param name="checkpointSettings">Checkpoint settings</param> /// <param name="serializerSettings">Serializer settings</param> /// <param name="comparer">FASTER equality comparer for key</param> /// <param name="variableLengthStructSettings"></param> public FasterKV(long size, LogSettings logSettings, CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null, IFasterEqualityComparer <Key> comparer = null, VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null) { if (comparer != null) { this.comparer = comparer; } else { if (typeof(IFasterEqualityComparer <Key>).IsAssignableFrom(typeof(Key))) { if (default(Key) != null) { this.comparer = default(Key) as IFasterEqualityComparer <Key>; } else if (typeof(Key).GetConstructor(Type.EmptyTypes) != null) { this.comparer = Activator.CreateInstance(typeof(Key)) as IFasterEqualityComparer <Key>; } } else { this.comparer = FasterEqualityComparer.Get <Key>(); } } if (checkpointSettings == null) { checkpointSettings = new CheckpointSettings(); } if (checkpointSettings.CheckpointDir != null && checkpointSettings.CheckpointManager != null) { throw new FasterException( "Specify either CheckpointManager or CheckpointDir for CheckpointSettings, not both"); } bool oldCheckpointManager = false; if (oldCheckpointManager) { checkpointManager = checkpointSettings.CheckpointManager ?? new LocalCheckpointManager(checkpointSettings.CheckpointDir ?? ""); } else { checkpointManager = checkpointSettings.CheckpointManager ?? new DeviceLogCommitCheckpointManager (new LocalStorageNamedDeviceFactory(), new DefaultCheckpointNamingScheme( new DirectoryInfo(checkpointSettings.CheckpointDir ?? ".").FullName), removeOutdated: checkpointSettings.RemoveOutdated); } if (checkpointSettings.CheckpointManager == null) { disposeCheckpointManager = true; } FoldOverSnapshot = checkpointSettings.CheckPointType == core.CheckpointType.FoldOver; CopyReadsToTail = logSettings.CopyReadsToTail; if (logSettings.ReadCacheSettings != null) { CopyReadsToTail = CopyReadsToTail.None; UseReadCache = true; } UpdateVarLen(ref variableLengthStructSettings); if ((!Utility.IsBlittable <Key>() && variableLengthStructSettings?.keyLength == null) || (!Utility.IsBlittable <Value>() && variableLengthStructSettings?.valueLength == null)) { WriteDefaultOnDelete = true; hlog = new GenericAllocator <Key, Value>(logSettings, serializerSettings, this.comparer, null, epoch); Log = new LogAccessor <Key, Value>(this, hlog); if (UseReadCache) { readcache = new GenericAllocator <Key, Value>( new LogSettings { LogDevice = new NullDevice(), ObjectLogDevice = new NullDevice(), PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits, MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits, SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits, MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction }, serializerSettings, this.comparer, ReadCacheEvict, epoch); readcache.Initialize(); ReadCache = new LogAccessor <Key, Value>(this, readcache); } } else if (variableLengthStructSettings != null) { hlog = new VariableLengthBlittableAllocator <Key, Value>(logSettings, variableLengthStructSettings, this.comparer, null, epoch); Log = new LogAccessor <Key, Value>(this, hlog); if (UseReadCache) { readcache = new VariableLengthBlittableAllocator <Key, Value>( new LogSettings { LogDevice = new NullDevice(), PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits, MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits, SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits, MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction }, variableLengthStructSettings, this.comparer, ReadCacheEvict, epoch); readcache.Initialize(); ReadCache = new LogAccessor <Key, Value>(this, readcache); } } else { hlog = new BlittableAllocator <Key, Value>(logSettings, this.comparer, null, epoch); Log = new LogAccessor <Key, Value>(this, hlog); if (UseReadCache) { readcache = new BlittableAllocator <Key, Value>( new LogSettings { LogDevice = new NullDevice(), PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits, MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits, SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits, MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction }, this.comparer, ReadCacheEvict, epoch); readcache.Initialize(); ReadCache = new LogAccessor <Key, Value>(this, readcache); } } hlog.Initialize(); sectorSize = (int)logSettings.LogDevice.SectorSize; Initialize(size, sectorSize); systemState = default; systemState.Phase = Phase.REST; systemState.Version = 1; }
private static void UpdateVarLen(ref VariableLengthStructSettings <Key, Value> variableLengthStructSettings) { if (typeof(Key) == typeof(SpanByte)) { if (variableLengthStructSettings == null) { variableLengthStructSettings = new VariableLengthStructSettings <SpanByte, Value>() as VariableLengthStructSettings <Key, Value>; } if (variableLengthStructSettings.keyLength == null) { (variableLengthStructSettings as VariableLengthStructSettings <SpanByte, Value>).keyLength = new SpanByteVarLenStruct(); } } else if (typeof(Key).IsGenericType && (typeof(Key).GetGenericTypeDefinition() == typeof(Memory <>)) && Utility.IsBlittableType(typeof(Key).GetGenericArguments()[0])) { if (variableLengthStructSettings == null) { variableLengthStructSettings = new VariableLengthStructSettings <Key, Value>(); } if (variableLengthStructSettings.keyLength == null) { var m = typeof(MemoryVarLenStruct <>).MakeGenericType(typeof(Key).GetGenericArguments()); object o = Activator.CreateInstance(m); variableLengthStructSettings.keyLength = o as IVariableLengthStruct <Key>; } } else if (typeof(Key).IsGenericType && (typeof(Key).GetGenericTypeDefinition() == typeof(ReadOnlyMemory <>)) && Utility.IsBlittableType(typeof(Key).GetGenericArguments()[0])) { if (variableLengthStructSettings == null) { variableLengthStructSettings = new VariableLengthStructSettings <Key, Value>(); } if (variableLengthStructSettings.keyLength == null) { var m = typeof(ReadOnlyMemoryVarLenStruct <>).MakeGenericType(typeof(Key).GetGenericArguments()); object o = Activator.CreateInstance(m); variableLengthStructSettings.keyLength = o as IVariableLengthStruct <Key>; } } if (typeof(Value) == typeof(SpanByte)) { if (variableLengthStructSettings == null) { variableLengthStructSettings = new VariableLengthStructSettings <Key, SpanByte>() as VariableLengthStructSettings <Key, Value>; } if (variableLengthStructSettings.valueLength == null) { (variableLengthStructSettings as VariableLengthStructSettings <Key, SpanByte>).valueLength = new SpanByteVarLenStruct(); } } else if (typeof(Value).IsGenericType && (typeof(Value).GetGenericTypeDefinition() == typeof(Memory <>)) && Utility.IsBlittableType(typeof(Value).GetGenericArguments()[0])) { if (variableLengthStructSettings == null) { variableLengthStructSettings = new VariableLengthStructSettings <Key, Value>(); } if (variableLengthStructSettings.valueLength == null) { var m = typeof(MemoryVarLenStruct <>).MakeGenericType(typeof(Value).GetGenericArguments()); object o = Activator.CreateInstance(m); variableLengthStructSettings.valueLength = o as IVariableLengthStruct <Value>; } } else if (typeof(Value).IsGenericType && (typeof(Value).GetGenericTypeDefinition() == typeof(ReadOnlyMemory <>)) && Utility.IsBlittableType(typeof(Value).GetGenericArguments()[0])) { if (variableLengthStructSettings == null) { variableLengthStructSettings = new VariableLengthStructSettings <Key, Value>(); } if (variableLengthStructSettings.valueLength == null) { var m = typeof(ReadOnlyMemoryVarLenStruct <>).MakeGenericType(typeof(Value).GetGenericArguments()); object o = Activator.CreateInstance(m); variableLengthStructSettings.valueLength = o as IVariableLengthStruct <Value>; } } }
/// <summary> /// Create FASTER instance /// </summary> /// <param name="size">Size of core index (#cache lines)</param> /// <param name="comparer">FASTER equality comparer for key</param> /// <param name="variableLengthStructSettings"></param> /// <param name="logSettings">Log settings</param> /// <param name="checkpointSettings">Checkpoint settings</param> /// <param name="serializerSettings">Serializer settings</param> public FasterKV(long size, LogSettings logSettings, CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null, IFasterEqualityComparer <Key> comparer = null, VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null) { if (comparer != null) { this.comparer = comparer; } else { if (typeof(IFasterEqualityComparer <Key>).IsAssignableFrom(typeof(Key))) { this.comparer = new Key() as IFasterEqualityComparer <Key>; } else { Console.WriteLine( "***WARNING*** Creating default FASTER key equality comparer based on potentially slow EqualityComparer<Key>.Default. To avoid this, provide a comparer (IFasterEqualityComparer<Key>) as an argument to FASTER's constructor, or make Key implement the interface IFasterEqualityComparer<Key>"); this.comparer = FasterEqualityComparer <Key> .Default; } } if (checkpointSettings == null) { checkpointSettings = new CheckpointSettings(); } if (checkpointSettings.CheckpointDir != null && checkpointSettings.CheckpointManager != null) { throw new FasterException( "Specify either CheckpointManager or CheckpointDir for CheckpointSettings, not both"); } bool oldCheckpointManager = false; if (oldCheckpointManager) { checkpointManager = checkpointSettings.CheckpointManager ?? new LocalCheckpointManager(checkpointSettings.CheckpointDir ?? ""); } else { checkpointManager = checkpointSettings.CheckpointManager ?? new DeviceLogCommitCheckpointManager (new LocalStorageNamedDeviceFactory(), new DefaultCheckpointNamingScheme( new DirectoryInfo(checkpointSettings.CheckpointDir ?? ".").FullName)); } if (checkpointSettings.CheckpointManager == null) { disposeCheckpointManager = true; } FoldOverSnapshot = checkpointSettings.CheckPointType == core.CheckpointType.FoldOver; CopyReadsToTail = logSettings.CopyReadsToTail; if (logSettings.ReadCacheSettings != null) { CopyReadsToTail = false; UseReadCache = true; } if (Utility.IsBlittable <Key>() && Utility.IsBlittable <Value>()) { if (variableLengthStructSettings != null) { hlog = new VariableLengthBlittableAllocator <Key, Value>(logSettings, variableLengthStructSettings, this.comparer, null, epoch); Log = new LogAccessor <Key, Value>(this, hlog); if (UseReadCache) { readcache = new VariableLengthBlittableAllocator <Key, Value>( new LogSettings { PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits, MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits, SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits, MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction }, variableLengthStructSettings, this.comparer, ReadCacheEvict, epoch); readcache.Initialize(); ReadCache = new LogAccessor <Key, Value>(this, readcache); } } else { hlog = new BlittableAllocator <Key, Value>(logSettings, this.comparer, null, epoch); Log = new LogAccessor <Key, Value>(this, hlog); if (UseReadCache) { readcache = new BlittableAllocator <Key, Value>( new LogSettings { PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits, MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits, SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits, MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction }, this.comparer, ReadCacheEvict, epoch); readcache.Initialize(); ReadCache = new LogAccessor <Key, Value>(this, readcache); } } } else { WriteDefaultOnDelete = true; hlog = new GenericAllocator <Key, Value>(logSettings, serializerSettings, this.comparer, null, epoch); Log = new LogAccessor <Key, Value>(this, hlog); if (UseReadCache) { readcache = new GenericAllocator <Key, Value>( new LogSettings { PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits, MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits, SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits, MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction }, serializerSettings, this.comparer, ReadCacheEvict, epoch); readcache.Initialize(); ReadCache = new LogAccessor <Key, Value>(this, readcache); } } hlog.Initialize(); sectorSize = (int)logSettings.LogDevice.SectorSize; Initialize(size, sectorSize); systemState = default; systemState.phase = Phase.REST; systemState.version = 1; }
/// <summary> /// Create FASTER instance /// </summary> /// <param name="size">Size of core index (#cache lines)</param> /// <param name="comparer">FASTER equality comparer for key</param> /// <param name="variableLengthStructSettings"></param> /// <param name="functions">Callback functions</param> /// <param name="logSettings">Log settings</param> /// <param name="checkpointSettings">Checkpoint settings</param> /// <param name="serializerSettings">Serializer settings</param> public FasterKV(long size, Functions functions, LogSettings logSettings, CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null, IFasterEqualityComparer <Key> comparer = null, VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null) { threadCtx = new FastThreadLocal <FasterExecutionContext>(); prevThreadCtx = new FastThreadLocal <FasterExecutionContext>(); if (comparer != null) { this.comparer = comparer; } else { if (typeof(IFasterEqualityComparer <Key>).IsAssignableFrom(typeof(Key))) { this.comparer = new Key() as IFasterEqualityComparer <Key>; } else { Console.WriteLine("***WARNING*** Creating default FASTER key equality comparer based on potentially slow EqualityComparer<Key>.Default. To avoid this, provide a comparer (IFasterEqualityComparer<Key>) as an argument to FASTER's constructor, or make Key implement the interface IFasterEqualityComparer<Key>"); this.comparer = FasterEqualityComparer <Key> .Default; } } if (checkpointSettings == null) { checkpointSettings = new CheckpointSettings(); } directoryConfiguration = new DirectoryConfiguration(checkpointSettings.CheckpointDir); FoldOverSnapshot = checkpointSettings.CheckPointType == core.CheckpointType.FoldOver; CopyReadsToTail = logSettings.CopyReadsToTail; this.functions = functions; if (logSettings.ReadCacheSettings != null) { CopyReadsToTail = false; UseReadCache = true; } if (Utility.IsBlittable <Key>() && Utility.IsBlittable <Value>()) { if (variableLengthStructSettings != null) { hlog = new VariableLengthBlittableAllocator <Key, Value> (logSettings, variableLengthStructSettings, this.comparer, null, epoch); Log = new LogAccessor <Key, Value, Input, Output, Context>(this, hlog); if (UseReadCache) { readcache = new VariableLengthBlittableAllocator <Key, Value>( new LogSettings { PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits, MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits, SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits, MutableFraction = logSettings.ReadCacheSettings.SecondChanceFraction }, variableLengthStructSettings, this.comparer, ReadCacheEvict, epoch); readcache.Initialize(); ReadCache = new LogAccessor <Key, Value, Input, Output, Context>(this, readcache); } } else { hlog = new BlittableAllocator <Key, Value>(logSettings, this.comparer, null, epoch); Log = new LogAccessor <Key, Value, Input, Output, Context>(this, hlog); if (UseReadCache) { readcache = new BlittableAllocator <Key, Value>( new LogSettings { PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits, MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits, SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits, MutableFraction = logSettings.ReadCacheSettings.SecondChanceFraction }, this.comparer, ReadCacheEvict, epoch); readcache.Initialize(); ReadCache = new LogAccessor <Key, Value, Input, Output, Context>(this, readcache); } } } else { hlog = new GenericAllocator <Key, Value>(logSettings, serializerSettings, this.comparer, null, epoch); Log = new LogAccessor <Key, Value, Input, Output, Context>(this, hlog); if (UseReadCache) { readcache = new GenericAllocator <Key, Value>( new LogSettings { PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits, MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits, SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits, MutableFraction = logSettings.ReadCacheSettings.SecondChanceFraction }, serializerSettings, this.comparer, ReadCacheEvict, epoch); readcache.Initialize(); ReadCache = new LogAccessor <Key, Value, Input, Output, Context>(this, readcache); } } hlog.Initialize(); sectorSize = (int)logSettings.LogDevice.SectorSize; Initialize(size, sectorSize); _systemState = default(SystemState); _systemState.phase = Phase.REST; _systemState.version = 1; _checkpointType = CheckpointType.HYBRID_LOG_ONLY; }