public BlittableAllocator(LogSettings settings, IFasterEqualityComparer <Key> comparer, Action <long, long> evictCallback = null, LightEpoch epoch = null, Action <CommitInfo> flushCallback = null) : base(settings, comparer, evictCallback, epoch, flushCallback) { overflowPagePool = new OverflowPool <PageUnit>(4, p => #if NET5_0_OR_GREATER { } #else p.handle.Free() #endif ); if (BufferSize > 0) { values = new byte[BufferSize][]; #if NET5_0_OR_GREATER pointers = GC.AllocateArray <long>(BufferSize, true); nativePointers = (long *)Unsafe.AsPointer(ref pointers[0]); #else pointers = new long[BufferSize]; handles = new GCHandle[BufferSize]; ptrHandle = GCHandle.Alloc(pointers, GCHandleType.Pinned); nativePointers = (long *)ptrHandle.AddrOfPinnedObject(); #endif } }
/// <inheritdoc /> public FasterKV(long size, Functions functions, LogSettings logSettings, CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null, IFasterEqualityComparer <Key> comparer = null, VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null) { _functions = functions; _fasterKV = new FasterKV <Key, Value>(size, logSettings, checkpointSettings, serializerSettings, comparer, variableLengthStructSettings); }
public BlittableAllocator(LogSettings settings, IFasterEqualityComparer <Key> comparer, Action <long, long> evictCallback = null, LightEpoch epoch = null, Action <CommitInfo> flushCallback = null) : base(settings, comparer, evictCallback, epoch, flushCallback) { values = new byte[BufferSize][]; handles = new GCHandle[BufferSize]; pointers = new long[BufferSize]; ptrHandle = GCHandle.Alloc(pointers, GCHandleType.Pinned); nativePointers = (long *)ptrHandle.AddrOfPinnedObject(); }
public BlittableAllocator(LogSettings settings, IFasterEqualityComparer <Key> comparer) : base(settings, comparer) { values = new byte[BufferSize][]; handles = new GCHandle[BufferSize]; pointers = new long[BufferSize]; epoch = LightEpoch.Instance; ptrHandle = GCHandle.Alloc(pointers, GCHandleType.Pinned); nativePointers = (long *)ptrHandle.AddrOfPinnedObject(); }
public KeyValueStore(string folder, int logSizeBits, IFasterEqualityComparer <TKey> keyComparer, TFunctions functions, IVariableLengthStruct <TKey> keyLength, IVariableLengthStruct <TValue> valueLength) { _dataFolder = folder; _logSizeBits = logSizeBits; _keyComparer = keyComparer; _functions = functions; _variableLenSettings = new VariableLengthStructSettings <TKey, TValue> { keyLength = keyLength, valueLength = valueLength }; }
public KeyValueStore(string folder, int logSizeBits, IFasterEqualityComparer <TKey> keyComparer, TFunctions functions, Func <IObjectSerializer <TKey> > keySerializer, Func <IObjectSerializer <TValue> > valueSerializer) { _dataFolder = folder; _logSizeBits = logSizeBits; _keyComparer = keyComparer; _functions = functions; _serializerSettings = new SerializerSettings <TKey, TValue> { keySerializer = keySerializer, valueSerializer = valueSerializer }; }
/// <inheritdoc /> public FasterKV(long size, Functions functions, LogSettings logSettings, CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null, IFasterEqualityComparer <Key> comparer = null, VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null, IVariableLengthStruct <Value, Input> variableLengthStructForInput = null) { _functions = functions; _fasterKV = new FasterKV <Key, Value>(size, logSettings, checkpointSettings, serializerSettings, comparer, variableLengthStructSettings); _variableLengthStructForInput = variableLengthStructForInput; if (_fasterKV.hlog is VariableLengthBlittableAllocator <Key, Value> allocator && _variableLengthStructForInput == default) { _variableLengthStructForInput = new DefaultVariableLengthStruct <Value, Input>(allocator.ValueLength); } }
/// <summary> /// Create FASTER instance /// </summary> /// <param name="size">Size of core index (#cache lines)</param> /// <param name="comparer">FASTER equality comparer for key</param> /// <param name="functions">Callback functions</param> /// <param name="logSettings">Log settings</param> /// <param name="checkpointSettings">Checkpoint settings</param> /// <param name="serializerSettings">Serializer settings</param> public FasterKV(long size, Functions functions, LogSettings logSettings, CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null, IFasterEqualityComparer <Key> comparer = null) { if (comparer != null) { this.comparer = comparer; } else { if (typeof(IFasterEqualityComparer <Key>).IsAssignableFrom(typeof(Key))) { this.comparer = new Key() as IFasterEqualityComparer <Key>; } else { Console.WriteLine("***WARNING*** Creating default FASTER key equality comparer based on potentially slow EqualityComparer<Key>.Default. To avoid this, provide a comparer (IFasterEqualityComparer<Key>) as an argument to FASTER's constructor, or make Key implement the interface IFasterEqualityComparer<Key>"); this.comparer = FasterEqualityComparer <Key> .Default; } } if (checkpointSettings == null) { checkpointSettings = new CheckpointSettings(); } directoryConfiguration = new DirectoryConfiguration(checkpointSettings.CheckpointDir); FoldOverSnapshot = checkpointSettings.CheckPointType == core.CheckpointType.FoldOver; CopyReadsToTail = logSettings.CopyReadsToTail; this.functions = functions; if (Utility.IsBlittable <Key>() && Utility.IsBlittable <Value>()) { hlog = new BlittableAllocator <Key, Value>(logSettings, this.comparer); } else { hlog = new GenericAllocator <Key, Value>(logSettings, serializerSettings, this.comparer); } hlog.Initialize(); sectorSize = (int)logSettings.LogDevice.SectorSize; Initialize(size, sectorSize); _systemState = default(SystemState); _systemState.phase = Phase.REST; _systemState.version = 1; _checkpointType = CheckpointType.HYBRID_LOG_ONLY; }
public FasterDictionary(IFasterEqualityComparer <TKey> keyComparer, Options options = default) { if (keyComparer == null) { throw new ArgumentNullException(nameof(keyComparer)); } _keyComparer = new KeyComparerAdapter(keyComparer); _options = Options.Default; if (options.PersistDirectoryPath != null) { _options.DeleteOnClose = options.DeleteOnClose; _options.DictionaryName = options.DictionaryName; _options.Logger = options.Logger; _options.PersistDirectoryPath = options.PersistDirectoryPath; if (options.CheckPointType != CheckpointType.Snapshot) { _options.CheckPointType = options.CheckPointType; } if (options.MemorySize != 0) { _options.MemorySize = options.MemorySize; } if (options.PageSize != 0) { _options.PageSize = options.PageSize; } if (options.SegmentSize != 0) { _options.SegmentSize = options.SegmentSize; } } JobQueue = Channel.CreateUnbounded <Job>(new UnboundedChannelOptions() { AllowSynchronousContinuations = true, SingleReader = true, SingleWriter = false }); StartConsumer(); }
/// <summary> /// Create FASTER instance /// </summary> /// <param name="size">Size of core index (#cache lines)</param> /// <param name="logSettings">Log settings</param> /// <param name="checkpointSettings">Checkpoint settings</param> /// <param name="serializerSettings">Serializer settings</param> /// <param name="comparer">FASTER equality comparer for key</param> /// <param name="variableLengthStructSettings"></param> public FasterKV(long size, LogSettings logSettings, CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null, IFasterEqualityComparer <Key> comparer = null, VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null) { if (comparer != null) { this.comparer = comparer; } else { if (typeof(IFasterEqualityComparer <Key>).IsAssignableFrom(typeof(Key))) { if (default(Key) != null) { this.comparer = default(Key) as IFasterEqualityComparer <Key>; } else if (typeof(Key).GetConstructor(Type.EmptyTypes) != null) { this.comparer = Activator.CreateInstance(typeof(Key)) as IFasterEqualityComparer <Key>; } } else { this.comparer = FasterEqualityComparer.Get <Key>(); } } if (checkpointSettings == null) { checkpointSettings = new CheckpointSettings(); } if (checkpointSettings.CheckpointDir != null && checkpointSettings.CheckpointManager != null) { throw new FasterException( "Specify either CheckpointManager or CheckpointDir for CheckpointSettings, not both"); } bool oldCheckpointManager = false; if (oldCheckpointManager) { checkpointManager = checkpointSettings.CheckpointManager ?? new LocalCheckpointManager(checkpointSettings.CheckpointDir ?? ""); } else { checkpointManager = checkpointSettings.CheckpointManager ?? new DeviceLogCommitCheckpointManager (new LocalStorageNamedDeviceFactory(), new DefaultCheckpointNamingScheme( new DirectoryInfo(checkpointSettings.CheckpointDir ?? ".").FullName), removeOutdated: checkpointSettings.RemoveOutdated); } if (checkpointSettings.CheckpointManager == null) { disposeCheckpointManager = true; } FoldOverSnapshot = checkpointSettings.CheckPointType == core.CheckpointType.FoldOver; CopyReadsToTail = logSettings.CopyReadsToTail; if (logSettings.ReadCacheSettings != null) { CopyReadsToTail = CopyReadsToTail.None; UseReadCache = true; } UpdateVarLen(ref variableLengthStructSettings); if ((!Utility.IsBlittable <Key>() && variableLengthStructSettings?.keyLength == null) || (!Utility.IsBlittable <Value>() && variableLengthStructSettings?.valueLength == null)) { WriteDefaultOnDelete = true; hlog = new GenericAllocator <Key, Value>(logSettings, serializerSettings, this.comparer, null, epoch); Log = new LogAccessor <Key, Value>(this, hlog); if (UseReadCache) { readcache = new GenericAllocator <Key, Value>( new LogSettings { LogDevice = new NullDevice(), ObjectLogDevice = new NullDevice(), PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits, MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits, SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits, MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction }, serializerSettings, this.comparer, ReadCacheEvict, epoch); readcache.Initialize(); ReadCache = new LogAccessor <Key, Value>(this, readcache); } } else if (variableLengthStructSettings != null) { hlog = new VariableLengthBlittableAllocator <Key, Value>(logSettings, variableLengthStructSettings, this.comparer, null, epoch); Log = new LogAccessor <Key, Value>(this, hlog); if (UseReadCache) { readcache = new VariableLengthBlittableAllocator <Key, Value>( new LogSettings { LogDevice = new NullDevice(), PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits, MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits, SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits, MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction }, variableLengthStructSettings, this.comparer, ReadCacheEvict, epoch); readcache.Initialize(); ReadCache = new LogAccessor <Key, Value>(this, readcache); } } else { hlog = new BlittableAllocator <Key, Value>(logSettings, this.comparer, null, epoch); Log = new LogAccessor <Key, Value>(this, hlog); if (UseReadCache) { readcache = new BlittableAllocator <Key, Value>( new LogSettings { LogDevice = new NullDevice(), PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits, MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits, SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits, MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction }, this.comparer, ReadCacheEvict, epoch); readcache.Initialize(); ReadCache = new LogAccessor <Key, Value>(this, readcache); } } hlog.Initialize(); sectorSize = (int)logSettings.LogDevice.SectorSize; Initialize(size, sectorSize); systemState = default; systemState.Phase = Phase.REST; systemState.Version = 1; }
internal KeyComparer(IFasterEqualityComparer <TKey> comparer) => this.comparer = comparer;
/// <summary> /// Create FASTER instance /// </summary> /// <param name="size">Size of core index (#cache lines)</param> /// <param name="comparer">FASTER equality comparer for key</param> /// <param name="variableLengthStructSettings"></param> /// <param name="logSettings">Log settings</param> /// <param name="checkpointSettings">Checkpoint settings</param> /// <param name="serializerSettings">Serializer settings</param> public FasterKV(long size, LogSettings logSettings, CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null, IFasterEqualityComparer <Key> comparer = null, VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null) { if (comparer != null) { this.comparer = comparer; } else { if (typeof(IFasterEqualityComparer <Key>).IsAssignableFrom(typeof(Key))) { this.comparer = new Key() as IFasterEqualityComparer <Key>; } else { Console.WriteLine( "***WARNING*** Creating default FASTER key equality comparer based on potentially slow EqualityComparer<Key>.Default. To avoid this, provide a comparer (IFasterEqualityComparer<Key>) as an argument to FASTER's constructor, or make Key implement the interface IFasterEqualityComparer<Key>"); this.comparer = FasterEqualityComparer <Key> .Default; } } if (checkpointSettings == null) { checkpointSettings = new CheckpointSettings(); } if (checkpointSettings.CheckpointDir != null && checkpointSettings.CheckpointManager != null) { throw new FasterException( "Specify either CheckpointManager or CheckpointDir for CheckpointSettings, not both"); } bool oldCheckpointManager = false; if (oldCheckpointManager) { checkpointManager = checkpointSettings.CheckpointManager ?? new LocalCheckpointManager(checkpointSettings.CheckpointDir ?? ""); } else { checkpointManager = checkpointSettings.CheckpointManager ?? new DeviceLogCommitCheckpointManager (new LocalStorageNamedDeviceFactory(), new DefaultCheckpointNamingScheme( new DirectoryInfo(checkpointSettings.CheckpointDir ?? ".").FullName)); } if (checkpointSettings.CheckpointManager == null) { disposeCheckpointManager = true; } FoldOverSnapshot = checkpointSettings.CheckPointType == core.CheckpointType.FoldOver; CopyReadsToTail = logSettings.CopyReadsToTail; if (logSettings.ReadCacheSettings != null) { CopyReadsToTail = false; UseReadCache = true; } if (Utility.IsBlittable <Key>() && Utility.IsBlittable <Value>()) { if (variableLengthStructSettings != null) { hlog = new VariableLengthBlittableAllocator <Key, Value>(logSettings, variableLengthStructSettings, this.comparer, null, epoch); Log = new LogAccessor <Key, Value>(this, hlog); if (UseReadCache) { readcache = new VariableLengthBlittableAllocator <Key, Value>( new LogSettings { PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits, MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits, SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits, MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction }, variableLengthStructSettings, this.comparer, ReadCacheEvict, epoch); readcache.Initialize(); ReadCache = new LogAccessor <Key, Value>(this, readcache); } } else { hlog = new BlittableAllocator <Key, Value>(logSettings, this.comparer, null, epoch); Log = new LogAccessor <Key, Value>(this, hlog); if (UseReadCache) { readcache = new BlittableAllocator <Key, Value>( new LogSettings { PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits, MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits, SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits, MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction }, this.comparer, ReadCacheEvict, epoch); readcache.Initialize(); ReadCache = new LogAccessor <Key, Value>(this, readcache); } } } else { WriteDefaultOnDelete = true; hlog = new GenericAllocator <Key, Value>(logSettings, serializerSettings, this.comparer, null, epoch); Log = new LogAccessor <Key, Value>(this, hlog); if (UseReadCache) { readcache = new GenericAllocator <Key, Value>( new LogSettings { PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits, MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits, SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits, MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction }, serializerSettings, this.comparer, ReadCacheEvict, epoch); readcache.Initialize(); ReadCache = new LogAccessor <Key, Value>(this, readcache); } } hlog.Initialize(); sectorSize = (int)logSettings.LogDevice.SectorSize; Initialize(size, sectorSize); systemState = default; systemState.phase = Phase.REST; systemState.version = 1; }
public GenericAllocator(LogSettings settings, SerializerSettings <Key, Value> serializerSettings, IFasterEqualityComparer <Key> comparer) : base(settings, comparer) { SerializerSettings = serializerSettings; if (default(Key) == null && ((SerializerSettings == null) || (SerializerSettings.keySerializer == null))) { throw new Exception("Key is a class, but no serializer specified via SerializerSettings"); } if (default(Value) == null && ((SerializerSettings == null) || (SerializerSettings.valueSerializer == null))) { throw new Exception("Value is a class, but no serializer specified via SerializerSettings"); } values = new Record <Key, Value> [BufferSize][]; segmentOffsets = new long[SegmentBufferSize]; objectLogDevice = settings.ObjectLogDevice; if (KeyHasObjects() || ValueHasObjects()) { if (objectLogDevice == null) { throw new Exception("Objects in key/value, but object log not provided during creation of FASTER instance"); } } epoch = LightEpoch.Instance; ioBufferPool = SectorAlignedBufferPool.GetPool(1, sectorSize); }
internal static FasterStore <TKey, TValue> Create <TKey, TValue>(string directory, IFasterEqualityComparer <TKey> comparer) { return(new FasterStore <TKey, TValue>(new StoreOptions(directory), comparer, new NullLogger <FasterStore <TKey, TValue> >())); }
public KeyComparerAdapter(IFasterEqualityComparer <TKey> keyComparer) { _keyComparer = keyComparer; }
/// <summary> /// Create FASTER instance /// </summary> /// <param name="size">Size of core index (#cache lines)</param> /// <param name="comparer">FASTER equality comparer for key</param> /// <param name="functions">Callback functions</param> /// <param name="logSettings">Log settings</param> /// <param name="checkpointSettings">Checkpoint settings</param> /// <param name="serializerSettings">Serializer settings</param> public FasterKV(long size, Functions functions, LogSettings logSettings, CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null, IFasterEqualityComparer <Key> comparer = null) { threadCtx = new FastThreadLocal <FasterExecutionContext>(); prevThreadCtx = new FastThreadLocal <FasterExecutionContext>(); if (comparer != null) { this.comparer = comparer; } else { if (typeof(IFasterEqualityComparer <Key>).IsAssignableFrom(typeof(Key))) { this.comparer = new Key() as IFasterEqualityComparer <Key>; } else { Console.WriteLine("***WARNING*** Creating default FASTER key equality comparer based on potentially slow EqualityComparer<Key>.Default. To avoid this, provide a comparer (IFasterEqualityComparer<Key>) as an argument to FASTER's constructor, or make Key implement the interface IFasterEqualityComparer<Key>"); this.comparer = FasterEqualityComparer <Key> .Default; } } if (checkpointSettings == null) { checkpointSettings = new CheckpointSettings(); } directoryConfiguration = new DirectoryConfiguration(checkpointSettings.CheckpointDir); FoldOverSnapshot = checkpointSettings.CheckPointType == core.CheckpointType.FoldOver; CopyReadsToTail = logSettings.CopyReadsToTail; this.functions = functions; if (logSettings.ReadCacheSettings != null) { CopyReadsToTail = false; UseReadCache = true; } if (Utility.IsBlittable <Key>() && Utility.IsBlittable <Value>()) { hlog = new BlittableAllocator <Key, Value>(logSettings, this.comparer, null, epoch); Log = new LogAccessor <Key, Value, Input, Output, Context>(this, hlog); if (UseReadCache) { readcache = new BlittableAllocator <Key, Value>( new LogSettings { PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits, MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits, SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits, MutableFraction = logSettings.ReadCacheSettings.SecondChanceFraction }, this.comparer, ReadCacheEvict, epoch); readcache.Initialize(); ReadCache = new LogAccessor <Key, Value, Input, Output, Context>(this, readcache); } } else { hlog = new GenericAllocator <Key, Value>(logSettings, serializerSettings, this.comparer, null, epoch); Log = new LogAccessor <Key, Value, Input, Output, Context>(this, hlog); if (UseReadCache) { readcache = new GenericAllocator <Key, Value>( new LogSettings { PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits, MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits, SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits, MutableFraction = logSettings.ReadCacheSettings.SecondChanceFraction }, serializerSettings, this.comparer, ReadCacheEvict, epoch); readcache.Initialize(); ReadCache = new LogAccessor <Key, Value, Input, Output, Context>(this, readcache); } } hlog.Initialize(); sectorSize = (int)logSettings.LogDevice.SectorSize; Initialize(size, sectorSize); _systemState = default(SystemState); _systemState.phase = Phase.REST; _systemState.version = 1; _checkpointType = CheckpointType.HYBRID_LOG_ONLY; }
public GenericAllocator(LogSettings settings, SerializerSettings <Key, Value> serializerSettings, IFasterEqualityComparer <Key> comparer, Action <long, long> evictCallback = null, LightEpoch epoch = null, Action <CommitInfo> flushCallback = null) : base(settings, comparer, evictCallback, epoch, flushCallback) { SerializerSettings = serializerSettings; if ((!keyBlittable) && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.keySerializer == null))) { throw new FasterException("Key is not blittable, but no serializer specified via SerializerSettings"); } if ((!valueBlittable) && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.valueSerializer == null))) { throw new FasterException("Value is not blittable, but no serializer specified via SerializerSettings"); } values = new Record <Key, Value> [BufferSize][]; segmentOffsets = new long[SegmentBufferSize]; objectLogDevice = settings.ObjectLogDevice; if ((settings.LogDevice as NullDevice == null) && (KeyHasObjects() || ValueHasObjects())) { if (objectLogDevice == null) { throw new FasterException("Objects in key/value, but object log not provided during creation of FASTER instance"); } if (objectLogDevice.SegmentSize != -1) { throw new FasterException("Object log device should not have fixed segment size. Set preallocateFile to false when calling CreateLogDevice for object log"); } } }
public GenericAllocator(LogSettings settings, SerializerSettings <Key, Value> serializerSettings, IFasterEqualityComparer <Key> comparer, Action <long, long> evictCallback = null, LightEpoch epoch = null, Action <CommitInfo> flushCallback = null) : base(settings, comparer, evictCallback, epoch, flushCallback) { overflowPagePool = new OverflowPool <Record <Key, Value>[]>(4); if (settings.ObjectLogDevice == null) { throw new FasterException("LogSettings.ObjectLogDevice needs to be specified (e.g., use Devices.CreateLogDevice, AzureStorageDevice, or NullDevice)"); } SerializerSettings = serializerSettings ?? new SerializerSettings <Key, Value>(); if ((!keyBlittable) && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.keySerializer == null))) { Debug.WriteLine("Key is not blittable, but no serializer specified via SerializerSettings. Using (slow) DataContractSerializer as default."); SerializerSettings.keySerializer = ObjectSerializer.Get <Key>(); } if ((!valueBlittable) && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.valueSerializer == null))) { Debug.WriteLine("Value is not blittable, but no serializer specified via SerializerSettings. Using (slow) DataContractSerializer as default."); SerializerSettings.valueSerializer = ObjectSerializer.Get <Value>(); } values = new Record <Key, Value> [BufferSize][]; segmentOffsets = new long[SegmentBufferSize]; objectLogDevice = settings.ObjectLogDevice; if ((settings.LogDevice as NullDevice == null) && (KeyHasObjects() || ValueHasObjects())) { if (objectLogDevice == null) { throw new FasterException("Objects in key/value, but object log not provided during creation of FASTER instance"); } if (objectLogDevice.SegmentSize != -1) { throw new FasterException("Object log device should not have fixed segment size. Set preallocateFile to false when calling CreateLogDevice for object log"); } } }
public GenericAllocator(LogSettings settings, SerializerSettings <Key, Value> serializerSettings, IFasterEqualityComparer <Key> comparer, Action <long, long> evictCallback = null, LightEpoch epoch = null) : base(settings, comparer, evictCallback, epoch) { SerializerSettings = serializerSettings; if (default(Key) == null && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.keySerializer == null))) { throw new Exception("Key is a class, but no serializer specified via SerializerSettings"); } if (default(Value) == null && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.valueSerializer == null))) { throw new Exception("Value is a class, but no serializer specified via SerializerSettings"); } values = new Record <Key, Value> [BufferSize][]; segmentOffsets = new long[SegmentBufferSize]; objectLogDevice = settings.ObjectLogDevice; if ((settings.LogDevice as NullDevice == null) && (KeyHasObjects() || ValueHasObjects())) { if (objectLogDevice == null) { throw new Exception("Objects in key/value, but object log not provided during creation of FASTER instance"); } } }
public VariableLengthBlittableAllocator(LogSettings settings, VariableLengthStructSettings <Key, Value> vlSettings, IFasterEqualityComparer <Key> comparer, Action <long, long> evictCallback = null, LightEpoch epoch = null, Action <CommitInfo> flushCallback = null) : base(settings, comparer, evictCallback, epoch, flushCallback) { values = new byte[BufferSize][]; handles = new GCHandle[BufferSize]; pointers = new long[BufferSize]; ptrHandle = GCHandle.Alloc(pointers, GCHandleType.Pinned); nativePointers = (long *)ptrHandle.AddrOfPinnedObject(); KeyLength = vlSettings.keyLength; ValueLength = vlSettings.valueLength; if (KeyLength == null) { fixedSizeKey = true; KeyLength = new FixedLengthStruct <Key>(); } if (ValueLength == null) { fixedSizeValue = true; ValueLength = new FixedLengthStruct <Value>(); } }
public FasterStore(IOptions <FasterStoreOptions> optionsProvider, IFasterEqualityComparer <TKey> keyComparer, ILogger <FasterStore <TKey, TValue> > logger) { _logger = logger; var options = optionsProvider.Value; var indexStorePath = Path.Combine(options.Directory, $"{options.Name}-index.log"); var objectStorePath = Path.Combine(options.Directory, $"{options.Name}-object.log"); _indexDevice = Devices.CreateLogDevice(indexStorePath, preallocateFile: false, deleteOnClose: false, recoverDevice: true); _objectDevice = Devices.CreateLogDevice(objectStorePath, preallocateFile: false, deleteOnClose: false, recoverDevice: true); _checkpointsPath = Path.Combine(options.Directory, "checkpoints"); if (!Directory.Exists(_checkpointsPath)) { _logger.LogInformation("Checkpoints directory not exists. Creating..."); Directory.CreateDirectory(_checkpointsPath); } var logSettings = new LogSettings { LogDevice = _indexDevice, ObjectLogDevice = _objectDevice, MemorySizeBits = (int)options.MemorySize, PageSizeBits = (int)options.PageSize, SegmentSizeBits = (int)options.SegmentSize, ReadCacheSettings = new ReadCacheSettings() { MemorySizeBits = (int)options.MemorySize + 1, PageSizeBits = (int)options.PageSize + 1, SecondChanceFraction = .2 } }; _serializerSettings = new SerializerSettings <KeyHolder, ValueHolder> { keySerializer = () => new KeySerializer(options.JsonOptions), valueSerializer = () => new ValueSerializer(options.JsonOptions) }; var comparer = new KeyComparerAdapter(keyComparer); var functions = new StoreFunctions(_logger); var checkpointsSettings = new CheckpointSettings { CheckpointDir = _checkpointsPath, CheckPointType = CheckpointType.Snapshot }; _keyValueStore = new FasterKV <KeyHolder, ValueHolder, ValueHolder, ValueHolder, StoreContext, StoreFunctions>( StoreSize, functions, logSettings, checkpointsSettings, _serializerSettings, comparer); var checkpoints = Directory.GetDirectories(_checkpointsPath).Length; if (checkpoints > 0) { _logger.LogInformation($"Found {checkpoints} checkpoints. Recovering store..."); _keyValueStore.Recover(); _logger.LogInformation("Store recovered from checkpoints"); } _session = _keyValueStore.NewSession(); }