public BlittableAllocator(LogSettings settings, IFasterEqualityComparer <Key> comparer, Action <long, long> evictCallback = null, LightEpoch epoch = null, Action <CommitInfo> flushCallback = null)
: base(settings, comparer, evictCallback, epoch, flushCallback)
{
overflowPagePool = new OverflowPool <PageUnit>(4, p =>
#if NET5_0_OR_GREATER
{ }
#else
p.handle.Free()
#endif
);
if (BufferSize > 0)
{
values = new byte[BufferSize][];
#if NET5_0_OR_GREATER
pointers = GC.AllocateArray <long>(BufferSize, true);
nativePointers = (long *)Unsafe.AsPointer(ref pointers[0]);
#else
pointers = new long[BufferSize];
handles = new GCHandle[BufferSize];
ptrHandle = GCHandle.Alloc(pointers, GCHandleType.Pinned);
nativePointers = (long *)ptrHandle.AddrOfPinnedObject();
#endif
}
}
/// <inheritdoc />
public FasterKV(long size, Functions functions, LogSettings logSettings,
CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null,
IFasterEqualityComparer <Key> comparer = null,
VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null)
{
_functions = functions;
_fasterKV = new FasterKV <Key, Value>(size, logSettings, checkpointSettings, serializerSettings, comparer, variableLengthStructSettings);
}
public BlittableAllocator(LogSettings settings, IFasterEqualityComparer <Key> comparer, Action <long, long> evictCallback = null, LightEpoch epoch = null, Action <CommitInfo> flushCallback = null)
: base(settings, comparer, evictCallback, epoch, flushCallback)
{
values = new byte[BufferSize][];
handles = new GCHandle[BufferSize];
pointers = new long[BufferSize];
ptrHandle = GCHandle.Alloc(pointers, GCHandleType.Pinned);
nativePointers = (long *)ptrHandle.AddrOfPinnedObject();
}
public BlittableAllocator(LogSettings settings, IFasterEqualityComparer <Key> comparer)
: base(settings, comparer)
{
values = new byte[BufferSize][];
handles = new GCHandle[BufferSize];
pointers = new long[BufferSize];
epoch = LightEpoch.Instance;
ptrHandle = GCHandle.Alloc(pointers, GCHandleType.Pinned);
nativePointers = (long *)ptrHandle.AddrOfPinnedObject();
}
public KeyValueStore(string folder, int logSizeBits, IFasterEqualityComparer <TKey> keyComparer, TFunctions functions, IVariableLengthStruct <TKey> keyLength, IVariableLengthStruct <TValue> valueLength)
{
_dataFolder = folder;
_logSizeBits = logSizeBits;
_keyComparer = keyComparer;
_functions = functions;
_variableLenSettings = new VariableLengthStructSettings <TKey, TValue>
{
keyLength = keyLength,
valueLength = valueLength
};
}
public KeyValueStore(string folder, int logSizeBits, IFasterEqualityComparer <TKey> keyComparer, TFunctions functions, Func <IObjectSerializer <TKey> > keySerializer, Func <IObjectSerializer <TValue> > valueSerializer)
{
_dataFolder = folder;
_logSizeBits = logSizeBits;
_keyComparer = keyComparer;
_functions = functions;
_serializerSettings = new SerializerSettings <TKey, TValue>
{
keySerializer = keySerializer,
valueSerializer = valueSerializer
};
}
/// <inheritdoc />
public FasterKV(long size, Functions functions, LogSettings logSettings,
CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null,
IFasterEqualityComparer <Key> comparer = null,
VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null, IVariableLengthStruct <Value, Input> variableLengthStructForInput = null)
{
_functions = functions;
_fasterKV = new FasterKV <Key, Value>(size, logSettings, checkpointSettings, serializerSettings, comparer, variableLengthStructSettings);
_variableLengthStructForInput = variableLengthStructForInput;
if (_fasterKV.hlog is VariableLengthBlittableAllocator <Key, Value> allocator && _variableLengthStructForInput == default)
{
_variableLengthStructForInput = new DefaultVariableLengthStruct <Value, Input>(allocator.ValueLength);
}
}
/// <summary>
/// Create FASTER instance
/// </summary>
/// <param name="size">Size of core index (#cache lines)</param>
/// <param name="comparer">FASTER equality comparer for key</param>
/// <param name="functions">Callback functions</param>
/// <param name="logSettings">Log settings</param>
/// <param name="checkpointSettings">Checkpoint settings</param>
/// <param name="serializerSettings">Serializer settings</param>
public FasterKV(long size, Functions functions, LogSettings logSettings, CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null, IFasterEqualityComparer <Key> comparer = null)
{
if (comparer != null)
{
this.comparer = comparer;
}
else
{
if (typeof(IFasterEqualityComparer <Key>).IsAssignableFrom(typeof(Key)))
{
this.comparer = new Key() as IFasterEqualityComparer <Key>;
}
else
{
Console.WriteLine("***WARNING*** Creating default FASTER key equality comparer based on potentially slow EqualityComparer<Key>.Default. To avoid this, provide a comparer (IFasterEqualityComparer<Key>) as an argument to FASTER's constructor, or make Key implement the interface IFasterEqualityComparer<Key>");
this.comparer = FasterEqualityComparer <Key> .Default;
}
}
if (checkpointSettings == null)
{
checkpointSettings = new CheckpointSettings();
}
directoryConfiguration = new DirectoryConfiguration(checkpointSettings.CheckpointDir);
FoldOverSnapshot = checkpointSettings.CheckPointType == core.CheckpointType.FoldOver;
CopyReadsToTail = logSettings.CopyReadsToTail;
this.functions = functions;
if (Utility.IsBlittable <Key>() && Utility.IsBlittable <Value>())
{
hlog = new BlittableAllocator <Key, Value>(logSettings, this.comparer);
}
else
{
hlog = new GenericAllocator <Key, Value>(logSettings, serializerSettings, this.comparer);
}
hlog.Initialize();
sectorSize = (int)logSettings.LogDevice.SectorSize;
Initialize(size, sectorSize);
_systemState = default(SystemState);
_systemState.phase = Phase.REST;
_systemState.version = 1;
_checkpointType = CheckpointType.HYBRID_LOG_ONLY;
}
public FasterDictionary(IFasterEqualityComparer <TKey> keyComparer, Options options = default)
{
if (keyComparer == null)
{
throw new ArgumentNullException(nameof(keyComparer));
}
_keyComparer = new KeyComparerAdapter(keyComparer);
_options = Options.Default;
if (options.PersistDirectoryPath != null)
{
_options.DeleteOnClose = options.DeleteOnClose;
_options.DictionaryName = options.DictionaryName;
_options.Logger = options.Logger;
_options.PersistDirectoryPath = options.PersistDirectoryPath;
if (options.CheckPointType != CheckpointType.Snapshot)
{
_options.CheckPointType = options.CheckPointType;
}
if (options.MemorySize != 0)
{
_options.MemorySize = options.MemorySize;
}
if (options.PageSize != 0)
{
_options.PageSize = options.PageSize;
}
if (options.SegmentSize != 0)
{
_options.SegmentSize = options.SegmentSize;
}
}
JobQueue = Channel.CreateUnbounded <Job>(new UnboundedChannelOptions()
{
AllowSynchronousContinuations = true,
SingleReader = true,
SingleWriter = false
});
StartConsumer();
}
/// <summary>
/// Create FASTER instance
/// </summary>
/// <param name="size">Size of core index (#cache lines)</param>
/// <param name="logSettings">Log settings</param>
/// <param name="checkpointSettings">Checkpoint settings</param>
/// <param name="serializerSettings">Serializer settings</param>
/// <param name="comparer">FASTER equality comparer for key</param>
/// <param name="variableLengthStructSettings"></param>
public FasterKV(long size, LogSettings logSettings,
CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null,
IFasterEqualityComparer <Key> comparer = null,
VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null)
{
if (comparer != null)
{
this.comparer = comparer;
}
else
{
if (typeof(IFasterEqualityComparer <Key>).IsAssignableFrom(typeof(Key)))
{
if (default(Key) != null)
{
this.comparer = default(Key) as IFasterEqualityComparer <Key>;
}
else if (typeof(Key).GetConstructor(Type.EmptyTypes) != null)
{
this.comparer = Activator.CreateInstance(typeof(Key)) as IFasterEqualityComparer <Key>;
}
}
else
{
this.comparer = FasterEqualityComparer.Get <Key>();
}
}
if (checkpointSettings == null)
{
checkpointSettings = new CheckpointSettings();
}
if (checkpointSettings.CheckpointDir != null && checkpointSettings.CheckpointManager != null)
{
throw new FasterException(
"Specify either CheckpointManager or CheckpointDir for CheckpointSettings, not both");
}
bool oldCheckpointManager = false;
if (oldCheckpointManager)
{
checkpointManager = checkpointSettings.CheckpointManager ??
new LocalCheckpointManager(checkpointSettings.CheckpointDir ?? "");
}
else
{
checkpointManager = checkpointSettings.CheckpointManager ??
new DeviceLogCommitCheckpointManager
(new LocalStorageNamedDeviceFactory(),
new DefaultCheckpointNamingScheme(
new DirectoryInfo(checkpointSettings.CheckpointDir ?? ".").FullName), removeOutdated: checkpointSettings.RemoveOutdated);
}
if (checkpointSettings.CheckpointManager == null)
{
disposeCheckpointManager = true;
}
FoldOverSnapshot = checkpointSettings.CheckPointType == core.CheckpointType.FoldOver;
CopyReadsToTail = logSettings.CopyReadsToTail;
if (logSettings.ReadCacheSettings != null)
{
CopyReadsToTail = CopyReadsToTail.None;
UseReadCache = true;
}
UpdateVarLen(ref variableLengthStructSettings);
if ((!Utility.IsBlittable <Key>() && variableLengthStructSettings?.keyLength == null) ||
(!Utility.IsBlittable <Value>() && variableLengthStructSettings?.valueLength == null))
{
WriteDefaultOnDelete = true;
hlog = new GenericAllocator <Key, Value>(logSettings, serializerSettings, this.comparer, null, epoch);
Log = new LogAccessor <Key, Value>(this, hlog);
if (UseReadCache)
{
readcache = new GenericAllocator <Key, Value>(
new LogSettings
{
LogDevice = new NullDevice(),
ObjectLogDevice = new NullDevice(),
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
}, serializerSettings, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value>(this, readcache);
}
}
else if (variableLengthStructSettings != null)
{
hlog = new VariableLengthBlittableAllocator <Key, Value>(logSettings, variableLengthStructSettings,
this.comparer, null, epoch);
Log = new LogAccessor <Key, Value>(this, hlog);
if (UseReadCache)
{
readcache = new VariableLengthBlittableAllocator <Key, Value>(
new LogSettings
{
LogDevice = new NullDevice(),
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
}, variableLengthStructSettings, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value>(this, readcache);
}
}
else
{
hlog = new BlittableAllocator <Key, Value>(logSettings, this.comparer, null, epoch);
Log = new LogAccessor <Key, Value>(this, hlog);
if (UseReadCache)
{
readcache = new BlittableAllocator <Key, Value>(
new LogSettings
{
LogDevice = new NullDevice(),
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
}, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value>(this, readcache);
}
}
hlog.Initialize();
sectorSize = (int)logSettings.LogDevice.SectorSize;
Initialize(size, sectorSize);
systemState = default;
systemState.Phase = Phase.REST;
systemState.Version = 1;
}
internal KeyComparer(IFasterEqualityComparer <TKey> comparer) => this.comparer = comparer;
/// <summary>
/// Create FASTER instance
/// </summary>
/// <param name="size">Size of core index (#cache lines)</param>
/// <param name="comparer">FASTER equality comparer for key</param>
/// <param name="variableLengthStructSettings"></param>
/// <param name="logSettings">Log settings</param>
/// <param name="checkpointSettings">Checkpoint settings</param>
/// <param name="serializerSettings">Serializer settings</param>
public FasterKV(long size, LogSettings logSettings,
CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null,
IFasterEqualityComparer <Key> comparer = null,
VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null)
{
if (comparer != null)
{
this.comparer = comparer;
}
else
{
if (typeof(IFasterEqualityComparer <Key>).IsAssignableFrom(typeof(Key)))
{
this.comparer = new Key() as IFasterEqualityComparer <Key>;
}
else
{
Console.WriteLine(
"***WARNING*** Creating default FASTER key equality comparer based on potentially slow EqualityComparer<Key>.Default. To avoid this, provide a comparer (IFasterEqualityComparer<Key>) as an argument to FASTER's constructor, or make Key implement the interface IFasterEqualityComparer<Key>");
this.comparer = FasterEqualityComparer <Key> .Default;
}
}
if (checkpointSettings == null)
{
checkpointSettings = new CheckpointSettings();
}
if (checkpointSettings.CheckpointDir != null && checkpointSettings.CheckpointManager != null)
{
throw new FasterException(
"Specify either CheckpointManager or CheckpointDir for CheckpointSettings, not both");
}
bool oldCheckpointManager = false;
if (oldCheckpointManager)
{
checkpointManager = checkpointSettings.CheckpointManager ??
new LocalCheckpointManager(checkpointSettings.CheckpointDir ?? "");
}
else
{
checkpointManager = checkpointSettings.CheckpointManager ??
new DeviceLogCommitCheckpointManager
(new LocalStorageNamedDeviceFactory(),
new DefaultCheckpointNamingScheme(
new DirectoryInfo(checkpointSettings.CheckpointDir ?? ".").FullName));
}
if (checkpointSettings.CheckpointManager == null)
{
disposeCheckpointManager = true;
}
FoldOverSnapshot = checkpointSettings.CheckPointType == core.CheckpointType.FoldOver;
CopyReadsToTail = logSettings.CopyReadsToTail;
if (logSettings.ReadCacheSettings != null)
{
CopyReadsToTail = false;
UseReadCache = true;
}
if (Utility.IsBlittable <Key>() && Utility.IsBlittable <Value>())
{
if (variableLengthStructSettings != null)
{
hlog = new VariableLengthBlittableAllocator <Key, Value>(logSettings, variableLengthStructSettings,
this.comparer, null, epoch);
Log = new LogAccessor <Key, Value>(this, hlog);
if (UseReadCache)
{
readcache = new VariableLengthBlittableAllocator <Key, Value>(
new LogSettings
{
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
}, variableLengthStructSettings, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value>(this, readcache);
}
}
else
{
hlog = new BlittableAllocator <Key, Value>(logSettings, this.comparer, null, epoch);
Log = new LogAccessor <Key, Value>(this, hlog);
if (UseReadCache)
{
readcache = new BlittableAllocator <Key, Value>(
new LogSettings
{
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
}, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value>(this, readcache);
}
}
}
else
{
WriteDefaultOnDelete = true;
hlog = new GenericAllocator <Key, Value>(logSettings, serializerSettings, this.comparer, null, epoch);
Log = new LogAccessor <Key, Value>(this, hlog);
if (UseReadCache)
{
readcache = new GenericAllocator <Key, Value>(
new LogSettings
{
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
}, serializerSettings, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value>(this, readcache);
}
}
hlog.Initialize();
sectorSize = (int)logSettings.LogDevice.SectorSize;
Initialize(size, sectorSize);
systemState = default;
systemState.phase = Phase.REST;
systemState.version = 1;
}
public GenericAllocator(LogSettings settings, SerializerSettings <Key, Value> serializerSettings, IFasterEqualityComparer <Key> comparer)
: base(settings, comparer)
{
SerializerSettings = serializerSettings;
if (default(Key) == null && ((SerializerSettings == null) || (SerializerSettings.keySerializer == null)))
{
throw new Exception("Key is a class, but no serializer specified via SerializerSettings");
}
if (default(Value) == null && ((SerializerSettings == null) || (SerializerSettings.valueSerializer == null)))
{
throw new Exception("Value is a class, but no serializer specified via SerializerSettings");
}
values = new Record <Key, Value> [BufferSize][];
segmentOffsets = new long[SegmentBufferSize];
objectLogDevice = settings.ObjectLogDevice;
if (KeyHasObjects() || ValueHasObjects())
{
if (objectLogDevice == null)
{
throw new Exception("Objects in key/value, but object log not provided during creation of FASTER instance");
}
}
epoch = LightEpoch.Instance;
ioBufferPool = SectorAlignedBufferPool.GetPool(1, sectorSize);
}
internal static FasterStore <TKey, TValue> Create <TKey, TValue>(string directory, IFasterEqualityComparer <TKey> comparer)
{
return(new FasterStore <TKey, TValue>(new StoreOptions(directory), comparer,
new NullLogger <FasterStore <TKey, TValue> >()));
}
public KeyComparerAdapter(IFasterEqualityComparer <TKey> keyComparer)
{
_keyComparer = keyComparer;
}
/// <summary>
/// Create FASTER instance
/// </summary>
/// <param name="size">Size of core index (#cache lines)</param>
/// <param name="comparer">FASTER equality comparer for key</param>
/// <param name="functions">Callback functions</param>
/// <param name="logSettings">Log settings</param>
/// <param name="checkpointSettings">Checkpoint settings</param>
/// <param name="serializerSettings">Serializer settings</param>
public FasterKV(long size, Functions functions, LogSettings logSettings, CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null, IFasterEqualityComparer <Key> comparer = null)
{
threadCtx = new FastThreadLocal <FasterExecutionContext>();
prevThreadCtx = new FastThreadLocal <FasterExecutionContext>();
if (comparer != null)
{
this.comparer = comparer;
}
else
{
if (typeof(IFasterEqualityComparer <Key>).IsAssignableFrom(typeof(Key)))
{
this.comparer = new Key() as IFasterEqualityComparer <Key>;
}
else
{
Console.WriteLine("***WARNING*** Creating default FASTER key equality comparer based on potentially slow EqualityComparer<Key>.Default. To avoid this, provide a comparer (IFasterEqualityComparer<Key>) as an argument to FASTER's constructor, or make Key implement the interface IFasterEqualityComparer<Key>");
this.comparer = FasterEqualityComparer <Key> .Default;
}
}
if (checkpointSettings == null)
{
checkpointSettings = new CheckpointSettings();
}
directoryConfiguration = new DirectoryConfiguration(checkpointSettings.CheckpointDir);
FoldOverSnapshot = checkpointSettings.CheckPointType == core.CheckpointType.FoldOver;
CopyReadsToTail = logSettings.CopyReadsToTail;
this.functions = functions;
if (logSettings.ReadCacheSettings != null)
{
CopyReadsToTail = false;
UseReadCache = true;
}
if (Utility.IsBlittable <Key>() && Utility.IsBlittable <Value>())
{
hlog = new BlittableAllocator <Key, Value>(logSettings, this.comparer, null, epoch);
Log = new LogAccessor <Key, Value, Input, Output, Context>(this, hlog);
if (UseReadCache)
{
readcache = new BlittableAllocator <Key, Value>(
new LogSettings {
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = logSettings.ReadCacheSettings.SecondChanceFraction
}, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value, Input, Output, Context>(this, readcache);
}
}
else
{
hlog = new GenericAllocator <Key, Value>(logSettings, serializerSettings, this.comparer, null, epoch);
Log = new LogAccessor <Key, Value, Input, Output, Context>(this, hlog);
if (UseReadCache)
{
readcache = new GenericAllocator <Key, Value>(
new LogSettings
{
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = logSettings.ReadCacheSettings.SecondChanceFraction
}, serializerSettings, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value, Input, Output, Context>(this, readcache);
}
}
hlog.Initialize();
sectorSize = (int)logSettings.LogDevice.SectorSize;
Initialize(size, sectorSize);
_systemState = default(SystemState);
_systemState.phase = Phase.REST;
_systemState.version = 1;
_checkpointType = CheckpointType.HYBRID_LOG_ONLY;
}
public GenericAllocator(LogSettings settings, SerializerSettings <Key, Value> serializerSettings, IFasterEqualityComparer <Key> comparer, Action <long, long> evictCallback = null, LightEpoch epoch = null, Action <CommitInfo> flushCallback = null)
: base(settings, comparer, evictCallback, epoch, flushCallback)
{
SerializerSettings = serializerSettings;
if ((!keyBlittable) && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.keySerializer == null)))
{
throw new FasterException("Key is not blittable, but no serializer specified via SerializerSettings");
}
if ((!valueBlittable) && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.valueSerializer == null)))
{
throw new FasterException("Value is not blittable, but no serializer specified via SerializerSettings");
}
values = new Record <Key, Value> [BufferSize][];
segmentOffsets = new long[SegmentBufferSize];
objectLogDevice = settings.ObjectLogDevice;
if ((settings.LogDevice as NullDevice == null) && (KeyHasObjects() || ValueHasObjects()))
{
if (objectLogDevice == null)
{
throw new FasterException("Objects in key/value, but object log not provided during creation of FASTER instance");
}
if (objectLogDevice.SegmentSize != -1)
{
throw new FasterException("Object log device should not have fixed segment size. Set preallocateFile to false when calling CreateLogDevice for object log");
}
}
}
public GenericAllocator(LogSettings settings, SerializerSettings <Key, Value> serializerSettings, IFasterEqualityComparer <Key> comparer, Action <long, long> evictCallback = null, LightEpoch epoch = null, Action <CommitInfo> flushCallback = null)
: base(settings, comparer, evictCallback, epoch, flushCallback)
{
overflowPagePool = new OverflowPool <Record <Key, Value>[]>(4);
if (settings.ObjectLogDevice == null)
{
throw new FasterException("LogSettings.ObjectLogDevice needs to be specified (e.g., use Devices.CreateLogDevice, AzureStorageDevice, or NullDevice)");
}
SerializerSettings = serializerSettings ?? new SerializerSettings <Key, Value>();
if ((!keyBlittable) && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.keySerializer == null)))
{
Debug.WriteLine("Key is not blittable, but no serializer specified via SerializerSettings. Using (slow) DataContractSerializer as default.");
SerializerSettings.keySerializer = ObjectSerializer.Get <Key>();
}
if ((!valueBlittable) && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.valueSerializer == null)))
{
Debug.WriteLine("Value is not blittable, but no serializer specified via SerializerSettings. Using (slow) DataContractSerializer as default.");
SerializerSettings.valueSerializer = ObjectSerializer.Get <Value>();
}
values = new Record <Key, Value> [BufferSize][];
segmentOffsets = new long[SegmentBufferSize];
objectLogDevice = settings.ObjectLogDevice;
if ((settings.LogDevice as NullDevice == null) && (KeyHasObjects() || ValueHasObjects()))
{
if (objectLogDevice == null)
{
throw new FasterException("Objects in key/value, but object log not provided during creation of FASTER instance");
}
if (objectLogDevice.SegmentSize != -1)
{
throw new FasterException("Object log device should not have fixed segment size. Set preallocateFile to false when calling CreateLogDevice for object log");
}
}
}
public GenericAllocator(LogSettings settings, SerializerSettings <Key, Value> serializerSettings, IFasterEqualityComparer <Key> comparer, Action <long, long> evictCallback = null, LightEpoch epoch = null)
: base(settings, comparer, evictCallback, epoch)
{
SerializerSettings = serializerSettings;
if (default(Key) == null && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.keySerializer == null)))
{
throw new Exception("Key is a class, but no serializer specified via SerializerSettings");
}
if (default(Value) == null && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.valueSerializer == null)))
{
throw new Exception("Value is a class, but no serializer specified via SerializerSettings");
}
values = new Record <Key, Value> [BufferSize][];
segmentOffsets = new long[SegmentBufferSize];
objectLogDevice = settings.ObjectLogDevice;
if ((settings.LogDevice as NullDevice == null) && (KeyHasObjects() || ValueHasObjects()))
{
if (objectLogDevice == null)
{
throw new Exception("Objects in key/value, but object log not provided during creation of FASTER instance");
}
}
}
public VariableLengthBlittableAllocator(LogSettings settings, VariableLengthStructSettings <Key, Value> vlSettings, IFasterEqualityComparer <Key> comparer, Action <long, long> evictCallback = null, LightEpoch epoch = null, Action <CommitInfo> flushCallback = null)
: base(settings, comparer, evictCallback, epoch, flushCallback)
{
values = new byte[BufferSize][];
handles = new GCHandle[BufferSize];
pointers = new long[BufferSize];
ptrHandle = GCHandle.Alloc(pointers, GCHandleType.Pinned);
nativePointers = (long *)ptrHandle.AddrOfPinnedObject();
KeyLength = vlSettings.keyLength;
ValueLength = vlSettings.valueLength;
if (KeyLength == null)
{
fixedSizeKey = true;
KeyLength = new FixedLengthStruct <Key>();
}
if (ValueLength == null)
{
fixedSizeValue = true;
ValueLength = new FixedLengthStruct <Value>();
}
}
public FasterStore(IOptions <FasterStoreOptions> optionsProvider, IFasterEqualityComparer <TKey> keyComparer, ILogger <FasterStore <TKey, TValue> > logger)
{
_logger = logger;
var options = optionsProvider.Value;
var indexStorePath = Path.Combine(options.Directory, $"{options.Name}-index.log");
var objectStorePath = Path.Combine(options.Directory, $"{options.Name}-object.log");
_indexDevice = Devices.CreateLogDevice(indexStorePath,
preallocateFile: false, deleteOnClose: false, recoverDevice: true);
_objectDevice = Devices.CreateLogDevice(objectStorePath,
preallocateFile: false, deleteOnClose: false, recoverDevice: true);
_checkpointsPath = Path.Combine(options.Directory, "checkpoints");
if (!Directory.Exists(_checkpointsPath))
{
_logger.LogInformation("Checkpoints directory not exists. Creating...");
Directory.CreateDirectory(_checkpointsPath);
}
var logSettings = new LogSettings
{
LogDevice = _indexDevice,
ObjectLogDevice = _objectDevice,
MemorySizeBits = (int)options.MemorySize,
PageSizeBits = (int)options.PageSize,
SegmentSizeBits = (int)options.SegmentSize,
ReadCacheSettings = new ReadCacheSettings()
{
MemorySizeBits = (int)options.MemorySize + 1,
PageSizeBits = (int)options.PageSize + 1,
SecondChanceFraction = .2
}
};
_serializerSettings = new SerializerSettings <KeyHolder, ValueHolder>
{
keySerializer = () => new KeySerializer(options.JsonOptions),
valueSerializer = () => new ValueSerializer(options.JsonOptions)
};
var comparer = new KeyComparerAdapter(keyComparer);
var functions = new StoreFunctions(_logger);
var checkpointsSettings = new CheckpointSettings
{
CheckpointDir = _checkpointsPath,
CheckPointType = CheckpointType.Snapshot
};
_keyValueStore = new FasterKV <KeyHolder, ValueHolder, ValueHolder, ValueHolder, StoreContext, StoreFunctions>(
StoreSize, functions, logSettings, checkpointsSettings, _serializerSettings, comparer);
var checkpoints = Directory.GetDirectories(_checkpointsPath).Length;
if (checkpoints > 0)
{
_logger.LogInformation($"Found {checkpoints} checkpoints. Recovering store...");
_keyValueStore.Recover();
_logger.LogInformation("Store recovered from checkpoints");
}
_session = _keyValueStore.NewSession();
}
