public GenericAllocator(LogSettings settings, SerializerSettings <Key, Value> serializerSettings, IFasterEqualityComparer <Key> comparer, Action <long, long> evictCallback = null)
: base(settings, comparer, evictCallback)
{
SerializerSettings = serializerSettings;
if (default(Key) == null && ((SerializerSettings == null) || (SerializerSettings.keySerializer == null)))
{
throw new Exception("Key is a class, but no serializer specified via SerializerSettings");
}
if (default(Value) == null && ((SerializerSettings == null) || (SerializerSettings.valueSerializer == null)))
{
throw new Exception("Value is a class, but no serializer specified via SerializerSettings");
}
values = new Record <Key, Value> [BufferSize][];
segmentOffsets = new long[SegmentBufferSize];
objectLogDevice = settings.ObjectLogDevice;
if (KeyHasObjects() || ValueHasObjects())
{
if (objectLogDevice == null)
{
throw new Exception("Objects in key/value, but object log not provided during creation of FASTER instance");
}
}
epoch = LightEpoch.Instance;
ioBufferPool = SectorAlignedBufferPool.GetPool(1, sectorSize);
}
public GenericAllocator(LogSettings settings, SerializerSettings <Key, Value> serializerSettings, IFasterEqualityComparer <Key> comparer, Action <long, long> evictCallback = null, LightEpoch epoch = null)
: base(settings, comparer, evictCallback, epoch)
{
SerializerSettings = serializerSettings;
if ((!keyBlittable) && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.keySerializer == null)))
{
throw new Exception("Key is not blittable, but no serializer specified via SerializerSettings");
}
if ((!valueBlittable) && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.valueSerializer == null)))
{
throw new Exception("Value is not blittable, but no serializer specified via SerializerSettings");
}
values = new Record <Key, Value> [BufferSize][];
segmentOffsets = new long[SegmentBufferSize];
objectLogDevice = settings.ObjectLogDevice;
if ((settings.LogDevice as NullDevice == null) && (KeyHasObjects() || ValueHasObjects()))
{
if (objectLogDevice == null)
{
throw new Exception("Objects in key/value, but object log not provided during creation of FASTER instance");
}
}
}
/// <inheritdoc />
public FasterKV(long size, Functions functions, LogSettings logSettings,
CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null,
IFasterEqualityComparer <Key> comparer = null,
VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null)
{
_functions = functions;
_fasterKV = new FasterKV <Key, Value>(size, logSettings, checkpointSettings, serializerSettings, comparer, variableLengthStructSettings);
}
/// <inheritdoc />
public FasterKV(long size, Functions functions, LogSettings logSettings,
CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null,
IFasterEqualityComparer <Key> comparer = null,
VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null, IVariableLengthStruct <Value, Input> variableLengthStructForInput = null)
{
_functions = functions;
_fasterKV = new FasterKV <Key, Value>(size, logSettings, checkpointSettings, serializerSettings, comparer, variableLengthStructSettings);
_variableLengthStructForInput = variableLengthStructForInput;
if (_fasterKV.hlog is VariableLengthBlittableAllocator <Key, Value> allocator && _variableLengthStructForInput == default)
{
_variableLengthStructForInput = new DefaultVariableLengthStruct <Value, Input>(allocator.ValueLength);
}
}
public GenericAllocator(LogSettings settings, SerializerSettings <Key, Value> serializerSettings, IFasterEqualityComparer <Key> comparer, Action <long, long> evictCallback = null, LightEpoch epoch = null, Action <CommitInfo> flushCallback = null)
: base(settings, comparer, evictCallback, epoch, flushCallback)
{
overflowPagePool = new OverflowPool <Record <Key, Value>[]>(4);
if (settings.ObjectLogDevice == null)
{
throw new FasterException("LogSettings.ObjectLogDevice needs to be specified (e.g., use Devices.CreateLogDevice, AzureStorageDevice, or NullDevice)");
}
SerializerSettings = serializerSettings ?? new SerializerSettings <Key, Value>();
if ((!keyBlittable) && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.keySerializer == null)))
{
#if DEBUG
if (typeof(Key) != typeof(byte[]) && typeof(Key) != typeof(string))
{
Debug.WriteLine("Key is not blittable, but no serializer specified via SerializerSettings. Using (slow) DataContractSerializer as default.");
}
#endif
SerializerSettings.keySerializer = ObjectSerializer.Get <Key>();
}
if ((!valueBlittable) && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.valueSerializer == null)))
{
#if DEBUG
if (typeof(Value) != typeof(byte[]) && typeof(Value) != typeof(string))
{
Debug.WriteLine("Value is not blittable, but no serializer specified via SerializerSettings. Using (slow) DataContractSerializer as default.");
}
#endif
SerializerSettings.valueSerializer = ObjectSerializer.Get <Value>();
}
values = new Record <Key, Value> [BufferSize][];
segmentOffsets = new long[SegmentBufferSize];
objectLogDevice = settings.ObjectLogDevice;
if ((settings.LogDevice as NullDevice == null) && (KeyHasObjects() || ValueHasObjects()))
{
if (objectLogDevice == null)
{
throw new FasterException("Objects in key/value, but object log not provided during creation of FASTER instance");
}
if (objectLogDevice.SegmentSize != -1)
{
throw new FasterException("Object log device should not have fixed segment size. Set preallocateFile to false when calling CreateLogDevice for object log");
}
}
}
/// <summary>
/// Create FASTER instance
/// </summary>
/// <param name="size">Size of core index (#cache lines)</param>
/// <param name="logSettings">Log settings</param>
/// <param name="checkpointSettings">Checkpoint settings</param>
/// <param name="serializerSettings">Serializer settings</param>
/// <param name="comparer">FASTER equality comparer for key</param>
/// <param name="variableLengthStructSettings"></param>
public FasterKV(long size, LogSettings logSettings,
CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null,
IFasterEqualityComparer <Key> comparer = null,
VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null)
{
if (comparer != null)
{
this.comparer = comparer;
}
else
{
if (typeof(IFasterEqualityComparer <Key>).IsAssignableFrom(typeof(Key)))
{
if (default(Key) != null)
{
this.comparer = default(Key) as IFasterEqualityComparer <Key>;
}
else if (typeof(Key).GetConstructor(Type.EmptyTypes) != null)
{
this.comparer = Activator.CreateInstance(typeof(Key)) as IFasterEqualityComparer <Key>;
}
}
else
{
this.comparer = FasterEqualityComparer.Get <Key>();
}
}
if (checkpointSettings == null)
{
checkpointSettings = new CheckpointSettings();
}
if (checkpointSettings.CheckpointDir != null && checkpointSettings.CheckpointManager != null)
{
throw new FasterException(
"Specify either CheckpointManager or CheckpointDir for CheckpointSettings, not both");
}
bool oldCheckpointManager = false;
if (oldCheckpointManager)
{
checkpointManager = checkpointSettings.CheckpointManager ??
new LocalCheckpointManager(checkpointSettings.CheckpointDir ?? "");
}
else
{
checkpointManager = checkpointSettings.CheckpointManager ??
new DeviceLogCommitCheckpointManager
(new LocalStorageNamedDeviceFactory(),
new DefaultCheckpointNamingScheme(
new DirectoryInfo(checkpointSettings.CheckpointDir ?? ".").FullName), removeOutdated: checkpointSettings.RemoveOutdated);
}
if (checkpointSettings.CheckpointManager == null)
{
disposeCheckpointManager = true;
}
FoldOverSnapshot = checkpointSettings.CheckPointType == core.CheckpointType.FoldOver;
CopyReadsToTail = logSettings.CopyReadsToTail;
if (logSettings.ReadCacheSettings != null)
{
CopyReadsToTail = CopyReadsToTail.None;
UseReadCache = true;
}
UpdateVarLen(ref variableLengthStructSettings);
if ((!Utility.IsBlittable <Key>() && variableLengthStructSettings?.keyLength == null) ||
(!Utility.IsBlittable <Value>() && variableLengthStructSettings?.valueLength == null))
{
WriteDefaultOnDelete = true;
hlog = new GenericAllocator <Key, Value>(logSettings, serializerSettings, this.comparer, null, epoch);
Log = new LogAccessor <Key, Value>(this, hlog);
if (UseReadCache)
{
readcache = new GenericAllocator <Key, Value>(
new LogSettings
{
LogDevice = new NullDevice(),
ObjectLogDevice = new NullDevice(),
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
}, serializerSettings, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value>(this, readcache);
}
}
else if (variableLengthStructSettings != null)
{
hlog = new VariableLengthBlittableAllocator <Key, Value>(logSettings, variableLengthStructSettings,
this.comparer, null, epoch);
Log = new LogAccessor <Key, Value>(this, hlog);
if (UseReadCache)
{
readcache = new VariableLengthBlittableAllocator <Key, Value>(
new LogSettings
{
LogDevice = new NullDevice(),
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
}, variableLengthStructSettings, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value>(this, readcache);
}
}
else
{
hlog = new BlittableAllocator <Key, Value>(logSettings, this.comparer, null, epoch);
Log = new LogAccessor <Key, Value>(this, hlog);
if (UseReadCache)
{
readcache = new BlittableAllocator <Key, Value>(
new LogSettings
{
LogDevice = new NullDevice(),
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
}, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value>(this, readcache);
}
}
hlog.Initialize();
sectorSize = (int)logSettings.LogDevice.SectorSize;
Initialize(size, sectorSize);
systemState = default;
systemState.Phase = Phase.REST;
systemState.Version = 1;
}
/// <summary>
/// Create FASTER instance
/// </summary>
/// <param name="size">Size of core index (#cache lines)</param>
/// <param name="comparer">FASTER equality comparer for key</param>
/// <param name="functions">Callback functions</param>
/// <param name="logSettings">Log settings</param>
/// <param name="checkpointSettings">Checkpoint settings</param>
/// <param name="serializerSettings">Serializer settings</param>
public FasterKV(long size, Functions functions, LogSettings logSettings, CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null, IFasterEqualityComparer <Key> comparer = null)
{
if (comparer != null)
{
this.comparer = comparer;
}
else
{
if (typeof(IFasterEqualityComparer <Key>).IsAssignableFrom(typeof(Key)))
{
this.comparer = new Key() as IFasterEqualityComparer <Key>;
}
else
{
Console.WriteLine("***WARNING*** Creating default FASTER key equality comparer based on potentially slow EqualityComparer<Key>.Default. To avoid this, provide a comparer (IFasterEqualityComparer<Key>) as an argument to FASTER's constructor, or make Key implement the interface IFasterEqualityComparer<Key>");
this.comparer = FasterEqualityComparer <Key> .Default;
}
}
if (checkpointSettings == null)
{
checkpointSettings = new CheckpointSettings();
}
directoryConfiguration = new DirectoryConfiguration(checkpointSettings.CheckpointDir);
FoldOverSnapshot = checkpointSettings.CheckPointType == core.CheckpointType.FoldOver;
CopyReadsToTail = logSettings.CopyReadsToTail;
this.functions = functions;
if (Utility.IsBlittable <Key>() && Utility.IsBlittable <Value>())
{
hlog = new BlittableAllocator <Key, Value>(logSettings, this.comparer);
}
else
{
hlog = new GenericAllocator <Key, Value>(logSettings, serializerSettings, this.comparer);
}
hlog.Initialize();
sectorSize = (int)logSettings.LogDevice.SectorSize;
Initialize(size, sectorSize);
_systemState = default(SystemState);
_systemState.phase = Phase.REST;
_systemState.version = 1;
_checkpointType = CheckpointType.HYBRID_LOG_ONLY;
}
/// <summary>
/// Create FASTER instance
/// </summary>
/// <param name="size">Size of core index (#cache lines)</param>
/// <param name="comparer">FASTER equality comparer for key</param>
/// <param name="variableLengthStructSettings"></param>
/// <param name="logSettings">Log settings</param>
/// <param name="checkpointSettings">Checkpoint settings</param>
/// <param name="serializerSettings">Serializer settings</param>
public FasterKV(long size, LogSettings logSettings,
CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null,
IFasterEqualityComparer <Key> comparer = null,
VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null)
{
if (comparer != null)
{
this.comparer = comparer;
}
else
{
if (typeof(IFasterEqualityComparer <Key>).IsAssignableFrom(typeof(Key)))
{
this.comparer = new Key() as IFasterEqualityComparer <Key>;
}
else
{
Console.WriteLine(
"***WARNING*** Creating default FASTER key equality comparer based on potentially slow EqualityComparer<Key>.Default. To avoid this, provide a comparer (IFasterEqualityComparer<Key>) as an argument to FASTER's constructor, or make Key implement the interface IFasterEqualityComparer<Key>");
this.comparer = FasterEqualityComparer <Key> .Default;
}
}
if (checkpointSettings == null)
{
checkpointSettings = new CheckpointSettings();
}
if (checkpointSettings.CheckpointDir != null && checkpointSettings.CheckpointManager != null)
{
throw new FasterException(
"Specify either CheckpointManager or CheckpointDir for CheckpointSettings, not both");
}
bool oldCheckpointManager = false;
if (oldCheckpointManager)
{
checkpointManager = checkpointSettings.CheckpointManager ??
new LocalCheckpointManager(checkpointSettings.CheckpointDir ?? "");
}
else
{
checkpointManager = checkpointSettings.CheckpointManager ??
new DeviceLogCommitCheckpointManager
(new LocalStorageNamedDeviceFactory(),
new DefaultCheckpointNamingScheme(
new DirectoryInfo(checkpointSettings.CheckpointDir ?? ".").FullName));
}
if (checkpointSettings.CheckpointManager == null)
{
disposeCheckpointManager = true;
}
FoldOverSnapshot = checkpointSettings.CheckPointType == core.CheckpointType.FoldOver;
CopyReadsToTail = logSettings.CopyReadsToTail;
if (logSettings.ReadCacheSettings != null)
{
CopyReadsToTail = false;
UseReadCache = true;
}
if (Utility.IsBlittable <Key>() && Utility.IsBlittable <Value>())
{
if (variableLengthStructSettings != null)
{
hlog = new VariableLengthBlittableAllocator <Key, Value>(logSettings, variableLengthStructSettings,
this.comparer, null, epoch);
Log = new LogAccessor <Key, Value>(this, hlog);
if (UseReadCache)
{
readcache = new VariableLengthBlittableAllocator <Key, Value>(
new LogSettings
{
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
}, variableLengthStructSettings, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value>(this, readcache);
}
}
else
{
hlog = new BlittableAllocator <Key, Value>(logSettings, this.comparer, null, epoch);
Log = new LogAccessor <Key, Value>(this, hlog);
if (UseReadCache)
{
readcache = new BlittableAllocator <Key, Value>(
new LogSettings
{
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
}, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value>(this, readcache);
}
}
}
else
{
WriteDefaultOnDelete = true;
hlog = new GenericAllocator <Key, Value>(logSettings, serializerSettings, this.comparer, null, epoch);
Log = new LogAccessor <Key, Value>(this, hlog);
if (UseReadCache)
{
readcache = new GenericAllocator <Key, Value>(
new LogSettings
{
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
}, serializerSettings, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value>(this, readcache);
}
}
hlog.Initialize();
sectorSize = (int)logSettings.LogDevice.SectorSize;
Initialize(size, sectorSize);
systemState = default;
systemState.phase = Phase.REST;
systemState.version = 1;
}
/// <summary>
/// Create FASTER instance
/// </summary>
/// <param name="size">Size of core index (#cache lines)</param>
/// <param name="comparer">FASTER equality comparer for key</param>
/// <param name="functions">Callback functions</param>
/// <param name="logSettings">Log settings</param>
/// <param name="checkpointSettings">Checkpoint settings</param>
/// <param name="serializerSettings">Serializer settings</param>
public FasterKV(long size, Functions functions, LogSettings logSettings, CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null, IFasterEqualityComparer <Key> comparer = null)
{
threadCtx = new FastThreadLocal <FasterExecutionContext>();
prevThreadCtx = new FastThreadLocal <FasterExecutionContext>();
if (comparer != null)
{
this.comparer = comparer;
}
else
{
if (typeof(IFasterEqualityComparer <Key>).IsAssignableFrom(typeof(Key)))
{
this.comparer = new Key() as IFasterEqualityComparer <Key>;
}
else
{
Console.WriteLine("***WARNING*** Creating default FASTER key equality comparer based on potentially slow EqualityComparer<Key>.Default. To avoid this, provide a comparer (IFasterEqualityComparer<Key>) as an argument to FASTER's constructor, or make Key implement the interface IFasterEqualityComparer<Key>");
this.comparer = FasterEqualityComparer <Key> .Default;
}
}
if (checkpointSettings == null)
{
checkpointSettings = new CheckpointSettings();
}
directoryConfiguration = new DirectoryConfiguration(checkpointSettings.CheckpointDir);
FoldOverSnapshot = checkpointSettings.CheckPointType == core.CheckpointType.FoldOver;
CopyReadsToTail = logSettings.CopyReadsToTail;
this.functions = functions;
if (logSettings.ReadCacheSettings != null)
{
CopyReadsToTail = false;
UseReadCache = true;
}
if (Utility.IsBlittable <Key>() && Utility.IsBlittable <Value>())
{
hlog = new BlittableAllocator <Key, Value>(logSettings, this.comparer, null, epoch);
Log = new LogAccessor <Key, Value, Input, Output, Context>(this, hlog);
if (UseReadCache)
{
readcache = new BlittableAllocator <Key, Value>(
new LogSettings {
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = logSettings.ReadCacheSettings.SecondChanceFraction
}, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value, Input, Output, Context>(this, readcache);
}
}
else
{
hlog = new GenericAllocator <Key, Value>(logSettings, serializerSettings, this.comparer, null, epoch);
Log = new LogAccessor <Key, Value, Input, Output, Context>(this, hlog);
if (UseReadCache)
{
readcache = new GenericAllocator <Key, Value>(
new LogSettings
{
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = logSettings.ReadCacheSettings.SecondChanceFraction
}, serializerSettings, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value, Input, Output, Context>(this, readcache);
}
}
hlog.Initialize();
sectorSize = (int)logSettings.LogDevice.SectorSize;
Initialize(size, sectorSize);
_systemState = default(SystemState);
_systemState.phase = Phase.REST;
_systemState.version = 1;
_checkpointType = CheckpointType.HYBRID_LOG_ONLY;
}
