public VariableLengthBlittableAllocator(LogSettings settings, VariableLengthStructSettings <Key, Value> vlSettings, IFasterEqualityComparer <Key> comparer, Action <long, long> evictCallback = null, LightEpoch epoch = null, Action <CommitInfo> flushCallback = null)
: base(settings, comparer, evictCallback, epoch, flushCallback)
{
values = new byte[BufferSize][];
handles = new GCHandle[BufferSize];
pointers = new long[BufferSize];
ptrHandle = GCHandle.Alloc(pointers, GCHandleType.Pinned);
nativePointers = (long *)ptrHandle.AddrOfPinnedObject();
KeyLength = vlSettings.keyLength;
ValueLength = vlSettings.valueLength;
if (KeyLength == null)
{
fixedSizeKey = true;
KeyLength = new FixedLengthStruct <Key>();
}
if (ValueLength == null)
{
fixedSizeValue = true;
ValueLength = new FixedLengthStruct <Value>();
}
}
public long Compact(long untilAddress, bool shiftBeginAddress)
{
if (allocator is VariableLengthBlittableAllocator <Key, Value> varLen)
{
if (typeof(Key).IsGenericType && (typeof(Key).GetGenericTypeDefinition() == typeof(ReadOnlyMemory <>)) && Utility.IsBlittableType(typeof(Key).GetGenericArguments()[0]) &&
typeof(Value).IsGenericType && (typeof(Value).GetGenericTypeDefinition() == typeof(Memory <>)) && Utility.IsBlittableType(typeof(Value).GetGenericArguments()[0]))
{
MethodInfo method = GetType().GetMethod("CompactReadOnly", BindingFlags.NonPublic | BindingFlags.Instance);
MethodInfo generic = method.MakeGenericMethod(typeof(Key).GetGenericArguments()[0]);
return((long)generic.Invoke(this, new object[] { untilAddress, shiftBeginAddress }));
}
else if (typeof(Key).IsGenericType && (typeof(Key).GetGenericTypeDefinition() == typeof(Memory <>)) && Utility.IsBlittableType(typeof(Key).GetGenericArguments()[0]) &&
typeof(Value).IsGenericType && (typeof(Value).GetGenericTypeDefinition() == typeof(Memory <>)) && Utility.IsBlittableType(typeof(Value).GetGenericArguments()[0]))
{
MethodInfo method = GetType().GetMethod("CompactMemory", BindingFlags.NonPublic | BindingFlags.Instance);
MethodInfo generic = method.MakeGenericMethod(typeof(Key).GetGenericArguments()[0]);
return((long)generic.Invoke(this, new object[] { untilAddress, shiftBeginAddress }));
}
else
{
var functions = new LogVariableCompactFunctions <Key, Value, DefaultVariableCompactionFunctions <Key, Value> >(varLen, default);
var variableLengthStructSettings = new VariableLengthStructSettings <Key, Value>
{
keyLength = varLen.KeyLength,
valueLength = varLen.ValueLength,
};
return(Compact(functions, default(DefaultVariableCompactionFunctions <Key, Value>), untilAddress, variableLengthStructSettings, shiftBeginAddress));
}
}
else
{
return(Compact(new LogCompactFunctions <Key, Value, DefaultCompactionFunctions <Key, Value> >(default), default(DefaultCompactionFunctions <Key, Value>), untilAddress, null, shiftBeginAddress));
private void Compact <T>(T functions, long untilAddress, VariableLengthStructSettings <Key, Value> variableLengthStructSettings)
where T : IFunctions <Key, Value, Input, Output, Context>
{
var originalUntilAddress = untilAddress;
var tempKv = new FasterKV <Key, Value, Input, Output, Context, T>
(fht.IndexSize, functions, new LogSettings(), comparer: fht.Comparer, variableLengthStructSettings: variableLengthStructSettings);
tempKv.StartSession();
int cnt = 0;
using (var iter1 = fht.Log.Scan(fht.Log.BeginAddress, untilAddress))
{
while (iter1.GetNext(out RecordInfo recordInfo))
{
ref var key = ref iter1.GetKey();
ref var value = ref iter1.GetValue();
if (recordInfo.Tombstone)
{
tempKv.Delete(ref key, default(Context), 0);
}
else
{
tempKv.Upsert(ref key, ref value, default(Context), 0);
}
if (++cnt % 1000 == 0)
{
fht.Refresh();
tempKv.Refresh();
}
}
/// <inheritdoc />
public FasterKV(long size, Functions functions, LogSettings logSettings,
CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null,
IFasterEqualityComparer <Key> comparer = null,
VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null)
{
_functions = functions;
_fasterKV = new FasterKV <Key, Value>(size, logSettings, checkpointSettings, serializerSettings, comparer, variableLengthStructSettings);
}
/// <inheritdoc />
public FasterKV(long size, Functions functions, LogSettings logSettings,
CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null,
IFasterEqualityComparer <Key> comparer = null,
VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null, IVariableLengthStruct <Value, Input> variableLengthStructForInput = null)
{
_functions = functions;
_fasterKV = new FasterKV <Key, Value>(size, logSettings, checkpointSettings, serializerSettings, comparer, variableLengthStructSettings);
_variableLengthStructForInput = variableLengthStructForInput;
if (_fasterKV.hlog is VariableLengthBlittableAllocator <Key, Value> allocator && _variableLengthStructForInput == default)
{
_variableLengthStructForInput = new DefaultVariableLengthStruct <Value, Input>(allocator.ValueLength);
}
}
/// <summary>
/// Compact the log until specified address, moving active
/// records to the tail of the log.
/// Uses default compaction functions that only deletes explicitly deleted records, copying is implemeted by shallow copying values from source to destination.
/// </summary>
/// <param name="untilAddress"></param>
public void Compact(long untilAddress)
{
if (allocator is VariableLengthBlittableAllocator <Key, Value> varLen)
{
var functions = new LogVariableCompactFunctions <Key, Value, DefaultVariableCompactionFunctions <Key, Value> >(varLen, default);
var variableLengthStructSettings = new VariableLengthStructSettings <Key, Value>
{
keyLength = varLen.KeyLength,
valueLength = varLen.ValueLength,
};
Compact(functions, default(DefaultVariableCompactionFunctions <Key, Value>), untilAddress, variableLengthStructSettings);
}
else
{
Compact(new LogCompactFunctions <Key, Value, DefaultCompactionFunctions <Key, Value> >(default), default(DefaultCompactionFunctions <Key, Value>), untilAddress, null);
/// <summary>
/// Iterator for all (distinct) live key-values stored in FASTER
/// </summary>
/// <param name="untilAddress">Report records until this address (tail by default)</param>
/// <returns>FASTER iterator</returns>
public IFasterScanIterator <Key, Value> Iterate(long untilAddress = -1)
{
if (untilAddress == -1)
{
untilAddress = Log.TailAddress;
}
if (hlog is VariableLengthBlittableAllocator <Key, Value> varLen)
{
var functions = new LogVariableCompactFunctions <Key, Value, DefaultVariableCompactionFunctions <Key, Value> >(varLen, default);
var variableLengthStructSettings = new VariableLengthStructSettings <Key, Value>
{
keyLength = varLen.KeyLength,
valueLength = varLen.ValueLength,
};
return(new FasterKVIterator <Key, Value, LogVariableCompactFunctions <Key, Value, DefaultVariableCompactionFunctions <Key, Value> >, DefaultVariableCompactionFunctions <Key, Value> >
(this, functions, default, untilAddress, variableLengthStructSettings));
public VariableLengthBlittableAllocator(LogSettings settings, VariableLengthStructSettings <Key, Value> vlSettings, IFasterEqualityComparer <Key> comparer, Action <long, long> evictCallback = null, LightEpoch epoch = null, Action <CommitInfo> flushCallback = null)
: base(settings, comparer, evictCallback, epoch, flushCallback)
{
overflowPagePool = new OverflowPool <PageUnit>(4, p =>
#if NET5_0_OR_GREATER
{ }
#else
p.handle.Free()
#endif
);
if (BufferSize > 0)
{
values = new byte[BufferSize][];
#if NET5_0_OR_GREATER
pointers = GC.AllocateArray <long>(BufferSize, true);
nativePointers = (long *)Unsafe.AsPointer(ref pointers[0]);
#else
pointers = new long[BufferSize];
handles = new GCHandle[BufferSize];
ptrHandle = GCHandle.Alloc(pointers, GCHandleType.Pinned);
nativePointers = (long *)ptrHandle.AddrOfPinnedObject();
#endif
}
KeyLength = vlSettings.keyLength;
ValueLength = vlSettings.valueLength;
if (KeyLength == null)
{
fixedSizeKey = true;
KeyLength = new FixedLengthStruct <Key>();
}
if (ValueLength == null)
{
fixedSizeValue = true;
ValueLength = new FixedLengthStruct <Value>();
}
}
/// <summary>
/// Compact the log until specified address using current session, moving active records to the tail of the log.
/// </summary>
/// <param name="untilAddress">Compact log until this address</param>
/// <param name="shiftBeginAddress">Whether to shift begin address to untilAddress after compaction. To avoid
/// <param name="compactionFunctions">User provided compaction functions (see <see cref="ICompactionFunctions{Key, Value}"/>).</param>
/// data loss on failure, set this to false, and shift begin address only after taking a checkpoint. This
/// ensures that records written to the tail during compaction are first made stable.</param>
/// <returns>Address until which compaction was done</returns>
public long Compact <CompactionFunctions>(long untilAddress, bool shiftBeginAddress, CompactionFunctions compactionFunctions)
where CompactionFunctions : ICompactionFunctions <Key, Value>
{
if (!SupportAsync)
{
throw new FasterException("Do not perform compaction using a threadAffinitized session");
}
VariableLengthStructSettings <Key, Value> vl = null;
if (fht.hlog is VariableLengthBlittableAllocator <Key, Value> varLen)
{
vl = new VariableLengthStructSettings <Key, Value>
{
keyLength = varLen.KeyLength,
valueLength = varLen.ValueLength,
};
}
return(fht.Log.Compact(this, functions, compactionFunctions, untilAddress, vl, shiftBeginAddress));
}
private void Compact <T>(T functions, long untilAddress, VariableLengthStructSettings <Key, Value> variableLengthStructSettings)
where T : IFunctions <Key, Value, Input, Output, Context>
{
var fhtSession = fht.NewSession();
var originalUntilAddress = untilAddress;
var tempKv = new FasterKV <Key, Value, Input, Output, Context, T>
(fht.IndexSize, functions, new LogSettings(), comparer: fht.Comparer, variableLengthStructSettings: variableLengthStructSettings);
var tempKvSession = tempKv.NewSession();
using (var iter1 = fht.Log.Scan(fht.Log.BeginAddress, untilAddress))
{
while (iter1.GetNext(out RecordInfo recordInfo))
{
ref var key = ref iter1.GetKey();
ref var value = ref iter1.GetValue();
if (recordInfo.Tombstone)
{
tempKvSession.Delete(ref key, default, 0);
/// <summary>
/// Compact the log until specified address, moving active records to the tail of the log.
/// </summary>
/// <param name="functions">Functions used to manage key-values during compaction</param>
/// <param name="cf">User provided compaction functions (see <see cref="ICompactionFunctions{Key, Value}"/>).</param>
/// <param name="untilAddress">Compact log until this address</param>
/// <param name="shiftBeginAddress">Whether to shift begin address to untilAddress after compaction. To avoid
/// data loss on failure, set this to false, and shift begin address only after taking a checkpoint. This
/// ensures that records written to the tail during compaction are first made stable.</param>
/// <returns>Address until which compaction was done</returns>
public long Compact <Input, Output, Context, Functions, CompactionFunctions>(Functions functions, CompactionFunctions cf, long untilAddress, bool shiftBeginAddress)
where Functions : IFunctions <Key, Value, Input, Output, Context>
where CompactionFunctions : ICompactionFunctions <Key, Value>
{
if (untilAddress > fht.Log.SafeReadOnlyAddress)
{
throw new FasterException("Can compact only until Log.SafeReadOnlyAddress");
}
var originalUntilAddress = untilAddress;
var lf = new LogCompactionFunctions <Key, Value, Input, Output, Context, Functions>(functions);
using var fhtSession = fht.For(lf).NewSession <LogCompactionFunctions <Key, Value, Input, Output, Context, Functions> >();
VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null;
if (allocator is VariableLengthBlittableAllocator <Key, Value> varLen)
{
variableLengthStructSettings = new VariableLengthStructSettings <Key, Value>
{
keyLength = varLen.KeyLength,
valueLength = varLen.ValueLength,
};
}
using (var tempKv = new FasterKV <Key, Value>(fht.IndexSize, new LogSettings {
LogDevice = new NullDevice(), ObjectLogDevice = new NullDevice()
}, comparer: fht.Comparer, variableLengthStructSettings: variableLengthStructSettings))
using (var tempKvSession = tempKv.NewSession <Input, Output, Context, Functions>(functions))
{
using (var iter1 = fht.Log.Scan(fht.Log.BeginAddress, untilAddress))
{
while (iter1.GetNext(out var recordInfo))
{
ref var key = ref iter1.GetKey();
ref var value = ref iter1.GetValue();
if (recordInfo.Tombstone || cf.IsDeleted(key, value))
{
tempKvSession.Delete(ref key, default, 0);
public FasterKVIterator(FasterKV <Key, Value> fht, Functions functions, long untilAddress)
{
this.fht = fht;
enumerationPhase = 0;
VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null;
if (fht.hlog is VariableLengthBlittableAllocator <Key, Value> varLen)
{
variableLengthStructSettings = new VariableLengthStructSettings <Key, Value>
{
keyLength = varLen.KeyLength,
valueLength = varLen.ValueLength,
};
}
tempKv = new FasterKV <Key, Value>(fht.IndexSize, new LogSettings {
LogDevice = new NullDevice(), ObjectLogDevice = new NullDevice(), MutableFraction = 1
}, comparer: fht.Comparer, variableLengthStructSettings: variableLengthStructSettings);
tempKvSession = tempKv.NewSession <Input, Output, Context, Functions>(functions);
iter1 = fht.Log.Scan(fht.Log.BeginAddress, untilAddress);
}
/// <summary>
/// Create FASTER instance
/// </summary>
/// <param name="size">Size of core index (#cache lines)</param>
/// <param name="logSettings">Log settings</param>
/// <param name="checkpointSettings">Checkpoint settings</param>
/// <param name="serializerSettings">Serializer settings</param>
/// <param name="comparer">FASTER equality comparer for key</param>
/// <param name="variableLengthStructSettings"></param>
public FasterKV(long size, LogSettings logSettings,
CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null,
IFasterEqualityComparer <Key> comparer = null,
VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null)
{
if (comparer != null)
{
this.comparer = comparer;
}
else
{
if (typeof(IFasterEqualityComparer <Key>).IsAssignableFrom(typeof(Key)))
{
if (default(Key) != null)
{
this.comparer = default(Key) as IFasterEqualityComparer <Key>;
}
else if (typeof(Key).GetConstructor(Type.EmptyTypes) != null)
{
this.comparer = Activator.CreateInstance(typeof(Key)) as IFasterEqualityComparer <Key>;
}
}
else
{
this.comparer = FasterEqualityComparer.Get <Key>();
}
}
if (checkpointSettings == null)
{
checkpointSettings = new CheckpointSettings();
}
if (checkpointSettings.CheckpointDir != null && checkpointSettings.CheckpointManager != null)
{
throw new FasterException(
"Specify either CheckpointManager or CheckpointDir for CheckpointSettings, not both");
}
bool oldCheckpointManager = false;
if (oldCheckpointManager)
{
checkpointManager = checkpointSettings.CheckpointManager ??
new LocalCheckpointManager(checkpointSettings.CheckpointDir ?? "");
}
else
{
checkpointManager = checkpointSettings.CheckpointManager ??
new DeviceLogCommitCheckpointManager
(new LocalStorageNamedDeviceFactory(),
new DefaultCheckpointNamingScheme(
new DirectoryInfo(checkpointSettings.CheckpointDir ?? ".").FullName), removeOutdated: checkpointSettings.RemoveOutdated);
}
if (checkpointSettings.CheckpointManager == null)
{
disposeCheckpointManager = true;
}
FoldOverSnapshot = checkpointSettings.CheckPointType == core.CheckpointType.FoldOver;
CopyReadsToTail = logSettings.CopyReadsToTail;
if (logSettings.ReadCacheSettings != null)
{
CopyReadsToTail = CopyReadsToTail.None;
UseReadCache = true;
}
UpdateVarLen(ref variableLengthStructSettings);
if ((!Utility.IsBlittable <Key>() && variableLengthStructSettings?.keyLength == null) ||
(!Utility.IsBlittable <Value>() && variableLengthStructSettings?.valueLength == null))
{
WriteDefaultOnDelete = true;
hlog = new GenericAllocator <Key, Value>(logSettings, serializerSettings, this.comparer, null, epoch);
Log = new LogAccessor <Key, Value>(this, hlog);
if (UseReadCache)
{
readcache = new GenericAllocator <Key, Value>(
new LogSettings
{
LogDevice = new NullDevice(),
ObjectLogDevice = new NullDevice(),
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
}, serializerSettings, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value>(this, readcache);
}
}
else if (variableLengthStructSettings != null)
{
hlog = new VariableLengthBlittableAllocator <Key, Value>(logSettings, variableLengthStructSettings,
this.comparer, null, epoch);
Log = new LogAccessor <Key, Value>(this, hlog);
if (UseReadCache)
{
readcache = new VariableLengthBlittableAllocator <Key, Value>(
new LogSettings
{
LogDevice = new NullDevice(),
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
}, variableLengthStructSettings, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value>(this, readcache);
}
}
else
{
hlog = new BlittableAllocator <Key, Value>(logSettings, this.comparer, null, epoch);
Log = new LogAccessor <Key, Value>(this, hlog);
if (UseReadCache)
{
readcache = new BlittableAllocator <Key, Value>(
new LogSettings
{
LogDevice = new NullDevice(),
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
}, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value>(this, readcache);
}
}
hlog.Initialize();
sectorSize = (int)logSettings.LogDevice.SectorSize;
Initialize(size, sectorSize);
systemState = default;
systemState.Phase = Phase.REST;
systemState.Version = 1;
}
private static void UpdateVarLen(ref VariableLengthStructSettings <Key, Value> variableLengthStructSettings)
{
if (typeof(Key) == typeof(SpanByte))
{
if (variableLengthStructSettings == null)
{
variableLengthStructSettings = new VariableLengthStructSettings <SpanByte, Value>() as VariableLengthStructSettings <Key, Value>;
}
if (variableLengthStructSettings.keyLength == null)
{
(variableLengthStructSettings as VariableLengthStructSettings <SpanByte, Value>).keyLength = new SpanByteVarLenStruct();
}
}
else if (typeof(Key).IsGenericType && (typeof(Key).GetGenericTypeDefinition() == typeof(Memory <>)) && Utility.IsBlittableType(typeof(Key).GetGenericArguments()[0]))
{
if (variableLengthStructSettings == null)
{
variableLengthStructSettings = new VariableLengthStructSettings <Key, Value>();
}
if (variableLengthStructSettings.keyLength == null)
{
var m = typeof(MemoryVarLenStruct <>).MakeGenericType(typeof(Key).GetGenericArguments());
object o = Activator.CreateInstance(m);
variableLengthStructSettings.keyLength = o as IVariableLengthStruct <Key>;
}
}
else if (typeof(Key).IsGenericType && (typeof(Key).GetGenericTypeDefinition() == typeof(ReadOnlyMemory <>)) && Utility.IsBlittableType(typeof(Key).GetGenericArguments()[0]))
{
if (variableLengthStructSettings == null)
{
variableLengthStructSettings = new VariableLengthStructSettings <Key, Value>();
}
if (variableLengthStructSettings.keyLength == null)
{
var m = typeof(ReadOnlyMemoryVarLenStruct <>).MakeGenericType(typeof(Key).GetGenericArguments());
object o = Activator.CreateInstance(m);
variableLengthStructSettings.keyLength = o as IVariableLengthStruct <Key>;
}
}
if (typeof(Value) == typeof(SpanByte))
{
if (variableLengthStructSettings == null)
{
variableLengthStructSettings = new VariableLengthStructSettings <Key, SpanByte>() as VariableLengthStructSettings <Key, Value>;
}
if (variableLengthStructSettings.valueLength == null)
{
(variableLengthStructSettings as VariableLengthStructSettings <Key, SpanByte>).valueLength = new SpanByteVarLenStruct();
}
}
else if (typeof(Value).IsGenericType && (typeof(Value).GetGenericTypeDefinition() == typeof(Memory <>)) && Utility.IsBlittableType(typeof(Value).GetGenericArguments()[0]))
{
if (variableLengthStructSettings == null)
{
variableLengthStructSettings = new VariableLengthStructSettings <Key, Value>();
}
if (variableLengthStructSettings.valueLength == null)
{
var m = typeof(MemoryVarLenStruct <>).MakeGenericType(typeof(Value).GetGenericArguments());
object o = Activator.CreateInstance(m);
variableLengthStructSettings.valueLength = o as IVariableLengthStruct <Value>;
}
}
else if (typeof(Value).IsGenericType && (typeof(Value).GetGenericTypeDefinition() == typeof(ReadOnlyMemory <>)) && Utility.IsBlittableType(typeof(Value).GetGenericArguments()[0]))
{
if (variableLengthStructSettings == null)
{
variableLengthStructSettings = new VariableLengthStructSettings <Key, Value>();
}
if (variableLengthStructSettings.valueLength == null)
{
var m = typeof(ReadOnlyMemoryVarLenStruct <>).MakeGenericType(typeof(Value).GetGenericArguments());
object o = Activator.CreateInstance(m);
variableLengthStructSettings.valueLength = o as IVariableLengthStruct <Value>;
}
}
}
/// <summary>
/// Create FASTER instance
/// </summary>
/// <param name="size">Size of core index (#cache lines)</param>
/// <param name="comparer">FASTER equality comparer for key</param>
/// <param name="variableLengthStructSettings"></param>
/// <param name="logSettings">Log settings</param>
/// <param name="checkpointSettings">Checkpoint settings</param>
/// <param name="serializerSettings">Serializer settings</param>
public FasterKV(long size, LogSettings logSettings,
CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null,
IFasterEqualityComparer <Key> comparer = null,
VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null)
{
if (comparer != null)
{
this.comparer = comparer;
}
else
{
if (typeof(IFasterEqualityComparer <Key>).IsAssignableFrom(typeof(Key)))
{
this.comparer = new Key() as IFasterEqualityComparer <Key>;
}
else
{
Console.WriteLine(
"***WARNING*** Creating default FASTER key equality comparer based on potentially slow EqualityComparer<Key>.Default. To avoid this, provide a comparer (IFasterEqualityComparer<Key>) as an argument to FASTER's constructor, or make Key implement the interface IFasterEqualityComparer<Key>");
this.comparer = FasterEqualityComparer <Key> .Default;
}
}
if (checkpointSettings == null)
{
checkpointSettings = new CheckpointSettings();
}
if (checkpointSettings.CheckpointDir != null && checkpointSettings.CheckpointManager != null)
{
throw new FasterException(
"Specify either CheckpointManager or CheckpointDir for CheckpointSettings, not both");
}
bool oldCheckpointManager = false;
if (oldCheckpointManager)
{
checkpointManager = checkpointSettings.CheckpointManager ??
new LocalCheckpointManager(checkpointSettings.CheckpointDir ?? "");
}
else
{
checkpointManager = checkpointSettings.CheckpointManager ??
new DeviceLogCommitCheckpointManager
(new LocalStorageNamedDeviceFactory(),
new DefaultCheckpointNamingScheme(
new DirectoryInfo(checkpointSettings.CheckpointDir ?? ".").FullName));
}
if (checkpointSettings.CheckpointManager == null)
{
disposeCheckpointManager = true;
}
FoldOverSnapshot = checkpointSettings.CheckPointType == core.CheckpointType.FoldOver;
CopyReadsToTail = logSettings.CopyReadsToTail;
if (logSettings.ReadCacheSettings != null)
{
CopyReadsToTail = false;
UseReadCache = true;
}
if (Utility.IsBlittable <Key>() && Utility.IsBlittable <Value>())
{
if (variableLengthStructSettings != null)
{
hlog = new VariableLengthBlittableAllocator <Key, Value>(logSettings, variableLengthStructSettings,
this.comparer, null, epoch);
Log = new LogAccessor <Key, Value>(this, hlog);
if (UseReadCache)
{
readcache = new VariableLengthBlittableAllocator <Key, Value>(
new LogSettings
{
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
}, variableLengthStructSettings, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value>(this, readcache);
}
}
else
{
hlog = new BlittableAllocator <Key, Value>(logSettings, this.comparer, null, epoch);
Log = new LogAccessor <Key, Value>(this, hlog);
if (UseReadCache)
{
readcache = new BlittableAllocator <Key, Value>(
new LogSettings
{
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
}, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value>(this, readcache);
}
}
}
else
{
WriteDefaultOnDelete = true;
hlog = new GenericAllocator <Key, Value>(logSettings, serializerSettings, this.comparer, null, epoch);
Log = new LogAccessor <Key, Value>(this, hlog);
if (UseReadCache)
{
readcache = new GenericAllocator <Key, Value>(
new LogSettings
{
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
}, serializerSettings, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value>(this, readcache);
}
}
hlog.Initialize();
sectorSize = (int)logSettings.LogDevice.SectorSize;
Initialize(size, sectorSize);
systemState = default;
systemState.phase = Phase.REST;
systemState.version = 1;
}
/// <summary>
/// Create FASTER instance
/// </summary>
/// <param name="size">Size of core index (#cache lines)</param>
/// <param name="comparer">FASTER equality comparer for key</param>
/// <param name="variableLengthStructSettings"></param>
/// <param name="functions">Callback functions</param>
/// <param name="logSettings">Log settings</param>
/// <param name="checkpointSettings">Checkpoint settings</param>
/// <param name="serializerSettings">Serializer settings</param>
public FasterKV(long size, Functions functions, LogSettings logSettings, CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null, IFasterEqualityComparer <Key> comparer = null, VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null)
{
threadCtx = new FastThreadLocal <FasterExecutionContext>();
prevThreadCtx = new FastThreadLocal <FasterExecutionContext>();
if (comparer != null)
{
this.comparer = comparer;
}
else
{
if (typeof(IFasterEqualityComparer <Key>).IsAssignableFrom(typeof(Key)))
{
this.comparer = new Key() as IFasterEqualityComparer <Key>;
}
else
{
Console.WriteLine("***WARNING*** Creating default FASTER key equality comparer based on potentially slow EqualityComparer<Key>.Default. To avoid this, provide a comparer (IFasterEqualityComparer<Key>) as an argument to FASTER's constructor, or make Key implement the interface IFasterEqualityComparer<Key>");
this.comparer = FasterEqualityComparer <Key> .Default;
}
}
if (checkpointSettings == null)
{
checkpointSettings = new CheckpointSettings();
}
directoryConfiguration = new DirectoryConfiguration(checkpointSettings.CheckpointDir);
FoldOverSnapshot = checkpointSettings.CheckPointType == core.CheckpointType.FoldOver;
CopyReadsToTail = logSettings.CopyReadsToTail;
this.functions = functions;
if (logSettings.ReadCacheSettings != null)
{
CopyReadsToTail = false;
UseReadCache = true;
}
if (Utility.IsBlittable <Key>() && Utility.IsBlittable <Value>())
{
if (variableLengthStructSettings != null)
{
hlog = new VariableLengthBlittableAllocator <Key, Value>
(logSettings, variableLengthStructSettings, this.comparer, null, epoch);
Log = new LogAccessor <Key, Value, Input, Output, Context>(this, hlog);
if (UseReadCache)
{
readcache = new VariableLengthBlittableAllocator <Key, Value>(
new LogSettings
{
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = logSettings.ReadCacheSettings.SecondChanceFraction
}, variableLengthStructSettings, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value, Input, Output, Context>(this, readcache);
}
}
else
{
hlog = new BlittableAllocator <Key, Value>(logSettings, this.comparer, null, epoch);
Log = new LogAccessor <Key, Value, Input, Output, Context>(this, hlog);
if (UseReadCache)
{
readcache = new BlittableAllocator <Key, Value>(
new LogSettings
{
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = logSettings.ReadCacheSettings.SecondChanceFraction
}, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value, Input, Output, Context>(this, readcache);
}
}
}
else
{
hlog = new GenericAllocator <Key, Value>(logSettings, serializerSettings, this.comparer, null, epoch);
Log = new LogAccessor <Key, Value, Input, Output, Context>(this, hlog);
if (UseReadCache)
{
readcache = new GenericAllocator <Key, Value>(
new LogSettings
{
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = logSettings.ReadCacheSettings.SecondChanceFraction
}, serializerSettings, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value, Input, Output, Context>(this, readcache);
}
}
hlog.Initialize();
sectorSize = (int)logSettings.LogDevice.SectorSize;
Initialize(size, sectorSize);
_systemState = default(SystemState);
_systemState.phase = Phase.REST;
_systemState.version = 1;
_checkpointType = CheckpointType.HYBRID_LOG_ONLY;
}
