public async Task LogCompaction_IncreasesThresholdAfterFiveConsecutiveCompactions()
{
// Arrange
var resultStringBuilder = new StringBuilder();
ILogger <MixedStorageKVStore <int, DummyClass> > dummyLogger = CreateLogger <int, DummyClass>(resultStringBuilder, LogLevel.Trace);
var logSettings = new LogSettings
{
LogDevice = Devices.CreateLogDevice(Path.Combine(_fixture.TempDirectory, $"{nameof(LogCompaction_IncreasesThresholdAfterFiveConsecutiveCompactions)}.log"),
deleteOnClose: true),
PageSizeBits = 12,
MemorySizeBits = 13
};
DummyClass dummyClassInstance = CreatePopulatedDummyClassInstance();
var dummyOptions = new MixedStorageKVStoreOptions()
{
TimeBetweenLogCompactionsMS = 1,
InitialLogCompactionThresholdBytes = 20_000 // So we compact 5 times in a row
};
// Create and populate faster KV store before passing it to MixedStorageKVStore, at which point the compaction loop starts.
// For quicker tests, use thread local sessions.
FasterKV <SpanByte, SpanByte>?dummyFasterKVStore = null;
ThreadLocal <ClientSession <SpanByte, SpanByte, SpanByte, SpanByteAndMemory, Empty, SpanByteFunctions <Empty> > >?dummyThreadLocalSession = null;
try
{
dummyFasterKVStore = new FasterKV <SpanByte, SpanByte>(1L << 20, logSettings);
FasterKV <SpanByte, SpanByte> .ClientSessionBuilder <SpanByte, SpanByteAndMemory, Empty> dummyClientSessionBuilder = dummyFasterKVStore.For(new SpanByteFunctions <Empty>());
dummyThreadLocalSession = new(() => dummyClientSessionBuilder.NewSession <SpanByteFunctions <Empty> >(), true);
MessagePackSerializerOptions dummyMessagePackSerializerOptions = MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray);
// Record size estimate:
// - dummyClassInstance serialized and compressed = ~73 bytes
// - int key serialized and compressed = ~3 bytes
// - key length metadata = 4 bytes
// - value length metadata = 4 bytes
// - record header = 8 bytes
// Total = ~92 bytes
//
// n * 92 - 8192 > InitialLogCompactionThresholdBytes
// Insert
byte[] dummyValueBytes = MessagePackSerializer.Serialize(dummyClassInstance, dummyMessagePackSerializerOptions);
Parallel.For(0, 500, key =>
{
byte[] dummyKeyBytes = MessagePackSerializer.Serialize(key, dummyMessagePackSerializerOptions);
unsafe
{
// Upsert
fixed(byte *keyPointer = dummyKeyBytes)
fixed(byte *valuePointer = dummyValueBytes)
{
var keySpanByte = SpanByte.FromPointer(keyPointer, dummyKeyBytes.Length);
var objSpanByte = SpanByte.FromPointer(valuePointer, dummyValueBytes.Length);
dummyThreadLocalSession.Value.Upsert(ref keySpanByte, ref objSpanByte);
}
}
});
// Update so compaction does something. Can't update in insert loop or we'll get a bunch of in-place updates.
dummyClassInstance.DummyInt++;
dummyValueBytes = MessagePackSerializer.Serialize(dummyClassInstance, dummyMessagePackSerializerOptions);
Parallel.For(0, 500, key =>
{
byte[] dummyKeyBytes = MessagePackSerializer.Serialize(key, dummyMessagePackSerializerOptions);
unsafe
{
// Upsert
fixed(byte *keyPointer = dummyKeyBytes)
fixed(byte *valuePointer = dummyValueBytes)
{
var keySpanByte = SpanByte.FromPointer(keyPointer, dummyKeyBytes.Length);
var objSpanByte = SpanByte.FromPointer(valuePointer, dummyValueBytes.Length);
dummyThreadLocalSession.Value.Upsert(ref keySpanByte, ref objSpanByte);
}
}
});
}
finally
{
if (dummyThreadLocalSession != null)
{
foreach (ClientSession <SpanByte, SpanByte, SpanByte, SpanByteAndMemory, Empty, SpanByteFunctions <Empty> > session in dummyThreadLocalSession.Values)
{
session.Dispose(); // Faster synchronously completes all pending operations, so we should not get exceptions if we're in the middle of log compaction
}
}
}
// Runs 5 consecutive compactions, increases threshold, runs 5 more consecutive compactions (all redundant), increases threshold above
// safe-readonly region size, skips compactions thereafter.
string expectedResultStart = @$ "{LogLevel.Trace}: {string.Format(Strings.LogTrace_LogCompacted, 90048, 72000, 1)}
{LogLevel.Trace}: {string.Format(Strings.LogTrace_LogCompacted, 72000, 57600, 2)}
{LogLevel.Trace}: {string.Format(Strings.LogTrace_LogCompacted, 57600, 46080, 3)}
public async Task LogCompaction_OccursIfSafeReadOnlyRegionIsLargerThanThreshold()
{
// Arrange
var resultStringBuilder = new StringBuilder();
ILogger <MixedStorageKVStore <int, DummyClass> > dummyLogger = CreateLogger <int, DummyClass>(resultStringBuilder, LogLevel.Trace);
var logSettings = new LogSettings
{
LogDevice = Devices.CreateLogDevice(Path.Combine(_fixture.TempDirectory, $"{nameof(LogCompaction_OccursIfSafeReadOnlyRegionIsLargerThanThreshold)}.log"),
deleteOnClose: true),
PageSizeBits = 12,
MemorySizeBits = 13
};
DummyClass dummyClassInstance = CreatePopulatedDummyClassInstance();
var dummyOptions = new MixedStorageKVStoreOptions()
{
TimeBetweenLogCompactionsMS = 1,
InitialLogCompactionThresholdBytes = 80_000
};
// Create and populate faster KV store before passing it to MixedStorageKVStore, at which point the compaction loop starts.
// For quicker tests, use thread local sessions.
FasterKV <SpanByte, SpanByte>?dummyFasterKVStore = null;
ThreadLocal <ClientSession <SpanByte, SpanByte, SpanByte, SpanByteAndMemory, Empty, SpanByteFunctions <Empty> > >?dummyThreadLocalSession = null;
try
{
dummyFasterKVStore = new FasterKV <SpanByte, SpanByte>(1L << 20, logSettings);
FasterKV <SpanByte, SpanByte> .ClientSessionBuilder <SpanByte, SpanByteAndMemory, Empty> dummyClientSessionBuilder = dummyFasterKVStore.For(new SpanByteFunctions <Empty>());
dummyThreadLocalSession = new(() => dummyClientSessionBuilder.NewSession <SpanByteFunctions <Empty> >(), true);
MessagePackSerializerOptions dummyMessagePackSerializerOptions = MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray);
// Record size estimate:
// - dummyClassInstance serialized and compressed = ~73 bytes
// - int key serialized and compressed = ~3 bytes
// - key length metadata = 4 bytes
// - value length metadata = 4 bytes
// - record header = 8 bytes
// Total = ~92 bytes
//
// n * 92 - 8192 > InitialLogCompactionThresholdBytes
// Insert
byte[] dummyValueBytes = MessagePackSerializer.Serialize(dummyClassInstance, dummyMessagePackSerializerOptions);
Parallel.For(0, 500, key =>
{
byte[] dummyKeyBytes = MessagePackSerializer.Serialize(key, dummyMessagePackSerializerOptions);
unsafe
{
// Upsert
fixed(byte *keyPointer = dummyKeyBytes)
fixed(byte *valuePointer = dummyValueBytes)
{
var keySpanByte = SpanByte.FromPointer(keyPointer, dummyKeyBytes.Length);
var objSpanByte = SpanByte.FromPointer(valuePointer, dummyValueBytes.Length);
dummyThreadLocalSession.Value.Upsert(ref keySpanByte, ref objSpanByte);
}
}
});
// Update so compaction does something. Can't update in insert loop or we'll get a bunch of in-place updates.
dummyClassInstance.DummyInt++;
dummyValueBytes = MessagePackSerializer.Serialize(dummyClassInstance, dummyMessagePackSerializerOptions);
Parallel.For(0, 500, key =>
{
byte[] dummyKeyBytes = MessagePackSerializer.Serialize(key, dummyMessagePackSerializerOptions);
unsafe
{
// Upsert
fixed(byte *keyPointer = dummyKeyBytes)
fixed(byte *valuePointer = dummyValueBytes)
{
var keySpanByte = SpanByte.FromPointer(keyPointer, dummyKeyBytes.Length);
var objSpanByte = SpanByte.FromPointer(valuePointer, dummyValueBytes.Length);
dummyThreadLocalSession.Value.Upsert(ref keySpanByte, ref objSpanByte);
}
}
});
}
finally
{
if (dummyThreadLocalSession != null)
{
foreach (ClientSession <SpanByte, SpanByte, SpanByte, SpanByteAndMemory, Empty, SpanByteFunctions <Empty> > session in dummyThreadLocalSession.Values)
{
session.Dispose(); // Faster synchronously completes all pending operations, so we should not get exceptions if we're in the middle of log compaction
}
}
}
// We compact 20% of the safe-readonly region of the log. Since we inserted then updated, compaction here means removal.
// 90048 * 0.8 = 72038, ~72000 (missing 38 bytes likely has to do with the fact that we can't remove only part of a record).
string expectedResultStart = $"{LogLevel.Trace}: {string.Format(Strings.LogTrace_LogCompacted, 90048, 72000, 1)}";
int expectedResultStartLength = expectedResultStart.Length;
// Act
using (var testSubject = new MixedStorageKVStore <int, DummyClass>(dummyOptions, dummyLogger, dummyFasterKVStore)) // Start log compaction
{
while (resultStringBuilder.Length <= expectedResultStartLength)
{
await Task.Delay(10).ConfigureAwait(false);
}
}
// Assert
Assert.StartsWith(expectedResultStart, resultStringBuilder.ToString());
// If compaction runs more than once, should be skipped after first compaction (< threshold behaviour verified in previous test)
}