private void FasterLogTest1(LogChecksumType logChecksum, IDevice device, ILogCommitManager logCommitManager)
{
log = new FasterLog(new FasterLogSettings {
PageSizeBits = 20, SegmentSizeBits = 20, LogDevice = device, LogChecksum = logChecksum, LogCommitManager = logCommitManager
});
byte[] entry = new byte[entryLength];
for (int i = 0; i < entryLength; i++)
{
entry[i] = (byte)i;
}
for (int i = 0; i < numEntries; i++)
{
log.Enqueue(entry);
}
log.Commit(true);
using (var iter = log.Scan(0, long.MaxValue))
{
int count = 0;
while (iter.GetNext(out byte[] result, out int length, out long currentAddress))
{
count++;
Assert.IsTrue(result.SequenceEqual(entry));
if (count % 100 == 0)
{
log.TruncateUntil(iter.NextAddress);
}
}
Assert.IsTrue(count == numEntries);
}
log.Dispose();
}
/// <summary>
/// Recover info from token
/// </summary>
/// <param name="logCommitManager"></param>
/// <returns></returns>
internal void Recover(ILogCommitManager logCommitManager)
{
var metadata = logCommitManager.GetCommitMetadata();
if (metadata == null)
{
throw new Exception("Invalid log commit metadata during recovery");
}
Initialize(new BinaryReader(new MemoryStream(metadata)));
}
/// <summary>
/// Create new log instance
/// </summary>
/// <param name="logSettings"></param>
public FasterLog(FasterLogSettings logSettings)
{
bool oldCommitManager = false;
if (oldCommitManager)
{
logCommitManager = logSettings.LogCommitManager ??
new LocalLogCommitManager(logSettings.LogCommitFile ??
logSettings.LogDevice.FileName + ".commit");
}
else
{
logCommitManager = logSettings.LogCommitManager ??
new DeviceLogCommitCheckpointManager
(new LocalStorageNamedDeviceFactory(),
new DefaultCheckpointNamingScheme(
new FileInfo(logSettings.LogDevice.FileName).Directory.FullName));
}
if (logSettings.LogCommitManager == null)
{
disposeLogCommitManager = true;
}
// Reserve 8 byte checksum in header if requested
logChecksum = logSettings.LogChecksum;
headerSize = logChecksum == LogChecksumType.PerEntry ? 12 : 4;
getMemory = logSettings.GetMemory;
epoch = new LightEpoch();
CommittedUntilAddress = Constants.kFirstValidAddress;
CommittedBeginAddress = Constants.kFirstValidAddress;
SafeTailAddress = Constants.kFirstValidAddress;
allocator = new BlittableAllocator <Empty, byte>(
logSettings.GetLogSettings(), null,
null, epoch, CommitCallback);
allocator.Initialize();
// FasterLog is used as a read-only iterator
if (logSettings.ReadOnlyMode)
{
readOnlyMode = true;
allocator.HeadAddress = long.MaxValue;
}
Restore(out RecoveredIterators);
}
/// <summary>
/// Create new log instance
/// </summary>
/// <param name="logSettings"></param>
public FasterLog(FasterLogSettings logSettings)
{
logCommitManager = logSettings.LogCommitManager ??
new LocalLogCommitManager(logSettings.LogCommitFile ??
logSettings.LogDevice.FileName + ".commit");
// Reserve 8 byte checksum in header if requested
logChecksum = logSettings.LogChecksum;
headerSize = logChecksum == LogChecksumType.PerEntry ? 12 : 4;
getMemory = logSettings.GetMemory;
epoch = new LightEpoch();
CommittedUntilAddress = Constants.kFirstValidAddress;
CommittedBeginAddress = Constants.kFirstValidAddress;
allocator = new BlittableAllocator <Empty, byte>(
logSettings.GetLogSettings(), null,
null, epoch, CommitCallback);
allocator.Initialize();
Restore(out RecoveredIterators);
}
private async ValueTask FasterLogTest1(LogChecksumType logChecksum, IDevice device, ILogCommitManager logCommitManager, FasterLogTests.IteratorType iteratorType)
{
var logSettings = new FasterLogSettings {
PageSizeBits = 20, SegmentSizeBits = 20, LogDevice = device, LogChecksum = logChecksum, LogCommitManager = logCommitManager
};
log = FasterLogTests.IsAsync(iteratorType) ? await FasterLog.CreateAsync(logSettings) : new FasterLog(logSettings);
byte[] entry = new byte[entryLength];
for (int i = 0; i < entryLength; i++)
{
entry[i] = (byte)i;
}
for (int i = 0; i < numEntries; i++)
{
log.Enqueue(entry);
}
log.Commit(true);
using (var iter = log.Scan(0, long.MaxValue))
{
var counter = new FasterLogTests.Counter(log);
switch (iteratorType)
{
case FasterLogTests.IteratorType.AsyncByteVector:
await foreach ((byte[] result, _, _, long nextAddress) in iter.GetAsyncEnumerable())
{
Assert.IsTrue(result.SequenceEqual(entry));
counter.IncrementAndMaybeTruncateUntil(nextAddress);
// MoveNextAsync() would hang at TailAddress, waiting for more entries (that we don't add).
// Note: If this happens and the test has to be canceled, there may be a leftover blob from the log.Commit(), because
// the log device isn't Dispose()d; the symptom is currently a numeric string format error in DefaultCheckpointNamingScheme.
if (nextAddress == log.TailAddress)
{
break;
}
}
break;
case FasterLogTests.IteratorType.AsyncMemoryOwner:
await foreach ((IMemoryOwner <byte> result, int _, long _, long nextAddress) in iter.GetAsyncEnumerable(MemoryPool <byte> .Shared))
{
Assert.IsTrue(result.Memory.Span.ToArray().Take(entry.Length).SequenceEqual(entry));
result.Dispose();
counter.IncrementAndMaybeTruncateUntil(nextAddress);
// MoveNextAsync() would hang at TailAddress, waiting for more entries (that we don't add).
// Note: If this happens and the test has to be canceled, there may be a leftover blob from the log.Commit(), because
// the log device isn't Dispose()d; the symptom is currently a numeric string format error in DefaultCheckpointNamingScheme.
if (nextAddress == log.TailAddress)
{
break;
}
}
break;
case FasterLogTests.IteratorType.Sync:
while (iter.GetNext(out byte[] result, out _, out _))
{
Assert.IsTrue(result.SequenceEqual(entry));
counter.IncrementAndMaybeTruncateUntil(iter.NextAddress);
}
break;
default:
Assert.Fail("Unknown IteratorType");
break;
}
Assert.IsTrue(counter.count == numEntries);
}
log.Dispose();
}
