public async Task FasterLogResumePersistedReaderSpec([Values] LogChecksumType logChecksum)
{
var input1 = new byte[] { 0, 1, 2, 3 };
var input2 = new byte[] { 4, 5, 6, 7, 8, 9, 10 };
var input3 = new byte[] { 11, 12 };
string readerName = "abc";
using (var l = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitFile = commitPath
}))
{
await l.EnqueueAsync(input1);
await l.EnqueueAsync(input2);
await l.EnqueueAsync(input3);
await l.CommitAsync();
using var originalIterator = l.Scan(0, long.MaxValue, readerName);
Assert.IsTrue(originalIterator.GetNext(out _, out _, out _, out long recoveryAddress));
originalIterator.CompleteUntil(recoveryAddress);
Assert.IsTrue(originalIterator.GetNext(out _, out _, out _, out _)); // move the reader ahead
await l.CommitAsync();
}
using (var l = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitFile = commitPath
}))
{
using var recoveredIterator = l.Scan(0, long.MaxValue, readerName);
Assert.IsTrue(recoveredIterator.GetNext(out byte[] outBuf, out _, out _, out _));
Assert.True(input2.SequenceEqual(outBuf)); // we should have read in input2, not input1 or input3
}
}
public async Task FasterLogResumePersistedReader2([Values] LogChecksumType logChecksum, [Values] bool overwriteLogCommits, [Values] bool removeOutdated)
{
var input1 = new byte[] { 0, 1, 2, 3 };
var input2 = new byte[] { 4, 5, 6, 7, 8, 9, 10 };
var input3 = new byte[] { 11, 12 };
string readerName = "abc";
using (var logCommitManager = new DeviceLogCommitCheckpointManager(new LocalStorageNamedDeviceFactory(), new DefaultCheckpointNamingScheme(commitPath), overwriteLogCommits, removeOutdated))
{
long originalCompleted;
using (var l = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitManager = logCommitManager
}))
{
await l.EnqueueAsync(input1);
await l.CommitAsync();
await l.EnqueueAsync(input2);
await l.CommitAsync();
await l.EnqueueAsync(input3);
await l.CommitAsync();
using (var originalIterator = l.Scan(0, long.MaxValue, readerName))
{
originalIterator.GetNext(out _, out _, out _, out long recoveryAddress);
originalIterator.CompleteUntil(recoveryAddress);
originalIterator.GetNext(out _, out _, out _, out _); // move the reader ahead
await l.CommitAsync();
originalCompleted = originalIterator.CompletedUntilAddress;
}
}
using (var l = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitManager = logCommitManager
}))
{
using (var recoveredIterator = l.Scan(0, long.MaxValue, readerName))
{
recoveredIterator.GetNext(out byte[] outBuf, out _, out _, out _);
// we should have read in input2, not input1 or input3
Assert.True(input2.SequenceEqual(outBuf), $"Original: {input2[0]}, Recovered: {outBuf[0]}, Original: {originalCompleted}, Recovered: {recoveredIterator.CompletedUntilAddress}");
// TestContext.Progress.WriteLine($"Original: {originalCompleted}, Recovered: {recoveredIterator.CompletedUntilAddress}");
}
}
}
}
static async Task LogWriterAsync(FasterLog log, byte[] entry)
{
// Enter in some entries then wait on this separate thread
await log.EnqueueAsync(entry);
await log.EnqueueAsync(entry);
await log.EnqueueAsync(entry);
await log.WaitForCommitAsync(log.TailAddress);
}
/*
* public async override Task OnDeactivateAsync()
* {
* string primaryKey = this.GetPrimaryKeyString();
* _log.Commit(true);
* Console.WriteLine($"Ending {_serviceName} - {_recordStart:g} - {_metric}");
* await base.OnDeactivateAsync();
* }*/
public async Task AddRecord(Record record)
{
_recordCount++;
var offset = (long)record.Time - _recordStart.Ticks;
foreach (var kvp in record.Attributes)
{
var entry = new LogEntry()
{
Offset = (uint)offset,
MetricValue = record.Metricvalue,
KeyName = kvp.Key,
KeyValue = kvp.Value,
RecordId = _recordCount
};
var bytes = entry.ToByteArray();
await _log.EnqueueAsync(bytes);
}
if (_recordCount % 10_000 == 0)
{
_log.Commit();
Console.WriteLine($"{_recordCount} {_recordStart.Ticks}");
}
}
public async void TestLog()
{
using var log = new FasterLog(new FasterLogSettings { LogDevice = _device });
await log.EnqueueAsync(new byte[2] {
0x12, 0x34
});
await log.CommitAsync();
// TODO:
Assert.True(File.Exists(_path + ".0"));
}
public async Task SaveManyAsync(IEnumerable <EventEntity> entities)
{
var source = new CancellationTokenSource();
foreach (var entity in entities)
{
var enqueueAsync = _log.EnqueueAsync(Encoding.UTF8.GetBytes(entity.ToString()), source.Token);
if (!enqueueAsync.IsCompleted)
{
await enqueueAsync;
}
}
var commitAsync = _log.CommitAsync(source.Token);
if (!commitAsync.IsCompleted)
{
await commitAsync;
}
}
/// <summary>
/// Async version of enqueue
/// </summary>
static async Task AsyncLogWriter(int id)
{
bool batched = false;
await Task.Yield();
if (!batched)
{
// Single commit version - append each item and wait for commit
// Needs high parallelism (NumParallelTasks) for perf
// Needs separate commit thread to perform regular commit
// Otherwise we commit only at page boundaries
while (true)
{
try
{
await log.EnqueueAndWaitForCommitAsync(staticEntry);
}
catch (Exception ex)
{
Console.WriteLine($"{nameof(AsyncLogWriter)}({id}): {ex}");
}
}
}
else
{
// Batched version - we enqueue many entries to memory,
// then wait for commit periodically
int count = 0;
while (true)
{
await log.EnqueueAsync(staticEntry);
if (count++ % 100 == 0)
{
await log.WaitForCommitAsync();
}
}
}
}
public async Task FasterLogResumePersistedReader3([Values] LogChecksumType logChecksum, [Values] bool overwriteLogCommits, [Values] bool removeOutdated)
{
var input1 = new byte[] { 0, 1, 2, 3 };
var input2 = new byte[] { 4, 5, 6, 7, 8, 9, 10 };
var input3 = new byte[] { 11, 12 };
string readerName = "abcd";
using (var logCommitManager = new DeviceLogCommitCheckpointManager(new LocalStorageNamedDeviceFactory(), new DefaultCheckpointNamingScheme(commitPath), overwriteLogCommits, removeOutdated))
{
long originalCompleted;
using (var l = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitManager = logCommitManager
}))
{
await l.EnqueueAsync(input1);
await l.CommitAsync();
await l.EnqueueAsync(input2);
await l.CommitAsync();
await l.EnqueueAsync(input3);
await l.CommitAsync();
using var originalIterator = l.Scan(0, l.TailAddress, readerName);
int count = 0;
await foreach (var item in originalIterator.GetAsyncEnumerable())
{
if (count < 2) // we complete 1st and 2nd item read
{
originalIterator.CompleteUntil(item.nextAddress);
}
if (count < 1) // we commit only 1st item read
{
await l.CommitAsync();
}
count++;
}
originalCompleted = originalIterator.CompletedUntilAddress;
}
using (var l = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitManager = logCommitManager
}))
{
using var recoveredIterator = l.Scan(0, l.TailAddress, readerName);
int count = 0;
await foreach (var item in recoveredIterator.GetAsyncEnumerable())
{
if (count == 0) // resumed iterator will start at item2
{
Assert.True(input2.SequenceEqual(item.entry), $"Original: {input2[0]}, Recovered: {item.entry[0]}, Original: {originalCompleted}, Recovered: {recoveredIterator.CompletedUntilAddress}");
}
count++;
}
Assert.IsTrue(count == 2);
}
}
}
public async ValueTask <long> EnqueueAsync(byte[] entry, CancellationToken token = default)
{
return(await logger.EnqueueAsync(entry, token));
}