// TODO: Currenly the last items are omitted with this variant. Use Listen() instead.
public async IAsyncEnumerable <(string, long, long)> ListenAsync_(int flushSize)
{
// Always start from beginning. Assume it is refilled or truncate works.
using (FasterLogScanIterator iter = logger.Scan(0, 1_000_000_000, name: "listen"))
{
int i = 0;
await foreach ((byte[] bytes, int length) in iter.GetAsyncEnumerable())
{
if (i >= flushSize)
{
nextAddress = iter.NextAddress;
break;
}
CancellationTokenSource cts = new CancellationTokenSource();
UTF8Encoding encoding = new UTF8Encoding();
i++;
// Probably shouldn't wait
// - https://microsoft.github.io/FASTER/docs/fasterlog#iteration
yield return(encoding.GetString(bytes), iter.CurrentAddress, iter.NextAddress);
}
}
}
public async Task <List <(string, long, long)> > GetList()
{
var result = new List <(string, long, long)>();
using (FasterLogScanIterator iter = logger.Scan(nextAddress, 100_000_000))
{
int i = 0;
byte[] entry;
int length;
while (iter.GetNext(out entry, out length))
{
UTF8Encoding encoding = new UTF8Encoding();
if (iter.CurrentAddress >= 1568)
{
Debugger.Break();
}
await iter.WaitAsync();
result.Add((encoding.GetString(entry), iter.CurrentAddress, iter.NextAddress));
i++;
if (i > 50)
{
nextAddress = iter.NextAddress;
break;
}
}
}
return(result);
}
public async IAsyncEnumerable <(string, long, long)> GetListAsync()
{
using (FasterLogScanIterator iter = logger.Scan(nextAddress, 100_000_000))
{
int i = 0;
await foreach ((byte[] bytes, int length) in iter.GetAsyncEnumerable())
{
if (i > 50)
{
nextAddress = iter.NextAddress;
break;
}
CancellationTokenSource cts = new CancellationTokenSource();
UTF8Encoding encoding = new UTF8Encoding();
try
{
await Task.WhenAny(WaitAsync(iter, cts.Token), SetTimeout(cts));
i++;
}
catch (Exception)
{
break;
}
yield return(encoding.GetString(bytes), iter.CurrentAddress, iter.NextAddress);
}
}
}
async IAsyncEnumerable <PartitionUpdateEvent> EventsToReplay(long from)
{
long to = this.log.TailAddress;
using (FasterLogScanIterator iter = this.log.Scan(from, to))
{
byte[] result;
int entryLength;
long currentAddress;
MemoryStream reassembly = null;
while (!this.cancellationToken.IsCancellationRequested)
{
PartitionUpdateEvent partitionEvent = null;
while (!iter.GetNext(out result, out entryLength, out currentAddress))
{
if (currentAddress >= to)
{
yield break;
}
await iter.WaitAsync(this.cancellationToken).ConfigureAwait(false);
}
if ((result[0] & first) != none)
{
if ((result[0] & last) != none)
{
partitionEvent = (PartitionUpdateEvent)Serializer.DeserializeEvent(new ArraySegment <byte>(result, 1, entryLength - 1));
}
else
{
reassembly = new MemoryStream();
reassembly.Write(result, 1, entryLength - 1);
}
}
else
{
reassembly.Write(result, 1, entryLength - 1);
if ((result[0] & last) != none)
{
reassembly.Position = 0;
partitionEvent = (PartitionUpdateEvent)Serializer.DeserializeEvent(reassembly);
reassembly = null;
}
}
if (partitionEvent != null)
{
partitionEvent.NextCommitLogPosition = iter.NextAddress;
yield return(partitionEvent);
}
}
}
}
static async Task AsyncScan()
{
using (iter = log.Scan(log.BeginAddress, long.MaxValue))
await foreach ((byte[] result, int length) in iter.GetAsyncEnumerable())
{
if (Different(result, staticEntry, out int location))
{
throw new Exception("Invalid entry found");
}
log.TruncateUntil(iter.NextAddress);
}
}
public List <(string, long, long)> Listen(int flushSize, Cursor cursor)
{
var nextAddress_ = cursor.NextAddress;
List <(string, long, long)> entries = new List <(string, long, long)>();
// CancellationTokenSource cts = new CancellationTokenSource();
UTF8Encoding encoding = new UTF8Encoding();
int i = 0;
using (FasterLogScanIterator iter = logger.Scan(0, 1_000_000_000))
{
while (true)
{
byte[] bytes;
while (!iter.GetNext(out bytes, out int entryLength))
{
// TODO: Cursor keeps an item at the end for some reason. Fix this!
if (entries.Count > 1)
{
return(entries);
}
return(new List <(string, long, long)>());
// TODO: for the moment make sure to return the final items instead of awaiting more results.
// if (iter.CurrentAddress >= 1_000_000_000) break;
// await iter.WaitAsync(cts.Token);
}
entries.Add((encoding.GetString(bytes), iter.CurrentAddress, iter.NextAddress));
if (cursor is DebugCursor debugCursor)
{
debugCursor.Increment();
if (debugCursor.Iteration > 1841)
{
int j = 0;
}
}
i++;
if (i >= flushSize)
{
return(entries);
}
}
}
}
public async Task <List <(string, long, long)> > StartScan(string devicePath)
{
IDevice device = Devices.CreateLogDevice(devicePath);
FasterLog logger = new FasterLog(new FasterLogSettings {
LogDevice = device
});
long nextAddress = 0;
bool keepGoing = true;
int i = 0;
var result = new List <(string, long, long)>();
// using (FasterLogScanIterator iter = logger.Scan(logger.BeginAddress, 100_000_000, name: nameof(GetListAsync)))
using (FasterLogScanIterator iter = logger.Scan(nextAddress, 1_000_000_000))
{
while (keepGoing)
{
Console.WriteLine("Going");
LocalTime timeOfDay;
await foreach ((byte[] bytes, int length) in iter.GetAsyncEnumerable())
{
DateTimeZone tz = DateTimeZoneProviders.Tzdb.GetSystemDefault();
timeOfDay = SystemClock.Instance.GetCurrentInstant().InZone(tz).TimeOfDay;
nextAddress = iter.NextAddress;
Console.WriteLine("Time={1} NextAddress={0}, Count={2}", iter.NextAddress, timeOfDay, i++);
var cts = new CancellationTokenSource();
UTF8Encoding encoding = new UTF8Encoding();
try
{
await Task.WhenAny(WaitAsync(iter), SetTimeout(cts));
}
catch (Exception e)
{
Console.Error.WriteLine($"Error={e.GetType()}, Message={e.ToString()}");
break;
}
timeOfDay = SystemClock.Instance.GetCurrentInstant().InZone(tz).TimeOfDay;
Console.WriteLine("Time={2} ContentLength={0}", bytes.Length, iter.NextAddress, timeOfDay);
}
await Task.Delay(5000);
}
}
return(result);
}
static void ScanThread()
{
Random r = new Random();
byte[] result;
using (iter = log.Scan(log.BeginAddress, long.MaxValue))
{
while (true)
{
while (!iter.GetNext(out result, out int length))
{
// For finite end address, check if iteration ended
// if (iter.CurrentAddress >= endAddress) return;
iter.WaitAsync().GetAwaiter().GetResult();
}
// Memory pool variant:
// iter.GetNext(pool, out IMemoryOwner<byte> resultMem, out int length))
if (Different(result, staticEntry, out int location))
{
throw new Exception("Invalid entry found");
}
// Re-insert entry with small probability
if (r.Next(100) < 10)
{
log.Enqueue(result);
}
// Example of random read from given address
// (result, _) = log.ReadAsync(iter.CurrentAddress).GetAwaiter().GetResult();
// Truncate log until after recently processed entry
log.TruncateUntil(iter.NextAddress);
// Safer truncate variant: truncate until start of page
// log.TruncateUntilPageStart(iter.NextAddress);
}
}
// Example of recoverable (named) iterator:
// using (iter = log.Scan(log.BeginAddress, long.MaxValue, "foo"))
}
public async Task <Option <(string, long, long)> > GetNext()
{
using FasterLogScanIterator iter = logger.Scan(nextAddress, 100_000_000);
while (true)
{
byte[] entry;
int length;
while (!iter.GetNext(out entry, out length))
{
if (iter.CurrentAddress >= 100_000_000)
{
return(Option.None <(string, long, long)>());
}
}
UTF8Encoding encoding = new UTF8Encoding();
await iter.WaitAsync();
nextAddress = iter.NextAddress;
return(Option.Some((encoding.GetString(entry), iter.CurrentAddress, iter.NextAddress))); // Possible to pipe
}
}
/// <summary>
/// Main program entry point
/// </summary>
static void Main()
{
bool sync = true;
// Populate entry being inserted
for (int i = 0; i < entryLength; i++)
{
staticEntry[i] = (byte)i;
}
var path = Path.GetTempPath() + "FasterLogSample/";
IDevice device = Devices.CreateLogDevice(path + "hlog.log");
// FasterLog will recover and resume if there is a previous commit found
log = new FasterLog(new FasterLogSettings {
LogDevice = device
});
using (iter = log.Scan(log.BeginAddress, long.MaxValue))
{
if (sync)
{
// Log writer thread: create as many as needed
new Thread(new ThreadStart(LogWriterThread)).Start();
// Threads for iterator scan: create as many as needed
new Thread(() => ScanThread()).Start();
// Threads for reporting, commit
new Thread(new ThreadStart(ReportThread)).Start();
var t = new Thread(new ThreadStart(CommitThread));
t.Start();
t.Join();
}
else
{
// Async version of demo: expect lower performance
// particularly for small payload sizes
const int NumParallelTasks = 10_000;
ThreadPool.SetMinThreads(2 * Environment.ProcessorCount, 2 * Environment.ProcessorCount);
TaskScheduler.UnobservedTaskException += (object sender, UnobservedTaskExceptionEventArgs e) =>
{
Console.WriteLine($"Unobserved task exception: {e.Exception}");
e.SetObserved();
};
Task[] tasks = new Task[NumParallelTasks];
for (int i = 0; i < NumParallelTasks; i++)
{
int local = i;
tasks[i] = Task.Run(() => AsyncLogWriter(local));
}
var scan = Task.Run(() => AsyncScan());
// Threads for reporting, commit
new Thread(new ThreadStart(ReportThread)).Start();
new Thread(new ThreadStart(CommitThread)).Start();
Task.WaitAll(tasks);
Task.WaitAll(scan);
}
}
}
public async Task <bool> WaitAsync(FasterLogScanIterator iter)
{
return(await iter.WaitAsync());
}
private async Task <bool> WaitAsync(FasterLogScanIterator iter, CancellationToken cancellationToken)
{
return(await iter.WaitAsync(cancellationToken));
}
