// TODO: Currenly the last items are omitted with this variant. Use Listen() instead. public async IAsyncEnumerable <(string, long, long)> ListenAsync_(int flushSize) { // Always start from beginning. Assume it is refilled or truncate works. using (FasterLogScanIterator iter = logger.Scan(0, 1_000_000_000, name: "listen")) { int i = 0; await foreach ((byte[] bytes, int length) in iter.GetAsyncEnumerable()) { if (i >= flushSize) { nextAddress = iter.NextAddress; break; } CancellationTokenSource cts = new CancellationTokenSource(); UTF8Encoding encoding = new UTF8Encoding(); i++; // Probably shouldn't wait // - https://microsoft.github.io/FASTER/docs/fasterlog#iteration yield return(encoding.GetString(bytes), iter.CurrentAddress, iter.NextAddress); } } }
public async Task <List <(string, long, long)> > GetList() { var result = new List <(string, long, long)>(); using (FasterLogScanIterator iter = logger.Scan(nextAddress, 100_000_000)) { int i = 0; byte[] entry; int length; while (iter.GetNext(out entry, out length)) { UTF8Encoding encoding = new UTF8Encoding(); if (iter.CurrentAddress >= 1568) { Debugger.Break(); } await iter.WaitAsync(); result.Add((encoding.GetString(entry), iter.CurrentAddress, iter.NextAddress)); i++; if (i > 50) { nextAddress = iter.NextAddress; break; } } } return(result); }
public async IAsyncEnumerable <(string, long, long)> GetListAsync() { using (FasterLogScanIterator iter = logger.Scan(nextAddress, 100_000_000)) { int i = 0; await foreach ((byte[] bytes, int length) in iter.GetAsyncEnumerable()) { if (i > 50) { nextAddress = iter.NextAddress; break; } CancellationTokenSource cts = new CancellationTokenSource(); UTF8Encoding encoding = new UTF8Encoding(); try { await Task.WhenAny(WaitAsync(iter, cts.Token), SetTimeout(cts)); i++; } catch (Exception) { break; } yield return(encoding.GetString(bytes), iter.CurrentAddress, iter.NextAddress); } } }
async IAsyncEnumerable <PartitionUpdateEvent> EventsToReplay(long from) { long to = this.log.TailAddress; using (FasterLogScanIterator iter = this.log.Scan(from, to)) { byte[] result; int entryLength; long currentAddress; MemoryStream reassembly = null; while (!this.cancellationToken.IsCancellationRequested) { PartitionUpdateEvent partitionEvent = null; while (!iter.GetNext(out result, out entryLength, out currentAddress)) { if (currentAddress >= to) { yield break; } await iter.WaitAsync(this.cancellationToken).ConfigureAwait(false); } if ((result[0] & first) != none) { if ((result[0] & last) != none) { partitionEvent = (PartitionUpdateEvent)Serializer.DeserializeEvent(new ArraySegment <byte>(result, 1, entryLength - 1)); } else { reassembly = new MemoryStream(); reassembly.Write(result, 1, entryLength - 1); } } else { reassembly.Write(result, 1, entryLength - 1); if ((result[0] & last) != none) { reassembly.Position = 0; partitionEvent = (PartitionUpdateEvent)Serializer.DeserializeEvent(reassembly); reassembly = null; } } if (partitionEvent != null) { partitionEvent.NextCommitLogPosition = iter.NextAddress; yield return(partitionEvent); } } } }
static async Task AsyncScan() { using (iter = log.Scan(log.BeginAddress, long.MaxValue)) await foreach ((byte[] result, int length) in iter.GetAsyncEnumerable()) { if (Different(result, staticEntry, out int location)) { throw new Exception("Invalid entry found"); } log.TruncateUntil(iter.NextAddress); } }
public List <(string, long, long)> Listen(int flushSize, Cursor cursor) { var nextAddress_ = cursor.NextAddress; List <(string, long, long)> entries = new List <(string, long, long)>(); // CancellationTokenSource cts = new CancellationTokenSource(); UTF8Encoding encoding = new UTF8Encoding(); int i = 0; using (FasterLogScanIterator iter = logger.Scan(0, 1_000_000_000)) { while (true) { byte[] bytes; while (!iter.GetNext(out bytes, out int entryLength)) { // TODO: Cursor keeps an item at the end for some reason. Fix this! if (entries.Count > 1) { return(entries); } return(new List <(string, long, long)>()); // TODO: for the moment make sure to return the final items instead of awaiting more results. // if (iter.CurrentAddress >= 1_000_000_000) break; // await iter.WaitAsync(cts.Token); } entries.Add((encoding.GetString(bytes), iter.CurrentAddress, iter.NextAddress)); if (cursor is DebugCursor debugCursor) { debugCursor.Increment(); if (debugCursor.Iteration > 1841) { int j = 0; } } i++; if (i >= flushSize) { return(entries); } } } }
public async Task <List <(string, long, long)> > StartScan(string devicePath) { IDevice device = Devices.CreateLogDevice(devicePath); FasterLog logger = new FasterLog(new FasterLogSettings { LogDevice = device }); long nextAddress = 0; bool keepGoing = true; int i = 0; var result = new List <(string, long, long)>(); // using (FasterLogScanIterator iter = logger.Scan(logger.BeginAddress, 100_000_000, name: nameof(GetListAsync))) using (FasterLogScanIterator iter = logger.Scan(nextAddress, 1_000_000_000)) { while (keepGoing) { Console.WriteLine("Going"); LocalTime timeOfDay; await foreach ((byte[] bytes, int length) in iter.GetAsyncEnumerable()) { DateTimeZone tz = DateTimeZoneProviders.Tzdb.GetSystemDefault(); timeOfDay = SystemClock.Instance.GetCurrentInstant().InZone(tz).TimeOfDay; nextAddress = iter.NextAddress; Console.WriteLine("Time={1} NextAddress={0}, Count={2}", iter.NextAddress, timeOfDay, i++); var cts = new CancellationTokenSource(); UTF8Encoding encoding = new UTF8Encoding(); try { await Task.WhenAny(WaitAsync(iter), SetTimeout(cts)); } catch (Exception e) { Console.Error.WriteLine($"Error={e.GetType()}, Message={e.ToString()}"); break; } timeOfDay = SystemClock.Instance.GetCurrentInstant().InZone(tz).TimeOfDay; Console.WriteLine("Time={2} ContentLength={0}", bytes.Length, iter.NextAddress, timeOfDay); } await Task.Delay(5000); } } return(result); }
static void ScanThread() { Random r = new Random(); byte[] result; using (iter = log.Scan(log.BeginAddress, long.MaxValue)) { while (true) { while (!iter.GetNext(out result, out int length)) { // For finite end address, check if iteration ended // if (iter.CurrentAddress >= endAddress) return; iter.WaitAsync().GetAwaiter().GetResult(); } // Memory pool variant: // iter.GetNext(pool, out IMemoryOwner<byte> resultMem, out int length)) if (Different(result, staticEntry, out int location)) { throw new Exception("Invalid entry found"); } // Re-insert entry with small probability if (r.Next(100) < 10) { log.Enqueue(result); } // Example of random read from given address // (result, _) = log.ReadAsync(iter.CurrentAddress).GetAwaiter().GetResult(); // Truncate log until after recently processed entry log.TruncateUntil(iter.NextAddress); // Safer truncate variant: truncate until start of page // log.TruncateUntilPageStart(iter.NextAddress); } } // Example of recoverable (named) iterator: // using (iter = log.Scan(log.BeginAddress, long.MaxValue, "foo")) }
public async Task <Option <(string, long, long)> > GetNext() { using FasterLogScanIterator iter = logger.Scan(nextAddress, 100_000_000); while (true) { byte[] entry; int length; while (!iter.GetNext(out entry, out length)) { if (iter.CurrentAddress >= 100_000_000) { return(Option.None <(string, long, long)>()); } } UTF8Encoding encoding = new UTF8Encoding(); await iter.WaitAsync(); nextAddress = iter.NextAddress; return(Option.Some((encoding.GetString(entry), iter.CurrentAddress, iter.NextAddress))); // Possible to pipe } }
/// <summary> /// Main program entry point /// </summary> static void Main() { bool sync = true; // Populate entry being inserted for (int i = 0; i < entryLength; i++) { staticEntry[i] = (byte)i; } var path = Path.GetTempPath() + "FasterLogSample/"; IDevice device = Devices.CreateLogDevice(path + "hlog.log"); // FasterLog will recover and resume if there is a previous commit found log = new FasterLog(new FasterLogSettings { LogDevice = device }); using (iter = log.Scan(log.BeginAddress, long.MaxValue)) { if (sync) { // Log writer thread: create as many as needed new Thread(new ThreadStart(LogWriterThread)).Start(); // Threads for iterator scan: create as many as needed new Thread(() => ScanThread()).Start(); // Threads for reporting, commit new Thread(new ThreadStart(ReportThread)).Start(); var t = new Thread(new ThreadStart(CommitThread)); t.Start(); t.Join(); } else { // Async version of demo: expect lower performance // particularly for small payload sizes const int NumParallelTasks = 10_000; ThreadPool.SetMinThreads(2 * Environment.ProcessorCount, 2 * Environment.ProcessorCount); TaskScheduler.UnobservedTaskException += (object sender, UnobservedTaskExceptionEventArgs e) => { Console.WriteLine($"Unobserved task exception: {e.Exception}"); e.SetObserved(); }; Task[] tasks = new Task[NumParallelTasks]; for (int i = 0; i < NumParallelTasks; i++) { int local = i; tasks[i] = Task.Run(() => AsyncLogWriter(local)); } var scan = Task.Run(() => AsyncScan()); // Threads for reporting, commit new Thread(new ThreadStart(ReportThread)).Start(); new Thread(new ThreadStart(CommitThread)).Start(); Task.WaitAll(tasks); Task.WaitAll(scan); } } }
public async Task <bool> WaitAsync(FasterLogScanIterator iter) { return(await iter.WaitAsync()); }
private async Task <bool> WaitAsync(FasterLogScanIterator iter, CancellationToken cancellationToken) { return(await iter.WaitAsync(cancellationToken)); }