コード例 #1
0
ファイル: FasterLogTests.cs プロジェクト: mito-csod/FASTER
        public void FasterLogTest6([Values] LogChecksumType logChecksum)
        {
            log = new FasterLog(new FasterLogSettings {
                LogDevice = device, MemorySizeBits = 20, PageSizeBits = 14, LogChecksum = logChecksum, LogCommitManager = manager
            });
            byte[] data1 = new byte[1000];
            for (int i = 0; i < 100; i++)
            {
                data1[i] = (byte)i;
            }

            for (int i = 0; i < 100; i++)
            {
                log.Enqueue(data1);
            }
            log.RefreshUncommitted();
            Assert.IsTrue(log.SafeTailAddress == log.TailAddress);

            Assert.IsTrue(log.CommittedUntilAddress < log.SafeTailAddress);

            using (var iter = log.Scan(0, long.MaxValue, scanUncommitted: true))
            {
                while (iter.GetNext(out _, out _, out _))
                {
                    log.TruncateUntil(iter.NextAddress);
                }
                Assert.IsTrue(iter.NextAddress == log.SafeTailAddress);
                log.Enqueue(data1);
                Assert.IsFalse(iter.GetNext(out _, out _, out _));
                log.RefreshUncommitted();
                Assert.IsTrue(iter.GetNext(out _, out _, out _));
            }
            log.Dispose();
        }
コード例 #2
0
        private void FasterLogTest1(LogChecksumType logChecksum, IDevice device, ILogCommitManager logCommitManager)
        {
            log = new FasterLog(new FasterLogSettings {
                PageSizeBits = 20, SegmentSizeBits = 20, LogDevice = device, LogChecksum = logChecksum, LogCommitManager = logCommitManager
            });

            byte[] entry = new byte[entryLength];
            for (int i = 0; i < entryLength; i++)
            {
                entry[i] = (byte)i;
            }

            for (int i = 0; i < numEntries; i++)
            {
                log.Enqueue(entry);
            }
            log.Commit(true);

            using (var iter = log.Scan(0, long.MaxValue))
            {
                int count = 0;
                while (iter.GetNext(out byte[] result, out int length, out long currentAddress))
                {
                    count++;
                    Assert.IsTrue(result.SequenceEqual(entry));
                    if (count % 100 == 0)
                    {
                        log.TruncateUntil(iter.NextAddress);
                    }
                }
                Assert.IsTrue(count == numEntries);
            }

            log.Dispose();
        }
コード例 #3
0
ファイル: FasterLogTests.cs プロジェクト: zheng1748/FASTER
        public void FasterLogTest1([Values] LogChecksumType logChecksum)
        {
            log = new FasterLog(new FasterLogSettings {
                LogDevice = device, LogChecksum = logChecksum
            });

            byte[] entry = new byte[entryLength];
            for (int i = 0; i < entryLength; i++)
            {
                entry[i] = (byte)i;
            }

            for (int i = 0; i < numEntries; i++)
            {
                log.Enqueue(entry);
            }
            log.Commit(true);

            using (var iter = log.Scan(0, long.MaxValue))
            {
                int count = 0;
                while (iter.GetNext(out byte[] result, out int length))
                {
                    count++;
                    Assert.IsTrue(result.SequenceEqual(entry));
                    if (count % 100 == 0)
                    {
                        log.TruncateUntil(iter.CurrentAddress);
                    }
                }
                Assert.IsTrue(count == numEntries);
            }

            log.Dispose();
        }
コード例 #4
0
        static void ScanThread()
        {
            Random r = new Random();

            byte[] result;

            using (iter = log.Scan(log.BeginAddress, long.MaxValue))
            {
                while (true)
                {
                    while (!iter.GetNext(out result, out int length))
                    {
                        // For finite end address, check if iteration ended
                        // if (iter.CurrentAddress >= endAddress) return;
                        iter.WaitAsync().GetAwaiter().GetResult();
                    }

                    // Memory pool variant:
                    // iter.GetNext(pool, out IMemoryOwner<byte> resultMem, out int length))

                    if (Different(result, staticEntry, out int location))
                    {
                        throw new Exception("Invalid entry found");
                    }

                    // Re-insert entry with small probability
                    if (r.Next(100) < 10)
                    {
                        log.Enqueue(result);
                    }

                    // Example of random read from given address
                    // (result, _) = log.ReadAsync(iter.CurrentAddress).GetAwaiter().GetResult();

                    // Truncate log until after recently processed entry
                    log.TruncateUntil(iter.NextAddress);

                    // Safer truncate variant: truncate until start of page
                    // log.TruncateUntilPageStart(iter.NextAddress);
                }
            }

            // Example of recoverable (named) iterator:
            // using (iter = log.Scan(log.BeginAddress, long.MaxValue, "foo"))
        }
コード例 #5
0
ファイル: Program.cs プロジェクト: hixio-mh/FASTER-1
        static async Task ConsumerAsync(FasterLog log, CancellationToken cancellationToken)
        {
            using var iter = log.Scan(log.BeginAddress, long.MaxValue, "foo", true, ScanBufferingMode.DoublePageBuffering, scanUncommitted: true);

            int count = 0;

            await foreach (var(result, length, currentAddress, nextAddress) in iter.GetAsyncEnumerable(cancellationToken))
            {
                Console.WriteLine($"Consuming {Encoding.UTF8.GetString(result)}");
                iter.CompleteUntil(nextAddress);
                log.TruncateUntil(nextAddress);

                // We simulate temporary slow down of data consumption
                // This will cause transient log spill to disk (observe folder on storage)
                if (count++ > 1000 && count < 1200)
                {
                    Thread.Sleep(100);
                }
            }
        }
コード例 #6
0
        public async Task FasterLogTest4([Values] LogChecksumType logChecksum)
        {
            log = new FasterLog(new FasterLogSettings {
                LogDevice = device, PageSizeBits = 14, LogChecksum = logChecksum
            });
            byte[] data1 = new byte[100];
            for (int i = 0; i < 100; i++)
            {
                data1[i] = (byte)i;
            }

            for (int i = 0; i < 100; i++)
            {
                log.Enqueue(data1);
            }

            Assert.IsTrue(log.CommittedUntilAddress == log.BeginAddress);
            await log.CommitAsync();

            Assert.IsTrue(log.CommittedUntilAddress == log.TailAddress);
            Assert.IsTrue(log.CommittedBeginAddress == log.BeginAddress);

            using (var iter = log.Scan(0, long.MaxValue))
            {
                // Should read the "hole" and return false
                var iterResult = iter.GetNext(out byte[] entry, out _, out _);
                log.TruncateUntil(iter.NextAddress);

                Assert.IsTrue(log.CommittedUntilAddress == log.TailAddress);
                Assert.IsTrue(log.CommittedBeginAddress < log.BeginAddress);
                Assert.IsTrue(iter.NextAddress == log.BeginAddress);

                await log.CommitAsync();

                Assert.IsTrue(log.CommittedUntilAddress == log.TailAddress);
                Assert.IsTrue(log.CommittedBeginAddress == log.BeginAddress);
            }
            log.Dispose();
        }
コード例 #7
0
 public void TruncateUntil(long address)
 {
     logger.TruncateUntil(address);
 }