private void FasterLogTest1(LogChecksumType logChecksum, IDevice device, ILogCommitManager logCommitManager) { log = new FasterLog(new FasterLogSettings { PageSizeBits = 20, SegmentSizeBits = 20, LogDevice = device, LogChecksum = logChecksum, LogCommitManager = logCommitManager }); byte[] entry = new byte[entryLength]; for (int i = 0; i < entryLength; i++) { entry[i] = (byte)i; } for (int i = 0; i < numEntries; i++) { log.Enqueue(entry); } log.Commit(true); using (var iter = log.Scan(0, long.MaxValue)) { int count = 0; while (iter.GetNext(out byte[] result, out int length, out long currentAddress)) { count++; Assert.IsTrue(result.SequenceEqual(entry)); if (count % 100 == 0) { log.TruncateUntil(iter.NextAddress); } } Assert.IsTrue(count == numEntries); } log.Dispose(); }
/// <summary> /// Recover info from token /// </summary> /// <param name="logCommitManager"></param> /// <returns></returns> internal void Recover(ILogCommitManager logCommitManager) { var metadata = logCommitManager.GetCommitMetadata(); if (metadata == null) { throw new Exception("Invalid log commit metadata during recovery"); } Initialize(new BinaryReader(new MemoryStream(metadata))); }
/// <summary> /// Create new log instance /// </summary> /// <param name="logSettings"></param> public FasterLog(FasterLogSettings logSettings) { bool oldCommitManager = false; if (oldCommitManager) { logCommitManager = logSettings.LogCommitManager ?? new LocalLogCommitManager(logSettings.LogCommitFile ?? logSettings.LogDevice.FileName + ".commit"); } else { logCommitManager = logSettings.LogCommitManager ?? new DeviceLogCommitCheckpointManager (new LocalStorageNamedDeviceFactory(), new DefaultCheckpointNamingScheme( new FileInfo(logSettings.LogDevice.FileName).Directory.FullName)); } if (logSettings.LogCommitManager == null) { disposeLogCommitManager = true; } // Reserve 8 byte checksum in header if requested logChecksum = logSettings.LogChecksum; headerSize = logChecksum == LogChecksumType.PerEntry ? 12 : 4; getMemory = logSettings.GetMemory; epoch = new LightEpoch(); CommittedUntilAddress = Constants.kFirstValidAddress; CommittedBeginAddress = Constants.kFirstValidAddress; SafeTailAddress = Constants.kFirstValidAddress; allocator = new BlittableAllocator <Empty, byte>( logSettings.GetLogSettings(), null, null, epoch, CommitCallback); allocator.Initialize(); // FasterLog is used as a read-only iterator if (logSettings.ReadOnlyMode) { readOnlyMode = true; allocator.HeadAddress = long.MaxValue; } Restore(out RecoveredIterators); }
/// <summary> /// Create new log instance /// </summary> /// <param name="logSettings"></param> public FasterLog(FasterLogSettings logSettings) { logCommitManager = logSettings.LogCommitManager ?? new LocalLogCommitManager(logSettings.LogCommitFile ?? logSettings.LogDevice.FileName + ".commit"); // Reserve 8 byte checksum in header if requested logChecksum = logSettings.LogChecksum; headerSize = logChecksum == LogChecksumType.PerEntry ? 12 : 4; getMemory = logSettings.GetMemory; epoch = new LightEpoch(); CommittedUntilAddress = Constants.kFirstValidAddress; CommittedBeginAddress = Constants.kFirstValidAddress; allocator = new BlittableAllocator <Empty, byte>( logSettings.GetLogSettings(), null, null, epoch, CommitCallback); allocator.Initialize(); Restore(out RecoveredIterators); }
private async ValueTask FasterLogTest1(LogChecksumType logChecksum, IDevice device, ILogCommitManager logCommitManager, FasterLogTests.IteratorType iteratorType) { var logSettings = new FasterLogSettings { PageSizeBits = 20, SegmentSizeBits = 20, LogDevice = device, LogChecksum = logChecksum, LogCommitManager = logCommitManager }; log = FasterLogTests.IsAsync(iteratorType) ? await FasterLog.CreateAsync(logSettings) : new FasterLog(logSettings); byte[] entry = new byte[entryLength]; for (int i = 0; i < entryLength; i++) { entry[i] = (byte)i; } for (int i = 0; i < numEntries; i++) { log.Enqueue(entry); } log.Commit(true); using (var iter = log.Scan(0, long.MaxValue)) { var counter = new FasterLogTests.Counter(log); switch (iteratorType) { case FasterLogTests.IteratorType.AsyncByteVector: await foreach ((byte[] result, _, _, long nextAddress) in iter.GetAsyncEnumerable()) { Assert.IsTrue(result.SequenceEqual(entry)); counter.IncrementAndMaybeTruncateUntil(nextAddress); // MoveNextAsync() would hang at TailAddress, waiting for more entries (that we don't add). // Note: If this happens and the test has to be canceled, there may be a leftover blob from the log.Commit(), because // the log device isn't Dispose()d; the symptom is currently a numeric string format error in DefaultCheckpointNamingScheme. if (nextAddress == log.TailAddress) { break; } } break; case FasterLogTests.IteratorType.AsyncMemoryOwner: await foreach ((IMemoryOwner <byte> result, int _, long _, long nextAddress) in iter.GetAsyncEnumerable(MemoryPool <byte> .Shared)) { Assert.IsTrue(result.Memory.Span.ToArray().Take(entry.Length).SequenceEqual(entry)); result.Dispose(); counter.IncrementAndMaybeTruncateUntil(nextAddress); // MoveNextAsync() would hang at TailAddress, waiting for more entries (that we don't add). // Note: If this happens and the test has to be canceled, there may be a leftover blob from the log.Commit(), because // the log device isn't Dispose()d; the symptom is currently a numeric string format error in DefaultCheckpointNamingScheme. if (nextAddress == log.TailAddress) { break; } } break; case FasterLogTests.IteratorType.Sync: while (iter.GetNext(out byte[] result, out _, out _)) { Assert.IsTrue(result.SequenceEqual(entry)); counter.IncrementAndMaybeTruncateUntil(iter.NextAddress); } break; default: Assert.Fail("Unknown IteratorType"); break; } Assert.IsTrue(counter.count == numEntries); } log.Dispose(); }