public unsafe void CountAndChecksum() { var path = TestUtils.GetPath(); var rounds = 10; var count = 100_000; var bufferSize = count * 16 + StreamBlock.DataOffset; var pool = new BlockMemoryPool(path, 64, LMDBEnvironmentFlags.WriteMap | LMDBEnvironmentFlags.NoSync, (Wpid)123, maxBufferLength: BitUtil.FindNextPositivePowerOfTwo(bufferSize)); // TODO step on what happens when buffer is too large var buffer = pool.RentMemory(bufferSize); var rm = buffer.RetainBlockMemory(false); var rm2 = rm.Clone(); var seed = 2u; StreamBlock.TryInitialize(new DirectBuffer(rm2), (StreamLogId)1, default, 16, 1, seed, default);
public StreamLogManager(ProcessConfig processConfig, string dataStoreName, string dataStorePath = null, uint maxLogSizeMb = 1024, bool disableNotificationLog = false, bool disablePacker = false, IStreamBlockStorage blockStorage = null) { if (LeaksDetection.Enabled) { Spreads.Buffers.BufferPool.PinnedArrayMemoryPool.AddStackTraceOnRent = true; } DataStorePath = dataStorePath ?? Path.Combine(processConfig.DataRootPath, dataStoreName); ProcessConfig = processConfig; _wpidValue = ProcessConfig.Wpid; DisableNotificationLog = disableNotificationLog; DisablePacker = disablePacker; var bufferPoolPath = Path.Combine(DataStorePath, "log", "logbuffer"); var bufferPoolFlags = StartupConfig.StreamLogBufferPoolFlags; BufferPool = new BlockMemoryPool(bufferPoolPath, maxLogSizeMb, bufferPoolFlags, processConfig.Wpid, maxBufferLength: MaxBufferSize, maxBuffersPerBucket: Environment.ProcessorCount * 4); if (blockStorage == null) { var dataStoragePath = Path.Combine(DataStorePath, "storage"); Directory.CreateDirectory(dataStoragePath); var path = Path.GetFullPath(Path.Combine(dataStoragePath, "data.db")); var uri = new Uri(path); var absoluteUri = uri.AbsoluteUri; blockStorage = new SQLiteStorage($@"Data Source={absoluteUri}?cache=shared"); } var blockIndexPath = Path.Combine(DataStorePath, "log", "blockindex"); var blockIndexFlags = StartupConfig.StreamBlockIndexFlags; var blockIndexSizeMb = Math.Max(StartupConfig.StreamBlockTableMaxSizeMb, 128); BlockIndex = new StreamBlockIndex(blockIndexPath, blockIndexSizeMb, blockIndexFlags, BufferPool, blockStorage); var logStateStoragePath = Path.Combine(DataStorePath, "log", "logstate"); StateStorage = new StreamLogStateStorage(logStateStoragePath); // For Log0 tests we need state but we removed it from Log0 ctor, so always init it. // In real code _disableNotificationLog is always false. Log0State = StateStorage.GetState(StreamLogId.Log0Id); Log0State.CheckInit(StreamLogId.Log0Id, StreamLogNotification.Size, StreamLogFlags.IsBinary | StreamLogFlags.NoTimestamp | StreamLogFlags.DropPacked | StreamLogFlags.Pow2Payload); Log0State.HintRatePerMinute(MaxBufferSize); if (!DisableNotificationLog) { Log0 = new NotificationLog(this); OpenStreams.TryAdd((long)StreamLogId.Log0Id, Log0); Log0Reader = new NotificationLog.Reader(Log0, Cts.Token); var lastVersion = (ulong)Log0.CurrentVersion; if (lastVersion > 1) { Log0Reader.MoveGT(lastVersion - 1); } } Packer = new Packer(this, StateStorage, disablePacker); StartLog0(dataStoreName); // TODO we need more general unlocking locking that detects missed updates // and works for all waiters, not only for ack. See Log0.Reader comments. //_unlockTimer = new Timer(o => //{ // TryCompleteAckRequests(); //}, null, 0, 1000); }