public StreamBlockManager(string path, LMDBEnvironmentFlags envFlags, Wpid wpid, IStreamBlockStorage blockStorage = null, long maxTotalSize = 1024L * 1024 * 1024 * 1024 * 1024, uint envSizeMb = 1024) // 1Gb takes 2 MB of TLB, do not make it too large TODO calculate realistic required size that is derived from maxTotalSize when using smallest buffers : base(path, envFlags, wpid, PageSize, maxTotalSize, envSizeMb) { _blocksDb = Environment.OpenDatabase("_streamBlocks", new DatabaseConfig(DbFlags.Create | DbFlags.DuplicatesSort | DbFlags.IntegerDuplicates | DbFlags.DuplicatesFixed) { DupSortPrefix = 64 * 64 } ); _pidDb = Environment.OpenDatabase("_streamLogState", new DatabaseConfig(DbFlags.Create | DbFlags.IntegerKey)); _buckets = new SharedMemoryBuckets(Path.Combine(path, "buckets"), pageSize: PageSize, maxBucketIndex: BufferRef.MaxBucketIdx); _blockStorage = blockStorage; }
public StreamLogManager(ProcessConfig processConfig, string dataStoreName, string dataStorePath = null, uint maxLogSizeMb = 1024, bool disableNotificationLog = false, bool disablePacker = false, IStreamBlockStorage blockStorage = null) { if (LeaksDetection.Enabled) { Spreads.Buffers.BufferPool.PinnedArrayMemoryPool.AddStackTraceOnRent = true; } DataStorePath = dataStorePath ?? Path.Combine(processConfig.DataRootPath, dataStoreName); ProcessConfig = processConfig; _wpidValue = ProcessConfig.Wpid; DisableNotificationLog = disableNotificationLog; DisablePacker = disablePacker; var bufferPoolPath = Path.Combine(DataStorePath, "log", "logbuffer"); var bufferPoolFlags = StartupConfig.StreamLogBufferPoolFlags; BufferPool = new BlockMemoryPool(bufferPoolPath, maxLogSizeMb, bufferPoolFlags, processConfig.Wpid, maxBufferLength: MaxBufferSize, maxBuffersPerBucket: Environment.ProcessorCount * 4); if (blockStorage == null) { var dataStoragePath = Path.Combine(DataStorePath, "storage"); Directory.CreateDirectory(dataStoragePath); var path = Path.GetFullPath(Path.Combine(dataStoragePath, "data.db")); var uri = new Uri(path); var absoluteUri = uri.AbsoluteUri; blockStorage = new SQLiteStorage($@"Data Source={absoluteUri}?cache=shared"); } var blockIndexPath = Path.Combine(DataStorePath, "log", "blockindex"); var blockIndexFlags = StartupConfig.StreamBlockIndexFlags; var blockIndexSizeMb = Math.Max(StartupConfig.StreamBlockTableMaxSizeMb, 128); BlockIndex = new StreamBlockIndex(blockIndexPath, blockIndexSizeMb, blockIndexFlags, BufferPool, blockStorage); var logStateStoragePath = Path.Combine(DataStorePath, "log", "logstate"); StateStorage = new StreamLogStateStorage(logStateStoragePath); // For Log0 tests we need state but we removed it from Log0 ctor, so always init it. // In real code _disableNotificationLog is always false. Log0State = StateStorage.GetState(StreamLogId.Log0Id); Log0State.CheckInit(StreamLogId.Log0Id, StreamLogNotification.Size, StreamLogFlags.IsBinary | StreamLogFlags.NoTimestamp | StreamLogFlags.DropPacked | StreamLogFlags.Pow2Payload); Log0State.HintRatePerMinute(MaxBufferSize); if (!DisableNotificationLog) { Log0 = new NotificationLog(this); OpenStreams.TryAdd((long)StreamLogId.Log0Id, Log0); Log0Reader = new NotificationLog.Reader(Log0, Cts.Token); var lastVersion = (ulong)Log0.CurrentVersion; if (lastVersion > 1) { Log0Reader.MoveGT(lastVersion - 1); } } Packer = new Packer(this, StateStorage, disablePacker); StartLog0(dataStoreName); // TODO we need more general unlocking locking that detects missed updates // and works for all waiters, not only for ack. See Log0.Reader comments. //_unlockTimer = new Timer(o => //{ // TryCompleteAckRequests(); //}, null, 0, 1000); }