public StreamBlockManager(string path,
LMDBEnvironmentFlags envFlags,
Wpid wpid,
IStreamBlockStorage blockStorage = null,
long maxTotalSize = 1024L * 1024 * 1024 * 1024 * 1024,
uint envSizeMb = 1024) // 1Gb takes 2 MB of TLB, do not make it too large TODO calculate realistic required size that is derived from maxTotalSize when using smallest buffers
: base(path, envFlags, wpid, PageSize, maxTotalSize, envSizeMb)
{
_blocksDb = Environment.OpenDatabase("_streamBlocks",
new DatabaseConfig(DbFlags.Create | DbFlags.DuplicatesSort | DbFlags.IntegerDuplicates | DbFlags.DuplicatesFixed)
{
DupSortPrefix = 64 * 64
}
);
_pidDb = Environment.OpenDatabase("_streamLogState", new DatabaseConfig(DbFlags.Create | DbFlags.IntegerKey));
_buckets = new SharedMemoryBuckets(Path.Combine(path, "buckets"), pageSize: PageSize, maxBucketIndex: BufferRef.MaxBucketIdx);
_blockStorage = blockStorage;
}
public StreamLogManager(ProcessConfig processConfig,
string dataStoreName,
string dataStorePath = null,
uint maxLogSizeMb = 1024,
bool disableNotificationLog = false,
bool disablePacker = false,
IStreamBlockStorage blockStorage = null)
{
if (LeaksDetection.Enabled)
{
Spreads.Buffers.BufferPool.PinnedArrayMemoryPool.AddStackTraceOnRent = true;
}
DataStorePath = dataStorePath ?? Path.Combine(processConfig.DataRootPath, dataStoreName);
ProcessConfig = processConfig;
_wpidValue = ProcessConfig.Wpid;
DisableNotificationLog = disableNotificationLog;
DisablePacker = disablePacker;
var bufferPoolPath = Path.Combine(DataStorePath, "log", "logbuffer");
var bufferPoolFlags = StartupConfig.StreamLogBufferPoolFlags;
BufferPool = new BlockMemoryPool(bufferPoolPath, maxLogSizeMb, bufferPoolFlags, processConfig.Wpid,
maxBufferLength: MaxBufferSize,
maxBuffersPerBucket: Environment.ProcessorCount * 4);
if (blockStorage == null)
{
var dataStoragePath = Path.Combine(DataStorePath, "storage");
Directory.CreateDirectory(dataStoragePath);
var path = Path.GetFullPath(Path.Combine(dataStoragePath, "data.db"));
var uri = new Uri(path);
var absoluteUri = uri.AbsoluteUri;
blockStorage = new SQLiteStorage($@"Data Source={absoluteUri}?cache=shared");
}
var blockIndexPath = Path.Combine(DataStorePath, "log", "blockindex");
var blockIndexFlags = StartupConfig.StreamBlockIndexFlags;
var blockIndexSizeMb = Math.Max(StartupConfig.StreamBlockTableMaxSizeMb, 128);
BlockIndex = new StreamBlockIndex(blockIndexPath, blockIndexSizeMb, blockIndexFlags, BufferPool, blockStorage);
var logStateStoragePath = Path.Combine(DataStorePath, "log", "logstate");
StateStorage = new StreamLogStateStorage(logStateStoragePath);
// For Log0 tests we need state but we removed it from Log0 ctor, so always init it.
// In real code _disableNotificationLog is always false.
Log0State = StateStorage.GetState(StreamLogId.Log0Id);
Log0State.CheckInit(StreamLogId.Log0Id, StreamLogNotification.Size, StreamLogFlags.IsBinary | StreamLogFlags.NoTimestamp | StreamLogFlags.DropPacked | StreamLogFlags.Pow2Payload);
Log0State.HintRatePerMinute(MaxBufferSize);
if (!DisableNotificationLog)
{
Log0 = new NotificationLog(this);
OpenStreams.TryAdd((long)StreamLogId.Log0Id, Log0);
Log0Reader = new NotificationLog.Reader(Log0, Cts.Token);
var lastVersion = (ulong)Log0.CurrentVersion;
if (lastVersion > 1)
{
Log0Reader.MoveGT(lastVersion - 1);
}
}
Packer = new Packer(this, StateStorage, disablePacker);
StartLog0(dataStoreName);
// TODO we need more general unlocking locking that detects missed updates
// and works for all waiters, not only for ack. See Log0.Reader comments.
//_unlockTimer = new Timer(o =>
//{
// TryCompleteAckRequests();
//}, null, 0, 1000);
}
