public StreamLogStateStorage(string directoryPath)
: base(directoryPath, StartupConfig.StreamStateEnvFlags, default, ProcessConfig.ProcessConfigRecord.BufferSize,
1024L * 1024 * 1024) // 1GB is way too much but it takes no resources above actually used
{
// BRA dbs and _env are set in base class
// InstanceId -> BufferRef
_stateDb = Environment.OpenDatabase("_streamLogState", new DatabaseConfig(DbFlags.Create | DbFlags.IntegerKey));
_buckets = new SharedMemoryBuckets(directoryPath, StreamLogState.StreamLogStateRecord.BufferSize, 0);
using (var txn = Environment.BeginTransaction())
{
try
{
long sharedLogId = 0;
if (_stateDb.TryGet(txn, ref sharedLogId, out BufferRef sharedBufferRef))
{
txn.Abort();
}
else
{
sharedBufferRef = Allocate(txn, 0, out var fromFreeList, null);
_stateDb.Put(txn, sharedLogId, sharedBufferRef, TransactionPutOptions.NoOverwrite);
txn.Commit();
if (!fromFreeList)
{
Environment.Sync(true);
}
}
SharedState = new StreamLogState(default, (IntPtr)_buckets[sharedBufferRef].Data);
internal StreamLog(StreamLogManager streamLogManager,
StreamLogState state,
int ratePerMinuteHint = 0,
string textId = null)
{
_streamLogManager = streamLogManager;
State = state;
WriteMode = state.GetWriteMode();
_streamLogFlags = State.StreamLogFlags;
if (state.GetRatePerMinute() == 0)
{
if (ratePerMinuteHint > 0)
{
// hinted rate stored as negative
State.SetRatePerMinute(-ratePerMinuteHint);
}
else
{
State.SetRatePerMinute(MinChunkSize);
}
}
else
{
if (ratePerMinuteHint > 0 && Slid != StreamLogId.Log0Id)
{
// hinted rate stored as negative
State.SetRatePerMinute(-ratePerMinuteHint);
}
}
// TODO State init must check for RO
if (_streamLogManager.BlockIndex.GetIsCompleted(State.StreamLogId))
{
State.SetIsCompleted();
}
_textId = textId;
}
public StreamLogManager(ProcessConfig processConfig,
string dataStoreName,
string dataStorePath = null,
uint maxLogSizeMb = 1024,
bool disableNotificationLog = false,
bool disablePacker = false,
IStreamBlockStorage blockStorage = null)
{
if (LeaksDetection.Enabled)
{
Spreads.Buffers.BufferPool.PinnedArrayMemoryPool.AddStackTraceOnRent = true;
}
DataStorePath = dataStorePath ?? Path.Combine(processConfig.DataRootPath, dataStoreName);
ProcessConfig = processConfig;
_wpidValue = ProcessConfig.Wpid;
DisableNotificationLog = disableNotificationLog;
DisablePacker = disablePacker;
var bufferPoolPath = Path.Combine(DataStorePath, "log", "logbuffer");
var bufferPoolFlags = StartupConfig.StreamLogBufferPoolFlags;
BufferPool = new BlockMemoryPool(bufferPoolPath, maxLogSizeMb, bufferPoolFlags, processConfig.Wpid,
maxBufferLength: MaxBufferSize,
maxBuffersPerBucket: Environment.ProcessorCount * 4);
if (blockStorage == null)
{
var dataStoragePath = Path.Combine(DataStorePath, "storage");
Directory.CreateDirectory(dataStoragePath);
var path = Path.GetFullPath(Path.Combine(dataStoragePath, "data.db"));
var uri = new Uri(path);
var absoluteUri = uri.AbsoluteUri;
blockStorage = new SQLiteStorage($@"Data Source={absoluteUri}?cache=shared");
}
var blockIndexPath = Path.Combine(DataStorePath, "log", "blockindex");
var blockIndexFlags = StartupConfig.StreamBlockIndexFlags;
var blockIndexSizeMb = Math.Max(StartupConfig.StreamBlockTableMaxSizeMb, 128);
BlockIndex = new StreamBlockIndex(blockIndexPath, blockIndexSizeMb, blockIndexFlags, BufferPool, blockStorage);
var logStateStoragePath = Path.Combine(DataStorePath, "log", "logstate");
StateStorage = new StreamLogStateStorage(logStateStoragePath);
// For Log0 tests we need state but we removed it from Log0 ctor, so always init it.
// In real code _disableNotificationLog is always false.
Log0State = StateStorage.GetState(StreamLogId.Log0Id);
Log0State.CheckInit(StreamLogId.Log0Id, StreamLogNotification.Size, StreamLogFlags.IsBinary | StreamLogFlags.NoTimestamp | StreamLogFlags.DropPacked | StreamLogFlags.Pow2Payload);
Log0State.HintRatePerMinute(MaxBufferSize);
if (!DisableNotificationLog)
{
Log0 = new NotificationLog(this);
OpenStreams.TryAdd((long)StreamLogId.Log0Id, Log0);
Log0Reader = new NotificationLog.Reader(Log0, Cts.Token);
var lastVersion = (ulong)Log0.CurrentVersion;
if (lastVersion > 1)
{
Log0Reader.MoveGT(lastVersion - 1);
}
}
Packer = new Packer(this, StateStorage, disablePacker);
StartLog0(dataStoreName);
// TODO we need more general unlocking locking that detects missed updates
// and works for all waiters, not only for ack. See Log0.Reader comments.
//_unlockTimer = new Timer(o =>
//{
// TryCompleteAckRequests();
//}, null, 0, 1000);
}