Ejemplo n.º 1
0
        public StreamLogStateStorage(string directoryPath)
            : base(directoryPath, StartupConfig.StreamStateEnvFlags, default, ProcessConfig.ProcessConfigRecord.BufferSize,
                   1024L * 1024 * 1024) // 1GB is way too much but it takes no resources above actually used
        {
            // BRA dbs and _env are set in base class

            // InstanceId -> BufferRef
            _stateDb = Environment.OpenDatabase("_streamLogState", new DatabaseConfig(DbFlags.Create | DbFlags.IntegerKey));

            _buckets = new SharedMemoryBuckets(directoryPath, StreamLogState.StreamLogStateRecord.BufferSize, 0);

            using (var txn = Environment.BeginTransaction())
            {
                try
                {
                    long sharedLogId = 0;

                    if (_stateDb.TryGet(txn, ref sharedLogId, out BufferRef sharedBufferRef))
                    {
                        txn.Abort();
                    }
                    else
                    {
                        sharedBufferRef = Allocate(txn, 0, out var fromFreeList, null);
                        _stateDb.Put(txn, sharedLogId, sharedBufferRef, TransactionPutOptions.NoOverwrite);
                        txn.Commit();
                        if (!fromFreeList)
                        {
                            Environment.Sync(true);
                        }
                    }

                    SharedState = new StreamLogState(default, (IntPtr)_buckets[sharedBufferRef].Data);
Ejemplo n.º 2
0
        internal StreamLog(StreamLogManager streamLogManager,
                           StreamLogState state,
                           int ratePerMinuteHint = 0,
                           string textId         = null)
        {
            _streamLogManager = streamLogManager;
            State             = state;

            WriteMode       = state.GetWriteMode();
            _streamLogFlags = State.StreamLogFlags;

            if (state.GetRatePerMinute() == 0)
            {
                if (ratePerMinuteHint > 0)
                {
                    // hinted rate stored as negative
                    State.SetRatePerMinute(-ratePerMinuteHint);
                }
                else
                {
                    State.SetRatePerMinute(MinChunkSize);
                }
            }
            else
            {
                if (ratePerMinuteHint > 0 && Slid != StreamLogId.Log0Id)
                {
                    // hinted rate stored as negative
                    State.SetRatePerMinute(-ratePerMinuteHint);
                }
            }

            // TODO State init must check for RO
            if (_streamLogManager.BlockIndex.GetIsCompleted(State.StreamLogId))
            {
                State.SetIsCompleted();
            }

            _textId = textId;
        }
Ejemplo n.º 3
0
        public StreamLogManager(ProcessConfig processConfig,
                                string dataStoreName,
                                string dataStorePath             = null,
                                uint maxLogSizeMb                = 1024,
                                bool disableNotificationLog      = false,
                                bool disablePacker               = false,
                                IStreamBlockStorage blockStorage = null)
        {
            if (LeaksDetection.Enabled)
            {
                Spreads.Buffers.BufferPool.PinnedArrayMemoryPool.AddStackTraceOnRent = true;
            }

            DataStorePath = dataStorePath ?? Path.Combine(processConfig.DataRootPath, dataStoreName);

            ProcessConfig = processConfig;
            _wpidValue    = ProcessConfig.Wpid;

            DisableNotificationLog = disableNotificationLog;
            DisablePacker          = disablePacker;

            var bufferPoolPath  = Path.Combine(DataStorePath, "log", "logbuffer");
            var bufferPoolFlags = StartupConfig.StreamLogBufferPoolFlags;

            BufferPool = new BlockMemoryPool(bufferPoolPath, maxLogSizeMb, bufferPoolFlags, processConfig.Wpid,
                                             maxBufferLength: MaxBufferSize,
                                             maxBuffersPerBucket: Environment.ProcessorCount * 4);

            if (blockStorage == null)
            {
                var dataStoragePath = Path.Combine(DataStorePath, "storage");
                Directory.CreateDirectory(dataStoragePath);
                var path        = Path.GetFullPath(Path.Combine(dataStoragePath, "data.db"));
                var uri         = new Uri(path);
                var absoluteUri = uri.AbsoluteUri;

                blockStorage = new SQLiteStorage($@"Data Source={absoluteUri}?cache=shared");
            }

            var blockIndexPath   = Path.Combine(DataStorePath, "log", "blockindex");
            var blockIndexFlags  = StartupConfig.StreamBlockIndexFlags;
            var blockIndexSizeMb = Math.Max(StartupConfig.StreamBlockTableMaxSizeMb, 128);

            BlockIndex = new StreamBlockIndex(blockIndexPath, blockIndexSizeMb, blockIndexFlags, BufferPool, blockStorage);

            var logStateStoragePath = Path.Combine(DataStorePath, "log", "logstate");

            StateStorage = new StreamLogStateStorage(logStateStoragePath);

            // For Log0 tests we need state but we removed it from Log0 ctor, so always init it.
            // In real code _disableNotificationLog is always false.
            Log0State = StateStorage.GetState(StreamLogId.Log0Id);
            Log0State.CheckInit(StreamLogId.Log0Id, StreamLogNotification.Size, StreamLogFlags.IsBinary | StreamLogFlags.NoTimestamp | StreamLogFlags.DropPacked | StreamLogFlags.Pow2Payload);

            Log0State.HintRatePerMinute(MaxBufferSize);

            if (!DisableNotificationLog)
            {
                Log0 = new NotificationLog(this);
                OpenStreams.TryAdd((long)StreamLogId.Log0Id, Log0);
                Log0Reader = new NotificationLog.Reader(Log0, Cts.Token);
                var lastVersion = (ulong)Log0.CurrentVersion;
                if (lastVersion > 1)
                {
                    Log0Reader.MoveGT(lastVersion - 1);
                }
            }

            Packer = new Packer(this, StateStorage, disablePacker);

            StartLog0(dataStoreName);

            // TODO we need more general unlocking locking that detects missed updates
            // and works for all waiters, not only for ack. See Log0.Reader comments.
            //_unlockTimer = new Timer(o =>
            //{
            //    TryCompleteAckRequests();
            //}, null, 0, 1000);
        }