Ejemplo n.º 1
0
        public void Init(long buildToPosition)
        {
            Log.Info("TableIndex initialization...");

            _tableIndex.Initialize(buildToPosition);
            _persistedPreparePos = _tableIndex.PrepareCheckpoint;
            _persistedCommitPos  = _tableIndex.CommitCheckpoint;
            _lastCommitPosition  = _tableIndex.CommitCheckpoint;

            if (_lastCommitPosition >= buildToPosition)
            {
                throw new Exception(string.Format("_lastCommitPosition {0} >= buildToPosition {1}", _lastCommitPosition, buildToPosition));
            }

            var startTime    = DateTime.UtcNow;
            var lastTime     = DateTime.UtcNow;
            var reportPeriod = TimeSpan.FromSeconds(5);

            Log.Info("ReadIndex building...");

            _indexRebuild = true;
            using (var reader = _backend.BorrowReader())
            {
                var startPosition = Math.Max(0, _persistedCommitPos);
                reader.Reposition(startPosition);

                var commitedPrepares = new List <PrepareLogRecord>();

                long          processed = 0;
                SeqReadResult result;
                while ((result = reader.TryReadNext()).Success && result.LogRecord.LogPosition < buildToPosition)
                {
                    switch (result.LogRecord.RecordType)
                    {
                    case LogRecordType.Prepare:
                    {
                        var prepare = (PrepareLogRecord)result.LogRecord;
                        if (prepare.Flags.HasAnyOf(PrepareFlags.IsCommitted))
                        {
                            if (prepare.Flags.HasAnyOf(PrepareFlags.Data | PrepareFlags.StreamDelete))
                            {
                                commitedPrepares.Add(prepare);
                            }
                            if (prepare.Flags.HasAnyOf(PrepareFlags.TransactionEnd))
                            {
                                Commit(commitedPrepares);
                                commitedPrepares.Clear();
                            }
                        }
                        break;
                    }

                    case LogRecordType.Commit:
                        Commit((CommitLogRecord)result.LogRecord);
                        break;

                    case LogRecordType.System:
                        break;

                    default:
                        throw new Exception(string.Format("Unknown RecordType: {0}", result.LogRecord.RecordType));
                    }

                    processed += 1;
                    if (DateTime.UtcNow - lastTime > reportPeriod || processed % 100000 == 0)
                    {
                        Log.Debug("ReadIndex Rebuilding: processed {0} records ({1:0.0}%).",
                                  processed, (result.RecordPostPosition - startPosition) * 100.0 / (buildToPosition - startPosition));
                        lastTime = DateTime.UtcNow;
                    }
                }
                Log.Debug("ReadIndex rebuilding done: total processed {0} records, time elapsed: {1}.", processed, DateTime.UtcNow - startTime);

                _backend.SetSystemSettings(GetSystemSettings());
            }

            _indexRebuild = false;
        }
Ejemplo n.º 2
0
        public void Init(long buildToPosition)
        {
            Log.Information("TableIndex initialization...");

            _tableIndex.Initialize(buildToPosition);
            _persistedPreparePos = _tableIndex.PrepareCheckpoint;
            _persistedCommitPos  = _tableIndex.CommitCheckpoint;
            //todo(clc) determin if this needs to move into the TableIndex re:project-io
            _indexChk.Write(_tableIndex.CommitCheckpoint);
            _indexChk.Flush();

            if (_indexChk.Read() >= buildToPosition)
            {
                throw new Exception(string.Format("_lastCommitPosition {0} >= buildToPosition {1}", _indexChk.Read(),
                                                  buildToPosition));
            }

            var startTime    = DateTime.UtcNow;
            var lastTime     = DateTime.UtcNow;
            var reportPeriod = TimeSpan.FromSeconds(5);

            Log.Information("ReadIndex building...");

            _indexRebuild = true;
            using (var reader = _backend.BorrowReader()) {
                var startPosition = Math.Max(0, _persistedCommitPos);
                reader.Reposition(startPosition);

                var commitedPrepares = new List <PrepareLogRecord>();

                long          processed = 0;
                SeqReadResult result;
                while ((result = reader.TryReadNext()).Success && result.LogRecord.LogPosition < buildToPosition)
                {
                    switch (result.LogRecord.RecordType)
                    {
                    case LogRecordType.Prepare: {
                        var prepare = (PrepareLogRecord)result.LogRecord;
                        if (prepare.Flags.HasAnyOf(PrepareFlags.IsCommitted))
                        {
                            if (prepare.Flags.HasAnyOf(PrepareFlags.SingleWrite))
                            {
                                Commit(commitedPrepares, false, false);
                                commitedPrepares.Clear();
                                Commit(new[] { prepare }, result.Eof, false);
                            }
                            else
                            {
                                if (prepare.Flags.HasAnyOf(PrepareFlags.Data | PrepareFlags.StreamDelete))
                                {
                                    commitedPrepares.Add(prepare);
                                }
                                if (prepare.Flags.HasAnyOf(PrepareFlags.TransactionEnd))
                                {
                                    Commit(commitedPrepares, result.Eof, false);
                                    commitedPrepares.Clear();
                                }
                            }
                        }

                        break;
                    }

                    case LogRecordType.Commit:
                        Commit((CommitLogRecord)result.LogRecord, result.Eof, false);
                        break;

                    case LogRecordType.System:
                        break;

                    default:
                        throw new Exception(string.Format("Unknown RecordType: {0}", result.LogRecord.RecordType));
                    }

                    processed += 1;
                    if (DateTime.UtcNow - lastTime > reportPeriod || processed % 100000 == 0)
                    {
                        Log.Debug("ReadIndex Rebuilding: processed {processed} records ({resultPosition:0.0}%).",
                                  processed,
                                  (result.RecordPostPosition - startPosition) * 100.0 / (buildToPosition - startPosition));
                        lastTime = DateTime.UtcNow;
                    }
                }

                Log.Debug("ReadIndex rebuilding done: total processed {processed} records, time elapsed: {elapsed}.",
                          processed, DateTime.UtcNow - startTime);
                _bus.Publish(new StorageMessage.TfEofAtNonCommitRecord());
                _backend.SetSystemSettings(GetSystemSettings());
            }
            _indexRebuild = false;
        }