private void RestoreChangelog(ChangelogMetadata changelogMetadata)
        {
            var numRecords = changelogMetadata.BufferedLimit;

            if (numRecords > 0)
            {
                var records = changelogMetadata.BufferedRecords.Take(numRecords);
                changelogMetadata.StateManager.Restore(changelogMetadata.StoreMetadata, records);

                if (numRecords >= changelogMetadata.BufferedRecords.Count)
                {
                    changelogMetadata.BufferedRecords.Clear();
                }

                long currentOffset = changelogMetadata.StoreMetadata.Offset.Value;
                changelogMetadata.CurrentOffset = currentOffset;
                log.LogDebug($"Restored {numRecords} records from " +
                             $"changelog {changelogMetadata.StoreMetadata.Store.Name} " +
                             $"to store {changelogMetadata.StoreMetadata.ChangelogTopicPartition}, " +
                             $"end offset is {(changelogMetadata.RestoreEndOffset.HasValue ? changelogMetadata.RestoreEndOffset.Value : "unknown")}, " +
                             $"current offset is {currentOffset}");

                changelogMetadata.BufferedLimit  = 0;
                changelogMetadata.TotalRestored += numRecords;

                // TODO : call trigger batchRestored

                var restorationRecordSensor = TaskMetrics.RestorationRecordsSensor(
                    threadId,
                    changelogMetadata.StateManager.taskId,
                    metricsRegistry);
                restorationRecordSensor.Record(Math.Max(changelogMetadata.RestoreEndOffset.Value - currentOffset, 0));
            }
            else if (changelogMetadata.StoreMetadata.Offset.HasValue)
            {
                changelogMetadata.CurrentOffset = changelogMetadata.StoreMetadata.Offset.Value;
            }


            if (HasRestoredToEnd(changelogMetadata))
            {
                log.LogInformation($"Finished restoring changelog {changelogMetadata.StoreMetadata.Store.Name} " +
                                   $"to store {changelogMetadata.StoreMetadata.ChangelogTopicPartition} " +
                                   $"with a total number of {changelogMetadata.TotalRestored} records");

                changelogMetadata.ChangelogState = ChangelogState.COMPLETED;

                if (!restoreConsumer.Assignment.Contains(changelogMetadata.StoreMetadata.ChangelogTopicPartition))
                {
                    throw new IllegalStateException($"The current assignment {string.Join(",", restoreConsumer.Assignment.Select(t => $"{t.Topic}-{t.Partition}"))} " +
                                                    $"does not contain the partition {changelogMetadata.StoreMetadata.ChangelogTopicPartition} for pausing.");
                }

                restoreConsumer.Pause(changelogMetadata.StoreMetadata.ChangelogTopicPartition.ToSingle());

                log.LogDebug($"Paused partition {changelogMetadata.StoreMetadata.ChangelogTopicPartition} from the restore consumer");

                // TODO : call trigger restoredEnd
            }
        }
        public void Init(ProcessorContext context, IStateStore root)
        {
            this.context        = context;
            expiredRecordSensor = TaskMetrics.DroppedRecordsSensor(
                Thread.CurrentThread.Name,
                context.Id,
                context.Metrics);

            segments.OpenExisting(context, observedStreamTime);
            context.Register(root, (k, v, t) => Put(k, v));

            isOpen = true;
        }
Esempio n. 3
0
        public virtual void Init(ProcessorContext context)
        {
            log.LogDebug("{LogPrefix}Initializing process context", logPrefix);
            Context = context;
            droppedRecordsSensor = TaskMetrics.DroppedRecordsSensor(
                Thread.CurrentThread.Name,
                Context.Id,
                Context.Metrics);

            foreach (var n in Next)
            {
                n.Init(context);
            }
            log.LogDebug("{LogPrefix}Process context initialized", logPrefix);
        }
Esempio n. 4
0
        public void Init(ProcessorContext context, IStateStore root)
        {
            expiredRecordSensor = TaskMetrics.DroppedRecordsSensor(
                Thread.CurrentThread.Name,
                context.Id,
                context.Metrics);
            this.context = context;

            if (root != null)
            {
                // register the store
                context.Register(root,
                                 (key, value, timestamp) => Put(key, value, timestamp));
            }

            IsOpen = true;
        }
        public StreamTask(string threadId, TaskId id, IEnumerable <TopicPartition> partitions,
                          ProcessorTopology processorTopology, IConsumer <byte[], byte[]> consumer, IStreamConfig configuration,
                          IKafkaSupplier kafkaSupplier, IProducer <byte[], byte[]> producer, IChangelogRegister changelogRegister,
                          StreamMetricsRegistry streamMetricsRegistry)
            : base(id, partitions, processorTopology, consumer, configuration, changelogRegister)
        {
            this.threadId              = threadId;
            this.kafkaSupplier         = kafkaSupplier;
            this.streamMetricsRegistry = streamMetricsRegistry;
            consumedOffsets            = new Dictionary <TopicPartition, long>();
            maxTaskIdleMs              = configuration.MaxTaskIdleMs;
            maxBufferedSize            = configuration.BufferedRecordsPerPartition;
            followMetadata             = configuration.FollowMetadata;
            idleStartTime              = -1;

            // eos enabled
            if (producer == null)
            {
                this.producer = CreateEOSProducer();
                InitializeTransaction();
                eosEnabled = true;
            }
            else
            {
                this.producer = producer;
            }

            var droppedRecordsSensor = TaskMetrics.DroppedRecordsSensor(this.threadId, Id, this.streamMetricsRegistry);

            collector = new RecordCollector(logPrefix, configuration, id, droppedRecordsSensor);
            collector.Init(ref this.producer);

            Context = new ProcessorContext(this, configuration, stateMgr, streamMetricsRegistry)
                      .UseRecordCollector(collector);
            Context.FollowMetadata = followMetadata;

            var partitionsQueue = new Dictionary <TopicPartition, RecordQueue>();

            foreach (var p in partitions)
            {
                var sourceProcessor = processorTopology.GetSourceProcessor(p.Topic);
                sourceProcessor.SetTaskId(id);
                var sourceTimestampExtractor = sourceProcessor.Extractor ?? configuration.DefaultTimestampExtractor;
                var queue = new RecordQueue(
                    logPrefix,
                    $"record-queue-{p.Topic}-{id.Id}-{id.Partition}",
                    sourceTimestampExtractor,
                    p,
                    sourceProcessor,
                    droppedRecordsSensor);
                partitionsQueue.Add(p, queue);
                processors.Add(sourceProcessor);
            }

            partitionGrouper = new PartitionGrouper(partitionsQueue);

            closeTaskSensor            = ThreadMetrics.ClosedTaskSensor(this.threadId, streamMetricsRegistry);
            activeBufferedRecordSensor = TaskMetrics.ActiveBufferedRecordsSensor(this.threadId, Id, streamMetricsRegistry);
            processSensor             = TaskMetrics.ProcessSensor(this.threadId, Id, streamMetricsRegistry);
            processLatencySensor      = TaskMetrics.ProcessLatencySensor(this.threadId, Id, streamMetricsRegistry);
            enforcedProcessingSensor  = TaskMetrics.EnforcedProcessingSensor(this.threadId, Id, streamMetricsRegistry);
            commitSensor              = TaskMetrics.CommitSensor(this.threadId, Id, streamMetricsRegistry);
            activeRestorationSensor   = TaskMetrics.ActiveRestorationSensor(this.threadId, Id, streamMetricsRegistry);
            restorationRecordsSendsor = TaskMetrics.RestorationRecordsSensor(this.threadId, Id, streamMetricsRegistry);
        }