private int AddToTasks(IEnumerable <ConsumeResult <byte[], byte[]> > records) { int count = 0; foreach (var record in records) { count++; var task = manager.ActiveTaskFor(record.TopicPartition); if (task != null) { if (task.IsClosed) { log.LogInformation( "Stream task {TaskId} is already closed, probably because it got unexpectedly migrated to another thread already. Notifying the thread to trigger a new rebalance immediately", task.Id); // TODO gesture this behaviour //throw new TaskMigratedException(task); } else { task.AddRecord(record); } } else if (consumer.Assignment.Contains(record.TopicPartition)) { log.LogError( "Unable to locate active task for received-record partition {TopicPartition}. Current tasks: {TaskIDs}. Current Consumer Assignment : {Assignment}", record.TopicPartition, string.Join(",", manager.ActiveTaskIds), string.Join(",", consumer.Assignment.Select(t => $"{t.Topic}-[{t.Partition}]"))); throw new NullReferenceException($"Task was unexpectedly missing for partition {record.TopicPartition}"); } } return(count); }
public void Run() { Exception exception = null; if (IsRunning) { while (!token.IsCancellationRequested) { if (exception != null) { Close(false); if (ThrowException) { throw new StreamsException(exception); } else { IsRunning = false; break; } } try { IEnumerable <ConsumeResult <byte[], byte[]> > records = null; long now = DateTime.Now.GetMilliseconds(); if (State == ThreadState.PARTITIONS_ASSIGNED) { records = PollRequest(TimeSpan.Zero); } else if (State == ThreadState.PARTITIONS_REVOKED) { records = PollRequest(TimeSpan.Zero); } else if (State == ThreadState.RUNNING || State == ThreadState.STARTING) { records = PollRequest(consumeTimeout); } else { log.Error($"{logPrefix}Unexpected state {State} during normal iteration"); throw new StreamsException($"Unexpected state {State} during normal iteration"); } DateTime n = DateTime.Now; if (records != null && records.Count() > 0) { foreach (var record in records) { var task = manager.ActiveTaskFor(record.TopicPartition); if (task != null) { if (task.IsClosed) { log.Info($"Stream task {task.Id} is already closed, probably because it got unexpectedly migrated to another thread already. Notifying the thread to trigger a new rebalance immediately."); // TODO gesture this behaviour //throw new TaskMigratedException(task); } else { task.AddRecord(record); } } else { log.Error($"Unable to locate active task for received-record partition {record.TopicPartition}. Current tasks: {string.Join(",", manager.ActiveTaskIds)}"); throw new NullReferenceException($"Task was unexpectedly missing for partition {record.TopicPartition}"); } } log.Info($"Add {records.Count()} records in tasks in {DateTime.Now - n}"); } int processed = 0; long timeSinceLastPoll = 0; do { processed = 0; for (int i = 0; i < numIterations; ++i) { processed = manager.Process(now); if (processed == 0) { break; } // NOT AVAILABLE NOW, NEED PROCESSOR API //if (processed > 0) // manager.MaybeCommitPerUserRequested(); //else // break; } timeSinceLastPoll = Math.Max(DateTime.Now.GetMilliseconds() - lastPollMs, 0); if (MaybeCommit()) { numIterations = numIterations > 1 ? numIterations / 2 : numIterations; } else if (timeSinceLastPoll > streamConfig.MaxPollIntervalMs.Value / 2) { numIterations = numIterations > 1 ? numIterations / 2 : numIterations; break; } else if (processed > 0) { numIterations++; } } while (processed > 0); if (State == ThreadState.RUNNING) { MaybeCommit(); } if (State == ThreadState.PARTITIONS_ASSIGNED) { SetState(ThreadState.RUNNING); } if (records.Any()) { log.Info($"Processing {records.Count()} records in {DateTime.Now - n}"); } } catch (KafkaException e) { log.Error($"{logPrefix}Encountered the following unexpected Kafka exception during processing, tis usually indicate Streams internal errors:", e); exception = e; } catch (Exception e) { log.Error($"{logPrefix}Encountered the following error during processing:", e); exception = e; } } while (IsRunning) { // Use for waiting end of disposing Thread.Sleep(100); } // Dispose consumer try { consumer.Dispose(); } catch (Exception e) { log.Error($"{logPrefix}Failed to close consumer due to the following error:", e); } } }
public void Run() { Exception exception = null; if (IsRunning) { while (!token.IsCancellationRequested) { try { if (exception != null) { Close(false); throw exception; } ConsumeResult <byte[], byte[]> record = null; if (State == ThreadState.PARTITIONS_ASSIGNED) { record = PollRequest(TimeSpan.Zero); } else if (State == ThreadState.PARTITIONS_REVOKED) { record = PollRequest(TimeSpan.Zero); } else if (State == ThreadState.RUNNING || State == ThreadState.STARTING) { record = PollRequest(consumeTimeout); } else { log.Error($"{logPrefix}Unexpected state {State} during normal iteration"); throw new StreamsException($"Unexpected state {State} during normal iteration"); } if (record != null) { var task = manager.ActiveTaskFor(record.TopicPartition); if (task != null) { if (task.IsClosed) { log.Info($"Stream task {task.Id} is already closed, probably because it got unexpectedly migrated to another thread already. Notifying the thread to trigger a new rebalance immediately."); // TODO gesture this behaviour //throw new TaskMigratedException(task); } else { task.AddRecords(new List <ConsumeResult <byte[], byte[]> > { record }); } } else { log.Error($"Unable to locate active task for received-record partition {record.TopicPartition}. Current tasks: {string.Join(",", manager.ActiveTaskIds)}"); throw new NullReferenceException($"Task was unexpectedly missing for partition {record.TopicPartition}"); } } foreach (var t in manager.ActiveTasks) { if (t.CanProcess) { bool b = t.Process(); if (b && t.CommitNeeded) { t.Commit(); } } } if (State == ThreadState.RUNNING) { MaybeCommit(); } if (State == ThreadState.PARTITIONS_ASSIGNED) { SetState(ThreadState.RUNNING); } } catch (KafkaException e) { log.Error($"{logPrefix}Encountered the following unexpected Kafka exception during processing, tis usually indicate Streams internal errors:", e); exception = e; } catch (Exception e) { log.Error($"{logPrefix}Encountered the following error during processing:", e); exception = e; } } while (IsRunning) { // Use for waiting end of disposing Thread.Sleep(100); } // Dispose consumer try { consumer.Dispose(); } catch (Exception e) { log.Error($"{logPrefix}Failed to close consumer due to the following error:", e); } } }
public void Run() { Exception exception = null; if (IsRunning) { while (!token.IsCancellationRequested) { try { if (exception != null) { Close(false); throw exception; } ConsumeResult <byte[], byte[]> record = null; if (State == ThreadState.PARTITIONS_ASSIGNED) { record = PollRequest(TimeSpan.Zero); } else if (State == ThreadState.PARTITIONS_REVOKED) { record = PollRequest(TimeSpan.Zero); } else if (State == ThreadState.RUNNING || State == ThreadState.STARTING) { record = PollRequest(consumeTimeout); } else { log.Error($"{logPrefix}Unexpected state {State} during normal iteration"); throw new StreamsException($"Unexpected state {State} during normal iteration"); } if (record != null) { var task = manager.ActiveTaskFor(record.TopicPartition); if (task != null) { task.AddRecords(new List <ConsumeResult <byte[], byte[]> > { record }); } } foreach (var t in manager.ActiveTasks) { if (t.CanProcess) { bool b = t.Process(); if (b && t.CommitNeeded) { t.Commit(); } } } if (State == ThreadState.RUNNING) { MaybeCommit(); } if (State == ThreadState.PARTITIONS_ASSIGNED) { SetState(ThreadState.RUNNING); } } catch (KafkaException e) { log.Error($"{logPrefix}Encountered the following unexpected Kafka exception during processing, tis usually indicate Streams internal errors:", e); exception = e; } catch (Exception e) { log.Error($"{logPrefix}Encountered the following error during processing:", e); exception = e; } } while (IsRunning) { // Use for waiting end of disposing Thread.Sleep(100); } // Dispose consumer try { consumer.Dispose(); } catch (Exception e) { log.Error($"{logPrefix}Failed to close consumer due to the following error:", e); } } }
public void StandardWorkflowTaskManager() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var topology = builder.Build(); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); var taskCreator = new TaskCreator(topology.Builder, config, "thread-0", supplier, producer); var taskManager = new TaskManager(topology.Builder, taskCreator, supplier.GetAdmin(config.ToAdminConfig("admin")), consumer); taskManager.CreateTasks( new List <TopicPartition> { new TopicPartition("topic", 0), new TopicPartition("topic", 1), new TopicPartition("topic", 2), new TopicPartition("topic", 3), }); Assert.AreEqual(4, taskManager.ActiveTasks.Count()); for (int i = 0; i < 4; ++i) { var task = taskManager.ActiveTaskFor(new TopicPartition("topic", i)); Assert.IsNotNull(task); Assert.AreEqual("test-app", task.ApplicationId); Assert.IsFalse(task.CanProcess(DateTime.Now.GetMilliseconds())); Assert.IsFalse(task.CommitNeeded); Assert.IsFalse(task.HasStateStores); } // Revoked 2 partitions taskManager.RevokeTasks(new List <TopicPartition> { new TopicPartition("topic", 2), new TopicPartition("topic", 3), }); Assert.AreEqual(2, taskManager.ActiveTasks.Count()); Assert.AreEqual(2, taskManager.RevokedTasks.Count()); for (int i = 0; i < 2; ++i) { var task = taskManager.ActiveTaskFor(new TopicPartition("topic", i)); Assert.IsNotNull(task); Assert.AreEqual("test-app", task.ApplicationId); Assert.IsFalse(task.CanProcess(DateTime.Now.GetMilliseconds())); Assert.IsFalse(task.CommitNeeded); Assert.IsFalse(task.HasStateStores); } var taskFailed = taskManager.ActiveTaskFor(new TopicPartition("topic", 2)); Assert.IsNull(taskFailed); taskManager.Close(); Assert.AreEqual(0, taskManager.ActiveTasks.Count()); Assert.AreEqual(0, taskManager.RevokedTasks.Count()); }
public void TaskManagerCommit() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; var serdes = new StringSerDes(); var builder = new StreamBuilder(); builder.Stream <string, string>("topic") .Map((k, v) => KeyValuePair.Create(k.ToUpper(), v.ToUpper())) .To("topic2"); var topology = builder.Build(); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); var taskCreator = new TaskCreator(topology.Builder, config, "thread-0", supplier, producer); var taskManager = new TaskManager(topology.Builder, taskCreator, supplier.GetAdmin(config.ToAdminConfig("admin")), consumer); taskManager.CreateTasks( new List <TopicPartition> { new TopicPartition("topic", 0), new TopicPartition("topic", 1), new TopicPartition("topic", 2), new TopicPartition("topic", 3), }); Assert.AreEqual(4, taskManager.ActiveTasks.Count()); var part = new TopicPartition("topic", 0); var task = taskManager.ActiveTaskFor(part); List <ConsumeResult <byte[], byte[]> > messages = new List <ConsumeResult <byte[], byte[]> >(); int offset = 0; for (int i = 0; i < 5; ++i) { messages.Add( new ConsumeResult <byte[], byte[]> { Message = new Message <byte[], byte[]> { Key = serdes.Serialize($"key{i + 1}", new SerializationContext()), Value = serdes.Serialize($"value{i + 1}", new SerializationContext()) }, TopicPartitionOffset = new TopicPartitionOffset(part, offset++) }); } task.AddRecords(messages); Assert.IsTrue(task.CanProcess(DateTime.Now.GetMilliseconds())); while (task.CanProcess(DateTime.Now.GetMilliseconds())) { Assert.IsTrue(task.Process()); } // ONLY ONE TASK HAVE BEEN RECORDS Assert.AreEqual(1, taskManager.CommitAll()); // CHECK IN TOPIC topic2 consumer.Subscribe("topic2"); List <ConsumeResult <byte[], byte[]> > results = new List <ConsumeResult <byte[], byte[]> >(); ConsumeResult <byte[], byte[]> result = null; do { result = consumer.Consume(100); if (result != null) { results.Add(result); consumer.Commit(result); } } while (result != null); Assert.AreEqual(5, results.Count); for (int i = 0; i < 5; ++i) { Assert.AreEqual($"KEY{i + 1}", serdes.Deserialize(results[i].Message.Key, new SerializationContext())); Assert.AreEqual($"VALUE{i+1}", serdes.Deserialize(results[i].Message.Value, new SerializationContext())); } // NO RECORD IN THIS TASKS part = new TopicPartition("topic", 2); task = taskManager.ActiveTaskFor(part); Assert.IsFalse(task.CanProcess(DateTime.Now.GetMilliseconds())); Assert.IsFalse(task.Process()); taskManager.Close(); }
private void TaskManagerRestorationChangelog(bool persistenStateStore = false) { var stateDir = Path.Combine(".", Guid.NewGuid().ToString()); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-restoration-changelog-app"; config.StateDir = stateDir; var builder = new StreamBuilder(); builder.Table("topic", persistenStateStore ? RocksDb <string, string> .As("store").WithLoggingEnabled(null) : InMemory <string, string> .As("store").WithLoggingEnabled(null)); var serdes = new StringSerDes(); var topology = builder.Build(); topology.Builder.RewriteTopology(config); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); var restoreConsumer = supplier.GetRestoreConsumer(config.ToConsumerConfig()); var storeChangelogReader = new StoreChangelogReader(config, restoreConsumer, "thread-0", new StreamMetricsRegistry()); var taskCreator = new TaskCreator(topology.Builder, config, "thread-0", supplier, producer, storeChangelogReader, new StreamMetricsRegistry()); var taskManager = new TaskManager(topology.Builder, taskCreator, supplier.GetAdmin(config.ToAdminConfig("admin")), consumer, storeChangelogReader); var part = new TopicPartition("topic", 0); taskManager.CreateTasks( new List <TopicPartition> { part }); var task = taskManager.ActiveTaskFor(part); IDictionary <TaskId, ITask> tasks = new Dictionary <TaskId, ITask>(); tasks.Add(task.Id, task); taskManager.TryToCompleteRestoration(); storeChangelogReader.Restore(); Assert.IsTrue(taskManager.TryToCompleteRestoration()); List <ConsumeResult <byte[], byte[]> > messages = new List <ConsumeResult <byte[], byte[]> >(); int offset = 0; for (int i = 0; i < 5; ++i) { messages.Add( new ConsumeResult <byte[], byte[]> { Message = new Message <byte[], byte[]> { Key = serdes.Serialize($"key{i + 1}", new SerializationContext()), Value = serdes.Serialize($"value{i + 1}", new SerializationContext()) }, TopicPartitionOffset = new TopicPartitionOffset(part, offset++) }); } task.AddRecords(messages); // Process messages while (task.CanProcess(DateTime.Now.GetMilliseconds())) { Assert.IsTrue(task.Process()); } taskManager.CommitAll(); // Simulate Close + new open taskManager.Close(); restoreConsumer.Resume(new TopicPartition("test-restoration-changelog-app-store-changelog", 0).ToSingle()); taskManager.CreateTasks( new List <TopicPartition> { part }); task = taskManager.ActiveTaskFor(part); tasks = new Dictionary <TaskId, ITask>(); tasks.Add(task.Id, task); Assert.IsFalse(taskManager.TryToCompleteRestoration()); storeChangelogReader.Restore(); Assert.IsTrue(taskManager.TryToCompleteRestoration()); var store = task.GetStore("store"); var items = (store as ITimestampedKeyValueStore <string, string>).All().ToList(); Assert.AreEqual(5, items.Count); taskManager.Close(); if (persistenStateStore) { Directory.Delete(stateDir, true); } }