private int AddToTasks(IEnumerable <ConsumeResult <byte[], byte[]> > records)
{
int count = 0;
foreach (var record in records)
{
count++;
var task = manager.ActiveTaskFor(record.TopicPartition);
if (task != null)
{
if (task.IsClosed)
{
log.LogInformation(
"Stream task {TaskId} is already closed, probably because it got unexpectedly migrated to another thread already. Notifying the thread to trigger a new rebalance immediately",
task.Id);
// TODO gesture this behaviour
//throw new TaskMigratedException(task);
}
else
{
task.AddRecord(record);
}
}
else if (consumer.Assignment.Contains(record.TopicPartition))
{
log.LogError(
"Unable to locate active task for received-record partition {TopicPartition}. Current tasks: {TaskIDs}. Current Consumer Assignment : {Assignment}",
record.TopicPartition, string.Join(",", manager.ActiveTaskIds),
string.Join(",", consumer.Assignment.Select(t => $"{t.Topic}-[{t.Partition}]")));
throw new NullReferenceException($"Task was unexpectedly missing for partition {record.TopicPartition}");
}
}
return(count);
}
public void Run()
{
Exception exception = null;
if (IsRunning)
{
while (!token.IsCancellationRequested)
{
if (exception != null)
{
Close(false);
if (ThrowException)
{
throw new StreamsException(exception);
}
else
{
IsRunning = false;
break;
}
}
try
{
IEnumerable <ConsumeResult <byte[], byte[]> > records = null;
long now = DateTime.Now.GetMilliseconds();
if (State == ThreadState.PARTITIONS_ASSIGNED)
{
records = PollRequest(TimeSpan.Zero);
}
else if (State == ThreadState.PARTITIONS_REVOKED)
{
records = PollRequest(TimeSpan.Zero);
}
else if (State == ThreadState.RUNNING || State == ThreadState.STARTING)
{
records = PollRequest(consumeTimeout);
}
else
{
log.Error($"{logPrefix}Unexpected state {State} during normal iteration");
throw new StreamsException($"Unexpected state {State} during normal iteration");
}
DateTime n = DateTime.Now;
if (records != null && records.Count() > 0)
{
foreach (var record in records)
{
var task = manager.ActiveTaskFor(record.TopicPartition);
if (task != null)
{
if (task.IsClosed)
{
log.Info($"Stream task {task.Id} is already closed, probably because it got unexpectedly migrated to another thread already. Notifying the thread to trigger a new rebalance immediately.");
// TODO gesture this behaviour
//throw new TaskMigratedException(task);
}
else
{
task.AddRecord(record);
}
}
else
{
log.Error($"Unable to locate active task for received-record partition {record.TopicPartition}. Current tasks: {string.Join(",", manager.ActiveTaskIds)}");
throw new NullReferenceException($"Task was unexpectedly missing for partition {record.TopicPartition}");
}
}
log.Info($"Add {records.Count()} records in tasks in {DateTime.Now - n}");
}
int processed = 0;
long timeSinceLastPoll = 0;
do
{
processed = 0;
for (int i = 0; i < numIterations; ++i)
{
processed = manager.Process(now);
if (processed == 0)
{
break;
}
// NOT AVAILABLE NOW, NEED PROCESSOR API
//if (processed > 0)
// manager.MaybeCommitPerUserRequested();
//else
// break;
}
timeSinceLastPoll = Math.Max(DateTime.Now.GetMilliseconds() - lastPollMs, 0);
if (MaybeCommit())
{
numIterations = numIterations > 1 ? numIterations / 2 : numIterations;
}
else if (timeSinceLastPoll > streamConfig.MaxPollIntervalMs.Value / 2)
{
numIterations = numIterations > 1 ? numIterations / 2 : numIterations;
break;
}
else if (processed > 0)
{
numIterations++;
}
} while (processed > 0);
if (State == ThreadState.RUNNING)
{
MaybeCommit();
}
if (State == ThreadState.PARTITIONS_ASSIGNED)
{
SetState(ThreadState.RUNNING);
}
if (records.Any())
{
log.Info($"Processing {records.Count()} records in {DateTime.Now - n}");
}
}
catch (KafkaException e)
{
log.Error($"{logPrefix}Encountered the following unexpected Kafka exception during processing, tis usually indicate Streams internal errors:", e);
exception = e;
}
catch (Exception e)
{
log.Error($"{logPrefix}Encountered the following error during processing:", e);
exception = e;
}
}
while (IsRunning)
{
// Use for waiting end of disposing
Thread.Sleep(100);
}
// Dispose consumer
try
{
consumer.Dispose();
}
catch (Exception e)
{
log.Error($"{logPrefix}Failed to close consumer due to the following error:", e);
}
}
}
public void Run()
{
Exception exception = null;
if (IsRunning)
{
while (!token.IsCancellationRequested)
{
try
{
if (exception != null)
{
Close(false);
throw exception;
}
ConsumeResult <byte[], byte[]> record = null;
if (State == ThreadState.PARTITIONS_ASSIGNED)
{
record = PollRequest(TimeSpan.Zero);
}
else if (State == ThreadState.PARTITIONS_REVOKED)
{
record = PollRequest(TimeSpan.Zero);
}
else if (State == ThreadState.RUNNING || State == ThreadState.STARTING)
{
record = PollRequest(consumeTimeout);
}
else
{
log.Error($"{logPrefix}Unexpected state {State} during normal iteration");
throw new StreamsException($"Unexpected state {State} during normal iteration");
}
if (record != null)
{
var task = manager.ActiveTaskFor(record.TopicPartition);
if (task != null)
{
if (task.IsClosed)
{
log.Info($"Stream task {task.Id} is already closed, probably because it got unexpectedly migrated to another thread already. Notifying the thread to trigger a new rebalance immediately.");
// TODO gesture this behaviour
//throw new TaskMigratedException(task);
}
else
{
task.AddRecords(new List <ConsumeResult <byte[], byte[]> > {
record
});
}
}
else
{
log.Error($"Unable to locate active task for received-record partition {record.TopicPartition}. Current tasks: {string.Join(",", manager.ActiveTaskIds)}");
throw new NullReferenceException($"Task was unexpectedly missing for partition {record.TopicPartition}");
}
}
foreach (var t in manager.ActiveTasks)
{
if (t.CanProcess)
{
bool b = t.Process();
if (b && t.CommitNeeded)
{
t.Commit();
}
}
}
if (State == ThreadState.RUNNING)
{
MaybeCommit();
}
if (State == ThreadState.PARTITIONS_ASSIGNED)
{
SetState(ThreadState.RUNNING);
}
}
catch (KafkaException e)
{
log.Error($"{logPrefix}Encountered the following unexpected Kafka exception during processing, tis usually indicate Streams internal errors:", e);
exception = e;
}
catch (Exception e)
{
log.Error($"{logPrefix}Encountered the following error during processing:", e);
exception = e;
}
}
while (IsRunning)
{
// Use for waiting end of disposing
Thread.Sleep(100);
}
// Dispose consumer
try
{
consumer.Dispose();
}
catch (Exception e)
{
log.Error($"{logPrefix}Failed to close consumer due to the following error:", e);
}
}
}
public void Run()
{
Exception exception = null;
if (IsRunning)
{
while (!token.IsCancellationRequested)
{
try
{
if (exception != null)
{
Close(false);
throw exception;
}
ConsumeResult <byte[], byte[]> record = null;
if (State == ThreadState.PARTITIONS_ASSIGNED)
{
record = PollRequest(TimeSpan.Zero);
}
else if (State == ThreadState.PARTITIONS_REVOKED)
{
record = PollRequest(TimeSpan.Zero);
}
else if (State == ThreadState.RUNNING || State == ThreadState.STARTING)
{
record = PollRequest(consumeTimeout);
}
else
{
log.Error($"{logPrefix}Unexpected state {State} during normal iteration");
throw new StreamsException($"Unexpected state {State} during normal iteration");
}
if (record != null)
{
var task = manager.ActiveTaskFor(record.TopicPartition);
if (task != null)
{
task.AddRecords(new List <ConsumeResult <byte[], byte[]> > {
record
});
}
}
foreach (var t in manager.ActiveTasks)
{
if (t.CanProcess)
{
bool b = t.Process();
if (b && t.CommitNeeded)
{
t.Commit();
}
}
}
if (State == ThreadState.RUNNING)
{
MaybeCommit();
}
if (State == ThreadState.PARTITIONS_ASSIGNED)
{
SetState(ThreadState.RUNNING);
}
}
catch (KafkaException e)
{
log.Error($"{logPrefix}Encountered the following unexpected Kafka exception during processing, tis usually indicate Streams internal errors:", e);
exception = e;
}
catch (Exception e)
{
log.Error($"{logPrefix}Encountered the following error during processing:", e);
exception = e;
}
}
while (IsRunning)
{
// Use for waiting end of disposing
Thread.Sleep(100);
}
// Dispose consumer
try
{
consumer.Dispose();
}
catch (Exception e)
{
log.Error($"{logPrefix}Failed to close consumer due to the following error:", e);
}
}
}
public void StandardWorkflowTaskManager()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-app";
var builder = new StreamBuilder();
builder.Stream <string, string>("topic").To("topic2");
var topology = builder.Build();
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(config.ToProducerConfig());
var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null);
var taskCreator = new TaskCreator(topology.Builder, config, "thread-0", supplier, producer);
var taskManager = new TaskManager(topology.Builder, taskCreator, supplier.GetAdmin(config.ToAdminConfig("admin")), consumer);
taskManager.CreateTasks(
new List <TopicPartition>
{
new TopicPartition("topic", 0),
new TopicPartition("topic", 1),
new TopicPartition("topic", 2),
new TopicPartition("topic", 3),
});
Assert.AreEqual(4, taskManager.ActiveTasks.Count());
for (int i = 0; i < 4; ++i)
{
var task = taskManager.ActiveTaskFor(new TopicPartition("topic", i));
Assert.IsNotNull(task);
Assert.AreEqual("test-app", task.ApplicationId);
Assert.IsFalse(task.CanProcess(DateTime.Now.GetMilliseconds()));
Assert.IsFalse(task.CommitNeeded);
Assert.IsFalse(task.HasStateStores);
}
// Revoked 2 partitions
taskManager.RevokeTasks(new List <TopicPartition> {
new TopicPartition("topic", 2),
new TopicPartition("topic", 3),
});
Assert.AreEqual(2, taskManager.ActiveTasks.Count());
Assert.AreEqual(2, taskManager.RevokedTasks.Count());
for (int i = 0; i < 2; ++i)
{
var task = taskManager.ActiveTaskFor(new TopicPartition("topic", i));
Assert.IsNotNull(task);
Assert.AreEqual("test-app", task.ApplicationId);
Assert.IsFalse(task.CanProcess(DateTime.Now.GetMilliseconds()));
Assert.IsFalse(task.CommitNeeded);
Assert.IsFalse(task.HasStateStores);
}
var taskFailed = taskManager.ActiveTaskFor(new TopicPartition("topic", 2));
Assert.IsNull(taskFailed);
taskManager.Close();
Assert.AreEqual(0, taskManager.ActiveTasks.Count());
Assert.AreEqual(0, taskManager.RevokedTasks.Count());
}
public void TaskManagerCommit()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-app";
var serdes = new StringSerDes();
var builder = new StreamBuilder();
builder.Stream <string, string>("topic")
.Map((k, v) => KeyValuePair.Create(k.ToUpper(), v.ToUpper()))
.To("topic2");
var topology = builder.Build();
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(config.ToProducerConfig());
var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null);
var taskCreator = new TaskCreator(topology.Builder, config, "thread-0", supplier, producer);
var taskManager = new TaskManager(topology.Builder, taskCreator, supplier.GetAdmin(config.ToAdminConfig("admin")), consumer);
taskManager.CreateTasks(
new List <TopicPartition>
{
new TopicPartition("topic", 0),
new TopicPartition("topic", 1),
new TopicPartition("topic", 2),
new TopicPartition("topic", 3),
});
Assert.AreEqual(4, taskManager.ActiveTasks.Count());
var part = new TopicPartition("topic", 0);
var task = taskManager.ActiveTaskFor(part);
List <ConsumeResult <byte[], byte[]> > messages = new List <ConsumeResult <byte[], byte[]> >();
int offset = 0;
for (int i = 0; i < 5; ++i)
{
messages.Add(
new ConsumeResult <byte[], byte[]>
{
Message = new Message <byte[], byte[]>
{
Key = serdes.Serialize($"key{i + 1}", new SerializationContext()),
Value = serdes.Serialize($"value{i + 1}", new SerializationContext())
},
TopicPartitionOffset = new TopicPartitionOffset(part, offset++)
});
}
task.AddRecords(messages);
Assert.IsTrue(task.CanProcess(DateTime.Now.GetMilliseconds()));
while (task.CanProcess(DateTime.Now.GetMilliseconds()))
{
Assert.IsTrue(task.Process());
}
// ONLY ONE TASK HAVE BEEN RECORDS
Assert.AreEqual(1, taskManager.CommitAll());
// CHECK IN TOPIC topic2
consumer.Subscribe("topic2");
List <ConsumeResult <byte[], byte[]> > results = new List <ConsumeResult <byte[], byte[]> >();
ConsumeResult <byte[], byte[]> result = null;
do
{
result = consumer.Consume(100);
if (result != null)
{
results.Add(result);
consumer.Commit(result);
}
} while (result != null);
Assert.AreEqual(5, results.Count);
for (int i = 0; i < 5; ++i)
{
Assert.AreEqual($"KEY{i + 1}", serdes.Deserialize(results[i].Message.Key, new SerializationContext()));
Assert.AreEqual($"VALUE{i+1}", serdes.Deserialize(results[i].Message.Value, new SerializationContext()));
}
// NO RECORD IN THIS TASKS
part = new TopicPartition("topic", 2);
task = taskManager.ActiveTaskFor(part);
Assert.IsFalse(task.CanProcess(DateTime.Now.GetMilliseconds()));
Assert.IsFalse(task.Process());
taskManager.Close();
}
private void TaskManagerRestorationChangelog(bool persistenStateStore = false)
{
var stateDir = Path.Combine(".", Guid.NewGuid().ToString());
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-restoration-changelog-app";
config.StateDir = stateDir;
var builder = new StreamBuilder();
builder.Table("topic",
persistenStateStore
? RocksDb <string, string> .As("store").WithLoggingEnabled(null)
: InMemory <string, string> .As("store").WithLoggingEnabled(null));
var serdes = new StringSerDes();
var topology = builder.Build();
topology.Builder.RewriteTopology(config);
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(config.ToProducerConfig());
var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null);
var restoreConsumer = supplier.GetRestoreConsumer(config.ToConsumerConfig());
var storeChangelogReader =
new StoreChangelogReader(config, restoreConsumer, "thread-0", new StreamMetricsRegistry());
var taskCreator = new TaskCreator(topology.Builder, config, "thread-0", supplier, producer,
storeChangelogReader, new StreamMetricsRegistry());
var taskManager = new TaskManager(topology.Builder, taskCreator,
supplier.GetAdmin(config.ToAdminConfig("admin")), consumer, storeChangelogReader);
var part = new TopicPartition("topic", 0);
taskManager.CreateTasks(
new List <TopicPartition>
{
part
});
var task = taskManager.ActiveTaskFor(part);
IDictionary <TaskId, ITask> tasks = new Dictionary <TaskId, ITask>();
tasks.Add(task.Id, task);
taskManager.TryToCompleteRestoration();
storeChangelogReader.Restore();
Assert.IsTrue(taskManager.TryToCompleteRestoration());
List <ConsumeResult <byte[], byte[]> > messages = new List <ConsumeResult <byte[], byte[]> >();
int offset = 0;
for (int i = 0; i < 5; ++i)
{
messages.Add(
new ConsumeResult <byte[], byte[]>
{
Message = new Message <byte[], byte[]>
{
Key = serdes.Serialize($"key{i + 1}", new SerializationContext()),
Value = serdes.Serialize($"value{i + 1}", new SerializationContext())
},
TopicPartitionOffset = new TopicPartitionOffset(part, offset++)
});
}
task.AddRecords(messages);
// Process messages
while (task.CanProcess(DateTime.Now.GetMilliseconds()))
{
Assert.IsTrue(task.Process());
}
taskManager.CommitAll();
// Simulate Close + new open
taskManager.Close();
restoreConsumer.Resume(new TopicPartition("test-restoration-changelog-app-store-changelog", 0).ToSingle());
taskManager.CreateTasks(
new List <TopicPartition>
{
part
});
task = taskManager.ActiveTaskFor(part);
tasks = new Dictionary <TaskId, ITask>();
tasks.Add(task.Id, task);
Assert.IsFalse(taskManager.TryToCompleteRestoration());
storeChangelogReader.Restore();
Assert.IsTrue(taskManager.TryToCompleteRestoration());
var store = task.GetStore("store");
var items = (store as ITimestampedKeyValueStore <string, string>).All().ToList();
Assert.AreEqual(5, items.Count);
taskManager.Close();
if (persistenStateStore)
{
Directory.Delete(stateDir, true);
}
}
