public IEnumerable <T> Stream(CancellationToken cancellationToken)
{
var kafkaConsumerBuilder = CreateKafkaConsumerBuilder();
using (var kafkaConsumer = kafkaConsumerBuilder.Build())
{
this.kafkaConsumer = kafkaConsumer;
kafkaConsumer.Subscribe(topic);
while (!cancellationToken.IsCancellationRequested)
{
ConsumeResult <K, T> consumedResult;
try
{
consumedResult = kafkaConsumer.Consume(cancellationToken);
if (consumedResult?.IsPartitionEOF == true)
{
OnEOF?.Invoke();
}
}
catch (ConsumeException ex)
{
OnError?.Invoke(new StreamingError {
Reason = ex.Message, IsFatal = false
});
consumedResult = null;
}
catch (OperationCanceledException) {
consumedResult = null;
}
if (consumedResult?.Message != null)
{
yield return(consumedResult.Message.Value);
}
}
// Gracefully close the consumer, liberating the group offsets, etc.
kafkaConsumer.Close();
}
}
/// <summary>
/// Retrieves the stream of messages from the change tracking source.
/// </summary>
/// <param name="cancellationToken"></param>
/// <returns>IEnumerable of messages of type Change</returns>
public IEnumerable <Change> Stream(CancellationToken cancellationToken)
{
Stopwatch processTime = Stopwatch.StartNew();
IEnumerable <Change> changes = null;
TimeSpan delay = TimeSpan.FromSeconds(0);
bool alreadyInvokeEOF = false;
try
{
if (CommitEnable)
{
ContextHandler.InitContext();
}
else
{
ContextHandler.InitContext(InitialChangeTable == null ? 0 : (long)InitialChangeTable);
}
}
catch (Exception ex)
{
OnError?.Invoke(new StreamingError {
IsFatal = true, Reason = ex.Message
});
}
while (!cancellationToken.IsCancellationRequested)
{
using (var conn = Repository.CreateConnection())
{
try
{
changes = null;
ContextHandler.UpdateDatabaseOffset(Repository.GetDatabaseOffset(conn));
if (ContextHandler.HasChanges())
{
changes = GetChanges(conn, ContextHandler);
alreadyInvokeEOF = false;
}
else if (ContextHandler.isEOF() && !alreadyInvokeEOF)
{
OnEOF?.Invoke();
alreadyInvokeEOF = true;
}
}
catch (Exception cte)
{
OnError?.Invoke(new StreamingError {
IsFatal = cte is ChangeTrackingException, Reason = cte.Message
});
ContextHandler.RegisterError();
delay = delay.Add(TimeSpan.FromSeconds(5));
if (delay > TimeSpan.FromSeconds(15))
{
delay = TimeSpan.FromSeconds(15);
}
}
if (null != changes)
{
bool mustSync = false;
foreach (var change in changes)
{
if (change.ChangeOperation.Equals(ChangeOperationEnum.SYNC))
{
mustSync = true;
}
else
{
yield return(change);
}
}
delay = TimeSpan.FromSeconds(0);
ContextHandler.UpdateApplicationOffset();
if (mustSync && CommitEnable)
{
Commit();
}
}
}
processTime.Stop();
var sleep = (delay + PollingInterval) - processTime.Elapsed;
if (sleep.TotalMilliseconds > 0)
{
Thread.Sleep(sleep);
}
processTime.Restart();
}
ContextHandler.Dispose();
}