private Action <DeliveryReport <K, V> > BuildSendCallback <TResult>(TaskCompletionSource <TResult> completion, Action <DeliveryReport <K, V> > onSuccess)
{
return(report =>
{
if (!report.Error.IsError)
{
onSuccess(report);
}
else
{
Log.Error(report.Error.Reason);
var exception = new KafkaException(report.Error);
switch (_decider(exception))
{
case Directive.Stop:
if (_stage.CloseProducerOnStop)
{
// TODO: address the missing Producer.Close() in the future
Producer.Dispose();
}
_failStageCallback(exception);
break;
default:
completion.SetException(exception);
break;
}
}
if (AwaitingConfirmation.DecrementAndGet() == 0 && _inIsClosed)
{
_checkForCompletionCallback();
}
});
}
private void LogCommitFailure(KafkaException kafkaException, TimeSpan timeSpan, int retryCount)
{
//this.logger.WriteLog(new KafkaCommitFailLog(this.topicName,
// timeSpan.TotalSeconds,
// retryCount,
// kafkaException));
}
/// <summary>
/// MANIFOLD use
/// </summary>
public FetchResponseWrapper FetchAndGetDetail(string clientId, string topic, int correlationId, int partitionId, long fetchOffset, int fetchSize
, int maxWaitTime, int minWaitSize)
{
FetchResponse response = this.Fetch(clientId,
topic,
correlationId,
partitionId,
fetchOffset,
fetchSize,
maxWaitTime,
minWaitSize);
if (response == null)
{
throw new KafkaConsumeException(string.Format("FetchRequest returned null FetchResponse,fetchOffset={0},leader={1},topic={2},partition={3}",
fetchOffset, this.Config.Broker, topic, partitionId));
}
PartitionData partitionData = response.PartitionData(topic, partitionId);
if (partitionData == null)
{
throw new KafkaConsumeException(string.Format("PartitionData int FetchResponse is null,fetchOffset={0},leader={1},topic={2},partition={3}",
fetchOffset, this.Config.Broker, topic, partitionId));
}
if (partitionData.Error != ErrorMapping.NoError)
{
string s = string.Format("Partition data in FetchResponse has error. {0} {1} fetchOffset={2},leader={3},topic={4},partition={5}"
, partitionData.Error, KafkaException.GetMessage(partitionData.Error)
, fetchOffset, this.Config.Broker, topic, partitionId);
Logger.Error(s);
throw new KafkaConsumeException(s, partitionData.Error);
}
return(new FetchResponseWrapper(partitionData.GetMessageAndOffsets(), response.Size, response.CorrelationId, topic, partitionId));
}
static bool IsExceptionTransient(Exception exception)
{
return(exception switch
{
KafkaException bue => bue.Error.IsLocalError,
_ => true
});
private void OnProducerError(object sender, Error error)
{
Log.Error(error.Reason);
if (!KafkaExtensions.IsBrokerErrorRetriable(error) && !KafkaExtensions.IsLocalErrorRetriable(error))
{
var exception = new KafkaException(error);
FailStage(exception);
}
}
private void HandleProduceError(IProducer <K, V> producer, Error error)
{
Log.Error(error.Reason);
if (!KafkaExtensions.IsBrokerErrorRetriable(error) && !KafkaExtensions.IsLocalErrorRetriable(error))
{
var exception = new KafkaException(error);
FailStage(exception);
}
}
public void Logging_kafka_exception_has_message_information()
{
var ex = new KafkaException(new Error(ErrorCode.BrokerNotAvailable, "Nope"));
_sut.LogKafkaException(_eventId, ex);
var entry = _sut.LogEntries.Single(l => l.LogLevel == LogLevel.Error && l.EventId.Id == 1);
Assert.Equal(ErrorCode.BrokerNotAvailable, entry.GetValue("Code"));
Assert.Equal("Nope", entry.GetValue("Reason"));
}
private bool AutoRecoveryIfEnabled(KafkaException ex)
{
if (_enableAutoRecovery)
{
_logger.LogWarning(ex, "KafkaException occurred. The consumer will try to recover.");
ResetInnerConsumer();
}
else
{
_logger.LogCritical(ex, "Fatal error occurred consuming a message. The consumer will be stopped. " +
"Enable auto recovery to allow Silverback to automatically try to reconnect " +
"(EnableAutoRecovery=true in the endpoint configuration).");
}
return(_enableAutoRecovery);
}
public static void LogKafkaException(this ILogger logger, EventId eventId, KafkaException kex)
{
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}
var e = kex.Error;
logger.LogError(
eventId,
kex,
"ConsumeException. Code: {Code}, IsBrokerError {IsBrokerError}, IsError: {IsError}, IsLocalError: {IsLocalError}, Reason: {Reason}",
e.Code,
e.IsBrokerError,
e.IsError,
e.IsLocalError,
e.Reason);
}
private void ProcessConsumingError(ConsumeException ex)
{
var error = ex.Error;
_log.Error(error.Reason);
if (!KafkaExtensions.IsBrokerErrorRetriable(error) && !KafkaExtensions.IsLocalErrorRetriable(error))
{
var exception = new KafkaException(error);
ProcessError(exception);
}
else if (KafkaExtensions.IsLocalValueSerializationError(error))
{
var exception = new SerializationException(error.Reason);
ProcessError(exception);
}
else
{
ProcessError(ex);
}
}
internal bool AutoRecoveryIfEnabled(KafkaException ex, CancellationToken cancellationToken)
{
if (!Endpoint.Configuration.EnableAutoRecovery)
{
const string errorMessage = "Fatal error occurred consuming a message. The consumer will be stopped. " +
"Enable auto recovery to allow Silverback to automatically try to reconnect " +
"(EnableAutoRecovery=true in the endpoint configuration). (topic(s): {topics})";
_logger.LogCritical(
KafkaEventIds.KafkaExceptionNoAutoRecovery,
ex,
errorMessage,
(object)Endpoint.Names);
return(false);
}
try
{
if (cancellationToken.IsCancellationRequested)
{
return(false);
}
_logger.LogWarning(
KafkaEventIds.KafkaExceptionAutoRecovery,
ex,
"KafkaException occurred. The consumer will try to recover. (topic(s): {topics})",
(object)Endpoint.Names);
ResetConfluentConsumer(cancellationToken);
return(true);
}
catch (OperationCanceledException)
{
return(false);
}
}
private void HandleProduceError(IProducer <K, V> producer, Error error)
{
Log.Error(error.Reason);
if (!KafkaExtensions.IsBrokerErrorRetriable(error) && !KafkaExtensions.IsLocalErrorRetriable(error))
{
var exception = new KafkaException(error);
switch (_decider(exception))
{
case Directive.Stop:
if (_stage.CloseProducerOnStop)
{
producer.Dispose();
}
_failStageCallback(exception);
break;
default:
break;
}
}
}
public ProducerStageLogic(ProducerStage <K, V> stage, Attributes attributes) : base(stage.Shape)
{
_stage = stage;
var supervisionStrategy = attributes.GetAttribute <ActorAttributes.SupervisionStrategy>(null);
var decider = supervisionStrategy != null ? supervisionStrategy.Decider : Deciders.ResumingDecider;
SetHandler(_stage.In,
onPush: () =>
{
var msg = Grab(_stage.In);
var result = new TaskCompletionSource <DeliveryReport <K, V> >();
_producer.Produce(msg.TopicPartition, msg.Message, report =>
{
if (!report.Error.HasError)
{
result.SetResult(report);
}
else
{
var exception = new KafkaException(report.Error);
switch (decider(exception))
{
case Directive.Stop:
if (_stage.CloseProducerOnStop)
{
_producer.Dispose();
}
FailStage(exception);
break;
default:
result.SetException(exception);
break;
}
}
if (_awaitingConfirmation.DecrementAndGet() == 0 && _inIsClosed)
{
CheckForCompletion();
}
});
_awaitingConfirmation.IncrementAndGet();
Push(_stage.Out, result.Task);
},
onUpstreamFinish: () =>
{
_inIsClosed = true;
_completionState.SetResult(NotUsed.Instance);
CheckForCompletion();
},
onUpstreamFailure: exception =>
{
_inIsClosed = true;
_completionState.SetException(exception);
CheckForCompletion();
});
SetHandler(_stage.Out, onPull: () =>
{
TryPull(_stage.In);
});
}
/// <summary> /// Decider for all kafka exceptions raised by the underlying Kafka producer /// </summary> /// <param name="exception"><see cref="KafkaException"/> thrown when producer produced</param> /// <returns><see cref="Directive"/>></returns> protected virtual Directive OnKafkaException(KafkaException exception) => Directive.Stop;
