/// <exception cref="Exception">A delegate callback throws an exception.</exception>
/// <exception cref="NullReferenceException">The address of <paramref name="location" /> is a null pointer. </exception>
public async Task <List <T> > SendAsync <T>(IKafkaRequest <T> request)
{
T result;
if (typeof(T) == typeof(ProduceResponse))
{
Interlocked.Increment(ref ProduceRequestCallCount);
result = (T)((object)await ProduceResponseFunction());
}
else if (typeof(T) == typeof(MetadataResponse))
{
Interlocked.Increment(ref MetadataRequestCallCount);
result = (T)(object) await MetadataResponseFunction();
}
else if (typeof(T) == typeof(OffsetResponse))
{
Interlocked.Increment(ref OffsetRequestCallCount);
result = (T)(object) await OffsetResponseFunction();
}
else if (typeof(T) == typeof(FetchResponse))
{
Interlocked.Increment(ref FetchRequestCallCount);
result = (T)(object) await FetchResponseFunction();
}
else
{
throw new Exception("no found implementation");
}
var resultlist = new List <T>();
resultlist.Add(result);
return(resultlist);
}
/// <summary>
/// Send kafka payload to server and receive a task event when response is received.
/// </summary>
/// <typeparam name="T">A Kafka response object return by decode function.</typeparam>
/// <param name="request">The IKafkaRequest to send to the kafka servers.</param>
/// <returns></returns>
public async Task <List <T> > SendAsync <T>(IKafkaRequest <T> request)
{
//assign unique correlationId
request.CorrelationId = NextCorrelationId();
//if response is expected, register a receive data task and send request
if (request.ExpectResponse)
{
var asyncRequest = new AsyncRequestItem(request.CorrelationId);
try
{
AddAsyncRequestItemToResponseQueue(asyncRequest);
await SendAsync(request.Encode()).ConfigureAwait(false);
}
catch (OperationCanceledException ex)
{
TriggerMessageTimeout(asyncRequest);
}
var response = await asyncRequest.ReceiveTask.Task.ConfigureAwait(false);
return(request.Decode(response).ToList());
}
//no response needed, just send
await SendAsync(request.Encode()).ConfigureAwait(false);
//TODO should this return a response of success for request?
return(new List <T>());
}
/// <summary>
/// Send kafka payload to server and receive a task event when response is received.
/// </summary>
/// <typeparam name="T">A Kafka response object return by decode function.</typeparam>
/// <param name="request">The IKafkaRequest to send to the kafka servers.</param>
/// <returns></returns>
public Task <List <T> > SendAsync <T>(IKafkaRequest <T> request)
{
//assign unique correlationId
request.CorrelationId = NextCorrelationId();
var tcs = new TaskCompletionSource <List <T> >();
var asynRequest = new AsyncRequestItem(request.CorrelationId);
asynRequest.ReceiveTask.Task.ContinueWith(data =>
{
try
{
var response = request.Decode(data.Result);
tcs.SetResult(response.ToList());
//TODO should we check for errors type here and throw?
}
catch (Exception ex)
{
tcs.SetException(ex);
}
});
if (_requestIndex.TryAdd(request.CorrelationId, asynRequest) == false)
{
throw new ApplicationException("Failed to register request for async response.");
}
SendAsync(request.Encode());
return(tcs.Task);
}
/// <summary>
/// Send kafka payload to server and receive a task event when response is received.
/// </summary>
/// <typeparam name="T">A Kafka response object return by decode function.</typeparam>
/// <param name="request">The IKafkaRequest to send to the kafka servers.</param>
/// <returns></returns>
public async Task <List <T> > SendAsync <T>(IKafkaRequest <T> request)
{
//assign unique correlationId
request.CorrelationId = NextCorrelationId();
//if response is expected, register a receive data task and send request
if (request.ExpectResponse)
{
var asyncRequest = new AsyncRequestItem(request.CorrelationId);
if (_requestIndex.TryAdd(request.CorrelationId, asyncRequest) == false)
{
throw new ApplicationException("Failed to register request for async response.");
}
SendAsync(request.Encode());
var response = await asyncRequest.ReceiveTask.Task.ConfigureAwait(false);
return(request.Decode(response).ToList());
}
//no response needed, just send
await SendAsync(request.Encode()).ConfigureAwait(false);
//TODO should this return a response of success for request?
return(new List <T>());
}
/// <summary>
/// Encode the common head for kafka request.
/// </summary>
/// <returns>KafkaMessagePacker with header populated</returns>
/// <remarks>Format: (hhihs) </remarks>
public static KafkaMessagePacker EncodeHeader <T>(IKafkaRequest <T> request)
{
return(new KafkaMessagePacker()
.Pack(((Int16)request.ApiKey))
.Pack(ApiVersion)
.Pack(request.CorrelationId)
.Pack(request.ClientId, StringPrefixEncoding.Int16));
}
/// <summary>
/// Encode the common head for kafka request.
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
/// <remarks>Format: (hhihs) </remarks>
public static byte[] EncodeHeader <T>(IKafkaRequest <T> request)
{
var message = new WriteByteStream();
message.Pack(((Int16)request.ApiKey).ToBytes(),
ApiVersion.ToBytes(),
request.CorrelationId.ToBytes(),
request.ClientId.ToInt16SizedBytes());
return(message.Payload());
}
/// <summary>
/// Send kafka payload to server and receive a task event when response is received.
/// </summary>
/// <typeparam name="T">A Kafka response object return by decode function.</typeparam>
/// <param name="request">The IKafkaRequest to send to the kafka servers.</param>
/// <returns></returns>
public async Task <List <T> > SendAsync <T>(IKafkaRequest <T> request)
{
//assign unique correlationId
request.CorrelationId = NextCorrelationId();
var asyncRequest = new AsyncRequestItem(request.CorrelationId);
if (_requestIndex.TryAdd(request.CorrelationId, asyncRequest) == false)
{
throw new ApplicationException("Failed to register request for async response.");
}
await SendAsync(request.Encode());
var response = await asyncRequest.ReceiveTask.Task;
return(request.Decode(response).ToList());
}
/// <summary>
/// Send kafka payload to server and receive a task event when response is received.
/// </summary>
/// <typeparam name="T">A Kafka response object return by decode function.</typeparam>
/// <param name="request">The IKafkaRequest to send to the kafka servers.</param>
/// <returns></returns>
public async Task <List <T> > SendAsync <T>(IKafkaRequest <T> request)
{
//assign unique correlationId
request.CorrelationId = NextCorrelationId();
_log.DebugFormat("Entered SendAsync for CorrelationId:{0} Connection:{1} ", request.CorrelationId, Endpoint);
//if response is expected, register a receive data task and send request
if (request.ExpectResponse)
{
using (var asyncRequest = new AsyncRequestItem(request.CorrelationId))
{
try
{
AddAsyncRequestItemToResponseQueue(asyncRequest);
ExceptionDispatchInfo exceptionDispatchInfo = null;
try
{
await _client.WriteAsync(request.Encode()).ConfigureAwait(false);
}
catch (Exception ex)
{
exceptionDispatchInfo = ExceptionDispatchInfo.Capture(ex);
}
asyncRequest.MarkRequestAsSent(exceptionDispatchInfo, _responseTimeoutMs, TriggerMessageTimeout);
}
catch (OperationCanceledException)
{
TriggerMessageTimeout(asyncRequest);
}
var response = await asyncRequest.ReceiveTask.Task.ConfigureAwait(false);
return(request.Decode(response).ToList());
}
}
//no response needed, just send
await _client.WriteAsync(request.Encode()).ConfigureAwait(false);
//TODO should this return a response of success for request?
return(new List <T>());
}
public Task <List <T> > SendAsync <T>(IKafkaRequest <T> request)
{
//start a thread to handle the request and return
var task = new Task <List <T> >(() =>
{
if (typeof(T) == typeof(ProduceResponse))
{
ProduceRequestCallCount++;
return(new List <T> {
(T)(object)ProduceResponseFunction()
});
}
else if (typeof(T) == typeof(MetadataResponse))
{
MetadataRequestCallCount++;
return(new List <T> {
(T)(object)MetadataResponseFunction()
});
}
else if (typeof(T) == typeof(OffsetResponse))
{
OffsetRequestCallCount++;
return(new List <T> {
(T)(object)OffsetResponseFunction()
});
}
else if (typeof(T) == typeof(FetchResponse))
{
FetchRequestCallCount++;
return(new List <T> {
(T)(object)FetchResponseFunction()
});
}
return(null);
});
task.Start();
return(task);
}
/// <summary>
/// Send kafka payload to server and receive a task event when response is received.
/// </summary>
/// <typeparam name="T">A Kafka response object return by decode function.</typeparam>
/// <param name="request">The IKafkaRequest to send to the kafka servers.</param>
/// <returns></returns>
public async Task <List <T> > SendAsync <T>(IKafkaRequest <T> request)
{
//assign unique correlationId
request.Correlation = NextCorrelationId();
var asyncRequest = new AsyncRequestItem(request.Correlation);
if (_requestIndex.TryAdd(request.Correlation, asyncRequest) == false)
{
throw new ApplicationException("Failed to register request for async response.");
}
// Encode the request into a stream
var stream = new BinaryStream();
request.Encode(stream);
// Write to the socket
await _client.WriteAsync(stream);
var response = await asyncRequest.ReceiveTask.Task;
return(request.Decode(response).ToList());
}
/// <exception cref="InvalidTopicMetadataException">Thrown if the returned metadata for the given topic is invalid or missing</exception>
/// <exception cref="InvalidPartitionException">Thrown if the give partitionId does not exist for the given topic.</exception>
/// <exception cref="ServerUnreachableException">Thrown if none of the default brokers can be contacted</exception>
/// <exception cref="ResponseTimeoutException">Thrown if there request times out</exception>
/// <exception cref="BrokerConnectionException">Thrown in case of network error contacting broker (after retries)</exception>
/// <exception cref="KafkaApplicationException">Thrown in case of an unexpected error in the request</exception>
/// <exception cref="FormatException">Thrown in case the topic name is invalid</exception>
public async Task <T> SendProtocolRequest <T>(IKafkaRequest <T> request, string topic, int partition) where T : class, IBaseResponse
{
ValidateTopic(topic);
T response = null;
int retryTime = 0;
while (retryTime < _maxRetry)
{
bool needToRefreshTopicMetadata = false;
ExceptionDispatchInfo exceptionInfo = null;
string errorDetails = "";
try
{
await _brokerRouter.RefreshMissingTopicMetadata(topic);
//find route it can chage after Metadata Refresh
var route = _brokerRouter.SelectBrokerRouteFromLocalCache(topic, partition);
var responses = await route.Connection.SendAsync(request).ConfigureAwait(false);
response = responses.FirstOrDefault();
//this can happened if you send ProduceRequest with ack level=0
if (response == null)
{
return(null);
}
var error = (ErrorResponseCode)response.Error;
if (error == ErrorResponseCode.NoError)
{
return(response);
}
//It means we had an error
errorDetails = error.ToString();
needToRefreshTopicMetadata = CanRecoverByRefreshMetadata(error);
}
catch (ResponseTimeoutException ex)
{
exceptionInfo = ExceptionDispatchInfo.Capture(ex);
}
catch (BrokerConnectionException ex)
{
exceptionInfo = ExceptionDispatchInfo.Capture(ex);
}
catch (NoLeaderElectedForPartition ex)
{
exceptionInfo = ExceptionDispatchInfo.Capture(ex);
}
catch (LeaderNotFoundException ex)//the numbar partition of can be change
{
exceptionInfo = ExceptionDispatchInfo.Capture(ex);
}
if (exceptionInfo != null)
{
needToRefreshTopicMetadata = true;
errorDetails = exceptionInfo.SourceException.GetType().Name;
}
retryTime++;
bool hasMoreRetry = retryTime < _maxRetry;
_brokerRouter.Log.WarnFormat("ProtocolGateway error sending request, retrying (attempt number {0}): {1}", retryTime, errorDetails);
if (needToRefreshTopicMetadata && hasMoreRetry)
{
await _brokerRouter.RefreshTopicMetadata(topic).ConfigureAwait(false);
}
else
{
_brokerRouter.Log.ErrorFormat("ProtocolGateway sending request failed");
// If an exception was thrown, we want to propagate it
if (exceptionInfo != null)
{
exceptionInfo.Throw();
}
// Otherwise, the error was from Kafka, throwing application exception
throw new KafkaApplicationException("FetchResponse received an error from Kafka: {0}", errorDetails)
{
ErrorCode = response.Error
};
}
}
return(response);
}
/// <exception cref="InvalidTopicMetadataException">Thrown if the returned metadata for the given topic is invalid or missing</exception>
/// <exception cref="InvalidPartitionException">Thrown if the give partitionId does not exist for the given topic.</exception>
/// <exception cref="ServerUnreachableException">Thrown if none of the default brokers can be contacted</exception>
/// <exception cref="ResponseTimeoutException">Thrown if there request times out</exception>
/// <exception cref="SocketException">Thrown in case of network error contacting broker (after retries)</exception>
/// <exception cref="KafkaApplicationException">Thrown in case of an unexpected error in the request</exception>
/// <exception cref="FormatException">Thrown in case the topic name is invalid</exception>
public async Task <T> SendProtocolRequest <T>(IKafkaRequest <T> request, string topic, int partition) where T : class, IBaseResponse
{
ValidateTopic(topic);
T response = null;
int retryTime = 0;
while (retryTime < _maxRetry)
{
bool needToRefreshTopicMetadata;
ExceptionDispatchInfo exception = null;
try
{
await _brokerRouter.RefreshMissingTopicMetadata(topic);
//find route it can chage after Metadata Refresh
var route = _brokerRouter.SelectBrokerRouteFromLocalCache(topic, partition);
var responses = await route.Connection.SendAsync(request);
response = responses.FirstOrDefault();
//this can happened if you send ProduceRequest with ack level=0
if (response == null)
{
return(null);
}
var error = (ErrorResponseCode)response.Error;
if (error == ErrorResponseCode.NoError)
{
return(response);
}
//It means we had an error
needToRefreshTopicMetadata = CanRecoverByRefreshMetadata(error);
}
catch (ResponseTimeoutException ex)
{
exception = ExceptionDispatchInfo.Capture(ex);
needToRefreshTopicMetadata = true;
}
catch (SocketException ex)
{
exception = ExceptionDispatchInfo.Capture(ex);
needToRefreshTopicMetadata = true;
}
bool hasMoreRetry = retryTime + 1 < _maxRetry;
if (needToRefreshTopicMetadata && hasMoreRetry)
{
retryTime++;
await _brokerRouter.RefreshTopicMetadata(topic);
}
else
{
if (exception != null)
{
exception.Throw();
}
throw new KafkaApplicationException("FetchResponse returned error condition. ErrorCode:{0}", response.Error)
{
ErrorCode = response.Error
};
}
}
throw new KafkaApplicationException("FetchResponse returned error condition. ErrorCode:{0}", response.Error);
}
