private string ConvertKafkaEventData2String(KafkaEventData kafkaEventData)
{
try
{
if (kafkaEventData.Value is byte[] binaryContent)
{
if (binaryContent != null)
{
return(Encoding.UTF8.GetString(binaryContent));
}
else
{
return(string.Empty);
}
}
else if (kafkaEventData.Value is GenericRecord genericRecord)
{
return(GenericRecord2String(genericRecord));
}
else
{
return(JsonConvert.SerializeObject(kafkaEventData));
}
}
catch (Exception ex)
{
logger.LogError(ex, $@"Unable to convert incoming data to string.");
throw;
}
}
private byte[] ConvertKafkaEventData2Bytes(KafkaEventData input)
{
if (input.Value is byte[] bytes)
{
return(bytes);
}
logger.LogWarning($@"Unable to convert incoming data to byte[] as underlying data stream was not byte[]. Returning [null]");
return(null);
}
public void Produce(string topic, KafkaEventData item)
{
if (item == null)
{
throw new ArgumentNullException(nameof(item));
}
if (item.Value == null)
{
throw new ArgumentException("Message value was not defined");
}
if (!(item.Value is TValue typedValue))
{
throw new ArgumentException($"Message value is not of the expected type. Expected: {typeof(TValue).Name}. Actual: {item.Value.GetType().Name}");
}
var msg = new Message <TKey, TValue>()
{
Value = typedValue,
};
if (item.Key != null)
{
if (!(item.Key is TKey keyValue))
{
throw new ArgumentException($"Key value is not of the expected type. Expected: {typeof(TKey).Name}. Actual: {item.Key.GetType().Name}");
}
msg.Key = keyValue;
}
var topicUsed = topic;
if (string.IsNullOrEmpty(topic))
{
topicUsed = item.Topic;
if (string.IsNullOrEmpty(topicUsed))
{
throw new ArgumentException("No topic was defined in Kafka attribute or in KafkaEventData");
}
}
try
{
this.producer.BeginProduce(topicUsed, msg, this.DeliveryHandler);
}
catch (Exception ex)
{
this.logger.LogError(ex, "Error producing into {topic}", topicUsed);
throw;
}
}
private ISpecificRecord ConvertKafkaEventData2AvroSpecific(KafkaEventData kafkaEventData)
{
if (kafkaEventData.Value is ISpecificRecord specRecord)
{
return(specRecord);
}
else
{
logger.LogWarning($@"Unable to convert incoming data to Avro format. Expected ISpecificRecord, got {kafkaEventData.Value.GetType()}. Returning [null]");
return(null);
}
}
public IKafkaEventData[] Convert(string[] input)
{
var result = new KafkaEventData <string> [input?.Length ?? 0];
if (input != null)
{
for (int i = 0; i < input.Length; i++)
{
result[i] = new KafkaEventData <string>(input[i]);
}
}
return(result);
}
public Task AddAsync(T item, CancellationToken cancellationToken)
{
if (item == null)
{
throw new InvalidOperationException("Cannot produce a null message instance.");
}
object messageToSend = item;
if (item.GetType() == typeof(string) || item.GetType() == typeof(byte[]))
{
messageToSend = new KafkaEventData <T>(item);
}
return(entity.SendAndCreateEntityIfNotExistsAsync(messageToSend, functionInstanceId, cancellationToken));
}
private string ConvertKafkaEventData2String(KafkaEventData kafkaEventData)
{
try
{
if (kafkaEventData.Value is GenericRecord genericRecord)
{
return(GenericRecord2String(genericRecord));
}
else
{
return(JsonConvert.SerializeObject(kafkaEventData));
}
}
catch (Exception ex)
{
logger.LogError(ex, $@"Unable to convert incoming data to string.");
throw;
}
}
private void ProcessSubscription(object parameter)
{
this.subscriberFinished = new SemaphoreSlim(0, 1);
var cancellationToken = (CancellationToken)parameter;
var maxBatchSize = this.options.MaxBatchSize;
var maxBatchReleaseTime = TimeSpan.FromSeconds(this.options.SubscriberIntervalInSeconds);
var localConsumer = this.consumer.Value;
try
{
var alreadyFlushedInCurrentExecution = false;
while (!cancellationToken.IsCancellationRequested)
{
var batchStart = DateTime.UtcNow;
var availableTime = maxBatchReleaseTime - (DateTime.UtcNow - batchStart);
alreadyFlushedInCurrentExecution = false;
while (availableTime > TimeSpan.Zero)
{
try
{
var consumeResult = localConsumer.Consume(availableTime);
// If no message was consumed during the available time, returns null
if (consumeResult != null)
{
if (consumeResult.IsPartitionEOF)
{
this.logger.LogInformation("Reached end of {topic} / {partition} / {offset}", consumeResult.Topic, consumeResult.Partition, consumeResult.Offset);
}
else
{
var kafkaEventData = this.requiresKey ?
(IKafkaEventData) new KafkaEventData <TKey, TValue>(consumeResult) :
KafkaEventData <TValue> .CreateFrom(consumeResult);
// add message to executor
// if executor pending items is full, flush it
var currentSize = this.functionExecutor.Add(kafkaEventData);
if (currentSize >= maxBatchSize)
{
this.functionExecutor.Flush();
alreadyFlushedInCurrentExecution = true;
}
}
availableTime = maxBatchReleaseTime - (DateTime.UtcNow - batchStart);
}
else
{
// TODO: maybe slow down if there isn't much incoming data
break;
}
}
catch (ConsumeException ex)
{
this.logger.LogError(ex, $"Consume error");
}
}
if (!alreadyFlushedInCurrentExecution)
{
this.functionExecutor.Flush();
}
}
}
catch (Exception ex)
{
this.logger.LogError(ex, "Error in Kafka subscriber");
}
finally
{
this.logger.LogInformation("Exiting {processName} for {topic}", nameof(ProcessSubscription), this.listenerConfiguration.Topic);
this.subscriberFinished.Release();
}
}
public async Task ProduceAsync(string topic, KafkaEventData item)
{
if (item == null)
{
throw new ArgumentNullException(nameof(item));
}
if (item.Value == null)
{
throw new ArgumentException("Message value was not defined");
}
if (!(item.Value is TValue typedValue))
{
throw new ArgumentException($"Message value is not of the expected type. Expected: {typeof(TValue).Name}. Actual: {item.Value.GetType().Name}");
}
var msg = new Message <TKey, TValue>()
{
Value = typedValue,
};
if (item.Key != null)
{
if (!(item.Key is TKey keyValue))
{
throw new ArgumentException($"Key value is not of the expected type. Expected: {typeof(TKey).Name}. Actual: {item.Key.GetType().Name}");
}
msg.Key = keyValue;
}
var topicUsed = topic;
if (string.IsNullOrEmpty(topic))
{
topicUsed = item.Topic;
if (string.IsNullOrEmpty(topicUsed))
{
throw new ArgumentException("No topic was defined in Kafka attribute or in KafkaEventData");
}
}
try
{
var deliveryResult = await this.producer.ProduceAsync(topicUsed, msg);
this.logger.LogDebug("Message delivered on {topic} / {partition} / {offset}", deliveryResult.Topic, (int)deliveryResult.Partition, (long)deliveryResult.Offset);
}
catch (ProduceException <TKey, TValue> produceException)
{
logger.LogError("Failed to delivery message to {topic} / {partition} / {offset}. Reason: {reason}. Full Error: {error}", produceException.DeliveryResult?.Topic, (int)produceException.DeliveryResult?.Partition, (long)produceException.DeliveryResult?.Offset, produceException.Error.Reason, produceException.Error.ToString());
throw;
}
catch (Exception ex)
{
this.logger.LogError(ex, "Error producing into {topic}", topicUsed);
throw;
}
}
/// <summary>
/// Adds an item, returning the current pending amount
/// </summary>
internal int Add(KafkaEventData kafkaEventData)
{
this.currentBatch.Add(kafkaEventData);
return(this.currentBatch.Count);
}
