public Task <DeliveryResult <T, TV> > SendMessage(T key, TV value)
{
var result = KafkaHandler.DeliverMessage(key, value, ProducerId);
var date = result.Result.Timestamp.UtcDateTime;
Devon4NetLogger.Information($"Message delivered. Key: {result.Result.Key} | Value : {result.Result.Value} | Topic: {result.Result.Topic} | UTC TimeStamp : {date.ToShortDateString()}-{date.ToLongTimeString()} | Status: {result.Result.Status}");
return(result);
}
private static ProducerBuilder <T, TV> GetProducerBuilderInstance <T, TV>(ProducerConfig configuration) where T : class where TV : class
{
var producer = new ProducerBuilder <T, TV>(configuration);
producer.SetErrorHandler((_, e) => Devon4NetLogger.Error(new ConsumerException($"Error code {e.Code} : {e.Reason}")));
producer.SetStatisticsHandler((_, json) => Devon4NetLogger.Information($"Statistics: {json}"));
producer.SetLogHandler((c, partitions) => { Devon4NetLogger.Information($"Kafka log handler: [{string.Join(", ", partitions)}]"); });
return(producer);
}
private static void SetupMediatRBackupLocalDatabase(ref IServiceCollection services, ref MediatROptions mediatROptions)
{
Devon4NetLogger.Information("Please setup your database in order to have the RabbitMq messaging backup feature");
if (mediatROptions.Backup == null || !mediatROptions.Backup.UseLocalBackup)
{
return;
}
Devon4NetLogger.Information("RabbitMq messaging backup feature is going to be used via LiteDb");
services.AddSingleton <ILiteDbContext, MediatRBackupLiteDbContext>();
services.AddTransient(typeof(IMediatRBackupLiteDbService), typeof(MediatRBackupLiteDbService));
}
private static async Task LogHttpResponse(HttpResponseMessage httpResponseMessage, string endPointName)
{
if (httpResponseMessage == null)
{
Devon4NetLogger.Information($"HttpResponse message for endpoint call {endPointName} is null");
return;
}
var contentResult = httpResponseMessage.IsSuccessStatusCode
? "The Http response is success"
: await LogHttpContent(httpResponseMessage.Content, endPointName).ConfigureAwait(false);
Devon4NetLogger.Information($"HttpResponse message for endpoint call {endPointName} : HttpRequest :{httpResponseMessage.RequestMessage} | httpResponse: {httpResponseMessage} | message Content: {contentResult}");
}
private static void ProcessDirectorySettings(string settingsItemDirectory)
{
Devon4NetLogger.Information($"ProcessDirectorySettings {settingsItemDirectory}");
if (string.IsNullOrEmpty(settingsItemDirectory) || string.IsNullOrWhiteSpace(settingsItemDirectory) || !Directory.Exists(settingsItemDirectory))
{
return;
}
var fileNameList = FileOperations.GetFilesFromPath("*", settingsItemDirectory);
foreach (var fileSettings in fileNameList)
{
Devon4NetLogger.Information($"Processing {fileSettings}");
AddConfigurationSettingsFile(fileSettings, true, false, settingsItemDirectory);
}
}
private static void CheckExtraSettingsFiles()
{
Devon4NetLogger.Information("CheckExtraSettingsFiles Initialized");
var appSettingsList = new List <string>();
Configuration.GetSection("ExtraSettingsFiles").Bind(appSettingsList);
if (appSettingsList?.Any() != true)
{
Devon4NetLogger.Information("CheckExtraSettingsFiles does not contains any settings file to be managed");
return;
}
Devon4NetLogger.Information($"CheckExtraSettingsFiles has detected the global settings : {appSettingsList}");
ManageSettingsFiles(appSettingsList);
}
public IConsumer <T, TV> GetConsumerBuilder <T, TV>(string consumerId) where T : class where TV : class
{
if (string.IsNullOrEmpty(consumerId))
{
throw new ConsumerNotFoundException($"The consumerId param can not be null or empty");
}
var consumerOptions = KafkaOptions.Consumers.FirstOrDefault(p => p.ConsumerId == consumerId);
if (consumerOptions == null)
{
throw new ConsumerNotFoundException($"Could not find consumer configuration with ConsumerId {consumerId}");
}
var configuration = GetDefaultKafkaConsumerConfiguration(consumerOptions);
var consumer = new ConsumerBuilder <T, TV>(configuration);
IConsumer <T, TV> result = null;
try
{
consumer.SetErrorHandler((_, e) => Devon4NetLogger.Error(new ConsumerException($"Error code {e.Code} : {e.Reason}")));
consumer.SetStatisticsHandler((_, json) => Devon4NetLogger.Information($"Statistics: {json}"));
consumer.SetPartitionsAssignedHandler((c, partitions) =>
{
Devon4NetLogger.Information($"Assigned partitions: [{string.Join(", ", partitions)}]");
});
consumer.SetPartitionsRevokedHandler((c, partitions) =>
{
Devon4NetLogger.Information($"Revoking assignment: [{string.Join(", ", partitions)}]");
});
result = consumer.Build();
if (!string.IsNullOrEmpty(consumerOptions.Topics))
{
result.Subscribe(consumerOptions.GetTopics());
}
}
catch (InvalidOperationException ex)
{
Devon4NetLogger.Error(ex);
}
return(result);
}
private async Task LogHttpResponseAsync(HttpResponseMessage httpResponseMessage, HttpContent httpContent = null)
{
var messageContent = string.Empty;
if (httpResponseMessage != null && httpResponseMessage.Content != null)
{
messageContent = await httpResponseMessage.Content.ReadAsStringAsync().ConfigureAwait(false);
}
if (httpResponseMessage != null)
{
Devon4NetLogger.Information($" HttpRequest :{httpResponseMessage.RequestMessage} | httpResponse: {httpResponseMessage} | message Content: {messageContent}");
}
if (httpContent != null)
{
Devon4NetLogger.Information($" HttpRequestBody :{await httpContent.ReadAsStringAsync().ConfigureAwait(false)}");
}
}
/// <summary>
/// If commit is set to true, the Commit method sends a "commit offsets" request to the Kafka cluster and synchronously waits for the response.
/// This is very slow compared to the rate at which the consumer is capable of consuming messages.
/// A high performance application will typically commit offsets relatively infrequently and be designed handle duplicate messages in the event of failure.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <typeparam name="TV"></typeparam>
/// <param name="commit"></param>
/// <param name="commitPeriod"></param>
private void Consume(bool commit, int commitPeriod)
{
var cancellationToken = new CancellationTokenSource();
Task.Run(() =>
{
try
{
using var consumer = KafkaHandler.GetConsumerBuilder <T, TV>(ConsumerId);
while (EnableConsumerFlag)
{
var consumeResult = consumer?.Consume(cancellationToken.Token);
if (consumeResult?.Message == null)
{
continue;
}
HandleCommand(consumeResult.Message.Key, consumeResult.Message.Value);
if (consumeResult.IsPartitionEOF)
{
Devon4NetLogger.Information($"Reached end of topic {consumeResult.Topic}, partition {consumeResult.Partition}, offset {consumeResult.Offset}.");
continue;
}
Devon4NetLogger.Debug($"Received message at {consumeResult.TopicPartitionOffset}: {consumeResult.Message.Value}");
if (!commit || consumeResult.Offset % commitPeriod != 0)
{
continue;
}
consumer?.Commit(consumeResult);
consumer?.Close();
}
}
catch (Exception ex)
{
Devon4NetLogger.Error(ex);
}
}, cancellationToken.Token);
}
private static void AddConfigurationSettingsFile(string filename, bool optional, bool reloadOnChange, string defaultDirectory = null)
{
if (string.IsNullOrEmpty(filename) || string.IsNullOrWhiteSpace(filename))
{
Devon4NetLogger.Information($"{filename} settings file does NOT exists!!!");
return;
}
SetupConfigurationBuilder();
var fileName = FileOperations.GetFileFullPath(filename, defaultDirectory);
if (string.IsNullOrEmpty(fileName) || !File.Exists(fileName))
{
return;
}
ConfigurationBuilder.AddJsonFile(filename, optional, reloadOnChange);
Configuration = ConfigurationBuilder.Build();
}
private static QueueActionsEnum PublishCommandTaskResult(T command, Task task)
{
var status = QueueActionsEnum.SetUp;
if (task.IsCompleted)
{
status = QueueActionsEnum.Sent;
Devon4NetLogger.Information($"Message {command.MessageType} with identifier '{command.InternalMessageIdentifier}' published");
}
if (!task.IsFaulted)
{
return(status);
}
status = QueueActionsEnum.Error;
Devon4NetLogger.Error($"Message {command.MessageType} with identifier '{command.InternalMessageIdentifier}' NOT published");
Devon4NetLogger.Error(task.Exception);
return(status);
}
private async Task Process()
{
try
{
while (!TokenSource.Token.IsCancellationRequested)
{
var messages = await SqsClientHandler.GetSqsMessages(SqsQueue.Url, SqsQueue.RedrivePolicy.MaxReceiveCount, SqsQueue.ReceiveMessageWaitTimeSeconds, TokenSource.Token).ConfigureAwait(false);
foreach (var message in messages)
{
var messageType = message.MessageAttributes.GetMessageTypeAttributeValue();
if (messageType == null)
{
throw new ArgumentNullException($"No MessageType attribute present in message {JsonSerializer.Serialize(message)}");
}
if (messageType.Equals(typeof(T).Name))
{
var processed = await ProcessMessage(message).ConfigureAwait(false);
if (processed)
{
_ = await SqsClientHandler.DeleteMessage(SqsQueue.Url, message.ReceiptHandle, TokenSource.Token).ConfigureAwait(false);
}
}
}
}
}
catch (OperationCanceledException ex)
{
Devon4NetLogger.Information("The message operation process has been cancelled");
Devon4NetLogger.Information(ex);
}
catch (Exception ex)
{
Devon4NetLogger.Error(ex);
throw;
}
}
private async Task <bool> BackupAndHandleCommand(T message)
{
var status = QueueActionsEnum.SetUp;
var errorMessage = string.Empty;
try
{
await HandleCommand(message).ContinueWith(async task =>
{
if (task.IsCompleted)
{
status = QueueActionsEnum.Handled;
Devon4NetLogger.Information($"Message {message.MessageType} with identifier '{message.InternalMessageIdentifier}' published");
}
if (task.IsFaulted)
{
status = QueueActionsEnum.Error;
errorMessage = $"Message {message.MessageType} with identifier '{message.InternalMessageIdentifier}' NOT published. {task.Exception?.Message} | {task.Exception?.InnerExceptions}";
Devon4NetLogger.Error(errorMessage);
Devon4NetLogger.Error(task.Exception);
}
}).ConfigureAwait(false);
await BackUpMessage(message, status, false, string.Empty, errorMessage).ConfigureAwait(false);
}
catch (Exception ex)
{
await BackUpMessage(message, QueueActionsEnum.Error, false, string.Empty,
$"{ex.Message} : {ex.InnerException}").ConfigureAwait(false);
Devon4NetLogger.Error($"Error handling message: {ex.Message}/{ex.InnerException}");
Devon4NetLogger.Error(ex);
return(false);
}
return(status == QueueActionsEnum.Handled);
}
private static string GetEncodedUrl(string baseAddress, string endPoint)
{
if (string.IsNullOrEmpty(baseAddress))
{
Devon4NetLogger.Information("GetEncodedUrl method inoveed with empty baseAddres");
return(string.Empty);
}
var result = string.Empty;
if (endPoint.Contains(baseAddress))
{
endPoint = endPoint.Replace(baseAddress, "/");
}
if (endPoint.Contains("//"))
{
endPoint = endPoint.Replace("//", "/");
}
if (baseAddress.EndsWith("/") && endPoint.StartsWith("/"))
{
result = string.Concat(baseAddress, endPoint.AsSpan(1));
}
if (!baseAddress.EndsWith("/") && !endPoint.StartsWith("/"))
{
result = $"{baseAddress}/{endPoint}";
}
if (!endPoint.StartsWith("/") || (baseAddress.EndsWith("/") && !endPoint.StartsWith("/")) || (!baseAddress.EndsWith("/") && endPoint.StartsWith("/")))
{
result = $"{baseAddress}{endPoint}";
}
return(Uri.EscapeDataString(result));
}
public async Task <bool> Publish(T command)
{
var status = QueueActionsEnum.SetUp;
try
{
status = QueueActionsEnum.SetUp;
await ServiceBus.PublishAsync(command).ContinueWith(async task =>
{
if (task.IsCompleted)
{
status = QueueActionsEnum.Sent;
Devon4NetLogger.Information($"Message {command.MessageType} with identifier '{command.InternalMessageIdentifier}' published");
}
if (task.IsFaulted)
{
status = QueueActionsEnum.Error;
Devon4NetLogger.Error($"Message {command.MessageType} with identifier '{command.InternalMessageIdentifier}' NOT published");
Devon4NetLogger.Error(task.Exception);
}
}).ConfigureAwait(false);
await BackUpMessage(command, status).ConfigureAwait(false);
}
catch (Exception ex)
{
await BackUpMessage(command, QueueActionsEnum.Error, false, string.Empty, $"{ex?.Message} : {ex?.InnerException}").ConfigureAwait(false);
Devon4NetLogger.Error($"Error publishing message: {ex.Message}/{ex.InnerException}");
Devon4NetLogger.Error(ex);
}
return(status == QueueActionsEnum.Sent);
}
private static void ManageSettingsFiles(IReadOnlyCollection <string> settingsItemList)
{
Devon4NetLogger.Information("Managing settings global settings files ...");
if (settingsItemList?.Any() != true)
{
Devon4NetLogger.Information("No global settings files found!");
return;
}
foreach (var settingsItem in settingsItemList)
{
if (string.IsNullOrEmpty(settingsItem))
{
continue;
}
if (Directory.Exists(settingsItem))
{
Devon4NetLogger.Information($"SettingsItem {settingsItem} is a directory. Checking the directory...");
ProcessDirectorySettings(settingsItem);
Devon4NetLogger.Information($"SettingsItem {settingsItem} processed");
}
else
{
if (!File.Exists(settingsItem))
{
Devon4NetLogger.Error($"The provided settings file '{settingsItem}' does not exists as directory or file");
continue;
}
Devon4NetLogger.Information($"SettingsItem {settingsItem} is a file. Checking the file...");
AddConfigurationSettingsFile(settingsItem, true, false);
Devon4NetLogger.Information($"SettingsItem {settingsItem} processed");
}
}
}
private static QueueActionsEnum HandleCommandTaskResult(T message, Task <bool> task, out string errorMessage)
{
var status = QueueActionsEnum.SetUp;
errorMessage = string.Empty;
if (task.IsCompleted)
{
status = QueueActionsEnum.Handled;
Devon4NetLogger.Information($"Message {message.MessageType} with identifier '{message.InternalMessageIdentifier}' published");
}
if (!task.IsFaulted)
{
return(status);
}
status = QueueActionsEnum.Error;
errorMessage = $"Message {message.MessageType} with identifier '{message.InternalMessageIdentifier}' NOT published. {task.Exception?.Message} | {task.Exception?.InnerExceptions}";
Devon4NetLogger.Error(errorMessage);
Devon4NetLogger.Error(task.Exception);
return(status);
}
public override void HandleCommand(string key, string value)
{
Devon4NetLogger.Information($"Received message key: {key} | value: {value}");
}
private static void LogEvent(string method, string result)
{
Devon4NetLogger.Information($"Result from {method}: {result}");
}
private static void LogEvent(string method, Microsoft.AspNetCore.Mvc.ObjectResult result)
{
Devon4NetLogger.Information($"Result from {method}: {result.StatusCode} | Value: {GetValue(result.Value)}");
}
