protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var consumerConfig = new ConsumerConfig
{
GroupId = "socket-server-consumer-command-response" + Guid.NewGuid(), // NOTE: If the socket server restarts it will never join the same group. This is to ensure it always reads the latest data
BootstrapServers = KafkaHelpers.BootstrapServers,
AutoOffsetReset = AutoOffsetReset.Latest,
};
KafkaHelpers.SetKafkaConfigKerberos(consumerConfig);
using (var c = new ConsumerBuilder <Ignore, string>(consumerConfig).Build())
{
c.Subscribe($"^{KafkaHelpers.CommandResponseTopicPrefix}");
_logger.LogInformation("Listening for responses from all command servers");
while (!stoppingToken.IsCancellationRequested)
{
// consumer does not have an async method. So it is wrapped in a task, so that the rest of the application doesn't hang here
var consumeResult = await Task.Factory.StartNew(() => c.Consume(stoppingToken));
Console.WriteLine("Found a result! " + consumeResult.Message.Value);
await _updatersHub.Clients.All.SendCommandResponses(consumeResult.Message.Value);
}
c.Close();
}
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var consumerConfig = new ConsumerConfig
{
GroupId = "socket-server-consumer",
BootstrapServers = KafkaHelpers.BootstrapServers,
AutoOffsetReset = AutoOffsetReset.Latest,
};
KafkaHelpers.SetKafkaConfigKerberos(consumerConfig);
using (var c = new ConsumerBuilder <Ignore, string>(consumerConfig).Build())
{
c.Subscribe(new List <string>
{
KafkaHelpers.OverviewTopic, KafkaHelpers.StatsTopic, KafkaHelpers.InspectTopic
});
_logger.LogInformation("Listening for updates");
while (!stoppingToken.IsCancellationRequested)
{
// consumer does not have an async method. So it is wrapped in a task, so that the rest of the application doesn't hang here
var consumeResult = await Task.Factory.StartNew(() => c.Consume(stoppingToken));
switch (consumeResult.Topic)
{
case KafkaHelpers.OverviewTopic:
await _updatersHub.Clients.All.SendOverviewData(consumeResult.Message.Value);
// NOTE: The method checks on LatestOverviewInfo, so it's important this is called before LatestOverviewInfo is set after the record has been saved
// await SaveStatusRecordInDb(consumeResult.Message.Value);
KafkaHelpers.LatestOverviewInfo = consumeResult.Message.Value;
break;
case KafkaHelpers.StatsTopic:
await _updatersHub.Clients.All.SendStatsData(consumeResult.Message.Value);
// NOTE: The method checks on LatestOverviewInfo, so it's important this is called before LatestOverviewInfo is set after the record has been saved
// await SaveRessourceUsageRecordInDb(consumeResult.Message.Value);
KafkaHelpers.LatestStatsInfo = consumeResult.Message.Value;
break;
case KafkaHelpers.InspectTopic:
var inspectData = JsonConvert.DeserializeObject <InspectData>(consumeResult.Message.Value);
await _updatersHub.Clients.All.SendInspectResponse(
JsonConvert.SerializeObject(inspectData));
break;
}
}
c.Close();
}
}
public SubscriptionWorker(ILogger <SubscriptionWorker> logger, IServiceProvider services)
{
_services = services;
_logger = logger;
var producerConfig = new ProducerConfig {
Acks = Acks.Leader
};
KafkaHelpers.SetKafkaConfigKerberos(producerConfig);
_producer = new ProducerBuilder <Null, string>(producerConfig).Build();
}
public static async void Start(CancellationTokenSource cts)
{
var consumerConfig = new ConsumerConfig {
GroupId = "command-server" + KafkaHelpers.Servername,
BootstrapServers = KafkaHelpers.BootstrapServers,
AutoOffsetReset = AutoOffsetReset.Latest,
};
KafkaHelpers.SetKafkaConfigKerberos(consumerConfig);
using (var c = new ConsumerBuilder <Ignore, string>(consumerConfig).Build()) {
c.Subscribe(KafkaHelpers.RequestTopic);
Console.WriteLine($"Listening for commands on topic {KafkaHelpers.RequestTopic}");
try {
var producerConfig = new ProducerConfig {
BootstrapServers = KafkaHelpers.BootstrapServers, Acks = Acks.Leader
};
KafkaHelpers.SetKafkaConfigKerberos(producerConfig);
using (var p = new ProducerBuilder <Null, string>(producerConfig).Build()) {
while (true)
{
try {
var consumeResult = c.Consume(cts.Token); // Polling for new messages, waiting here until message recieved
var messageJsonString = consumeResult.Message.Value;
ContainerRequest request = JsonConvert.DeserializeObject <ContainerRequest>(messageJsonString);
await CallAction(request.Action, messageJsonString, p);
} catch (ConsumeException ex) {
Console.Error.WriteLine(ex.Error);
} catch (Newtonsoft.Json.JsonException ex) {
await KafkaHelpers.SendMessageAsync(KafkaHelpers.ResponseTopic, new ContainerResponse {
ResponseStatusCode = 400,
Message = ex.Message
}, p);
}
}
}
} catch (OperationCanceledException) { } finally {
// Ensure the consumer leaves the group cleanly and final offsets are committed.
c.Close();
}
}
}
public static void Start()
{
var producerConfig = new ProducerConfig {
BootstrapServers = KafkaHelpers.BootstrapServers, Acks = Acks.Leader
};
KafkaHelpers.SetKafkaConfigKerberos(producerConfig);
using var p = new ProducerBuilder <Null, string>(producerConfig).Build();
Task[] tester = new Task[2];
if (_processesToStart.Contains("overviewdata"))
{
var overviewTask = SendOverViewData(p);
tester[0] = overviewTask;
}
if (_processesToStart.Contains("statsdata"))
{
var statsTask = SendStatsData(p);
tester[1] = statsTask;
}
Task.WaitAll(tester);
}
public async void ListenForSubscriptionsActions(CancellationToken stoppingToken, List <CurrentConditionSubscription> currentConditionSubscriptions, List <ForecastSubscription> forecastSubscriptions)
{
// Listening and waiting for new subscription requests from kafka
var consumerConfig = new ConsumerConfig
{
GroupId = "weather-station-new-subscription-group",
AutoOffsetReset = AutoOffsetReset.Earliest,
};
KafkaHelpers.SetKafkaConfigKerberos(consumerConfig);
using (var consumer = new ConsumerBuilder <Ignore, string>(consumerConfig).Build())
{
using (var scope = _services.CreateScope())
{
var repo = scope.ServiceProvider.GetRequiredService <ISubscriptionRepo>();
consumer.Subscribe(KafkaHelpers.SubscriptionActionsTopic);
_logger.LogInformation("Listening for subscription action requests");
while (!stoppingToken.IsCancellationRequested)
{
try
{
// consumer does not have an async method. So it is wrapped in a task, so that the rest of the application doesn't hang here
var consumeResult = await Task.Factory.StartNew(() => consumer.Consume(stoppingToken));
var messageJsonString = consumeResult.Message.Value;
// Checking format required.
SubscriptionRequestDTO request = JsonConvert.DeserializeObject <SubscriptionRequestDTO>(messageJsonString);
_logger.LogInformation("A subscription action request has arrived");
switch (request.Action)
{
case SubscriptionAction.CREATECURRENTCONDITION:
var currentConditionRequest = JsonConvert.DeserializeObject <NewCurrentConditionSubscriptionRequestDTO>(messageJsonString);
var currentConditionSubscription = new CurrentConditionSubscription
{
StationId = currentConditionRequest.StationId,
IntervalSeconds = currentConditionRequest.IntervalSeconds
};
// persisting the newly added subscription
await repo.AddCurrentConditionSubscription(currentConditionSubscription);
// adding it to the list of subscriptions
currentConditionSubscriptions.Add(currentConditionSubscription);
// starting the subscription. I want this to run in the background. So I discard the result with '_'
_ = Task.Run(() => StartTimer(currentConditionSubscription));
break;
case SubscriptionAction.CREATEFORECAST:
var forecastSubscriptionRequest = JsonConvert.DeserializeObject <ForecastSubscription>(messageJsonString);
var forecastSubscription = new ForecastSubscription
{
GeoCode = forecastSubscriptionRequest.GeoCode,
IntervalSeconds = forecastSubscriptionRequest.IntervalSeconds
};
// persisting the newly added subscription
await repo.AddForecastSubscription(forecastSubscription);
// adding it to the list of subscriptions
forecastSubscriptions.Add(forecastSubscription);
// starting the subscription. I want this to run in the background. So I discard the result with '_'
_ = Task.Run(() => StartTimer(forecastSubscription));
break;
case SubscriptionAction.DELETE:
// TODO: Delete
break;
default:
throw new Newtonsoft.Json.JsonException("Invalid Subscription action");
}
}
catch (ConsumeException ex)
{
_logger.LogError(ex.Error.ToString());
}
// If the request from kafka is not correctly formatted an error will be thrown and catched here
catch (Newtonsoft.Json.JsonException ex)
{
_logger.LogError(ex.Message);
}
// Cancelled background worker
catch (OperationCanceledException)
{
consumer.Close();
}
}
}
}
// If we reach this point it means the application is shutting down. Therefore we clean up
_logger.LogInformation("Cleaning up");
_producer.Flush(TimeSpan.FromSeconds(5));
_producer.Dispose();
}