public async void RefetchStatsData(string serverRequestTopic) { await KafkaHelpers.SendMessageAsync(serverRequestTopic, new ContainerRequest { Action = ContainerActionType.REFETCH_STATS }); }
protected override async Task ExecuteAsync(CancellationToken stoppingToken) { var consumerConfig = new ConsumerConfig { GroupId = "socket-server-consumer-command-response" + Guid.NewGuid(), // NOTE: If the socket server restarts it will never join the same group. This is to ensure it always reads the latest data BootstrapServers = KafkaHelpers.BootstrapServers, AutoOffsetReset = AutoOffsetReset.Latest, }; KafkaHelpers.SetKafkaConfigKerberos(consumerConfig); using (var c = new ConsumerBuilder <Ignore, string>(consumerConfig).Build()) { c.Subscribe($"^{KafkaHelpers.CommandResponseTopicPrefix}"); _logger.LogInformation("Listening for responses from all command servers"); while (!stoppingToken.IsCancellationRequested) { // consumer does not have an async method. So it is wrapped in a task, so that the rest of the application doesn't hang here var consumeResult = await Task.Factory.StartNew(() => c.Consume(stoppingToken)); Console.WriteLine("Found a result! " + consumeResult.Message.Value); await _updatersHub.Clients.All.SendCommandResponses(consumeResult.Message.Value); } c.Close(); } }
public async void RefetchOverviewData(string serverRequestTopic) { await KafkaHelpers.SendMessageAsync(serverRequestTopic, new ContainerRequest { Action = ContainerActionType.REFETCH_OVERVIEW }); }
public async static Task RenameContainer(RenameContainerParameter parameters, IProducer <Null, string> p) { try { CancellationToken cancellation = new CancellationToken(); await client.Containers.RenameContainerAsync(parameters.ContainerId, new ContainerRenameParameters { NewName = parameters.NewName }, cancellation); // Updating the containers with the new configurations right away and sending it out var updatedContainers = await DockerUpdater.FetchOverviewData(); var overviewContainerUpdate = DockerUpdater.CreateOverViewData(updatedContainers); await KafkaHelpers.SendMessageAsync(DockerUpdater.OverviewTopic, overviewContainerUpdate, p); // Send response await KafkaHelpers.SendMessageAsync(_responseTopic, new ContainerResponse { ResponseStatusCode = 200, Message = ResponseMessageContracts.CONTAINER_RENAMED, ContainerIds = new string[] { parameters.ContainerId } }, p); } catch (DockerApiException ex) { await KafkaHelpers.SendMessageAsync(_responseTopic, new ContainerResponse { ResponseStatusCode = 400, Message = ex.Message, ContainerIds = new string[] { parameters.ContainerId } }, p); } }
public async static Task InspectContainer(InspectContainerParameters parameters, IProducer <Null, string> p) { try { CancellationToken cancellation = new CancellationToken(); var inspectResponse = await client.Containers.InspectContainerAsync(parameters.ContainerId, cancellation); var containerId = inspectResponse.ID.Substring(0, 10); var inspectResponseStr = JsonConvert.SerializeObject(inspectResponse, Formatting.Indented); var inspectContainerResponse = new InspectContainerResponse { ServerName = KafkaHelpers.Servername, ContainerId = containerId, RawData = inspectResponseStr }; await KafkaHelpers.SendMessageAsync(DockerUpdater.InspectTopic, inspectContainerResponse, p); // Send response await KafkaHelpers.SendMessageAsync(_responseTopic, new ContainerResponse { ResponseStatusCode = 200, Message = ResponseMessageContracts.CONTAINER_INSPECTED, ContainerIds = new string[] { parameters.ContainerId } }, p); } catch (DockerApiException ex) { await KafkaHelpers.SendMessageAsync(_responseTopic, new ContainerResponse { ResponseStatusCode = 400, Message = ex.Message, ContainerIds = new string[] { parameters.ContainerId } }, p); } }
protected override async Task ExecuteAsync(CancellationToken stoppingToken) { var consumerConfig = new ConsumerConfig { GroupId = "socket-server-consumer", BootstrapServers = KafkaHelpers.BootstrapServers, AutoOffsetReset = AutoOffsetReset.Latest, }; KafkaHelpers.SetKafkaConfigKerberos(consumerConfig); using (var c = new ConsumerBuilder <Ignore, string>(consumerConfig).Build()) { c.Subscribe(new List <string> { KafkaHelpers.OverviewTopic, KafkaHelpers.StatsTopic, KafkaHelpers.InspectTopic }); _logger.LogInformation("Listening for updates"); while (!stoppingToken.IsCancellationRequested) { // consumer does not have an async method. So it is wrapped in a task, so that the rest of the application doesn't hang here var consumeResult = await Task.Factory.StartNew(() => c.Consume(stoppingToken)); switch (consumeResult.Topic) { case KafkaHelpers.OverviewTopic: await _updatersHub.Clients.All.SendOverviewData(consumeResult.Message.Value); // NOTE: The method checks on LatestOverviewInfo, so it's important this is called before LatestOverviewInfo is set after the record has been saved // await SaveStatusRecordInDb(consumeResult.Message.Value); KafkaHelpers.LatestOverviewInfo = consumeResult.Message.Value; break; case KafkaHelpers.StatsTopic: await _updatersHub.Clients.All.SendStatsData(consumeResult.Message.Value); // NOTE: The method checks on LatestOverviewInfo, so it's important this is called before LatestOverviewInfo is set after the record has been saved // await SaveRessourceUsageRecordInDb(consumeResult.Message.Value); KafkaHelpers.LatestStatsInfo = consumeResult.Message.Value; break; case KafkaHelpers.InspectTopic: var inspectData = JsonConvert.DeserializeObject <InspectData>(consumeResult.Message.Value); await _updatersHub.Clients.All.SendInspectResponse( JsonConvert.SerializeObject(inspectData)); break; } } c.Close(); } }
public SubscriptionWorker(ILogger <SubscriptionWorker> logger, IServiceProvider services) { _services = services; _logger = logger; var producerConfig = new ProducerConfig { Acks = Acks.Leader }; KafkaHelpers.SetKafkaConfigKerberos(producerConfig); _producer = new ProducerBuilder <Null, string>(producerConfig).Build(); }
public async static Task RefetchOverviewData(IProducer <Null, string> p) { var updatedContainers = await DockerUpdater.FetchOverviewData(); var overviewContainerUpdate = DockerUpdater.CreateOverViewData(updatedContainers); await KafkaHelpers.SendMessageAsync(DockerUpdater.OverviewTopic, overviewContainerUpdate, p); // Send response await KafkaHelpers.SendMessageAsync(_responseTopic, new ContainerResponse { ResponseStatusCode = 200, Message = ResponseMessageContracts.OVERVIEW_DATA_REFETCHED, ContainerIds = updatedContainers.Select(container => container.Id).ToArray() }, p); }
public async Task FetchData(ForecastSubscription subscription) { try { using var scope = _services.CreateScope(); var wundergroundService = scope.ServiceProvider.GetRequiredService <IWundergroundService>(); var data = await wundergroundService.Get5DayForecast(subscription.GeoCode); await KafkaHelpers.SendMessageAsync(KafkaHelpers.WeatherDataTopic, data, _producer); } catch (HttpRequestException e) { _logger.LogError($"Fetch for GeoCode={subscription.GeoCode} threw an error: {e.Message}"); } }
public async static Task RefetchStatsData(IProducer <Null, string> p) { // Updating the containers with the new configurations right away and sending it out var updatedContainers = await DockerUpdater.FetchStatsData(); var overviewContainerUpdate = DockerUpdater.CreateStatsData(updatedContainers); await KafkaHelpers.SendMessageAsync(DockerUpdater.StatsTopic, overviewContainerUpdate, p); // Send response await KafkaHelpers.SendMessageAsync(_responseTopic, new ContainerResponse { ResponseStatusCode = 200, Message = ResponseMessageContracts.STATS_DATA_REFETCHED, ContainerIds = updatedContainers.Select(container => container.Id).ToArray() }, p); }
public static async void Start(CancellationTokenSource cts) { var consumerConfig = new ConsumerConfig { GroupId = "command-server" + KafkaHelpers.Servername, BootstrapServers = KafkaHelpers.BootstrapServers, AutoOffsetReset = AutoOffsetReset.Latest, }; KafkaHelpers.SetKafkaConfigKerberos(consumerConfig); using (var c = new ConsumerBuilder <Ignore, string>(consumerConfig).Build()) { c.Subscribe(KafkaHelpers.RequestTopic); Console.WriteLine($"Listening for commands on topic {KafkaHelpers.RequestTopic}"); try { var producerConfig = new ProducerConfig { BootstrapServers = KafkaHelpers.BootstrapServers, Acks = Acks.Leader }; KafkaHelpers.SetKafkaConfigKerberos(producerConfig); using (var p = new ProducerBuilder <Null, string>(producerConfig).Build()) { while (true) { try { var consumeResult = c.Consume(cts.Token); // Polling for new messages, waiting here until message recieved var messageJsonString = consumeResult.Message.Value; ContainerRequest request = JsonConvert.DeserializeObject <ContainerRequest>(messageJsonString); await CallAction(request.Action, messageJsonString, p); } catch (ConsumeException ex) { Console.Error.WriteLine(ex.Error); } catch (Newtonsoft.Json.JsonException ex) { await KafkaHelpers.SendMessageAsync(KafkaHelpers.ResponseTopic, new ContainerResponse { ResponseStatusCode = 400, Message = ex.Message }, p); } } } } catch (OperationCanceledException) { } finally { // Ensure the consumer leaves the group cleanly and final offsets are committed. c.Close(); } } }
public async void CreateNewContainer(string serverRequestTopic, string parametersSerialized) { try { // Check that all parameters is present and is correct by deserializing if (parametersSerialized == null)// TODO: client error { return; } var parameters = JsonConvert.DeserializeObject <RunNewContainerParameters>(parametersSerialized); parameters.Action = ContainerActionType.RUN_NEW; await KafkaHelpers.SendMessageAsync(serverRequestTopic, parameters); } catch (Newtonsoft.Json.JsonException ex) { Console.WriteLine(parametersSerialized); Console.WriteLine("ERROR! : " + ex.Message); // TODO: client error } }
private static async Task SendOverViewData(IProducer <Null, string> p) { var latestRead = CreateOverViewData(); var latestSendTime = DateTime.Now; while (true) { new Task(async() => { var containerData = await FetchOverviewData(); if (ContainerHelpers.OverviewContainersAreDifferent(latestRead.Containers, containerData) || latestSendTime.AddMinutes(_sendInterval) < DateTime.Now) { latestRead.Containers = containerData; latestSendTime = DateTime.Now; await KafkaHelpers.SendMessageAsync(OverviewTopic, latestRead, p); } }).Start(); await Task.Delay(TimeSpan.FromSeconds(_checkInterval)); } }
public async static Task UpdateConfigContainer(UpdateConfigContainerParameters parameters, IProducer <Null, string> p) { try { CancellationToken cancellation = new CancellationToken(); await client.Containers.UpdateContainerAsync(parameters.ContainerId, new ContainerUpdateParameters { BlkioWeight = parameters.BlkioWeight, Memory = parameters.MemoryLimit, MemoryReservation = parameters.MemoryParameters, MemorySwap = parameters.MemorySwapLimit, KernelMemory = parameters.KernelMemory, // RestartPolicy = parameters.RestartPolicy, TODO: CPUShares = parameters.CPUShares, CPUPeriod = parameters.CPUPeriod, CpusetCpus = parameters.CPUSetCPUs, CpusetMems = parameters.CPUSetMems }, cancellation); // Updating the containers with the new configurations right away and sending it out var updatedContainers = await DockerUpdater.FetchStatsData(); var overviewContainerUpdate = DockerUpdater.CreateStatsData(updatedContainers); await KafkaHelpers.SendMessageAsync(DockerUpdater.StatsTopic, overviewContainerUpdate, p); // Send response await KafkaHelpers.SendMessageAsync(_responseTopic, new ContainerResponse { ResponseStatusCode = 200, Message = ResponseMessageContracts.CONTAINER_CONFIGURATION_UPDATED, ContainerIds = new string[] { parameters.ContainerId } }, p); } catch (DockerApiException ex) { await KafkaHelpers.SendMessageAsync(_responseTopic, new ContainerResponse { ResponseStatusCode = 400, Message = ex.Message, ContainerIds = new string[] { parameters.ContainerId } }, p); } }
public static void Start() { var producerConfig = new ProducerConfig { BootstrapServers = KafkaHelpers.BootstrapServers, Acks = Acks.Leader }; KafkaHelpers.SetKafkaConfigKerberos(producerConfig); using var p = new ProducerBuilder <Null, string>(producerConfig).Build(); Task[] tester = new Task[2]; if (_processesToStart.Contains("overviewdata")) { var overviewTask = SendOverViewData(p); tester[0] = overviewTask; } if (_processesToStart.Contains("statsdata")) { var statsTask = SendStatsData(p); tester[1] = statsTask; } Task.WaitAll(tester); }
private async static Task CallAction(ContainerActionType action, string jsonParameterString, IProducer <Null, String> p) { switch (action) { case ContainerActionType.RUN_NEW: var runNewParam = JsonConvert.DeserializeObject <RunNewContainerParameters>(jsonParameterString); await ContainerAction.RunNewContainer(runNewParam, p); break; case ContainerActionType.START: var startParam = JsonConvert.DeserializeObject <StartContainerParameters>(jsonParameterString); await ContainerAction.StartContainer(startParam, p); break; case ContainerActionType.STOP: var stopParam = JsonConvert.DeserializeObject <StopContainerParameters>(jsonParameterString); await ContainerAction.StopContainer(stopParam, p); break; case ContainerActionType.REMOVE: var removeParam = JsonConvert.DeserializeObject <RemoveContainerParameters>(jsonParameterString); await ContainerAction.RemoveContainer(removeParam, p); break; case ContainerActionType.RESTART: var restartParam = JsonConvert.DeserializeObject <RestartContainerParameters>(jsonParameterString); await ContainerAction.RestartContainer(restartParam, p); break; case ContainerActionType.RENAME: var parameters = JsonConvert.DeserializeObject <RenameContainerParameter>(jsonParameterString); await ContainerAction.RenameContainer(parameters, p); break; case ContainerActionType.UPDATE_CONFIGURATION: var updateParam = JsonConvert.DeserializeObject <UpdateConfigContainerParameters>(jsonParameterString); await ContainerAction.UpdateConfigContainer(updateParam, p); break; case ContainerActionType.REFETCH_OVERVIEW: await ContainerAction.RefetchOverviewData(p); break; case ContainerActionType.REFETCH_STATS: await ContainerAction.RefetchStatsData(p); break; case ContainerActionType.INSPECT: var inspectParam = JsonConvert.DeserializeObject <InspectContainerParameters>(jsonParameterString); await ContainerAction.InspectContainer(inspectParam, p); break; default: await KafkaHelpers.SendMessageAsync(KafkaHelpers.ResponseTopic, new ContainerResponse { ResponseStatusCode = 404, Message = ResponseMessageContracts.METHOD_CALL_NOT_VIABLE }, p); break; } }
public async void UpdateContainerConfiguration(string serverRequestTopic, string containerId) { await KafkaHelpers.SendMessageAsync(serverRequestTopic, new UpdateConfigContainerParameters { ContainerId = containerId, }); // TODO: }
public async void ListenForSubscriptionsActions(CancellationToken stoppingToken, List <CurrentConditionSubscription> currentConditionSubscriptions, List <ForecastSubscription> forecastSubscriptions) { // Listening and waiting for new subscription requests from kafka var consumerConfig = new ConsumerConfig { GroupId = "weather-station-new-subscription-group", AutoOffsetReset = AutoOffsetReset.Earliest, }; KafkaHelpers.SetKafkaConfigKerberos(consumerConfig); using (var consumer = new ConsumerBuilder <Ignore, string>(consumerConfig).Build()) { using (var scope = _services.CreateScope()) { var repo = scope.ServiceProvider.GetRequiredService <ISubscriptionRepo>(); consumer.Subscribe(KafkaHelpers.SubscriptionActionsTopic); _logger.LogInformation("Listening for subscription action requests"); while (!stoppingToken.IsCancellationRequested) { try { // consumer does not have an async method. So it is wrapped in a task, so that the rest of the application doesn't hang here var consumeResult = await Task.Factory.StartNew(() => consumer.Consume(stoppingToken)); var messageJsonString = consumeResult.Message.Value; // Checking format required. SubscriptionRequestDTO request = JsonConvert.DeserializeObject <SubscriptionRequestDTO>(messageJsonString); _logger.LogInformation("A subscription action request has arrived"); switch (request.Action) { case SubscriptionAction.CREATECURRENTCONDITION: var currentConditionRequest = JsonConvert.DeserializeObject <NewCurrentConditionSubscriptionRequestDTO>(messageJsonString); var currentConditionSubscription = new CurrentConditionSubscription { StationId = currentConditionRequest.StationId, IntervalSeconds = currentConditionRequest.IntervalSeconds }; // persisting the newly added subscription await repo.AddCurrentConditionSubscription(currentConditionSubscription); // adding it to the list of subscriptions currentConditionSubscriptions.Add(currentConditionSubscription); // starting the subscription. I want this to run in the background. So I discard the result with '_' _ = Task.Run(() => StartTimer(currentConditionSubscription)); break; case SubscriptionAction.CREATEFORECAST: var forecastSubscriptionRequest = JsonConvert.DeserializeObject <ForecastSubscription>(messageJsonString); var forecastSubscription = new ForecastSubscription { GeoCode = forecastSubscriptionRequest.GeoCode, IntervalSeconds = forecastSubscriptionRequest.IntervalSeconds }; // persisting the newly added subscription await repo.AddForecastSubscription(forecastSubscription); // adding it to the list of subscriptions forecastSubscriptions.Add(forecastSubscription); // starting the subscription. I want this to run in the background. So I discard the result with '_' _ = Task.Run(() => StartTimer(forecastSubscription)); break; case SubscriptionAction.DELETE: // TODO: Delete break; default: throw new Newtonsoft.Json.JsonException("Invalid Subscription action"); } } catch (ConsumeException ex) { _logger.LogError(ex.Error.ToString()); } // If the request from kafka is not correctly formatted an error will be thrown and catched here catch (Newtonsoft.Json.JsonException ex) { _logger.LogError(ex.Message); } // Cancelled background worker catch (OperationCanceledException) { consumer.Close(); } } } } // If we reach this point it means the application is shutting down. Therefore we clean up _logger.LogInformation("Cleaning up"); _producer.Flush(TimeSpan.FromSeconds(5)); _producer.Dispose(); }
public async void RenameContainer(string serverRequestTopic, string containerId, string newContainerName) { await KafkaHelpers.SendMessageAsync(serverRequestTopic, new RenameContainerParameter { ContainerId = containerId, NewName = newContainerName }); }
public async void RestartContainer(string serverRequestTopic, string containerId) { await KafkaHelpers.SendMessageAsync(serverRequestTopic, new RestartContainerParameters { ContainerId = containerId }); }
public async static Task RunNewContainer(RunNewContainerParameters parameters, IProducer <Null, string> p) { try { var exposedPorts = new Dictionary <string, EmptyStruct>(); var hostPortBindings = new Dictionary <string, IList <PortBinding> >(); if (parameters.Ports != null) { foreach (var portBinding in parameters.Ports) { exposedPorts.Add(portBinding.HostPort, default(EmptyStruct)); hostPortBindings.Add(portBinding.ContainerPort, new List <PortBinding> { new PortBinding { HostPort = portBinding.HostPort } }); } } RestartPolicy restartPolicy = new RestartPolicy { Name = RestartPolicyKind.No }; if (parameters.RestartPolicy != null) { restartPolicy.MaximumRetryCount = parameters.RestartPolicy.MaximumRetryCount ?? 0; switch (parameters.RestartPolicy.RestartPolicy) { case ContainerRestartPolicy.Always: restartPolicy.Name = RestartPolicyKind.Always; break; case ContainerRestartPolicy.OnFailure: restartPolicy.Name = RestartPolicyKind.OnFailure; break; case ContainerRestartPolicy.UnlessStopped: restartPolicy.Name = RestartPolicyKind.UnlessStopped; break; } } var environment = new List <string>(); if (parameters.Environment != null) { foreach (var environmentEntry in parameters.Environment) { environment.Add(environmentEntry.Key + "=" + environmentEntry.Value); } } var volumes = new List <string>(); if (parameters.Volumes != null) { foreach (var volumeEntry in parameters.Volumes) { volumes.Add(volumeEntry.HostPath + ":" + volumeEntry.ContainerPath); } } var dockerResponse = await client.Containers.CreateContainerAsync(new CreateContainerParameters { Image = parameters.Image, Cmd = parameters.Command == null ? null : parameters.Command.Split(" "), Name = parameters.Name, ExposedPorts = exposedPorts, HostConfig = new HostConfig { PortBindings = hostPortBindings, PublishAllPorts = true, RestartPolicy = restartPolicy, VolumesFrom = parameters.VolumesFrom, Binds = volumes, NetworkMode = parameters.NetworkMode, }, Env = environment, }); // Update general info as quickly as possible and send it out so clients can see the newly created container quickly var updatedContainers = await DockerUpdater.FetchOverviewData(); var overviewContainerUpdate = DockerUpdater.CreateOverViewData(updatedContainers); await KafkaHelpers.SendMessageAsync(DockerUpdater.OverviewTopic, overviewContainerUpdate, p); // Send response await KafkaHelpers.SendMessageAsync(_responseTopic, new ContainerResponse { ResponseStatusCode = 201, Message = ResponseMessageContracts.CONTAINER_CREATED, ContainerIds = new string[] { dockerResponse.ID } }, p); await StartContainer(new StartContainerParameters { ContainerId = dockerResponse.ID }, p); } catch (DockerApiException ex) { await KafkaHelpers.SendMessageAsync(_responseTopic, new ContainerResponse { ResponseStatusCode = 400, Message = ex.Message, ContainerIds = null }, p); // TODO: create contract for these error messages } }
public async void RemoveContainer(string serverRequestTopic, string containerId, bool removeVolumes) { await KafkaHelpers.SendMessageAsync(serverRequestTopic, new RemoveContainerParameters { ContainerId = containerId, RemoveVolumes = removeVolumes }); }