public async Task <IEnumerable <MessageModel> > GetMessages(string environment) { List <MessageModel> toReturn = new List <MessageModel>(); // Get Kafka messages List <Kafka.Models.KafkaMessageModel> kafkaMessages = new List <Kafka.Models.KafkaMessageModel>(); KafkaConfigurationModel kConfig = this.GetKafkaConfiguration(environment); Kafka.MessageProvider kmProvider = new Kafka.MessageProvider() { GroupId = kConfig.GroupId, BootstrapServers = kConfig.BootstrapServers, TimeoutSeconds = kConfig.TimeoutSeconds }; await kConfig.Topics.ForEachAsync(async t => kafkaMessages.AddRange(await kmProvider.GetMessages(t, kConfig.MaxMessageCount))); // Validate messages ValidatorProvider vProvider = new ValidatorProvider(); toReturn.AddRange(kafkaMessages.Select(k => MessageModel.FromKafkaMessage(k, vProvider.Validate(k.Value)))); // Get elastic messages ElasticConfigurationModel eConfig = this.GetElasticConfiguration(environment); Elastic.MessageProvider emProvider = new Elastic.MessageProvider() { Nodes = eConfig.Nodes, MaxValuesPerQuery = eConfig.MaxValuesPerQuery }; // Check action ID for those found in kafka IEnumerable <Core.MessageModel> actionIdMessages = await emProvider.GetMessages(m => m.ActionId, toReturn.Where(m => m.FoundInKafka).Select(m => m.Event.ActionId).ToList(), eConfig.EnvironmentTldCsv); actionIdMessages .ForEach(em => { MessageModel m = toReturn.FirstOrDefault(m => m.FoundInKafka && m.Event.ActionId == em.ActionId); if (m != null) { m.FoundInElastic = true; m.KibanaUrl = MessageModel.GetKibanaUrl(em, eConfig); } }); // Add in messages missing in Kafka, and get events from elastic EventModel[] allEvents = this.Configuration.GetSection("Events").Get <EventModel[]>(); int[] currentEventIds = toReturn.Select(m => m.Event.Id).Distinct().ToArray(); await allEvents.Where(e => !currentEventIds.Contains(e.Id)).ForEachAsync(async e => { IEnumerable <Core.MessageModel> elasticMessages = await emProvider.GetMessages(m => m.EventId, new List <int>() { e.Id }, eConfig.EnvironmentTldCsv, this.Configuration.GetSection("NumberOfElasticEventsForMissingFromKafka").Get <int>()); if (!elasticMessages.Any()) { toReturn.Add(new MessageModel(e)); } else { toReturn.AddRange(elasticMessages.Select(m => MessageModel.FromElasticMessage(m, vProvider.Validate(m), eConfig))); } }); return(toReturn); }