/// <summary> /// 运行kafka消费者 /// </summary> /// <param name="serviceScope"></param> /// <returns></returns> private IList <IStopable> StartConsumers(IServiceScope serviceScope) { var consumers = new List <IStopable>(); var kafkaSetting = serviceScope.ServiceProvider.GetService <KafkaSetting>(); // 盒子状态消费者 var kafkaBoxStateConsumerSetting = new KafkaConsumerSetting { BootstrapServers = kafkaSetting.BootstrapServers, GroupId = "BoxState" + kafkaSetting.ConsumerGroupPostFix, Topic = kafkaSetting.RequestStateTopic }; for (var i = 0; i < kafkaSetting.BoxStateConsumerCount; i++) { var boxStateConsumerManager = serviceScope.ServiceProvider.GetService <KafkaBoxStateConsumerManager>(); consumers.Add(boxStateConsumerManager); Task.Factory.StartNew(async() => await boxStateConsumerManager.Init(kafkaBoxStateConsumerSetting), TaskCreationOptions.LongRunning); } // 监控点kafka消费者 var kafkaDmonDataConsumerSetting = new KafkaConsumerSetting { BootstrapServers = kafkaSetting.BootstrapServers, GroupId = "DmonData" + kafkaSetting.ConsumerGroupPostFix, Topic = kafkaSetting.OpenOrCloseMonitorTopic }; for (var i = 0; i < kafkaSetting.OpenOrCloseMonitorDataConsumerCount; i++) { var dmonDataConsumerManager = serviceScope.ServiceProvider.GetService <KafkaMonitorDataConsumerManager>(); consumers.Add(dmonDataConsumerManager); Task.Factory.StartNew( async() => await dmonDataConsumerManager.Init(kafkaDmonDataConsumerSetting), TaskCreationOptions.LongRunning); } // 写值kafka消费者 var kafkaWriteDataConsumerSetting = new KafkaConsumerSetting { BootstrapServers = kafkaSetting.BootstrapServers, GroupId = "WriteData" + kafkaSetting.ConsumerGroupPostFix, Topic = kafkaSetting.WriteDataTopic }; for (var i = 0; i < kafkaSetting.WriteDataConsumerCount; i++) { var writeDataConsumerManager = serviceScope.ServiceProvider.GetService <KafkaWriteDataConsumerManager>(); consumers.Add(writeDataConsumerManager); Task.Factory.StartNew( async() => await writeDataConsumerManager.Init(kafkaWriteDataConsumerSetting), TaskCreationOptions.LongRunning); } return(consumers); }
public static ServiceProvider InjectService(KafkaConsumerSetting setting) { var serviceProvider = new ServiceCollection().AddSingleton <IAsyncClient, AsyncClient>(c => new AsyncClient(setting.AerospikeAddress, setting.AerospikePort)) .AddSingleton <IRedisCache, RedisCache>() .AddSingleton <ISegmentCache, SegmentCache>() .BuildServiceProvider(); return(serviceProvider); }
/// <summary> /// 初始化当前客户端 /// </summary> /// <param name="setting"></param> /// <returns></returns> public async Task Init(KafkaConsumerSetting setting) { var config = new ConsumerConfig { // 设置当前Kafka客户端分组, // 只要不更改group.id,每次重新消费kafka,都是从上次消费结束的地方继续开始 GroupId = setting.GroupId, BootstrapServers = setting.BootstrapServers, EnableAutoCommit = true, AutoOffsetReset = AutoOffsetReset.Earliest }; var consuming = true; var builder = new ConsumerBuilder <T1, T2>(config).SetErrorHandler((_, e) => { Logger.LogError($"Error: {e.Reason}"); consuming = !e.IsFatal; }); if (setting.ConsumeOffset != ConsumeOffset.Unspecified) { builder.SetPartitionsAssignedHandler((c, partitions) => { Logger.LogInformation($"Assigned partitions: [{string.Join(", ", partitions)}]"); return(partitions.Select(y => new TopicPartitionOffset(y, new Offset((long)setting.ConsumeOffset)))); }); } if (_keyDeserializer != null) { builder.SetKeyDeserializer(_keyDeserializer); } if (_valueDeserializer != null) { builder.SetValueDeserializer(_valueDeserializer); } using (_consumer = builder.Build()) { _consumer.Subscribe(setting.Topic); while (consuming && !_token.IsCancellationRequested) { try { var cr = _consumer.Consume(_token); try { await Consume(cr.Key, cr.Value, cr.Timestamp.UtcDateTime); } catch (Exception e) { Logger.LogError("KafkaConsumerManager.Consume error", e); } } catch (ConsumeException e) { Logger.LogError($"Error occured: {e.Error.Reason}"); } } } }
public static async Task DoJob(KafkaConsumerSetting setting) { var services = InjectService(setting); var _client = services.GetService <IAsyncClient>(); var segmentCache = services.GetService <ISegmentCache>(); var conf = new ConsumerConfig { GroupId = "default", BootstrapServers = $"{setting.KafkaAddress}:{setting.KafkaPort}", AutoOffsetReset = AutoOffsetReset.Earliest, }; using (var c = new ConsumerBuilder <Ignore, string>(conf).Build()) { c.Subscribe("Sticky"); #pragma warning disable IDE0067 // Dispose objects before losing scope CancellationTokenSource cts = new CancellationTokenSource(); #pragma warning restore IDE0067 // Dispose objects before losing scope Console.CancelKeyPress += (_, e) => { e.Cancel = true; // prevent the process from terminating. cts.Cancel(); }; try { var counter = 0; var pagedSegments = (await segmentCache.GetAllActiveSegments()).Where(x => x.AudienceId == 1).ToList(); while (true) { try { var cr = c.Consume(cts.Token); counter++; if (counter > 1000 && counter % 1000 == 0) { pagedSegments = (await segmentCache.GetAllActiveSegments()).Where(x => x.AudienceId == 1 && !string.IsNullOrEmpty(x.AudienceExtra)).ToList(); } DruidData data = Newtonsoft.Json.JsonConvert.DeserializeObject <DruidData>(cr.Value); if (data != null) { Key matchKey = new Key("Sticky", "Activity", $"{ data.UserId }_{data.HostId}"); var record = _client.Get(new Policy(), matchKey); if (!string.IsNullOrEmpty(data.ProductId) && data.StatType != "PageView") { if (record == null) { var bin = new Bin(data.StatType, data.ProductId); _client.Put(new WritePolicy() { }, matchKey, bin); } else { var listStr = record.GetString(data.StatType); listStr += "," + data.ProductId; var list = listStr.Split(",").ToList(); while (list.Count() > setting.MaxProductRecordPerUser) { list.RemoveAt(0); } var bin = new Bin(data.StatType, string.Join(",", list)); _client.Put(new WritePolicy(), matchKey, bin); } Console.WriteLine(data.UserId); } else if (data.UserId == "PageView") { var pageSegmentIds = new List <string>(); if (record != null) { var membershipString = record.GetString("PagedMemberships"); if (!string.IsNullOrEmpty(membershipString)) { pageSegmentIds = membershipString.Split(",").ToList(); } } var hostPageSegment = pagedSegments.Where(x => x.HostId.ToString() == data.HostId).ToList(); foreach (var item in hostPageSegment) { if (data.PageAddress != null && data.PageAddress.Contains(item.AudienceExtra) && !pageSegmentIds.Contains(item.Id.ToString())) { pageSegmentIds.Add(item.Id.ToString()); } } var bin = new Bin("PagedMemberships", string.Join(",", pageSegmentIds)); _client.Put(new WritePolicy(), matchKey, bin); } } } catch (Exception ex) { Console.Write(ex.Message); } } } catch (OperationCanceledException) { // Ensure the consumer leaves the group cleanly and final offsets are committed. c.Close(); } } }