Exemplo n.º 1
0
        /// <summary>
        ///     运行kafka消费者
        /// </summary>
        /// <param name="serviceScope"></param>
        /// <returns></returns>
        private IList <IStopable> StartConsumers(IServiceScope serviceScope)
        {
            var consumers    = new List <IStopable>();
            var kafkaSetting = serviceScope.ServiceProvider.GetService <KafkaSetting>();

            //    盒子状态消费者
            var kafkaBoxStateConsumerSetting = new KafkaConsumerSetting
            {
                BootstrapServers = kafkaSetting.BootstrapServers,
                GroupId          = "BoxState" + kafkaSetting.ConsumerGroupPostFix,
                Topic            = kafkaSetting.RequestStateTopic
            };

            for (var i = 0; i < kafkaSetting.BoxStateConsumerCount; i++)
            {
                var boxStateConsumerManager = serviceScope.ServiceProvider.GetService <KafkaBoxStateConsumerManager>();
                consumers.Add(boxStateConsumerManager);
                Task.Factory.StartNew(async() => await boxStateConsumerManager.Init(kafkaBoxStateConsumerSetting),
                                      TaskCreationOptions.LongRunning);
            }

            //    监控点kafka消费者
            var kafkaDmonDataConsumerSetting = new KafkaConsumerSetting
            {
                BootstrapServers = kafkaSetting.BootstrapServers,
                GroupId          = "DmonData" + kafkaSetting.ConsumerGroupPostFix,
                Topic            = kafkaSetting.OpenOrCloseMonitorTopic
            };

            for (var i = 0; i < kafkaSetting.OpenOrCloseMonitorDataConsumerCount; i++)
            {
                var dmonDataConsumerManager = serviceScope.ServiceProvider.GetService <KafkaMonitorDataConsumerManager>();
                consumers.Add(dmonDataConsumerManager);
                Task.Factory.StartNew(
                    async() => await dmonDataConsumerManager.Init(kafkaDmonDataConsumerSetting),
                    TaskCreationOptions.LongRunning);
            }

            //    写值kafka消费者
            var kafkaWriteDataConsumerSetting = new KafkaConsumerSetting
            {
                BootstrapServers = kafkaSetting.BootstrapServers,
                GroupId          = "WriteData" + kafkaSetting.ConsumerGroupPostFix,
                Topic            = kafkaSetting.WriteDataTopic
            };

            for (var i = 0; i < kafkaSetting.WriteDataConsumerCount; i++)
            {
                var writeDataConsumerManager = serviceScope.ServiceProvider.GetService <KafkaWriteDataConsumerManager>();
                consumers.Add(writeDataConsumerManager);
                Task.Factory.StartNew(
                    async() => await writeDataConsumerManager.Init(kafkaWriteDataConsumerSetting),
                    TaskCreationOptions.LongRunning);
            }

            return(consumers);
        }
Exemplo n.º 2
0
        public static ServiceProvider InjectService(KafkaConsumerSetting setting)
        {
            var serviceProvider = new ServiceCollection().AddSingleton <IAsyncClient, AsyncClient>(c => new AsyncClient(setting.AerospikeAddress, setting.AerospikePort))
                                  .AddSingleton <IRedisCache, RedisCache>()
                                  .AddSingleton <ISegmentCache, SegmentCache>()
                                  .BuildServiceProvider();

            return(serviceProvider);
        }
        /// <summary>
        ///     初始化当前客户端
        /// </summary>
        /// <param name="setting"></param>
        /// <returns></returns>
        public async Task Init(KafkaConsumerSetting setting)
        {
            var config = new ConsumerConfig
            {
                //    设置当前Kafka客户端分组,
                //    只要不更改group.id,每次重新消费kafka,都是从上次消费结束的地方继续开始
                GroupId          = setting.GroupId,
                BootstrapServers = setting.BootstrapServers,
                EnableAutoCommit = true,
                AutoOffsetReset  = AutoOffsetReset.Earliest
            };
            var consuming = true;
            var builder   = new ConsumerBuilder <T1, T2>(config).SetErrorHandler((_, e) =>
            {
                Logger.LogError($"Error: {e.Reason}");
                consuming = !e.IsFatal;
            });

            if (setting.ConsumeOffset != ConsumeOffset.Unspecified)
            {
                builder.SetPartitionsAssignedHandler((c, partitions) =>
                {
                    Logger.LogInformation($"Assigned partitions: [{string.Join(", ", partitions)}]");
                    return(partitions.Select(y =>
                                             new TopicPartitionOffset(y, new Offset((long)setting.ConsumeOffset))));
                });
            }

            if (_keyDeserializer != null)
            {
                builder.SetKeyDeserializer(_keyDeserializer);
            }
            if (_valueDeserializer != null)
            {
                builder.SetValueDeserializer(_valueDeserializer);
            }

            using (_consumer = builder.Build())
            {
                _consumer.Subscribe(setting.Topic);
                while (consuming && !_token.IsCancellationRequested)
                {
                    try
                    {
                        var cr = _consumer.Consume(_token);
                        try
                        {
                            await Consume(cr.Key, cr.Value, cr.Timestamp.UtcDateTime);
                        }
                        catch (Exception e)
                        {
                            Logger.LogError("KafkaConsumerManager.Consume error", e);
                        }
                    }
                    catch (ConsumeException e)
                    {
                        Logger.LogError($"Error occured: {e.Error.Reason}");
                    }
                }
            }
        }
Exemplo n.º 4
0
        public static async Task DoJob(KafkaConsumerSetting setting)
        {
            var services     = InjectService(setting);
            var _client      = services.GetService <IAsyncClient>();
            var segmentCache = services.GetService <ISegmentCache>();

            var conf = new ConsumerConfig
            {
                GroupId          = "default",
                BootstrapServers = $"{setting.KafkaAddress}:{setting.KafkaPort}",
                AutoOffsetReset  = AutoOffsetReset.Earliest,
            };

            using (var c = new ConsumerBuilder <Ignore, string>(conf).Build())
            {
                c.Subscribe("Sticky");

#pragma warning disable IDE0067 // Dispose objects before losing scope
                CancellationTokenSource cts = new CancellationTokenSource();
#pragma warning restore IDE0067 // Dispose objects before losing scope
                Console.CancelKeyPress += (_, e) =>
                {
                    e.Cancel = true; // prevent the process from terminating.
                    cts.Cancel();
                };

                try
                {
                    var counter       = 0;
                    var pagedSegments = (await segmentCache.GetAllActiveSegments()).Where(x => x.AudienceId == 1).ToList();

                    while (true)
                    {
                        try
                        {
                            var cr = c.Consume(cts.Token);
                            counter++;
                            if (counter > 1000 && counter % 1000 == 0)
                            {
                                pagedSegments = (await segmentCache.GetAllActiveSegments()).Where(x => x.AudienceId == 1 && !string.IsNullOrEmpty(x.AudienceExtra)).ToList();
                            }
                            DruidData data = Newtonsoft.Json.JsonConvert.DeserializeObject <DruidData>(cr.Value);
                            if (data != null)
                            {
                                Key matchKey = new Key("Sticky", "Activity", $"{ data.UserId }_{data.HostId}");
                                var record   = _client.Get(new Policy(), matchKey);
                                if (!string.IsNullOrEmpty(data.ProductId) && data.StatType != "PageView")
                                {
                                    if (record == null)
                                    {
                                        var bin = new Bin(data.StatType, data.ProductId);
                                        _client.Put(new WritePolicy()
                                        {
                                        }, matchKey, bin);
                                    }
                                    else
                                    {
                                        var listStr = record.GetString(data.StatType);
                                        listStr += "," + data.ProductId;
                                        var list = listStr.Split(",").ToList();
                                        while (list.Count() > setting.MaxProductRecordPerUser)
                                        {
                                            list.RemoveAt(0);
                                        }

                                        var bin = new Bin(data.StatType, string.Join(",", list));

                                        _client.Put(new WritePolicy(), matchKey, bin);
                                    }
                                    Console.WriteLine(data.UserId);
                                }
                                else if (data.UserId == "PageView")
                                {
                                    var pageSegmentIds = new List <string>();
                                    if (record != null)
                                    {
                                        var membershipString = record.GetString("PagedMemberships");
                                        if (!string.IsNullOrEmpty(membershipString))
                                        {
                                            pageSegmentIds = membershipString.Split(",").ToList();
                                        }
                                    }
                                    var hostPageSegment = pagedSegments.Where(x => x.HostId.ToString() == data.HostId).ToList();
                                    foreach (var item in hostPageSegment)
                                    {
                                        if (data.PageAddress != null && data.PageAddress.Contains(item.AudienceExtra) && !pageSegmentIds.Contains(item.Id.ToString()))
                                        {
                                            pageSegmentIds.Add(item.Id.ToString());
                                        }
                                    }
                                    var bin = new Bin("PagedMemberships", string.Join(",", pageSegmentIds));
                                    _client.Put(new WritePolicy(), matchKey, bin);
                                }
                            }
                        }
                        catch (Exception ex)
                        {
                            Console.Write(ex.Message);
                        }
                    }
                }
                catch (OperationCanceledException)
                {
                    // Ensure the consumer leaves the group cleanly and final offsets are committed.
                    c.Close();
                }
            }
        }