/// <summary>
/// 运行kafka消费者
/// </summary>
/// <param name="serviceScope"></param>
/// <returns></returns>
private IList <IStopable> StartConsumers(IServiceScope serviceScope)
{
var consumers = new List <IStopable>();
var kafkaSetting = serviceScope.ServiceProvider.GetService <KafkaSetting>();
// 盒子状态消费者
var kafkaBoxStateConsumerSetting = new KafkaConsumerSetting
{
BootstrapServers = kafkaSetting.BootstrapServers,
GroupId = "BoxState" + kafkaSetting.ConsumerGroupPostFix,
Topic = kafkaSetting.RequestStateTopic
};
for (var i = 0; i < kafkaSetting.BoxStateConsumerCount; i++)
{
var boxStateConsumerManager = serviceScope.ServiceProvider.GetService <KafkaBoxStateConsumerManager>();
consumers.Add(boxStateConsumerManager);
Task.Factory.StartNew(async() => await boxStateConsumerManager.Init(kafkaBoxStateConsumerSetting),
TaskCreationOptions.LongRunning);
}
// 监控点kafka消费者
var kafkaDmonDataConsumerSetting = new KafkaConsumerSetting
{
BootstrapServers = kafkaSetting.BootstrapServers,
GroupId = "DmonData" + kafkaSetting.ConsumerGroupPostFix,
Topic = kafkaSetting.OpenOrCloseMonitorTopic
};
for (var i = 0; i < kafkaSetting.OpenOrCloseMonitorDataConsumerCount; i++)
{
var dmonDataConsumerManager = serviceScope.ServiceProvider.GetService <KafkaMonitorDataConsumerManager>();
consumers.Add(dmonDataConsumerManager);
Task.Factory.StartNew(
async() => await dmonDataConsumerManager.Init(kafkaDmonDataConsumerSetting),
TaskCreationOptions.LongRunning);
}
// 写值kafka消费者
var kafkaWriteDataConsumerSetting = new KafkaConsumerSetting
{
BootstrapServers = kafkaSetting.BootstrapServers,
GroupId = "WriteData" + kafkaSetting.ConsumerGroupPostFix,
Topic = kafkaSetting.WriteDataTopic
};
for (var i = 0; i < kafkaSetting.WriteDataConsumerCount; i++)
{
var writeDataConsumerManager = serviceScope.ServiceProvider.GetService <KafkaWriteDataConsumerManager>();
consumers.Add(writeDataConsumerManager);
Task.Factory.StartNew(
async() => await writeDataConsumerManager.Init(kafkaWriteDataConsumerSetting),
TaskCreationOptions.LongRunning);
}
return(consumers);
}
public static ServiceProvider InjectService(KafkaConsumerSetting setting)
{
var serviceProvider = new ServiceCollection().AddSingleton <IAsyncClient, AsyncClient>(c => new AsyncClient(setting.AerospikeAddress, setting.AerospikePort))
.AddSingleton <IRedisCache, RedisCache>()
.AddSingleton <ISegmentCache, SegmentCache>()
.BuildServiceProvider();
return(serviceProvider);
}
/// <summary>
/// 初始化当前客户端
/// </summary>
/// <param name="setting"></param>
/// <returns></returns>
public async Task Init(KafkaConsumerSetting setting)
{
var config = new ConsumerConfig
{
// 设置当前Kafka客户端分组,
// 只要不更改group.id，每次重新消费kafka，都是从上次消费结束的地方继续开始
GroupId = setting.GroupId,
BootstrapServers = setting.BootstrapServers,
EnableAutoCommit = true,
AutoOffsetReset = AutoOffsetReset.Earliest
};
var consuming = true;
var builder = new ConsumerBuilder <T1, T2>(config).SetErrorHandler((_, e) =>
{
Logger.LogError($"Error: {e.Reason}");
consuming = !e.IsFatal;
});
if (setting.ConsumeOffset != ConsumeOffset.Unspecified)
{
builder.SetPartitionsAssignedHandler((c, partitions) =>
{
Logger.LogInformation($"Assigned partitions: [{string.Join(", ", partitions)}]");
return(partitions.Select(y =>
new TopicPartitionOffset(y, new Offset((long)setting.ConsumeOffset))));
});
}
if (_keyDeserializer != null)
{
builder.SetKeyDeserializer(_keyDeserializer);
}
if (_valueDeserializer != null)
{
builder.SetValueDeserializer(_valueDeserializer);
}
using (_consumer = builder.Build())
{
_consumer.Subscribe(setting.Topic);
while (consuming && !_token.IsCancellationRequested)
{
try
{
var cr = _consumer.Consume(_token);
try
{
await Consume(cr.Key, cr.Value, cr.Timestamp.UtcDateTime);
}
catch (Exception e)
{
Logger.LogError("KafkaConsumerManager.Consume error", e);
}
}
catch (ConsumeException e)
{
Logger.LogError($"Error occured: {e.Error.Reason}");
}
}
}
}
public static async Task DoJob(KafkaConsumerSetting setting)
{
var services = InjectService(setting);
var _client = services.GetService <IAsyncClient>();
var segmentCache = services.GetService <ISegmentCache>();
var conf = new ConsumerConfig
{
GroupId = "default",
BootstrapServers = $"{setting.KafkaAddress}:{setting.KafkaPort}",
AutoOffsetReset = AutoOffsetReset.Earliest,
};
using (var c = new ConsumerBuilder <Ignore, string>(conf).Build())
{
c.Subscribe("Sticky");
#pragma warning disable IDE0067 // Dispose objects before losing scope
CancellationTokenSource cts = new CancellationTokenSource();
#pragma warning restore IDE0067 // Dispose objects before losing scope
Console.CancelKeyPress += (_, e) =>
{
e.Cancel = true; // prevent the process from terminating.
cts.Cancel();
};
try
{
var counter = 0;
var pagedSegments = (await segmentCache.GetAllActiveSegments()).Where(x => x.AudienceId == 1).ToList();
while (true)
{
try
{
var cr = c.Consume(cts.Token);
counter++;
if (counter > 1000 && counter % 1000 == 0)
{
pagedSegments = (await segmentCache.GetAllActiveSegments()).Where(x => x.AudienceId == 1 && !string.IsNullOrEmpty(x.AudienceExtra)).ToList();
}
DruidData data = Newtonsoft.Json.JsonConvert.DeserializeObject <DruidData>(cr.Value);
if (data != null)
{
Key matchKey = new Key("Sticky", "Activity", $"{ data.UserId }_{data.HostId}");
var record = _client.Get(new Policy(), matchKey);
if (!string.IsNullOrEmpty(data.ProductId) && data.StatType != "PageView")
{
if (record == null)
{
var bin = new Bin(data.StatType, data.ProductId);
_client.Put(new WritePolicy()
{
}, matchKey, bin);
}
else
{
var listStr = record.GetString(data.StatType);
listStr += "," + data.ProductId;
var list = listStr.Split(",").ToList();
while (list.Count() > setting.MaxProductRecordPerUser)
{
list.RemoveAt(0);
}
var bin = new Bin(data.StatType, string.Join(",", list));
_client.Put(new WritePolicy(), matchKey, bin);
}
Console.WriteLine(data.UserId);
}
else if (data.UserId == "PageView")
{
var pageSegmentIds = new List <string>();
if (record != null)
{
var membershipString = record.GetString("PagedMemberships");
if (!string.IsNullOrEmpty(membershipString))
{
pageSegmentIds = membershipString.Split(",").ToList();
}
}
var hostPageSegment = pagedSegments.Where(x => x.HostId.ToString() == data.HostId).ToList();
foreach (var item in hostPageSegment)
{
if (data.PageAddress != null && data.PageAddress.Contains(item.AudienceExtra) && !pageSegmentIds.Contains(item.Id.ToString()))
{
pageSegmentIds.Add(item.Id.ToString());
}
}
var bin = new Bin("PagedMemberships", string.Join(",", pageSegmentIds));
_client.Put(new WritePolicy(), matchKey, bin);
}
}
}
catch (Exception ex)
{
Console.Write(ex.Message);
}
}
}
catch (OperationCanceledException)
{
// Ensure the consumer leaves the group cleanly and final offsets are committed.
c.Close();
}
}
}