public KafkaTopicHandler(string zkConnectionStr)
{
Client = new KafkaClient(zkConnectionStr);
Topics = new ConcurrentDictionary <KafkaTopicType, IKafkaTopic>();
PushQueue = new BlockingCollection <KafkaTopicEventData>(new ConcurrentQueue <KafkaTopicEventData>());
Task.Factory.StartNew(() =>
{
foreach (var eventData in PushQueue.GetConsumingEnumerable())
{
try
{
var topic = Topics.GetOrAdd(eventData.TopicType, type => Client.Topic(type.ToString().ToLowerInvariant()));
var data = eventData.Data as string ?? eventData.Data.ToNewtonsoftJson();
topic.Send(new Message
{
Key = eventData.Key ?? string.Empty,
Value = data,
Codec = eventData.Codec
});
}
catch (Exception e)
{
// 时长事件不补发
if (eventData.TopicType != KafkaTopicType.MissionEventOnline)
{
_listKafkaEventDataMissedHandler.LeftPush(null, eventData.ToJson());
}
_logger.Error("KafkaTopicHandler", e);
}
}
}, TaskCreationOptions.LongRunning);
}
public int Start(SimpleConsumerOptions opts)
{
var zookeeperString = opts.ZkConnect;
var testTopic = opts.Topic;
var partition = opts.Partition;
var partitions = partition == null
? null
: new[] { partition.Value };
var timer = Metric.Timer("Received", Unit.Events);
Metric.Config.WithReporting(r => r.WithConsoleReport(TimeSpan.FromSeconds(5)));
using (var client = new KafkaClient(zookeeperString))
using (var consumer = new Kafka.Basic.SimpleConsumer(client, testTopic, partitions))
{
ListenToConsole(consumer);
consumer.Start(
message =>
{
var time = DateTime.UtcNow.Ticks;
var value = long.Parse(message.Value);
var diff = (time - value) / 10000;
timer.Record(diff, TimeUnit.Milliseconds);
},
e => Console.Error.WriteLine(e.Message)
);
}
return(0);
}
public TTSettlesSubscriber(KafkaClient awsKafkaClient, KafkaClient njKafkaClient, WorkerDispatcher dispatcher, InstrumentType instrumentType)
{
this._awsKafkaClient = awsKafkaClient;
this._njKafkaClient = njKafkaClient;
this._dispatcher = dispatcher;
this._instrumentType = instrumentType;
}
private static void listInstance(KafkaClient client)
{
var req = new ListInstancesRequest
{
Engine = ListInstancesRequest.EngineEnum.KAFKA,
};
try
{
var resp = client.ListInstances(req);
Console.WriteLine(resp.GetHttpBody());
}
catch (RequestTimeoutException requestTimeoutException)
{
Console.WriteLine(requestTimeoutException.ErrorMessage);
}
catch (ServiceResponseException clientRequestException)
{
Console.WriteLine(clientRequestException.HttpStatusCode);
Console.WriteLine(clientRequestException.ErrorCode);
Console.WriteLine(clientRequestException.ErrorMsg);
}
catch (ConnectionException connectionException)
{
Console.WriteLine(connectionException.ErrorMessage);
}
}
private static void resetPassword(KafkaClient client)
{
var req = new ResetPasswordRequest
{
InstanceId = instanceId,
Body = new ResetPasswordReq
{
NewPassword = "******",
}
};
try
{
var resp = client.ResetPassword(req);
Console.WriteLine(resp.GetHttpBody());
}
catch (RequestTimeoutException requestTimeoutException)
{
Console.WriteLine(requestTimeoutException.ErrorMessage);
}
catch (ServiceResponseException clientRequestException)
{
Console.WriteLine(clientRequestException.HttpStatusCode);
Console.WriteLine(clientRequestException.ErrorCode);
Console.WriteLine(clientRequestException.ErrorMsg);
}
catch (ConnectionException connectionException)
{
Console.WriteLine(connectionException.ErrorMessage);
}
}
private static void batchDeleteInstanceTopic(KafkaClient client)
{
var req = new BatchDeleteInstanceTopicRequest()
{
InstanceId = instanceId,
Body = new BatchDeleteInstanceTopicReq
{
Topics = new List <string>()
{
"topic-test"
},
}
};
try
{
var resp = client.BatchDeleteInstanceTopic(req);
Console.WriteLine(resp.GetHttpBody());
}
catch (RequestTimeoutException requestTimeoutException)
{
Console.WriteLine(requestTimeoutException.ErrorMessage);
}
catch (ServiceResponseException clientRequestException)
{
Console.WriteLine(clientRequestException.HttpStatusCode);
Console.WriteLine(clientRequestException.ErrorCode);
Console.WriteLine(clientRequestException.ErrorMsg);
}
catch (ConnectionException connectionException)
{
Console.WriteLine(connectionException.ErrorMessage);
}
}
private static void createInstanceTopic(KafkaClient client)
{
var req = new CreateInstanceTopicRequest
{
InstanceId = instanceId,
Body = new CreateInstanceTopicReq
{
Name = "topic-test",
}
};
try
{
var resp = client.CreateInstanceTopic(req);
Console.WriteLine(resp.GetHttpBody());
}
catch (RequestTimeoutException requestTimeoutException)
{
Console.WriteLine(requestTimeoutException.ErrorMessage);
}
catch (ServiceResponseException clientRequestException)
{
Console.WriteLine(clientRequestException.HttpStatusCode);
Console.WriteLine(clientRequestException.ErrorCode);
Console.WriteLine(clientRequestException.ErrorMsg);
}
catch (ConnectionException connectionException)
{
Console.WriteLine(connectionException.ErrorMessage);
}
}
private static void batchRestartOrDeleteInstances(KafkaClient client)
{
var req = new BatchRestartOrDeleteInstancesRequest
{
Body = new BatchRestartOrDeleteInstanceReq
{
Action = BatchRestartOrDeleteInstanceReq.ActionEnum.DELETE,
Instances = new List <string>()
{
instanceId
},
}
};
try
{
var resp = client.BatchRestartOrDeleteInstances(req);
Console.WriteLine(resp.GetHttpBody());
}
catch (RequestTimeoutException requestTimeoutException)
{
Console.WriteLine(requestTimeoutException.ErrorMessage);
}
catch (ServiceResponseException clientRequestException)
{
Console.WriteLine(clientRequestException.HttpStatusCode);
Console.WriteLine(clientRequestException.ErrorCode);
Console.WriteLine(clientRequestException.ErrorMsg);
}
catch (ConnectionException connectionException)
{
Console.WriteLine(connectionException.ErrorMessage);
}
}
protected override void ConfigureApplicationContainer(TinyIoCContainer container)
{
base.ConfigureApplicationContainer(container);
var kafkaClient = new KafkaClient("localhost:9092", "Kafka.Demo.Api");
container.Register <IKafkaClient, KafkaClient>(kafkaClient);
}
public RDFServClient(string kafkaServer, string kafkaClientId, string serverTopic)
{
ServerTopic = serverTopic;
Client = new KafkaClient(kafkaClientId, kafkaServer);
QueryResultPath = ServerTopic + "_" + kafkaClientId + "_response_query";
QueryPath = ServerTopic + "_query";
AddServiceResultPath = ServerTopic + "_" + kafkaClientId + "_response_addservice";
AddServicePath = ServerTopic + "_add-service";
}
private async Task When_a_capability_is_created()
{
var kafkaClient = new KafkaClient();
var message = new
{
CapabilityId = Guid.NewGuid().ToString(),
CapabilityName = "VerbNoun",
};
await kafkaClient.SendMessageAsync(message);
}
public void CreateTopicTest()
{
try
{
KafkaClient client = new KafkaClient(_zkConnection);
client.CreateTopic("testtopic1");
}
catch (Exception ex)
{
Console.WriteLine(ex.GetBaseException().Message);
throw;
}
}
private static void InstanceMgtMain(string[] args)
{
var config = HttpConfig.GetDefaultConfig();
config.IgnoreSslVerification = true;
var auth = new BasicCredentials(ak, sk, projectId);
var kafkaClient = KafkaClient.NewBuilder()
.WithCredential(auth)
.WithEndPoint(endpoint)
.WithHttpConfig(config).Build();
resetPassword(kafkaClient);
}
public static void Main(string[] args)
{
AutoResetEvent waitForTTInit = new AutoResetEvent(false);
Action <string> onInit = str => {
Console.WriteLine(str);
waitForTTInit.Set();
};
WorkerDispatcher dispatcher = new WorkerDispatcher();
TTDriver driver = new TTDriver(onInit, dispatcher);
driver.start();
waitForTTInit.WaitOne();
var aws_bootstrap_servers =
"b-1.kafka01.mb3vi8.c4.kafka.us-east-1.amazonaws.com:9092,b-2.kafka01.mb3vi8.c4.kafka.us-east-1.amazonaws.com:9092,b-3.kafka01.mb3vi8.c4.kafka.us-east-1.amazonaws.com:9092";
var nj_bootstrap_servers = "njxmd01.hetco.com:9092";
var awsKafkaClient = new KafkaClient(aws_bootstrap_servers);
var njKafkaClient = new KafkaClient(nj_bootstrap_servers);
// kafkaClient.Run();
Parser.Default.ParseArguments <Options>(args)
.WithParsed <Options>(o =>
{
if (o.LiveFutures)
{
var liveFuturesSubscriber = new LiveFuturesSubscriber(awsKafkaClient, njKafkaClient, dispatcher);
liveFuturesSubscriber.start();
}
else if (o.SettlesFutures)
{
var ttSettlesFuturesSubscriber = new TTSettlesSubscriber(awsKafkaClient, njKafkaClient, dispatcher, InstrumentType.Future);
ttSettlesFuturesSubscriber.start();
}
else if (o.SettlesOptions)
{
var ttSettlesOptionsSubscriber =
new TTSettlesSubscriber(awsKafkaClient, njKafkaClient, dispatcher, InstrumentType.Option);
ttSettlesOptionsSubscriber.start();
}
else
{
Console.WriteLine($"Choose one of the options (-f -l -l -o)");
Environment.Exit(-1);
}
});
Console.WriteLine("{0}", Thread.CurrentThread.ManagedThreadId);
Thread.Sleep(Timeout.Infinite);
}
public async Task StartProducing(int messagesToProduce)
{
var kafkaBootstrapServer = "kafka:9092";
var topicConfiguration = new Dictionary <string, object>
{
{ "request.required.acks", 1 },
{ "request.timeout.ms", 2000 },
{ "message.timeout.ms", 2000 },
};
// Create the producer configuration
var configuration = new Dictionary <string, object>
{
{ "api.version.request", true },
{ "bootstrap.servers", kafkaBootstrapServer },
{ "default.topic.config", topicConfiguration }
};
// Create a new topic producer
var kafkaProducer = KafkaClient.Connect(SchemaRegistryConnection, configuration).GetTopicProducer(Topic, true);
try
{
var transaction = CreateTransctionModel();
var message = await kafkaProducer.ProduceAsync(transaction.CustomerId, transaction);
}
catch (Exception e)
{
Console.WriteLine(e);
Console.WriteLine("Something went wrong aborting");
return;
}
// create 10 tasks to produce messages
var tasks = Enumerable.Range(0, 10)
.Select(x =>
{
var i1 = messagesToProduce / 10;
var dave = new Task[i1];
for (var i = 0; i < i1; i++)
{
var transaction = CreateTransctionModel();
dave[i] = kafkaProducer.ProduceAsync(transaction.CustomerId, transaction);
}
return(Task.WhenAll(dave));
});
//wait for everything to produce
await Task.WhenAll(tasks);
}
private static int Run(Options options)
{
var timer = Metric.Timer("Sent", Unit.Events);
Metric.Config.WithReporting(r => r.WithConsoleReport(TimeSpan.FromSeconds(5)));
var client = new KafkaClient(options.ZkConnect);
var topic = client.Topic(options.Topic);
var published = 0;
var stop = false;
Console.CancelKeyPress += (sender, eventArgs) =>
{
eventArgs.Cancel = true;
stop = true;
};
var start = DateTime.Now;
while (!stop)
{
Thread.Sleep(options.Sleep);
var batch =
Enumerable.Range(0, options.BatchSize)
.Select(i =>
new Message
{
Key = (published + i).ToString(),
Value = DateTime.UtcNow.Ticks.ToString()
})
.ToArray();
var time = DateTime.UtcNow.Ticks;
topic.Send(batch);
var diff = (DateTime.UtcNow.Ticks - time) / 10000;
timer.Record(diff, TimeUnit.Milliseconds);
published += options.BatchSize;
if (published >= options.Messages)
{
break;
}
}
var end = DateTime.Now;
var timeTake = (end - start).TotalSeconds;
Console.WriteLine("Published {0} messages in {1} seconds", published, timeTake);
return(0);
}
public void CanSendEmptyMessageBatch()
{
// Given
var options = new KafkaSinkOptions("topic", new[] { new Uri("http://sample") });
var kafkaClient = new KafkaClient(options);
var systemUnderTest = new KafkaSink(kafkaClient, options);
var logEvents = new LogEvent[0];
// When
var result = systemUnderTest.EmitBatchInternalAsync(logEvents);
// Then
Assert.IsTrue(result.IsCompleted);
Assert.IsFalse(result.IsFaulted);
}
public static LoggerConfiguration Kafka(
this LoggerSinkConfiguration loggerConfiguration,
KafkaSinkOptions options)
{
Contract.Requires(loggerConfiguration != null);
Contract.Requires(options != null);
Contract.Ensures(Contract.Result <LoggerConfiguration>() != null);
var kafkaClient = new KafkaClient(options);
var kafkaSink = new KafkaSink(kafkaClient, options);
var result = loggerConfiguration.Sink(kafkaSink);
Contract.Assume(result != null);
return(result);
}
public void CanSendEmptyMessageBatch()
{
// Given
var options = new KafkaSinkOptions("topic", new[] { new Uri("http://sample") });
var kafkaClient = new KafkaClient(options);
var systemUnderTest = new KafkaSink(kafkaClient, options);
var logEvents = new LogEvent[0];
// When
var result = systemUnderTest.EmitBatchInternalAsync(logEvents);
// Then
Assert.IsTrue(result.IsCompleted);
Assert.IsFalse(result.IsFaulted);
}
private async Task <int> OnExecute(IConsole console)
{
try
{
await KafkaClient.DeleteTopicAsync(TopicName);
console.WriteLine($"Topic '{TopicName}' was deleted.");
return(await Task.FromResult(0));
}
catch (Exception e)
{
console.WriteLine($"An error occured deleting topic '{TopicName}': {e.Message}");
return(await Task.FromResult(1));
}
}
private static void LifeCycleMgtMain(string[] args)
{
var config = HttpConfig.GetDefaultConfig();
config.IgnoreSslVerification = true;
var auth = new BasicCredentials(ak, sk, projectId);
var kafkaClient = KafkaClient.NewBuilder()
.WithCredential(auth)
.WithEndPoint(endpoint)
.WithHttpConfig(config).Build();
//listInstance(kafkaClient);
createInstance(kafkaClient);
//batchRestartOrDeleteInstances(kafkaClient);
}
private static void OtherInterfaceMain(string[] args)
{
var config = HttpConfig.GetDefaultConfig();
config.IgnoreSslVerification = true;
var auth = new BasicCredentials(ak, sk, projectId);
var kafkaClient = KafkaClient.NewBuilder()
.WithCredential(auth)
.WithEndPoint(endpoint)
.WithHttpConfig(config).Build();
listAvailableZones(kafkaClient);
listProducts(kafkaClient);
showMaintainWindows(kafkaClient);
}
private static void createInstance(KafkaClient client)
{
var req = new CreatePostPaidInstanceRequest
{
Body = new CreatePostPaidInstanceReq
{
Name = "Csharp-sdk-create",
AvailableZones = new List <string> {
az
},
Engine = CreatePostPaidInstanceReq.EngineEnum.KAFKA,
EngineVersion = CreatePostPaidInstanceReq.EngineVersionEnum._1_1_0,
KafkaManagerUser = managerUser,
KafkaManagerPassword = managerPasswd,
PartitionNum = CreatePostPaidInstanceReq.PartitionNumEnum.NUMBER_300,
ProductId = "00300-30308-0--0",
SecurityGroupId = sgId,
Specification = CreatePostPaidInstanceReq.SpecificationEnum._100MB,
StorageSpecCode = CreatePostPaidInstanceReq.StorageSpecCodeEnum.DMS_PHYSICAL_STORAGE_NORMAL,
StorageSpace = 300,
SubnetId = subNetId,
VpcId = vpcId,
SslEnable = true,
AccessUser = accessUser,
Password = accessPasswd,
}
};
try
{
var resp = client.CreatePostPaidInstance(req);
Console.WriteLine(resp.GetHttpBody());
}
catch (RequestTimeoutException requestTimeoutException)
{
Console.WriteLine(requestTimeoutException.ErrorMessage);
}
catch (ServiceResponseException clientRequestException)
{
Console.WriteLine(clientRequestException.HttpStatusCode);
Console.WriteLine(clientRequestException.ErrorCode);
Console.WriteLine(clientRequestException.ErrorMsg);
}
catch (ConnectionException connectionException)
{
Console.WriteLine(connectionException.ErrorMessage);
}
}
private async Task and_a_capability()
{
var capabilityId = Guid.NewGuid();
var kafkaClient = new KafkaClient();
var message = new
{
CapabilityId = capabilityId.ToString(),
CapabilityName = "VerbNoun",
};
await kafkaClient.SendMessageAsync(message);
Thread.Sleep(1500); // Magic sleep that ensures that the service have complected the create operation.
_capabilityId = capabilityId;
}
public async Task KafkaClientConnectionTest()
{
// Given
const string Topic = "test";
var brokers = new[] { new Uri("localhost:9092") };
var options = new KafkaSinkOptions(Topic, brokers);
var kafkaClient = new KafkaClient(options);
var message = new Message { Value = Encoding.UTF8.GetBytes("{'message' : 'This is a test message'}") };
// When
await kafkaClient.SendMessagesAsync(new[] { message }).ConfigureAwait(false);
kafkaClient.Dispose();
// Then
Assert.IsTrue(true);
}
private async Task When_a_capability_is_deleted()
{
var kafkaClient = new KafkaClient();
var capability = new Capability
{
Id = "1e0a8f85-de38-42ef-a4f4-87c3b4f9a5f9",
Name = "devx-acltest4"
};
var message = new
{
CapabilityId = capability.Id,
CapabilityName = capability.Name
};
await kafkaClient.SendMessageAsync(message, "capability_deleted");
}
private async Task <int> OnExecute(IConsole console)
{
try
{
var topics = KafkaClient.ListTopics(Filter);
foreach (var topic in topics)
{
console.WriteLine(topic);
}
return(await Task.FromResult(0));
}
catch (Exception e)
{
console.WriteLine($"An error occured listing topics: {e.Message}");
return(await Task.FromResult(1));
}
}
public async Task KafkaClientConnectionTest()
{
// Given
const string Topic = "test";
var brokers = new[] { new Uri("localhost:9092") };
var options = new KafkaSinkOptions(Topic, brokers);
var kafkaClient = new KafkaClient(options);
var message = new Message {
Value = Encoding.UTF8.GetBytes("{'message' : 'This is a test message'}")
};
// When
await kafkaClient.SendMessagesAsync(new[] { message }).ConfigureAwait(false);
kafkaClient.Dispose();
// Then
Assert.IsTrue(true);
}
private async Task <int> OnExecute(IConsole console)
{
var config = ConfigService.Get();
console.WriteLine($"Waiting for messages in {TopicName}:");
KafkaClient.ConsumeMessage(TopicName, GroupId, Commit, (string topicInfo, string message) =>
{
console.WriteLine($"[{DateTime.Now.ToString("HH:mm:ss.ff")}] at {topicInfo}: {message}");
if (Pause > 0)
{
Thread.Sleep(Pause);
}
},
(string error) =>
{
console.WriteLine($"Error occured: {error}");
});
return(await Task.FromResult(0));
}
public int Start(HighLevelConsumerOptions opts)
{
var timer = Metric.Timer("Received", Unit.Events);
Metric.Config.WithReporting(r => r.WithConsoleReport(TimeSpan.FromSeconds(5)));
using (var client = new KafkaClient(opts.ZkConnect))
using (var consumer = new BalancedConsumer(client, opts.Group, opts.Topic, opts.Threads))
{
ListenToConsole(consumer);
consumer.Start(message =>
{
var time = DateTime.UtcNow.Ticks;
var value = long.Parse(message.Value);
var diff = (time - value) / 10000;
timer.Record(diff, TimeUnit.Milliseconds);
});
}
return(0);
}
static void Main(string[] args)
{
KafkaClient kafkaClient = new KafkaClient("localhost:9092", "Kafka.Demo.Consumer");
var cancelled = false;
Console.CancelKeyPress += (_, e) => {
e.Cancel = true; // prevent the process from terminating.
cancelled = true;
};
Console.WriteLine("Ctrl-C to exit.");
while (!cancelled)
{
Behavior behavior = kafkaClient.ConsumeAsync <Behavior>("behaviors");
string behaviorStr = JsonConvert.SerializeObject(behavior);
Console.WriteLine(behaviorStr);
}
}
public async Task Publish_Message_Consumer_Should_Receive()
{
KafkaClient kafkaClient = new KafkaClient(new KafkaClientConfiguration
{
ProducerConfig = new ProducerConfig
{
BootstrapServers = BootstrapServers
},
Topic = Topic
});
string message = $"hello kafka";
await kafkaClient.PublishAsync(message);
await Task.Delay(100);
var conf = new ConsumerConfig
{
GroupId = "test-consumer-group",
BootstrapServers = BootstrapServers,
AutoOffsetReset = AutoOffsetReset.Earliest
};
using (var c = new ConsumerBuilder <Ignore, string>(conf).Build())
{
c.Subscribe(Topic);
try
{
var cr = c.Consume(1000);
cr.Message.Value.ShouldBe(message);
}
catch (ConsumeException e)
{
Console.WriteLine($"Error occured: {e.Error.Reason}");
}
}
}
