public KafkaTopicHandler(string zkConnectionStr) { Client = new KafkaClient(zkConnectionStr); Topics = new ConcurrentDictionary <KafkaTopicType, IKafkaTopic>(); PushQueue = new BlockingCollection <KafkaTopicEventData>(new ConcurrentQueue <KafkaTopicEventData>()); Task.Factory.StartNew(() => { foreach (var eventData in PushQueue.GetConsumingEnumerable()) { try { var topic = Topics.GetOrAdd(eventData.TopicType, type => Client.Topic(type.ToString().ToLowerInvariant())); var data = eventData.Data as string ?? eventData.Data.ToNewtonsoftJson(); topic.Send(new Message { Key = eventData.Key ?? string.Empty, Value = data, Codec = eventData.Codec }); } catch (Exception e) { // 时长事件不补发 if (eventData.TopicType != KafkaTopicType.MissionEventOnline) { _listKafkaEventDataMissedHandler.LeftPush(null, eventData.ToJson()); } _logger.Error("KafkaTopicHandler", e); } } }, TaskCreationOptions.LongRunning); }
public int Start(SimpleConsumerOptions opts) { var zookeeperString = opts.ZkConnect; var testTopic = opts.Topic; var partition = opts.Partition; var partitions = partition == null ? null : new[] { partition.Value }; var timer = Metric.Timer("Received", Unit.Events); Metric.Config.WithReporting(r => r.WithConsoleReport(TimeSpan.FromSeconds(5))); using (var client = new KafkaClient(zookeeperString)) using (var consumer = new Kafka.Basic.SimpleConsumer(client, testTopic, partitions)) { ListenToConsole(consumer); consumer.Start( message => { var time = DateTime.UtcNow.Ticks; var value = long.Parse(message.Value); var diff = (time - value) / 10000; timer.Record(diff, TimeUnit.Milliseconds); }, e => Console.Error.WriteLine(e.Message) ); } return(0); }
public TTSettlesSubscriber(KafkaClient awsKafkaClient, KafkaClient njKafkaClient, WorkerDispatcher dispatcher, InstrumentType instrumentType) { this._awsKafkaClient = awsKafkaClient; this._njKafkaClient = njKafkaClient; this._dispatcher = dispatcher; this._instrumentType = instrumentType; }
private static void listInstance(KafkaClient client) { var req = new ListInstancesRequest { Engine = ListInstancesRequest.EngineEnum.KAFKA, }; try { var resp = client.ListInstances(req); Console.WriteLine(resp.GetHttpBody()); } catch (RequestTimeoutException requestTimeoutException) { Console.WriteLine(requestTimeoutException.ErrorMessage); } catch (ServiceResponseException clientRequestException) { Console.WriteLine(clientRequestException.HttpStatusCode); Console.WriteLine(clientRequestException.ErrorCode); Console.WriteLine(clientRequestException.ErrorMsg); } catch (ConnectionException connectionException) { Console.WriteLine(connectionException.ErrorMessage); } }
private static void resetPassword(KafkaClient client) { var req = new ResetPasswordRequest { InstanceId = instanceId, Body = new ResetPasswordReq { NewPassword = "******", } }; try { var resp = client.ResetPassword(req); Console.WriteLine(resp.GetHttpBody()); } catch (RequestTimeoutException requestTimeoutException) { Console.WriteLine(requestTimeoutException.ErrorMessage); } catch (ServiceResponseException clientRequestException) { Console.WriteLine(clientRequestException.HttpStatusCode); Console.WriteLine(clientRequestException.ErrorCode); Console.WriteLine(clientRequestException.ErrorMsg); } catch (ConnectionException connectionException) { Console.WriteLine(connectionException.ErrorMessage); } }
private static void batchDeleteInstanceTopic(KafkaClient client) { var req = new BatchDeleteInstanceTopicRequest() { InstanceId = instanceId, Body = new BatchDeleteInstanceTopicReq { Topics = new List <string>() { "topic-test" }, } }; try { var resp = client.BatchDeleteInstanceTopic(req); Console.WriteLine(resp.GetHttpBody()); } catch (RequestTimeoutException requestTimeoutException) { Console.WriteLine(requestTimeoutException.ErrorMessage); } catch (ServiceResponseException clientRequestException) { Console.WriteLine(clientRequestException.HttpStatusCode); Console.WriteLine(clientRequestException.ErrorCode); Console.WriteLine(clientRequestException.ErrorMsg); } catch (ConnectionException connectionException) { Console.WriteLine(connectionException.ErrorMessage); } }
private static void createInstanceTopic(KafkaClient client) { var req = new CreateInstanceTopicRequest { InstanceId = instanceId, Body = new CreateInstanceTopicReq { Name = "topic-test", } }; try { var resp = client.CreateInstanceTopic(req); Console.WriteLine(resp.GetHttpBody()); } catch (RequestTimeoutException requestTimeoutException) { Console.WriteLine(requestTimeoutException.ErrorMessage); } catch (ServiceResponseException clientRequestException) { Console.WriteLine(clientRequestException.HttpStatusCode); Console.WriteLine(clientRequestException.ErrorCode); Console.WriteLine(clientRequestException.ErrorMsg); } catch (ConnectionException connectionException) { Console.WriteLine(connectionException.ErrorMessage); } }
private static void batchRestartOrDeleteInstances(KafkaClient client) { var req = new BatchRestartOrDeleteInstancesRequest { Body = new BatchRestartOrDeleteInstanceReq { Action = BatchRestartOrDeleteInstanceReq.ActionEnum.DELETE, Instances = new List <string>() { instanceId }, } }; try { var resp = client.BatchRestartOrDeleteInstances(req); Console.WriteLine(resp.GetHttpBody()); } catch (RequestTimeoutException requestTimeoutException) { Console.WriteLine(requestTimeoutException.ErrorMessage); } catch (ServiceResponseException clientRequestException) { Console.WriteLine(clientRequestException.HttpStatusCode); Console.WriteLine(clientRequestException.ErrorCode); Console.WriteLine(clientRequestException.ErrorMsg); } catch (ConnectionException connectionException) { Console.WriteLine(connectionException.ErrorMessage); } }
protected override void ConfigureApplicationContainer(TinyIoCContainer container) { base.ConfigureApplicationContainer(container); var kafkaClient = new KafkaClient("localhost:9092", "Kafka.Demo.Api"); container.Register <IKafkaClient, KafkaClient>(kafkaClient); }
public RDFServClient(string kafkaServer, string kafkaClientId, string serverTopic) { ServerTopic = serverTopic; Client = new KafkaClient(kafkaClientId, kafkaServer); QueryResultPath = ServerTopic + "_" + kafkaClientId + "_response_query"; QueryPath = ServerTopic + "_query"; AddServiceResultPath = ServerTopic + "_" + kafkaClientId + "_response_addservice"; AddServicePath = ServerTopic + "_add-service"; }
private async Task When_a_capability_is_created() { var kafkaClient = new KafkaClient(); var message = new { CapabilityId = Guid.NewGuid().ToString(), CapabilityName = "VerbNoun", }; await kafkaClient.SendMessageAsync(message); }
public void CreateTopicTest() { try { KafkaClient client = new KafkaClient(_zkConnection); client.CreateTopic("testtopic1"); } catch (Exception ex) { Console.WriteLine(ex.GetBaseException().Message); throw; } }
private static void InstanceMgtMain(string[] args) { var config = HttpConfig.GetDefaultConfig(); config.IgnoreSslVerification = true; var auth = new BasicCredentials(ak, sk, projectId); var kafkaClient = KafkaClient.NewBuilder() .WithCredential(auth) .WithEndPoint(endpoint) .WithHttpConfig(config).Build(); resetPassword(kafkaClient); }
public static void Main(string[] args) { AutoResetEvent waitForTTInit = new AutoResetEvent(false); Action <string> onInit = str => { Console.WriteLine(str); waitForTTInit.Set(); }; WorkerDispatcher dispatcher = new WorkerDispatcher(); TTDriver driver = new TTDriver(onInit, dispatcher); driver.start(); waitForTTInit.WaitOne(); var aws_bootstrap_servers = "b-1.kafka01.mb3vi8.c4.kafka.us-east-1.amazonaws.com:9092,b-2.kafka01.mb3vi8.c4.kafka.us-east-1.amazonaws.com:9092,b-3.kafka01.mb3vi8.c4.kafka.us-east-1.amazonaws.com:9092"; var nj_bootstrap_servers = "njxmd01.hetco.com:9092"; var awsKafkaClient = new KafkaClient(aws_bootstrap_servers); var njKafkaClient = new KafkaClient(nj_bootstrap_servers); // kafkaClient.Run(); Parser.Default.ParseArguments <Options>(args) .WithParsed <Options>(o => { if (o.LiveFutures) { var liveFuturesSubscriber = new LiveFuturesSubscriber(awsKafkaClient, njKafkaClient, dispatcher); liveFuturesSubscriber.start(); } else if (o.SettlesFutures) { var ttSettlesFuturesSubscriber = new TTSettlesSubscriber(awsKafkaClient, njKafkaClient, dispatcher, InstrumentType.Future); ttSettlesFuturesSubscriber.start(); } else if (o.SettlesOptions) { var ttSettlesOptionsSubscriber = new TTSettlesSubscriber(awsKafkaClient, njKafkaClient, dispatcher, InstrumentType.Option); ttSettlesOptionsSubscriber.start(); } else { Console.WriteLine($"Choose one of the options (-f -l -l -o)"); Environment.Exit(-1); } }); Console.WriteLine("{0}", Thread.CurrentThread.ManagedThreadId); Thread.Sleep(Timeout.Infinite); }
public async Task StartProducing(int messagesToProduce) { var kafkaBootstrapServer = "kafka:9092"; var topicConfiguration = new Dictionary <string, object> { { "request.required.acks", 1 }, { "request.timeout.ms", 2000 }, { "message.timeout.ms", 2000 }, }; // Create the producer configuration var configuration = new Dictionary <string, object> { { "api.version.request", true }, { "bootstrap.servers", kafkaBootstrapServer }, { "default.topic.config", topicConfiguration } }; // Create a new topic producer var kafkaProducer = KafkaClient.Connect(SchemaRegistryConnection, configuration).GetTopicProducer(Topic, true); try { var transaction = CreateTransctionModel(); var message = await kafkaProducer.ProduceAsync(transaction.CustomerId, transaction); } catch (Exception e) { Console.WriteLine(e); Console.WriteLine("Something went wrong aborting"); return; } // create 10 tasks to produce messages var tasks = Enumerable.Range(0, 10) .Select(x => { var i1 = messagesToProduce / 10; var dave = new Task[i1]; for (var i = 0; i < i1; i++) { var transaction = CreateTransctionModel(); dave[i] = kafkaProducer.ProduceAsync(transaction.CustomerId, transaction); } return(Task.WhenAll(dave)); }); //wait for everything to produce await Task.WhenAll(tasks); }
private static int Run(Options options) { var timer = Metric.Timer("Sent", Unit.Events); Metric.Config.WithReporting(r => r.WithConsoleReport(TimeSpan.FromSeconds(5))); var client = new KafkaClient(options.ZkConnect); var topic = client.Topic(options.Topic); var published = 0; var stop = false; Console.CancelKeyPress += (sender, eventArgs) => { eventArgs.Cancel = true; stop = true; }; var start = DateTime.Now; while (!stop) { Thread.Sleep(options.Sleep); var batch = Enumerable.Range(0, options.BatchSize) .Select(i => new Message { Key = (published + i).ToString(), Value = DateTime.UtcNow.Ticks.ToString() }) .ToArray(); var time = DateTime.UtcNow.Ticks; topic.Send(batch); var diff = (DateTime.UtcNow.Ticks - time) / 10000; timer.Record(diff, TimeUnit.Milliseconds); published += options.BatchSize; if (published >= options.Messages) { break; } } var end = DateTime.Now; var timeTake = (end - start).TotalSeconds; Console.WriteLine("Published {0} messages in {1} seconds", published, timeTake); return(0); }
public void CanSendEmptyMessageBatch() { // Given var options = new KafkaSinkOptions("topic", new[] { new Uri("http://sample") }); var kafkaClient = new KafkaClient(options); var systemUnderTest = new KafkaSink(kafkaClient, options); var logEvents = new LogEvent[0]; // When var result = systemUnderTest.EmitBatchInternalAsync(logEvents); // Then Assert.IsTrue(result.IsCompleted); Assert.IsFalse(result.IsFaulted); }
public static LoggerConfiguration Kafka( this LoggerSinkConfiguration loggerConfiguration, KafkaSinkOptions options) { Contract.Requires(loggerConfiguration != null); Contract.Requires(options != null); Contract.Ensures(Contract.Result <LoggerConfiguration>() != null); var kafkaClient = new KafkaClient(options); var kafkaSink = new KafkaSink(kafkaClient, options); var result = loggerConfiguration.Sink(kafkaSink); Contract.Assume(result != null); return(result); }
private async Task <int> OnExecute(IConsole console) { try { await KafkaClient.DeleteTopicAsync(TopicName); console.WriteLine($"Topic '{TopicName}' was deleted."); return(await Task.FromResult(0)); } catch (Exception e) { console.WriteLine($"An error occured deleting topic '{TopicName}': {e.Message}"); return(await Task.FromResult(1)); } }
private static void LifeCycleMgtMain(string[] args) { var config = HttpConfig.GetDefaultConfig(); config.IgnoreSslVerification = true; var auth = new BasicCredentials(ak, sk, projectId); var kafkaClient = KafkaClient.NewBuilder() .WithCredential(auth) .WithEndPoint(endpoint) .WithHttpConfig(config).Build(); //listInstance(kafkaClient); createInstance(kafkaClient); //batchRestartOrDeleteInstances(kafkaClient); }
private static void OtherInterfaceMain(string[] args) { var config = HttpConfig.GetDefaultConfig(); config.IgnoreSslVerification = true; var auth = new BasicCredentials(ak, sk, projectId); var kafkaClient = KafkaClient.NewBuilder() .WithCredential(auth) .WithEndPoint(endpoint) .WithHttpConfig(config).Build(); listAvailableZones(kafkaClient); listProducts(kafkaClient); showMaintainWindows(kafkaClient); }
private static void createInstance(KafkaClient client) { var req = new CreatePostPaidInstanceRequest { Body = new CreatePostPaidInstanceReq { Name = "Csharp-sdk-create", AvailableZones = new List <string> { az }, Engine = CreatePostPaidInstanceReq.EngineEnum.KAFKA, EngineVersion = CreatePostPaidInstanceReq.EngineVersionEnum._1_1_0, KafkaManagerUser = managerUser, KafkaManagerPassword = managerPasswd, PartitionNum = CreatePostPaidInstanceReq.PartitionNumEnum.NUMBER_300, ProductId = "00300-30308-0--0", SecurityGroupId = sgId, Specification = CreatePostPaidInstanceReq.SpecificationEnum._100MB, StorageSpecCode = CreatePostPaidInstanceReq.StorageSpecCodeEnum.DMS_PHYSICAL_STORAGE_NORMAL, StorageSpace = 300, SubnetId = subNetId, VpcId = vpcId, SslEnable = true, AccessUser = accessUser, Password = accessPasswd, } }; try { var resp = client.CreatePostPaidInstance(req); Console.WriteLine(resp.GetHttpBody()); } catch (RequestTimeoutException requestTimeoutException) { Console.WriteLine(requestTimeoutException.ErrorMessage); } catch (ServiceResponseException clientRequestException) { Console.WriteLine(clientRequestException.HttpStatusCode); Console.WriteLine(clientRequestException.ErrorCode); Console.WriteLine(clientRequestException.ErrorMsg); } catch (ConnectionException connectionException) { Console.WriteLine(connectionException.ErrorMessage); } }
private async Task and_a_capability() { var capabilityId = Guid.NewGuid(); var kafkaClient = new KafkaClient(); var message = new { CapabilityId = capabilityId.ToString(), CapabilityName = "VerbNoun", }; await kafkaClient.SendMessageAsync(message); Thread.Sleep(1500); // Magic sleep that ensures that the service have complected the create operation. _capabilityId = capabilityId; }
public async Task KafkaClientConnectionTest() { // Given const string Topic = "test"; var brokers = new[] { new Uri("localhost:9092") }; var options = new KafkaSinkOptions(Topic, brokers); var kafkaClient = new KafkaClient(options); var message = new Message { Value = Encoding.UTF8.GetBytes("{'message' : 'This is a test message'}") }; // When await kafkaClient.SendMessagesAsync(new[] { message }).ConfigureAwait(false); kafkaClient.Dispose(); // Then Assert.IsTrue(true); }
private async Task When_a_capability_is_deleted() { var kafkaClient = new KafkaClient(); var capability = new Capability { Id = "1e0a8f85-de38-42ef-a4f4-87c3b4f9a5f9", Name = "devx-acltest4" }; var message = new { CapabilityId = capability.Id, CapabilityName = capability.Name }; await kafkaClient.SendMessageAsync(message, "capability_deleted"); }
private async Task <int> OnExecute(IConsole console) { try { var topics = KafkaClient.ListTopics(Filter); foreach (var topic in topics) { console.WriteLine(topic); } return(await Task.FromResult(0)); } catch (Exception e) { console.WriteLine($"An error occured listing topics: {e.Message}"); return(await Task.FromResult(1)); } }
private async Task <int> OnExecute(IConsole console) { var config = ConfigService.Get(); console.WriteLine($"Waiting for messages in {TopicName}:"); KafkaClient.ConsumeMessage(TopicName, GroupId, Commit, (string topicInfo, string message) => { console.WriteLine($"[{DateTime.Now.ToString("HH:mm:ss.ff")}] at {topicInfo}: {message}"); if (Pause > 0) { Thread.Sleep(Pause); } }, (string error) => { console.WriteLine($"Error occured: {error}"); }); return(await Task.FromResult(0)); }
public int Start(HighLevelConsumerOptions opts) { var timer = Metric.Timer("Received", Unit.Events); Metric.Config.WithReporting(r => r.WithConsoleReport(TimeSpan.FromSeconds(5))); using (var client = new KafkaClient(opts.ZkConnect)) using (var consumer = new BalancedConsumer(client, opts.Group, opts.Topic, opts.Threads)) { ListenToConsole(consumer); consumer.Start(message => { var time = DateTime.UtcNow.Ticks; var value = long.Parse(message.Value); var diff = (time - value) / 10000; timer.Record(diff, TimeUnit.Milliseconds); }); } return(0); }
static void Main(string[] args) { KafkaClient kafkaClient = new KafkaClient("localhost:9092", "Kafka.Demo.Consumer"); var cancelled = false; Console.CancelKeyPress += (_, e) => { e.Cancel = true; // prevent the process from terminating. cancelled = true; }; Console.WriteLine("Ctrl-C to exit."); while (!cancelled) { Behavior behavior = kafkaClient.ConsumeAsync <Behavior>("behaviors"); string behaviorStr = JsonConvert.SerializeObject(behavior); Console.WriteLine(behaviorStr); } }
public async Task Publish_Message_Consumer_Should_Receive() { KafkaClient kafkaClient = new KafkaClient(new KafkaClientConfiguration { ProducerConfig = new ProducerConfig { BootstrapServers = BootstrapServers }, Topic = Topic }); string message = $"hello kafka"; await kafkaClient.PublishAsync(message); await Task.Delay(100); var conf = new ConsumerConfig { GroupId = "test-consumer-group", BootstrapServers = BootstrapServers, AutoOffsetReset = AutoOffsetReset.Earliest }; using (var c = new ConsumerBuilder <Ignore, string>(conf).Build()) { c.Subscribe(Topic); try { var cr = c.Consume(1000); cr.Message.Value.ShouldBe(message); } catch (ConsumeException e) { Console.WriteLine($"Error occured: {e.Error.Reason}"); } } }