public void GetMessagePartition() { // arrange var publisherA = new ProducerSettings(); new ProducerBuilder <MessageA>(publisherA) .PartitionProvider((m, t) => 10); var publisherB = new ProducerSettings(); new ProducerBuilder <MessageB>(publisherB); MbSettings.Producers.Add(publisherA); MbSettings.Producers.Add(publisherB); var msgA = new MessageA(); var msgB = new MessageB(); // act var msgAPartition = KafkaMb.Value.Public_GetMessagePartition(msgA.GetType(), msgA, "topic1"); var msgBPartition = KafkaMb.Value.Public_GetMessagePartition(msgB.GetType(), msgB, "topic1"); // assert msgAPartition.Should().Be(10); msgBPartition.Should().Be(-1); }
protected void Setup() { testTopic = "akka100"; subscription = Subscriptions.Topics(testTopic); probe = this.CreateTestProbe(); string configText = File.ReadAllText("akka.test.conf"); var config = ConfigurationFactory.ParseString(configText); var system_producer = ActorSystem.Create("TestKafka", config); materializer_producer = system_producer.Materializer(); var system_consumer = ActorSystem.Create("TestKafka", config); materializer_consumer = system_producer.Materializer(); this.Sys.Settings.Config.WithFallback(config); producerSettings = ProducerSettings <Null, string> .Create(system_producer, null, null) .WithBootstrapServers("kafka:9092"); consumerSettings = ConsumerSettings <Null, string> .Create(system_consumer, null, null) .WithBootstrapServers("kafka:9092") .WithGroupId("group1"); }
protected virtual TimeSpan GetDefaultRequestTimeout(Type requestType, ProducerSettings producerSettings) { var timeout = producerSettings.Timeout ?? Settings.RequestResponse.Timeout; Log.DebugFormat(CultureInfo.InvariantCulture, "Applying default timeout {0} for message type {1}", timeout, requestType); return(timeout); }
protected override void PreStart() { #if DEBUG Console.WriteLine("PreStart"); #endif base.PreStart(); var producerSettings = ProducerSettings <string, byte[]> .Create(settings.KafkaConfig, Serializers.Utf8, Serializers.ByteArray) .WithBootstrapServers(settings.KafkaConfig.GetString("bootstrap.servers")); materializer = Context.Materializer(); producer = producerSettings.CreateKafkaProducer(); topicMapper = settings.EventTopicMapper; enabledTopicPartitions = new HashSet <TopicPartition>(settings.EnabledTopicPartitions); #if DEBUG Console.WriteLine("Gettings RocksDB"); #endif var rocksDbSettings = settings.RocksDbSettings; rocksDbReadOptions = new ReadOptions().SetVerifyChecksums(rocksDbSettings.Checksum); rocksDbWriteOptions = new WriteOptions().SetSync(rocksDbSettings.FSync); //retrieve offsets for the consumer to start reading; as rocksdb possible already knows about database = RocksDb.Open(rocksDbOptions, rocksDbSettings.Path); #if DEBUG Console.WriteLine("Database opened"); #endif idMap = ReadIdMap(); currentOffsets = ReadOffsetsForPartitions(); StartConsuming(); }
public void GetMessageKey() { // arrange var producerA = new ProducerSettings(); new ProducerBuilder <MessageA>(producerA) .KeyProvider((m, t) => m.Key); var producerB = new ProducerSettings(); new ProducerBuilder <MessageB>(producerB); MbSettings.Producers.Add(producerA); MbSettings.Producers.Add(producerB); var msgA = new MessageA(); var msgB = new MessageB(); // act var msgAKey = KafkaMb.Value.Public_GetMessageKey(msgA.GetType(), msgA, "topic1"); var msgBKey = KafkaMb.Value.Public_GetMessageKey(msgB.GetType(), msgB, "topic1"); // assert msgAKey.Should().BeSameAs(msgA.Key); msgBKey.Should().BeNull(); }
public static void UserKafkaProducer(this ProducerSettings settings, Action <KafkaSettings> settignsAction) { var settingsDict = new KafkaSettings(); settignsAction(settingsDict); settings.Producer = new KafkaProducer(settingsDict.GetDictionaryConfig()); }
public void Initialize(ProducerSettings producerSettings, string topic, int queueCount, IPEndPoint replyAddress) { _topic = topic; _producer = new Producer(producerSettings); _producer.RegisterTopic(topic, queueCount); _replyAddress = replyAddress; }
public void PublishService_SendBatch_CallSerializer() { var moqProducerSettings = new ProducerSettings() { Producer = A.Fake <IProducer>() }; var moqSerializer = A.Fake <ISerializer <string> >(); var moqPublishSettings = new PublishSettings <string>() { Serializer = moqSerializer }; var moqOptionsSnapshot = A.Fake <IOptionsSnapshot <PublishSettings <string> > >(); A.CallTo(() => moqOptionsSnapshot.Get(A <string> ._)).Returns(moqPublishSettings); var publishSerivce = new PublishService <string>(moqProducerSettings, moqOptionsSnapshot, "topicName"); publishSerivce.SendBatch("test msg", "test msg1", "test msg2"); A.CallTo( () => moqSerializer.Serialize(A <string> .That.IsSameAs("test msg")) ).MustHaveHappenedOnceExactly(); A.CallTo( () => moqSerializer.Serialize(A <string> .That.IsSameAs("test msg1")) ).MustHaveHappenedOnceExactly(); A.CallTo( () => moqSerializer.Serialize(A <string> .That.IsSameAs("test msg2")) ).MustHaveHappenedOnceExactly(); }
public void PublishService_Send_CallProducer() { var moqProducer = A.Fake <IProducer>(); var moqProducerSettings = new ProducerSettings() { Producer = moqProducer }; var moqSerializer = A.Fake <ISerializer <string> >(); A.CallTo(() => moqSerializer.Serialize(A <string> ._)) .Returns(new byte[] { 0x0, 0x1, 0x2 }); var moqPublishSettings = new PublishSettings <string>() { Serializer = moqSerializer }; var moqOptionsSnapshot = A.Fake <IOptionsSnapshot <PublishSettings <string> > >(); A.CallTo(() => moqOptionsSnapshot.Get(A <string> ._)).Returns(moqPublishSettings); var publishSerivce = new PublishService <string>(moqProducerSettings, moqOptionsSnapshot, "topicName"); publishSerivce.Send("test msg"); A.CallTo(() => moqProducer.Send("topicName", A <byte[]> .That.IsSameSequenceAs(new byte[] { 0x0, 0x1, 0x2 }))) .MustHaveHappenedOnceExactly(); }
public ProducerStageLogic(ProducerSettings <K, V> settings, Shape shape) : base(shape) { In = shape.Inlets.FirstOrDefault(); Out = shape.Outlets.FirstOrDefault(); producer = new Producer <K, V>(settings.Properties, settings.KeySerializer, settings.ValueSerializer); SetHandler(In, onPush: () => { var msg = Grab <ProduceRecord <K, V> >(In); var task = producer.ProduceAsync(msg.Topic, msg.Key, msg.Value).GetAwaiter().GetResult(); Console.WriteLine($"Partition: {task.Partition}, Offset: {task.Offset}"); Push(Out, Task.FromResult(new Result <K, V>(task, msg))); }, onUpstreamFinish: () => { inIsClosed = true; completionState.SetResult(true); CheckForCompletion(); }, onUpstreamFailure: exception => { inIsClosed = true; completionState.SetException(exception); CheckForCompletion(); }); SetHandler(Out, onPull: () => { TryPull(In); }); }
public static void Main(string[] args) { Config fallbackConfig = ConfigurationFactory.ParseString(@" akka.suppress-json-serializer-warning=true akka.loglevel = DEBUG ").WithFallback(ConfigurationFactory.FromResource <ConsumerSettings <object, object> >("Akka.Streams.Kafka.reference.conf")); var system = ActorSystem.Create("TestKafka", fallbackConfig); var materializer = system.Materializer(); var producerSettings = ProducerSettings <Null, string> .Create(system, null, null) .WithBootstrapServers("localhost:29092"); Source .Cycle(() => Enumerable.Range(1, 100).GetEnumerator()) .Select(c => c.ToString()) .Select(elem => ProducerMessage.Single(new ProducerRecord <Null, string>("akka100", elem))) .Via(KafkaProducer.FlexiFlow <Null, string, NotUsed>(producerSettings)) .Select(result => { var response = result as Result <Null, string, NotUsed>; Console.WriteLine($"Producer: {response.Metadata.Topic}/{response.Metadata.Partition} {response.Metadata.Offset}: {response.Metadata.Value}"); return(result); }) .RunWith(Sink.Ignore <IResults <Null, string, NotUsed> >(), materializer); // TODO: producer as a Commitable Sink // TODO: Sharing KafkaProducer Console.ReadLine(); }
public static void Main(string[] args) { var system = ActorSystem.Create("TestKafka"); var materializer = system.Materializer(); var producerSettings = new ProducerSettings <Null, string>(system, null, new StringSerializer(Encoding.UTF8)) .WithBootstrapServers("localhost:9092"); // producer as a Sink Source .From(Enumerable.Range(500, 601)) .Select(c => c.ToString()) .Select(elem => new ProduceRecord <Null, string>("akka", null, elem)) .RunWith(Producer.PlainSink(producerSettings), materializer); // producer as a Flow Source .From(Enumerable.Range(1, 100)) .Select(c => c.ToString()) .Select(elem => new ProduceRecord <Null, string>("akka", null, elem)) .Via(Producer.CreateFlow(producerSettings)) .Select(result => { var record = result.Result.Metadata; Console.WriteLine($"{record.Topic}/{record.Partition} {result.Result.Offset}: {record.Value}"); return(result); }) .RunWith(Sink.Ignore <Task <Result <Null, string> > >(), materializer); // TODO: producer as a Commitable Sink // TODO: Sharing KafkaProducer Console.ReadLine(); }
public static void Main(string[] args) { Config fallbackConfig = ConfigurationFactory.ParseString(@" akka.suppress-json-serializer-warning=true akka.loglevel = DEBUG ").WithFallback(ConfigurationFactory.FromResource <ConsumerSettings <object, object> >("Akka.Streams.Kafka.reference.conf")); var system = ActorSystem.Create("TestKafka", fallbackConfig); var materializer = system.Materializer(); var producerSettings = ProducerSettings <Null, string> .Create(system, null, new StringSerializer(Encoding.UTF8)) .WithBootstrapServers("localhost:29092"); Source .Cycle(() => Enumerable.Range(1, 100).GetEnumerator()) .Select(c => c.ToString()) .Select(elem => new MessageAndMeta <Null, string> { Topic = "akka100", Message = new Message <Null, string> { Value = elem } }) .Via(KafkaProducer.PlainFlow(producerSettings)) .Select(record => { Console.WriteLine($"Producer: {record.Topic}/{record.Partition} {record.Offset}: {record.Value}"); return(record); }) .RunWith(Sink.Ignore <DeliveryReport <Null, string> >(), materializer); // TODO: producer as a Commitable Sink // TODO: Sharing KafkaProducer Console.ReadLine(); }
public async Task <ActionResult <string> > ProducerAsync([FromBody] UserViewModel userViewModel) { try { string topic = "insidetech-5"; string host = "127.0.0.1:9092"; var producerSettings = new ProducerSettings(host); var producer = new Producer(producerSettings); var payload = new Model.User { Email = userViewModel.Email, Name = userViewModel.Name }; var result = await producer.SendAsync(topic, payload); return(Ok(result.Value)); } catch { return(new StatusCodeResult(500)); } }
/// <summary> /// Initialize RabbitMQ /// </summary> /// <param name="settings"></param> /// <param name="topic"></param> /// <param name="queueCount"></param> /// <param name="autoConfig"></param> /// <returns></returns> public ApplicationMessagePublisher InitializeRabbitMQ(ProducerSettings settings, string topic, int queueCount = 4, bool autoConfig = true) { InitializeENode(); _producer = new Producer(settings, false, autoConfig); _producer.RegisterTopic(topic, queueCount); return(this); }
private static Flow <IEnvelope <K, V, TPassThrough>, IResults <K, V, TPassThrough>, NotUsed> FlowWithDispatcher <K, V, TPassThrough>( ProducerSettings <K, V> settings, Flow <IEnvelope <K, V, TPassThrough>, IResults <K, V, TPassThrough>, NotUsed> flow) { return(string.IsNullOrEmpty(settings.DispatcherId) ? flow : flow.WithAttributes(ActorAttributes.CreateDispatcher(settings.DispatcherId))); }
/// <summary> /// TransactionalProducerStage /// </summary> public TransactionalProducerStage(bool closeProducerOnStop, ProducerSettings <K, V> settings) { _settings = settings; CloseProducerOnStop = closeProducerOnStop; ProducerProvider = errorHandler => _settings.CreateKafkaProducer(errorHandler); Shape = new FlowShape <IEnvelope <K, V, TPassThrough>, Task <IResults <K, V, TPassThrough> > >(In, Out); }
public async Task<ProducerList> CreateAsync(ProducerListSettings listSettings) { Guard.ArgumentNotNull(() => listSettings, listSettings); ProducerList producerList = new ProducerList(); SchemeInfo schemeInfo = await FetchSchemeInfo(listSettings.OrganisationID); string approvalNumber = SanitizeApprovalNumber(schemeInfo.ApprovalNumber); string tradingName = SanitizeTradingName(schemeInfo.TradingName); producerList.SchemaVersion = listSettings.SchemaVersion; producerList.ApprovalNumber = approvalNumber; producerList.ComplianceYear = listSettings.ComplianceYear; producerList.TradingName = tradingName; producerList.SchemeBusiness = SchemeBusiness.Create(listSettings); for (int index = 0; index < listSettings.NumberOfNewProducers; ++index) { ProducerSettings producerSettings = new ProducerSettings(); producerSettings.SchemaVersion = listSettings.SchemaVersion; producerSettings.IsNew = true; producerSettings.IgnoreStringLengthConditions = listSettings.IgnoreStringLengthConditions; Producer producer = Producer.Create(producerSettings, listSettings.NoCompaniesForNewProducers); producerList.Producers.Add(producer); } int numberOfExistingProducersToInclude = listSettings.NumberOfExistingProducers; List<string> registrationNumbers = await dataAccess.GetRegistrationNumbers(listSettings.OrganisationID, listSettings.ComplianceYear, numberOfExistingProducersToInclude); int numberOfExistingProducers = registrationNumbers.Count; if (numberOfExistingProducersToInclude > numberOfExistingProducers) { numberOfExistingProducersToInclude = numberOfExistingProducers; } for (int index = 0; index < numberOfExistingProducersToInclude; ++index) { ProducerSettings producerSettings = new ProducerSettings(); producerSettings.SchemaVersion = listSettings.SchemaVersion; producerSettings.IsNew = false; producerSettings.RegistrationNumber = registrationNumbers[index]; producerSettings.IgnoreStringLengthConditions = listSettings.IgnoreStringLengthConditions; Producer producer = Producer.Create(producerSettings, listSettings.NoCompaniesForNewProducers); producerList.Producers.Add(producer); } return producerList; }
public async Task <ProducerList> CreateAsync(ProducerListSettings listSettings) { Guard.ArgumentNotNull(() => listSettings, listSettings); ProducerList producerList = new ProducerList(); SchemeInfo schemeInfo = await FetchSchemeInfo(listSettings.OrganisationID); string approvalNumber = SanitizeApprovalNumber(schemeInfo.ApprovalNumber); string tradingName = SanitizeTradingName(schemeInfo.TradingName); producerList.SchemaVersion = listSettings.SchemaVersion; producerList.ApprovalNumber = approvalNumber; producerList.ComplianceYear = listSettings.ComplianceYear; producerList.TradingName = tradingName; producerList.SchemeBusiness = SchemeBusiness.Create(listSettings); for (int index = 0; index < listSettings.NumberOfNewProducers; ++index) { ProducerSettings producerSettings = new ProducerSettings(); producerSettings.SchemaVersion = listSettings.SchemaVersion; producerSettings.IsNew = true; producerSettings.IgnoreStringLengthConditions = listSettings.IgnoreStringLengthConditions; Producer producer = Producer.Create(producerSettings, listSettings.NoCompaniesForNewProducers); producerList.Producers.Add(producer); } int numberOfExistingProducersToInclude = listSettings.NumberOfExistingProducers; List <string> registrationNumbers = await dataAccess.GetRegistrationNumbers(listSettings.OrganisationID, listSettings.ComplianceYear, numberOfExistingProducersToInclude); int numberOfExistingProducers = registrationNumbers.Count; if (numberOfExistingProducersToInclude > numberOfExistingProducers) { numberOfExistingProducersToInclude = numberOfExistingProducers; } for (int index = 0; index < numberOfExistingProducersToInclude; ++index) { ProducerSettings producerSettings = new ProducerSettings(); producerSettings.SchemaVersion = listSettings.SchemaVersion; producerSettings.IsNew = false; producerSettings.RegistrationNumber = registrationNumbers[index]; producerSettings.IgnoreStringLengthConditions = listSettings.IgnoreStringLengthConditions; Producer producer = Producer.Create(producerSettings, listSettings.NoCompaniesForNewProducers); producerList.Producers.Add(producer); } return(producerList); }
private async Task GivenInitializedTopic(string topic) { using (var producer = ProducerSettings.CreateKafkaProducer()) { await producer.ProduceAsync(topic, new Message <Null, string> { Value = InitialMsg }); } }
/// <summary> /// Initialize RabbitMQ /// </summary> /// <param name="settings"></param> /// <param name="topic"></param> /// <param name="queueCount"></param> /// <param name="delayedCommandEnabled">Enable delayed command or not. If enable, please make sure that plugin 'rabbitmq_delayed_message_exchange' enabled.</param> /// <param name="autoConfig"></param> /// <param name="commandResultProcessor"></param> /// <param name="sagaIdCommandItemKey">Command items's key for store saga id</param> /// <returns></returns> public CommandService InitializeRabbitMQ(ProducerSettings settings, string topic, int queueCount = 4, bool delayedCommandEnabled = false, bool autoConfig = true, CommandResultProcessor commandResultProcessor = null, string sagaIdCommandItemKey = "SagaId") { InitializeENode(); _producer = new Producer(settings, delayedCommandEnabled, autoConfig); _producer.RegisterTopic(topic, queueCount); _commandResultProcessor = commandResultProcessor; _sagaIdCommandItemKey = sagaIdCommandItemKey; return(this); }
public void Start(ProducerAkkaOption producerAkkaOption) { materializer_producer = producerSystem.Materializer(); var producer = ProducerSettings <Null, string> .Create(producerSystem, null, null) .WithBootstrapServers(producerAkkaOption.BootstrapServers); producerList[producerAkkaOption.ProducerName] = producer; }
public async Task PlainSink_should_resume_on_deserialization_errors() { var callCount = 0; Directive Decider(Exception cause) { callCount++; switch (cause) { case ProduceException <Null, string> ex when ex.Error.IsSerializationError(): return(Directive.Resume); default: return(Directive.Stop); } } var elementsCount = 10; var topic1 = CreateTopic(1); var group1 = CreateGroup(1); var producerSettings = ProducerSettings <Null, string> .Create(Sys, null, new FailingSerializer()) .WithBootstrapServers(Fixture.KafkaServer); var sink = KafkaProducer.PlainSink(producerSettings) .AddAttributes(ActorAttributes.CreateSupervisionStrategy(Decider)); var sourceTask = Source .From(new [] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }) .Select(elem => new ProducerRecord <Null, string>(new TopicPartition(topic1, 0), elem.ToString())) .RunWith(sink, Materializer); var timeoutTask = Task.Delay(TimeSpan.FromSeconds(5)); var completeTask = await Task.WhenAny(sourceTask, timeoutTask); if (completeTask == timeoutTask) { throw new Exception("Producer timed out"); } var settings = CreateConsumerSettings <Null, string>(group1).WithValueDeserializer(new StringDeserializer()); var probe = KafkaConsumer .PlainSource(settings, Subscriptions.Assignment(new TopicPartition(topic1, 0))) .Select(c => c.Value) .RunWith(this.SinkProbe <string>(), Materializer); probe.Request(elementsCount); for (var i = 0; i < 9; i++) { Log.Info($">>>>>>>>>>> {i}"); probe.ExpectNext(); } callCount.Should().Be(1); probe.Cancel(); }
/// <summary> /// <para> /// Create a flow to conditionally publish records to Kafka topics and then pass it on. /// </para> /// /// <para> /// It publishes records to Kafka topics conditionally: /// <list type="bullet"> /// <item><description> /// <see cref="Message{K,V,TPassThrough}"/> publishes a single message to its topic, and continues in the stream as <see cref="Result{K,V,TPassThrough}"/> /// </description></item> /// <item><description> /// <see cref="MultiMessage{K,V,TPassThrough}"/> publishes all messages in its `records` field, and continues in the stream as <see cref="MultiResult{K,V,TPassThrough}"/> /// </description></item> /// <item><description> /// <see cref="PassThroughMessage{K,V,TPassThrough}"/> does not publish anything, and continues in the stream as <see cref="PassThroughResult{K,V,TPassThrough}"/> /// </description></item> /// </list> /// </para> /// /// <para> /// The messages support the possibility to pass through arbitrary data, which can for example be a <see cref="CommittableOffset"/> /// or <see cref="CommittableOffsetBatch"/> that can be committed later in the flow. /// </para> /// </summary> public static Flow <IEnvelope <TKey, TValue, TPassThrough>, IResults <TKey, TValue, TPassThrough>, NotUsed> FlexiFlow <TKey, TValue, TPassThrough>( ProducerSettings <TKey, TValue> settings) { var flow = Flow.FromGraph(new DefaultProducerStage <TKey, TValue, TPassThrough, IEnvelope <TKey, TValue, TPassThrough>, IResults <TKey, TValue, TPassThrough> >( settings, closeProducerOnStop: true)) .SelectAsync(settings.Parallelism, x => x); return(FlowWithDispatcher(settings, flow)); }
public ProducerStage( ProducerSettings <K, V> settings, bool closeProducerOnStop, Func <IProducer <K, V> > producerProvider) { Settings = settings; CloseProducerOnStop = closeProducerOnStop; ProducerProvider = producerProvider; Shape = new FlowShape <MessageAndMeta <K, V>, Task <DeliveryReport <K, V> > >(In, Out); }
public async Task SendWithAkka(SequenceFlowInput inputMessage) { var producerSettings = ProducerSettings <Null, string> .Create(Context.System, null, null) .WithBootstrapServers(KafkaEndpoint); await Source .Single("Akka: " + inputMessage.ProcessInstanceId.ToString()) .Select(kafkaMessage => ProducerMessage.Single(new ProducerRecord <Null, string>(TopicName, kafkaMessage))) .Via(KafkaProducer.FlexiFlow <Null, string, NotUsed>(producerSettings)) .RunWith(Sink.Ignore <IResults <Null, string, NotUsed> >(), Materializer); }
protected virtual string GetDefaultName(Type messageType, ProducerSettings producerSettings) { var name = producerSettings.DefaultTopic; if (name == null) { throw new PublishMessageBusException($"An attempt to produce message of type {messageType} without specifying name, but there was no default name configured. Double check your configuration."); } Log.DebugFormat(CultureInfo.InvariantCulture, "Applying default name {0} for message type {1}", name, messageType); return(name); }
private async Task Produce(string topic, IEnumerable <int> range, ProducerSettings <Null, string> producerSettings) { await Source .From(range) .Select(elem => new MessageAndMeta <Null, string> { Topic = topic, Message = new Message <Null, string> { Value = elem.ToString() } }) .RunWith(KafkaProducer.PlainSink(producerSettings), _materializer); }
public DefaultProducerStage( ProducerSettings <K, V> settings, bool closeProducerOnStop, Func <IProducer <K, V> > customProducerProvider = null) { ProducerProvider = errorHandler => customProducerProvider?.Invoke() ?? Settings.CreateKafkaProducer(errorHandler); Settings = settings; CloseProducerOnStop = closeProducerOnStop; Shape = new FlowShape <TIn, Task <TOut> >(In, Out); }
public void GivenDefaultWhenGetKeyProviderThenReturnsNull() { // arrange var ps = new ProducerSettings(); // act var keyProvider = ps.GetKeyProvider(); // assert keyProvider.Should().BeNull(); }
private void OnProducedHook(object message, string name, ProducerSettings producerSettings) { try { producerSettings.OnMessageProduced?.Invoke(this, producerSettings, message, name); Settings.OnMessageProduced?.Invoke(this, producerSettings, message, name); } catch (Exception eh) { HookFailed(_logger, eh, nameof(IProducerEvents.OnMessageProduced)); } }