protected void PreSend(ProducerMessage message)
{
var partitionNo = partitioningAlgo.ComputePartitionNo(
message.PartitionKey,
metaService.ListPartitionsByTopic(message.Topic).Count);
message.Partition = partitionNo;
}
public static void Main(string[] args)
{
Config fallbackConfig = ConfigurationFactory.ParseString(@"
akka.suppress-json-serializer-warning=true
akka.loglevel = DEBUG
").WithFallback(ConfigurationFactory.FromResource <ConsumerSettings <object, object> >("Akka.Streams.Kafka.reference.conf"));
var system = ActorSystem.Create("TestKafka", fallbackConfig);
var materializer = system.Materializer();
var producerSettings = ProducerSettings <Null, string> .Create(system, null, null)
.WithBootstrapServers("localhost:29092");
Source
.Cycle(() => Enumerable.Range(1, 100).GetEnumerator())
.Select(c => c.ToString())
.Select(elem => ProducerMessage.Single(new ProducerRecord <Null, string>("akka100", elem)))
.Via(KafkaProducer.FlexiFlow <Null, string, NotUsed>(producerSettings))
.Select(result =>
{
var response = result as Result <Null, string, NotUsed>;
Console.WriteLine($"Producer: {response.Metadata.Topic}/{response.Metadata.Partition} {response.Metadata.Offset}: {response.Metadata.Value}");
return(result);
})
.RunWith(Sink.Ignore <IResults <Null, string, NotUsed> >(), materializer);
// TODO: producer as a Commitable Sink
// TODO: Sharing KafkaProducer
Console.ReadLine();
}
public async Task Transactional_source_with_sink_Should_work()
{
var settings = CreateConsumerSettings <string>(CreateGroup(1));
var sourceTopic = CreateTopic(1);
var targetTopic = CreateTopic(2);
var transactionalId = Guid.NewGuid().ToString();
const int totalMessages = 10;
var control = KafkaConsumer.TransactionalSource(settings, Subscriptions.Topics(sourceTopic))
.Via(Business <TransactionalMessage <Null, string> >())
.Select(message =>
{
return(ProducerMessage.Single(
new ProducerRecord <Null, string>(targetTopic, message.Record.Key, message.Record.Value),
passThrough: message.PartitionOffset));
})
.ToMaterialized(KafkaProducer.TransactionalSink(ProducerSettings, transactionalId), Keep.Both)
.MapMaterializedValue(DrainingControl <NotUsed> .Create)
.Run(Materializer);
var consumer = ConsumeStrings(targetTopic, totalMessages);
await ProduceStrings(sourceTopic, Enumerable.Range(1, totalMessages), ProducerSettings);
AssertTaskCompletesWithin(TimeSpan.FromSeconds(totalMessages), consumer.IsShutdown);
AssertTaskCompletesWithin(TimeSpan.FromSeconds(totalMessages), control.DrainAndShutdown());
consumer.DrainAndShutdown().Result.Should().HaveCount(totalMessages);
}
public void Finish(ProducerMessage message)
{
using (var command = database.GetSqlStringCommand(FINISH_SQL))
{
database.AddInParameter(command, "@messageid", DbType.AnsiString, message.Base.MessageId);
database.ExecuteNonQuery(command);
}
}
public void ProduceMessage(ProducerMessage eventMessage)
{
var body = Encoding.UTF8.GetBytes(eventMessage.Content);
channel.BasicPublish(exchange: "",
routingKey: QueueConfiguration.AggregatedTopicsQueueName,
basicProperties: null,
body: body);
}
public void Error(ProducerMessage message, int status)
{
using (var command = database.GetSqlStringCommand(ERROR_SQL))
{
database.AddInParameter(command, "@status", DbType.Int16, status);
database.AddInParameter(command, "@updatetime", DbType.DateTime, DateTime.Now);
database.AddInParameter(command, "@messageid", DbType.AnsiString, message.Base.MessageId);
database.ExecuteNonQuery(command);
}
}
public void Save(ProducerMessage message)
{
using (var command = database.GetSqlStringCommand(INSERT_SQL))
{
database.AddInParameter(command, "@messageid", DbType.AnsiString, message.Base.MessageId);
database.AddInParameter(command, "@content", DbType.String, message.Base.ToString());
database.AddInParameter(command, "@createtime", DbType.DateTime, DateTime.Now);
database.ExecuteNonQuery(command);
}
}
public async Task SendWithAkka(SequenceFlowInput inputMessage)
{
var producerSettings = ProducerSettings <Null, string> .Create(Context.System, null, null)
.WithBootstrapServers(KafkaEndpoint);
await Source
.Single("Akka: " + inputMessage.ProcessInstanceId.ToString())
.Select(kafkaMessage => ProducerMessage.Single(new ProducerRecord <Null, string>(TopicName, kafkaMessage)))
.Via(KafkaProducer.FlexiFlow <Null, string, NotUsed>(producerSettings))
.RunWith(Sink.Ignore <IResults <Null, string, NotUsed> >(), Materializer);
}
private void enrichPartitionKey(ProducerMessage msg, String ip)
{
if (string.IsNullOrEmpty(msg.PartitionKey))
{
if (logEnrichInfo)
{
log.Info(string.Format("Parition key not set, will set ip as partition key(topic={0}, ip={1})", msg.Topic, ip));
}
msg.PartitionKey = ip;
}
}
private void enrichRefKey(ProducerMessage msg)
{
if (string.IsNullOrEmpty(msg.Key))
{
String refKey = System.Guid.NewGuid().ToString();
if (logEnrichInfo)
{
log.Info(string.Format("Ref key not set, will set uuid as ref key(topic={0}, ref key={1})", msg.Topic, refKey));
}
msg.Key = refKey;
}
}
public byte[] Encode(ProducerMessage message, byte version)
{
var bodyCodec = PayloadCodecFactory.GetCodecByTopicName(message.Topic);
var body = bodyCodec.Encode(message.Topic, message.GetBody<object>());
var buf = IoBuffer.Allocate(body.Length + 150);
buf.AutoExpand = true;
Magic.WriteMagic(buf);
buf.Put(version);
Encode(message, buf, body, bodyCodec.Type);
return null;
}
public DefaultMessageHolder(
string topic,
string partitionKey,
object body,
IPipeline<IFuture<SendResult>> pipeline,
ISystemClockService systemClockService)
{
this.message = new ProducerMessage(topic, body);
this.message.PartitionKey = partitionKey;
this.pipeline = pipeline;
this.systemClockService = systemClockService;
}
public IProducerMessage ToProducerMessage <TKey, TAggregate>(IEvent <TKey, TAggregate> @event) where TAggregate : IAggregate <TKey>
{
@event.Subject = GetSubject(@event);
var message = new ProducerMessage()
{
MessageBytes = Serializer.Serialize(@event),
Subject = @event.Subject,
MessageType = @event.GetType()
};
return(message);
}
[InlineData(20, 10)] //20 개의 메시지를 생산하고,소비한다,테스트는 10초이내에 완료되어야함(완료시 종료됨)
public void Test1(int limit, int cutoff)
{
string lastSignal = Guid.NewGuid().ToString();
int readyTimeForConsume = 3;
int recCnt = 0;
KafkaConsumer.CommittableSource(consumerSettings, subscription)
.RunForeach(result =>
{
Console.WriteLine($"Consumer: {result.Record.Topic}/{result.Record.Partition} {result.Record.Offset}: {result.Record.Value}");
if (lastSignal == result.Record.Value)
{
probe.Tell("처리모두완료");
}
result.CommitableOffset.Commit();
}, materializer_consumer);
Source <int, NotUsed> source = Source.From(Enumerable.Range(1, limit));
source
.Throttle(2, TimeSpan.FromSeconds(1), 1, ThrottleMode.Shaping) //출력 조절 : 초당 2개처리
.Select(c =>
{
var result = $"No:{c.ToString()}";
if (c == limit)
{
result = lastSignal;
}
return(result);
})
.Select(elem => ProducerMessage.Single(new ProducerRecord <Null, string>(testTopic, elem)))
.Via(KafkaProducer.FlexiFlow <Null, string, NotUsed>(producerSettings))
.Select(result =>
{
var response = result as Result <Null, string, NotUsed>;
Console.WriteLine($"Producer: {response.Metadata.Topic}/{response.Metadata.Partition} {response.Metadata.Offset}: {response.Metadata.Value}");
return(result);
})
.RunWith(Sink.Ignore <IResults <Null, string, NotUsed> >(), materializer_producer);
Within(TimeSpan.FromSeconds(cutoff), () =>
{
probe.ExpectMsg("처리모두완료", TimeSpan.FromSeconds(cutoff));
});
}
public void AddMessage(ProducerMessage message, SettableFuture<SendResult> future)
{
Validate(message);
message.SequenceNo = msgCounter.AtomicIncrementAndGet();
if (message.IsPriority)
{
messages.TryAdd(0, new List<ProducerMessage>());
messages[0].Add(message);
}
else
{
messages.TryAdd(1, new List<ProducerMessage>());
messages[1].Add(message);
}
futures[message.SequenceNo] = future;
}
public void SinkMessage(string producerName, string topic, List <string> message, int tps)
{
int maxBuster = 1; //메시지가 해당값에 도달하면, TPS 제약기가 발동합니다.
ProducerSettings <Null, string> producerSettings = producerList[producerName];
Source <string, NotUsed> source = Source.From(message);
source
.Select(c =>
{
return(c);
})
.Select(elem => ProducerMessage.Single(new ProducerRecord <Null, string>(topic, elem)))
.Throttle(tps, TimeSpan.FromSeconds(1), maxBuster, ThrottleMode.Shaping) //TPS
.Via(KafkaProducer.FlexiFlow <Null, string, NotUsed>(producerSettings))
.Select(result =>
{
var response = result as Result <Null, string, NotUsed>;
Console.WriteLine($"[{DateTime.Now}] Producer: {response.Metadata.Topic}/{response.Metadata.Partition} {response.Metadata.Offset}: {response.Metadata.Value}");
return(result);
})
.RunWith(Sink.Ignore <IResults <Null, string, NotUsed> >(), materializer_producer);
}
static async Task Main(string[] args)
{
Config fallbackConfig = ConfigurationFactory.ParseString(@"
akka.suppress-json-serializer-warning=true
akka.loglevel = DEBUG
").WithFallback(ConfigurationFactory.FromResource <ConsumerSettings <object, object> >("Akka.Streams.Kafka.reference.conf"));
var system = ActorSystem.Create("TestKafka", fallbackConfig);
var materializer = system.Materializer();
var producerSettings = ProducerSettings <Null, string> .Create(system, null, null)
.WithBootstrapServers($"{EventHubNamespace}.servicebus.windows.net:9093")
.WithProperties(new Dictionary <string, string>
{
{ "security.protocol", "SASL_SSL" },
{ "sasl.mechanism", "PLAIN" },
{ "sasl.username", "$ConnectionString" },
{ "sasl.password", EventHubConnectionString },
});
await Source.From(Enumerable.Range(1, 100))
.Select(c => c.ToString())
.Select(elem => ProducerMessage.Single(new ProducerRecord <Null, string>(EventHubName, elem)))
.Via(KafkaProducer.FlexiFlow <Null, string, NotUsed>(producerSettings))
.Select(result =>
{
var response = result as Result <Null, string, NotUsed>;
Console.WriteLine($"Producer: {response.Metadata.Topic}/{response.Metadata.Partition} {response.Metadata.Offset}: {response.Metadata.Value}");
return(result);
})
.RunWith(Sink.Ignore <IResults <Null, string, NotUsed> >(), materializer);
Console.ReadKey();
await system.Terminate();
}
private void Encode(ProducerMessage message, IoBuffer buf, byte[] body, string codecType)
{
var codec = new HermesPrimitiveCodec(buf);
var indexBeginning = buf.Position;
codec.WriteInt(-1); // placeholder for whole length
var indexAfterWholeLen = buf.Position;
codec.WriteInt(-1); // placeholder for header length
codec.WriteInt(-1); // placeholder for body length
var indexBeforeHeader = buf.Position;
// header begin
codec.WriteString(message.Key);
codec.WriteLong(message.BornTime);
codec.WriteInt(0); //remaining retries
codec.WriteString(codecType);
var propertiesHolder = message.PropertiesHolder;
WriteProperties(propertiesHolder.DurableProperties, buf, codec);
WriteProperties(propertiesHolder.VolatileProperties, buf, codec);
// header end
var headerLen = buf.Position - indexBeforeHeader;
//body begin
var indexBeforeBody = buf.Position;
buf.Put(body);
var bodyLen = buf.Position - indexBeforeBody;
//body end
//crc
codec.WriteLong(ChecksumUtil.Crc32(buf.GetSlice(indexBeforeHeader, headerLen + bodyLen)));
var indexEnd = buf.Position;
var wholeLen = indexEnd - indexAfterWholeLen;
// refill whole length
buf.Position = indexBeginning;
codec.WriteInt(wholeLen);
// refill header length
codec.WriteInt(headerLen);
// refill body length
codec.WriteInt(bodyLen);
buf.Position = indexEnd;
}
protected abstract IFuture<SendResult> DoSend(ProducerMessage message);
public IFuture<SendResult> Send(ProducerMessage message)
{
PreSend(message);
return DoSend(message);
}
private void Validate(ProducerMessage message)
{
if (string.IsNullOrEmpty(Topic))
throw new ArgumentNullException("SendMessageCommand.Topic");
if (string.IsNullOrEmpty(message.Topic))
throw new ArgumentNullException("ProducerMessage.Topic");
if (!Topic.Equals(message.Topic, StringComparison.OrdinalIgnoreCase)
|| Partition != message.Partition)
{
throw new ArgumentException(string.Format("Illegal message[topic={0}, partition={1}] try to add to SendMessageCommand[topic={0}, partition={1}]",
message.Topic, message.Partition, Topic, Partition));
}
}
public async Task ProducerFlowWithContext_should_work_with_source_with_context()
{
bool Duplicate(string value) => value == "1";
bool Ignore(string value) => value == "2";
var consumerSettings = CreateConsumerSettings <string, string>(CreateGroup(1));
var topic1 = CreateTopic(1);
var topic2 = CreateTopic(2);
var topic3 = CreateTopic(3);
var topic4 = CreateTopic(4);
var producerSettings = BuildProducerSettings <string, string>();
var committerSettings = CommitterSettings;
var totalMessages = 10;
var totalConsumed = 0;
await ProduceStrings(topic1, Enumerable.Range(1, totalMessages), producerSettings);
var(control2, result) = KafkaConsumer.PlainSource(consumerSettings, Subscriptions.Topics(topic2, topic3, topic4))
.Scan(0, (c, _) => c + 1)
.Select(consumed =>
{
totalConsumed = consumed;
return(consumed);
})
.ToMaterialized(Sink.Last <int>(), Keep.Both)
.Run(Materializer);
var control = KafkaConsumer.SourceWithOffsetContext(consumerSettings, Subscriptions.Topics(topic1))
.Select(record =>
{
IEnvelope <string, string, NotUsed> output;
if (Duplicate(record.Message.Value))
{
output = ProducerMessage.Multi(new[]
{
new ProducerRecord <string, string>(topic2, record.Message.Key, record.Message.Value),
new ProducerRecord <string, string>(topic3, record.Message.Key, record.Message.Value)
}.ToImmutableSet());
}
else if (Ignore(record.Message.Value))
{
output = ProducerMessage.PassThrough <string, string>();
}
else
{
output = ProducerMessage.Single(new ProducerRecord <string, string>(topic4, record.Message.Key, record.Message.Value));
}
Log.Debug($"Giving message of type {output.GetType().Name}");
return(output);
})
.Via(KafkaProducer.FlowWithContext <string, string, ICommittableOffset>(producerSettings))
.AsSource()
.Log("Produced messages", r => $"Committing {r.Item2.Offset.Topic}:{r.Item2.Offset.Partition}[{r.Item2.Offset.Offset}]")
.ToMaterialized(Committer.SinkWithOffsetContext <IResults <string, string, ICommittableOffset> >(committerSettings), Keep.Both)
.MapMaterializedValue(tuple => DrainingControl <NotUsed> .Create(tuple.Item1, tuple.Item2))
.Run(Materializer);
// One by one, wait while all `totalMessages` will be consumed
for (var i = 1; i < totalMessages; ++i)
{
var consumedExpect = i;
Log.Info($"Waiting for {consumedExpect} to be consumed...");
try
{
await AwaitConditionAsync(() => totalConsumed >= consumedExpect, TimeSpan.FromSeconds(30));
}
finally
{
Log.Info($"Finished waiting for {consumedExpect} messages. Total: {totalConsumed}");
}
Log.Info($"Confirmed that {consumedExpect} messages are consumed");
}
AssertTaskCompletesWithin(TimeSpan.FromSeconds(10), control.DrainAndShutdown());
AssertTaskCompletesWithin(TimeSpan.FromSeconds(10), control2.Shutdown());
AssertTaskCompletesWithin(TimeSpan.FromSeconds(10), result).Should().Be(totalConsumed);
}
public byte[] Encode (ProducerMessage message)
{
return CURRENT_VERSION.Handler.Encode (message, CURRENT_VERSION.Version);
}
public void Encode (ProducerMessage message, IoBuffer buf)
{
Magic.WriteMagic (buf);
buf.Put (CURRENT_VERSION.Version);
CURRENT_VERSION.Handler.Encode (message, buf);
}
public JMCEngine()
{
producerMessage = new ProducerMessage(uri, userName, passWord, exChange);
}
public async Task SupervisionStrategy_Decider_on_complex_stream_should_work()
{
var topic = CreateTopic(1);
var group = CreateGroup(1);
var topicPartition = new TopicPartition(topic, 0);
var committedTopicPartition = new TopicPartition($"{topic}-done", 0);
var callCount = 0;
Directive Decider(Exception cause)
{
callCount++;
return(Directive.Resume);
}
var committerSettings = CommitterSettings.Create(Sys);
var consumerSettings = CreateConsumerSettings <string>(group);
var counter = 0;
// arrange
await Source.From(Enumerable.Range(1, 10))
.Select(elem => new ProducerRecord <Null, string>(topicPartition, elem.ToString()))
.RunWith(KafkaProducer.PlainSink(ProducerSettings), Materializer);
// act
var drainingControl = KafkaConsumer.CommittableSource(consumerSettings, Subscriptions.Assignment(topicPartition))
.Via(Flow.Create <CommittableMessage <Null, string> >().Select(x =>
{
counter++;
// Exception happened here, fail once, when counter is 5
if (counter == 5)
{
throw new Exception("BOOM!");
}
return(x);
}))
.WithAttributes(Attributes.CreateName("CommitableSource").And(ActorAttributes.CreateSupervisionStrategy(Decider)))
.Select(c => (c.Record.Topic, c.Record.Message.Value, c.CommitableOffset))
.SelectAsync(1, async t =>
{
Log.Info($"[{t.Topic}]: {t.Value}");
// simulate a request-response call that takes 10ms to complete here
await Task.Delay(10);
return(t);
})
.Select(t => ProducerMessage.Single(new ProducerRecord <Null, string>(committedTopicPartition, t.Value),
t.CommitableOffset))
.Via(KafkaProducer.FlexiFlow <Null, string, ICommittableOffset>(ProducerSettings)).WithAttributes(Attributes.CreateName("FlexiFlow"))
.Select(m => (ICommittable)m.PassThrough)
.AlsoToMaterialized(Committer.Sink(committerSettings), DrainingControl <NotUsed> .Create)
.To(Flow.Create <ICommittable>()
.Async()
.GroupedWithin(1000, TimeSpan.FromSeconds(1))
.Select(c => c.Count())
.Log("MsgCount").AddAttributes(Attributes.CreateLogLevels(LogLevel.InfoLevel))
.To(Sink.Ignore <int>()))
.Run(Sys.Materializer());
await Task.Delay(TimeSpan.FromSeconds(5));
await GuardWithTimeoutAsync(drainingControl.DrainAndShutdown(), TimeSpan.FromSeconds(10));
// There should be only 1 decider call
callCount.Should().Be(1);
// Assert that all of the messages, except for those that failed in the stage, got committed
var settings = CreateConsumerSettings <Null, string>(group);
var probe = KafkaConsumer
.PlainSource(settings, Subscriptions.Assignment(committedTopicPartition))
.Select(c => c.Message.Value)
.RunWith(this.SinkProbe <string>(), Materializer);
probe.Request(9);
var messages = new List <string>();
for (var i = 0; i < 9; ++i)
{
var message = probe.RequestNext();
messages.Add(message);
}
// Message "5" is missing because the exception happened downstream of the source and we chose to
// ignore it in the decider
messages.Should().BeEquivalentTo(new[] { "1", "2", "3", "4", "6", "7", "8", "9", "10" });
probe.Cancel();
}
public void Encode(ProducerMessage message, IoBuffer buf)
{
var bodyCodec = PayloadCodecFactory.GetCodecByTopicName(message.Topic);
var body = bodyCodec.Encode(message.Topic, message.GetBody<object>());
Encode(message, buf, body, bodyCodec.Type);
}
private void enrichMessageProperties(ProducerMessage msg, String ip)
{
msg.AddDurableSysProperty(MessagePropertyNames.PRODUCER_IP, ip);
}
