public void Send <K, V>(string topic, K key, V value, Headers headers, long timestamp, ISerDes <K> keySerializer, ISerDes <V> valueSerializer)
        {
            var k = key != null?keySerializer.Serialize(key) : null;

            var v = value != null?valueSerializer.Serialize(value) : null;

            producer?.Produce(
                topic,
                new Message <byte[], byte[]> {
                Key = k, Value = v
            },
                (report) => {
                if (report.Error.Code == ErrorCode.NoError && report.Status == PersistenceStatus.Persisted)
                {
                    if (offsets.ContainsKey(report.TopicPartition) && offsets[report.TopicPartition] <= report.Offset)
                    {
                        offsets[report.TopicPartition] = report.Offset;
                    }
                    else
                    {
                        offsets.Add(report.TopicPartition, report.Offset);
                    }
                }
            });
        }
Exemplo n.º 2
0
        public TestInputTopic <K, V> CreateInputTopic <K, V>(string topicName, ISerDes <K> keySerdes, ISerDes <V> valueSerdes)
        {
            var pipeBuilder = new SyncPipeBuilder(GetTask(topicName));
            var pipeInput   = pipeBuilder.Input(topicName, configuration);

            return(new TestInputTopic <K, V>(pipeInput, configuration, keySerdes, valueSerdes));
        }
        public TestMultiInputTopic <K, V> CreateMultiInputTopic <K, V>(string[] topics, ISerDes <K> keySerdes = null, ISerDes <V> valueSerdes = null)
        {
            Dictionary <string, IPipeInput> pipes = new Dictionary <string, IPipeInput>();

            foreach (var t in topics)
            {
                var pipeInput = pipeBuilder.Input(t, configuration);
                pipes.Add(t, pipeInput);
            }

            return(new TestMultiInputTopic <K, V>(pipes, configuration, keySerdes, valueSerdes));
        }
 internal SourceProcessor(string name, string topicName, ISerDes <K> keySerdes, ISerDes <V> valueSerdes, ITimestampExtractor extractor)
     : base(name, keySerdes, valueSerdes)
 {
     TopicName = topicName;
     Extractor = extractor;
 }
Exemplo n.º 5
0
 public GetEventStoreSubscriber(ISerDes serDes, ILogger <GetEventStoreSubscriber> logger)
 {
     _serDes = serDes;
     _logger = logger;
 }
Exemplo n.º 6
0
 internal SinkProcessor(string name, ITopicNameExtractor <K, V> topicNameExtractor, ISerDes <K> keySerdes, ISerDes <V> valueSerdes)
     : base(name, keySerdes, valueSerdes)
 {
     this.topicNameExtractor = topicNameExtractor;
 }
 public TimestampedKeyValueStoreImpl(IKeyValueStore <Bytes, byte[]> wrapped, ISerDes <K> keySerdes, ISerDes <ValueAndTimestamp <V> > valueSerdes)
     : base(wrapped, keySerdes, valueSerdes)
 {
 }
Exemplo n.º 8
0
 internal KTable(string name, ISerDes <K> keySerde, ISerDes <V> valSerde, List <string> sourceNodes, String queryableStoreName, IProcessorSupplier <K, S> processorSupplier, StreamGraphNode streamsGraphNode, InternalStreamBuilder builder)
     : base(name, keySerde, valSerde, sourceNodes, streamsGraphNode, builder)
 {
     this.processorSupplier  = processorSupplier;
     this.queryableStoreName = queryableStoreName;
 }
Exemplo n.º 9
0
        // TODO : use default partitioner .net
        //private readonly DefaultPartitioner defaultPartitioner;

        public DefaultStreamPartitioner(ISerDes <K> keySerializer, Metadata cluster)
        {
            this.cluster       = cluster;
            this.keySerializer = keySerializer;
            //this.defaultPartitioner = new DefaultPartitioner();
        }
Exemplo n.º 10
0
        public TimeWindowedKStream(WindowOptions <W> windowOptions, GroupedStreamAggregateBuilder <K, V> aggBuilder, string name, ISerDes <K> keySerde, ISerDes <V> valSerde, List <string> sourceNodes, StreamGraphNode streamsGraphNode, InternalStreamBuilder builder)
            : base(name, keySerde, valSerde, sourceNodes, streamsGraphNode, builder)
        {
            CheckIfParamNull(windowOptions, "windowOptions");

            this.windowOptions = windowOptions;
            this.aggBuilder    = aggBuilder;
        }
Exemplo n.º 11
0
        /// <summary>
        /// Create <see cref="TestOutputTopic{K, V}"/> to be used for reading records from topic.
        /// </summary>
        /// <typeparam name="K">Key type</typeparam>
        /// <typeparam name="V">Value type</typeparam>
        /// <param name="topicName">the name of the topic</param>
        /// <param name="consumeTimeout">Consumer timeout</param>
        /// <param name="keySerdes">Key deserializer</param>
        /// <param name="valueSerdes">Value deserializer</param>
        /// <returns><see cref="TestOutputTopic{K, V}"/> instance</returns>
        public TestOutputTopic <K, V> CreateOuputTopic <K, V>(string topicName, TimeSpan consumeTimeout, ISerDes <K> keySerdes = null, ISerDes <V> valueSerdes = null)
        {
            var output = behavior.CreateOutputTopic(topicName, consumeTimeout, keySerdes, valueSerdes);

            outputs.Add(topicName, output.Pipe);
            return(output);
        }
Exemplo n.º 12
0
        /// <summary>
        /// Create <see cref="TestInputTopic{K, V}"/> to be used for piping records to topic
        /// </summary>
        /// <typeparam name="K">key type</typeparam>
        /// <typeparam name="V">value type</typeparam>
        /// <param name="keySerdes">Key serializer</param>
        /// <param name="valueSerdes">Value serializer</param>
        /// <param name="topicName">the name of the topic</param>
        /// <returns><see cref="TestInputTopic{K, V}"/> instance</returns>
        public TestInputTopic <K, V> CreateInputTopic <K, V>(string topicName, ISerDes <K> keySerdes, ISerDes <V> valueSerdes)
        {
            var input = behavior.CreateInputTopic(topicName, keySerdes, valueSerdes);

            inputs.Add(topicName, input.Pipe);
            return(input);
        }
Exemplo n.º 13
0
 internal TestOutputTopic(IPipeOutput pipe, IStreamConfig configuration, ISerDes <K> keySerdes, ISerDes <V> valueSerdes)
 {
     this.pipe          = pipe;
     this.configuration = configuration;
     this.keySerdes     = keySerdes;
     this.valueSerdes   = valueSerdes;
 }
Exemplo n.º 14
0
        public TestOutputTopic <K, V> CreateOutputTopic <K, V>(string topicName, TimeSpan consumeTimeout, ISerDes <K> keySerdes = null, ISerDes <V> valueSerdes = null)
        {
            var pipeBuilder = new SyncPipeBuilder(null, producer);
            var pipeOutput  = pipeBuilder.Output(topicName, consumeTimeout, configuration, token);

            return(new TestOutputTopic <K, V>(pipeOutput, topicConfiguration, keySerdes, valueSerdes));
        }
 protected AbstractStoreBuilder(String name, ISerDes <K> keySerde, ISerDes <V> valueSerde)
 {
     this.name   = name;
     keySerdes   = keySerde;
     valueSerdes = valueSerde;
 }
 /// <summary>
 /// Configure key and value serdes
 /// </summary>
 /// <param name="keySerdes">Key serdes</param>
 /// <param name="valueSerdes">Value serdes</param>
 /// <returns>Itself</returns>
 public Materialized <K, V, S> With(ISerDes <K> keySerdes, ISerDes <V> valueSerdes)
 {
     KeySerdes   = keySerdes;
     ValueSerdes = valueSerdes;
     return(this);
 }
Exemplo n.º 17
0
        /// <summary>
        /// Create <see cref="TestMultiInputTopic{K, V}"/> to be used for piping records to multiple topics in same time.
        /// Need to call <see cref="TestMultiInputTopic{K, V}.Flush"/> at the end of writting !
        /// </summary>
        /// <typeparam name="K">key type</typeparam>
        /// <typeparam name="V">value type</typeparam>
        /// <param name="keySerdes">key serializer instance</param>
        /// <param name="valueSerdes">value serializer instance</param>
        /// <param name="topics">the list of topics</param>
        /// <returns><see cref="TestMultiInputTopic{K, V}"/> instance</returns>
        public TestMultiInputTopic <K, V> CreateMultiInputTopic <K, V>(ISerDes <K> keySerdes, ISerDes <V> valueSerdes, params string[] topics)
        {
            var multi = behavior.CreateMultiInputTopic(topics, keySerdes, valueSerdes);

            foreach (var topic in topics)
            {
                inputs.Add(topic, multi.GetPipe(topic));
            }
            return(multi);
        }