static void Main(string[] args) { string accessToken = "874229294-vTpHJDcl8K0I7Ae6H29Ezvpw5ZVsX3uT2wbtDNkD"; string accessTokenSecret = "3XNHrHBanj3x2fOTCm53t7L6hd7NSDlLqWIgc24um3x83"; string customerKey = "kO6JlIwLa4czaQSqvHXLFfOhb"; string customerSecret = "rSbhwvMR0vq1UCpkztfl3PvazveNHCKg6879J8yd0kLu7Q0xSF"; var config = new StreamConfig() { ConsumerKey = customerKey, ConsumerSecret = customerSecret, AccessToken = accessToken, AccessSecret = accessTokenSecret, GeoOnly = true }; var stream = new TwitterStreamClient(config); // subscribe to the event handler stream.TweetReceivedEvent += (sender, arguments) => { Console.WriteLine(arguments.Tweet.coordinates); }; stream.ExceptionReceived += (sender, exception) => Console.WriteLine(exception.TwitterException.ResponseMessage); stream.Start(); }
public TwitterStreamClient(StreamConfig options) { _consumerKey = options.ConsumerKey; _consumerSecret = options.ConsumerSecret; _accessToken = options.AccessToken; _accessSecret = options.AccessSecret; _isGeoEnabled = options.GeoOnly; _keywords = options.TrackKeywords; }
public void ProductionExceptionFatalHandlerFailTest() { bool errorState = false; var _return = new List <KeyValuePair <string, string> >(); var config = new StreamConfig <StringSerDes, StringSerDes>(); var dt = DateTime.Now; var timeout = TimeSpan.FromSeconds(10); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; config.PollMs = 10; config.ProductionExceptionHandler += (r) => ExceptionHandlerResponse.FAIL; var options = new ProducerSyncExceptionOptions { IsFatal = true }; var supplier = new ProducerSyncException(options); var builder = new StreamBuilder(); builder .Stream <string, string>("test") .To("test-output"); builder.Stream <string, string>("test-output") .Peek((k, v) => _return.Add(KeyValuePair.Create(k, v))); var t = builder.Build(); using (var driver = new TopologyTestDriver(t.Builder, config, TopologyTestDriver.Mode.ASYNC_CLUSTER_IN_MEMORY, supplier)) { var inputtopic = driver.CreateInputTopic <string, string>("test"); inputtopic.PipeInput("coucou"); while (!errorState) { errorState = driver.IsError; Thread.Sleep(10); if (DateTime.Now > dt + timeout) { break; } } Assert.IsTrue(driver.IsError); } Assert.AreEqual(new List <KeyValuePair <string, string> >(), _return); }
public void WithNullMaterialize() { // CERTIFIED THAT SAME IF Materialize is null, a state store exist for count processor with a generated namestore var config = new StreamConfig <StringSerDes, StringSerDes>(); var serdes = new StringSerDes(); config.ApplicationId = "test-window-count"; var builder = new StreamBuilder(); Materialized <string, int, IWindowStore <Bytes, byte[]> > m = null; builder .Stream <string, string>("topic") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(2000)) .Aggregate( () => 0, (k, v, agg) => Math.Max(v.Length, agg), m); var topology = builder.Build(); TaskId id = new TaskId { Id = 0, Partition = 0 }; var processorTopology = topology.Builder.BuildTopology(id); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); var part = new TopicPartition("topic", 0); StreamTask task = new StreamTask( "thread-0", id, new List <TopicPartition> { part }, processorTopology, consumer, config, supplier, null); task.GroupMetadata = consumer as SyncConsumer; Assert.Throws <StreamsException>(() => task.InitializeStateStores()); }
public void StateStoreRange() { var source = new CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-config"; config.PollMs = 50; var topicConfiguration = config.Clone(); topicConfiguration.ApplicationId = $"test-driver-{config.ApplicationId}"; var builder = new StreamBuilder(); builder.Table("test", InMemory <string, string> .As("store")); var driver = new ClusterInMemoryTopologyDriver("client", builder.Build().Builder, config, topicConfiguration, TimeSpan.FromSeconds(1), source.Token); driver.StartDriver(); var input = driver.CreateInputTopic("test", new StringSerDes(), new StringSerDes()); var store = driver.GetStateStore <string, string>("store"); Assert.IsNotNull(store); Assert.IsInstanceOf <MockReadOnlyKeyValueStore <string, string> >(store); input.PipeInput("coucou", "1"); input.PipeInput("test", "2"); Thread.Sleep(100); var range = ((MockReadOnlyKeyValueStore <string, string>)store).Range("coucou", "test").ToList(); Assert.AreEqual(2, range.Count); Assert.AreEqual("coucou", range[0].Key); Assert.AreEqual("1", range[0].Value); Assert.AreEqual("test", range[1].Key); Assert.AreEqual("2", range[1].Value); var reverseRange = ((MockReadOnlyKeyValueStore <string, string>)store).ReverseRange("coucou", "test").ToList(); Assert.AreEqual(2, reverseRange.Count); Assert.AreEqual("test", reverseRange[0].Key); Assert.AreEqual("2", reverseRange[0].Value); Assert.AreEqual("coucou", reverseRange[1].Key); Assert.AreEqual("1", reverseRange[1].Value); source.Cancel(); driver.Dispose(); }
public void GetKVStateStoreInvalidStateStoreException() { var timeout = TimeSpan.FromSeconds(10); var source = new CancellationTokenSource(); bool state = false; DateTime dt = DateTime.Now; var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; var builder = new StreamBuilder(); builder .Stream <string, string>("test") .GroupByKey() .Count(InMemory <string, long> .As("store")); var t = builder.Build(); var stream = new KafkaStream(t, config, new SyncKafkaSupplier()); stream.StateChanged += (old, @new) => { if ([email protected](KafkaStream.State.RUNNING)) { if (!state) { Assert.Throws <InvalidStateStoreException>(() => stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, long>()))); state = true; } } }; stream.Start(source.Token); while (!state) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } Assert.IsTrue(state); source.Cancel(); stream.Close(); }
public void TaskManagerAssignedUnknownPartitions() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; var serdes = new StringSerDes(); var builder = new StreamBuilder(); builder.Stream <string, string>("topic") .Map((k, v) => KeyValuePair.Create(k.ToUpper(), v.ToUpper())) .To("topic2"); var topology = builder.Build(); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); var taskCreator = new TaskCreator(topology.Builder, config, "thread-0", supplier, producer); var taskManager = new TaskManager(topology.Builder, taskCreator, supplier.GetAdmin(config.ToAdminConfig("admin")), consumer); taskManager.CreateTasks( new List <TopicPartition> { new TopicPartition("topic", 0), new TopicPartition("topic", 1) }); taskManager.RevokeTasks( new List <TopicPartition> { new TopicPartition("topic", 1) }); taskManager.CreateTasks( new List <TopicPartition> { new TopicPartition("topic", 0), new TopicPartition("topic", 1), new TopicPartition("topic", 2) }); Assert.AreEqual(3, taskManager.ActiveTasks.Count()); Assert.AreEqual(0, taskManager.RevokedTasks.Count()); taskManager.Close(); }
public void ApplicationIdProperty() { var config = new StreamConfig(); config.ApplicationId = "test"; var stateMgt = new ProcessorStateManager( new TaskId() { Id = 0, Partition = 0 }, new List <Confluent.Kafka.TopicPartition> { new Confluent.Kafka.TopicPartition("test", 0) }); ProcessorContext context = new ProcessorContext(config, stateMgt); Assert.AreEqual("test", context.ApplicationId); }
internal static bool IsCompatible(this StreamConfig config, StreamConfig otherConfig) { switch (config) { case VideoStreamConfig videoConfig: return(videoConfig.IsCompatible(otherConfig as VideoStreamConfig)); case AudioStreamConfig audioConfig: return(audioConfig.IsCompatible(otherConfig as AudioStreamConfig)); case null: throw new ArgumentNullException(nameof(config), "Config cannot be null"); default: throw new ArgumentException("Unsupported configuration type", nameof(config)); } }
public void DeserializeOK() { var mockSchemaClient = new MockSchemaRegistryClient(); var config = new StreamConfig(); var serdes = new MockProtoSerDes(mockSchemaClient); serdes.Initialize(new Net.SerDes.SerDesContext(config)); var person = new Helpers.Proto.Person { Age = 18, FirstName = "TEST", LastName = "TEST" }; var bytes = serdes.Serialize(person, new Confluent.Kafka.SerializationContext(Confluent.Kafka.MessageComponentType.Value, topic)); var pbis = serdes.Deserialize(bytes, new Confluent.Kafka.SerializationContext(Confluent.Kafka.MessageComponentType.Value, topic)); Assert.AreEqual(18, pbis.Age); Assert.AreEqual("TEST", pbis.FirstName); Assert.AreEqual("TEST", pbis.LastName); }
public void GetStateStoreBeforeRunningState() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; var builder = new StreamBuilder(); builder.Table("topic", InMemory <string, string> .As("store")); var t = builder.Build(); var stream = new KafkaStream(t, config, new SyncKafkaSupplier()); Assert.Throws <IllegalStateException>(() => stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, string>()))); stream.Dispose(); }
public void GetStateStore() { var timeout = TimeSpan.FromSeconds(10); var source = new CancellationTokenSource(); bool isRunningState = false; DateTime dt = DateTime.Now; var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; var builder = new StreamBuilder(); builder.Table("topic", InMemory <string, string> .As("store")); var t = builder.Build(); var stream = new KafkaStream(t, config, new SyncKafkaSupplier()); stream.StateChanged += (old, @new) => { if (@new.Equals(KafkaStream.State.RUNNING)) { isRunningState = true; } }; stream.Start(source.Token); while (!isRunningState) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } Assert.IsTrue(isRunningState); if (isRunningState) { var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, string>()));; Assert.IsNotNull(store); } source.Cancel(); stream.Close(); }
static async Task Main(string[] args) { CancellationTokenSource source = new CancellationTokenSource(); var config = new StreamConfig(); config.ApplicationId = "test-app"; config.BootstrapServers = "192.168.56.1:9092"; config.SaslMechanism = SaslMechanism.Plain; config.SaslUsername = "******"; config.SaslPassword = "******"; config.SecurityProtocol = SecurityProtocol.SaslPlaintext; config.AutoOffsetReset = AutoOffsetReset.Earliest; // NEED FOR SchemaAvroSerDes config.SchemaRegistryUrl = "http://192.168.56.1:8081"; config.AutoRegisterSchemas = true; StreamBuilder builder = new StreamBuilder(); var table = builder.Table("product", new Int32SerDes(), new SchemaAvroSerDes <Product>(), InMemory <int, Product> .As("product-store")); builder.Stream <int, Order, Int32SerDes, SchemaAvroSerDes <Order> >("orders") .Join(table, (order, product) => new OrderProduct { order_id = order.order_id, price = order.price, product_id = product.product_id, product_name = product.name, product_price = product.price }) .To <Int32SerDes, SchemaAvroSerDes <OrderProduct> >("orders-output"); Topology t = builder.Build(); KafkaStream stream = new KafkaStream(t, config); Console.CancelKeyPress += (o, e) => { stream.Dispose(); }; await stream.StartAsync(); }
public void SchemaRegistryConfig() { var mockSchemaClient = new MockSchemaRegistryClient(); var config = new StreamConfig(); config.AutoRegisterSchemas = true; config.SchemaRegistryMaxCachedSchemas = 1; config.SchemaRegistryRequestTimeoutMs = 30; config.SubjectNameStrategy = SubjectNameStrategy.TopicRecord; var serdes = new MockProtoSerDes(mockSchemaClient); serdes.Initialize(new Net.SerDes.SerDesContext(config)); Assert.AreEqual(1, mockSchemaClient.MaxCachedSchemas); Assert.AreEqual(30, mockSchemaClient.RequestTimeoutMs); }
public void ReduceAndQueryInStateStore() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-reduce"; var builder = new StreamBuilder(); var stream = builder .Stream <string, string>("topic") .MapValues(v => v.Length) .GroupBy((k, v) => k.ToUpper()); stream.Count(InMemory <string, long> .As("count-store")); stream.Reduce( (v1, v2) => Math.Max(v1, v2), InMemory <string, int> .As("reduce-store").WithValueSerdes <Int32SerDes>()); var topology = builder.Build(); using var driver = new TopologyTestDriver(topology, config); var input = driver.CreateInputTopic <string, string>("topic"); input.PipeInput("test", "1"); input.PipeInput("test", "120"); input.PipeInput("test", "30"); input.PipeInput("coucou", "120"); var store = driver.GetKeyValueStore <string, int>("reduce-store"); Assert.IsNotNull(store); Assert.AreEqual(2, store.ApproximateNumEntries()); var el = store.Get("TEST"); Assert.IsNotNull(el); Assert.AreEqual(3, el); var storeCount = driver.GetKeyValueStore <string, long>("count-store"); Assert.IsNotNull(storeCount); Assert.AreEqual(2, store.ApproximateNumEntries()); var e = storeCount.Get("TEST"); Assert.IsNotNull(e); Assert.AreEqual(3, e); }
public async Task GetStateStoreDoesntExists() { var timeout = TimeSpan.FromSeconds(10); bool isRunningState = false; DateTime dt = DateTime.Now; var supplier = new SyncKafkaSupplier(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; var builder = new StreamBuilder(); builder.Table("topic", InMemory <string, string> .As("store")); var t = builder.Build(); var stream = new KafkaStream(t, config, supplier); stream.StateChanged += (old, @new) => { if (@new.Equals(KafkaStream.State.RUNNING)) { isRunningState = true; } }; await stream.StartAsync(); while (!isRunningState) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } Assert.IsTrue(isRunningState); if (isRunningState) { Assert.Throws <InvalidStateStoreException>(() => stream.Store(StoreQueryParameters.FromNameAndType("stodfdsfdsfre", QueryableStoreTypes.KeyValueStore <string, string>()))); } stream.Dispose(); }
public void MultiBranchWithElements() { var builder = new StreamBuilder(); var branchs = builder.Stream <string, int>("topic") .Branch((k, v) => v % 2 == 0, (k, v) => v % 2 > 0); branchs[0].To("topic-pair"); branchs[1].To("topic-impair"); var config = new StreamConfig <StringSerDes, Int32SerDes>(); config.ApplicationId = "test-branch"; Topology t = builder.Build(); using var driver = new TopologyTestDriver(t, config); var inputTopic = driver.CreateInputTopic <string, int>("topic"); var outputTopicPair = driver.CreateOuputTopic <string, int>("topic-pair"); var outputTopicImpair = driver.CreateOuputTopic <string, int>("topic-impair"); var expectedPair = new List <KeyValuePair <string, int> >(); var expectedImpair = new List <KeyValuePair <string, int> >(); for (int i = 0; i < 10; i++) { string key = i.ToString(); int value = i; inputTopic.PipeInput(key, value); if (i % 2 == 0) { expectedPair.Add(KeyValuePair.Create(key, value)); } else { expectedImpair.Add(KeyValuePair.Create(key, value)); } } var listPair = outputTopicPair.ReadKeyValueList().Select(r => KeyValuePair.Create(r.Message.Key, r.Message.Value)).ToList(); var listImpair = outputTopicImpair.ReadKeyValueList().Select(r => KeyValuePair.Create(r.Message.Key, r.Message.Value)).ToList(); Assert.AreEqual(expectedPair, listPair); Assert.AreEqual(expectedImpair, listImpair); }
public void StreamAddCorrectConfig() { var stream = new StreamConfig(); stream.ApplicationId = "unittest"; stream.AddConfig("sasl.password", "coucou"); var adminConfig = stream.ToAdminConfig("admin"); var consumerConfig = stream.ToConsumerConfig(); var producerConfig = stream.ToProducerConfig(); var globalConfig = stream.ToGlobalConsumerConfig("global"); Assert.AreEqual("coucou", adminConfig.SaslPassword); Assert.AreEqual("coucou", consumerConfig.SaslPassword); Assert.AreEqual("coucou", producerConfig.SaslPassword); Assert.AreEqual("coucou", globalConfig.SaslPassword); }
public void TableTableOuterJoinGetterSupplier() { var config = new StreamConfig <StringSerDes, StringSerDes> { ApplicationId = "test-table-table-outer-join" }; StreamBuilder builder = new StreamBuilder(); var table1 = builder.Table("users", InMemory <string, string> .As("store-users")); var table2 = builder.Table("regions", InMemory <string, string> .As("store-regions")); var stream = builder.Stream <string, string>("orders"); var tableJoin = table1.OuterJoin(table2, (v1, v2) => $"{v1 ?? "?"}-{v2 ?? "?"}"); stream .Join(tableJoin, (order, ur) => $"Order:{order}|UserRegion:{ur}") .To("topic-output"); Topology t = builder.Build(); using (var driver = new TopologyTestDriver(t, config)) { var inputTopic1 = driver.CreateInputTopic <string, string>("users"); var inputTopic2 = driver.CreateInputTopic <string, string>("regions"); var inputTopic3 = driver.CreateInputTopic <string, string>("orders"); var outputTopic = driver.CreateOuputTopic <string, string>("topic-output"); inputTopic1.PipeInput("sylvain", "sylvain"); inputTopic1.PipeInput("lise", "lise"); inputTopic2.PipeInput("sylvain", "France"); inputTopic2.PipeInput("remi", "USA"); inputTopic3.PipeInput("sylvain", "iPhone12Pro"); inputTopic3.PipeInput("lise", "PixelA4"); inputTopic3.PipeInput("remi", "Tab"); var records = outputTopic.ReadKeyValuesToMap(); Assert.IsNotNull(records); Assert.AreEqual(3, records.Count); Assert.AreEqual("Order:iPhone12Pro|UserRegion:sylvain-France", records["sylvain"]); Assert.AreEqual("Order:PixelA4|UserRegion:lise-?", records["lise"]); Assert.AreEqual("Order:Tab|UserRegion:?-USA", records["remi"]); } }
public void Begin() { valueAndTimestampSerDes = new ValueAndTimestampSerDes <string>(stringSerDes); config = new StreamConfig(); config.ApplicationId = $"unit-test-changelogging-tkv"; id = new TaskId { Id = 0, Partition = 0 }; partition = new TopicPartition("source", 0); kafkaSupplier = new SyncKafkaSupplier(); var producerConfig = new ProducerConfig(); producerConfig.ClientId = "producer-1"; var producerClient = kafkaSupplier.GetProducer(producerConfig); recordCollector = new RecordCollector("p-1", config, id, new NoRunnableSensor("s", "s", MetricsRecordingLevel.DEBUG)); recordCollector.Init(ref producerClient); var changelogsTopics = new Dictionary <string, string> { { "test-store", "test-store-changelog" } }; stateManager = new ProcessorStateManager( id, new List <TopicPartition> { partition }, changelogsTopics, new MockChangelogRegister(), new MockOffsetCheckpointManager()); task = new Mock <AbstractTask>(); task.Setup(k => k.Id).Returns(id); context = new ProcessorContext(task.Object, config, stateManager, new StreamMetricsRegistry()); context.UseRecordCollector(recordCollector); var inmemorystore = new InMemoryKeyValueStore("test-store"); store = new ChangeLoggingTimestampedKeyValueBytesStore(inmemorystore); store.Init(context, store); }
public void MergeTwoStreamsWithElements() { var data1 = new List <KeyValuePair <string, int> > { KeyValuePair.Create("key1", 1), KeyValuePair.Create("key2", 2) }; var data2 = new List <KeyValuePair <string, int> > { KeyValuePair.Create("key3", 3) }; var observed = new List <KeyValuePair <string, int> >(); var builder = new StreamBuilder(); var stream1 = builder.Stream <string, int>("topic1"); var stream2 = builder.Stream <string, int>("topic2"); var merged = stream1.Merge(stream2); merged.Peek((k, v) => observed.Add(KeyValuePair.Create(k, v))); var config = new StreamConfig <StringSerDes, Int32SerDes> { ApplicationId = "test-merge" }; Topology t = builder.Build(); using var driver = new TopologyTestDriver(t, config); var inputTopic1 = driver.CreateInputTopic <string, int>("topic1"); inputTopic1.PipeInputs(data1); var inputTopic2 = driver.CreateInputTopic <string, int>("topic2"); inputTopic2.PipeInputs(data2); var expected = new List <KeyValuePair <string, int> >(); expected.AddRange(data1); expected.AddRange(data2); Assert.AreEqual(expected, observed); }
public async Task StartKafkaStream() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var t = builder.Build(); var stream = new KafkaStream(t, config, new SyncKafkaSupplier()); await stream.StartAsync(); Thread.Sleep(1500); stream.Dispose(); }
public void StartDriverKO() { var source = new CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-config"; var topicConfiguration = config.Clone(); topicConfiguration.ApplicationId = $"test-driver-{config.ApplicationId}"; var builder = new StreamBuilder(); var driver = new ClusterInMemoryTopologyDriver("client", builder.Build().Builder, config, topicConfiguration, TimeSpan.FromSeconds(1), source.Token); Assert.Throws <StreamsException>(() => driver.StartDriver()); source.Cancel(); driver.Dispose(); }
public void SchemaRegistryAvroSerializerConfig() { var config = new StreamConfig { SubjectNameStrategy = SubjectNameStrategy.TopicRecord, AutoRegisterSchemas = true, UseLatestVersion = false, BufferBytes = 1024 }; var serdes = new SchemaAvroSerDes <Order>(); var schemaConfig = serdes.GetSerializerConfig(config); Assert.AreEqual(Confluent.SchemaRegistry.SubjectNameStrategy.TopicRecord, schemaConfig.SubjectNameStrategy); Assert.AreEqual(true, schemaConfig.AutoRegisterSchemas); Assert.AreEqual(false, schemaConfig.UseLatestVersion); Assert.AreEqual(1024, schemaConfig.BufferBytes); }
public void StreamWithNullLeftStream() { var config = new StreamConfig <StringSerDes, StringSerDes> { ApplicationId = "test-stream-stream-left-join" }; StreamBuilder builder = new StreamBuilder(); var stream = builder.Stream <string, string>("topic1"); Assert.Throws <ArgumentNullException>(() => builder .Stream <string, string>("topic2") .LeftJoin( null, new MyJoinerMapper(), JoinWindowOptions.Of(TimeSpan.FromSeconds(10)))); }
public void KTableSourceNoMaterialize2() { var builder = new StreamBuilder(); builder.Table <string, string, StringSerDes, StringSerDes>("table-topic"); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-map"; Topology t = builder.Build(); using var driver = new TopologyTestDriver(t, config); var inputTopic = driver.CreateInputTopic <string, string>("table-topic"); inputTopic.PipeInput("key1", "1"); inputTopic.PipeInput("key2", "2"); }
public void GetStateStoreNotExist() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-config"; var builder = new StreamBuilder(); builder.Stream <string, string>("test").To("test2"); var driver = new TaskSynchronousTopologyDriver("client", builder.Build().Builder, config, config, default); driver.CreateInputTopic("test", new StringSerDes(), new StringSerDes()); var store = driver.GetStateStore <string, string>("store"); Assert.IsNull(store); driver.Dispose(); }
/// <summary> /// Sets Stream configuration /// If no argument is provided, current configuration will be pushed. /// Otherwise provided configuration will be set as current and pushed. /// <param name="config">StreamConfig</param> /// </summary> public void SetStreamConfiguration(StreamConfig config = null) { logger.Info($"{streamType}: Using {(config == null ? "stored" : "provided")} configuration"); if (config == null) { if (Configuration == null) { throw new ArgumentNullException(nameof(Configuration), "Current configuration is null"); } } else { Configuration = config; } PushStreamConfig(Configuration); }
public void TableTableMergeJoinForwardSendOldValues() { var config = new StreamConfig <StringSerDes, StringSerDes> { ApplicationId = "test-table-table-left-join" }; StreamBuilder builder = new StreamBuilder(); var table1 = builder.Table("users", InMemory <string, string> .As("store-users")); var table2 = builder.Table("regions", InMemory <string, string> .As("store-regions")); var table3 = builder.Table("country", InMemory <string, string> .As("store-country")); var stream = builder.Stream <string, string>("orders"); var tableJoin = table1.LeftJoin(table2, (v1, v2) => $"{v1}-{v2 ?? "?"}", InMemory <string, string> .As("merge1")); var tableJoin2 = tableJoin.LeftJoin(table3, (v1, v2) => $"{v1}-{v2 ?? "?"}"); stream .Join(tableJoin2, (order, ur) => $"Order:{order}|UserRegionCountry:{ur}") .To("topic-output"); Topology t = builder.Build(); using (var driver = new TopologyTestDriver(t, config)) { var inputTopic1 = driver.CreateInputTopic <string, string>("users"); var inputTopic2 = driver.CreateInputTopic <string, string>("regions"); var inputTopic3 = driver.CreateInputTopic <string, string>("orders"); var inputTopic4 = driver.CreateInputTopic <string, string>("country"); var outputTopic = driver.CreateOuputTopic <string, string>("topic-output"); inputTopic1.PipeInput("sylvain", "sylvain"); inputTopic2.PipeInput("sylvain", "Europe"); inputTopic4.PipeInput("sylvain", "France"); inputTopic3.PipeInput("sylvain", "iPhone12Pro"); var records = outputTopic.ReadKeyValuesToMap(); Assert.IsNotNull(records); Assert.AreEqual(1, records.Count); Assert.AreEqual("Order:iPhone12Pro|UserRegionCountry:sylvain-Europe-France", records["sylvain"]); } }
public void WithNullMaterialize() { // CERTIFIED THAT SAME IF Materialize is null, a state store exist for count processor with a generated namestore var config = new StreamConfig(); var serdes = new StringSerDes(); config.ApplicationId = "test-window-reduce"; var builder = new StreamBuilder(); builder .Stream <string, string>("topic") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(2000)) .Reduce((v1, v2) => v1.Length > v2.Length ? v1 : v2); var topology = builder.Build(); TaskId id = new TaskId { Id = 0, Partition = 0 }; var processorTopology = topology.Builder.BuildTopology(id); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); var part = new TopicPartition("topic", 0); StreamTask task = new StreamTask( "thread-0", id, new List <TopicPartition> { part }, processorTopology, consumer, config, supplier, null, new MockChangelogRegister(), new StreamMetricsRegistry()); task.GroupMetadata = consumer as SyncConsumer; Assert.Throws <StreamsException>(() => task.InitializeStateStores()); }
public void ProductionExceptionRecoverableHandlerFailTest() { var _return = new List <KeyValuePair <string, string> >(); var config = new StreamConfig <StringSerDes, StringSerDes>(); var dt = DateTime.Now; var timeout = TimeSpan.FromSeconds(10); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; config.PollMs = 10; config.ProductionExceptionHandler += (r) => ExceptionHandlerResponse.FAIL; var options = new ProducerSyncExceptionOptions { IsRecoverable = true }; var supplier = new ProducerSyncException(options); var builder = new StreamBuilder(); builder .Stream <string, string>("test") .To("test-output"); builder.Stream <string, string>("test-output") .Peek((k, v) => _return.Add(KeyValuePair.Create(k, v))); var t = builder.Build(); using (var driver = new TopologyTestDriver(t.Builder, config, TopologyTestDriver.Mode.ASYNC_CLUSTER_IN_MEMORY, supplier)) { var inputtopic = driver.CreateInputTopic <string, string>("test"); var outputTopic = driver.CreateOuputTopic <string, string>("test-output"); inputtopic.PipeInput("coucou"); inputtopic.PipeInput("coucou"); while (_return.Count == 0) { ; } var expected = new List <KeyValuePair <string, string> >(); expected.Add(KeyValuePair.Create <string, string>(null, "coucou")); Assert.AreEqual(expected, _return); } }
public async Task StartKafkaStreamWaitRunningState() { var timeout = TimeSpan.FromSeconds(10); bool isRunningState = false; DateTime dt = DateTime.Now; var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var t = builder.Build(); var stream = new KafkaStream(t, config, new SyncKafkaSupplier()); stream.StateChanged += (old, @new) => { if (@new.Equals(KafkaStream.State.RUNNING)) { isRunningState = true; } }; await stream.StartAsync(); while (!isRunningState) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } var sensors = stream.Metrics(); Assert.IsTrue(sensors.Any()); stream.Dispose(); Assert.IsTrue(isRunningState); }
public TwitterConnection() { _context = GlobalHost.ConnectionManager.GetHubContext<TwitterHub>(); var config = new StreamConfig() { ConsumerKey = _consumerKey, ConsumerSecret = _consumerSecret, AccessToken = _accessKey, AccessSecret = _accessToken, GeoOnly = true }; var stream = new TwitterStreamClient(config); stream.TweetReceivedEvent += (sender, args) => { Debug.WriteLine(args.Tweet.ToString()); _context.Clients.All.broadcast(args.Tweet); }; stream.Start(); }