static void Main(string[] args)
{
string accessToken = "874229294-vTpHJDcl8K0I7Ae6H29Ezvpw5ZVsX3uT2wbtDNkD";
string accessTokenSecret = "3XNHrHBanj3x2fOTCm53t7L6hd7NSDlLqWIgc24um3x83";
string customerKey = "kO6JlIwLa4czaQSqvHXLFfOhb";
string customerSecret = "rSbhwvMR0vq1UCpkztfl3PvazveNHCKg6879J8yd0kLu7Q0xSF";
var config = new StreamConfig()
{
ConsumerKey = customerKey,
ConsumerSecret = customerSecret,
AccessToken = accessToken,
AccessSecret = accessTokenSecret,
GeoOnly = true
};
var stream = new TwitterStreamClient(config);
// subscribe to the event handler
stream.TweetReceivedEvent += (sender, arguments) =>
{
Console.WriteLine(arguments.Tweet.coordinates);
};
stream.ExceptionReceived += (sender, exception) => Console.WriteLine(exception.TwitterException.ResponseMessage);
stream.Start();
}
public TwitterStreamClient(StreamConfig options)
{
_consumerKey = options.ConsumerKey;
_consumerSecret = options.ConsumerSecret;
_accessToken = options.AccessToken;
_accessSecret = options.AccessSecret;
_isGeoEnabled = options.GeoOnly;
_keywords = options.TrackKeywords;
}
public void ProductionExceptionFatalHandlerFailTest()
{
bool errorState = false;
var _return = new List <KeyValuePair <string, string> >();
var config = new StreamConfig <StringSerDes, StringSerDes>();
var dt = DateTime.Now;
var timeout = TimeSpan.FromSeconds(10);
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
config.PollMs = 10;
config.ProductionExceptionHandler += (r) => ExceptionHandlerResponse.FAIL;
var options = new ProducerSyncExceptionOptions {
IsFatal = true
};
var supplier = new ProducerSyncException(options);
var builder = new StreamBuilder();
builder
.Stream <string, string>("test")
.To("test-output");
builder.Stream <string, string>("test-output")
.Peek((k, v) => _return.Add(KeyValuePair.Create(k, v)));
var t = builder.Build();
using (var driver = new TopologyTestDriver(t.Builder, config, TopologyTestDriver.Mode.ASYNC_CLUSTER_IN_MEMORY, supplier))
{
var inputtopic = driver.CreateInputTopic <string, string>("test");
inputtopic.PipeInput("coucou");
while (!errorState)
{
errorState = driver.IsError;
Thread.Sleep(10);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(driver.IsError);
}
Assert.AreEqual(new List <KeyValuePair <string, string> >(), _return);
}
public void WithNullMaterialize()
{
// CERTIFIED THAT SAME IF Materialize is null, a state store exist for count processor with a generated namestore
var config = new StreamConfig <StringSerDes, StringSerDes>();
var serdes = new StringSerDes();
config.ApplicationId = "test-window-count";
var builder = new StreamBuilder();
Materialized <string, int, IWindowStore <Bytes, byte[]> > m = null;
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Aggregate(
() => 0,
(k, v, agg) => Math.Max(v.Length, agg),
m);
var topology = builder.Build();
TaskId id = new TaskId {
Id = 0, Partition = 0
};
var processorTopology = topology.Builder.BuildTopology(id);
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(config.ToProducerConfig());
var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null);
var part = new TopicPartition("topic", 0);
StreamTask task = new StreamTask(
"thread-0",
id,
new List <TopicPartition> {
part
},
processorTopology,
consumer,
config,
supplier,
null);
task.GroupMetadata = consumer as SyncConsumer;
Assert.Throws <StreamsException>(() => task.InitializeStateStores());
}
public void StateStoreRange()
{
var source = new CancellationTokenSource();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-config";
config.PollMs = 50;
var topicConfiguration = config.Clone();
topicConfiguration.ApplicationId = $"test-driver-{config.ApplicationId}";
var builder = new StreamBuilder();
builder.Table("test", InMemory <string, string> .As("store"));
var driver = new ClusterInMemoryTopologyDriver("client", builder.Build().Builder, config, topicConfiguration, TimeSpan.FromSeconds(1), source.Token);
driver.StartDriver();
var input = driver.CreateInputTopic("test", new StringSerDes(), new StringSerDes());
var store = driver.GetStateStore <string, string>("store");
Assert.IsNotNull(store);
Assert.IsInstanceOf <MockReadOnlyKeyValueStore <string, string> >(store);
input.PipeInput("coucou", "1");
input.PipeInput("test", "2");
Thread.Sleep(100);
var range = ((MockReadOnlyKeyValueStore <string, string>)store).Range("coucou", "test").ToList();
Assert.AreEqual(2, range.Count);
Assert.AreEqual("coucou", range[0].Key);
Assert.AreEqual("1", range[0].Value);
Assert.AreEqual("test", range[1].Key);
Assert.AreEqual("2", range[1].Value);
var reverseRange = ((MockReadOnlyKeyValueStore <string, string>)store).ReverseRange("coucou", "test").ToList();
Assert.AreEqual(2, reverseRange.Count);
Assert.AreEqual("test", reverseRange[0].Key);
Assert.AreEqual("2", reverseRange[0].Value);
Assert.AreEqual("coucou", reverseRange[1].Key);
Assert.AreEqual("1", reverseRange[1].Value);
source.Cancel();
driver.Dispose();
}
public void GetKVStateStoreInvalidStateStoreException()
{
var timeout = TimeSpan.FromSeconds(10);
var source = new CancellationTokenSource();
bool state = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder
.Stream <string, string>("test")
.GroupByKey()
.Count(InMemory <string, long> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
stream.StateChanged += (old, @new) =>
{
if ([email protected](KafkaStream.State.RUNNING))
{
if (!state)
{
Assert.Throws <InvalidStateStoreException>(() => stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, long>())));
state = true;
}
}
};
stream.Start(source.Token);
while (!state)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(state);
source.Cancel();
stream.Close();
}
public void TaskManagerAssignedUnknownPartitions()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-app";
var serdes = new StringSerDes();
var builder = new StreamBuilder();
builder.Stream <string, string>("topic")
.Map((k, v) => KeyValuePair.Create(k.ToUpper(), v.ToUpper()))
.To("topic2");
var topology = builder.Build();
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(config.ToProducerConfig());
var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null);
var taskCreator = new TaskCreator(topology.Builder, config, "thread-0", supplier, producer);
var taskManager = new TaskManager(topology.Builder, taskCreator, supplier.GetAdmin(config.ToAdminConfig("admin")), consumer);
taskManager.CreateTasks(
new List <TopicPartition>
{
new TopicPartition("topic", 0),
new TopicPartition("topic", 1)
});
taskManager.RevokeTasks(
new List <TopicPartition>
{
new TopicPartition("topic", 1)
});
taskManager.CreateTasks(
new List <TopicPartition>
{
new TopicPartition("topic", 0),
new TopicPartition("topic", 1),
new TopicPartition("topic", 2)
});
Assert.AreEqual(3, taskManager.ActiveTasks.Count());
Assert.AreEqual(0, taskManager.RevokedTasks.Count());
taskManager.Close();
}
public void ApplicationIdProperty()
{
var config = new StreamConfig();
config.ApplicationId = "test";
var stateMgt = new ProcessorStateManager(
new TaskId()
{
Id = 0, Partition = 0
},
new List <Confluent.Kafka.TopicPartition> {
new Confluent.Kafka.TopicPartition("test", 0)
});
ProcessorContext context = new ProcessorContext(config, stateMgt);
Assert.AreEqual("test", context.ApplicationId);
}
internal static bool IsCompatible(this StreamConfig config, StreamConfig otherConfig)
{
switch (config)
{
case VideoStreamConfig videoConfig:
return(videoConfig.IsCompatible(otherConfig as VideoStreamConfig));
case AudioStreamConfig audioConfig:
return(audioConfig.IsCompatible(otherConfig as AudioStreamConfig));
case null:
throw new ArgumentNullException(nameof(config), "Config cannot be null");
default:
throw new ArgumentException("Unsupported configuration type", nameof(config));
}
}
public void DeserializeOK()
{
var mockSchemaClient = new MockSchemaRegistryClient();
var config = new StreamConfig();
var serdes = new MockProtoSerDes(mockSchemaClient);
serdes.Initialize(new Net.SerDes.SerDesContext(config));
var person = new Helpers.Proto.Person {
Age = 18, FirstName = "TEST", LastName = "TEST"
};
var bytes = serdes.Serialize(person, new Confluent.Kafka.SerializationContext(Confluent.Kafka.MessageComponentType.Value, topic));
var pbis = serdes.Deserialize(bytes, new Confluent.Kafka.SerializationContext(Confluent.Kafka.MessageComponentType.Value, topic));
Assert.AreEqual(18, pbis.Age);
Assert.AreEqual("TEST", pbis.FirstName);
Assert.AreEqual("TEST", pbis.LastName);
}
public void GetStateStoreBeforeRunningState()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder.Table("topic", InMemory <string, string> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
Assert.Throws <IllegalStateException>(() => stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, string>())));
stream.Dispose();
}
public void GetStateStore()
{
var timeout = TimeSpan.FromSeconds(10);
var source = new CancellationTokenSource();
bool isRunningState = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder.Table("topic", InMemory <string, string> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
stream.StateChanged += (old, @new) =>
{
if (@new.Equals(KafkaStream.State.RUNNING))
{
isRunningState = true;
}
};
stream.Start(source.Token);
while (!isRunningState)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(isRunningState);
if (isRunningState)
{
var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, string>()));;
Assert.IsNotNull(store);
}
source.Cancel();
stream.Close();
}
static async Task Main(string[] args)
{
CancellationTokenSource source = new CancellationTokenSource();
var config = new StreamConfig();
config.ApplicationId = "test-app";
config.BootstrapServers = "192.168.56.1:9092";
config.SaslMechanism = SaslMechanism.Plain;
config.SaslUsername = "******";
config.SaslPassword = "******";
config.SecurityProtocol = SecurityProtocol.SaslPlaintext;
config.AutoOffsetReset = AutoOffsetReset.Earliest;
// NEED FOR SchemaAvroSerDes
config.SchemaRegistryUrl = "http://192.168.56.1:8081";
config.AutoRegisterSchemas = true;
StreamBuilder builder = new StreamBuilder();
var table = builder.Table("product",
new Int32SerDes(),
new SchemaAvroSerDes <Product>(),
InMemory <int, Product> .As("product-store"));
builder.Stream <int, Order, Int32SerDes, SchemaAvroSerDes <Order> >("orders")
.Join(table, (order, product) => new OrderProduct
{
order_id = order.order_id,
price = order.price,
product_id = product.product_id,
product_name = product.name,
product_price = product.price
})
.To <Int32SerDes, SchemaAvroSerDes <OrderProduct> >("orders-output");
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, config);
Console.CancelKeyPress += (o, e) =>
{
stream.Dispose();
};
await stream.StartAsync();
}
public void SchemaRegistryConfig()
{
var mockSchemaClient = new MockSchemaRegistryClient();
var config = new StreamConfig();
config.AutoRegisterSchemas = true;
config.SchemaRegistryMaxCachedSchemas = 1;
config.SchemaRegistryRequestTimeoutMs = 30;
config.SubjectNameStrategy = SubjectNameStrategy.TopicRecord;
var serdes = new MockProtoSerDes(mockSchemaClient);
serdes.Initialize(new Net.SerDes.SerDesContext(config));
Assert.AreEqual(1, mockSchemaClient.MaxCachedSchemas);
Assert.AreEqual(30, mockSchemaClient.RequestTimeoutMs);
}
public void ReduceAndQueryInStateStore()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-reduce";
var builder = new StreamBuilder();
var stream = builder
.Stream <string, string>("topic")
.MapValues(v => v.Length)
.GroupBy((k, v) => k.ToUpper());
stream.Count(InMemory <string, long> .As("count-store"));
stream.Reduce(
(v1, v2) => Math.Max(v1, v2),
InMemory <string, int> .As("reduce-store").WithValueSerdes <Int32SerDes>());
var topology = builder.Build();
using var driver = new TopologyTestDriver(topology, config);
var input = driver.CreateInputTopic <string, string>("topic");
input.PipeInput("test", "1");
input.PipeInput("test", "120");
input.PipeInput("test", "30");
input.PipeInput("coucou", "120");
var store = driver.GetKeyValueStore <string, int>("reduce-store");
Assert.IsNotNull(store);
Assert.AreEqual(2, store.ApproximateNumEntries());
var el = store.Get("TEST");
Assert.IsNotNull(el);
Assert.AreEqual(3, el);
var storeCount = driver.GetKeyValueStore <string, long>("count-store");
Assert.IsNotNull(storeCount);
Assert.AreEqual(2, store.ApproximateNumEntries());
var e = storeCount.Get("TEST");
Assert.IsNotNull(e);
Assert.AreEqual(3, e);
}
public async Task GetStateStoreDoesntExists()
{
var timeout = TimeSpan.FromSeconds(10);
bool isRunningState = false;
DateTime dt = DateTime.Now;
var supplier = new SyncKafkaSupplier();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder.Table("topic", InMemory <string, string> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, supplier);
stream.StateChanged += (old, @new) =>
{
if (@new.Equals(KafkaStream.State.RUNNING))
{
isRunningState = true;
}
};
await stream.StartAsync();
while (!isRunningState)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(isRunningState);
if (isRunningState)
{
Assert.Throws <InvalidStateStoreException>(() => stream.Store(StoreQueryParameters.FromNameAndType("stodfdsfdsfre", QueryableStoreTypes.KeyValueStore <string, string>())));
}
stream.Dispose();
}
public void MultiBranchWithElements()
{
var builder = new StreamBuilder();
var branchs = builder.Stream <string, int>("topic")
.Branch((k, v) => v % 2 == 0, (k, v) => v % 2 > 0);
branchs[0].To("topic-pair");
branchs[1].To("topic-impair");
var config = new StreamConfig <StringSerDes, Int32SerDes>();
config.ApplicationId = "test-branch";
Topology t = builder.Build();
using var driver = new TopologyTestDriver(t, config);
var inputTopic = driver.CreateInputTopic <string, int>("topic");
var outputTopicPair = driver.CreateOuputTopic <string, int>("topic-pair");
var outputTopicImpair = driver.CreateOuputTopic <string, int>("topic-impair");
var expectedPair = new List <KeyValuePair <string, int> >();
var expectedImpair = new List <KeyValuePair <string, int> >();
for (int i = 0; i < 10; i++)
{
string key = i.ToString();
int value = i;
inputTopic.PipeInput(key, value);
if (i % 2 == 0)
{
expectedPair.Add(KeyValuePair.Create(key, value));
}
else
{
expectedImpair.Add(KeyValuePair.Create(key, value));
}
}
var listPair = outputTopicPair.ReadKeyValueList().Select(r => KeyValuePair.Create(r.Message.Key, r.Message.Value)).ToList();
var listImpair = outputTopicImpair.ReadKeyValueList().Select(r => KeyValuePair.Create(r.Message.Key, r.Message.Value)).ToList();
Assert.AreEqual(expectedPair, listPair);
Assert.AreEqual(expectedImpair, listImpair);
}
public void StreamAddCorrectConfig()
{
var stream = new StreamConfig();
stream.ApplicationId = "unittest";
stream.AddConfig("sasl.password", "coucou");
var adminConfig = stream.ToAdminConfig("admin");
var consumerConfig = stream.ToConsumerConfig();
var producerConfig = stream.ToProducerConfig();
var globalConfig = stream.ToGlobalConsumerConfig("global");
Assert.AreEqual("coucou", adminConfig.SaslPassword);
Assert.AreEqual("coucou", consumerConfig.SaslPassword);
Assert.AreEqual("coucou", producerConfig.SaslPassword);
Assert.AreEqual("coucou", globalConfig.SaslPassword);
}
public void TableTableOuterJoinGetterSupplier()
{
var config = new StreamConfig <StringSerDes, StringSerDes>
{
ApplicationId = "test-table-table-outer-join"
};
StreamBuilder builder = new StreamBuilder();
var table1 = builder.Table("users", InMemory <string, string> .As("store-users"));
var table2 = builder.Table("regions", InMemory <string, string> .As("store-regions"));
var stream = builder.Stream <string, string>("orders");
var tableJoin = table1.OuterJoin(table2, (v1, v2) => $"{v1 ?? "?"}-{v2 ?? "?"}");
stream
.Join(tableJoin,
(order, ur) => $"Order:{order}|UserRegion:{ur}")
.To("topic-output");
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
var inputTopic1 = driver.CreateInputTopic <string, string>("users");
var inputTopic2 = driver.CreateInputTopic <string, string>("regions");
var inputTopic3 = driver.CreateInputTopic <string, string>("orders");
var outputTopic = driver.CreateOuputTopic <string, string>("topic-output");
inputTopic1.PipeInput("sylvain", "sylvain");
inputTopic1.PipeInput("lise", "lise");
inputTopic2.PipeInput("sylvain", "France");
inputTopic2.PipeInput("remi", "USA");
inputTopic3.PipeInput("sylvain", "iPhone12Pro");
inputTopic3.PipeInput("lise", "PixelA4");
inputTopic3.PipeInput("remi", "Tab");
var records = outputTopic.ReadKeyValuesToMap();
Assert.IsNotNull(records);
Assert.AreEqual(3, records.Count);
Assert.AreEqual("Order:iPhone12Pro|UserRegion:sylvain-France", records["sylvain"]);
Assert.AreEqual("Order:PixelA4|UserRegion:lise-?", records["lise"]);
Assert.AreEqual("Order:Tab|UserRegion:?-USA", records["remi"]);
}
}
public void Begin()
{
valueAndTimestampSerDes = new ValueAndTimestampSerDes <string>(stringSerDes);
config = new StreamConfig();
config.ApplicationId = $"unit-test-changelogging-tkv";
id = new TaskId {
Id = 0, Partition = 0
};
partition = new TopicPartition("source", 0);
kafkaSupplier = new SyncKafkaSupplier();
var producerConfig = new ProducerConfig();
producerConfig.ClientId = "producer-1";
var producerClient = kafkaSupplier.GetProducer(producerConfig);
recordCollector = new RecordCollector("p-1", config, id, new NoRunnableSensor("s", "s", MetricsRecordingLevel.DEBUG));
recordCollector.Init(ref producerClient);
var changelogsTopics = new Dictionary <string, string> {
{ "test-store", "test-store-changelog" }
};
stateManager = new ProcessorStateManager(
id,
new List <TopicPartition> {
partition
},
changelogsTopics,
new MockChangelogRegister(),
new MockOffsetCheckpointManager());
task = new Mock <AbstractTask>();
task.Setup(k => k.Id).Returns(id);
context = new ProcessorContext(task.Object, config, stateManager, new StreamMetricsRegistry());
context.UseRecordCollector(recordCollector);
var inmemorystore = new InMemoryKeyValueStore("test-store");
store = new ChangeLoggingTimestampedKeyValueBytesStore(inmemorystore);
store.Init(context, store);
}
public void MergeTwoStreamsWithElements()
{
var data1 = new List <KeyValuePair <string, int> >
{
KeyValuePair.Create("key1", 1),
KeyValuePair.Create("key2", 2)
};
var data2 = new List <KeyValuePair <string, int> >
{
KeyValuePair.Create("key3", 3)
};
var observed = new List <KeyValuePair <string, int> >();
var builder = new StreamBuilder();
var stream1 = builder.Stream <string, int>("topic1");
var stream2 = builder.Stream <string, int>("topic2");
var merged = stream1.Merge(stream2);
merged.Peek((k, v) => observed.Add(KeyValuePair.Create(k, v)));
var config = new StreamConfig <StringSerDes, Int32SerDes>
{
ApplicationId = "test-merge"
};
Topology t = builder.Build();
using var driver = new TopologyTestDriver(t, config);
var inputTopic1 = driver.CreateInputTopic <string, int>("topic1");
inputTopic1.PipeInputs(data1);
var inputTopic2 = driver.CreateInputTopic <string, int>("topic2");
inputTopic2.PipeInputs(data2);
var expected = new List <KeyValuePair <string, int> >();
expected.AddRange(data1);
expected.AddRange(data2);
Assert.AreEqual(expected, observed);
}
public async Task StartKafkaStream()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder.Stream <string, string>("topic").To("topic2");
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
await stream.StartAsync();
Thread.Sleep(1500);
stream.Dispose();
}
public void StartDriverKO()
{
var source = new CancellationTokenSource();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-config";
var topicConfiguration = config.Clone();
topicConfiguration.ApplicationId = $"test-driver-{config.ApplicationId}";
var builder = new StreamBuilder();
var driver = new ClusterInMemoryTopologyDriver("client", builder.Build().Builder, config, topicConfiguration, TimeSpan.FromSeconds(1), source.Token);
Assert.Throws <StreamsException>(() => driver.StartDriver());
source.Cancel();
driver.Dispose();
}
public void SchemaRegistryAvroSerializerConfig()
{
var config = new StreamConfig
{
SubjectNameStrategy = SubjectNameStrategy.TopicRecord,
AutoRegisterSchemas = true,
UseLatestVersion = false,
BufferBytes = 1024
};
var serdes = new SchemaAvroSerDes <Order>();
var schemaConfig = serdes.GetSerializerConfig(config);
Assert.AreEqual(Confluent.SchemaRegistry.SubjectNameStrategy.TopicRecord, schemaConfig.SubjectNameStrategy);
Assert.AreEqual(true, schemaConfig.AutoRegisterSchemas);
Assert.AreEqual(false, schemaConfig.UseLatestVersion);
Assert.AreEqual(1024, schemaConfig.BufferBytes);
}
public void StreamWithNullLeftStream()
{
var config = new StreamConfig <StringSerDes, StringSerDes>
{
ApplicationId = "test-stream-stream-left-join"
};
StreamBuilder builder = new StreamBuilder();
var stream = builder.Stream <string, string>("topic1");
Assert.Throws <ArgumentNullException>(() => builder
.Stream <string, string>("topic2")
.LeftJoin(
null,
new MyJoinerMapper(),
JoinWindowOptions.Of(TimeSpan.FromSeconds(10))));
}
public void KTableSourceNoMaterialize2()
{
var builder = new StreamBuilder();
builder.Table <string, string, StringSerDes, StringSerDes>("table-topic");
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-map";
Topology t = builder.Build();
using var driver = new TopologyTestDriver(t, config);
var inputTopic = driver.CreateInputTopic <string, string>("table-topic");
inputTopic.PipeInput("key1", "1");
inputTopic.PipeInput("key2", "2");
}
public void GetStateStoreNotExist()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-config";
var builder = new StreamBuilder();
builder.Stream <string, string>("test").To("test2");
var driver = new TaskSynchronousTopologyDriver("client", builder.Build().Builder, config, config, default);
driver.CreateInputTopic("test", new StringSerDes(), new StringSerDes());
var store = driver.GetStateStore <string, string>("store");
Assert.IsNull(store);
driver.Dispose();
}
/// <summary>
/// Sets Stream configuration
/// If no argument is provided, current configuration will be pushed.
/// Otherwise provided configuration will be set as current and pushed.
/// <param name="config">StreamConfig</param>
/// </summary>
public void SetStreamConfiguration(StreamConfig config = null)
{
logger.Info($"{streamType}: Using {(config == null ? "stored" : "provided")} configuration");
if (config == null)
{
if (Configuration == null)
{
throw new ArgumentNullException(nameof(Configuration), "Current configuration is null");
}
}
else
{
Configuration = config;
}
PushStreamConfig(Configuration);
}
public void TableTableMergeJoinForwardSendOldValues()
{
var config = new StreamConfig <StringSerDes, StringSerDes>
{
ApplicationId = "test-table-table-left-join"
};
StreamBuilder builder = new StreamBuilder();
var table1 = builder.Table("users", InMemory <string, string> .As("store-users"));
var table2 = builder.Table("regions", InMemory <string, string> .As("store-regions"));
var table3 = builder.Table("country", InMemory <string, string> .As("store-country"));
var stream = builder.Stream <string, string>("orders");
var tableJoin = table1.LeftJoin(table2, (v1, v2) => $"{v1}-{v2 ?? "?"}", InMemory <string, string> .As("merge1"));
var tableJoin2 = tableJoin.LeftJoin(table3, (v1, v2) => $"{v1}-{v2 ?? "?"}");
stream
.Join(tableJoin2,
(order, ur) => $"Order:{order}|UserRegionCountry:{ur}")
.To("topic-output");
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
var inputTopic1 = driver.CreateInputTopic <string, string>("users");
var inputTopic2 = driver.CreateInputTopic <string, string>("regions");
var inputTopic3 = driver.CreateInputTopic <string, string>("orders");
var inputTopic4 = driver.CreateInputTopic <string, string>("country");
var outputTopic = driver.CreateOuputTopic <string, string>("topic-output");
inputTopic1.PipeInput("sylvain", "sylvain");
inputTopic2.PipeInput("sylvain", "Europe");
inputTopic4.PipeInput("sylvain", "France");
inputTopic3.PipeInput("sylvain", "iPhone12Pro");
var records = outputTopic.ReadKeyValuesToMap();
Assert.IsNotNull(records);
Assert.AreEqual(1, records.Count);
Assert.AreEqual("Order:iPhone12Pro|UserRegionCountry:sylvain-Europe-France", records["sylvain"]);
}
}
public void WithNullMaterialize()
{
// CERTIFIED THAT SAME IF Materialize is null, a state store exist for count processor with a generated namestore
var config = new StreamConfig();
var serdes = new StringSerDes();
config.ApplicationId = "test-window-reduce";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Reduce((v1, v2) => v1.Length > v2.Length ? v1 : v2);
var topology = builder.Build();
TaskId id = new TaskId {
Id = 0, Partition = 0
};
var processorTopology = topology.Builder.BuildTopology(id);
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(config.ToProducerConfig());
var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null);
var part = new TopicPartition("topic", 0);
StreamTask task = new StreamTask(
"thread-0",
id,
new List <TopicPartition> {
part
},
processorTopology,
consumer,
config,
supplier,
null,
new MockChangelogRegister(),
new StreamMetricsRegistry());
task.GroupMetadata = consumer as SyncConsumer;
Assert.Throws <StreamsException>(() => task.InitializeStateStores());
}
public void ProductionExceptionRecoverableHandlerFailTest()
{
var _return = new List <KeyValuePair <string, string> >();
var config = new StreamConfig <StringSerDes, StringSerDes>();
var dt = DateTime.Now;
var timeout = TimeSpan.FromSeconds(10);
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
config.PollMs = 10;
config.ProductionExceptionHandler += (r) => ExceptionHandlerResponse.FAIL;
var options = new ProducerSyncExceptionOptions {
IsRecoverable = true
};
var supplier = new ProducerSyncException(options);
var builder = new StreamBuilder();
builder
.Stream <string, string>("test")
.To("test-output");
builder.Stream <string, string>("test-output")
.Peek((k, v) => _return.Add(KeyValuePair.Create(k, v)));
var t = builder.Build();
using (var driver = new TopologyTestDriver(t.Builder, config,
TopologyTestDriver.Mode.ASYNC_CLUSTER_IN_MEMORY, supplier))
{
var inputtopic = driver.CreateInputTopic <string, string>("test");
var outputTopic = driver.CreateOuputTopic <string, string>("test-output");
inputtopic.PipeInput("coucou");
inputtopic.PipeInput("coucou");
while (_return.Count == 0)
{
;
}
var expected = new List <KeyValuePair <string, string> >();
expected.Add(KeyValuePair.Create <string, string>(null, "coucou"));
Assert.AreEqual(expected, _return);
}
}
public async Task StartKafkaStreamWaitRunningState()
{
var timeout = TimeSpan.FromSeconds(10);
bool isRunningState = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder.Stream <string, string>("topic").To("topic2");
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
stream.StateChanged += (old, @new) =>
{
if (@new.Equals(KafkaStream.State.RUNNING))
{
isRunningState = true;
}
};
await stream.StartAsync();
while (!isRunningState)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
var sensors = stream.Metrics();
Assert.IsTrue(sensors.Any());
stream.Dispose();
Assert.IsTrue(isRunningState);
}
public TwitterConnection()
{
_context = GlobalHost.ConnectionManager.GetHubContext<TwitterHub>();
var config = new StreamConfig()
{
ConsumerKey = _consumerKey,
ConsumerSecret = _consumerSecret,
AccessToken = _accessKey,
AccessSecret = _accessToken,
GeoOnly = true
};
var stream = new TwitterStreamClient(config);
stream.TweetReceivedEvent += (sender, args) =>
{
Debug.WriteLine(args.Tweet.ToString());
_context.Clients.All.broadcast(args.Tweet);
};
stream.Start();
}
