public void ShouldNotReturnKeyValueStoreAsTimestampedStore() { GlobalStateStoreProvider provider = new GlobalStateStoreProvider(stores); var result = provider.Stores(StoreQueryParameters.FromNameAndType("kv-store", QueryableStoreTypes.TimestampedKeyValueStore <object, object>())); Assert.AreEqual(0, result.Count()); }
public void ShouldReturnTimestampedKeyValueStore() { GlobalStateStoreProvider provider = new GlobalStateStoreProvider(stores); var result = provider.Stores(StoreQueryParameters.FromNameAndType("ts-kv-store", QueryableStoreTypes.TimestampedKeyValueStore <object, object>())); Assert.AreEqual(timestampedKVStore, result.Single()); }
public void ShouldReturnEmptyItemListIfStoreDoesntExist() { GlobalStateStoreProvider provider = new GlobalStateStoreProvider(new Dictionary <string, IStateStore>()); var result = provider.Stores(StoreQueryParameters.FromNameAndType("test", QueryableStoreTypes.KeyValueStore <object, object>())); Assert.AreEqual(0, result.Count()); }
public void ShouldReturnWindowStoreAsTimestampedWindowStore() { GlobalStateStoreProvider provider = new GlobalStateStoreProvider(stores); var result = provider.Stores(StoreQueryParameters.FromNameAndType("ts-w-store", QueryableStoreTypes.WindowStore <object, object>())); Assert.AreEqual(1, result.Count()); }
public void ShouldReturnWindowStore() { GlobalStateStoreProvider provider = new GlobalStateStoreProvider(stores); var result = provider.Stores(StoreQueryParameters.FromNameAndType("ws-store", QueryableStoreTypes.WindowStore <object, object>())); Assert.AreEqual(wStore, result.Single()); }
public void ShouldReturnTimestampedKeyValueStoreAsKeyValueStore() { GlobalStateStoreProvider provider = new GlobalStateStoreProvider(stores); var result = provider.Stores(StoreQueryParameters.FromNameAndType("ts-kv-store", QueryableStoreTypes.KeyValueStore <object, object>())); Assert.IsInstanceOf(typeof(ReadOnlyKeyValueStore <object, object>), result.Single()); Assert.IsNotInstanceOf(typeof(TimestampedKeyValueStore <object, object>), result.Single()); }
public void ShouldThrowExceptionIfStoreIsNotOpen() { var mockStore = CreateMockStore <IKeyValueStore <object, object> >(isOpen: false); GlobalStateStoreProvider provider = new GlobalStateStoreProvider(new Dictionary <string, IStateStore> { { "test", mockStore } }); Assert.Throws <InvalidStateStoreException>(() => provider.Stores(StoreQueryParameters.FromNameAndType("test", QueryableStoreTypes.KeyValueStore <object, object>()))); }
/// <summary> /// Create a <see cref="KafkaStream"/> instance with your own <see cref="IKafkaSupplier" /> /// Please DO NOT FORGET to call Close to avoid resources leak ! /// </summary> /// <param name="topology">the topology specifying the computational logic</param> /// <param name="configuration">configuration about this stream</param> /// <param name="kafkaSupplier">the Kafka clients supplier which provides underlying producer and consumer clients for the new <see cref="KafkaStream"/> instance</param> public KafkaStream(Topology topology, IStreamConfig configuration, IKafkaSupplier kafkaSupplier) { this.topology = topology; this.kafkaSupplier = kafkaSupplier; // check if ApplicationId & BootstrapServers has been set if (string.IsNullOrEmpty(configuration.ApplicationId) || string.IsNullOrEmpty(configuration.BootstrapServers)) { throw new StreamConfigException($"Stream configuration is not correct. Please set ApplicationId and BootstrapServers as minimal."); } var processID = Guid.NewGuid(); clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{configuration.ApplicationId.ToLower()}-{processID}" : configuration.ClientId; logPrefix = $"stream-application[{configuration.ApplicationId}] "; logger.Info($"{logPrefix} Start creation of the stream application with this configuration: {configuration}"); // re-write the physical topology according to the config topology.Builder.RewriteTopology(configuration); // sanity check var processorTopology = topology.Builder.BuildTopology(); int numStreamThreads = topology.Builder.HasNoNonGlobalTopology ? 0 : configuration.NumStreamThreads; threads = new IThread[numStreamThreads]; var threadState = new Dictionary <long, Processors.ThreadState>(); ProcessorTopology globalTaskTopology = topology.Builder.BuildGlobalStateTopology(); bool hasGlobalTopology = globalTaskTopology != null; if (numStreamThreads == 0 && !hasGlobalTopology) { throw new TopologyException("Topology has no stream threads and no global threads, " + "must subscribe to at least one source topic or global table."); } GlobalThreadState globalThreadState = null; if (hasGlobalTopology) { string globalThreadId = $"{clientId}-GlobalStreamThread"; GlobalStreamThreadFactory globalStreamThreadFactory = new GlobalStreamThreadFactory(globalTaskTopology, globalThreadId, kafkaSupplier.GetGlobalConsumer(configuration.ToGlobalConsumerConfig(globalThreadId)), configuration, kafkaSupplier.GetAdmin(configuration.ToAdminConfig(clientId))); globalStreamThread = globalStreamThreadFactory.GetGlobalStreamThread(); globalThreadState = globalStreamThread.State; } List <StreamThreadStateStoreProvider> stateStoreProviders = new List <StreamThreadStateStoreProvider>(); for (int i = 0; i < numStreamThreads; ++i) { var threadId = $"{configuration.ApplicationId.ToLower()}-stream-thread-{i}"; var adminClient = this.kafkaSupplier.GetAdmin(configuration.ToAdminConfig(StreamThread.GetSharedAdminClientId(clientId))); threads[i] = StreamThread.Create( threadId, clientId, this.topology.Builder, configuration, this.kafkaSupplier, adminClient, i); threadState.Add(threads[i].Id, threads[i].State); stateStoreProviders.Add(new StreamThreadStateStoreProvider(threads[i], this.topology.Builder)); } var manager = new StreamStateManager(this, threadState, globalThreadState); if (hasGlobalTopology) { globalStreamThread.StateChanged += manager.OnGlobalThreadStateChange; } foreach (var t in threads) { t.StateChanged += manager.OnChange; } var globalStateStoreProvider = new GlobalStateStoreProvider(topology.Builder.GlobalStateStores); queryableStoreProvider = new QueryableStoreProvider(stateStoreProviders, globalStateStoreProvider); StreamState = State.CREATED; }