private static async Task GetKsqlDbInformationAsync(KSqlDbRestApiProvider restApiProvider) { Console.WriteLine($"{Environment.NewLine}Available topics:"); var topicsResponses = await restApiProvider.GetTopicsAsync(); Console.WriteLine(string.Join(',', topicsResponses[0].Topics.Select(c => c.Name))); TopicsResponse[] allTopicsResponses = await restApiProvider.GetAllTopicsAsync(); TopicsExtendedResponse[] topicsExtendedResponses = await restApiProvider.GetTopicsExtendedAsync(); var allTopicsExtendedResponses = await restApiProvider.GetAllTopicsExtendedAsync(); Console.WriteLine($"{Environment.NewLine}Available tables:"); var tablesResponse = await restApiProvider.GetTablesAsync(); Console.WriteLine(string.Join(',', tablesResponse[0].Tables.Select(c => c.Name))); Console.WriteLine($"{Environment.NewLine}Available streams:"); var streamsResponse = await restApiProvider.GetStreamsAsync(); Console.WriteLine(string.Join(',', streamsResponse[0].Streams.Select(c => c.Name))); Console.WriteLine($"{Environment.NewLine}Available connectors:"); var connectorsResponse = await restApiProvider.GetConnectorsAsync(); Console.WriteLine(string.Join(',', connectorsResponse[0].Connectors.Select(c => c.Name))); }
public static async Task ClassInitialize(TestContext context) { RestApiProvider = KSqlDbRestApiProvider.Create(); moviesProvider = new MoviesProvider(RestApiProvider); await moviesProvider.CreateTablesAsync(); await moviesProvider.InsertMovieAsync(MoviesProvider.Movie1); }
protected static async Task InitializeDatabase() { RestApiProvider = KSqlDbRestApiProvider.Create(); MoviesProvider = new MoviesProvider(RestApiProvider); await MoviesProvider.CreateTablesAsync(); await MoviesProvider.InsertMovieAsync(MoviesProvider.Movie1); }
public static async Task Main(string[] args) { var ksqlDbUrl = @"http:\\localhost:8088"; var httpClientFactory = new HttpClientFactory(new Uri(ksqlDbUrl)); var restApiProvider = new KSqlDbRestApiProvider(httpClientFactory); var moviesProvider = new MoviesProvider(restApiProvider); await moviesProvider.CreateTablesAsync(); var contextOptions = CreateQueryStreamOptions(ksqlDbUrl); var context = new KSqlDBContext(contextOptions); var subscription = context.CreateQuery <Movie>() .Where(p => p.Title != "E.T.") .Where(c => K.Functions.Like(c.Title.ToLower(), "%hard%".ToLower()) || c.Id == 1) .Where(p => p.RowTime >= 1510923225000) //AND RowTime >= 1510923225000 .Select(l => new { Id2 = l.Id, l.Title, l.Release_Year, l.RowTime }) .Take(2) // LIMIT 2 .ToObservable() // client side processing starts here lazily after subscription. Switches to Rx.NET .ObserveOn(TaskPoolScheduler.Default) .Subscribe(onNext: movie => { Console.WriteLine($"{nameof(Movie)}: {movie.Id2} - {movie.Title} - {movie.RowTime}"); Console.WriteLine(); }, onError: error => { Console.WriteLine($"Exception: {error.Message}"); }, onCompleted: () => Console.WriteLine("Completed")); await moviesProvider.InsertMovieAsync(MoviesProvider.Movie1); await moviesProvider.InsertMovieAsync(MoviesProvider.Movie2); await moviesProvider.InsertLeadAsync(MoviesProvider.LeadActor1); try { await new PullQueryExample().ExecuteAsync(); } catch (Exception e) { Console.WriteLine(e); } Console.WriteLine("Press any key to stop the subscription"); Console.ReadKey(); await context.DisposeAsync(); using (subscription) { } Console.WriteLine("Subscription completed"); }
public async Task CreateOrReplaceStreamStatement_ToStatementString_ComplexQueryWasGenerated() { //Arrange var restApiClient = KSqlDbRestApiProvider.Create(); var statement = new KSqlDbStatement(StatementTemplates.DropStream(StreamName)); var response = await restApiClient.ExecuteStatementAsync(statement); EntityCreationMetadata metadata = new() { EntityName = StreamEntityName, KafkaTopic = nameof(Movie) + "Test2", Partitions = 1, Replicas = 1 }; var httpResponseMessage = await restApiClient.CreateStreamAsync <Movie>(metadata, ifNotExists : false); var creationMetadata = new CreationMetadata { KafkaTopic = "moviesByTitle", KeyFormat = SerializationFormats.Json, ValueFormat = SerializationFormats.Json, Replicas = 1, Partitions = 1 }; var createStatement = Context.CreateOrReplaceStreamStatement(StreamName) .With(creationMetadata) .As <Movie>(StreamEntityName) .Where(c => c.Id < 3) .Select(c => new { c.Id, c.Title, ReleaseYear = c.Release_Year }) .PartitionBy(c => c.Id); //Act var ksql = createStatement.ToStatementString(); httpResponseMessage = await createStatement.ExecuteStatementAsync(); //Assert ksql.Should().BeEquivalentTo(@$ "CREATE OR REPLACE STREAM {StreamName} WITH ( KAFKA_TOPIC='moviesByTitle', KEY_FORMAT='Json', VALUE_FORMAT='Json', PARTITIONS='1', REPLICAS='1' ) AS SELECT Id, Title, Release_Year AS ReleaseYear FROM {StreamEntityName} WHERE Id < 3 PARTITION BY Id EMIT CHANGES;"); var responses = await httpResponseMessage.ToStatementResponsesAsync(); responses[0].CommandStatus.Status.Should().BeOneOf("SUCCESS", "EXECUTING"); }
public static async Task ClassInitialize(TestContext context) { RestApiProvider = KSqlDbRestApiProvider.Create(); var response = await RestApiProvider.CreateTypeAsync <TestState>(); response = await RestApiProvider.CreateTypeAsync <TestCity>(); response = await RestApiProvider.CreateTableAsync <TestCity>(new EntityCreationMetadata { EntityName = CitiesTableName, KafkaTopic = CitiesTableName, Partitions = 1 }); var c = await response.Content.ReadAsStringAsync(); response = await RestApiProvider.InsertIntoAsync(new TestCity { RegionCode = "sk", State = State1 }, new InsertProperties { EntityName = CitiesTableName }); }
public static async Task ClassInitialize(TestContext context) { RestApiProvider = KSqlDbRestApiProvider.Create(); moviesProvider = new MoviesProvider(RestApiProvider); await moviesProvider.DropTablesAsync(); await Task.Delay(TimeSpan.FromSeconds(1)); await moviesProvider.CreateTablesAsync(); await moviesProvider.InsertMovieAsync(MoviesProvider.Movie1); await moviesProvider.InsertLeadAsync(MoviesProvider.LeadActor1); }
protected static async Task InitializeDatabase() { RestApiProvider = KSqlDbRestApiProvider.Create(); var tweetsProvider = new TweetsProvider(RestApiProvider); var result = await tweetsProvider.CreateTweetsStream(StreamName, topicName); result.Should().BeTrue(); result = await tweetsProvider.InsertTweetAsync(Tweet1, StreamName); result.Should().BeTrue(); result = await tweetsProvider.InsertTweetAsync(Tweet2, StreamName); result.Should().BeTrue(); }
public async Task AvroExampleAsync(string ksqlDbUrl) { var schemaRegistryConfig = new SchemaRegistryConfig { Url = "http://localhost:8081" }; using var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig); var schema = IoTSensor._SCHEMA.ToString(); var subject = "IoTSensors-value"; var registrationResult = await schemaRegistry.RegisterSchemaAsync(subject, schema); //http://localhost:8081/subjects/IoTSensors-value/versions/latest/schema var latestSchema = await schemaRegistry.GetLatestSchemaAsync(subject); var httpClientFactory = new HttpClientFactory(new Uri(ksqlDbUrl)); var restApiClient = new KSqlDbRestApiProvider(httpClientFactory); EntityCreationMetadata metadata = new() { EntityName = "avroSensors", KafkaTopic = "IoTSensors", ValueFormat = SerializationFormats.Avro, Partitions = 1, Replicas = 1 }; var httpResponseMessage = await restApiClient.CreateStreamAsync <IoTSensor>(metadata, ifNotExists : false) .ConfigureAwait(false); var httpResponse = await httpResponseMessage.Content.ReadAsStringAsync(); Console.WriteLine(httpResponse); // Stream Name | Kafka Topic | Key Format | Value Format | Windowed // ------------------------------------------------------------------------------------------------------ // AVROSENSORS | IoTSensors | KAFKA | AVRO | false } }
public static async Task ClassInitialize(TestContext context) { RestApiProvider = KSqlDbRestApiProvider.Create(); moviesProvider = new MoviesProvider(RestApiProvider); await moviesProvider.CreateTablesAsync(); await moviesProvider.InsertMovieAsync(MoviesProvider.Movie1); await moviesProvider.InsertMovieAsync(MoviesProvider.Movie2); tweetsProvider = new TweetsProvider(RestApiProvider); await tweetsProvider.CreateTweetsStream(TweetsStreamName, tweetsTopicName); await tweetsProvider.InsertTweetAsync(TweetsProvider.Tweet1, TweetsStreamName); await tweetsProvider.InsertTweetAsync(TweetsProvider.Tweet2, TweetsStreamName); }
public async Task CreateOrReplaceTableStatement_ExecuteStatementAsync_ResponseWasReceived() { //Arrange var restApiClient = KSqlDbRestApiProvider.Create(); await restApiClient.CreateStreamAsync <Movie>(new EntityCreationMetadata(EntityName, 1) { EntityName = EntityName, ShouldPluralizeEntityName = false }); var statement = new KSqlDbStatement(StatementTemplates.DropTable(TableName)); var response = await restApiClient.ExecuteStatementAsync(statement); int retryCount = 0; while ((await KSqlDbRestApiProvider.Create().GetTablesAsync()).SelectMany(c => c.Tables).Any(c => c.Name == TableName.ToUpper())) { if (retryCount++ > 5) { return; } await Task.Delay(TimeSpan.FromSeconds(1)); } var createStatement = Context.CreateTableStatement(TableName) .As <Movie>(EntityName) .GroupBy(c => c.Title) .Select(c => new { Title = c.Key, Count = c.Count() }); //Act var httpResponseMessage = await createStatement.ExecuteStatementAsync(); //Assert string responseContent = await httpResponseMessage.Content.ReadAsStringAsync(); responseContent.Should().NotBeNull(); var responses = await httpResponseMessage.ToStatementResponsesAsync(); responses[0].CommandStatus.Status.Should().Be("SUCCESS"); }
public static async Task ClassInitialize(TestContext context) { RestApiProvider = KSqlDbRestApiProvider.Create(); var statement = new KSqlDbStatement( @"CREATE STREAM stream2 (id INT, lambda_arr ARRAY<INTEGER>) WITH (kafka_topic = 'stream2', partitions = 1, value_format = 'json');"); var response = await RestApiProvider.ExecuteStatementAsync(statement); var statement2 = new KSqlDbStatement( @"CREATE OR REPLACE STREAM stream4 (id INT, lambda_map MAP<STRING,ARRAY<INTEGER>>) WITH (kafka_topic = 'stream4', partitions = 1, value_format = 'json');"); response = await RestApiProvider.ExecuteStatementAsync(statement2); string insertIntoStream3 = "insert into stream4 (id, lambda_map) values (1, MAP('hello':= ARRAY [1,2,3], 'goodbye':= ARRAY [-1,-2,-3]) );"; response = await RestApiProvider.ExecuteStatementAsync( new KSqlDbStatement("insert into stream2 (id, lambda_arr) values (1, ARRAY [1,2,3]);")); response = await RestApiProvider.ExecuteStatementAsync( new KSqlDbStatement(insertIntoStream3)); }
public MoviesProvider(KSqlDbRestApiProvider restApiProvider) { this.restApiProvider = restApiProvider ?? throw new ArgumentNullException(nameof(restApiProvider)); }
public static async Task ClassInitialize(TestContext context) { var restApiClient = KSqlDbRestApiProvider.Create(); await restApiClient.CreateStreamAsync <Movie>(new EntityCreationMetadata(EntityName, 1) { EntityName = EntityName, ShouldPluralizeEntityName = false }); }
public static async Task Main(string[] args) { var ksqlDbUrl = @"http:\\localhost:8088"; var loggerFactory = CreateLoggerFactory(); var httpClientFactory = new HttpClientFactory(new Uri(ksqlDbUrl)); var restApiProvider = new KSqlDbRestApiProvider(httpClientFactory, loggerFactory) { DisposeHttpClient = false }; restApiProvider.SetCredentials(new BasicAuthCredentials("fred", "letmein")); var moviesProvider = new MoviesProvider(restApiProvider); await moviesProvider.CreateTablesAsync(); var contextOptions = CreateQueryStreamOptions(ksqlDbUrl); await using var context = new KSqlDBContext(contextOptions, loggerFactory); var query = context.CreateQueryStream <Movie>() // Http 2.0 // var query = context.CreateQuery<Movie>() // Http 1.0 .Where(p => p.Title != "E.T.") .Where(c => c.Title.ToLower().Contains("hard".ToLower()) || c.Id == 1) .Where(p => p.RowTime >= 1510923225000) .Select(l => new { Id = l.Id, l.Title, l.Release_Year, l.RowTime }) .Take(2); // LIMIT 2 var ksql = query.ToQueryString(); Console.WriteLine("Generated ksql:"); Console.WriteLine(ksql); Console.WriteLine(); using var disposable = query .ToObservable() // client side processing starts here lazily after subscription. Switches to Rx.NET .Finally(() => { Console.WriteLine("Finally"); }) .Subscribe(onNext: movie => { Console.WriteLine($"{nameof(Movie)}: {movie.Id} - {movie.Title} - {movie.RowTime}"); Console.WriteLine(); }, onError: error => { Console.WriteLine($"Exception: {error.Message}"); }, onCompleted: () => Console.WriteLine("Completed")); await CreateOrReplaceTableStatement(context); await moviesProvider.InsertMovieAsync(MoviesProvider.Movie1); await moviesProvider.InsertMovieAsync(MoviesProvider.Movie2); await moviesProvider.InsertLeadAsync(MoviesProvider.LeadActor1); try { await new PullQueryExample().ExecuteAsync(); } catch (Exception e) { Console.WriteLine(); Console.WriteLine(e.Message); } string explain = await query.ExplainAsStringAsync(); ExplainResponse[] explainResponses = await query.ExplainAsync(); Console.WriteLine($"{Environment.NewLine} Explain => ExecutionPlan:"); Console.WriteLine(explainResponses[0].QueryDescription?.ExecutionPlan); Console.WriteLine("Press any key to stop the subscription"); Console.ReadKey(); await moviesProvider.DropTablesAsync(); Console.WriteLine("Finished."); }