public async Task ReceiveArrayWithComplexElements() { //Arrange var httpResponseMessage = await restApiClient.ExecuteStatementAsync(new KSqlDbStatement(@" Drop type EventCategory; Drop table Events; ")); //Act httpResponseMessage = await restApiClient.CreateTypeAsync <EventCategory>(); httpResponseMessage = await restApiClient.CreateTableAsync <Event>(new EntityCreationMetadata() { KafkaTopic = "Events", Partitions = 1 }); var eventCategory = new EventCategory() { Name = "xyz" }; var testEvent = new Event { Id = 1, Places = new[] { "Place1", "Place2" }, Categories = new[] { eventCategory, new EventCategory { Name = "puk" } } }; var semaphoreSlim = new SemaphoreSlim(0, 1); var receivedValues = new List <Event>(); var subscription = Context.CreateQueryStream <Event>().Take(1) .Subscribe(value => { receivedValues.Add(value); }, error => { semaphoreSlim.Release(); }, () => { semaphoreSlim.Release(); }); httpResponseMessage = await restApiClient.InsertIntoAsync(testEvent); string responseContent = await httpResponseMessage.Content.ReadAsStringAsync(); //Assert await semaphoreSlim.WaitAsync(TimeSpan.FromSeconds(5)); receivedValues.Count.Should().Be(1); receivedValues[0].Places.Length.Should().Be(2); receivedValues[0].Categories.ToArray()[0].Name.Should().Be("xyz"); receivedValues[0].Categories.ToArray()[1].Name.Should().Be("puk"); using (subscription) { } }
private static IDisposable DynamicFunctionCall(KSqlDBContext context) { var ifNullQueryString = context.CreateQueryStream <Tweet>() .Select(c => new { c.Id, c.Amount, Col = K.F.Dynamic("IFNULL(Message, 'n/a')") as string }) .ToQueryString(); return(context.CreateQueryStream <Tweet>() .Select(c => K.Functions.Dynamic("ARRAY_DISTINCT(ARRAY[1, 1, 2, 3, 1, 2])") as int[]) .Subscribe( message => Console.WriteLine($"{message[0]} - {message[^1]}"),
private static async Task GroupBy() { var ksqlDbUrl = @"http:\\localhost:8088"; var contextOptions = new KSqlDBContextOptions(ksqlDbUrl); contextOptions.QueryStreamParameters["auto.offset.reset"] = "latest"; await using var context = new KSqlDBContext(contextOptions); context.CreateQueryStream <Tweet>() .GroupBy(c => c.Id) .Select(g => new { Id = g.Key, Count = g.Count() }) .Subscribe(count => { Console.WriteLine($"{count.Id} Count: {count.Count}"); Console.WriteLine(); }, error => { Console.WriteLine($"Exception: {error.Message}"); }, () => Console.WriteLine("Completed")); context.CreateQueryStream <Tweet>() .GroupBy(c => c.Id) .Select(g => g.Count()) .Subscribe(count => { Console.WriteLine($"Count: {count}"); Console.WriteLine(); }, error => { Console.WriteLine($"Exception: {error.Message}"); }, () => Console.WriteLine("Completed")); context.CreateQueryStream <Tweet>() .GroupBy(c => c.Id) .Select(g => new { Count = g.Count() }) .Subscribe(count => { Console.WriteLine($"Count: {count}"); Console.WriteLine(); }, error => { Console.WriteLine($"Exception: {error.Message}"); }, () => Console.WriteLine("Completed")); //Sum var subscription = context.CreateQueryStream <Tweet>() .GroupBy(c => c.Id) //.Select(g => g.Sum(c => c.Id)) .Select(g => new { Id = g.Key, MySum = g.Sum(c => c.Id) }) .Subscribe(sum => { Console.WriteLine($"{sum}"); Console.WriteLine(); }, error => { Console.WriteLine($"Exception: {error.Message}"); }, () => Console.WriteLine("Completed")); var groupBySubscription = context.CreateQueryStream <IoTSensorChange>("sqlserversensors") .GroupBy(c => new { c.Op, c.After !.Value })
private static IDisposable NotNull(KSqlDBContext context) { return(context.CreateQueryStream <Click>() .Where(c => c.IP_ADDRESS != null) .Select(c => new { c.IP_ADDRESS, c.URL, c.TIMESTAMP }) .Subscribe(message => Console.WriteLine(message), error => { Console.WriteLine($"Exception: {error.Message}"); })); }
private static IDisposable FullOuterJoinTables(KSqlDBContext context) { var query = context.CreateQueryStream <MovieNullableFields>("Movies") .FullOuterJoin( Source.Of <Lead_Actor>(nameof(Lead_Actor)), movie => movie.Title, actor => actor.Title, (movie, actor) => new { movie.Id, Title = movie.Title, movie.Release_Year, ActorTitle = actor.Title, ActorName = actor.Actor_Name } ); var joinQueryString = query.ToQueryString(); return(query .Subscribe(c => { if (c.Id.HasValue) { Console.WriteLine($"{c.Id}: {c.ActorName} - {c.Title} - {c.ActorTitle}"); } else { Console.WriteLine($"No movie id: {c.ActorName} - {c.Title} - {c.ActorTitle}"); } }, exception => { Console.WriteLine(exception.Message); })); }
public static async Task Main(string[] args) { var ksqlDbUrl = @"http:\\localhost:8088"; var contextOptions = new KSqlDBContextOptions(ksqlDbUrl); await using var context = new KSqlDBContext(contextOptions); using var disposable = context.CreateQueryStream <Tweet>() .Where(p => p.Message != "Hello world" || p.Id == 1) .Where(c => K.Functions.Like(c.Message.ToLower(), "%ALL%".ToLower())) .Where(p => p.RowTime >= 1510923225000) //AND RowTime >= 1510923225000 .Select(l => new { l.Id, l.Message, l.RowTime }) .Take(2) // LIMIT 2 .ToObservable() // client side processing starts here lazily after subscription .ObserveOn(TaskPoolScheduler.Default) .Subscribe(onNext: tweetMessage => { Console.WriteLine($"{nameof(Tweet)}: {tweetMessage.Id} - {tweetMessage.Message}"); Console.WriteLine(); }, onError: error => { Console.WriteLine($"Exception: {error.Message}"); }, onCompleted: () => Console.WriteLine("Completed")); Console.WriteLine("Press any key to stop the subscription"); Console.ReadKey(); Console.WriteLine("Subscription completed"); }
private static async Task ToQueryStringExample(string ksqlDbUrl) { var contextOptions = new KSqlDBContextOptions(ksqlDbUrl); await using var context = new KSqlDBContext(contextOptions); var ksql = context.CreateQueryStream <Person>().ToQueryString(); //prints SELECT * FROM People EMIT CHANGES; Console.WriteLine(ksql); }
private static async Task AsyncEnumerable(KSqlDBContext context) { var cts = new CancellationTokenSource(); var asyncTweetsEnumerable = context.CreateQueryStream <Tweet>().ToAsyncEnumerable(); await foreach (var tweet in asyncTweetsEnumerable.WithCancellation(cts.Token)) { Console.WriteLine(tweet.Message); cts.Cancel(); } }
public static async Task Main(string[] args) { var ksqlDbUrl = @"http:\\localhost:8088"; var httpClientFactory = new HttpClientFactory(new Uri(ksqlDbUrl)); var restApiProvider = new KSqlDbRestApiProvider(httpClientFactory); var moviesProvider = new MoviesProvider(restApiProvider); await moviesProvider.CreateTablesAsync(); var contextOptions = CreateQueryStreamOptions(ksqlDbUrl); await using var context = new KSqlDBContext(contextOptions); using var disposable = context.CreateQueryStream <Movie>() // Http 2.0 // using var disposable = context.CreateQuery<Movie>() // Http 1.0 .Where(p => p.Title != "E.T.") .Where(c => K.Functions.Like(c.Title.ToLower(), "%hard%".ToLower()) || c.Id == 1) .Where(p => p.RowTime >= 1510923225000) //AND RowTime >= 1510923225000 .Select(l => new { Id = l.Id, l.Title, l.Release_Year, l.RowTime }) .Take(2) // LIMIT 2 .ToObservable() // client side processing starts here lazily after subscription. Switches to Rx.NET .ObserveOn(TaskPoolScheduler.Default) .Subscribe(onNext: movie => { Console.WriteLine($"{nameof(Movie)}: {movie.Id} - {movie.Title} - {movie.RowTime}"); Console.WriteLine(); }, onError: error => { Console.WriteLine($"Exception: {error.Message}"); }, onCompleted: () => Console.WriteLine("Completed")); await CreateOrReplaceTableStatement(context); await moviesProvider.InsertMovieAsync(MoviesProvider.Movie1); await moviesProvider.InsertMovieAsync(MoviesProvider.Movie2); await moviesProvider.InsertLeadAsync(MoviesProvider.LeadActor1); try { await new PullQueryExample().ExecuteAsync(); } catch (Exception e) { Console.WriteLine(e); } Console.WriteLine("Press any key to stop the subscription"); Console.ReadKey(); await moviesProvider.DropTablesAsync(); Console.WriteLine("Subscription completed"); }
private static IDisposable ToObservableExample(string ksqlDbUrl) { var contextOptions = new KSqlDBContextOptions(ksqlDbUrl); var context = new KSqlDBContext(contextOptions); var subscriptions = context.CreateQueryStream <Tweet>() .ToObservable() .Delay(TimeSpan.FromSeconds(2)) // IObservable extensions .Subscribe(new TweetsObserver()); return(subscriptions); }
private static IDisposable KQueryWithObserver(string ksqlDbUrl) { var contextOptions = new KSqlDBContextOptions(ksqlDbUrl); var context = new KSqlDBContext(contextOptions); var subscription = context.CreateQueryStream <Tweet>() .Where(p => p.Message != "Hello world" && p.Id != 1) .Take(2) .Subscribe(new TweetsObserver()); return(subscription); }
private static IDisposable Window(KSqlDBContext context) { var subscription1 = context.CreateQueryStream <Tweet>() .GroupBy(c => c.Id) .WindowedBy(new TimeWindows(Duration.OfSeconds(5)).WithGracePeriod(Duration.OfHours(2))) .Select(g => new { g.WindowStart, g.WindowEnd, Id = g.Key, Count = g.Count() }) .Subscribe(c => { Console.WriteLine($"{c.Id}: {c.Count}: {c.WindowStart}: {c.WindowEnd}"); }, exception => { Console.WriteLine(exception.Message); }); var query = context.CreateQueryStream <Tweet>() .GroupBy(c => c.Id) .WindowedBy(new HoppingWindows(Duration.OfSeconds(5)).WithAdvanceBy(Duration.OfSeconds(4)) .WithRetention(Duration.OfDays(7))) .Select(g => new { Id = g.Key, Count = g.Count() }); var hoppingWindowQueryString = query.ToQueryString(); var subscription2 = query .Subscribe(c => { Console.WriteLine($"{c.Id}: {c.Count}"); }, exception => { Console.WriteLine(exception.Message); }); return(new CompositeDisposable { subscription1, subscription2 }); }
private static IDisposable JoinTables(KSqlDBContext context) { var rightJoinQueryString = context.CreateQueryStream <Movie>() .RightJoin( Source.Of <Lead_Actor>(nameof(Lead_Actor)), movie => movie.Title, actor => actor.Title, (movie, actor) => new { movie.Id, actor.Title, } ).ToQueryString(); var query = context.CreateQueryStream <Movie>() .Join( //.LeftJoin( Source.Of <Lead_Actor>(nameof(Lead_Actor)), movie => movie.Title, actor => actor.Title, (movie, actor) => new { movie.Id, Title = movie.Title, movie.Release_Year, ActorName = K.Functions.RPad(K.Functions.LPad(actor.Actor_Name.ToUpper(), 15, "*"), 25, "^"), ActorTitle = actor.Title, Substr = K.Functions.Substring(actor.Title, 2, 4) } ); var joinQueryString = query.ToQueryString(); return(query .Subscribe(c => { Console.WriteLine($"{c.Id}: {c.ActorName} - {c.Title} - {c.ActorTitle}"); }, exception => { Console.WriteLine(exception.Message); })); }
private static IDisposable ClientSideBatching(KSqlDBContext context) { var disposable = context.CreateQueryStream <Tweet>() .ToObservable() .Buffer(TimeSpan.FromMilliseconds(250), 100) .Where(c => c.Count > 0) //.ObserveOn(System.Reactive.Concurrency.DispatcherScheduler.Current) //WPF .Subscribe(tweets => { foreach (var tweet in tweets) { Console.WriteLine(tweet.Message); } }); return(disposable); }
public static async Task Main(string[] args) { var ksqlDbUrl = @"http:\\localhost:8088"; var loggerFactory = CreateLoggerFactory(); var httpClientFactory = new HttpClientFactory(new Uri(ksqlDbUrl)); var restApiProvider = new KSqlDbRestApiProvider(httpClientFactory, loggerFactory) { DisposeHttpClient = false }; restApiProvider.SetCredentials(new BasicAuthCredentials("fred", "letmein")); var moviesProvider = new MoviesProvider(restApiProvider); await moviesProvider.CreateTablesAsync(); var contextOptions = CreateQueryStreamOptions(ksqlDbUrl); await using var context = new KSqlDBContext(contextOptions, loggerFactory); var query = context.CreateQueryStream <Movie>() // Http 2.0 // var query = context.CreateQuery<Movie>() // Http 1.0 .Where(p => p.Title != "E.T.") .Where(c => c.Title.ToLower().Contains("hard".ToLower()) || c.Id == 1) .Where(p => p.RowTime >= 1510923225000) .Select(l => new { Id = l.Id, l.Title, l.Release_Year, l.RowTime }) .Take(2); // LIMIT 2 var ksql = query.ToQueryString(); Console.WriteLine("Generated ksql:"); Console.WriteLine(ksql); Console.WriteLine(); using var disposable = query .ToObservable() // client side processing starts here lazily after subscription. Switches to Rx.NET .Finally(() => { Console.WriteLine("Finally"); }) .Subscribe(onNext: movie => { Console.WriteLine($"{nameof(Movie)}: {movie.Id} - {movie.Title} - {movie.RowTime}"); Console.WriteLine(); }, onError: error => { Console.WriteLine($"Exception: {error.Message}"); }, onCompleted: () => Console.WriteLine("Completed")); await CreateOrReplaceTableStatement(context); await moviesProvider.InsertMovieAsync(MoviesProvider.Movie1); await moviesProvider.InsertMovieAsync(MoviesProvider.Movie2); await moviesProvider.InsertLeadAsync(MoviesProvider.LeadActor1); try { await new PullQueryExample().ExecuteAsync(); } catch (Exception e) { Console.WriteLine(); Console.WriteLine(e.Message); } string explain = await query.ExplainAsStringAsync(); ExplainResponse[] explainResponses = await query.ExplainAsync(); Console.WriteLine($"{Environment.NewLine} Explain => ExecutionPlan:"); Console.WriteLine(explainResponses[0].QueryDescription?.ExecutionPlan); Console.WriteLine("Press any key to stop the subscription"); Console.ReadKey(); await moviesProvider.DropTablesAsync(); Console.WriteLine("Finished."); }