public async Task ReceiveArrayWithComplexElements()
{
//Arrange
var httpResponseMessage = await restApiClient.ExecuteStatementAsync(new KSqlDbStatement(@"
Drop type EventCategory;
Drop table Events;
"));
//Act
httpResponseMessage = await restApiClient.CreateTypeAsync <EventCategory>();
httpResponseMessage = await restApiClient.CreateTableAsync <Event>(new EntityCreationMetadata()
{
KafkaTopic = "Events", Partitions = 1
});
var eventCategory = new EventCategory()
{
Name = "xyz"
};
var testEvent = new Event
{
Id = 1,
Places = new[] { "Place1", "Place2" },
Categories = new[] { eventCategory, new EventCategory {
Name = "puk"
} }
};
var semaphoreSlim = new SemaphoreSlim(0, 1);
var receivedValues = new List <Event>();
var subscription = Context.CreateQueryStream <Event>().Take(1)
.Subscribe(value =>
{
receivedValues.Add(value);
}, error =>
{
semaphoreSlim.Release();
},
() =>
{
semaphoreSlim.Release();
});
httpResponseMessage = await restApiClient.InsertIntoAsync(testEvent);
string responseContent = await httpResponseMessage.Content.ReadAsStringAsync();
//Assert
await semaphoreSlim.WaitAsync(TimeSpan.FromSeconds(5));
receivedValues.Count.Should().Be(1);
receivedValues[0].Places.Length.Should().Be(2);
receivedValues[0].Categories.ToArray()[0].Name.Should().Be("xyz");
receivedValues[0].Categories.ToArray()[1].Name.Should().Be("puk");
using (subscription) { }
}
private static IDisposable DynamicFunctionCall(KSqlDBContext context)
{
var ifNullQueryString = context.CreateQueryStream <Tweet>()
.Select(c => new { c.Id, c.Amount, Col = K.F.Dynamic("IFNULL(Message, 'n/a')") as string })
.ToQueryString();
return(context.CreateQueryStream <Tweet>()
.Select(c => K.Functions.Dynamic("ARRAY_DISTINCT(ARRAY[1, 1, 2, 3, 1, 2])") as int[])
.Subscribe(
message => Console.WriteLine($"{message[0]} - {message[^1]}"),
private static async Task GroupBy()
{
var ksqlDbUrl = @"http:\\localhost:8088";
var contextOptions = new KSqlDBContextOptions(ksqlDbUrl);
contextOptions.QueryStreamParameters["auto.offset.reset"] = "latest";
await using var context = new KSqlDBContext(contextOptions);
context.CreateQueryStream <Tweet>()
.GroupBy(c => c.Id)
.Select(g => new { Id = g.Key, Count = g.Count() })
.Subscribe(count =>
{
Console.WriteLine($"{count.Id} Count: {count.Count}");
Console.WriteLine();
}, error => { Console.WriteLine($"Exception: {error.Message}"); }, () => Console.WriteLine("Completed"));
context.CreateQueryStream <Tweet>()
.GroupBy(c => c.Id)
.Select(g => g.Count())
.Subscribe(count =>
{
Console.WriteLine($"Count: {count}");
Console.WriteLine();
}, error => { Console.WriteLine($"Exception: {error.Message}"); }, () => Console.WriteLine("Completed"));
context.CreateQueryStream <Tweet>()
.GroupBy(c => c.Id)
.Select(g => new { Count = g.Count() })
.Subscribe(count =>
{
Console.WriteLine($"Count: {count}");
Console.WriteLine();
}, error => { Console.WriteLine($"Exception: {error.Message}"); }, () => Console.WriteLine("Completed"));
//Sum
var subscription = context.CreateQueryStream <Tweet>()
.GroupBy(c => c.Id)
//.Select(g => g.Sum(c => c.Id))
.Select(g => new { Id = g.Key, MySum = g.Sum(c => c.Id) })
.Subscribe(sum =>
{
Console.WriteLine($"{sum}");
Console.WriteLine();
}, error => { Console.WriteLine($"Exception: {error.Message}"); }, () => Console.WriteLine("Completed"));
var groupBySubscription = context.CreateQueryStream <IoTSensorChange>("sqlserversensors")
.GroupBy(c => new { c.Op, c.After !.Value })
private static IDisposable NotNull(KSqlDBContext context)
{
return(context.CreateQueryStream <Click>()
.Where(c => c.IP_ADDRESS != null)
.Select(c => new { c.IP_ADDRESS, c.URL, c.TIMESTAMP })
.Subscribe(message => Console.WriteLine(message), error => { Console.WriteLine($"Exception: {error.Message}"); }));
}
private static IDisposable FullOuterJoinTables(KSqlDBContext context)
{
var query = context.CreateQueryStream <MovieNullableFields>("Movies")
.FullOuterJoin(
Source.Of <Lead_Actor>(nameof(Lead_Actor)),
movie => movie.Title,
actor => actor.Title,
(movie, actor) => new
{
movie.Id,
Title = movie.Title,
movie.Release_Year,
ActorTitle = actor.Title,
ActorName = actor.Actor_Name
}
);
var joinQueryString = query.ToQueryString();
return(query
.Subscribe(c =>
{
if (c.Id.HasValue)
{
Console.WriteLine($"{c.Id}: {c.ActorName} - {c.Title} - {c.ActorTitle}");
}
else
{
Console.WriteLine($"No movie id: {c.ActorName} - {c.Title} - {c.ActorTitle}");
}
}, exception => { Console.WriteLine(exception.Message); }));
}
public static async Task Main(string[] args)
{
var ksqlDbUrl = @"http:\\localhost:8088";
var contextOptions = new KSqlDBContextOptions(ksqlDbUrl);
await using var context = new KSqlDBContext(contextOptions);
using var disposable = context.CreateQueryStream <Tweet>()
.Where(p => p.Message != "Hello world" || p.Id == 1)
.Where(c => K.Functions.Like(c.Message.ToLower(), "%ALL%".ToLower()))
.Where(p => p.RowTime >= 1510923225000) //AND RowTime >= 1510923225000
.Select(l => new { l.Id, l.Message, l.RowTime })
.Take(2) // LIMIT 2
.ToObservable() // client side processing starts here lazily after subscription
.ObserveOn(TaskPoolScheduler.Default)
.Subscribe(onNext: tweetMessage =>
{
Console.WriteLine($"{nameof(Tweet)}: {tweetMessage.Id} - {tweetMessage.Message}");
Console.WriteLine();
}, onError: error => { Console.WriteLine($"Exception: {error.Message}"); }, onCompleted: () => Console.WriteLine("Completed"));
Console.WriteLine("Press any key to stop the subscription");
Console.ReadKey();
Console.WriteLine("Subscription completed");
}
private static async Task ToQueryStringExample(string ksqlDbUrl)
{
var contextOptions = new KSqlDBContextOptions(ksqlDbUrl);
await using var context = new KSqlDBContext(contextOptions);
var ksql = context.CreateQueryStream <Person>().ToQueryString();
//prints SELECT * FROM People EMIT CHANGES;
Console.WriteLine(ksql);
}
private static async Task AsyncEnumerable(KSqlDBContext context)
{
var cts = new CancellationTokenSource();
var asyncTweetsEnumerable = context.CreateQueryStream <Tweet>().ToAsyncEnumerable();
await foreach (var tweet in asyncTweetsEnumerable.WithCancellation(cts.Token))
{
Console.WriteLine(tweet.Message);
cts.Cancel();
}
}
public static async Task Main(string[] args)
{
var ksqlDbUrl = @"http:\\localhost:8088";
var httpClientFactory = new HttpClientFactory(new Uri(ksqlDbUrl));
var restApiProvider = new KSqlDbRestApiProvider(httpClientFactory);
var moviesProvider = new MoviesProvider(restApiProvider);
await moviesProvider.CreateTablesAsync();
var contextOptions = CreateQueryStreamOptions(ksqlDbUrl);
await using var context = new KSqlDBContext(contextOptions);
using var disposable = context.CreateQueryStream <Movie>() // Http 2.0
// using var disposable = context.CreateQuery<Movie>() // Http 1.0
.Where(p => p.Title != "E.T.")
.Where(c => K.Functions.Like(c.Title.ToLower(), "%hard%".ToLower()) || c.Id == 1)
.Where(p => p.RowTime >= 1510923225000) //AND RowTime >= 1510923225000
.Select(l => new { Id = l.Id, l.Title, l.Release_Year, l.RowTime })
.Take(2) // LIMIT 2
.ToObservable() // client side processing starts here lazily after subscription. Switches to Rx.NET
.ObserveOn(TaskPoolScheduler.Default)
.Subscribe(onNext: movie =>
{
Console.WriteLine($"{nameof(Movie)}: {movie.Id} - {movie.Title} - {movie.RowTime}");
Console.WriteLine();
}, onError: error => { Console.WriteLine($"Exception: {error.Message}"); }, onCompleted: () => Console.WriteLine("Completed"));
await CreateOrReplaceTableStatement(context);
await moviesProvider.InsertMovieAsync(MoviesProvider.Movie1);
await moviesProvider.InsertMovieAsync(MoviesProvider.Movie2);
await moviesProvider.InsertLeadAsync(MoviesProvider.LeadActor1);
try
{
await new PullQueryExample().ExecuteAsync();
}
catch (Exception e)
{
Console.WriteLine(e);
}
Console.WriteLine("Press any key to stop the subscription");
Console.ReadKey();
await moviesProvider.DropTablesAsync();
Console.WriteLine("Subscription completed");
}
private static IDisposable ToObservableExample(string ksqlDbUrl)
{
var contextOptions = new KSqlDBContextOptions(ksqlDbUrl);
var context = new KSqlDBContext(contextOptions);
var subscriptions = context.CreateQueryStream <Tweet>()
.ToObservable()
.Delay(TimeSpan.FromSeconds(2)) // IObservable extensions
.Subscribe(new TweetsObserver());
return(subscriptions);
}
private static IDisposable KQueryWithObserver(string ksqlDbUrl)
{
var contextOptions = new KSqlDBContextOptions(ksqlDbUrl);
var context = new KSqlDBContext(contextOptions);
var subscription = context.CreateQueryStream <Tweet>()
.Where(p => p.Message != "Hello world" && p.Id != 1)
.Take(2)
.Subscribe(new TweetsObserver());
return(subscription);
}
private static IDisposable Window(KSqlDBContext context)
{
var subscription1 = context.CreateQueryStream <Tweet>()
.GroupBy(c => c.Id)
.WindowedBy(new TimeWindows(Duration.OfSeconds(5)).WithGracePeriod(Duration.OfHours(2)))
.Select(g => new { g.WindowStart, g.WindowEnd, Id = g.Key, Count = g.Count() })
.Subscribe(c => { Console.WriteLine($"{c.Id}: {c.Count}: {c.WindowStart}: {c.WindowEnd}"); }, exception => { Console.WriteLine(exception.Message); });
var query = context.CreateQueryStream <Tweet>()
.GroupBy(c => c.Id)
.WindowedBy(new HoppingWindows(Duration.OfSeconds(5)).WithAdvanceBy(Duration.OfSeconds(4))
.WithRetention(Duration.OfDays(7)))
.Select(g => new { Id = g.Key, Count = g.Count() });
var hoppingWindowQueryString = query.ToQueryString();
var subscription2 = query
.Subscribe(c => { Console.WriteLine($"{c.Id}: {c.Count}"); }, exception => { Console.WriteLine(exception.Message); });
return(new CompositeDisposable {
subscription1, subscription2
});
}
private static IDisposable JoinTables(KSqlDBContext context)
{
var rightJoinQueryString = context.CreateQueryStream <Movie>()
.RightJoin(
Source.Of <Lead_Actor>(nameof(Lead_Actor)),
movie => movie.Title,
actor => actor.Title,
(movie, actor) => new
{
movie.Id,
actor.Title,
}
).ToQueryString();
var query = context.CreateQueryStream <Movie>()
.Join(
//.LeftJoin(
Source.Of <Lead_Actor>(nameof(Lead_Actor)),
movie => movie.Title,
actor => actor.Title,
(movie, actor) => new
{
movie.Id,
Title = movie.Title,
movie.Release_Year,
ActorName = K.Functions.RPad(K.Functions.LPad(actor.Actor_Name.ToUpper(), 15, "*"), 25, "^"),
ActorTitle = actor.Title,
Substr = K.Functions.Substring(actor.Title, 2, 4)
}
);
var joinQueryString = query.ToQueryString();
return(query
.Subscribe(c => { Console.WriteLine($"{c.Id}: {c.ActorName} - {c.Title} - {c.ActorTitle}"); }, exception => { Console.WriteLine(exception.Message); }));
}
private static IDisposable ClientSideBatching(KSqlDBContext context)
{
var disposable = context.CreateQueryStream <Tweet>()
.ToObservable()
.Buffer(TimeSpan.FromMilliseconds(250), 100)
.Where(c => c.Count > 0)
//.ObserveOn(System.Reactive.Concurrency.DispatcherScheduler.Current) //WPF
.Subscribe(tweets =>
{
foreach (var tweet in tweets)
{
Console.WriteLine(tweet.Message);
}
});
return(disposable);
}
public static async Task Main(string[] args)
{
var ksqlDbUrl = @"http:\\localhost:8088";
var loggerFactory = CreateLoggerFactory();
var httpClientFactory = new HttpClientFactory(new Uri(ksqlDbUrl));
var restApiProvider = new KSqlDbRestApiProvider(httpClientFactory, loggerFactory)
{
DisposeHttpClient = false
};
restApiProvider.SetCredentials(new BasicAuthCredentials("fred", "letmein"));
var moviesProvider = new MoviesProvider(restApiProvider);
await moviesProvider.CreateTablesAsync();
var contextOptions = CreateQueryStreamOptions(ksqlDbUrl);
await using var context = new KSqlDBContext(contextOptions, loggerFactory);
var query = context.CreateQueryStream <Movie>() // Http 2.0
// var query = context.CreateQuery<Movie>() // Http 1.0
.Where(p => p.Title != "E.T.")
.Where(c => c.Title.ToLower().Contains("hard".ToLower()) || c.Id == 1)
.Where(p => p.RowTime >= 1510923225000)
.Select(l => new { Id = l.Id, l.Title, l.Release_Year, l.RowTime })
.Take(2); // LIMIT 2
var ksql = query.ToQueryString();
Console.WriteLine("Generated ksql:");
Console.WriteLine(ksql);
Console.WriteLine();
using var disposable = query
.ToObservable() // client side processing starts here lazily after subscription. Switches to Rx.NET
.Finally(() => { Console.WriteLine("Finally"); })
.Subscribe(onNext: movie =>
{
Console.WriteLine($"{nameof(Movie)}: {movie.Id} - {movie.Title} - {movie.RowTime}");
Console.WriteLine();
}, onError: error => { Console.WriteLine($"Exception: {error.Message}"); }, onCompleted: () => Console.WriteLine("Completed"));
await CreateOrReplaceTableStatement(context);
await moviesProvider.InsertMovieAsync(MoviesProvider.Movie1);
await moviesProvider.InsertMovieAsync(MoviesProvider.Movie2);
await moviesProvider.InsertLeadAsync(MoviesProvider.LeadActor1);
try
{
await new PullQueryExample().ExecuteAsync();
}
catch (Exception e)
{
Console.WriteLine();
Console.WriteLine(e.Message);
}
string explain = await query.ExplainAsStringAsync();
ExplainResponse[] explainResponses = await query.ExplainAsync();
Console.WriteLine($"{Environment.NewLine} Explain => ExecutionPlan:");
Console.WriteLine(explainResponses[0].QueryDescription?.ExecutionPlan);
Console.WriteLine("Press any key to stop the subscription");
Console.ReadKey();
await moviesProvider.DropTablesAsync();
Console.WriteLine("Finished.");
}
