private static async Task GetKsqlDbInformationAsync(KSqlDbRestApiProvider restApiProvider)
{
Console.WriteLine($"{Environment.NewLine}Available topics:");
var topicsResponses = await restApiProvider.GetTopicsAsync();
Console.WriteLine(string.Join(',', topicsResponses[0].Topics.Select(c => c.Name)));
TopicsResponse[] allTopicsResponses = await restApiProvider.GetAllTopicsAsync();
TopicsExtendedResponse[] topicsExtendedResponses = await restApiProvider.GetTopicsExtendedAsync();
var allTopicsExtendedResponses = await restApiProvider.GetAllTopicsExtendedAsync();
Console.WriteLine($"{Environment.NewLine}Available tables:");
var tablesResponse = await restApiProvider.GetTablesAsync();
Console.WriteLine(string.Join(',', tablesResponse[0].Tables.Select(c => c.Name)));
Console.WriteLine($"{Environment.NewLine}Available streams:");
var streamsResponse = await restApiProvider.GetStreamsAsync();
Console.WriteLine(string.Join(',', streamsResponse[0].Streams.Select(c => c.Name)));
Console.WriteLine($"{Environment.NewLine}Available connectors:");
var connectorsResponse = await restApiProvider.GetConnectorsAsync();
Console.WriteLine(string.Join(',', connectorsResponse[0].Connectors.Select(c => c.Name)));
}
public static async Task ClassInitialize(TestContext context)
{
RestApiProvider = KSqlDbRestApiProvider.Create();
moviesProvider = new MoviesProvider(RestApiProvider);
await moviesProvider.CreateTablesAsync();
await moviesProvider.InsertMovieAsync(MoviesProvider.Movie1);
}
protected static async Task InitializeDatabase()
{
RestApiProvider = KSqlDbRestApiProvider.Create();
MoviesProvider = new MoviesProvider(RestApiProvider);
await MoviesProvider.CreateTablesAsync();
await MoviesProvider.InsertMovieAsync(MoviesProvider.Movie1);
}
public static async Task Main(string[] args)
{
var ksqlDbUrl = @"http:\\localhost:8088";
var httpClientFactory = new HttpClientFactory(new Uri(ksqlDbUrl));
var restApiProvider = new KSqlDbRestApiProvider(httpClientFactory);
var moviesProvider = new MoviesProvider(restApiProvider);
await moviesProvider.CreateTablesAsync();
var contextOptions = CreateQueryStreamOptions(ksqlDbUrl);
var context = new KSqlDBContext(contextOptions);
var subscription = context.CreateQuery <Movie>()
.Where(p => p.Title != "E.T.")
.Where(c => K.Functions.Like(c.Title.ToLower(), "%hard%".ToLower()) || c.Id == 1)
.Where(p => p.RowTime >= 1510923225000) //AND RowTime >= 1510923225000
.Select(l => new { Id2 = l.Id, l.Title, l.Release_Year, l.RowTime })
.Take(2) // LIMIT 2
.ToObservable() // client side processing starts here lazily after subscription. Switches to Rx.NET
.ObserveOn(TaskPoolScheduler.Default)
.Subscribe(onNext: movie =>
{
Console.WriteLine($"{nameof(Movie)}: {movie.Id2} - {movie.Title} - {movie.RowTime}");
Console.WriteLine();
}, onError: error => { Console.WriteLine($"Exception: {error.Message}"); }, onCompleted: () => Console.WriteLine("Completed"));
await moviesProvider.InsertMovieAsync(MoviesProvider.Movie1);
await moviesProvider.InsertMovieAsync(MoviesProvider.Movie2);
await moviesProvider.InsertLeadAsync(MoviesProvider.LeadActor1);
try
{
await new PullQueryExample().ExecuteAsync();
}
catch (Exception e)
{
Console.WriteLine(e);
}
Console.WriteLine("Press any key to stop the subscription");
Console.ReadKey();
await context.DisposeAsync();
using (subscription)
{
}
Console.WriteLine("Subscription completed");
}
public async Task CreateOrReplaceStreamStatement_ToStatementString_ComplexQueryWasGenerated()
{
//Arrange
var restApiClient = KSqlDbRestApiProvider.Create();
var statement = new KSqlDbStatement(StatementTemplates.DropStream(StreamName));
var response = await restApiClient.ExecuteStatementAsync(statement);
EntityCreationMetadata metadata = new()
{
EntityName = StreamEntityName,
KafkaTopic = nameof(Movie) + "Test2",
Partitions = 1,
Replicas = 1
};
var httpResponseMessage =
await restApiClient.CreateStreamAsync <Movie>(metadata, ifNotExists : false);
var creationMetadata = new CreationMetadata
{
KafkaTopic = "moviesByTitle",
KeyFormat = SerializationFormats.Json,
ValueFormat = SerializationFormats.Json,
Replicas = 1,
Partitions = 1
};
var createStatement = Context.CreateOrReplaceStreamStatement(StreamName)
.With(creationMetadata)
.As <Movie>(StreamEntityName)
.Where(c => c.Id < 3)
.Select(c => new { c.Id, c.Title, ReleaseYear = c.Release_Year })
.PartitionBy(c => c.Id);
//Act
var ksql = createStatement.ToStatementString();
httpResponseMessage = await createStatement.ExecuteStatementAsync();
//Assert
ksql.Should().BeEquivalentTo(@$ "CREATE OR REPLACE STREAM {StreamName}
WITH ( KAFKA_TOPIC='moviesByTitle', KEY_FORMAT='Json', VALUE_FORMAT='Json', PARTITIONS='1', REPLICAS='1' )
AS SELECT Id, Title, Release_Year AS ReleaseYear FROM {StreamEntityName}
WHERE Id < 3 PARTITION BY Id EMIT CHANGES;");
var responses = await httpResponseMessage.ToStatementResponsesAsync();
responses[0].CommandStatus.Status.Should().BeOneOf("SUCCESS", "EXECUTING");
}
public static async Task ClassInitialize(TestContext context)
{
RestApiProvider = KSqlDbRestApiProvider.Create();
var response = await RestApiProvider.CreateTypeAsync <TestState>();
response = await RestApiProvider.CreateTypeAsync <TestCity>();
response = await RestApiProvider.CreateTableAsync <TestCity>(new EntityCreationMetadata { EntityName = CitiesTableName, KafkaTopic = CitiesTableName, Partitions = 1 });
var c = await response.Content.ReadAsStringAsync();
response = await RestApiProvider.InsertIntoAsync(new TestCity
{ RegionCode = "sk", State = State1 }, new InsertProperties { EntityName = CitiesTableName });
}
public static async Task ClassInitialize(TestContext context)
{
RestApiProvider = KSqlDbRestApiProvider.Create();
moviesProvider = new MoviesProvider(RestApiProvider);
await moviesProvider.DropTablesAsync();
await Task.Delay(TimeSpan.FromSeconds(1));
await moviesProvider.CreateTablesAsync();
await moviesProvider.InsertMovieAsync(MoviesProvider.Movie1);
await moviesProvider.InsertLeadAsync(MoviesProvider.LeadActor1);
}
protected static async Task InitializeDatabase()
{
RestApiProvider = KSqlDbRestApiProvider.Create();
var tweetsProvider = new TweetsProvider(RestApiProvider);
var result = await tweetsProvider.CreateTweetsStream(StreamName, topicName);
result.Should().BeTrue();
result = await tweetsProvider.InsertTweetAsync(Tweet1, StreamName);
result.Should().BeTrue();
result = await tweetsProvider.InsertTweetAsync(Tweet2, StreamName);
result.Should().BeTrue();
}
public async Task AvroExampleAsync(string ksqlDbUrl)
{
var schemaRegistryConfig = new SchemaRegistryConfig
{
Url = "http://localhost:8081"
};
using var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig);
var schema = IoTSensor._SCHEMA.ToString();
var subject = "IoTSensors-value";
var registrationResult = await schemaRegistry.RegisterSchemaAsync(subject, schema);
//http://localhost:8081/subjects/IoTSensors-value/versions/latest/schema
var latestSchema = await schemaRegistry.GetLatestSchemaAsync(subject);
var httpClientFactory = new HttpClientFactory(new Uri(ksqlDbUrl));
var restApiClient = new KSqlDbRestApiProvider(httpClientFactory);
EntityCreationMetadata metadata = new()
{
EntityName = "avroSensors",
KafkaTopic = "IoTSensors",
ValueFormat = SerializationFormats.Avro,
Partitions = 1,
Replicas = 1
};
var httpResponseMessage = await restApiClient.CreateStreamAsync <IoTSensor>(metadata, ifNotExists : false)
.ConfigureAwait(false);
var httpResponse = await httpResponseMessage.Content.ReadAsStringAsync();
Console.WriteLine(httpResponse);
// Stream Name | Kafka Topic | Key Format | Value Format | Windowed
// ------------------------------------------------------------------------------------------------------
// AVROSENSORS | IoTSensors | KAFKA | AVRO | false
}
}
public static async Task ClassInitialize(TestContext context)
{
RestApiProvider = KSqlDbRestApiProvider.Create();
moviesProvider = new MoviesProvider(RestApiProvider);
await moviesProvider.CreateTablesAsync();
await moviesProvider.InsertMovieAsync(MoviesProvider.Movie1);
await moviesProvider.InsertMovieAsync(MoviesProvider.Movie2);
tweetsProvider = new TweetsProvider(RestApiProvider);
await tweetsProvider.CreateTweetsStream(TweetsStreamName, tweetsTopicName);
await tweetsProvider.InsertTweetAsync(TweetsProvider.Tweet1, TweetsStreamName);
await tweetsProvider.InsertTweetAsync(TweetsProvider.Tweet2, TweetsStreamName);
}
public async Task CreateOrReplaceTableStatement_ExecuteStatementAsync_ResponseWasReceived()
{
//Arrange
var restApiClient = KSqlDbRestApiProvider.Create();
await restApiClient.CreateStreamAsync <Movie>(new EntityCreationMetadata(EntityName, 1) { EntityName = EntityName, ShouldPluralizeEntityName = false });
var statement = new KSqlDbStatement(StatementTemplates.DropTable(TableName));
var response = await restApiClient.ExecuteStatementAsync(statement);
int retryCount = 0;
while ((await KSqlDbRestApiProvider.Create().GetTablesAsync()).SelectMany(c => c.Tables).Any(c => c.Name == TableName.ToUpper()))
{
if (retryCount++ > 5)
{
return;
}
await Task.Delay(TimeSpan.FromSeconds(1));
}
var createStatement = Context.CreateTableStatement(TableName)
.As <Movie>(EntityName)
.GroupBy(c => c.Title)
.Select(c => new { Title = c.Key, Count = c.Count() });
//Act
var httpResponseMessage = await createStatement.ExecuteStatementAsync();
//Assert
string responseContent = await httpResponseMessage.Content.ReadAsStringAsync();
responseContent.Should().NotBeNull();
var responses = await httpResponseMessage.ToStatementResponsesAsync();
responses[0].CommandStatus.Status.Should().Be("SUCCESS");
}
public static async Task ClassInitialize(TestContext context)
{
RestApiProvider = KSqlDbRestApiProvider.Create();
var statement =
new KSqlDbStatement(
@"CREATE STREAM stream2 (id INT, lambda_arr ARRAY<INTEGER>) WITH (kafka_topic = 'stream2', partitions = 1, value_format = 'json');");
var response = await RestApiProvider.ExecuteStatementAsync(statement);
var statement2 =
new KSqlDbStatement(
@"CREATE OR REPLACE STREAM stream4 (id INT, lambda_map MAP<STRING,ARRAY<INTEGER>>) WITH (kafka_topic = 'stream4', partitions = 1, value_format = 'json');");
response = await RestApiProvider.ExecuteStatementAsync(statement2);
string insertIntoStream3 = "insert into stream4 (id, lambda_map) values (1, MAP('hello':= ARRAY [1,2,3], 'goodbye':= ARRAY [-1,-2,-3]) );";
response = await RestApiProvider.ExecuteStatementAsync(
new KSqlDbStatement("insert into stream2 (id, lambda_arr) values (1, ARRAY [1,2,3]);"));
response = await RestApiProvider.ExecuteStatementAsync(
new KSqlDbStatement(insertIntoStream3));
}
public MoviesProvider(KSqlDbRestApiProvider restApiProvider)
{
this.restApiProvider = restApiProvider ?? throw new ArgumentNullException(nameof(restApiProvider));
}
public static async Task ClassInitialize(TestContext context)
{
var restApiClient = KSqlDbRestApiProvider.Create();
await restApiClient.CreateStreamAsync <Movie>(new EntityCreationMetadata(EntityName, 1) { EntityName = EntityName, ShouldPluralizeEntityName = false });
}
public static async Task Main(string[] args)
{
var ksqlDbUrl = @"http:\\localhost:8088";
var loggerFactory = CreateLoggerFactory();
var httpClientFactory = new HttpClientFactory(new Uri(ksqlDbUrl));
var restApiProvider = new KSqlDbRestApiProvider(httpClientFactory, loggerFactory)
{
DisposeHttpClient = false
};
restApiProvider.SetCredentials(new BasicAuthCredentials("fred", "letmein"));
var moviesProvider = new MoviesProvider(restApiProvider);
await moviesProvider.CreateTablesAsync();
var contextOptions = CreateQueryStreamOptions(ksqlDbUrl);
await using var context = new KSqlDBContext(contextOptions, loggerFactory);
var query = context.CreateQueryStream <Movie>() // Http 2.0
// var query = context.CreateQuery<Movie>() // Http 1.0
.Where(p => p.Title != "E.T.")
.Where(c => c.Title.ToLower().Contains("hard".ToLower()) || c.Id == 1)
.Where(p => p.RowTime >= 1510923225000)
.Select(l => new { Id = l.Id, l.Title, l.Release_Year, l.RowTime })
.Take(2); // LIMIT 2
var ksql = query.ToQueryString();
Console.WriteLine("Generated ksql:");
Console.WriteLine(ksql);
Console.WriteLine();
using var disposable = query
.ToObservable() // client side processing starts here lazily after subscription. Switches to Rx.NET
.Finally(() => { Console.WriteLine("Finally"); })
.Subscribe(onNext: movie =>
{
Console.WriteLine($"{nameof(Movie)}: {movie.Id} - {movie.Title} - {movie.RowTime}");
Console.WriteLine();
}, onError: error => { Console.WriteLine($"Exception: {error.Message}"); }, onCompleted: () => Console.WriteLine("Completed"));
await CreateOrReplaceTableStatement(context);
await moviesProvider.InsertMovieAsync(MoviesProvider.Movie1);
await moviesProvider.InsertMovieAsync(MoviesProvider.Movie2);
await moviesProvider.InsertLeadAsync(MoviesProvider.LeadActor1);
try
{
await new PullQueryExample().ExecuteAsync();
}
catch (Exception e)
{
Console.WriteLine();
Console.WriteLine(e.Message);
}
string explain = await query.ExplainAsStringAsync();
ExplainResponse[] explainResponses = await query.ExplainAsync();
Console.WriteLine($"{Environment.NewLine} Explain => ExecutionPlan:");
Console.WriteLine(explainResponses[0].QueryDescription?.ExecutionPlan);
Console.WriteLine("Press any key to stop the subscription");
Console.ReadKey();
await moviesProvider.DropTablesAsync();
Console.WriteLine("Finished.");
}
