public void Init()
{
creationMetadata = new EntityCreationMetadata()
{
KafkaTopic = nameof(MyMovie),
Partitions = 1,
Replicas = 1
};
}
public static string CreateStream <T>(EntityCreationMetadata creationMetadata, bool ifNotExists = false)
{
var statementContext = new StatementContext
{
CreationType = CreationType.Create,
KSqlEntityType = KSqlEntityType.Stream
};
return(CreateOrReplace <T>(statementContext, creationMetadata, ifNotExists));
}
public static string CreateOrReplaceTable <T>(EntityCreationMetadata creationMetadata)
{
var statementContext = new StatementContext
{
CreationType = CreationType.CreateOrReplace,
KSqlEntityType = KSqlEntityType.Table
};
return(CreateOrReplace <T>(statementContext, creationMetadata, ifNotExists: null));
}
/// <summary>
/// Create a new table or replace an existing one with the specified columns and properties.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="creationMetadata">Table properties, specify details about your table by using the WITH clause.</param>
/// <param name="cancellationToken">Optional cancellation token to cancel the operation</param>
/// <returns>Http response object.</returns>
public Task <HttpResponseMessage> CreateOrReplaceTableAsync <T>(EntityCreationMetadata creationMetadata, CancellationToken cancellationToken = default)
{
if (creationMetadata == null)
{
throw new ArgumentNullException(nameof(creationMetadata));
}
var ksql = StatementGenerator.CreateOrReplaceTable <T>(creationMetadata);
return(ExecuteAsync <T>(ksql, cancellationToken));
}
/// <summary>
/// Create a new stream with the specified columns and properties.
/// </summary>
/// <typeparam name="T">The type that represents the stream.</typeparam>
/// <param name="creationMetadata">Stream properties, specify details about your stream by using the WITH clause.</param>
/// <param name="ifNotExists">If the IF NOT EXISTS clause is present, the statement won't fail if a stream with the same name already exists.</param>
/// <param name="cancellationToken">Optional cancellation token to cancel the operation</param>
/// <returns>Http response object.</returns>
public Task <HttpResponseMessage> CreateStreamAsync <T>(EntityCreationMetadata creationMetadata, bool ifNotExists = false, CancellationToken cancellationToken = default)
{
if (creationMetadata == null)
{
throw new ArgumentNullException(nameof(creationMetadata));
}
var ksql = StatementGenerator.CreateStream <T>(creationMetadata, ifNotExists);
return(ExecuteAsync <T>(ksql, cancellationToken));
}
public void GenerateWithClause_WindowSize()
{
//Arrange
var metadata = new EntityCreationMetadata
{
WindowSize = "10 MINUTES"
};
//Act
var withClause = CreateStatements.GenerateWithClause(metadata);
//Assert
withClause.Should().BeEquivalentTo(@$ " WITH ( WINDOW_SIZE='{metadata.WindowSize}', VALUE_FORMAT='Json' )");
}
public void GenerateWithClause_WindowType()
{
//Arrange
var metadata = new EntityCreationMetadata
{
WindowType = WindowType.Hopping
};
//Act
var withClause = CreateStatements.GenerateWithClause(metadata);
//Assert
withClause.Should().BeEquivalentTo(@$ " WITH ( WINDOW_TYPE='{metadata.WindowType}', VALUE_FORMAT='Json' )");
}
private static EntityCreationMetadata GetEntityCreationMetadata(string topicName)
{
EntityCreationMetadata metadata = new EntityCreationMetadata()
{
KeyFormat = SerializationFormats.Json,
KafkaTopic = topicName,
Partitions = 1,
Replicas = 1,
WindowType = WindowType.Tumbling,
WindowSize = "10 SECONDS",
TimestampFormat = "yyyy-MM-dd''T''HH:mm:ssX"
};
return(metadata);
}
public static string CreateTable <T>(EntityCreationMetadata creationMetadata, bool ifNotExists = false)
{
if (creationMetadata == null)
{
throw new ArgumentNullException(nameof(creationMetadata));
}
var statementContext = new StatementContext
{
CreationType = CreationType.Create,
KSqlEntityType = KSqlEntityType.Table
};
return(CreateOrReplace <T>(statementContext, creationMetadata, ifNotExists));
}
internal static string GenerateWithClause(EntityCreationMetadata metadata)
{
if (metadata == null)
{
throw new ArgumentNullException(nameof(metadata));
}
var properties = new List <string>();
if (metadata.WindowType.HasValue)
{
properties.Add(@$ "WINDOW_TYPE='{metadata.WindowType}'");
}
if (metadata.WindowSize.IsNotNullOrEmpty())
{
properties.Add(@$ "WINDOW_SIZE='{metadata.WindowSize}'");
}
return(GenerateWithClause(metadata, properties));
}
public async Task SubscribeAsync()
{
var entityCreationMetadata = new EntityCreationMetadata
{
KafkaTopic = nameof(Order) + "-Join",
Partitions = 1
};
var response = await restApiClient.CreateStreamAsync <Order>(entityCreationMetadata, ifNotExists : true);
var r = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
response = await restApiClient.CreateTableAsync <Payment>(entityCreationMetadata with {
KafkaTopic = nameof(Payment) + "-Join"
}, ifNotExists : true);
response = await restApiClient.CreateTableAsync <Shipment>(entityCreationMetadata with {
KafkaTopic = nameof(Shipment) + "-Join"
}, ifNotExists : true);
var value = new Foo {
Prop = 42
};
var query = (from o in context.CreateQueryStream <Order>()
join p1 in Source.Of <Payment>() on o.PaymentId equals p1.Id
join s1 in Source.Of <Shipment>() on o.ShipmentId equals s1.Id into gj
from sa in gj.DefaultIfEmpty()
select new
{
value,
orderId = o.OrderId,
shipmentId = sa.Id,
paymentId = p1.Id,
})
.Take(5);
string ksql = query.ToQueryString();
using var subscription = query
.Subscribe(c =>
{
Console.WriteLine($"{nameof(Order.OrderId)}: {c.orderId}");
Console.WriteLine($"{nameof(Order.PaymentId)}: {c.paymentId}");
if (c.shipmentId.HasValue)
{
Console.WriteLine($"{nameof(Order.ShipmentId)}: {c.shipmentId}");
}
}, error =>
{
Console.WriteLine(error.Message);
});
var order = new Order {
OrderId = 1, PaymentId = 1, ShipmentId = 1
};
var payment = new Payment {
Id = 1
};
var shipment = new Shipment {
Id = 1
};
response = await restApiClient.InsertIntoAsync(order);
r = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
response = await restApiClient.InsertIntoAsync(payment);
await Task.Delay(TimeSpan.FromMilliseconds(250));
response = await restApiClient.InsertIntoAsync(shipment);
}
private static string CreateOrReplace <T>(StatementContext statementContext, EntityCreationMetadata creationMetadata, bool?ifNotExists)
{
string ksql = new CreateEntity().Print <T>(statementContext, creationMetadata, ifNotExists);
return(ksql);
}
