public void CreateOrReplaceStreamStatement_ToStatementString_ComplexQueryWasGenerated()
{
//Arrange
var creationMetadata = new CreationMetadata
{
KafkaTopic = "tweetsByTitle",
KeyFormat = SerializationFormats.Json,
ValueFormat = SerializationFormats.Json,
Replicas = 1,
Partitions = 1
};
var query = DbProvider.CreateOrReplaceStreamStatement(StreamName)
.With(creationMetadata)
.As <Movie>()
.Where(c => c.Id < 3)
.Select(c => new { c.Title, ReleaseYear = c.Release_Year })
.PartitionBy(c => c.Title);
//Act
var ksql = query.ToStatementString();
//Assert
ksql.Should().BeEquivalentTo(@$ "CREATE OR REPLACE STREAM {StreamName}
WITH ( KAFKA_TOPIC='tweetsByTitle', KEY_FORMAT='Json', VALUE_FORMAT='Json', PARTITIONS='1', REPLICAS='1' )
AS SELECT Title, Release_Year AS ReleaseYear FROM Movies
WHERE Id < 3 PARTITION BY Title EMIT CHANGES;");
}
private static async Task CreateOrReplaceTableStatement(IKSqlDBStatementsContext context)
{
var creationMetadata = new CreationMetadata
{
KafkaTopic = "tweetsByTitle",
KeyFormat = SerializationFormats.Json,
ValueFormat = SerializationFormats.Json,
Replicas = 1,
Partitions = 1
};
var httpResponseMessage = await context.CreateOrReplaceTableStatement(tableName : "TweetsByTitle")
.With(creationMetadata)
.As <Movie>()
.Where(c => c.Id < 3)
.Select(c => new { c.Title, ReleaseYear = c.Release_Year })
.PartitionBy(c => c.Title)
.ExecuteStatementAsync();
/*
* CREATE OR REPLACE TABLE TweetsByTitle
* WITH ( KAFKA_TOPIC='tweetsByTitle', KEY_FORMAT='Json', VALUE_FORMAT='Json', PARTITIONS = '1', REPLICAS='1' )
* AS SELECT Title, Release_Year AS ReleaseYear FROM Movies
* WHERE Id < 3 PARTITION BY Title EMIT CHANGES;
*/
string responseContent = await httpResponseMessage.Content.ReadAsStringAsync();
var statementResponse = await httpResponseMessage.ToStatementResponsesAsync();
}
public void GenerateWithClause_ValueSchemaFullName()
{
//Arrange
var metadata = new CreationMetadata
{
ValueSchemaFullName = "ProductInfo"
};
//Act
var withClause = CreateStatements.GenerateWithClause(metadata);
//Assert
withClause.Should().BeEquivalentTo(@$ " WITH ( VALUE_SCHEMA_FULL_NAME={metadata.ValueSchemaFullName} )");
}
public void GenerateWithClause_ValueFormat()
{
//Arrange
var metadata = new CreationMetadata
{
ValueFormat = SerializationFormats.Json,
};
//Act
var withClause = CreateStatements.GenerateWithClause(metadata);
//Assert
withClause.Should().BeEquivalentTo(@$ " WITH ( VALUE_FORMAT='{metadata.ValueFormat}' )");
}
public void GenerateWithClause_ValueDelimiter()
{
//Arrange
var metadata = new CreationMetadata
{
ValueDelimiter = "SPACE"
};
//Act
var withClause = CreateStatements.GenerateWithClause(metadata);
//Assert
withClause.Should().BeEquivalentTo(@$ " WITH ( VALUE_DELIMITER='{metadata.ValueDelimiter}' )");
}
public void GenerateWithClause_Replicas()
{
//Arrange
var metadata = new CreationMetadata
{
Replicas = 3
};
//Act
var withClause = CreateStatements.GenerateWithClause(metadata);
//Assert
withClause.Should().BeEquivalentTo(@$ " WITH ( REPLICAS='{metadata.Replicas}' )");
}
public void GenerateWithClause_TimestampFormat()
{
//Arrange
var metadata = new CreationMetadata
{
TimestampFormat = "yyyy-MM-dd''T''HH:mm:ssX"
};
//Act
var withClause = CreateStatements.GenerateWithClause(metadata);
//Assert
withClause.Should().BeEquivalentTo(@$ " WITH ( TIMESTAMP_FORMAT='{metadata.TimestampFormat}' )");
}
public void GenerateWithClause_WrapSingleValue()
{
//Arrange
var metadata = new CreationMetadata
{
WrapSingleValue = true
};
//Act
var withClause = CreateStatements.GenerateWithClause(metadata);
//Assert
withClause.Should().BeEquivalentTo(@$ " WITH ( WRAP_SINGLE_VALUE='{metadata.WrapSingleValue}' )");
}
public void GenerateWithClause_KafkaTopic()
{
//Arrange
var metadata = new CreationMetadata
{
KafkaTopic = "tweetsByTitle"
};
//Act
var withClause = CreateStatements.GenerateWithClause(metadata);
//Assert
withClause.Should().BeEquivalentTo(@$ " WITH ( KAFKA_TOPIC='{metadata.KafkaTopic}' )");
}
public void GenerateWithClause_Timestamp()
{
//Arrange
var metadata = new CreationMetadata
{
Timestamp = "t2"
};
//Act
var withClause = CreateStatements.GenerateWithClause(metadata);
//Assert
withClause.Should().BeEquivalentTo(@$ " WITH ( TIMESTAMP='{metadata.Timestamp}' )");
}
public async Task CreateOrReplaceStreamStatement_ToStatementString_ComplexQueryWasGenerated()
{
//Arrange
var restApiClient = KSqlDbRestApiProvider.Create();
var statement = new KSqlDbStatement(StatementTemplates.DropStream(StreamName));
var response = await restApiClient.ExecuteStatementAsync(statement);
EntityCreationMetadata metadata = new()
{
EntityName = StreamEntityName,
KafkaTopic = nameof(Movie) + "Test2",
Partitions = 1,
Replicas = 1
};
var httpResponseMessage =
await restApiClient.CreateStreamAsync <Movie>(metadata, ifNotExists : false);
var creationMetadata = new CreationMetadata
{
KafkaTopic = "moviesByTitle",
KeyFormat = SerializationFormats.Json,
ValueFormat = SerializationFormats.Json,
Replicas = 1,
Partitions = 1
};
var createStatement = Context.CreateOrReplaceStreamStatement(StreamName)
.With(creationMetadata)
.As <Movie>(StreamEntityName)
.Where(c => c.Id < 3)
.Select(c => new { c.Id, c.Title, ReleaseYear = c.Release_Year })
.PartitionBy(c => c.Id);
//Act
var ksql = createStatement.ToStatementString();
httpResponseMessage = await createStatement.ExecuteStatementAsync();
//Assert
ksql.Should().BeEquivalentTo(@$ "CREATE OR REPLACE STREAM {StreamName}
WITH ( KAFKA_TOPIC='moviesByTitle', KEY_FORMAT='Json', VALUE_FORMAT='Json', PARTITIONS='1', REPLICAS='1' )
AS SELECT Id, Title, Release_Year AS ReleaseYear FROM {StreamEntityName}
WHERE Id < 3 PARTITION BY Id EMIT CHANGES;");
var responses = await httpResponseMessage.ToStatementResponsesAsync();
responses[0].CommandStatus.Status.Should().BeOneOf("SUCCESS", "EXECUTING");
}
public void GenerateWithClause_Partitions()
{
//Arrange
var metadata = new CreationMetadata
{
Partitions = 3
};
//Act
var withClause = CreateStatements.GenerateWithClause(metadata);
//Assert
withClause.Should().BeEquivalentTo(@$ " WITH ( PARTITIONS='{metadata.Partitions}' )");
}
public void GenerateWithClause_ValueSchemaId()
{
//Arrange
var metadata = new CreationMetadata
{
ValueSchemaId = 2
};
//Act
var withClause = CreateStatements.GenerateWithClause(metadata);
//Assert
withClause.Should().BeEquivalentTo(@$ " WITH ( VALUE_SCHEMA_ID={metadata.ValueSchemaId} )");
}
public void GenerateWithClause_MultipleValues_AreSeparated()
{
//Arrange
var metadata = new CreationMetadata
{
TimestampFormat = "yyyy-MM-dd''T''HH:mm:ssX",
WrapSingleValue = true
};
//Act
var withClause = CreateStatements.GenerateWithClause(metadata);
//Assert
withClause.Should().BeEquivalentTo(@$ " WITH ( TIMESTAMP_FORMAT='{metadata.TimestampFormat}', WRAP_SINGLE_VALUE='{metadata.WrapSingleValue}' )");
}
internal static string GenerateWithClause(CreationMetadata metadata, IList <string> properties = null)
{
if (metadata == null)
{
throw new ArgumentNullException(nameof(metadata));
}
properties ??= new List <string>();
if (metadata.KafkaTopic.IsNotNullOrEmpty())
{
properties.Add(@$ "KAFKA_TOPIC='{metadata.KafkaTopic}'");
}
if (metadata.KeyFormat.HasValue)
{
properties.Add(@$ "KEY_FORMAT='{metadata.KeyFormat}'");
}
if (metadata.ValueFormat.HasValue)
{
properties.Add(@$ "VALUE_FORMAT='{metadata.ValueFormat}'");
}
if (metadata.ValueDelimiter.IsNotNullOrEmpty())
{
properties.Add(@$ "VALUE_DELIMITER='{metadata.ValueDelimiter}'");
}
if (metadata.Partitions.HasValue)
{
properties.Add(@$ "PARTITIONS='{metadata.Partitions}'");
}
if (metadata.Replicas.HasValue)
{
properties.Add(@$ "REPLICAS='{metadata.Replicas}'");
}
if (metadata.Timestamp.IsNotNullOrEmpty())
{
properties.Add(@$ "TIMESTAMP='{metadata.Timestamp}'");
}
if (metadata.TimestampFormat.IsNotNullOrEmpty())
{
properties.Add(@$ "TIMESTAMP_FORMAT='{metadata.TimestampFormat}'");
}
if (metadata.WrapSingleValue.HasValue)
{
properties.Add(@$ "WRAP_SINGLE_VALUE='{metadata.WrapSingleValue}'");
}
if (metadata.KeySchemaId.HasValue)
{
properties.Add(@$ "KEY_SCHEMA_ID={metadata.KeySchemaId}");
}
if (metadata.ValueSchemaId.HasValue)
{
properties.Add(@$ "VALUE_SCHEMA_ID={metadata.ValueSchemaId}");
}
if (metadata.KeySchemaFullName.IsNotNullOrEmpty())
{
properties.Add(@$ "KEY_SCHEMA_FULL_NAME={metadata.KeySchemaFullName}");
}
if (metadata.ValueSchemaFullName.IsNotNullOrEmpty())
{
properties.Add(@$ "VALUE_SCHEMA_FULL_NAME={metadata.ValueSchemaFullName}");
}
string result = string.Join(", ", properties);
if (!string.IsNullOrEmpty(result))
{
result = $" WITH ( {result} )";
}
return(result);
}
public static void SetCustomCollectionNameCreationSettings(CreationMetadata meta, Type targetType, string databaseName, CreateCollectionOptions options) => meta[MakeSettingsKey(databaseName, targetType)] = options;
public static ManagerBuilder AddMongoFromTypes(this ManagerBuilder builder, IComponentContext context, string database, CreationMetadata metadata, params Type[] types)
{
metadata[MongoDataBaseFactory.MongoDatabaseNameMeta] = database;
return(builder.AddTypes(context.Resolve <MongoDataBaseFactory>(), metadata, types));
}
public IAsClause With(CreationMetadata creationMetadata)
{
string withClause = CreateStatements.GenerateWithClause(creationMetadata);
statementContext.Statement = @$ "{statementContext.Statement}
{withClause}";
public static void SetCustomCollectionName(CreationMetadata meta, Type targetType, string databaseName, string name) => meta[MakeKey(databaseName, targetType)] = name;