public void TestSignatures()
{
using var tempDirectory = new TemporaryDirectory();
string path = Path.Combine(tempDirectory.Path, "delta-table");
DataFrame rangeRate = _spark.Range(15);
rangeRate.Write().Format("delta").Save(path);
DeltaTable table = Assert.IsType <DeltaTable>(DeltaTable.ForPath(path));
table = Assert.IsType <DeltaTable>(DeltaTable.ForPath(_spark, path));
Assert.IsType <bool>(DeltaTable.IsDeltaTable(_spark, path));
Assert.IsType <bool>(DeltaTable.IsDeltaTable(path));
Assert.IsType <DeltaTable>(table.As("oldTable"));
Assert.IsType <DeltaTable>(table.Alias("oldTable"));
Assert.IsType <DataFrame>(table.History());
Assert.IsType <DataFrame>(table.History(200));
Assert.IsType <DataFrame>(table.ToDF());
DataFrame newTable = _spark.Range(10, 15).As("newTable");
Assert.IsType <DeltaMergeBuilder>(
table.Merge(newTable, Functions.Exp("oldTable.id == newTable.id")));
DeltaMergeBuilder mergeBuilder = Assert.IsType <DeltaMergeBuilder>(
table.Merge(newTable, "oldTable.id == newTable.id"));
// Validate the MergeBuilder matched signatures.
Assert.IsType <DeltaMergeMatchedActionBuilder>(mergeBuilder.WhenMatched());
Assert.IsType <DeltaMergeMatchedActionBuilder>(mergeBuilder.WhenMatched("id = 5"));
DeltaMergeMatchedActionBuilder matchedActionBuilder =
Assert.IsType <DeltaMergeMatchedActionBuilder>(
mergeBuilder.WhenMatched(Functions.Expr("id = 5")));
Assert.IsType <DeltaMergeBuilder>(
matchedActionBuilder.Update(new Dictionary <string, Column>()));
Assert.IsType <DeltaMergeBuilder>(
matchedActionBuilder.UpdateExpr(new Dictionary <string, string>()));
Assert.IsType <DeltaMergeBuilder>(matchedActionBuilder.UpdateAll());
Assert.IsType <DeltaMergeBuilder>(matchedActionBuilder.Delete());
// Validate the MergeBuilder not-matched signatures.
Assert.IsType <DeltaMergeNotMatchedActionBuilder>(mergeBuilder.WhenNotMatched());
Assert.IsType <DeltaMergeNotMatchedActionBuilder>(
mergeBuilder.WhenNotMatched("id = 5"));
DeltaMergeNotMatchedActionBuilder notMatchedActionBuilder =
Assert.IsType <DeltaMergeNotMatchedActionBuilder>(
mergeBuilder.WhenNotMatched(Functions.Expr("id = 5")));
Assert.IsType <DeltaMergeBuilder>(
notMatchedActionBuilder.Insert(new Dictionary <string, Column>()));
Assert.IsType <DeltaMergeBuilder>(
notMatchedActionBuilder.InsertExpr(new Dictionary <string, string>()));
Assert.IsType <DeltaMergeBuilder>(notMatchedActionBuilder.InsertAll());
// Update and UpdateExpr should return void.
table.Update(new Dictionary <string, Column>()
{
});
table.Update(Functions.Expr("id % 2 == 0"), new Dictionary <string, Column>()
{
});
table.UpdateExpr(new Dictionary <string, string>()
{
});
table.UpdateExpr("id % 2 == 1", new Dictionary <string, string>()
{
});
Assert.IsType <DataFrame>(table.Vacuum());
Assert.IsType <DataFrame>(table.Vacuum(168));
// Generate should return void.
table.Generate("symlink_format_manifest");
// Delete should return void.
table.Delete("id > 10");
table.Delete(Functions.Expr("id > 5"));
table.Delete();
// Load the table as a streaming source.
Assert.IsType <DataFrame>(_spark
.ReadStream()
.Format("delta")
.Option("path", path)
.Load());
Assert.IsType <DataFrame>(_spark.ReadStream().Format("delta").Load(path));
// Create Parquet data and convert it to DeltaTables.
string parquetIdentifier = $"parquet.`{path}`";
rangeRate.Write().Mode(SaveMode.Overwrite).Parquet(path);
Assert.IsType <DeltaTable>(DeltaTable.ConvertToDelta(_spark, parquetIdentifier));
rangeRate
.Select(Functions.Col("id"), Functions.Expr($"(`id` + 1) AS `id_plus_one`"))
.Write()
.PartitionBy("id")
.Mode(SaveMode.Overwrite)
.Parquet(path);
Assert.IsType <DeltaTable>(DeltaTable.ConvertToDelta(
_spark,
parquetIdentifier,
"id bigint"));
Assert.IsType <DeltaTable>(DeltaTable.ConvertToDelta(
_spark,
parquetIdentifier,
new StructType(new[]
{
new StructField("id", new IntegerType())
})));
}
public void TestSignatures()
{
using (var tempDirectory = new TemporaryDirectory())
{
string path = Path.Combine(tempDirectory.Path, "delta-table");
DataFrame rangeRate = _spark.Range(15);
rangeRate.Write().Format("delta").Save(path);
DeltaTable table = Assert.IsType <DeltaTable>(DeltaTable.ForPath(path));
table = Assert.IsType <DeltaTable>(DeltaTable.ForPath(_spark, path));
Assert.IsType <DeltaTable>(table.As("oldTable"));
Assert.IsType <DataFrame>(table.History());
Assert.IsType <DataFrame>(table.History(200));
Assert.IsType <DataFrame>(table.ToDF());
DataFrame newTable = _spark.Range(10, 15).As("newTable");
Assert.IsType <DeltaMergeBuilder>(
table.Merge(newTable, Functions.Exp("oldTable.id == newTable.id")));
DeltaMergeBuilder mergeBuilder = Assert.IsType <DeltaMergeBuilder>(
table.Merge(newTable, "oldTable.id == newTable.id"));
// Validate the MergeBuilder matched signatures.
Assert.IsType <DeltaMergeMatchedActionBuilder>(mergeBuilder.WhenMatched());
Assert.IsType <DeltaMergeMatchedActionBuilder>(mergeBuilder.WhenMatched("id = 5"));
DeltaMergeMatchedActionBuilder matchedActionBuilder =
Assert.IsType <DeltaMergeMatchedActionBuilder>(
mergeBuilder.WhenMatched(Functions.Expr("id = 5")));
Assert.IsType <DeltaMergeBuilder>(
matchedActionBuilder.Update(new Dictionary <string, Column>()));
Assert.IsType <DeltaMergeBuilder>(
matchedActionBuilder.UpdateExpr(new Dictionary <string, string>()));
Assert.IsType <DeltaMergeBuilder>(matchedActionBuilder.UpdateAll());
Assert.IsType <DeltaMergeBuilder>(matchedActionBuilder.Delete());
// Validate the MergeBuilder not-matched signatures.
Assert.IsType <DeltaMergeNotMatchedActionBuilder>(mergeBuilder.WhenNotMatched());
Assert.IsType <DeltaMergeNotMatchedActionBuilder>(
mergeBuilder.WhenNotMatched("id = 5"));
DeltaMergeNotMatchedActionBuilder notMatchedActionBuilder =
Assert.IsType <DeltaMergeNotMatchedActionBuilder>(
mergeBuilder.WhenNotMatched(Functions.Expr("id = 5")));
Assert.IsType <DeltaMergeBuilder>(
notMatchedActionBuilder.Insert(new Dictionary <string, Column>()));
Assert.IsType <DeltaMergeBuilder>(
notMatchedActionBuilder.InsertExpr(new Dictionary <string, string>()));
Assert.IsType <DeltaMergeBuilder>(notMatchedActionBuilder.InsertAll());
// Update and UpdateExpr should return void.
table.Update(new Dictionary <string, Column>()
{
});
table.Update(Functions.Expr("id % 2 == 0"), new Dictionary <string, Column>()
{
});
table.UpdateExpr(new Dictionary <string, string>()
{
});
table.UpdateExpr("id % 2 == 1", new Dictionary <string, string>()
{
});
Assert.IsType <DataFrame>(table.Vacuum());
Assert.IsType <DataFrame>(table.Vacuum(168));
// Delete should return void.
table.Delete("id > 10");
table.Delete(Functions.Expr("id > 5"));
table.Delete();
}
}