예제 #1
0
        public void TestSignatures()
        {
            using var tempDirectory = new TemporaryDirectory();
            string path = Path.Combine(tempDirectory.Path, "delta-table");

            DataFrame rangeRate = _spark.Range(15);

            rangeRate.Write().Format("delta").Save(path);

            DeltaTable table = Assert.IsType <DeltaTable>(DeltaTable.ForPath(path));

            table = Assert.IsType <DeltaTable>(DeltaTable.ForPath(_spark, path));

            Assert.IsType <bool>(DeltaTable.IsDeltaTable(_spark, path));
            Assert.IsType <bool>(DeltaTable.IsDeltaTable(path));

            Assert.IsType <DeltaTable>(table.As("oldTable"));
            Assert.IsType <DeltaTable>(table.Alias("oldTable"));
            Assert.IsType <DataFrame>(table.History());
            Assert.IsType <DataFrame>(table.History(200));
            Assert.IsType <DataFrame>(table.ToDF());

            DataFrame newTable = _spark.Range(10, 15).As("newTable");

            Assert.IsType <DeltaMergeBuilder>(
                table.Merge(newTable, Functions.Exp("oldTable.id == newTable.id")));
            DeltaMergeBuilder mergeBuilder = Assert.IsType <DeltaMergeBuilder>(
                table.Merge(newTable, "oldTable.id == newTable.id"));

            // Validate the MergeBuilder matched signatures.
            Assert.IsType <DeltaMergeMatchedActionBuilder>(mergeBuilder.WhenMatched());
            Assert.IsType <DeltaMergeMatchedActionBuilder>(mergeBuilder.WhenMatched("id = 5"));
            DeltaMergeMatchedActionBuilder matchedActionBuilder =
                Assert.IsType <DeltaMergeMatchedActionBuilder>(
                    mergeBuilder.WhenMatched(Functions.Expr("id = 5")));

            Assert.IsType <DeltaMergeBuilder>(
                matchedActionBuilder.Update(new Dictionary <string, Column>()));
            Assert.IsType <DeltaMergeBuilder>(
                matchedActionBuilder.UpdateExpr(new Dictionary <string, string>()));
            Assert.IsType <DeltaMergeBuilder>(matchedActionBuilder.UpdateAll());
            Assert.IsType <DeltaMergeBuilder>(matchedActionBuilder.Delete());

            // Validate the MergeBuilder not-matched signatures.
            Assert.IsType <DeltaMergeNotMatchedActionBuilder>(mergeBuilder.WhenNotMatched());
            Assert.IsType <DeltaMergeNotMatchedActionBuilder>(
                mergeBuilder.WhenNotMatched("id = 5"));
            DeltaMergeNotMatchedActionBuilder notMatchedActionBuilder =
                Assert.IsType <DeltaMergeNotMatchedActionBuilder>(
                    mergeBuilder.WhenNotMatched(Functions.Expr("id = 5")));

            Assert.IsType <DeltaMergeBuilder>(
                notMatchedActionBuilder.Insert(new Dictionary <string, Column>()));
            Assert.IsType <DeltaMergeBuilder>(
                notMatchedActionBuilder.InsertExpr(new Dictionary <string, string>()));
            Assert.IsType <DeltaMergeBuilder>(notMatchedActionBuilder.InsertAll());

            // Update and UpdateExpr should return void.
            table.Update(new Dictionary <string, Column>()
            {
            });
            table.Update(Functions.Expr("id % 2 == 0"), new Dictionary <string, Column>()
            {
            });
            table.UpdateExpr(new Dictionary <string, string>()
            {
            });
            table.UpdateExpr("id % 2 == 1", new Dictionary <string, string>()
            {
            });

            Assert.IsType <DataFrame>(table.Vacuum());
            Assert.IsType <DataFrame>(table.Vacuum(168));

            // Generate should return void.
            table.Generate("symlink_format_manifest");

            // Delete should return void.
            table.Delete("id > 10");
            table.Delete(Functions.Expr("id > 5"));
            table.Delete();

            // Load the table as a streaming source.
            Assert.IsType <DataFrame>(_spark
                                      .ReadStream()
                                      .Format("delta")
                                      .Option("path", path)
                                      .Load());
            Assert.IsType <DataFrame>(_spark.ReadStream().Format("delta").Load(path));

            // Create Parquet data and convert it to DeltaTables.
            string parquetIdentifier = $"parquet.`{path}`";

            rangeRate.Write().Mode(SaveMode.Overwrite).Parquet(path);
            Assert.IsType <DeltaTable>(DeltaTable.ConvertToDelta(_spark, parquetIdentifier));
            rangeRate
            .Select(Functions.Col("id"), Functions.Expr($"(`id` + 1) AS `id_plus_one`"))
            .Write()
            .PartitionBy("id")
            .Mode(SaveMode.Overwrite)
            .Parquet(path);
            Assert.IsType <DeltaTable>(DeltaTable.ConvertToDelta(
                                           _spark,
                                           parquetIdentifier,
                                           "id bigint"));
            Assert.IsType <DeltaTable>(DeltaTable.ConvertToDelta(
                                           _spark,
                                           parquetIdentifier,
                                           new StructType(new[]
            {
                new StructField("id", new IntegerType())
            })));
        }
예제 #2
0
        public void TestSignatures()
        {
            using (var tempDirectory = new TemporaryDirectory())
            {
                string path = Path.Combine(tempDirectory.Path, "delta-table");

                DataFrame rangeRate = _spark.Range(15);
                rangeRate.Write().Format("delta").Save(path);

                DeltaTable table = Assert.IsType <DeltaTable>(DeltaTable.ForPath(path));
                table = Assert.IsType <DeltaTable>(DeltaTable.ForPath(_spark, path));

                Assert.IsType <DeltaTable>(table.As("oldTable"));
                Assert.IsType <DataFrame>(table.History());
                Assert.IsType <DataFrame>(table.History(200));
                Assert.IsType <DataFrame>(table.ToDF());

                DataFrame newTable = _spark.Range(10, 15).As("newTable");
                Assert.IsType <DeltaMergeBuilder>(
                    table.Merge(newTable, Functions.Exp("oldTable.id == newTable.id")));
                DeltaMergeBuilder mergeBuilder = Assert.IsType <DeltaMergeBuilder>(
                    table.Merge(newTable, "oldTable.id == newTable.id"));

                // Validate the MergeBuilder matched signatures.
                Assert.IsType <DeltaMergeMatchedActionBuilder>(mergeBuilder.WhenMatched());
                Assert.IsType <DeltaMergeMatchedActionBuilder>(mergeBuilder.WhenMatched("id = 5"));
                DeltaMergeMatchedActionBuilder matchedActionBuilder =
                    Assert.IsType <DeltaMergeMatchedActionBuilder>(
                        mergeBuilder.WhenMatched(Functions.Expr("id = 5")));

                Assert.IsType <DeltaMergeBuilder>(
                    matchedActionBuilder.Update(new Dictionary <string, Column>()));
                Assert.IsType <DeltaMergeBuilder>(
                    matchedActionBuilder.UpdateExpr(new Dictionary <string, string>()));
                Assert.IsType <DeltaMergeBuilder>(matchedActionBuilder.UpdateAll());
                Assert.IsType <DeltaMergeBuilder>(matchedActionBuilder.Delete());

                // Validate the MergeBuilder not-matched signatures.
                Assert.IsType <DeltaMergeNotMatchedActionBuilder>(mergeBuilder.WhenNotMatched());
                Assert.IsType <DeltaMergeNotMatchedActionBuilder>(
                    mergeBuilder.WhenNotMatched("id = 5"));
                DeltaMergeNotMatchedActionBuilder notMatchedActionBuilder =
                    Assert.IsType <DeltaMergeNotMatchedActionBuilder>(
                        mergeBuilder.WhenNotMatched(Functions.Expr("id = 5")));

                Assert.IsType <DeltaMergeBuilder>(
                    notMatchedActionBuilder.Insert(new Dictionary <string, Column>()));
                Assert.IsType <DeltaMergeBuilder>(
                    notMatchedActionBuilder.InsertExpr(new Dictionary <string, string>()));
                Assert.IsType <DeltaMergeBuilder>(notMatchedActionBuilder.InsertAll());

                // Update and UpdateExpr should return void.
                table.Update(new Dictionary <string, Column>()
                {
                });
                table.Update(Functions.Expr("id % 2 == 0"), new Dictionary <string, Column>()
                {
                });
                table.UpdateExpr(new Dictionary <string, string>()
                {
                });
                table.UpdateExpr("id % 2 == 1", new Dictionary <string, string>()
                {
                });

                Assert.IsType <DataFrame>(table.Vacuum());
                Assert.IsType <DataFrame>(table.Vacuum(168));

                // Delete should return void.
                table.Delete("id > 10");
                table.Delete(Functions.Expr("id > 5"));
                table.Delete();
            }
        }