コード例 #1
0
        public void TestSignaturesV2_4_X()
        {
            DataStreamReader dsr = _spark.ReadStream();

            Assert.IsType <DataStreamReader>(dsr.Format("parquet"));

            Assert.IsType <DataStreamReader>(
                dsr.Schema(
                    new StructType(new[]
            {
                new StructField("columnName", new IntegerType())
            })));
            Assert.IsType <DataStreamReader>(dsr.Schema("columnName bigint"));

            Assert.IsType <DataStreamReader>(dsr.Option("key", "value"));
            Assert.IsType <DataStreamReader>(dsr.Option("key", true));
            Assert.IsType <DataStreamReader>(dsr.Option("key", long.MaxValue));
            Assert.IsType <DataStreamReader>(dsr.Option("key", double.MaxValue));
            Assert.IsType <DataStreamReader>(dsr.Options(new Dictionary <string, string>()));
            Assert.IsType <DataStreamReader>(
                dsr.Options(
                    new Dictionary <string, string>
            {
                { "key", "value" }
            }));

            string jsonFilePath = Path.Combine(TestEnvironment.ResourceDirectory, "people.json");

            Assert.IsType <DataFrame>(dsr.Format("json").Load(jsonFilePath));
            Assert.IsType <DataFrame>(dsr.Json(jsonFilePath));
            Assert.IsType <DataFrame>(
                dsr.Csv(Path.Combine(TestEnvironment.ResourceDirectory, "people.csv")));
            Assert.IsType <DataFrame>(
                dsr.Orc(Path.Combine(TestEnvironment.ResourceDirectory, "users.orc")));
            Assert.IsType <DataFrame>(
                dsr.Parquet(Path.Combine(TestEnvironment.ResourceDirectory, "users.parquet")));
            Assert.IsType <DataFrame>
                (dsr.Text(Path.Combine(TestEnvironment.ResourceDirectory, "people.txt")));

            // In Spark 3.1.1+ setting the `path` Option and then calling .Load(path) is not
            // supported unless `spark.sql.legacy.pathOptionBehavior.enabled` conf is set.
            // .Json(path), .Parquet(path), etc follow the same code path so the conf
            // needs to be set in these scenarios as well.
            Assert.IsType <DataFrame>(dsr.Format("json").Option("path", jsonFilePath).Load());
        }
コード例 #2
0
        public void TestSignaturesV2_3_X()
        {
            DataStreamReader dsr = _spark.ReadStream();

            Assert.IsType <DataStreamReader>(dsr.Format("parquet"));

            Assert.IsType <DataStreamReader>(
                dsr.Schema(
                    new StructType(new[]
            {
                new StructField("columnName", new IntegerType())
            })));
            Assert.IsType <DataStreamReader>(dsr.Schema("columnName bigint"));

            Assert.IsType <DataStreamReader>(dsr.Option("key", "value"));
            Assert.IsType <DataStreamReader>(dsr.Option("key", true));
            Assert.IsType <DataStreamReader>(dsr.Option("key", long.MaxValue));
            Assert.IsType <DataStreamReader>(dsr.Option("key", double.MaxValue));
            Assert.IsType <DataStreamReader>(dsr.Options(new Dictionary <string, string>()));
            Assert.IsType <DataStreamReader>(
                dsr.Options(
                    new Dictionary <string, string>
            {
                { "key", "value" }
            }));

            string jsonFilePath = Path.Combine(TestEnvironment.ResourceDirectory, "people.json");

            Assert.IsType <DataFrame>(dsr.Format("json").Option("path", jsonFilePath).Load());
            Assert.IsType <DataFrame>(dsr.Format("json").Load(jsonFilePath));
            Assert.IsType <DataFrame>(dsr.Json(jsonFilePath));
            Assert.IsType <DataFrame>(
                dsr.Csv(Path.Combine(TestEnvironment.ResourceDirectory, "people.csv")));
            Assert.IsType <DataFrame>(
                dsr.Orc(Path.Combine(TestEnvironment.ResourceDirectory, "users.orc")));
            Assert.IsType <DataFrame>(
                dsr.Parquet(Path.Combine(TestEnvironment.ResourceDirectory, "users.parquet")));
            Assert.IsType <DataFrame>
                (dsr.Text(Path.Combine(TestEnvironment.ResourceDirectory, "people.txt")));
        }