public void TestSignaturesV2_4_X()
{
DataStreamReader dsr = _spark.ReadStream();
Assert.IsType <DataStreamReader>(dsr.Format("parquet"));
Assert.IsType <DataStreamReader>(
dsr.Schema(
new StructType(new[]
{
new StructField("columnName", new IntegerType())
})));
Assert.IsType <DataStreamReader>(dsr.Schema("columnName bigint"));
Assert.IsType <DataStreamReader>(dsr.Option("key", "value"));
Assert.IsType <DataStreamReader>(dsr.Option("key", true));
Assert.IsType <DataStreamReader>(dsr.Option("key", long.MaxValue));
Assert.IsType <DataStreamReader>(dsr.Option("key", double.MaxValue));
Assert.IsType <DataStreamReader>(dsr.Options(new Dictionary <string, string>()));
Assert.IsType <DataStreamReader>(
dsr.Options(
new Dictionary <string, string>
{
{ "key", "value" }
}));
string jsonFilePath = Path.Combine(TestEnvironment.ResourceDirectory, "people.json");
Assert.IsType <DataFrame>(dsr.Format("json").Load(jsonFilePath));
Assert.IsType <DataFrame>(dsr.Json(jsonFilePath));
Assert.IsType <DataFrame>(
dsr.Csv(Path.Combine(TestEnvironment.ResourceDirectory, "people.csv")));
Assert.IsType <DataFrame>(
dsr.Orc(Path.Combine(TestEnvironment.ResourceDirectory, "users.orc")));
Assert.IsType <DataFrame>(
dsr.Parquet(Path.Combine(TestEnvironment.ResourceDirectory, "users.parquet")));
Assert.IsType <DataFrame>
(dsr.Text(Path.Combine(TestEnvironment.ResourceDirectory, "people.txt")));
// In Spark 3.1.1+ setting the `path` Option and then calling .Load(path) is not
// supported unless `spark.sql.legacy.pathOptionBehavior.enabled` conf is set.
// .Json(path), .Parquet(path), etc follow the same code path so the conf
// needs to be set in these scenarios as well.
Assert.IsType <DataFrame>(dsr.Format("json").Option("path", jsonFilePath).Load());
}
public void TestSignaturesV2_3_X()
{
DataStreamReader dsr = _spark.ReadStream();
Assert.IsType <DataStreamReader>(dsr.Format("parquet"));
Assert.IsType <DataStreamReader>(
dsr.Schema(
new StructType(new[]
{
new StructField("columnName", new IntegerType())
})));
Assert.IsType <DataStreamReader>(dsr.Schema("columnName bigint"));
Assert.IsType <DataStreamReader>(dsr.Option("key", "value"));
Assert.IsType <DataStreamReader>(dsr.Option("key", true));
Assert.IsType <DataStreamReader>(dsr.Option("key", long.MaxValue));
Assert.IsType <DataStreamReader>(dsr.Option("key", double.MaxValue));
Assert.IsType <DataStreamReader>(dsr.Options(new Dictionary <string, string>()));
Assert.IsType <DataStreamReader>(
dsr.Options(
new Dictionary <string, string>
{
{ "key", "value" }
}));
string jsonFilePath = Path.Combine(TestEnvironment.ResourceDirectory, "people.json");
Assert.IsType <DataFrame>(dsr.Format("json").Option("path", jsonFilePath).Load());
Assert.IsType <DataFrame>(dsr.Format("json").Load(jsonFilePath));
Assert.IsType <DataFrame>(dsr.Json(jsonFilePath));
Assert.IsType <DataFrame>(
dsr.Csv(Path.Combine(TestEnvironment.ResourceDirectory, "people.csv")));
Assert.IsType <DataFrame>(
dsr.Orc(Path.Combine(TestEnvironment.ResourceDirectory, "users.orc")));
Assert.IsType <DataFrame>(
dsr.Parquet(Path.Combine(TestEnvironment.ResourceDirectory, "users.parquet")));
Assert.IsType <DataFrame>
(dsr.Text(Path.Combine(TestEnvironment.ResourceDirectory, "people.txt")));
}
