public void TestSignaturesV2_3_X()
{
DataFrameReader dfr = _spark.Read();
Assert.IsType <DataFrameReader>(dfr.Format("json"));
Assert.IsType <DataFrameReader>(
dfr.Schema(
new StructType(new[]
{
new StructField("age", new IntegerType()),
new StructField("name", new StringType())
})));
Assert.IsType <DataFrameReader>(dfr.Schema("age INT, name STRING"));
Assert.IsType <DataFrameReader>(dfr.Option("stringOption", "value"));
Assert.IsType <DataFrameReader>(dfr.Option("boolOption", true));
Assert.IsType <DataFrameReader>(dfr.Option("longOption", 1L));
Assert.IsType <DataFrameReader>(dfr.Option("doubleOption", 3D));
Assert.IsType <DataFrameReader>(
dfr.Options(
new Dictionary <string, string>
{
{ "option1", "value1" },
{ "option2", "value2" }
}));
string jsonFile = $"{TestEnvironment.ResourceDirectory}people.json";
Assert.IsType <DataFrame>(dfr.Load());
Assert.IsType <DataFrame>(dfr.Load(jsonFile));
Assert.IsType <DataFrame>(dfr.Load(jsonFile, jsonFile));
Assert.IsType <DataFrame>(dfr.Json(jsonFile));
Assert.IsType <DataFrame>(dfr.Json(jsonFile, jsonFile));
string csvFile = $"{TestEnvironment.ResourceDirectory}people.csv";
Assert.IsType <DataFrame>(dfr.Csv(csvFile));
Assert.IsType <DataFrame>(dfr.Csv(csvFile, csvFile));
string parquetFile = $"{TestEnvironment.ResourceDirectory}users.parquet";
Assert.IsType <DataFrame>(dfr.Parquet(parquetFile));
Assert.IsType <DataFrame>(dfr.Parquet(parquetFile, parquetFile));
string orcFile = $"{TestEnvironment.ResourceDirectory}users.orc";
Assert.IsType <DataFrame>(dfr.Orc(orcFile));
Assert.IsType <DataFrame>(dfr.Orc(orcFile, orcFile));
dfr = _spark.Read();
string textFile = $"{TestEnvironment.ResourceDirectory}people.txt";
Assert.IsType <DataFrame>(dfr.Text(textFile));
Assert.IsType <DataFrame>(dfr.Text(textFile, textFile));
}
private static void LoginSql()
{
string query = "select * from tblUser";
var spark = SparkSession.Builder().GetOrCreate();
DataFrameReader dataFrameReader = spark.Read();
dataFrameReader = dataFrameReader.Format("jdbc");
dataFrameReader = dataFrameReader.Option("Driver", "com.microsoft.sqlserver.jdbc.SQLServerDriver");
dataFrameReader = dataFrameReader.Option("url", "jdbc:sqlserver://*****:*****@@Error15;");
dataFrameReader = dataFrameReader.Option("dbtable", query);
DataFrame dataFrame = dataFrameReader.Load();
dataFrame.Show();
spark.Stop();
}
public void TestOption()
{
// arrange
mockDataFrameReaderProxy.Setup(m => m.Options(It.IsAny <Dictionary <string, string> >()));
var dataFrameReader = new DataFrameReader(mockDataFrameReaderProxy.Object, sparkContext);
const string key = "path";
const string value = "path_value";
// Act
dataFrameReader.Option(key, value);
// Assert
mockDataFrameReaderProxy.Verify(m => m.Options(
It.Is <Dictionary <string, string> >(dict => dict[key] == value && dict.Count == 1)), Times.Once);
}
/// <summary>
/// Set up a reader with the options for a header.
/// </summary>
/// <param name="reader">The <see cref="DataFrameReader"/>.</param>
/// <returns>The <see cref="DataFrameReader"/> with configured option.</returns>
public static DataFrameReader HasHeader(this DataFrameReader reader)
{
return(reader.Option("header", true));
}