public void TestSignaturesV2_3_X()
{
DataFrameReader dfr = _spark.Read();
Assert.IsType <DataFrameReader>(dfr.Format("json"));
Assert.IsType <DataFrameReader>(
dfr.Schema(
new StructType(new[]
{
new StructField("age", new IntegerType()),
new StructField("name", new StringType())
})));
Assert.IsType <DataFrameReader>(dfr.Schema("age INT, name STRING"));
Assert.IsType <DataFrameReader>(dfr.Option("stringOption", "value"));
Assert.IsType <DataFrameReader>(dfr.Option("boolOption", true));
Assert.IsType <DataFrameReader>(dfr.Option("longOption", 1L));
Assert.IsType <DataFrameReader>(dfr.Option("doubleOption", 3D));
Assert.IsType <DataFrameReader>(
dfr.Options(
new Dictionary <string, string>
{
{ "option1", "value1" },
{ "option2", "value2" }
}));
string jsonFile = $"{TestEnvironment.ResourceDirectory}people.json";
Assert.IsType <DataFrame>(dfr.Load());
Assert.IsType <DataFrame>(dfr.Load(jsonFile));
Assert.IsType <DataFrame>(dfr.Load(jsonFile, jsonFile));
Assert.IsType <DataFrame>(dfr.Json(jsonFile));
Assert.IsType <DataFrame>(dfr.Json(jsonFile, jsonFile));
string csvFile = $"{TestEnvironment.ResourceDirectory}people.csv";
Assert.IsType <DataFrame>(dfr.Csv(csvFile));
Assert.IsType <DataFrame>(dfr.Csv(csvFile, csvFile));
string parquetFile = $"{TestEnvironment.ResourceDirectory}users.parquet";
Assert.IsType <DataFrame>(dfr.Parquet(parquetFile));
Assert.IsType <DataFrame>(dfr.Parquet(parquetFile, parquetFile));
string orcFile = $"{TestEnvironment.ResourceDirectory}users.orc";
Assert.IsType <DataFrame>(dfr.Orc(orcFile));
Assert.IsType <DataFrame>(dfr.Orc(orcFile, orcFile));
dfr = _spark.Read();
string textFile = $"{TestEnvironment.ResourceDirectory}people.txt";
Assert.IsType <DataFrame>(dfr.Text(textFile));
Assert.IsType <DataFrame>(dfr.Text(textFile, textFile));
}
static void Main(string[] args)
{
var spark = SparkSession.Builder().GetOrCreate();
DataFrameReader reader =
spark.Read().Format("csv").Option("header", true).Option("sep", ",");
var dataFrame = reader.Load("./csv_file.csv");
dataFrame.Show();
}
public void TestLoad()
{
// arrange
mockDataFrameReaderProxy.Setup(m => m.Load());
var dataFrameReader = new DataFrameReader(mockDataFrameReaderProxy.Object, sparkContext);
// Act
dataFrameReader.Load();
// Assert
mockDataFrameReaderProxy.Verify(m => m.Load(), Times.Once);
}
private static void LoginSql()
{
string query = "select * from tblUser";
var spark = SparkSession.Builder().GetOrCreate();
DataFrameReader dataFrameReader = spark.Read();
dataFrameReader = dataFrameReader.Format("jdbc");
dataFrameReader = dataFrameReader.Option("Driver", "com.microsoft.sqlserver.jdbc.SQLServerDriver");
dataFrameReader = dataFrameReader.Option("url", "jdbc:sqlserver://*****:*****@@Error15;");
dataFrameReader = dataFrameReader.Option("dbtable", query);
DataFrame dataFrame = dataFrameReader.Load();
dataFrame.Show();
spark.Stop();
}
public void TestLoadWithPath()
{
// arrange
mockDataFrameReaderProxy.Setup(m => m.Load());
mockDataFrameReaderProxy.Setup(m => m.Options(It.IsAny <Dictionary <string, string> >()));
var dataFrameReader = new DataFrameReader(mockDataFrameReaderProxy.Object, sparkContext);
const string key = "path";
const string path = "path_value";
// Act
dataFrameReader.Load(path);
// Assert
mockDataFrameReaderProxy.Verify(m => m.Options(
It.Is <Dictionary <string, string> >(dict => dict[key] == path && dict.Count == 1)), Times.Once);
mockDataFrameReaderProxy.Verify(m => m.Load(), Times.Once);
}