public void TestSqlContextCreateDataFrame()
{
// arrange
var mockSparkContextProxy = new Mock <ISparkContextProxy>();
mockSparkContextProxy.Setup(m => m.CreateCSharpRdd(It.IsAny <IRDDProxy>(), It.IsAny <byte[]>(), It.IsAny <Dictionary <string, string> >(),
It.IsAny <List <string> >(), It.IsAny <bool>(), It.IsAny <List <Broadcast> >(), It.IsAny <List <byte[]> >()));
var rddProxy = new Mock <IRDDProxy>();
var rdd = new RDD <object[]>(rddProxy.Object, new SparkContext(mockSparkContextProxy.Object, new SparkConf()));
var dataFrameProxy = new DataFrameIpcProxy(new JvmObjectReference("1"), mockSqlContextProxy.Object);
mockSqlContextProxy.Setup(m => m.CreateDataFrame(It.IsAny <IRDDProxy>(), It.IsAny <IStructTypeProxy>())).Returns(dataFrameProxy);
var sqlContext = new SqlContext(new SparkContext("", ""), mockSqlContextProxy.Object);
var structTypeProxy = new Mock <IStructTypeProxy>();
const string schemaJson = @"{
""fields"": [{
""metadata"": {},
""name"": ""guid"",
""nullable"": false,
""type"": ""string""
}],
""type"": ""struct""
}";
structTypeProxy.Setup(m => m.ToJson()).Returns(schemaJson);
// act
var dataFrame = sqlContext.CreateDataFrame(rdd, new StructType(structTypeProxy.Object));
// assert
Assert.AreEqual(dataFrameProxy, dataFrame.DataFrameProxy);
}
public void TestSqlContextRegisterDataFrameAsTable()
{
// arrange
mockSqlContextProxy.Setup(m => m.RegisterDataFrameAsTable(It.IsAny <IDataFrameProxy>(), It.IsAny <string>()));
var sqlContext = new SqlContext(new SparkContext("", ""), mockSqlContextProxy.Object);
var dataFrameProxy = new DataFrameIpcProxy(new JvmObjectReference("1"), mockSqlContextProxy.Object);
var dataFrame = new DataFrame(dataFrameProxy, new SparkContext(new SparkConf()));
// act
sqlContext.RegisterDataFrameAsTable(dataFrame, "table");
// assert
mockSqlContextProxy.Verify(m => m.RegisterDataFrameAsTable(dataFrameProxy, "table"));
}
public void TestSqlContextTable()
{
// arrange
var sqlContext = new SqlContext(new SparkContext("", ""), mockSqlContextProxy.Object);
var dataFrameProxy = new DataFrameIpcProxy(new JvmObjectReference("1"), mockSqlContextProxy.Object);
mockSqlContextProxy.Setup(m => m.Table(It.IsAny <string>())).Returns(dataFrameProxy);
// act
var actualTableDataFrame = sqlContext.Table("table");
// assert
Assert.AreEqual(dataFrameProxy, actualTableDataFrame.DataFrameProxy);
}
public void TestSqlContextReadDataFrame()
{
var dataFrameProxy = new DataFrameIpcProxy(new JvmObjectReference("1"), mockSqlContextProxy.Object);
mockSqlContextProxy.Setup(
m => m.ReadDataFrame(It.IsAny <string>(), It.IsAny <StructType>(), It.IsAny <Dictionary <string, string> >()))
.Returns(dataFrameProxy);
var sqlContext = new SqlContext(new SparkContext("", ""), mockSqlContextProxy.Object);
var structTypeProxy = new Mock <IStructTypeProxy>();
structTypeProxy.Setup(m => m.ToJson()).Returns(RowHelper.BasicJsonSchema);
// act
var dataFrame = sqlContext.ReadDataFrame(@"c:\path\to\input.txt", new StructType(structTypeProxy.Object), null);
// assert
Assert.AreEqual(dataFrameProxy, dataFrame.DataFrameProxy);
}
public void TestSqlContextCreateDataFrame()
{
// arrange
var mockSparkContextProxy = new Mock <ISparkContextProxy>();
mockSparkContextProxy.Setup(m => m.CreateCSharpRdd(It.IsAny <IRDDProxy>(), It.IsAny <byte[]>(), It.IsAny <Dictionary <string, string> >(),
It.IsAny <List <string> >(), It.IsAny <bool>(), It.IsAny <List <Broadcast> >(), It.IsAny <List <byte[]> >()));
var rddProxy = new Mock <IRDDProxy>();
var rdd = new RDD <object[]>(rddProxy.Object, new SparkContext(mockSparkContextProxy.Object, new SparkConf()));
var dataFrameProxy = new DataFrameIpcProxy(new JvmObjectReference("1"), mockSqlContextProxy.Object);
mockSqlContextProxy.Setup(m => m.CreateDataFrame(It.IsAny <IRDDProxy>(), It.IsAny <IStructTypeProxy>())).Returns(dataFrameProxy);
var sqlContext = new SqlContext(new SparkContext("", ""), mockSqlContextProxy.Object);
var structTypeProxy = new Mock <IStructTypeProxy>();
structTypeProxy.Setup(m => m.ToJson()).Returns(RowHelper.ComplexJsonSchema);
// act
var dataFrame = sqlContext.CreateDataFrame(rdd, new StructType(structTypeProxy.Object));
// assert
Assert.AreEqual(dataFrameProxy, dataFrame.DataFrameProxy);
}
