public void TestSqlContextTables()
{
// arrange
var sqlContext = new SqlContext(new SparkContext("", ""), mockSqlContextProxy.Object);
var dataFrameProxy = new DataFrameIpcProxy(new JvmObjectReference("1"), mockSqlContextProxy.Object);
mockSqlContextProxy.Setup(m => m.Tables()).Returns(dataFrameProxy);
mockSqlContextProxy.Setup(m => m.Tables(It.IsAny<string>())).Returns(dataFrameProxy);
// act
var actualTablesDataFrame = sqlContext.Tables();
// assert
Assert.AreEqual(dataFrameProxy, actualTablesDataFrame.DataFrameProxy);
// act
actualTablesDataFrame = sqlContext.Tables("db");
// assert
Assert.AreEqual(dataFrameProxy, actualTablesDataFrame.DataFrameProxy);
}
public void TestSqlContextRegisterDataFrameAsTable()
{
// arrange
mockSqlContextProxy.Setup(m => m.RegisterDataFrameAsTable(It.IsAny<IDataFrameProxy>(), It.IsAny<string>()));
var sqlContext = new SqlContext(new SparkContext("", ""), mockSqlContextProxy.Object);
var dataFrameProxy = new DataFrameIpcProxy(new JvmObjectReference("1"), mockSqlContextProxy.Object);
var dataFrame = new DataFrame(dataFrameProxy, new SparkContext(new SparkConf()));
// act
sqlContext.RegisterDataFrameAsTable(dataFrame, "table");
// assert
mockSqlContextProxy.Verify(m => m.RegisterDataFrameAsTable(dataFrameProxy, "table"));
}
public void TestSqlContextCreateDataFrame()
{
// arrange
var mockSparkContextProxy = new Mock<ISparkContextProxy>();
mockSparkContextProxy.Setup(m => m.CreateCSharpRdd(It.IsAny<IRDDProxy>(), It.IsAny<byte[]>(), It.IsAny<Dictionary<string, string>>(),
It.IsAny<List<string>>(), It.IsAny<bool>(), It.IsAny<List<Broadcast>>(), It.IsAny<List<byte[]>>()));
var rddProxy = new Mock<IRDDProxy>();
var rdd = new RDD<object[]>(rddProxy.Object, new SparkContext(mockSparkContextProxy.Object, new SparkConf()));
var dataFrameProxy = new DataFrameIpcProxy(new JvmObjectReference("1"), mockSqlContextProxy.Object);
mockSqlContextProxy.Setup(m => m.CreateDataFrame(It.IsAny<IRDDProxy>(), It.IsAny<IStructTypeProxy>())).Returns(dataFrameProxy);
var sqlContext = new SqlContext(new SparkContext("", ""), mockSqlContextProxy.Object);
var structTypeProxy = new Mock<IStructTypeProxy>();
structTypeProxy.Setup(m => m.ToJson()).Returns(RowHelper.ComplexJsonSchema);
// act
var dataFrame = sqlContext.CreateDataFrame(rdd, new StructType(structTypeProxy.Object));
// assert
Assert.AreEqual(dataFrameProxy, dataFrame.DataFrameProxy);
}
public void TestSqlContextCreateDataFrame()
{
// arrange
var mockSparkContextProxy = new Mock<ISparkContextProxy>();
mockSparkContextProxy.Setup(m => m.CreateCSharpRdd(It.IsAny<IRDDProxy>(), It.IsAny<byte[]>(), It.IsAny<Dictionary<string, string>>(),
It.IsAny<List<string>>(), It.IsAny<bool>(), It.IsAny<List<Broadcast>>(), It.IsAny<List<byte[]>>()));
var rddProxy = new Mock<IRDDProxy>();
var rdd = new RDD<object[]>(rddProxy.Object, new SparkContext(mockSparkContextProxy.Object, new SparkConf()));
var dataFrameProxy = new DataFrameIpcProxy(new JvmObjectReference("1"), mockSqlContextProxy.Object);
mockSqlContextProxy.Setup(m => m.CreateDataFrame(It.IsAny<IRDDProxy>(), It.IsAny<IStructTypeProxy>())).Returns(dataFrameProxy);
var sqlContext = new SqlContext(new SparkContext("", ""), mockSqlContextProxy.Object);
var structTypeProxy = new Mock<IStructTypeProxy>();
const string schemaJson = @"{
""fields"": [{
""metadata"": {},
""name"": ""guid"",
""nullable"": false,
""type"": ""string""
}],
""type"": ""struct""
}";
structTypeProxy.Setup(m => m.ToJson()).Returns(schemaJson);
// act
var dataFrame = sqlContext.CreateDataFrame(rdd, new StructType(structTypeProxy.Object));
// assert
Assert.AreEqual(dataFrameProxy, dataFrame.DataFrameProxy);
}
public void TestSqlContextReadDataFrame()
{
var dataFrameProxy = new DataFrameIpcProxy(new JvmObjectReference("1"), mockSqlContextProxy.Object);
mockSqlContextProxy.Setup(
m => m.ReadDataFrame(It.IsAny<string>(), It.IsAny<StructType>(), It.IsAny<Dictionary<string, string>>()))
.Returns(dataFrameProxy);
var sqlContext = new SqlContext(new SparkContext("", ""), mockSqlContextProxy.Object);
var structTypeProxy = new Mock<IStructTypeProxy>();
structTypeProxy.Setup(m => m.ToJson()).Returns(RowHelper.BasicJsonSchema);
// act
var dataFrame = sqlContext.ReadDataFrame(@"c:\path\to\input.txt", new StructType(structTypeProxy.Object), null);
// assert
Assert.AreEqual(dataFrameProxy, dataFrame.DataFrameProxy);
}