public void TestSqlContextTables() { // arrange var sqlContext = new SqlContext(new SparkContext("", ""), mockSqlContextProxy.Object); var dataFrameProxy = new DataFrameIpcProxy(new JvmObjectReference("1"), mockSqlContextProxy.Object); mockSqlContextProxy.Setup(m => m.Tables()).Returns(dataFrameProxy); mockSqlContextProxy.Setup(m => m.Tables(It.IsAny<string>())).Returns(dataFrameProxy); // act var actualTablesDataFrame = sqlContext.Tables(); // assert Assert.AreEqual(dataFrameProxy, actualTablesDataFrame.DataFrameProxy); // act actualTablesDataFrame = sqlContext.Tables("db"); // assert Assert.AreEqual(dataFrameProxy, actualTablesDataFrame.DataFrameProxy); }
public void TestSqlContextRegisterDataFrameAsTable() { // arrange mockSqlContextProxy.Setup(m => m.RegisterDataFrameAsTable(It.IsAny<IDataFrameProxy>(), It.IsAny<string>())); var sqlContext = new SqlContext(new SparkContext("", ""), mockSqlContextProxy.Object); var dataFrameProxy = new DataFrameIpcProxy(new JvmObjectReference("1"), mockSqlContextProxy.Object); var dataFrame = new DataFrame(dataFrameProxy, new SparkContext(new SparkConf())); // act sqlContext.RegisterDataFrameAsTable(dataFrame, "table"); // assert mockSqlContextProxy.Verify(m => m.RegisterDataFrameAsTable(dataFrameProxy, "table")); }
public void TestSqlContextCreateDataFrame() { // arrange var mockSparkContextProxy = new Mock<ISparkContextProxy>(); mockSparkContextProxy.Setup(m => m.CreateCSharpRdd(It.IsAny<IRDDProxy>(), It.IsAny<byte[]>(), It.IsAny<Dictionary<string, string>>(), It.IsAny<List<string>>(), It.IsAny<bool>(), It.IsAny<List<Broadcast>>(), It.IsAny<List<byte[]>>())); var rddProxy = new Mock<IRDDProxy>(); var rdd = new RDD<object[]>(rddProxy.Object, new SparkContext(mockSparkContextProxy.Object, new SparkConf())); var dataFrameProxy = new DataFrameIpcProxy(new JvmObjectReference("1"), mockSqlContextProxy.Object); mockSqlContextProxy.Setup(m => m.CreateDataFrame(It.IsAny<IRDDProxy>(), It.IsAny<IStructTypeProxy>())).Returns(dataFrameProxy); var sqlContext = new SqlContext(new SparkContext("", ""), mockSqlContextProxy.Object); var structTypeProxy = new Mock<IStructTypeProxy>(); structTypeProxy.Setup(m => m.ToJson()).Returns(RowHelper.ComplexJsonSchema); // act var dataFrame = sqlContext.CreateDataFrame(rdd, new StructType(structTypeProxy.Object)); // assert Assert.AreEqual(dataFrameProxy, dataFrame.DataFrameProxy); }
public void TestSqlContextCreateDataFrame() { // arrange var mockSparkContextProxy = new Mock<ISparkContextProxy>(); mockSparkContextProxy.Setup(m => m.CreateCSharpRdd(It.IsAny<IRDDProxy>(), It.IsAny<byte[]>(), It.IsAny<Dictionary<string, string>>(), It.IsAny<List<string>>(), It.IsAny<bool>(), It.IsAny<List<Broadcast>>(), It.IsAny<List<byte[]>>())); var rddProxy = new Mock<IRDDProxy>(); var rdd = new RDD<object[]>(rddProxy.Object, new SparkContext(mockSparkContextProxy.Object, new SparkConf())); var dataFrameProxy = new DataFrameIpcProxy(new JvmObjectReference("1"), mockSqlContextProxy.Object); mockSqlContextProxy.Setup(m => m.CreateDataFrame(It.IsAny<IRDDProxy>(), It.IsAny<IStructTypeProxy>())).Returns(dataFrameProxy); var sqlContext = new SqlContext(new SparkContext("", ""), mockSqlContextProxy.Object); var structTypeProxy = new Mock<IStructTypeProxy>(); const string schemaJson = @"{ ""fields"": [{ ""metadata"": {}, ""name"": ""guid"", ""nullable"": false, ""type"": ""string"" }], ""type"": ""struct"" }"; structTypeProxy.Setup(m => m.ToJson()).Returns(schemaJson); // act var dataFrame = sqlContext.CreateDataFrame(rdd, new StructType(structTypeProxy.Object)); // assert Assert.AreEqual(dataFrameProxy, dataFrame.DataFrameProxy); }
public void TestSqlContextReadDataFrame() { var dataFrameProxy = new DataFrameIpcProxy(new JvmObjectReference("1"), mockSqlContextProxy.Object); mockSqlContextProxy.Setup( m => m.ReadDataFrame(It.IsAny<string>(), It.IsAny<StructType>(), It.IsAny<Dictionary<string, string>>())) .Returns(dataFrameProxy); var sqlContext = new SqlContext(new SparkContext("", ""), mockSqlContextProxy.Object); var structTypeProxy = new Mock<IStructTypeProxy>(); structTypeProxy.Setup(m => m.ToJson()).Returns(RowHelper.BasicJsonSchema); // act var dataFrame = sqlContext.ReadDataFrame(@"c:\path\to\input.txt", new StructType(structTypeProxy.Object), null); // assert Assert.AreEqual(dataFrameProxy, dataFrame.DataFrameProxy); }