Beispiel #1
0
        public void TestSqlContextTables()
        {
            // arrange
            var sqlContext = new SqlContext(new SparkContext("", ""), mockSqlContextProxy.Object);
            var dataFrameProxy = new DataFrameIpcProxy(new JvmObjectReference("1"), mockSqlContextProxy.Object);
            mockSqlContextProxy.Setup(m => m.Tables()).Returns(dataFrameProxy);
            mockSqlContextProxy.Setup(m => m.Tables(It.IsAny<string>())).Returns(dataFrameProxy);

            // act
            var actualTablesDataFrame = sqlContext.Tables();

            // assert
            Assert.AreEqual(dataFrameProxy, actualTablesDataFrame.DataFrameProxy);

            // act
            actualTablesDataFrame = sqlContext.Tables("db");

            // assert
            Assert.AreEqual(dataFrameProxy, actualTablesDataFrame.DataFrameProxy);
        }
Beispiel #2
0
        public void TestSqlContextRegisterDataFrameAsTable()
        {
            // arrange
            mockSqlContextProxy.Setup(m => m.RegisterDataFrameAsTable(It.IsAny<IDataFrameProxy>(), It.IsAny<string>()));
            var sqlContext = new SqlContext(new SparkContext("", ""), mockSqlContextProxy.Object);
            var dataFrameProxy = new DataFrameIpcProxy(new JvmObjectReference("1"), mockSqlContextProxy.Object);
            var dataFrame = new DataFrame(dataFrameProxy, new SparkContext(new SparkConf()));

            // act
            sqlContext.RegisterDataFrameAsTable(dataFrame, "table");

            // assert
            mockSqlContextProxy.Verify(m => m.RegisterDataFrameAsTable(dataFrameProxy, "table"));
        }
Beispiel #3
0
        public void TestSqlContextCreateDataFrame()
        {
            // arrange
            var mockSparkContextProxy = new Mock<ISparkContextProxy>();
            mockSparkContextProxy.Setup(m => m.CreateCSharpRdd(It.IsAny<IRDDProxy>(), It.IsAny<byte[]>(), It.IsAny<Dictionary<string, string>>(),
                It.IsAny<List<string>>(), It.IsAny<bool>(), It.IsAny<List<Broadcast>>(), It.IsAny<List<byte[]>>()));
            var rddProxy = new Mock<IRDDProxy>();
            var rdd = new RDD<object[]>(rddProxy.Object, new SparkContext(mockSparkContextProxy.Object, new SparkConf()));
            var dataFrameProxy = new DataFrameIpcProxy(new JvmObjectReference("1"), mockSqlContextProxy.Object);
            mockSqlContextProxy.Setup(m => m.CreateDataFrame(It.IsAny<IRDDProxy>(), It.IsAny<IStructTypeProxy>())).Returns(dataFrameProxy);
            var sqlContext = new SqlContext(new SparkContext("", ""), mockSqlContextProxy.Object);
            var structTypeProxy = new Mock<IStructTypeProxy>();
            structTypeProxy.Setup(m => m.ToJson()).Returns(RowHelper.ComplexJsonSchema);
            // act
            var dataFrame = sqlContext.CreateDataFrame(rdd, new StructType(structTypeProxy.Object));

            // assert
            Assert.AreEqual(dataFrameProxy, dataFrame.DataFrameProxy);
        }
Beispiel #4
0
        public void TestSqlContextCreateDataFrame()
        {
            // arrange
            var mockSparkContextProxy = new Mock<ISparkContextProxy>();
            mockSparkContextProxy.Setup(m => m.CreateCSharpRdd(It.IsAny<IRDDProxy>(), It.IsAny<byte[]>(), It.IsAny<Dictionary<string, string>>(),
                It.IsAny<List<string>>(), It.IsAny<bool>(), It.IsAny<List<Broadcast>>(), It.IsAny<List<byte[]>>()));
            var rddProxy = new Mock<IRDDProxy>();
            var rdd = new RDD<object[]>(rddProxy.Object, new SparkContext(mockSparkContextProxy.Object, new SparkConf()));
            var dataFrameProxy = new DataFrameIpcProxy(new JvmObjectReference("1"), mockSqlContextProxy.Object);
            mockSqlContextProxy.Setup(m => m.CreateDataFrame(It.IsAny<IRDDProxy>(), It.IsAny<IStructTypeProxy>())).Returns(dataFrameProxy);
            var sqlContext = new SqlContext(new SparkContext("", ""), mockSqlContextProxy.Object);
            var structTypeProxy = new Mock<IStructTypeProxy>();
            const string schemaJson = @"{
	                                ""fields"": [{
		                                ""metadata"": {},
		                                ""name"": ""guid"",
		                                ""nullable"": false,
		                                ""type"": ""string""
	                                }],
	                                ""type"": ""struct""
                                    }";
            structTypeProxy.Setup(m => m.ToJson()).Returns(schemaJson);
            // act
            var dataFrame = sqlContext.CreateDataFrame(rdd, new StructType(structTypeProxy.Object));

            // assert
            Assert.AreEqual(dataFrameProxy, dataFrame.DataFrameProxy);
        }
Beispiel #5
0
        public void TestSqlContextReadDataFrame()
        {
            var dataFrameProxy = new DataFrameIpcProxy(new JvmObjectReference("1"), mockSqlContextProxy.Object);
            mockSqlContextProxy.Setup(
                m => m.ReadDataFrame(It.IsAny<string>(), It.IsAny<StructType>(), It.IsAny<Dictionary<string, string>>()))
                .Returns(dataFrameProxy);
            var sqlContext = new SqlContext(new SparkContext("", ""), mockSqlContextProxy.Object);
            var structTypeProxy = new Mock<IStructTypeProxy>();
            structTypeProxy.Setup(m => m.ToJson()).Returns(RowHelper.BasicJsonSchema);
            // act
            var dataFrame = sqlContext.ReadDataFrame(@"c:\path\to\input.txt", new StructType(structTypeProxy.Object), null);

            // assert
            Assert.AreEqual(dataFrameProxy, dataFrame.DataFrameProxy);
        }