public void TestSqlContextCreateDataFrame() { // arrange var mockSparkContextProxy = new Mock <ISparkContextProxy>(); mockSparkContextProxy.Setup(m => m.CreateCSharpRdd(It.IsAny <IRDDProxy>(), It.IsAny <byte[]>(), It.IsAny <Dictionary <string, string> >(), It.IsAny <List <string> >(), It.IsAny <bool>(), It.IsAny <List <Broadcast> >(), It.IsAny <List <byte[]> >())); var rddProxy = new Mock <IRDDProxy>(); var rdd = new RDD <object[]>(rddProxy.Object, new SparkContext(mockSparkContextProxy.Object, new SparkConf())); var dataFrameProxy = new DataFrameIpcProxy(new JvmObjectReference("1"), mockSqlContextProxy.Object); mockSqlContextProxy.Setup(m => m.CreateDataFrame(It.IsAny <IRDDProxy>(), It.IsAny <IStructTypeProxy>())).Returns(dataFrameProxy); var sqlContext = new SqlContext(new SparkContext("", ""), mockSqlContextProxy.Object); var structTypeProxy = new Mock <IStructTypeProxy>(); const string schemaJson = @"{ ""fields"": [{ ""metadata"": {}, ""name"": ""guid"", ""nullable"": false, ""type"": ""string"" }], ""type"": ""struct"" }"; structTypeProxy.Setup(m => m.ToJson()).Returns(schemaJson); // act var dataFrame = sqlContext.CreateDataFrame(rdd, new StructType(structTypeProxy.Object)); // assert Assert.AreEqual(dataFrameProxy, dataFrame.DataFrameProxy); }
public void TestSqlContextCreateDataFrame() { // arrange var mockSparkContextProxy = new Mock <ISparkContextProxy>(); mockSparkContextProxy.Setup(m => m.CreateCSharpRdd(It.IsAny <IRDDProxy>(), It.IsAny <byte[]>(), It.IsAny <Dictionary <string, string> >(), It.IsAny <List <string> >(), It.IsAny <bool>(), It.IsAny <List <Broadcast> >(), It.IsAny <List <byte[]> >())); var rddProxy = new Mock <IRDDProxy>(); var rdd = new RDD <object[]>(rddProxy.Object, new SparkContext(mockSparkContextProxy.Object, new SparkConf())); var dataFrameProxy = new DataFrameIpcProxy(new JvmObjectReference("1"), mockSqlContextProxy.Object); mockSqlContextProxy.Setup(m => m.CreateDataFrame(It.IsAny <IRDDProxy>(), It.IsAny <IStructTypeProxy>())).Returns(dataFrameProxy); var sqlContext = new SqlContext(new SparkContext("", ""), mockSqlContextProxy.Object); var structTypeProxy = new Mock <IStructTypeProxy>(); structTypeProxy.Setup(m => m.ToJson()).Returns(RowHelper.ComplexJsonSchema); // act var dataFrame = sqlContext.CreateDataFrame(rdd, new StructType(structTypeProxy.Object)); // assert Assert.AreEqual(dataFrameProxy, dataFrame.DataFrameProxy); }
public void TestSqlContextCreateDataFrame() { // arrange var mockSparkContextProxy = new Mock<ISparkContextProxy>(); mockSparkContextProxy.Setup(m => m.CreateCSharpRdd(It.IsAny<IRDDProxy>(), It.IsAny<byte[]>(), It.IsAny<Dictionary<string, string>>(), It.IsAny<List<string>>(), It.IsAny<bool>(), It.IsAny<List<Broadcast>>(), It.IsAny<List<byte[]>>())); var rddProxy = new Mock<IRDDProxy>(); var rdd = new RDD<object[]>(rddProxy.Object, new SparkContext(mockSparkContextProxy.Object, new SparkConf())); var dataFrameProxy = new DataFrameIpcProxy(new JvmObjectReference("1"), mockSqlContextProxy.Object); mockSqlContextProxy.Setup(m => m.CreateDataFrame(It.IsAny<IRDDProxy>(), It.IsAny<IStructTypeProxy>())).Returns(dataFrameProxy); var sqlContext = new SqlContext(new SparkContext("", ""), mockSqlContextProxy.Object); var structTypeProxy = new Mock<IStructTypeProxy>(); const string schemaJson = @"{ ""fields"": [{ ""metadata"": {}, ""name"": ""guid"", ""nullable"": false, ""type"": ""string"" }], ""type"": ""struct"" }"; structTypeProxy.Setup(m => m.ToJson()).Returns(schemaJson); // act var dataFrame = sqlContext.CreateDataFrame(rdd, new StructType(structTypeProxy.Object)); // assert Assert.AreEqual(dataFrameProxy, dataFrame.DataFrameProxy); }
public void TestSqlContextCreateDataFrame() { // arrange var mockSparkContextProxy = new Mock<ISparkContextProxy>(); mockSparkContextProxy.Setup(m => m.CreateCSharpRdd(It.IsAny<IRDDProxy>(), It.IsAny<byte[]>(), It.IsAny<Dictionary<string, string>>(), It.IsAny<List<string>>(), It.IsAny<bool>(), It.IsAny<List<Broadcast>>(), It.IsAny<List<byte[]>>())); var rddProxy = new Mock<IRDDProxy>(); var rdd = new RDD<object[]>(rddProxy.Object, new SparkContext(mockSparkContextProxy.Object, new SparkConf())); var dataFrameProxy = new DataFrameIpcProxy(new JvmObjectReference("1"), mockSqlContextProxy.Object); mockSqlContextProxy.Setup(m => m.CreateDataFrame(It.IsAny<IRDDProxy>(), It.IsAny<IStructTypeProxy>())).Returns(dataFrameProxy); var sqlContext = new SqlContext(new SparkContext("", ""), mockSqlContextProxy.Object); var structTypeProxy = new Mock<IStructTypeProxy>(); structTypeProxy.Setup(m => m.ToJson()).Returns(RowHelper.ComplexJsonSchema); // act var dataFrame = sqlContext.CreateDataFrame(rdd, new StructType(structTypeProxy.Object)); // assert Assert.AreEqual(dataFrameProxy, dataFrame.DataFrameProxy); }