示例#1
0
        public void TestSchema()
        {
            // arrange
            mockDataFrameReaderProxy.Setup(m => m.Schema(It.IsAny<StructType>()));
            var dataFrameReader = new DataFrameReader(mockDataFrameReaderProxy.Object, sparkContext);
            const string jsonSchema = @"
                {
                  ""type"" : ""struct"",
                  ""fields"" : [ {
                    ""name"" : ""address"",
                    ""type"" : {
                      ""type"" : ""struct"",
                      ""fields"" : [ {
                        ""name"" : ""city"",
                        ""type"" : ""string"",
                        ""nullable"" : true,
                        ""metadata"" : { }
                      }, {
                        ""name"" : ""state"",
                        ""type"" : ""string"",
                        ""nullable"" : true,
                        ""metadata"" : { }
                      } ]
                    },
                    ""nullable"" : true,
                    ""metadata"" : { }
                  }, {
                    ""name"" : ""age"",
                    ""type"" : ""long"",
                    ""nullable"" : true,
                    ""metadata"" : { }
                  }, {
                    ""name"" : ""id"",
                    ""type"" : ""string"",
                    ""nullable"" : true,
                    ""metadata"" : { }
                  }, {
                    ""name"" : ""name"",
                    ""type"" : ""string"",
                    ""nullable"" : true,
                    ""metadata"" : { }
                  } ]
                }";
            var mockStructTypeProxy = new MockStructTypeProxy(jsonSchema);
            var schema = new StructType(mockStructTypeProxy);

            // act
            var reader = dataFrameReader.Schema(schema);

            // verify
            Assert.IsNotNull(reader);
            Assert.AreSame(reader, dataFrameReader);
            mockDataFrameReaderProxy.Verify(m => m.Schema(schema), Times.Once);
        }
示例#2
0
        public void TestDataFrameCollect()
        {
            string jsonSchema = @"
                {
                  ""type"" : ""struct"",
                  ""fields"" : [ {
                    ""name"" : ""address"",
                    ""type"" : {
                      ""type"" : ""struct"",
                      ""fields"" : [ {
                        ""name"" : ""city"",
                        ""type"" : ""string"",
                        ""nullable"" : true,
                        ""metadata"" : { }
                      }, {
                        ""name"" : ""state"",
                        ""type"" : ""string"",
                        ""nullable"" : true,
                        ""metadata"" : { }
                      } ]
                    },
                    ""nullable"" : true,
                    ""metadata"" : { }
                  }, {
                    ""name"" : ""age"",
                    ""type"" : ""long"",
                    ""nullable"" : true,
                    ""metadata"" : { }
                  }, {
                    ""name"" : ""id"",
                    ""type"" : ""string"",
                    ""nullable"" : true,
                    ""metadata"" : { }
                  }, {
                    ""name"" : ""name"",
                    ""type"" : ""string"",
                    ""nullable"" : true,
                    ""metadata"" : { }
                  } ]
                }";

            int localPort = 4000;
            object row1 = new object[] {
                new object[] {"Columbus", "Ohio"},
                34,
                "123",
                "Bill"
            };

            object row2 = new object[] {
                new object[] {"Seattle", "Washington"},
                43,
                "789",
                "Bill"
            };

            IStructTypeProxy structTypeProxy = new MockStructTypeProxy(jsonSchema);
            IDataFrameProxy dataFrameProxy =
                new MockDataFrameProxy(localPort,
                                       new List<object>() { row1, row2 },
                                       structTypeProxy);
            DataFrame dataFrame = new DataFrame(dataFrameProxy, null);

            List<Row> rows = new List<Row>();
            foreach (var row in dataFrame.Collect())
            {
                rows.Add(row);
                Console.WriteLine("{0}", row);
            }

            Assert.AreEqual(rows.Count, 2);
            Row firstRow = rows[0];

            string id = firstRow.GetAs<string>("id");
            Assert.IsTrue(id.Equals("123"));
            string name = firstRow.GetAs<string>("name");
            Assert.IsTrue(name.Equals("Bill"));
            int age = firstRow.GetAs<int>("age");
            Assert.AreEqual(age, 34);

            Row address = firstRow.GetAs<Row>("address");
            Assert.AreNotEqual(address, null);
            string city = address.GetAs<string>("city");
            Assert.IsTrue(city.Equals("Columbus"));
            string state = address.GetAs<string>("state");
            Assert.IsTrue(state.Equals("Ohio"));
        }