public void TestSchema() { // arrange mockDataFrameReaderProxy.Setup(m => m.Schema(It.IsAny<StructType>())); var dataFrameReader = new DataFrameReader(mockDataFrameReaderProxy.Object, sparkContext); const string jsonSchema = @" { ""type"" : ""struct"", ""fields"" : [ { ""name"" : ""address"", ""type"" : { ""type"" : ""struct"", ""fields"" : [ { ""name"" : ""city"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""state"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } } ] }, ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""age"", ""type"" : ""long"", ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""id"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""name"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } } ] }"; var mockStructTypeProxy = new MockStructTypeProxy(jsonSchema); var schema = new StructType(mockStructTypeProxy); // act var reader = dataFrameReader.Schema(schema); // verify Assert.IsNotNull(reader); Assert.AreSame(reader, dataFrameReader); mockDataFrameReaderProxy.Verify(m => m.Schema(schema), Times.Once); }
public void TestDataFrameCollect() { string jsonSchema = @" { ""type"" : ""struct"", ""fields"" : [ { ""name"" : ""address"", ""type"" : { ""type"" : ""struct"", ""fields"" : [ { ""name"" : ""city"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""state"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } } ] }, ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""age"", ""type"" : ""long"", ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""id"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""name"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } } ] }"; int localPort = 4000; object row1 = new object[] { new object[] {"Columbus", "Ohio"}, 34, "123", "Bill" }; object row2 = new object[] { new object[] {"Seattle", "Washington"}, 43, "789", "Bill" }; IStructTypeProxy structTypeProxy = new MockStructTypeProxy(jsonSchema); IDataFrameProxy dataFrameProxy = new MockDataFrameProxy(localPort, new List<object>() { row1, row2 }, structTypeProxy); DataFrame dataFrame = new DataFrame(dataFrameProxy, null); List<Row> rows = new List<Row>(); foreach (var row in dataFrame.Collect()) { rows.Add(row); Console.WriteLine("{0}", row); } Assert.AreEqual(rows.Count, 2); Row firstRow = rows[0]; string id = firstRow.GetAs<string>("id"); Assert.IsTrue(id.Equals("123")); string name = firstRow.GetAs<string>("name"); Assert.IsTrue(name.Equals("Bill")); int age = firstRow.GetAs<int>("age"); Assert.AreEqual(age, 34); Row address = firstRow.GetAs<Row>("address"); Assert.AreNotEqual(address, null); string city = address.GetAs<string>("city"); Assert.IsTrue(city.Equals("Columbus")); string state = address.GetAs<string>("state"); Assert.IsTrue(state.Equals("Ohio")); }