public void TestSchema() { // arrange mockDataFrameReaderProxy.Setup(m => m.Schema(It.IsAny <StructType>())); var dataFrameReader = new DataFrameReader(mockDataFrameReaderProxy.Object, sparkContext); const string jsonSchema = @" { ""type"" : ""struct"", ""fields"" : [ { ""name"" : ""address"", ""type"" : { ""type"" : ""struct"", ""fields"" : [ { ""name"" : ""city"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""state"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } } ] }, ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""age"", ""type"" : ""long"", ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""id"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""name"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } } ] }"; var mockStructTypeProxy = new MockStructTypeProxy(jsonSchema); var schema = new StructType(mockStructTypeProxy); // act var reader = dataFrameReader.Schema(schema); // verify Assert.IsNotNull(reader); Assert.AreSame(reader, dataFrameReader); mockDataFrameReaderProxy.Verify(m => m.Schema(schema), Times.Once); }
public void TestDataFrameCollect() { string jsonSchema = @" { ""type"" : ""struct"", ""fields"" : [ { ""name"" : ""address"", ""type"" : { ""type"" : ""struct"", ""fields"" : [ { ""name"" : ""city"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""state"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } } ] }, ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""age"", ""type"" : ""long"", ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""id"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""name"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } } ] }"; int localPort = 4000; object row1 = new object[] { new object[] { "Columbus", "Ohio" }, 34, "123", "Bill" }; object row2 = new object[] { new object[] { "Seattle", "Washington" }, 43, "789", "Bill" }; IStructTypeProxy structTypeProxy = new MockStructTypeProxy(jsonSchema); IDataFrameProxy dataFrameProxy = new MockDataFrameProxy(localPort, new List <object>() { row1, row2 }, structTypeProxy); DataFrame dataFrame = new DataFrame(dataFrameProxy, null); List <Row> rows = new List <Row>(); foreach (var row in dataFrame.Collect()) { rows.Add(row); Console.WriteLine("{0}", row); } Assert.AreEqual(rows.Count, 2); Row firstRow = rows[0]; string id = firstRow.GetAs <string>("id"); Assert.IsTrue(id.Equals("123")); string name = firstRow.GetAs <string>("name"); Assert.IsTrue(name.Equals("Bill")); int age = firstRow.GetAs <int>("age"); Assert.AreEqual(age, 34); Row address = firstRow.GetAs <Row>("address"); Assert.AreNotEqual(address, null); string city = address.GetAs <string>("city"); Assert.IsTrue(city.Equals("Columbus")); string state = address.GetAs <string>("state"); Assert.IsTrue(state.Equals("Ohio")); }