public void TestSchema()
{
// arrange
mockDataFrameReaderProxy.Setup(m => m.Schema(It.IsAny<StructType>()));
var dataFrameReader = new DataFrameReader(mockDataFrameReaderProxy.Object, sparkContext);
const string jsonSchema = @"
{
""type"" : ""struct"",
""fields"" : [ {
""name"" : ""address"",
""type"" : {
""type"" : ""struct"",
""fields"" : [ {
""name"" : ""city"",
""type"" : ""string"",
""nullable"" : true,
""metadata"" : { }
}, {
""name"" : ""state"",
""type"" : ""string"",
""nullable"" : true,
""metadata"" : { }
} ]
},
""nullable"" : true,
""metadata"" : { }
}, {
""name"" : ""age"",
""type"" : ""long"",
""nullable"" : true,
""metadata"" : { }
}, {
""name"" : ""id"",
""type"" : ""string"",
""nullable"" : true,
""metadata"" : { }
}, {
""name"" : ""name"",
""type"" : ""string"",
""nullable"" : true,
""metadata"" : { }
} ]
}";
var mockStructTypeProxy = new MockStructTypeProxy(jsonSchema);
var schema = new StructType(mockStructTypeProxy);
// act
var reader = dataFrameReader.Schema(schema);
// verify
Assert.IsNotNull(reader);
Assert.AreSame(reader, dataFrameReader);
mockDataFrameReaderProxy.Verify(m => m.Schema(schema), Times.Once);
}
public void TestDataFrameCollect()
{
string jsonSchema = @"
{
""type"" : ""struct"",
""fields"" : [ {
""name"" : ""address"",
""type"" : {
""type"" : ""struct"",
""fields"" : [ {
""name"" : ""city"",
""type"" : ""string"",
""nullable"" : true,
""metadata"" : { }
}, {
""name"" : ""state"",
""type"" : ""string"",
""nullable"" : true,
""metadata"" : { }
} ]
},
""nullable"" : true,
""metadata"" : { }
}, {
""name"" : ""age"",
""type"" : ""long"",
""nullable"" : true,
""metadata"" : { }
}, {
""name"" : ""id"",
""type"" : ""string"",
""nullable"" : true,
""metadata"" : { }
}, {
""name"" : ""name"",
""type"" : ""string"",
""nullable"" : true,
""metadata"" : { }
} ]
}";
int localPort = 4000;
object row1 = new object[] {
new object[] {"Columbus", "Ohio"},
34,
"123",
"Bill"
};
object row2 = new object[] {
new object[] {"Seattle", "Washington"},
43,
"789",
"Bill"
};
IStructTypeProxy structTypeProxy = new MockStructTypeProxy(jsonSchema);
IDataFrameProxy dataFrameProxy =
new MockDataFrameProxy(localPort,
new List<object>() { row1, row2 },
structTypeProxy);
DataFrame dataFrame = new DataFrame(dataFrameProxy, null);
List<Row> rows = new List<Row>();
foreach (var row in dataFrame.Collect())
{
rows.Add(row);
Console.WriteLine("{0}", row);
}
Assert.AreEqual(rows.Count, 2);
Row firstRow = rows[0];
string id = firstRow.GetAs<string>("id");
Assert.IsTrue(id.Equals("123"));
string name = firstRow.GetAs<string>("name");
Assert.IsTrue(name.Equals("Bill"));
int age = firstRow.GetAs<int>("age");
Assert.AreEqual(age, 34);
Row address = firstRow.GetAs<Row>("address");
Assert.AreNotEqual(address, null);
string city = address.GetAs<string>("city");
Assert.IsTrue(city.Equals("Columbus"));
string state = address.GetAs<string>("state");
Assert.IsTrue(state.Equals("Ohio"));
}