public void TestSchema()
{
// arrange
mockDataFrameReaderProxy.Setup(m => m.Schema(It.IsAny <StructType>()));
var dataFrameReader = new DataFrameReader(mockDataFrameReaderProxy.Object, sparkContext);
const string jsonSchema = @"
{
""type"" : ""struct"",
""fields"" : [ {
""name"" : ""address"",
""type"" : {
""type"" : ""struct"",
""fields"" : [ {
""name"" : ""city"",
""type"" : ""string"",
""nullable"" : true,
""metadata"" : { }
}, {
""name"" : ""state"",
""type"" : ""string"",
""nullable"" : true,
""metadata"" : { }
} ]
},
""nullable"" : true,
""metadata"" : { }
}, {
""name"" : ""age"",
""type"" : ""long"",
""nullable"" : true,
""metadata"" : { }
}, {
""name"" : ""id"",
""type"" : ""string"",
""nullable"" : true,
""metadata"" : { }
}, {
""name"" : ""name"",
""type"" : ""string"",
""nullable"" : true,
""metadata"" : { }
} ]
}";
var mockStructTypeProxy = new MockStructTypeProxy(jsonSchema);
var schema = new StructType(mockStructTypeProxy);
// act
var reader = dataFrameReader.Schema(schema);
// verify
Assert.IsNotNull(reader);
Assert.AreSame(reader, dataFrameReader);
mockDataFrameReaderProxy.Verify(m => m.Schema(schema), Times.Once);
}
public void TestDataFrameCollect()
{
string jsonSchema = @"
{
""type"" : ""struct"",
""fields"" : [ {
""name"" : ""address"",
""type"" : {
""type"" : ""struct"",
""fields"" : [ {
""name"" : ""city"",
""type"" : ""string"",
""nullable"" : true,
""metadata"" : { }
}, {
""name"" : ""state"",
""type"" : ""string"",
""nullable"" : true,
""metadata"" : { }
} ]
},
""nullable"" : true,
""metadata"" : { }
}, {
""name"" : ""age"",
""type"" : ""long"",
""nullable"" : true,
""metadata"" : { }
}, {
""name"" : ""id"",
""type"" : ""string"",
""nullable"" : true,
""metadata"" : { }
}, {
""name"" : ""name"",
""type"" : ""string"",
""nullable"" : true,
""metadata"" : { }
} ]
}";
int localPort = 4000;
object row1 = new object[] {
new object[] { "Columbus", "Ohio" },
34,
"123",
"Bill"
};
object row2 = new object[] {
new object[] { "Seattle", "Washington" },
43,
"789",
"Bill"
};
IStructTypeProxy structTypeProxy = new MockStructTypeProxy(jsonSchema);
IDataFrameProxy dataFrameProxy =
new MockDataFrameProxy(localPort,
new List <object>()
{
row1, row2
},
structTypeProxy);
DataFrame dataFrame = new DataFrame(dataFrameProxy, null);
List <Row> rows = new List <Row>();
foreach (var row in dataFrame.Collect())
{
rows.Add(row);
Console.WriteLine("{0}", row);
}
Assert.AreEqual(rows.Count, 2);
Row firstRow = rows[0];
string id = firstRow.GetAs <string>("id");
Assert.IsTrue(id.Equals("123"));
string name = firstRow.GetAs <string>("name");
Assert.IsTrue(name.Equals("Bill"));
int age = firstRow.GetAs <int>("age");
Assert.AreEqual(age, 34);
Row address = firstRow.GetAs <Row>("address");
Assert.AreNotEqual(address, null);
string city = address.GetAs <string>("city");
Assert.IsTrue(city.Equals("Columbus"));
string state = address.GetAs <string>("state");
Assert.IsTrue(state.Equals("Ohio"));
}