public void TestSchema() { // arrange mockDataFrameReaderProxy.Setup(m => m.Schema(It.IsAny<StructType>())); var dataFrameReader = new DataFrameReader(mockDataFrameReaderProxy.Object, sparkContext); const string jsonSchema = @" { ""type"" : ""struct"", ""fields"" : [ { ""name"" : ""address"", ""type"" : { ""type"" : ""struct"", ""fields"" : [ { ""name"" : ""city"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""state"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } } ] }, ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""age"", ""type"" : ""long"", ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""id"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""name"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } } ] }"; var mockStructTypeProxy = new MockStructTypeProxy(jsonSchema); var schema = new StructType(mockStructTypeProxy); // act var reader = dataFrameReader.Schema(schema); // verify Assert.IsNotNull(reader); Assert.AreSame(reader, dataFrameReader); mockDataFrameReaderProxy.Verify(m => m.Schema(schema), Times.Once); }
public void TestFormat() { // arrange mockDataFrameReaderProxy.Setup(m => m.Format(It.IsAny<string>())); var dataFrameReader = new DataFrameReader(mockDataFrameReaderProxy.Object, sparkContext); // act var reader = dataFrameReader.Format("json"); // verify Assert.IsNotNull(reader); Assert.AreSame(reader, dataFrameReader); mockDataFrameReaderProxy.Verify(m => m.Format("json"), Times.Once); }
public void TestJson() { // arrange mockDataFrameReaderProxy.Setup(m => m.Format(It.IsAny<string>())); var dataFrameReader = new DataFrameReader(mockDataFrameReaderProxy.Object, sparkContext); // act const string path = "path/to/json"; var reader = dataFrameReader.Json(path); // Assert Assert.IsNotNull(reader); mockDataFrameReaderProxy.Verify(m => m.Format("json"), Times.Once); mockDataFrameReaderProxy.Verify(m => m.Options( It.Is<Dictionary<string, string>>(dict => dict["path"] == path && dict.Count == 1)), Times.Once); mockDataFrameReaderProxy.Verify(m => m.Load(), Times.Once); }
public void TestParquet() { // arrange mockDataFrameReaderProxy.Setup(m => m.Parquet(It.IsAny<string[]>())); var dataFrameReader = new DataFrameReader(mockDataFrameReaderProxy.Object, sparkContext); // act const string path1 = "path/to/json"; const string path2 = "path/to/json"; var reader = dataFrameReader.Parquet(path1, path2); // Assert Assert.IsNotNull(reader); mockDataFrameReaderProxy.Verify(m => m.Parquet( It.Is<string[]>(strArray => strArray.Length == 2 && strArray[0] == path1 && strArray[1] == path2)), Times.Once); }
public void TestJdbc3() { // arrange mockDataFrameReaderProxy.Setup(m => m.Jdbc(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string[]>(), It.IsAny<Dictionary<string, string>>())); var dataFrameReader = new DataFrameReader(mockDataFrameReaderProxy.Object, sparkContext); // Act const string url = "url"; const string table = "table_name"; var predicates = new[] { "predicate1", "predicate2" }; var connectionProperties = new Dictionary<string, string>() { {"prop1", "value1"}, {"prop2", "value2"} }; dataFrameReader.Jdbc(url, table, predicates, connectionProperties); // Assert mockDataFrameReaderProxy.Verify(m => m.Jdbc(url, table, predicates, connectionProperties), Times.Once); }
public void TestJdbc2() { // arrange mockDataFrameReaderProxy.Setup(m => m.Jdbc(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<int>(), It.IsAny<Dictionary<string, string>>())); var dataFrameReader = new DataFrameReader(mockDataFrameReaderProxy.Object, sparkContext); // Act const string url = "url"; const string table = "table_name"; const string columnName = "col1"; const string lowerBound = "a"; const string upperBound = "z"; const int numPartitions = 5; var connectionProperties = new Dictionary<string, string>() { {"prop1", "value1"}, {"prop2", "value2"} }; dataFrameReader.Jdbc(url, table, columnName, lowerBound, upperBound, numPartitions, connectionProperties); // Assert mockDataFrameReaderProxy.Verify(m => m.Jdbc(url, table, columnName, lowerBound, upperBound, numPartitions, connectionProperties), Times.Once); }
public void TestLoad() { // arrange mockDataFrameReaderProxy.Setup(m => m.Load()); var dataFrameReader = new DataFrameReader(mockDataFrameReaderProxy.Object, sparkContext); // Act dataFrameReader.Load(); // Assert mockDataFrameReaderProxy.Verify(m => m.Load(), Times.Once); }
public void TestLoadWithPath() { // arrange mockDataFrameReaderProxy.Setup(m => m.Load()); mockDataFrameReaderProxy.Setup(m => m.Options(It.IsAny<Dictionary<string, string>>())); var dataFrameReader = new DataFrameReader(mockDataFrameReaderProxy.Object, sparkContext); const string key = "path"; const string path = "path_value"; // Act dataFrameReader.Load(path); // Assert mockDataFrameReaderProxy.Verify(m => m.Options( It.Is<Dictionary<string, string>>(dict => dict[key] == path && dict.Count == 1)), Times.Once); mockDataFrameReaderProxy.Verify(m => m.Load(), Times.Once); }
public void TestOptions() { // arrange mockDataFrameReaderProxy.Setup(m => m.Options(It.IsAny<Dictionary<string, string>>())); var dataFrameReader = new DataFrameReader(mockDataFrameReaderProxy.Object, sparkContext); const string key1 = "key1"; const string value1 = "value1"; const string key2 = "key2"; const string value2 = "value2"; var opts = new Dictionary<string, string>() { {key1, value1}, {key2, value2} }; // Act dataFrameReader.Options(opts); // Assert mockDataFrameReaderProxy.Verify(m => m.Options(It.Is<Dictionary<string, string>>( dict => dict[key1] == value1 && dict[key2] == value2 && dict.Count == 2) ), Times.Once ); }