public void TestSignaturesV2_3_X() { DataFrameReader dfr = _spark.Read(); Assert.IsType <DataFrameReader>(dfr.Format("json")); Assert.IsType <DataFrameReader>( dfr.Schema( new StructType(new[] { new StructField("age", new IntegerType()), new StructField("name", new StringType()) }))); Assert.IsType <DataFrameReader>(dfr.Schema("age INT, name STRING")); Assert.IsType <DataFrameReader>(dfr.Option("stringOption", "value")); Assert.IsType <DataFrameReader>(dfr.Option("boolOption", true)); Assert.IsType <DataFrameReader>(dfr.Option("longOption", 1L)); Assert.IsType <DataFrameReader>(dfr.Option("doubleOption", 3D)); Assert.IsType <DataFrameReader>( dfr.Options( new Dictionary <string, string> { { "option1", "value1" }, { "option2", "value2" } })); string jsonFile = $"{TestEnvironment.ResourceDirectory}people.json"; Assert.IsType <DataFrame>(dfr.Load()); Assert.IsType <DataFrame>(dfr.Load(jsonFile)); Assert.IsType <DataFrame>(dfr.Load(jsonFile, jsonFile)); Assert.IsType <DataFrame>(dfr.Json(jsonFile)); Assert.IsType <DataFrame>(dfr.Json(jsonFile, jsonFile)); string csvFile = $"{TestEnvironment.ResourceDirectory}people.csv"; Assert.IsType <DataFrame>(dfr.Csv(csvFile)); Assert.IsType <DataFrame>(dfr.Csv(csvFile, csvFile)); string parquetFile = $"{TestEnvironment.ResourceDirectory}users.parquet"; Assert.IsType <DataFrame>(dfr.Parquet(parquetFile)); Assert.IsType <DataFrame>(dfr.Parquet(parquetFile, parquetFile)); string orcFile = $"{TestEnvironment.ResourceDirectory}users.orc"; Assert.IsType <DataFrame>(dfr.Orc(orcFile)); Assert.IsType <DataFrame>(dfr.Orc(orcFile, orcFile)); dfr = _spark.Read(); string textFile = $"{TestEnvironment.ResourceDirectory}people.txt"; Assert.IsType <DataFrame>(dfr.Text(textFile)); Assert.IsType <DataFrame>(dfr.Text(textFile, textFile)); }
public void TestSchema() { // arrange mockDataFrameReaderProxy.Setup(m => m.Schema(It.IsAny <StructType>())); var dataFrameReader = new DataFrameReader(mockDataFrameReaderProxy.Object, sparkContext); const string jsonSchema = @" { ""type"" : ""struct"", ""fields"" : [ { ""name"" : ""address"", ""type"" : { ""type"" : ""struct"", ""fields"" : [ { ""name"" : ""city"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""state"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } } ] }, ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""age"", ""type"" : ""long"", ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""id"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""name"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } } ] }"; var mockStructTypeProxy = new MockStructTypeProxy(jsonSchema); var schema = new StructType(mockStructTypeProxy); // act var reader = dataFrameReader.Schema(schema); // verify Assert.IsNotNull(reader); Assert.AreSame(reader, dataFrameReader); mockDataFrameReaderProxy.Verify(m => m.Schema(schema), Times.Once); }