/// <summary> /// Returns all of Rows in this DataFrame /// </summary> public IEnumerable <Row> Collect() { if (rowSchema == null) { rowSchema = RowSchema.ParseRowSchemaFromJson(Schema.ToJson()); } IRDDProxy rddProxy = dataFrameProxy.JavaToCSharp(); RDD <Row> rdd = new RDD <Row>(rddProxy, sparkContext, SerializedMode.Row); int port = rddProxy.CollectAndServe(); foreach (var item in rdd.Collect(port)) { yield return(new RowImpl(item, rowSchema)); } }
internal static ColumnSchema ParseColumnSchemaFromJson(string json) { ColumnSchema col = new ColumnSchema(); JObject joField = JObject.Parse(json); col.name = joField["name"].ToString(); col.nullable = (bool)(joField["nullable"]); JToken jtType = joField["type"]; if (jtType.Type == JTokenType.String) { col.type = new RowSchema(joField["type"].ToString()); } else { col.type = RowSchema.ParseRowSchemaFromJson(joField["type"].ToString()); } return(col); }