public void TestCache() { // arrange mockDataFrameProxy.Setup(m => m.Persist(It.IsAny <StorageLevelType>())); var sc = new SparkContext(null); var dataFrame = new DataFrame(mockDataFrameProxy.Object, sc); dataFrame.Cache(); // assert mockDataFrameProxy.Verify(m => m.Persist(StorageLevelType.MEMORY_AND_DISK), Times.Once()); }
private static void ReviewsCleanup(DataFrame dataFrame) { Console.WriteLine("Ratings Clean-up"); dataFrame = dataFrame .Filter( dataFrame["reviewerID"].IsNotNull() .And(dataFrame["asin"].IsNotNull()) .And(dataFrame["reviewText"].IsNotNull())); dataFrame = dataFrame .WithColumnRenamed("reviewerID", "rid") .WithColumnRenamed("reviewText", "review_text") .WithColumnRenamed("unixReviewTime", "unix_time"); dataFrame.Cache(); dataFrame.CreateOrReplaceTempView("ElectronicsReviews"); Console.WriteLine($"Reviews Count: {dataFrame.Count()}"); Console.WriteLine("Done"); Console.WriteLine(); }
public void TestSignaturesV2_3_X() { Column col = _df["name"]; col = _df["age"]; DataFrame df = _df.ToDF(); df = df.ToDF("name2", "age2"); StructType schema = _df.Schema(); Assert.NotNull(schema); _df.PrintSchema(); _df.Explain(); _df.Explain(true); _df.Explain(false); Assert.Equal(2, _df.Columns().ToArray().Length); _df.IsLocal(); _df.IsStreaming(); using (var tempDir = new TemporaryDirectory()) { // The following is required for *CheckPoint(). _spark.SparkContext.SetCheckpointDir(tempDir.Path); _df.Checkpoint(); _df.Checkpoint(false); _df.LocalCheckpoint(); _df.LocalCheckpoint(false); } _df.WithWatermark("time", "10 minutes"); _df.Show(); _df.Show(10); _df.Show(10, 10); _df.Show(10, 10, true); _df.Join(_df); _df.Join(_df, "name"); _df.Join(_df, new[] { "name" }); _df.Join(_df, new[] { "name" }, "outer"); _df.Join(_df, _df["age"] == _df["age"]); _df.Join(_df, _df["age"] == _df["age"], "outer"); _df.CrossJoin(_df); _df.SortWithinPartitions("age"); _df.SortWithinPartitions("age", "name"); _df.SortWithinPartitions(); _df.SortWithinPartitions(_df["age"]); _df.SortWithinPartitions(_df["age"], _df["name"]); _df.Sort("age"); _df.Sort("age", "name"); _df.Sort(); _df.Sort(_df["age"]); _df.Sort(_df["age"], _df["name"]); _df.OrderBy("age"); _df.OrderBy("age", "name"); _df.OrderBy(); _df.OrderBy(_df["age"]); _df.OrderBy(_df["age"], _df["name"]); _df.Hint("broadcast"); _df.Hint("broadcast", new[] { "hello", "world" }); _df.Col("age"); _df.ColRegex("age"); _df.As("alias"); _df.Alias("alias"); _df.Select("age"); _df.Select("age", "name"); _df.Select(); _df.Select(_df["age"]); _df.Select(_df["age"], _df["name"]); _df.SelectExpr(); _df.SelectExpr("age * 2"); _df.SelectExpr("age * 2", "abs(age)"); _df.Filter(_df["age"] > 21); _df.Filter("age > 21"); _df.Where(_df["age"] > 21); _df.Where("age > 21"); _df.GroupBy("age"); _df.GroupBy("age", "name"); _df.GroupBy(); _df.GroupBy(_df["age"]); _df.GroupBy(_df["age"], _df["name"]); _df.Rollup("age"); _df.Rollup("age", "name"); _df.Rollup(); _df.Rollup(_df["age"]); _df.Rollup(_df["age"], _df["name"]); _df.Cube("age"); _df.Cube("age", "name"); _df.Cube(); _df.Cube(_df["age"]); _df.Cube(_df["age"], _df["name"]); _df.Agg(Avg(_df["age"])); _df.Agg(Avg(_df["age"]), Avg(_df["name"])); _df.Limit(10); _df.Union(_df); _df.UnionByName(_df); _df.Intersect(_df); _df.Except(_df); _df.Sample(0.5); _df.Sample(0.5, true); _df.Sample(0.5, false, 12345); _df.RandomSplit(new[] { 0.2, 0.8 }); _df.RandomSplit(new[] { 0.2, 0.8 }, 12345); _df.WithColumn("age2", _df["age"]); _df.WithColumnRenamed("age", "age2"); _df.Drop(); _df.Drop("age"); _df.Drop("age", "name"); _df.Drop(_df["age"]); _df.DropDuplicates(); _df.DropDuplicates("age"); _df.DropDuplicates("age", "name"); _df.Describe(); _df.Describe("age"); _df.Describe("age", "name"); _df.Summary(); _df.Summary("count"); _df.Summary("count", "mean"); _df.Head(2); _df.Head(); _df.First(); _df.Take(3).ToArray(); _df.Collect().ToArray(); _df.ToLocalIterator().ToArray(); _df.Count(); _df.Repartition(2); _df.Repartition(2, _df["age"]); _df.Repartition(_df["age"]); _df.Repartition(); _df.RepartitionByRange(2, _df["age"]); _df.RepartitionByRange(_df["age"]); _df.Coalesce(1); _df.Distinct(); _df.Persist(); _df.Cache(); _df.Unpersist(); _df.CreateTempView("view"); _df.CreateOrReplaceTempView("view"); _df.CreateGlobalTempView("global_view"); _df.CreateOrReplaceGlobalTempView("global_view"); }
private static void MetadataCleanup(DataFrame dataFrame) { Console.WriteLine("Metadata Clean-up"); var priceCleanup = Udf <string, float>( p => { if (!string.IsNullOrEmpty(p)) { var index = 0; for (var i = 0; i < p.Length; i++) { if (char.IsDigit(p[i])) { index = i; break; } } if (float.TryParse(p.Substring(index), out var result)) { return(result); } } return(-1f); }); var dateCleanup = Udf <string, double>( d => { if (!string.IsNullOrEmpty(d) && DateTime.TryParse(d, out var result)) { return((result.ToUniversalTime() - new DateTime(1970, 1, 1)).TotalSeconds); } return(-1L); }); var rankCleanup = Udf <string, long>( r => { if (!string.IsNullOrEmpty(r)) { var regex = new Regex(@"\d+(,\d+)*", RegexOptions.Singleline); var match = regex.Match(r); if (match.Success && long.TryParse(match.Value.Replace(",", string.Empty), out var result)) { return(result); } } return(-1L); }); dataFrame = dataFrame .Filter( dataFrame["asin"].IsNotNull() .And(dataFrame["title"].IsNotNull()) .And(dataFrame["main_cat"].IsNotNull()) .And(dataFrame["brand"].IsNotNull()) .And(Not(dataFrame["main_cat"].IsIn("Grocery", "Pet Supplies", "Baby", "Books", "Appstore for Android", "Gift Cards")))); dataFrame = dataFrame .WithColumn("clean_price", priceCleanup(dataFrame["price"])) .WithColumn("clean-date", dateCleanup(dataFrame["date"])) .WithColumn("clean-rank", rankCleanup(dataFrame["rank"])) .Drop(dataFrame["price"]) .Drop(dataFrame["date"]) .Drop(dataFrame["rank"]) .WithColumnRenamed("clean_price", "price") .WithColumnRenamed("clean-date", "unixTime") .WithColumnRenamed("clean-rank", "rank"); dataFrame.Cache(); dataFrame.CreateOrReplaceTempView("ElectronicsMetadata"); Console.WriteLine($"Metadata Count: {dataFrame.Count()}"); Console.WriteLine("Done"); Console.WriteLine(); }