public static void ForEachRDD <V>(string title, DStream <KeyValuePair <string, V> > reducedStream, string prefix, string suffix = ".txt") { Logger.LogDebug("ForEachRDD " + title); reducedStream.ForeachRDD(new SumCountStatic().ForeachRDD <V>); if (!string.IsNullOrWhiteSpace(Options.SaveTxtDirectory)) { reducedStream.Map(kv => $"{kv.Key} = {TestUtils.GetValueText(kv.Value)}").SaveAsTextFiles(Path.Combine(Options.SaveTxtDirectory, prefix), suffix); } }
static void StartOneTest(SparkContext sc, DStream <string> lines, long elements, string prefix, string suffix = ".txt") { var isReduceByKey = Options.IsReduceByKey(); Logger.LogDebug("isReduceByKey = {0}", isReduceByKey); if (!Options.IsArrayValue) { //var pairs = lines.Map(line => new ParseKeyValue(0).Parse(line)); var pairs = lines.Map(new ParseKeyValue(0, Options.PrintReceivedLines).Parse); var reducedStream = isReduceByKey ? pairs.ReduceByKey(Sum) : pairs.ReduceByKeyAndWindow(Sum, InverseSum, Options.WindowSeconds, Options.SlideSeconds); ForEachRDD("KeyValue", reducedStream, prefix, suffix); } else { //var pairs = lines.Map(line => new ParseKeyValueUnevenArray(elements).Parse(line)); var pairs = Options.IsUnevenArray ? lines.Map(new ParseKeyValueUnevenArray(elements, Options.PrintReceivedLines).Parse) : lines.Map(new ParseKeyValueArray(elements, Options.PrintReceivedLines).Parse); var reducedStream = isReduceByKey ? pairs.ReduceByKey(new ReduceHelper(Options.CheckArray).Sum) : pairs.ReduceByKeyAndWindow(new ReduceHelper(Options.CheckArray).Sum, new ReduceHelper(Options.CheckArray).InverseSum, Options.WindowSeconds, Options.SlideSeconds); ForEachRDD(Options.IsUnevenArray ? "KeyValueUnevenArray" : "KeyValueArray", reducedStream, prefix, suffix); } }