public static void ForEachRDD <V>(string title, DStream <KeyValuePair <string, V> > reducedStream, string prefix, string suffix = ".txt")
{
Logger.LogDebug("ForEachRDD " + title);
reducedStream.ForeachRDD(new SumCountStatic().ForeachRDD <V>);
if (!string.IsNullOrWhiteSpace(Options.SaveTxtDirectory))
{
reducedStream.Map(kv => $"{kv.Key} = {TestUtils.GetValueText(kv.Value)}").SaveAsTextFiles(Path.Combine(Options.SaveTxtDirectory, prefix), suffix);
}
}
static void StartOneTest(SparkContext sc, DStream <string> lines, long elements, string prefix, string suffix = ".txt")
{
var isReduceByKey = Options.IsReduceByKey();
Logger.LogDebug("isReduceByKey = {0}", isReduceByKey);
if (!Options.IsArrayValue)
{
//var pairs = lines.Map(line => new ParseKeyValue(0).Parse(line));
var pairs = lines.Map(new ParseKeyValue(0, Options.PrintReceivedLines).Parse);
var reducedStream = isReduceByKey ? pairs.ReduceByKey(Sum)
: pairs.ReduceByKeyAndWindow(Sum, InverseSum, Options.WindowSeconds, Options.SlideSeconds);
ForEachRDD("KeyValue", reducedStream, prefix, suffix);
}
else
{
//var pairs = lines.Map(line => new ParseKeyValueUnevenArray(elements).Parse(line));
var pairs = Options.IsUnevenArray ? lines.Map(new ParseKeyValueUnevenArray(elements, Options.PrintReceivedLines).Parse) : lines.Map(new ParseKeyValueArray(elements, Options.PrintReceivedLines).Parse);
var reducedStream = isReduceByKey ? pairs.ReduceByKey(new ReduceHelper(Options.CheckArray).Sum)
: pairs.ReduceByKeyAndWindow(new ReduceHelper(Options.CheckArray).Sum, new ReduceHelper(Options.CheckArray).InverseSum, Options.WindowSeconds, Options.SlideSeconds);
ForEachRDD(Options.IsUnevenArray ? "KeyValueUnevenArray" : "KeyValueArray", reducedStream, prefix, suffix);
}
}
