protected void SaveStreamToFile <U>(DStream <U> reducedStream)
{
if (!string.IsNullOrWhiteSpace(Options.SaveTxtDirectory))
{
reducedStream.SaveAsTextFiles(Path.Combine(Options.SaveTxtDirectory, typeof(ClassName).Name), ".txt");
}
}
public static void ForEachRDD <V>(string title, DStream <KeyValuePair <string, V> > reducedStream, string prefix, string suffix = ".txt")
{
Logger.LogDebug("ForEachRDD " + title);
reducedStream.ForeachRDD(new SumCountStatic().ForeachRDD <V>);
if (!string.IsNullOrWhiteSpace(Options.SaveTxtDirectory))
{
reducedStream.Map(kv => $"{kv.Key} = {TestUtils.GetValueText(kv.Value)}").SaveAsTextFiles(Path.Combine(Options.SaveTxtDirectory, prefix), suffix);
}
}
public static byte[] ObjectToBinary(Object obj, Boolean compress)
{
MemoryStream MemStream = null;
DeflateStream DStream = null;
MemoryStream OutStream = null;
BinaryFormatter BinFormatter = default(BinaryFormatter);
byte[] B = null;
try
{
MemStream = new MemoryStream();
BinFormatter = new BinaryFormatter();
BinFormatter.Serialize(MemStream, obj);
if (compress)
{
B = MemStream.ToArray();
OutStream = new MemoryStream();
DStream = new DeflateStream(OutStream, CompressionMode.Compress, false);
DStream.Write(MemStream.ToArray(), 0, B.Length);
DStream.Flush();
B = OutStream.ToArray();
}
else
{
B = MemStream.ToArray();
}
return(B);
}
finally
{
if ((OutStream != null))
{
OutStream.Dispose();
}
if ((DStream != null))
{
DStream.Dispose();
}
if ((MemStream != null))
{
MemStream.Dispose();
}
}
}
public static bool BinaryToObject(byte[] Data, ref object obj, bool IsCompressed)
{
obj = null;
MemoryStream MemStream = null;
MemoryStream InputStream = null;
DeflateStream DStream = null;
BinaryFormatter BinFormatter = default(BinaryFormatter);
byte[] B = null;
try
{
MemStream = new MemoryStream(Data);
BinFormatter = new BinaryFormatter();
BinFormatter.AssemblyFormat = System.Runtime.Serialization.Formatters.FormatterAssemblyStyle.Simple;
if (IsCompressed)
{
DStream = new DeflateStream(MemStream, CompressionMode.Decompress, false);
B = ReadFullStream(DStream);
DStream.Flush();
InputStream = new MemoryStream(B);
obj = BinFormatter.Deserialize(InputStream);
}
else
{
obj = BinFormatter.Deserialize(MemStream);
}
return(true);
}
finally
{
if ((DStream != null))
{
DStream.Dispose();
}
if ((InputStream != null))
{
InputStream.Dispose();
}
if ((MemStream != null))
{
MemStream.Dispose();
}
}
}
public void TestDStreamMapWithState()
{
var mapwithStateDStreamProxy = new Mock <IDStreamProxy>();
var streamingContextProxy = new Mock <IStreamingContextProxy>();
streamingContextProxy.Setup(p =>
p.CreateCSharpStateDStream(It.IsAny <IDStreamProxy>(), It.IsAny <byte[]>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>()))
.Returns(mapwithStateDStreamProxy.Object);
var sparkContextProxy = new Mock <ISparkContextProxy>();
var sparkConfProxy = new Mock <ISparkConfProxy>();
var sparkClrProxy = new Mock <ISparkCLRProxy>();
sparkClrProxy.Setup(p => p.StreamingContextProxy).Returns(streamingContextProxy.Object);
sparkClrProxy.Setup(p => p.SparkContextProxy).Returns(sparkContextProxy.Object);
sparkClrProxy.Setup(p => p.CreateSparkContext(It.IsAny <ISparkConfProxy>())).Returns(sparkContextProxy.Object);
sparkClrProxy.Setup(p => p.CreateSparkConf(It.IsAny <bool>())).Returns(sparkConfProxy.Object);
// reset sparkCLRProxy for after test completes
var originalSparkCLRProxy = SparkCLREnvironment.SparkCLRProxy;
try
{
SparkCLREnvironment.SparkCLRProxy = sparkClrProxy.Object;
var sparkConf = new SparkConf(false);
var ssc = new StreamingContext(new SparkContext(sparkContextProxy.Object, sparkConf), 10);
var dstreamProxy = new Mock <IDStreamProxy>();
var pairDStream = new DStream <KeyValuePair <string, int> >(dstreamProxy.Object, ssc);
var stateSpec = new StateSpec <string, int, int, int>((k, v, s) => v);
var stateDStream = pairDStream.MapWithState(stateSpec);
var snapshotDStream = stateDStream.StateSnapshots();
Assert.IsNotNull(stateDStream);
Assert.IsNotNull(snapshotDStream);
}
finally
{
SparkCLREnvironment.SparkCLRProxy = originalSparkCLRProxy;
}
}
static void StartOneTest(SparkContext sc, DStream <string> lines, long elements, string prefix, string suffix = ".txt")
{
var isReduceByKey = Options.IsReduceByKey();
Logger.LogDebug("isReduceByKey = {0}", isReduceByKey);
if (!Options.IsArrayValue)
{
//var pairs = lines.Map(line => new ParseKeyValue(0).Parse(line));
var pairs = lines.Map(new ParseKeyValue(0, Options.PrintReceivedLines).Parse);
var reducedStream = isReduceByKey ? pairs.ReduceByKey(Sum)
: pairs.ReduceByKeyAndWindow(Sum, InverseSum, Options.WindowSeconds, Options.SlideSeconds);
ForEachRDD("KeyValue", reducedStream, prefix, suffix);
}
else
{
//var pairs = lines.Map(line => new ParseKeyValueUnevenArray(elements).Parse(line));
var pairs = Options.IsUnevenArray ? lines.Map(new ParseKeyValueUnevenArray(elements, Options.PrintReceivedLines).Parse) : lines.Map(new ParseKeyValueArray(elements, Options.PrintReceivedLines).Parse);
var reducedStream = isReduceByKey ? pairs.ReduceByKey(new ReduceHelper(Options.CheckArray).Sum)
: pairs.ReduceByKeyAndWindow(new ReduceHelper(Options.CheckArray).Sum, new ReduceHelper(Options.CheckArray).InverseSum, Options.WindowSeconds, Options.SlideSeconds);
ForEachRDD(Options.IsUnevenArray ? "KeyValueUnevenArray" : "KeyValueArray", reducedStream, prefix, suffix);
}
}
public void TestDStreamMapWithState()
{
var mapwithStateDStreamProxy = new Mock<IDStreamProxy>();
var streamingContextProxy = new Mock<IStreamingContextProxy>();
streamingContextProxy.Setup(p =>
p.CreateCSharpStateDStream(It.IsAny<IDStreamProxy>(), It.IsAny<byte[]>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>()))
.Returns(mapwithStateDStreamProxy.Object);
var sparkContextProxy = new Mock<ISparkContextProxy>();
var sparkConfProxy = new Mock<ISparkConfProxy>();
var sparkClrProxy = new Mock<ISparkCLRProxy>();
sparkClrProxy.Setup(p => p.StreamingContextProxy).Returns(streamingContextProxy.Object);
sparkClrProxy.Setup(p => p.SparkContextProxy).Returns(sparkContextProxy.Object);
sparkClrProxy.Setup(p => p.CreateSparkContext(It.IsAny<ISparkConfProxy>())).Returns(sparkContextProxy.Object);
sparkClrProxy.Setup(p => p.CreateSparkConf(It.IsAny<bool>())).Returns(sparkConfProxy.Object);
// reset sparkCLRProxy for after test completes
var originalSparkCLRProxy = SparkCLREnvironment.SparkCLRProxy;
try
{
SparkCLREnvironment.SparkCLRProxy = sparkClrProxy.Object;
var sparkConf = new SparkConf(false);
var ssc = new StreamingContext(new SparkContext(sparkContextProxy.Object, sparkConf), 10);
var dstreamProxy = new Mock<IDStreamProxy>();
var pairDStream = new DStream<KeyValuePair<string, int>>(dstreamProxy.Object, ssc);
var stateSpec = new StateSpec<string, int, int, int>((k, v, s) => v);
var stateDStream = pairDStream.MapWithState(stateSpec);
var snapshotDStream = stateDStream.StateSnapshots();
Assert.IsNotNull(stateDStream);
Assert.IsNotNull(snapshotDStream);
}
finally
{
SparkCLREnvironment.SparkCLRProxy = originalSparkCLRProxy;
}
}
