public void TestStreamingAwaitTimeout()
{
var ssc = new StreamingContext(new SparkContext("", ""), 1000);
Assert.IsNotNull((ssc.streamingContextProxy as MockStreamingContextProxy));
ssc.Start();
ssc.Remember(1000);
ssc.Checkpoint(Path.GetTempPath());
var textFile = ssc.TextFileStream(Path.GetTempPath());
Assert.IsNotNull(textFile.DStreamProxy);
var socketStream = ssc.SocketTextStream(IPAddress.Loopback.ToString(), 12345);
Assert.IsNotNull(socketStream.DStreamProxy);
var union = ssc.Union(textFile, socketStream);
Assert.IsNotNull(union.DStreamProxy);
ssc.AwaitTerminationOrTimeout(3000);
ssc.Stop();
}
public void TestStreamingContext()
{
var ssc = new StreamingContext(new SparkContext("", ""), 1);
Assert.IsNotNull((ssc.streamingContextProxy as MockStreamingContextProxy));
ssc.Start();
ssc.Remember(1);
ssc.Checkpoint(Path.GetTempPath());
var textFile = ssc.TextFileStream(Path.GetTempPath());
Assert.IsNotNull(textFile.DStreamProxy);
var socketStream = ssc.SocketTextStream(IPAddress.Loopback.ToString(), 12345);
Assert.IsNotNull(socketStream.DStreamProxy);
var kafkaStream = KafkaUtils.CreateStream(ssc, IPAddress.Loopback + ":2181", "testGroupId", new Dictionary <string, int> {
{ "testTopic1", 1 }
}, new Dictionary <string, string>());
Assert.IsNotNull(kafkaStream.DStreamProxy);
var directKafkaStream = KafkaUtils.CreateDirectStream(ssc, new List <string> {
"testTopic2"
}, new Dictionary <string, string>(), new Dictionary <string, long>());
Assert.IsNotNull(directKafkaStream.DStreamProxy);
var directKafkaStreamWithRepartition = KafkaUtils.CreateDirectStreamWithRepartition(ssc, new List <string> {
"testTopic3"
}, new Dictionary <string, string>(), new Dictionary <string, long>(), 10);
Assert.IsNotNull(directKafkaStreamWithRepartition.DStreamProxy);
var directKafkaStreamWithRepartitionAndReadFunc = KafkaUtils.CreateDirectStreamWithRepartitionAndReadFunc(
ssc,
new List <string> {
"testTopic3"
},
new Dictionary <string, string>(), new Dictionary <string, long>(),
10,
(int pid, IEnumerable <KeyValuePair <byte[], byte[]> > input) => { return(input); });
Assert.IsNotNull(directKafkaStreamWithRepartitionAndReadFunc.DStreamProxy);
var union = ssc.Union(textFile, socketStream);
Assert.IsNotNull(union.DStreamProxy);
ssc.AwaitTermination();
ssc.Stop();
}
public void TestStreamingContext()
{
var ssc = new StreamingContext(new SparkContext(Env.SPARK_MASTER_URL, "xxxx"), 1000);
Assert.IsNotNull((ssc.streamingContextProxy as MockStreamingContextProxy));
ssc.Start();
ssc.Remember(1000);
ssc.Checkpoint(Path.GetTempPath());
var textFile = ssc.TextFileStream(Path.GetTempPath());
Assert.IsNotNull(textFile.DStreamProxy);
var socketStream = ssc.SocketTextStream("127.0.0.1", 12345);
Assert.IsNotNull(socketStream.DStreamProxy);
var kafkaStream = ssc.KafkaStream("127.0.0.1:2181", "testGroupId", new Dictionary <string, int> {
{ "testTopic1", 1 }
}, new Dictionary <string, string>());
Assert.IsNotNull(kafkaStream.DStreamProxy);
var directKafkaStream = ssc.DirectKafkaStream(new List <string> {
"testTopic2"
}, new Dictionary <string, string>(), new Dictionary <string, long>());
Assert.IsNotNull(directKafkaStream.DStreamProxy);
var union = ssc.Union(textFile, socketStream);
Assert.IsNotNull(union.DStreamProxy);
ssc.AwaitTermination();
ssc.Stop();
}
public void TestStreamingContext()
{
var ssc = new StreamingContext(new SparkContext("", ""), 1000);
Assert.IsNotNull((ssc.streamingContextProxy as MockStreamingContextProxy));
ssc.Start();
ssc.Remember(1000);
ssc.Checkpoint(Path.GetTempPath());
var textFile = ssc.TextFileStream(Path.GetTempPath());
Assert.IsNotNull(textFile.DStreamProxy);
var socketStream = ssc.SocketTextStream(IPAddress.Loopback.ToString(), 12345);
Assert.IsNotNull(socketStream.DStreamProxy);
var kafkaStream = KafkaUtils.CreateStream(ssc, IPAddress.Loopback + ":2181", "testGroupId", new Dictionary <string, int> {
{ "testTopic1", 1 }
}, new Dictionary <string, string>());
Assert.IsNotNull(kafkaStream.DStreamProxy);
var directKafkaStream = KafkaUtils.CreateDirectStream(ssc, new List <string> {
"testTopic2"
}, new Dictionary <string, string>(), new Dictionary <string, long>());
Assert.IsNotNull(directKafkaStream.DStreamProxy);
var union = ssc.Union(textFile, socketStream);
Assert.IsNotNull(union.DStreamProxy);
ssc.AwaitTermination();
ssc.Stop();
}
public void TestStreamingContext()
{
var ssc = new StreamingContext(new SparkContext("", ""), 1000L);
Assert.IsNotNull((ssc.streamingContextProxy as MockStreamingContextProxy));
ssc.Start();
ssc.Remember(1000L);
ssc.Checkpoint(Path.GetTempPath());
var textFile = ssc.TextFileStream(Path.GetTempPath());
Assert.IsNotNull(textFile.DStreamProxy);
var socketStream = ssc.SocketTextStream(IPAddress.Loopback.ToString(), 12345);
Assert.IsNotNull(socketStream.DStreamProxy);
var kafkaStream = KafkaUtils.CreateStream(ssc, IPAddress.Loopback + ":2181", "testGroupId", new [] { Tuple.Create("testTopic1", 1) }, null);
Assert.IsNotNull(kafkaStream.DStreamProxy);
var directKafkaStream = KafkaUtils.CreateDirectStream(ssc, new List <string> {
"testTopic2"
}, new List <Tuple <string, string> >(), new List <Tuple <string, long> >());
Assert.IsNotNull(directKafkaStream.DStreamProxy);
ssc.SparkContext.SparkConf.Set("spark.mobius.streaming.kafka.numPartitions.testTopic3", "10");
var directKafkaStreamWithRepartition = KafkaUtils.CreateDirectStream(ssc, new List <string> {
"testTopic3"
}, new List <Tuple <string, string> >(), new List <Tuple <string, long> >());
Assert.IsNotNull(directKafkaStreamWithRepartition.DStreamProxy);
var directKafkaStreamWithRepartitionAndReadFunc = KafkaUtils.CreateDirectStream(
ssc,
new List <string> {
"testTopic3"
},
new List <Tuple <string, string> >(), new List <Tuple <string, long> >(),
(int pid, IEnumerable <Tuple <byte[], byte[]> > input) => { return(input); });
Assert.IsNotNull(directKafkaStreamWithRepartitionAndReadFunc);
ssc.SparkContext.SparkConf.Set("spark.mobius.streaming.kafka.numReceivers", "10");
var directKafkaReceiver = KafkaUtils.CreateDirectStream(
ssc,
new List <string> {
"testTopic3"
},
new List <Tuple <string, string> >(), new List <Tuple <string, long> >(),
(int pid, IEnumerable <Tuple <byte[], byte[]> > input) => { return(input); });
Assert.IsNotNull(directKafkaReceiver.DStreamProxy);
var union = ssc.Union(textFile, socketStream);
Assert.IsNotNull(union.DStreamProxy);
ssc.AwaitTermination();
ssc.Stop();
}
public static void Main(string[] args)
{
Logger.LogInfo(EnvironmentInfo);
var config = AppDomain.CurrentDomain.SetupInformation.ConfigurationFile;
var isParseOK = false;
//Options = ParserByCommandLine.Parse(args, out isParseOK);
Options = ArgParser.Parse <ArgOptions>(args, out isParseOK, "-Help");
if (!isParseOK)
{
return;
}
Logger.LogDebug("{0} configuration {1}", File.Exists(config) ? "Exist" : "Not Exist", config);
if (Options.WaitSecondsForAttachDebug > 0)
{
var waitBegin = DateTime.Now;
var waitEnd = waitBegin + TimeSpan.FromSeconds(Options.WaitSecondsForAttachDebug);
var currentPID = Process.GetCurrentProcess().Id;
Logger.LogWarn($"Will wait {Options.WaitSecondsForAttachDebug} seconds for you to debug this process : please attach PID {currentPID} before {waitEnd}");
Thread.Sleep(Options.WaitSecondsForAttachDebug * 1000);
}
Logger.LogInfo("will connect " + Options.Host + ":" + Options.Port + " batchSeconds = " + Options.BatchSeconds + " s , windowSeconds = " + Options.WindowSeconds + " s, slideSeconds = " + Options.SlideSeconds + " s."
+ " checkpointDirectory = " + Options.CheckPointDirectory + ", is-array-test = " + Options.IsArrayValue);
var prefix = ExeName + (Options.IsArrayValue ? "-array" + (Options.IsUnevenArray ? "-uneven" : "-even") : "-single");
var beginTime = DateTime.Now;
var sc = new SparkContext(new SparkConf());
Action <long> testOneStreaming = (testTime) =>
{
var timesInfo = "[" + testTime + "]-" + Options.TestTimes + " ";
Logger.LogInfo($"Begin test{timesInfo} : {GetCurrentProcessInfo()}");
if (Options.DeleteCheckPointDirectoryTimes >= testTime)
{
TestUtils.DeleteDirectory(Options.CheckPointDirectory);
}
var ssc = new StreamingContext(sc, Options.BatchSeconds * 1000L);
ssc.Checkpoint(Options.CheckPointDirectory);
var lines = ssc.SocketTextStream(Options.Host, Options.Port, StorageLevelType.MEMORY_AND_DISK_SER);
var oldSum = new SumCount(SumCountStatic.GetStaticSumCount());
StartOneTest(sc, lines, Options.ElementCount, prefix);
var newSum = SumCountStatic.GetStaticSumCount();
// var sum = newSum - oldSum; // newSum maybe same as oldSum
ssc.Start();
var startTime = DateTime.Now;
ssc.AwaitTerminationOrTimeout(Options.RunningSeconds * 1000);
ssc.Stop();
var sum = newSum - oldSum;
var isSameLineCount = Options.LineCount <= 0 || Options.LineCount == sum.LineCount;
var message = Options.LineCount <= 0 ? string.Empty :
(isSameLineCount ? ". LineCount same" : string.Format(". LineCount different : expected = {0}, but line count = {1}", Options.LineCount, sum.LineCount));
Logger.LogInfo("oldSum = {0}, newSum = {1}, sum = {2}", oldSum, newSum, sum);
Logger.LogInfo($"End test{timesInfo}, used time = {(DateTime.Now - startTime).TotalSeconds} s, total cost = {(DateTime.Now - beginTime).TotalSeconds} s, started at {startTime.ToString(TestUtils.MilliTimeFormat)} . Reduced final sumCount : {sum.ToString()} {message}. {GetCurrentProcessInfo()}");
};
for (var times = 1; times <= Options.TestTimes; times++)
{
testOneStreaming(times);
if (times < Options.TestTimes)
{
Thread.Sleep(TimeSpan.FromSeconds(Options.TestIntervalSeconds));
}
}
Logger.LogInfo($"Finished all tests, test times = {Options.TestTimes}, used time = {(DateTime.Now - beginTime).TotalSeconds} s = {DateTime.Now - beginTime} . {GetCurrentProcessInfo(true, "Final info: ")}");
}
