public void TestStreamingAwaitTimeout() { var ssc = new StreamingContext(new SparkContext("", ""), 1000); Assert.IsNotNull((ssc.streamingContextProxy as MockStreamingContextProxy)); ssc.Start(); ssc.Remember(1000); ssc.Checkpoint(Path.GetTempPath()); var textFile = ssc.TextFileStream(Path.GetTempPath()); Assert.IsNotNull(textFile.DStreamProxy); var socketStream = ssc.SocketTextStream(IPAddress.Loopback.ToString(), 12345); Assert.IsNotNull(socketStream.DStreamProxy); var union = ssc.Union(textFile, socketStream); Assert.IsNotNull(union.DStreamProxy); ssc.AwaitTerminationOrTimeout(3000); ssc.Stop(); }
public void TestStreamingContext() { var ssc = new StreamingContext(new SparkContext("", ""), 1); Assert.IsNotNull((ssc.streamingContextProxy as MockStreamingContextProxy)); ssc.Start(); ssc.Remember(1); ssc.Checkpoint(Path.GetTempPath()); var textFile = ssc.TextFileStream(Path.GetTempPath()); Assert.IsNotNull(textFile.DStreamProxy); var socketStream = ssc.SocketTextStream(IPAddress.Loopback.ToString(), 12345); Assert.IsNotNull(socketStream.DStreamProxy); var kafkaStream = KafkaUtils.CreateStream(ssc, IPAddress.Loopback + ":2181", "testGroupId", new Dictionary <string, int> { { "testTopic1", 1 } }, new Dictionary <string, string>()); Assert.IsNotNull(kafkaStream.DStreamProxy); var directKafkaStream = KafkaUtils.CreateDirectStream(ssc, new List <string> { "testTopic2" }, new Dictionary <string, string>(), new Dictionary <string, long>()); Assert.IsNotNull(directKafkaStream.DStreamProxy); var directKafkaStreamWithRepartition = KafkaUtils.CreateDirectStreamWithRepartition(ssc, new List <string> { "testTopic3" }, new Dictionary <string, string>(), new Dictionary <string, long>(), 10); Assert.IsNotNull(directKafkaStreamWithRepartition.DStreamProxy); var directKafkaStreamWithRepartitionAndReadFunc = KafkaUtils.CreateDirectStreamWithRepartitionAndReadFunc( ssc, new List <string> { "testTopic3" }, new Dictionary <string, string>(), new Dictionary <string, long>(), 10, (int pid, IEnumerable <KeyValuePair <byte[], byte[]> > input) => { return(input); }); Assert.IsNotNull(directKafkaStreamWithRepartitionAndReadFunc.DStreamProxy); var union = ssc.Union(textFile, socketStream); Assert.IsNotNull(union.DStreamProxy); ssc.AwaitTermination(); ssc.Stop(); }
public void TestStreamingContext() { var ssc = new StreamingContext(new SparkContext(Env.SPARK_MASTER_URL, "xxxx"), 1000); Assert.IsNotNull((ssc.streamingContextProxy as MockStreamingContextProxy)); ssc.Start(); ssc.Remember(1000); ssc.Checkpoint(Path.GetTempPath()); var textFile = ssc.TextFileStream(Path.GetTempPath()); Assert.IsNotNull(textFile.DStreamProxy); var socketStream = ssc.SocketTextStream("127.0.0.1", 12345); Assert.IsNotNull(socketStream.DStreamProxy); var kafkaStream = ssc.KafkaStream("127.0.0.1:2181", "testGroupId", new Dictionary <string, int> { { "testTopic1", 1 } }, new Dictionary <string, string>()); Assert.IsNotNull(kafkaStream.DStreamProxy); var directKafkaStream = ssc.DirectKafkaStream(new List <string> { "testTopic2" }, new Dictionary <string, string>(), new Dictionary <string, long>()); Assert.IsNotNull(directKafkaStream.DStreamProxy); var union = ssc.Union(textFile, socketStream); Assert.IsNotNull(union.DStreamProxy); ssc.AwaitTermination(); ssc.Stop(); }
public void TestStreamingContext() { var ssc = new StreamingContext(new SparkContext("", ""), 1000); Assert.IsNotNull((ssc.streamingContextProxy as MockStreamingContextProxy)); ssc.Start(); ssc.Remember(1000); ssc.Checkpoint(Path.GetTempPath()); var textFile = ssc.TextFileStream(Path.GetTempPath()); Assert.IsNotNull(textFile.DStreamProxy); var socketStream = ssc.SocketTextStream(IPAddress.Loopback.ToString(), 12345); Assert.IsNotNull(socketStream.DStreamProxy); var kafkaStream = KafkaUtils.CreateStream(ssc, IPAddress.Loopback + ":2181", "testGroupId", new Dictionary <string, int> { { "testTopic1", 1 } }, new Dictionary <string, string>()); Assert.IsNotNull(kafkaStream.DStreamProxy); var directKafkaStream = KafkaUtils.CreateDirectStream(ssc, new List <string> { "testTopic2" }, new Dictionary <string, string>(), new Dictionary <string, long>()); Assert.IsNotNull(directKafkaStream.DStreamProxy); var union = ssc.Union(textFile, socketStream); Assert.IsNotNull(union.DStreamProxy); ssc.AwaitTermination(); ssc.Stop(); }
public void TestStreamingContext() { var ssc = new StreamingContext(new SparkContext("", ""), 1000L); Assert.IsNotNull((ssc.streamingContextProxy as MockStreamingContextProxy)); ssc.Start(); ssc.Remember(1000L); ssc.Checkpoint(Path.GetTempPath()); var textFile = ssc.TextFileStream(Path.GetTempPath()); Assert.IsNotNull(textFile.DStreamProxy); var socketStream = ssc.SocketTextStream(IPAddress.Loopback.ToString(), 12345); Assert.IsNotNull(socketStream.DStreamProxy); var kafkaStream = KafkaUtils.CreateStream(ssc, IPAddress.Loopback + ":2181", "testGroupId", new [] { Tuple.Create("testTopic1", 1) }, null); Assert.IsNotNull(kafkaStream.DStreamProxy); var directKafkaStream = KafkaUtils.CreateDirectStream(ssc, new List <string> { "testTopic2" }, new List <Tuple <string, string> >(), new List <Tuple <string, long> >()); Assert.IsNotNull(directKafkaStream.DStreamProxy); ssc.SparkContext.SparkConf.Set("spark.mobius.streaming.kafka.numPartitions.testTopic3", "10"); var directKafkaStreamWithRepartition = KafkaUtils.CreateDirectStream(ssc, new List <string> { "testTopic3" }, new List <Tuple <string, string> >(), new List <Tuple <string, long> >()); Assert.IsNotNull(directKafkaStreamWithRepartition.DStreamProxy); var directKafkaStreamWithRepartitionAndReadFunc = KafkaUtils.CreateDirectStream( ssc, new List <string> { "testTopic3" }, new List <Tuple <string, string> >(), new List <Tuple <string, long> >(), (int pid, IEnumerable <Tuple <byte[], byte[]> > input) => { return(input); }); Assert.IsNotNull(directKafkaStreamWithRepartitionAndReadFunc); ssc.SparkContext.SparkConf.Set("spark.mobius.streaming.kafka.numReceivers", "10"); var directKafkaReceiver = KafkaUtils.CreateDirectStream( ssc, new List <string> { "testTopic3" }, new List <Tuple <string, string> >(), new List <Tuple <string, long> >(), (int pid, IEnumerable <Tuple <byte[], byte[]> > input) => { return(input); }); Assert.IsNotNull(directKafkaReceiver.DStreamProxy); var union = ssc.Union(textFile, socketStream); Assert.IsNotNull(union.DStreamProxy); ssc.AwaitTermination(); ssc.Stop(); }
public static void Main(string[] args) { Logger.LogInfo(EnvironmentInfo); var config = AppDomain.CurrentDomain.SetupInformation.ConfigurationFile; var isParseOK = false; //Options = ParserByCommandLine.Parse(args, out isParseOK); Options = ArgParser.Parse <ArgOptions>(args, out isParseOK, "-Help"); if (!isParseOK) { return; } Logger.LogDebug("{0} configuration {1}", File.Exists(config) ? "Exist" : "Not Exist", config); if (Options.WaitSecondsForAttachDebug > 0) { var waitBegin = DateTime.Now; var waitEnd = waitBegin + TimeSpan.FromSeconds(Options.WaitSecondsForAttachDebug); var currentPID = Process.GetCurrentProcess().Id; Logger.LogWarn($"Will wait {Options.WaitSecondsForAttachDebug} seconds for you to debug this process : please attach PID {currentPID} before {waitEnd}"); Thread.Sleep(Options.WaitSecondsForAttachDebug * 1000); } Logger.LogInfo("will connect " + Options.Host + ":" + Options.Port + " batchSeconds = " + Options.BatchSeconds + " s , windowSeconds = " + Options.WindowSeconds + " s, slideSeconds = " + Options.SlideSeconds + " s." + " checkpointDirectory = " + Options.CheckPointDirectory + ", is-array-test = " + Options.IsArrayValue); var prefix = ExeName + (Options.IsArrayValue ? "-array" + (Options.IsUnevenArray ? "-uneven" : "-even") : "-single"); var beginTime = DateTime.Now; var sc = new SparkContext(new SparkConf()); Action <long> testOneStreaming = (testTime) => { var timesInfo = "[" + testTime + "]-" + Options.TestTimes + " "; Logger.LogInfo($"Begin test{timesInfo} : {GetCurrentProcessInfo()}"); if (Options.DeleteCheckPointDirectoryTimes >= testTime) { TestUtils.DeleteDirectory(Options.CheckPointDirectory); } var ssc = new StreamingContext(sc, Options.BatchSeconds * 1000L); ssc.Checkpoint(Options.CheckPointDirectory); var lines = ssc.SocketTextStream(Options.Host, Options.Port, StorageLevelType.MEMORY_AND_DISK_SER); var oldSum = new SumCount(SumCountStatic.GetStaticSumCount()); StartOneTest(sc, lines, Options.ElementCount, prefix); var newSum = SumCountStatic.GetStaticSumCount(); // var sum = newSum - oldSum; // newSum maybe same as oldSum ssc.Start(); var startTime = DateTime.Now; ssc.AwaitTerminationOrTimeout(Options.RunningSeconds * 1000); ssc.Stop(); var sum = newSum - oldSum; var isSameLineCount = Options.LineCount <= 0 || Options.LineCount == sum.LineCount; var message = Options.LineCount <= 0 ? string.Empty : (isSameLineCount ? ". LineCount same" : string.Format(". LineCount different : expected = {0}, but line count = {1}", Options.LineCount, sum.LineCount)); Logger.LogInfo("oldSum = {0}, newSum = {1}, sum = {2}", oldSum, newSum, sum); Logger.LogInfo($"End test{timesInfo}, used time = {(DateTime.Now - startTime).TotalSeconds} s, total cost = {(DateTime.Now - beginTime).TotalSeconds} s, started at {startTime.ToString(TestUtils.MilliTimeFormat)} . Reduced final sumCount : {sum.ToString()} {message}. {GetCurrentProcessInfo()}"); }; for (var times = 1; times <= Options.TestTimes; times++) { testOneStreaming(times); if (times < Options.TestTimes) { Thread.Sleep(TimeSpan.FromSeconds(Options.TestIntervalSeconds)); } } Logger.LogInfo($"Finished all tests, test times = {Options.TestTimes}, used time = {(DateTime.Now - beginTime).TotalSeconds} s = {DateTime.Now - beginTime} . {GetCurrentProcessInfo(true, "Final info: ")}"); }