Beispiel #1
0
        static void Main(string[] args)
        {
            Logger.LogInfo(EnvironmentInfo);

            var config = AppDomain.CurrentDomain.SetupInformation.ConfigurationFile;

            if (args.Length < 1 || args[0] == "-h" || args[0] == "--help")
            {
                ShowUsage();
                return;
            }

            Logger.LogDebug("{0} configuration {1}", File.Exists(config) ? "Exist" : "Not Exist", config);

            var className = args[0];
            var type      = TestClasses.Find(tp => tp.Name.Equals(className, StringComparison.OrdinalIgnoreCase));

            Logger.LogDebug($"Test class = {type}");
            if (type == null)
            {
                Logger.LogWarn($"Please use one test-name as first parameter : {string.Join(", ", TestClasses.Select(tp => tp.Name))} ");
                ShowUsage();
                return;
            }

            var kafkaTest = Activator.CreateInstance(type) as ITestKafkaBase;

            var testArgs = new List <string>(args);

            testArgs.RemoveAt(0);
            if (testArgs.Count == 0)
            {
                testArgs.Add("-h");
            }

            var testTimesAndInterval = kafkaTest.GetTestTimesAndInterval(testArgs.ToArray());
            var testTimes            = testTimesAndInterval.Item1;
            var allBeginTime         = DateTime.Now;

            for (var t = 1; t <= testTimes; t++)
            {
                SumCountStatic.GetStaticSumCount().Set();
                var sparkContext = new Lazy <SparkContext>(() => new SparkContext(new SparkConf()));
                var beginTime    = DateTime.Now;
                Logger.LogInfo($"Begin test[{t}]-{testTimes} of {type.Name} : {GetCurrentProcessInfo()}");
                kafkaTest.Run(sparkContext, t, testTimes);
                var usedTime = DateTime.Now - beginTime;
                Logger.LogInfo($"End test[{t}]-{testTimes} of {type.Name}, used time = {usedTime.TotalSeconds} s = {usedTime} . SumCount : {SumCountStatic.GetStaticSumCount().ToString()}. {GetCurrentProcessInfo()}");
                if (t < testTimes)
                {
                    Thread.Sleep(TimeSpan.FromSeconds(testTimesAndInterval.Item2));
                }
            }

            var totalUsedTime = DateTime.Now - allBeginTime;

            Logger.LogInfo($"Finished all tests of {type.Name}, test times = {testTimes}, used time = {totalUsedTime.TotalSeconds} s = {totalUsedTime}. {GetCurrentProcessInfo(true, "Final Info: ")}");
        }
        public static void Main(string[] args)
        {
            Logger.LogInfo(EnvironmentInfo);
            var config = AppDomain.CurrentDomain.SetupInformation.ConfigurationFile;

            var isParseOK = false;

            //Options = ParserByCommandLine.Parse(args, out isParseOK);
            Options = ArgParser.Parse <ArgOptions>(args, out isParseOK, "-Help");

            if (!isParseOK)
            {
                return;
            }

            Logger.LogDebug("{0} configuration {1}", File.Exists(config) ? "Exist" : "Not Exist", config);

            if (Options.WaitSecondsForAttachDebug > 0)
            {
                var waitBegin  = DateTime.Now;
                var waitEnd    = waitBegin + TimeSpan.FromSeconds(Options.WaitSecondsForAttachDebug);
                var currentPID = Process.GetCurrentProcess().Id;
                Logger.LogWarn($"Will wait {Options.WaitSecondsForAttachDebug} seconds for you to debug this process : please attach PID {currentPID} before {waitEnd}");
                Thread.Sleep(Options.WaitSecondsForAttachDebug * 1000);
            }

            Logger.LogInfo("will connect " + Options.Host + ":" + Options.Port + " batchSeconds = " + Options.BatchSeconds + " s , windowSeconds = " + Options.WindowSeconds + " s, slideSeconds = " + Options.SlideSeconds + " s."
                           + " checkpointDirectory = " + Options.CheckPointDirectory + ", is-array-test = " + Options.IsArrayValue);

            var prefix = ExeName + (Options.IsArrayValue ? "-array" + (Options.IsUnevenArray ? "-uneven" : "-even") : "-single");

            var beginTime = DateTime.Now;

            var sc = new SparkContext(new SparkConf());

            Action <long> testOneStreaming = (testTime) =>
            {
                var timesInfo = "[" + testTime + "]-" + Options.TestTimes + " ";
                Logger.LogInfo($"Begin test{timesInfo} : {GetCurrentProcessInfo()}");
                if (Options.DeleteCheckPointDirectoryTimes >= testTime)
                {
                    TestUtils.DeleteDirectory(Options.CheckPointDirectory);
                }

                var ssc = new StreamingContext(sc, Options.BatchSeconds * 1000L);
                ssc.Checkpoint(Options.CheckPointDirectory);
                var lines = ssc.SocketTextStream(Options.Host, Options.Port, StorageLevelType.MEMORY_AND_DISK_SER);


                var oldSum = new SumCount(SumCountStatic.GetStaticSumCount());
                StartOneTest(sc, lines, Options.ElementCount, prefix);
                var newSum = SumCountStatic.GetStaticSumCount();
                // var sum = newSum - oldSum; // newSum maybe same as oldSum

                ssc.Start();
                var startTime = DateTime.Now;
                ssc.AwaitTerminationOrTimeout(Options.RunningSeconds * 1000);
                ssc.Stop();

                var sum             = newSum - oldSum;
                var isSameLineCount = Options.LineCount <= 0 || Options.LineCount == sum.LineCount;
                var message         = Options.LineCount <= 0 ? string.Empty :
                                      (isSameLineCount ? ". LineCount same" : string.Format(". LineCount different : expected = {0}, but line count = {1}", Options.LineCount, sum.LineCount));

                Logger.LogInfo("oldSum = {0}, newSum = {1}, sum = {2}", oldSum, newSum, sum);
                Logger.LogInfo($"End test{timesInfo}, used time = {(DateTime.Now - startTime).TotalSeconds} s, total cost = {(DateTime.Now - beginTime).TotalSeconds} s, started at {startTime.ToString(TestUtils.MilliTimeFormat)} . Reduced final sumCount : {sum.ToString()} {message}. {GetCurrentProcessInfo()}");
            };

            for (var times = 1; times <= Options.TestTimes; times++)
            {
                testOneStreaming(times);
                if (times < Options.TestTimes)
                {
                    Thread.Sleep(TimeSpan.FromSeconds(Options.TestIntervalSeconds));
                }
            }

            Logger.LogInfo($"Finished all tests, test times = {Options.TestTimes}, used time = {(DateTime.Now - beginTime).TotalSeconds} s = {DateTime.Now - beginTime} . {GetCurrentProcessInfo(true, "Final info: ")}");
        }