public void MemorySampleEvent(string payload, bool expectEvent, long expectedMemoryUtil, long expectedTotalUtil)
        {
            var processingNotificationsCollector = new ProcessingNotificationsCollector(10);

            using (var eventProcessor = new ResourceManagerEventsProcessor(_testWriterFactory, processingNotificationsCollector))
            {
                eventProcessor.ProcessEvent(_testBaseEvent, payload, _testLogLine, TestProcessName);
            }

            var memorySampleWriter = _testWriterFactory.GetOneWriterAndVerifyOthersAreEmptyAndDisposed <ResourceManagerMemorySample>("ResourceManagerMemorySamples", 5);

            memorySampleWriter.ReceivedObjects.Count.Should().Be(expectEvent ? 1 : 0);
            processingNotificationsCollector.TotalErrorsReported.Should().Be(expectEvent ? 0 : 1);
            if (expectEvent)
            {
                var expectedRecord = new
                {
                    FileName   = TestLogFileInfo.FileName,
                    FilePath   = TestLogFileInfo.FilePath,
                    LineNumber = _testLogLine.LineNumber,
                    Timestamp  = _testBaseEvent.Timestamp,
                    Worker     = TestLogFileInfo.Worker,

                    ProcessId    = _testBaseEvent.ProcessId,
                    ProcessIndex = (int?)null,
                    ProcessName  = TestProcessName,

                    ProcessMemoryUtil = expectedMemoryUtil,
                    TotalMemoryUtil   = expectedTotalUtil,
                };
                memorySampleWriter.ReceivedObjects[0].Should().BeEquivalentTo(expectedRecord);
            }
        }
Пример #2
0
        public void BadOrNoOpInput()
        {
            var testWriterFactory = new TestWriterFactory();
            var processingNotificationsCollector = new ProcessingNotificationsCollector(10);

            using (var plugin = new LogShark.Plugins.ResourceManager.ResourceManagerPlugin())
            {
                plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());

                var wrongContentFormat = new LogLine(new ReadLogLineResult(123, "ResourceManagerPlugin doesn't expect string"), VizqlServerCppFileInfo);
                var nullContent        = new LogLine(new ReadLogLineResult(123, null), VizqlServerCppFileInfo);
                var nonMsgContent      = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
                    EventType = "something else"
                }), VizqlServerCppFileInfo);
                var payloadIsNotAString = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
                    EventType = "msg", EventPayload = JToken.FromObject(new { Key = "value" })
                }), VizqlServerCppFileInfo);
                var vizqlAndSrmInternal = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
                    EventType = "srm-internal", EventPayload = "Resource Manager: CPU info: 0%"
                }), VizqlServerCppFileInfo);
                var hyperAndMessage = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
                    EventType = "msg", EventPayload = "Resource Manager: CPU info: 0%"
                }), HyperFileInfo);

                plugin.ProcessLogLine(wrongContentFormat, LogType.VizqlserverCpp);
                plugin.ProcessLogLine(nullContent, LogType.VizqlserverCpp);
                plugin.ProcessLogLine(nonMsgContent, LogType.VizqlserverCpp);
                plugin.ProcessLogLine(payloadIsNotAString, LogType.VizqlserverCpp);
                plugin.ProcessLogLine(vizqlAndSrmInternal, LogType.VizqlserverCpp);
                plugin.ProcessLogLine(hyperAndMessage, LogType.Hyper);
            }

            testWriterFactory.AssertAllWritersAreDisposedAndEmpty(5);
            processingNotificationsCollector.TotalErrorsReported.Should().Be(2);
        }
        public void BadAndNoOpInput()
        {
            var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
            var testWriterFactory = new TestWriterFactory();

            using (var plugin = new ServerTelemetryPlugin())
            {
                plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());

                var wrongContentFormat = new LogLine(new ReadLogLineResult(123, "ServerTelemetry doesn't expect string"), TestLogFileInfo);
                var nullContent        = new LogLine(new ReadLogLineResult(123, null), TestLogFileInfo);
                var differentEventType = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
                    EventType = "msg"
                }), TestLogFileInfo);                                                                                                                // This doesn't generate error, as it is a normal condition
                var payloadIsNull = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
                    EventType = "server-telemetry"
                }), TestLogFileInfo);

                plugin.ProcessLogLine(wrongContentFormat, LogType.VizqlserverCpp);
                plugin.ProcessLogLine(nullContent, LogType.VizqlserverCpp);
                plugin.ProcessLogLine(differentEventType, LogType.VizqlserverCpp);
                plugin.ProcessLogLine(payloadIsNull, LogType.VizqlserverCpp);
            }

            testWriterFactory.AssertAllWritersAreDisposedAndEmpty(2);
            processingNotificationsCollector.TotalErrorsReported.Should().Be(3);
        }
Пример #4
0
        public void BadOrNoOpInput()
        {
            var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
            var testWriterFactory = new TestWriterFactory();

            using (var plugin = new ArtPlugin())
            {
                plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());

                var wrongContentFormat = new LogLine(new ReadLogLineResult(123, "Art doesn't expect string"), TestLogFileInfo);
                var nullContent        = new LogLine(new ReadLogLineResult(123, null), TestLogFileInfo);
                var payloadIsNull      = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
                    EventType = "msg", ArtData = null
                }), TestLogFileInfo);
                var incorrectArtPayload = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
                    EventType = "msg", ArtData = JToken.FromObject("JSON was expected here")
                }), TestLogFileInfo);

                plugin.ProcessLogLine(wrongContentFormat, LogType.VizqlserverCpp);
                plugin.ProcessLogLine(nullContent, LogType.VizqlserverCpp);
                plugin.ProcessLogLine(payloadIsNull, LogType.VizqlserverCpp);
                plugin.ProcessLogLine(incorrectArtPayload, LogType.VizqlserverCpp);
            }

            testWriterFactory.AssertAllWritersAreDisposedAndEmpty(2);
            processingNotificationsCollector.TotalErrorsReported.Should().Be(3);
        }
Пример #5
0
 private static void AppendDetailedErrorsToStringBuilder(ProcessingNotificationsCollector errorsCollector, StringBuilder report)
 {
     foreach (var error in errorsCollector.ProcessingErrorsDetails ?? new List <ProcessingNotification>())
     {
         report.AppendLine($"--> File: `{error.FilePath ?? "(null)"}`. Line: {error.LineNumber}. Reported by: `{error.ReportedBy ?? "(null)"}`. Error: `{error.Message ?? "(null)"}`");
     }
 }
        public void BadInput()
        {
            var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
            var testWriterFactory = new TestWriterFactory();

            using (var plugin = new ClusterControllerPlugin())
            {
                plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());

                var ccWrongContentFormat = new LogLine(new ReadLogLineResult(123, 1234), _testClusterControllerLogFileInfo);
                var ccNullContent        = new LogLine(new ReadLogLineResult(123, null), _testClusterControllerLogFileInfo);
                var ccIncorrectString    = new LogLine(new ReadLogLineResult(123, "I am not a Java log!"), _testClusterControllerLogFileInfo);
                var zkWrongContentFormat = new LogLine(new ReadLogLineResult(123, 1234), _testZookeeperLogFileInfo);
                var zkNullContent        = new LogLine(new ReadLogLineResult(123, null), _testZookeeperLogFileInfo);
                var zkIncorrectString    = new LogLine(new ReadLogLineResult(123, "I am not a Java log!"), _testZookeeperLogFileInfo);

                plugin.ProcessLogLine(ccWrongContentFormat, LogType.ClusterController);
                plugin.ProcessLogLine(ccNullContent, LogType.ClusterController);
                plugin.ProcessLogLine(ccIncorrectString, LogType.ClusterController);
                plugin.ProcessLogLine(zkWrongContentFormat, LogType.Zookeeper);
                plugin.ProcessLogLine(zkNullContent, LogType.Zookeeper);
                plugin.ProcessLogLine(zkIncorrectString, LogType.Zookeeper);
            }

            testWriterFactory.AssertAllWritersAreDisposedAndEmpty(_testWriterCount);
            processingNotificationsCollector.TotalErrorsReported.Should().Be(_testWriterCount);
        }
Пример #7
0
        public void ValidLinesThatAreNotPersisted(string input)
        {
            var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
            var parser = new BackgrounderEventParser(_persisterMock.Object, processingNotificationsCollector);

            var logLine = _startTestCases[0].GetLogLine();

            parser.ParseAndPersistLine(logLine, input);

            _persisterMock.VerifyNoOtherCalls();
            processingNotificationsCollector.TotalErrorsReported.Should().Be(0);
        }
Пример #8
0
        public void BadOrNoOpInput()
        {
            var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
            var parser = new BackgrounderEventParser(_persisterMock.Object, processingNotificationsCollector);

            var logLine = _startTestCases[0].GetLogLine();

            parser.ParseAndPersistLine(logLine, null);                                                                                                                                                                                                                                                                          // Bad input
            parser.ParseAndPersistLine(logLine, "I am not a backgrounder line!");                                                                                                                                                                                                                                               // Good input, but doesn't match regex
            parser.ParseAndPersistLine(logLine, "2018-08-08 11:17:13.491 +1000 (,,,,,:purge_expired_wgsessions,-) scheduled-background-job-runner-1 backgrounder: INFO  com.tableausoftware.backgrounder.runner.BackgroundJobRunner - Running job of type PurgeExpiredWgSessions; no timeout; priority: 0; id: 9; args: null"); // Good input, but not error nor has ID

            _persisterMock.VerifyNoOtherCalls();
            processingNotificationsCollector.TotalErrorsReported.Should().Be(2);
        }
Пример #9
0
        public void TestVersionOutput()
        {
            var testWriterFactory            = new TestWriterFactory();
            var processNotificationCollector = new ProcessingNotificationsCollector(10);

            using (var plugin = new LogShark.Plugins.Config.ConfigPlugin())
            {
                plugin.Configure(testWriterFactory, null, processNotificationCollector, new NullLoggerFactory());
                plugin.ProcessLogLine(_workgroupYamlWithVersionInfo.GetLogLine(), _workgroupYamlWithVersionInfo.LogType);
                var pluginsExecutionResults = plugin.CompleteProcessing();
                pluginsExecutionResults.HasAdditionalTags.Should().BeTrue();
                pluginsExecutionResults.AdditionalTags.Count.Should().Be(2);
                pluginsExecutionResults.AdditionalTags[0].Should().Be("9000.15.0427.2036");
                pluginsExecutionResults.AdditionalTags[1].Should().Be("9.0.1");
            }
        }
Пример #10
0
 public static RunSummary SuccessfulRunSummary(
     string runId,
     TimeSpan elapsed,
     ProcessingNotificationsCollector processingNotificationsCollector,
     ProcessLogSetResult processLogSetResult,
     WorkbookGeneratorResults workbookGeneratorResults,
     PublisherResults publisherResults)
 {
     return(new RunSummary(
                runId,
                elapsed,
                processingNotificationsCollector,
                processLogSetResult,
                workbookGeneratorResults,
                publisherResults,
                null,
                ExitReason.CompletedSuccessfully));
 }
Пример #11
0
        public void BadInput()
        {
            var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
            var testWriterFactory = new TestWriterFactory();

            using (var plugin = new FilestorePlugin())
            {
                plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());

                var wrongContentFormat = new LogLine(new ReadLogLineResult(123, 1234), TestLogFileInfo);
                var nullContent        = new LogLine(new ReadLogLineResult(123, null), TestLogFileInfo);

                plugin.ProcessLogLine(wrongContentFormat, LogType.Apache);
                plugin.ProcessLogLine(nullContent, LogType.Apache);
            }

            testWriterFactory.AssertAllWritersAreDisposedAndEmpty(1);
            processingNotificationsCollector.TotalErrorsReported.Should().Be(2);
        }
Пример #12
0
 private RunSummary(
     string runId,
     TimeSpan elapsed,
     ProcessingNotificationsCollector processingNotificationsCollector,
     ProcessLogSetResult processLogSetResult,
     WorkbookGeneratorResults workbookGeneratorResults,
     PublisherResults publisherResults,
     string reasonForFailure,
     ExitReason exitReason)
 {
     RunId      = runId;
     Elapsed    = elapsed;
     ExitReason = exitReason;
     ProcessingNotificationsCollector = processingNotificationsCollector;
     WorkbookGeneratorResults         = workbookGeneratorResults;
     ProcessLogSetResult = processLogSetResult;
     PublisherResults    = publisherResults;
     ReasonForFailure    = reasonForFailure;
 }
Пример #13
0
        public void BadOrNoOpInput()
        {
            var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
            var testWriterFactory = new TestWriterFactory();

            using (var plugin = new HyperPlugin())
            {
                plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());

                var wrongContentFormat = new LogLine(new ReadLogLineResult(123, "Hyper doesn't expect string"), TestLogFileInfo);
                var nullContent        = new LogLine(new ReadLogLineResult(123, null), TestLogFileInfo);

                plugin.ProcessLogLine(wrongContentFormat, LogType.VizqlserverCpp);
                plugin.ProcessLogLine(nullContent, LogType.VizqlserverCpp);
            }

            testWriterFactory.AssertAllWritersAreDisposedAndEmpty(2);
            processingNotificationsCollector.TotalErrorsReported.Should().Be(2);
        }
Пример #14
0
        public void ReadTestFileWithPlainLines()
        {
            var expectedResults = new List <ReadLogLineResult>();

            for (var i = 1; i <= 6; ++i)
            {
                expectedResults.Add(new ReadLogLineResult(i, null));
            }

            var processingNotificationsCollector = new ProcessingNotificationsCollector(10);

            using (var stream = TestLogFiles.OpenTestFileWithPlainLines())
            {
                var reader  = new NativeJsonLogsReader(stream, "testFile.txt", processingNotificationsCollector);
                var results = reader.ReadLines().ToList();
                results.Should().BeEquivalentTo(expectedResults);
            }

            processingNotificationsCollector.TotalErrorsReported.Should().Be(6);
        }
Пример #15
0
 public static RunSummary FailedRunSummary(
     string runId,
     string errorMessage,
     ExitReason exitReason = ExitReason.UnclassifiedError,
     ProcessingNotificationsCollector processingNotificationsCollector = null,
     TimeSpan?elapsed = null,
     ProcessLogSetResult processLogSetResult           = null,
     WorkbookGeneratorResults workbookGeneratorResults = null,
     PublisherResults publisherResults = null)
 {
     return(new RunSummary(
                runId,
                elapsed ?? TimeSpan.Zero,
                processingNotificationsCollector,
                processLogSetResult,
                workbookGeneratorResults,
                publisherResults,
                errorMessage,
                exitReason));
 }
Пример #16
0
        private static void AddErrorsCountByReporterToStringBuilder(ProcessingNotificationsCollector errorsCollector, StringBuilder report)
        {
            if (errorsCollector.ErrorCountByReporter == null)
            {
                return;
            }

            if (errorsCollector.ErrorCountByReporter.Count == 1)
            {
                report.AppendLine($"All errors reported by {errorsCollector.ErrorCountByReporter.First().Key}.");
            }
            else
            {
                report.AppendLine($"Error count by reporter:");
                foreach (var(reporter, count) in errorsCollector.ErrorCountByReporter.OrderBy(kvp => kvp.Key))
                {
                    report.AppendLine($"{reporter?.PadRight(30, ' ') ?? "(null)"}: {count}");
                }
            }
        }
Пример #17
0
        public void BadInput()
        {
            var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
            var testWriterFactory = new TestWriterFactory();

            using (var plugin = new LogShark.Plugins.Config.ConfigPlugin())
            {
                plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());

                var wrongContentFormat = new LogLine(new ReadLogLineResult(123, "stringIsNotWhatConfigPluginExpects"), TestWorkgroupYmlInfo);
                var nullContent        = new LogLine(new ReadLogLineResult(123, null), TestWorkgroupYmlInfo);

                plugin.ProcessLogLine(wrongContentFormat, LogType.WorkgroupYml);
                plugin.ProcessLogLine(nullContent, LogType.WorkgroupYml);

                plugin.CompleteProcessing();
            }

            VerifyWritersState(testWriterFactory.Writers, new List <ConfigEntry>(), new List <ConfigProcessInfo>());
            processingNotificationsCollector.TotalErrorsReported.Should().Be(2);
        }
Пример #18
0
        public void BadInput()
        {
            var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
            var testWriterFactory = new TestWriterFactory();

            using (var plugin = new NetstatPlugin())
            {
                plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());
                var logFileInfo = new LogFileInfo("netstat-anp.txt", @"folder1/netstat-anp.txt", "worker1", new DateTime(2019, 04, 12, 13, 33, 31));

                var wrongContentFormat = new LogLine(new ReadLogLineResult(123, 1234), logFileInfo);
                var nullContent        = new LogLine(new ReadLogLineResult(123, null), logFileInfo);

                plugin.ProcessLogLine(wrongContentFormat, LogType.NetstatLinux);
                plugin.ProcessLogLine(nullContent, LogType.NetstatLinux);
                plugin.ProcessLogLine(nullContent, LogType.NetstatWindows);
            }

            testWriterFactory.AssertAllWritersAreDisposedAndEmpty(1);
            processingNotificationsCollector.TotalErrorsReported.Should().Be(3);
        }
Пример #19
0
        public void VerifyDifferentJsonStrings()
        {
            var processingNotificationsCollector = new ProcessingNotificationsCollector(10);

            using (var stream = TestLogFiles.OpenTestFileWithJsonData())
            {
                var reader  = new NativeJsonLogsReader(stream, "testFile.txt", processingNotificationsCollector);
                var results = reader.ReadLines().ToList();
                results.Should().BeEquivalentTo(ExpectedResults);

                // This extra steps are needed to confirm JToken equality (BeEquivalentTo above cannot do it)
                var actualPayload   = ExtractPayloadAsStrings(results);
                var expectedPayload = ExtractPayloadAsStrings(ExpectedResults);
                actualPayload.Should().BeEquivalentTo(expectedPayload);

                var actualArtData   = ExtractArtDataAsStrings(results);
                var expectedArtData = ExtractArtDataAsStrings(ExpectedResults);
                actualArtData.Should().BeEquivalentTo(expectedArtData);
            }

            processingNotificationsCollector.TotalErrorsReported.Should().Be(3);
        }
Пример #20
0
        public void BadOrNoOpInput()
        {
            var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
            var testWriterFactory = new TestWriterFactory();

            using (var plugin = new VizqlDesktopPlugin())
            {
                plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());

                var wrongContentFormat = new LogLine(new ReadLogLineResult(123, "VizqlDesktop doesn't expect string"), TestLogFileInfo);
                var nullContent        = new LogLine(new ReadLogLineResult(123, null), TestLogFileInfo);
                var noOpContentType    = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
                    EventType = "something else"
                }), TestLogFileInfo);

                plugin.ProcessLogLine(wrongContentFormat, LogType.BackgrounderJava);
                plugin.ProcessLogLine(nullContent, LogType.BackgrounderJava);
                plugin.ProcessLogLine(noOpContentType, LogType.BackgrounderJava);
            }

            testWriterFactory.AssertAllWritersAreDisposedAndEmpty(4);
            processingNotificationsCollector.TotalErrorsReported.Should().Be(2);
        }
Пример #21
0
 public static bool ReceivedAnything(this ProcessingNotificationsCollector processingNotificationsCollector)
 {
     return(processingNotificationsCollector.TotalErrorsReported != 0 ||
            processingNotificationsCollector.TotalWarningsReported != 0);
 }
Пример #22
0
 public TableauLogsExtractorTests()
 {
     DeleteTempDir();
     _processingNotificationsCollector = new ProcessingNotificationsCollector(10);
 }