public void MemorySampleEvent(string payload, bool expectEvent, long expectedMemoryUtil, long expectedTotalUtil)
{
var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
using (var eventProcessor = new ResourceManagerEventsProcessor(_testWriterFactory, processingNotificationsCollector))
{
eventProcessor.ProcessEvent(_testBaseEvent, payload, _testLogLine, TestProcessName);
}
var memorySampleWriter = _testWriterFactory.GetOneWriterAndVerifyOthersAreEmptyAndDisposed <ResourceManagerMemorySample>("ResourceManagerMemorySamples", 5);
memorySampleWriter.ReceivedObjects.Count.Should().Be(expectEvent ? 1 : 0);
processingNotificationsCollector.TotalErrorsReported.Should().Be(expectEvent ? 0 : 1);
if (expectEvent)
{
var expectedRecord = new
{
FileName = TestLogFileInfo.FileName,
FilePath = TestLogFileInfo.FilePath,
LineNumber = _testLogLine.LineNumber,
Timestamp = _testBaseEvent.Timestamp,
Worker = TestLogFileInfo.Worker,
ProcessId = _testBaseEvent.ProcessId,
ProcessIndex = (int?)null,
ProcessName = TestProcessName,
ProcessMemoryUtil = expectedMemoryUtil,
TotalMemoryUtil = expectedTotalUtil,
};
memorySampleWriter.ReceivedObjects[0].Should().BeEquivalentTo(expectedRecord);
}
}
public void BadOrNoOpInput()
{
var testWriterFactory = new TestWriterFactory();
var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
using (var plugin = new LogShark.Plugins.ResourceManager.ResourceManagerPlugin())
{
plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());
var wrongContentFormat = new LogLine(new ReadLogLineResult(123, "ResourceManagerPlugin doesn't expect string"), VizqlServerCppFileInfo);
var nullContent = new LogLine(new ReadLogLineResult(123, null), VizqlServerCppFileInfo);
var nonMsgContent = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
EventType = "something else"
}), VizqlServerCppFileInfo);
var payloadIsNotAString = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
EventType = "msg", EventPayload = JToken.FromObject(new { Key = "value" })
}), VizqlServerCppFileInfo);
var vizqlAndSrmInternal = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
EventType = "srm-internal", EventPayload = "Resource Manager: CPU info: 0%"
}), VizqlServerCppFileInfo);
var hyperAndMessage = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
EventType = "msg", EventPayload = "Resource Manager: CPU info: 0%"
}), HyperFileInfo);
plugin.ProcessLogLine(wrongContentFormat, LogType.VizqlserverCpp);
plugin.ProcessLogLine(nullContent, LogType.VizqlserverCpp);
plugin.ProcessLogLine(nonMsgContent, LogType.VizqlserverCpp);
plugin.ProcessLogLine(payloadIsNotAString, LogType.VizqlserverCpp);
plugin.ProcessLogLine(vizqlAndSrmInternal, LogType.VizqlserverCpp);
plugin.ProcessLogLine(hyperAndMessage, LogType.Hyper);
}
testWriterFactory.AssertAllWritersAreDisposedAndEmpty(5);
processingNotificationsCollector.TotalErrorsReported.Should().Be(2);
}
public void BadAndNoOpInput()
{
var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
var testWriterFactory = new TestWriterFactory();
using (var plugin = new ServerTelemetryPlugin())
{
plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());
var wrongContentFormat = new LogLine(new ReadLogLineResult(123, "ServerTelemetry doesn't expect string"), TestLogFileInfo);
var nullContent = new LogLine(new ReadLogLineResult(123, null), TestLogFileInfo);
var differentEventType = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
EventType = "msg"
}), TestLogFileInfo); // This doesn't generate error, as it is a normal condition
var payloadIsNull = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
EventType = "server-telemetry"
}), TestLogFileInfo);
plugin.ProcessLogLine(wrongContentFormat, LogType.VizqlserverCpp);
plugin.ProcessLogLine(nullContent, LogType.VizqlserverCpp);
plugin.ProcessLogLine(differentEventType, LogType.VizqlserverCpp);
plugin.ProcessLogLine(payloadIsNull, LogType.VizqlserverCpp);
}
testWriterFactory.AssertAllWritersAreDisposedAndEmpty(2);
processingNotificationsCollector.TotalErrorsReported.Should().Be(3);
}
public void BadOrNoOpInput()
{
var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
var testWriterFactory = new TestWriterFactory();
using (var plugin = new ArtPlugin())
{
plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());
var wrongContentFormat = new LogLine(new ReadLogLineResult(123, "Art doesn't expect string"), TestLogFileInfo);
var nullContent = new LogLine(new ReadLogLineResult(123, null), TestLogFileInfo);
var payloadIsNull = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
EventType = "msg", ArtData = null
}), TestLogFileInfo);
var incorrectArtPayload = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
EventType = "msg", ArtData = JToken.FromObject("JSON was expected here")
}), TestLogFileInfo);
plugin.ProcessLogLine(wrongContentFormat, LogType.VizqlserverCpp);
plugin.ProcessLogLine(nullContent, LogType.VizqlserverCpp);
plugin.ProcessLogLine(payloadIsNull, LogType.VizqlserverCpp);
plugin.ProcessLogLine(incorrectArtPayload, LogType.VizqlserverCpp);
}
testWriterFactory.AssertAllWritersAreDisposedAndEmpty(2);
processingNotificationsCollector.TotalErrorsReported.Should().Be(3);
}
private static void AppendDetailedErrorsToStringBuilder(ProcessingNotificationsCollector errorsCollector, StringBuilder report)
{
foreach (var error in errorsCollector.ProcessingErrorsDetails ?? new List <ProcessingNotification>())
{
report.AppendLine($"--> File: `{error.FilePath ?? "(null)"}`. Line: {error.LineNumber}. Reported by: `{error.ReportedBy ?? "(null)"}`. Error: `{error.Message ?? "(null)"}`");
}
}
public void BadInput()
{
var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
var testWriterFactory = new TestWriterFactory();
using (var plugin = new ClusterControllerPlugin())
{
plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());
var ccWrongContentFormat = new LogLine(new ReadLogLineResult(123, 1234), _testClusterControllerLogFileInfo);
var ccNullContent = new LogLine(new ReadLogLineResult(123, null), _testClusterControllerLogFileInfo);
var ccIncorrectString = new LogLine(new ReadLogLineResult(123, "I am not a Java log!"), _testClusterControllerLogFileInfo);
var zkWrongContentFormat = new LogLine(new ReadLogLineResult(123, 1234), _testZookeeperLogFileInfo);
var zkNullContent = new LogLine(new ReadLogLineResult(123, null), _testZookeeperLogFileInfo);
var zkIncorrectString = new LogLine(new ReadLogLineResult(123, "I am not a Java log!"), _testZookeeperLogFileInfo);
plugin.ProcessLogLine(ccWrongContentFormat, LogType.ClusterController);
plugin.ProcessLogLine(ccNullContent, LogType.ClusterController);
plugin.ProcessLogLine(ccIncorrectString, LogType.ClusterController);
plugin.ProcessLogLine(zkWrongContentFormat, LogType.Zookeeper);
plugin.ProcessLogLine(zkNullContent, LogType.Zookeeper);
plugin.ProcessLogLine(zkIncorrectString, LogType.Zookeeper);
}
testWriterFactory.AssertAllWritersAreDisposedAndEmpty(_testWriterCount);
processingNotificationsCollector.TotalErrorsReported.Should().Be(_testWriterCount);
}
public void ValidLinesThatAreNotPersisted(string input)
{
var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
var parser = new BackgrounderEventParser(_persisterMock.Object, processingNotificationsCollector);
var logLine = _startTestCases[0].GetLogLine();
parser.ParseAndPersistLine(logLine, input);
_persisterMock.VerifyNoOtherCalls();
processingNotificationsCollector.TotalErrorsReported.Should().Be(0);
}
public void BadOrNoOpInput()
{
var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
var parser = new BackgrounderEventParser(_persisterMock.Object, processingNotificationsCollector);
var logLine = _startTestCases[0].GetLogLine();
parser.ParseAndPersistLine(logLine, null); // Bad input
parser.ParseAndPersistLine(logLine, "I am not a backgrounder line!"); // Good input, but doesn't match regex
parser.ParseAndPersistLine(logLine, "2018-08-08 11:17:13.491 +1000 (,,,,,:purge_expired_wgsessions,-) scheduled-background-job-runner-1 backgrounder: INFO com.tableausoftware.backgrounder.runner.BackgroundJobRunner - Running job of type PurgeExpiredWgSessions; no timeout; priority: 0; id: 9; args: null"); // Good input, but not error nor has ID
_persisterMock.VerifyNoOtherCalls();
processingNotificationsCollector.TotalErrorsReported.Should().Be(2);
}
public void TestVersionOutput()
{
var testWriterFactory = new TestWriterFactory();
var processNotificationCollector = new ProcessingNotificationsCollector(10);
using (var plugin = new LogShark.Plugins.Config.ConfigPlugin())
{
plugin.Configure(testWriterFactory, null, processNotificationCollector, new NullLoggerFactory());
plugin.ProcessLogLine(_workgroupYamlWithVersionInfo.GetLogLine(), _workgroupYamlWithVersionInfo.LogType);
var pluginsExecutionResults = plugin.CompleteProcessing();
pluginsExecutionResults.HasAdditionalTags.Should().BeTrue();
pluginsExecutionResults.AdditionalTags.Count.Should().Be(2);
pluginsExecutionResults.AdditionalTags[0].Should().Be("9000.15.0427.2036");
pluginsExecutionResults.AdditionalTags[1].Should().Be("9.0.1");
}
}
public static RunSummary SuccessfulRunSummary(
string runId,
TimeSpan elapsed,
ProcessingNotificationsCollector processingNotificationsCollector,
ProcessLogSetResult processLogSetResult,
WorkbookGeneratorResults workbookGeneratorResults,
PublisherResults publisherResults)
{
return(new RunSummary(
runId,
elapsed,
processingNotificationsCollector,
processLogSetResult,
workbookGeneratorResults,
publisherResults,
null,
ExitReason.CompletedSuccessfully));
}
public void BadInput()
{
var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
var testWriterFactory = new TestWriterFactory();
using (var plugin = new FilestorePlugin())
{
plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());
var wrongContentFormat = new LogLine(new ReadLogLineResult(123, 1234), TestLogFileInfo);
var nullContent = new LogLine(new ReadLogLineResult(123, null), TestLogFileInfo);
plugin.ProcessLogLine(wrongContentFormat, LogType.Apache);
plugin.ProcessLogLine(nullContent, LogType.Apache);
}
testWriterFactory.AssertAllWritersAreDisposedAndEmpty(1);
processingNotificationsCollector.TotalErrorsReported.Should().Be(2);
}
private RunSummary(
string runId,
TimeSpan elapsed,
ProcessingNotificationsCollector processingNotificationsCollector,
ProcessLogSetResult processLogSetResult,
WorkbookGeneratorResults workbookGeneratorResults,
PublisherResults publisherResults,
string reasonForFailure,
ExitReason exitReason)
{
RunId = runId;
Elapsed = elapsed;
ExitReason = exitReason;
ProcessingNotificationsCollector = processingNotificationsCollector;
WorkbookGeneratorResults = workbookGeneratorResults;
ProcessLogSetResult = processLogSetResult;
PublisherResults = publisherResults;
ReasonForFailure = reasonForFailure;
}
public void BadOrNoOpInput()
{
var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
var testWriterFactory = new TestWriterFactory();
using (var plugin = new HyperPlugin())
{
plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());
var wrongContentFormat = new LogLine(new ReadLogLineResult(123, "Hyper doesn't expect string"), TestLogFileInfo);
var nullContent = new LogLine(new ReadLogLineResult(123, null), TestLogFileInfo);
plugin.ProcessLogLine(wrongContentFormat, LogType.VizqlserverCpp);
plugin.ProcessLogLine(nullContent, LogType.VizqlserverCpp);
}
testWriterFactory.AssertAllWritersAreDisposedAndEmpty(2);
processingNotificationsCollector.TotalErrorsReported.Should().Be(2);
}
public void ReadTestFileWithPlainLines()
{
var expectedResults = new List <ReadLogLineResult>();
for (var i = 1; i <= 6; ++i)
{
expectedResults.Add(new ReadLogLineResult(i, null));
}
var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
using (var stream = TestLogFiles.OpenTestFileWithPlainLines())
{
var reader = new NativeJsonLogsReader(stream, "testFile.txt", processingNotificationsCollector);
var results = reader.ReadLines().ToList();
results.Should().BeEquivalentTo(expectedResults);
}
processingNotificationsCollector.TotalErrorsReported.Should().Be(6);
}
public static RunSummary FailedRunSummary(
string runId,
string errorMessage,
ExitReason exitReason = ExitReason.UnclassifiedError,
ProcessingNotificationsCollector processingNotificationsCollector = null,
TimeSpan?elapsed = null,
ProcessLogSetResult processLogSetResult = null,
WorkbookGeneratorResults workbookGeneratorResults = null,
PublisherResults publisherResults = null)
{
return(new RunSummary(
runId,
elapsed ?? TimeSpan.Zero,
processingNotificationsCollector,
processLogSetResult,
workbookGeneratorResults,
publisherResults,
errorMessage,
exitReason));
}
private static void AddErrorsCountByReporterToStringBuilder(ProcessingNotificationsCollector errorsCollector, StringBuilder report)
{
if (errorsCollector.ErrorCountByReporter == null)
{
return;
}
if (errorsCollector.ErrorCountByReporter.Count == 1)
{
report.AppendLine($"All errors reported by {errorsCollector.ErrorCountByReporter.First().Key}.");
}
else
{
report.AppendLine($"Error count by reporter:");
foreach (var(reporter, count) in errorsCollector.ErrorCountByReporter.OrderBy(kvp => kvp.Key))
{
report.AppendLine($"{reporter?.PadRight(30, ' ') ?? "(null)"}: {count}");
}
}
}
public void BadInput()
{
var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
var testWriterFactory = new TestWriterFactory();
using (var plugin = new LogShark.Plugins.Config.ConfigPlugin())
{
plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());
var wrongContentFormat = new LogLine(new ReadLogLineResult(123, "stringIsNotWhatConfigPluginExpects"), TestWorkgroupYmlInfo);
var nullContent = new LogLine(new ReadLogLineResult(123, null), TestWorkgroupYmlInfo);
plugin.ProcessLogLine(wrongContentFormat, LogType.WorkgroupYml);
plugin.ProcessLogLine(nullContent, LogType.WorkgroupYml);
plugin.CompleteProcessing();
}
VerifyWritersState(testWriterFactory.Writers, new List <ConfigEntry>(), new List <ConfigProcessInfo>());
processingNotificationsCollector.TotalErrorsReported.Should().Be(2);
}
public void BadInput()
{
var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
var testWriterFactory = new TestWriterFactory();
using (var plugin = new NetstatPlugin())
{
plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());
var logFileInfo = new LogFileInfo("netstat-anp.txt", @"folder1/netstat-anp.txt", "worker1", new DateTime(2019, 04, 12, 13, 33, 31));
var wrongContentFormat = new LogLine(new ReadLogLineResult(123, 1234), logFileInfo);
var nullContent = new LogLine(new ReadLogLineResult(123, null), logFileInfo);
plugin.ProcessLogLine(wrongContentFormat, LogType.NetstatLinux);
plugin.ProcessLogLine(nullContent, LogType.NetstatLinux);
plugin.ProcessLogLine(nullContent, LogType.NetstatWindows);
}
testWriterFactory.AssertAllWritersAreDisposedAndEmpty(1);
processingNotificationsCollector.TotalErrorsReported.Should().Be(3);
}
public void VerifyDifferentJsonStrings()
{
var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
using (var stream = TestLogFiles.OpenTestFileWithJsonData())
{
var reader = new NativeJsonLogsReader(stream, "testFile.txt", processingNotificationsCollector);
var results = reader.ReadLines().ToList();
results.Should().BeEquivalentTo(ExpectedResults);
// This extra steps are needed to confirm JToken equality (BeEquivalentTo above cannot do it)
var actualPayload = ExtractPayloadAsStrings(results);
var expectedPayload = ExtractPayloadAsStrings(ExpectedResults);
actualPayload.Should().BeEquivalentTo(expectedPayload);
var actualArtData = ExtractArtDataAsStrings(results);
var expectedArtData = ExtractArtDataAsStrings(ExpectedResults);
actualArtData.Should().BeEquivalentTo(expectedArtData);
}
processingNotificationsCollector.TotalErrorsReported.Should().Be(3);
}
public void BadOrNoOpInput()
{
var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
var testWriterFactory = new TestWriterFactory();
using (var plugin = new VizqlDesktopPlugin())
{
plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());
var wrongContentFormat = new LogLine(new ReadLogLineResult(123, "VizqlDesktop doesn't expect string"), TestLogFileInfo);
var nullContent = new LogLine(new ReadLogLineResult(123, null), TestLogFileInfo);
var noOpContentType = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
EventType = "something else"
}), TestLogFileInfo);
plugin.ProcessLogLine(wrongContentFormat, LogType.BackgrounderJava);
plugin.ProcessLogLine(nullContent, LogType.BackgrounderJava);
plugin.ProcessLogLine(noOpContentType, LogType.BackgrounderJava);
}
testWriterFactory.AssertAllWritersAreDisposedAndEmpty(4);
processingNotificationsCollector.TotalErrorsReported.Should().Be(2);
}
public static bool ReceivedAnything(this ProcessingNotificationsCollector processingNotificationsCollector)
{
return(processingNotificationsCollector.TotalErrorsReported != 0 ||
processingNotificationsCollector.TotalWarningsReported != 0);
}
public TableauLogsExtractorTests()
{
DeleteTempDir();
_processingNotificationsCollector = new ProcessingNotificationsCollector(10);
}