public void BadAndNoOpInput()
{
var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
var testWriterFactory = new TestWriterFactory();
using (var plugin = new ServerTelemetryPlugin())
{
plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());
var wrongContentFormat = new LogLine(new ReadLogLineResult(123, "ServerTelemetry doesn't expect string"), TestLogFileInfo);
var nullContent = new LogLine(new ReadLogLineResult(123, null), TestLogFileInfo);
var differentEventType = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
EventType = "msg"
}), TestLogFileInfo); // This doesn't generate error, as it is a normal condition
var payloadIsNull = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
EventType = "server-telemetry"
}), TestLogFileInfo);
plugin.ProcessLogLine(wrongContentFormat, LogType.VizqlserverCpp);
plugin.ProcessLogLine(nullContent, LogType.VizqlserverCpp);
plugin.ProcessLogLine(differentEventType, LogType.VizqlserverCpp);
plugin.ProcessLogLine(payloadIsNull, LogType.VizqlserverCpp);
}
testWriterFactory.AssertAllWritersAreDisposedAndEmpty(2);
processingNotificationsCollector.TotalErrorsReported.Should().Be(3);
}
public void ServerTelemetryPluginTest()
{
var testWriterFactory = new TestWriterFactory();
using (var plugin = new ServerTelemetryPlugin())
{
plugin.Configure(testWriterFactory, null, null, new NullLoggerFactory());
foreach (var testCase in _testCases)
{
var logLine = testCase.GetLogLine();
plugin.ProcessLogLine(logLine, testCase.LogType);
}
}
var testEventWriter = testWriterFactory.Writers.Values.First() as TestWriter <ServerTelemetryEvent>;
var testMetricWriter = testWriterFactory.Writers.Values.Last() as TestWriter <ServerTelemetryMetric>;
testWriterFactory.Writers.Count.Should().Be(2);
testEventWriter.WasDisposed.Should().Be(true);
testMetricWriter.WasDisposed.Should().Be(true);
var expectedOutput = _testCases.Select(testCase => (Tuple <object, object[]>)testCase.ExpectedOutput).ToList();
var expectedEvents = expectedOutput.Select(i => i.Item1);
var expectedMetrics = expectedOutput.SelectMany(i => i.Item2);
testEventWriter.ReceivedObjects.Should().BeEquivalentTo(expectedEvents);
testMetricWriter.ReceivedObjects.Should().BeEquivalentTo(expectedMetrics);
}
private static (TestWriterFactory, PluginManager) InitPluginsWithTestWriter(
string selectedPlugins,
string defaultSet = null,
string excludedPlugins = null,
bool?usePluginsFromLogSharkAssembly = null)
{
var testWriterFactory = new TestWriterFactory();
var configDictionary = new Dictionary <string, string>()
{
{ "PluginsConfiguration:ApachePlugin:IncludeGatewayChecks", "true" },
{ "PluginsConfiguration:DefaultPluginSet:PluginsToRunByDefault", defaultSet },
{ "PluginsConfiguration:DefaultPluginSet:PluginsToExcludeFromDefaultSet", excludedPlugins }
};
if (usePluginsFromLogSharkAssembly.HasValue)
{
configDictionary["EnvironmentConfig:UsePluginsFromLogSharkAssembly"] = usePluginsFromLogSharkAssembly.Value.ToString();
}
var config = new LogSharkConfiguration(
new LogSharkCommandLineParameters()
{
RequestedPlugins = selectedPlugins
},
ConfigGenerator.GetConfigFromDictionary(configDictionary),
null);
var pluginManager = new PluginManager(config, new NullLoggerFactory());
pluginManager.CreatePlugins(testWriterFactory, null);
return(testWriterFactory, pluginManager);
}
public void RunTestCases_HyperQuery()
{
var hyperQueryTestCases = _testCases;
var testWriterFactory = new TestWriterFactory();
using (var plugin = new HyperPlugin())
{
plugin.Configure(testWriterFactory, null, null, new NullLoggerFactory());
foreach (var testCase in hyperQueryTestCases)
{
var logLine = testCase.GetLogLine();
plugin.ProcessLogLine(logLine, LogType.Hyper);
}
plugin.CompleteProcessing();
}
var expectedOutput = hyperQueryTestCases.Select(testCase => testCase.ExpectedOutput).ToList();
var hyperQueryWriter = testWriterFactory.Writers.Values.OfType <TestWriter <HyperEvent> >().First();
testWriterFactory.Writers.Count.Should().Be(2);
hyperQueryWriter.WasDisposed.Should().Be(true);
hyperQueryWriter.ReceivedObjects.Should().BeEquivalentTo(expectedOutput);
}
public void BadOrNoOpInput()
{
var testWriterFactory = new TestWriterFactory();
var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
using (var plugin = new LogShark.Plugins.ResourceManager.ResourceManagerPlugin())
{
plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());
var wrongContentFormat = new LogLine(new ReadLogLineResult(123, "ResourceManagerPlugin doesn't expect string"), VizqlServerCppFileInfo);
var nullContent = new LogLine(new ReadLogLineResult(123, null), VizqlServerCppFileInfo);
var nonMsgContent = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
EventType = "something else"
}), VizqlServerCppFileInfo);
var payloadIsNotAString = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
EventType = "msg", EventPayload = JToken.FromObject(new { Key = "value" })
}), VizqlServerCppFileInfo);
var vizqlAndSrmInternal = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
EventType = "srm-internal", EventPayload = "Resource Manager: CPU info: 0%"
}), VizqlServerCppFileInfo);
var hyperAndMessage = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
EventType = "msg", EventPayload = "Resource Manager: CPU info: 0%"
}), HyperFileInfo);
plugin.ProcessLogLine(wrongContentFormat, LogType.VizqlserverCpp);
plugin.ProcessLogLine(nullContent, LogType.VizqlserverCpp);
plugin.ProcessLogLine(nonMsgContent, LogType.VizqlserverCpp);
plugin.ProcessLogLine(payloadIsNotAString, LogType.VizqlserverCpp);
plugin.ProcessLogLine(vizqlAndSrmInternal, LogType.VizqlserverCpp);
plugin.ProcessLogLine(hyperAndMessage, LogType.Hyper);
}
testWriterFactory.AssertAllWritersAreDisposedAndEmpty(5);
processingNotificationsCollector.TotalErrorsReported.Should().Be(2);
}
public void BadInput()
{
var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
var testWriterFactory = new TestWriterFactory();
using (var plugin = new ClusterControllerPlugin())
{
plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());
var ccWrongContentFormat = new LogLine(new ReadLogLineResult(123, 1234), _testClusterControllerLogFileInfo);
var ccNullContent = new LogLine(new ReadLogLineResult(123, null), _testClusterControllerLogFileInfo);
var ccIncorrectString = new LogLine(new ReadLogLineResult(123, "I am not a Java log!"), _testClusterControllerLogFileInfo);
var zkWrongContentFormat = new LogLine(new ReadLogLineResult(123, 1234), _testZookeeperLogFileInfo);
var zkNullContent = new LogLine(new ReadLogLineResult(123, null), _testZookeeperLogFileInfo);
var zkIncorrectString = new LogLine(new ReadLogLineResult(123, "I am not a Java log!"), _testZookeeperLogFileInfo);
plugin.ProcessLogLine(ccWrongContentFormat, LogType.ClusterController);
plugin.ProcessLogLine(ccNullContent, LogType.ClusterController);
plugin.ProcessLogLine(ccIncorrectString, LogType.ClusterController);
plugin.ProcessLogLine(zkWrongContentFormat, LogType.Zookeeper);
plugin.ProcessLogLine(zkNullContent, LogType.Zookeeper);
plugin.ProcessLogLine(zkIncorrectString, LogType.Zookeeper);
}
testWriterFactory.AssertAllWritersAreDisposedAndEmpty(_testWriterCount);
processingNotificationsCollector.TotalErrorsReported.Should().Be(_testWriterCount);
}
public void RunTestCases_SkipHealthChecks()
{
var testWriterFactory = new TestWriterFactory();
using (var plugin = new ApachePlugin())
{
plugin.Configure(testWriterFactory, _configSkippingHealthChecks.GetSection("PluginsConfiguration:ApachePlugin"), null, new NullLoggerFactory());
foreach (var testCase in _testCases)
{
var logLine = testCase.GetLogLine();
plugin.ProcessLogLine(logLine, LogType.Apache);
}
}
var expectedOutput = _testCases
.Select(testCase => testCase.ExpectedOutput)
.Where(@event => ((dynamic)@event).RequestBody != "/favicon.ico")
.ToList();
testWriterFactory.Writers.Count.Should().Be(1);
var testWriter = testWriterFactory.Writers.Values.First() as TestWriter <ApacheEvent>;
testWriter.WasDisposed.Should().Be(true);
testWriter.ReceivedObjects.Should().BeEquivalentTo(expectedOutput);
}
public void BackgrounderPluginTest()
{
var testWriterFactory = new TestWriterFactory();
using (var plugin = new BackgrounderPlugin())
{
plugin.Configure(testWriterFactory, null, null, new NullLoggerFactory());
foreach (var testCase in _testCases)
{
var logLine = testCase.GetLogLine();
plugin.ProcessLogLine(logLine, testCase.LogType);
}
plugin.CompleteProcessing();
}
var expectedOutput = _testCases
.Where(@case => @case.ExpectedOutput != null)
.Select(@case => @case.ExpectedOutput)
.ToList();
var jobWriter = testWriterFactory.GetOneWriterAndVerifyOthersAreEmptyAndDisposed <BackgrounderJob>("BackgrounderJobs", 4);
jobWriter.ReceivedObjects.Should().BeEquivalentTo(expectedOutput);
}
public void BadOrNoOpInput()
{
var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
var testWriterFactory = new TestWriterFactory();
using (var plugin = new ArtPlugin())
{
plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());
var wrongContentFormat = new LogLine(new ReadLogLineResult(123, "Art doesn't expect string"), TestLogFileInfo);
var nullContent = new LogLine(new ReadLogLineResult(123, null), TestLogFileInfo);
var payloadIsNull = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
EventType = "msg", ArtData = null
}), TestLogFileInfo);
var incorrectArtPayload = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
EventType = "msg", ArtData = JToken.FromObject("JSON was expected here")
}), TestLogFileInfo);
plugin.ProcessLogLine(wrongContentFormat, LogType.VizqlserverCpp);
plugin.ProcessLogLine(nullContent, LogType.VizqlserverCpp);
plugin.ProcessLogLine(payloadIsNull, LogType.VizqlserverCpp);
plugin.ProcessLogLine(incorrectArtPayload, LogType.VizqlserverCpp);
}
testWriterFactory.AssertAllWritersAreDisposedAndEmpty(2);
processingNotificationsCollector.TotalErrorsReported.Should().Be(3);
}
public void RunTestCases_TabadminErrors()
{
var tabadminErrorTestCases = new List <PluginTestCase>
{
new PluginTestCase
{
LogContents =
@"2018-07-10 11:28:33.510 -0700_FATAL_10.10.1.153:DEVSRV1_:_pid=12068_0x188ac8a3__user=__request=__ Error during restore: MultiCommand::ExternalCommandFailure 'Command '""E:/ Tableau Server / 10.3 / pgsql / bin / pg_restore.exe"" -h localhost -p 8060 -U tblwgadmin -C -d postgres -j 1 ""E:/ Tableau Server / workgroup.pg_dump""' failed with code 1, result was pg_restore: [archiver] unsupported version (1.13) in file header
'",
LogFileInfo = TestLogFileInfo,
LineNumber = 51335,
ExpectedOutput = new TabadminError
{
File = "tabadmin.log",
FilePath = "logs/tabadmin.log",
Hostname = "DEVSRV1",
Id = "logs/tabadmin.log-51335",
Line = 51335,
Message = @"Error during restore: MultiCommand::ExternalCommandFailure 'Command '""E:/ Tableau Server / 10.3 / pgsql / bin / pg_restore.exe"" -h localhost -p 8060 -U tblwgadmin -C -d postgres -j 1 ""E:/ Tableau Server / workgroup.pg_dump""' failed with code 1, result was pg_restore: [archiver] unsupported version (1.13) in file header
'",
Severity = "FATAL",
Timestamp = DateTime.Parse(@"2018-07-10 11:28:33.510"),
TimestampGmt = DateTime.Parse(@"2018-07-10 04:28:33.510"),
TimestampOffset = "-0700",
Version = null,
VersionId = null,
VersionLong = null,
Worker = "worker0",
}
},
};
var testWriterFactory = new TestWriterFactory();
using (var plugin = new TabadminPlugin())
{
plugin.Configure(testWriterFactory, null, null, new NullLoggerFactory());
foreach (var testCase in tabadminErrorTestCases)
{
var logLine = testCase.GetLogLine();
plugin.ProcessLogLine(logLine, LogType.Tabadmin);
}
plugin.CompleteProcessing();
}
var expectedOutput = tabadminErrorTestCases.Select(testCase => testCase.ExpectedOutput).ToList();
var tabadminActionWriter = testWriterFactory.Writers.Values.OfType <TestWriter <TabadminError> >().First();
testWriterFactory.Writers.Count.Should().Be(3);
tabadminActionWriter.WasDisposed.Should().Be(true);
tabadminActionWriter.ReceivedObjects.Should().BeEquivalentTo(expectedOutput);
}
public void RunTestCases_TabadminActions()
{
var tabadminActionTestCases = new List <PluginTestCase>
{
new PluginTestCase()
{
LogContents = @"2018-07-06 13:54:19.875 -0700_DEBUG_10.10.1.153:DEVSRV1_:_pid=18044_0x7a0e7ecd__user=__request=__ run as: <script> validate --skiptempIPv6 --output E:\Tableau Server\2018.1\temp\validation_results.txt",
LogFileInfo = TestLogFileInfo,
LineNumber = 3,
ExpectedOutput = new TabadminAction
{
Arguments = @"--skiptempIPv6 --output E:\Tableau Server\2018.1\temp\validation_results.txt",
Command = "validate",
File = "tabadmin.log",
FilePath = "logs/tabadmin.log",
Hostname = "DEVSRV1",
Id = "logs/tabadmin.log-3",
Line = 3,
Timestamp = DateTime.Parse(@"2018-07-06 13:54:19.875"),
TimestampGmt = DateTime.Parse(@"2018-07-06 06:54:19.875"),
TimestampOffset = "-0700",
Version = null,
VersionId = null,
VersionLong = null,
Worker = "worker0",
}
},
};
var testWriterFactory = new TestWriterFactory();
using (var plugin = new TabadminPlugin())
{
plugin.Configure(testWriterFactory, null, null, new NullLoggerFactory());
foreach (var testCase in tabadminActionTestCases)
{
var logLine = testCase.GetLogLine();
plugin.ProcessLogLine(logLine, LogType.Tabadmin);
}
plugin.CompleteProcessing();
}
var expectedOutput = tabadminActionTestCases.Select(testCase => testCase.ExpectedOutput).ToList();
var tabadminActionWriter = testWriterFactory.Writers.Values.OfType <TestWriter <TabadminAction> >().First();
testWriterFactory.Writers.Count.Should().Be(3);
tabadminActionWriter.WasDisposed.Should().Be(true);
tabadminActionWriter.ReceivedObjects.Should().BeEquivalentTo(expectedOutput);
}
public void TestWithDifferentMaxLimitConfigured(bool useConfig, int?maxLength)
{
var config = useConfig
? CreateConfig(maxLength)
: null;
var testWriterFactory = new TestWriterFactory();
using (var plugin = new VizqlDesktopPlugin())
{
plugin.Configure(testWriterFactory, config, null, new NullLoggerFactory());
foreach (var testCase in _testCases)
{
var logLine = testCase.GetLogLine();
plugin.ProcessLogLine(logLine, LogType.VizqlDesktop);
}
plugin.CompleteProcessing();
}
testWriterFactory.AssertAllWritersDisposedState(true);
var expectedSessions = GetExpectedOutputForName("session");
var sessionsWriter = testWriterFactory.GetWriterByName <VizqlDesktopSession>("VizqlDesktopSessions");
sessionsWriter.ReceivedObjects.Should().BeEquivalentTo(expectedSessions);
var expectedErrors = GetExpectedOutputForName("error");
var errorsWriter = testWriterFactory.GetWriterByName <VizqlDesktopErrorEvent>("VizqlDesktopErrorEvents");
errorsWriter.ReceivedObjects.Should().BeEquivalentTo(expectedErrors);
var expectedEndQueryEvents = GetExpectedOutputForName("endQuery");
if (useConfig && maxLength != null)
{
expectedEndQueryEvents = TruncateQueryText(expectedEndQueryEvents, maxLength.Value);
}
var endQueryEventsWriter = testWriterFactory.GetWriterByName <VizqlEndQueryEvent>("VizqlDesktopEndQueryEvents");
endQueryEventsWriter.ReceivedObjects.Should().BeEquivalentTo(expectedEndQueryEvents);
var expectedPerformanceEvents = GetExpectedOutputForName("performance");
var extraEvent = GetPerformanceEvent(8.985, "end-query", 126, @"{""cols"":1,""elapsed"":8.985,""is-command"":false,""protocol-class"":""postgres"",""protocol-id"":44,""query-category"":""Data"",""query-hash"":3535682211,""query-trunc"":""SELECT * FROM very_long_table_name_so_we_have_something_to_truncate;"",""rows"":3}");
expectedPerformanceEvents.Insert(0, extraEvent);
var performanceEventsWriter = testWriterFactory.GetWriterByName <VizqlPerformanceEvent>("VizqlDesktopPerformanceEvents");
performanceEventsWriter.ReceivedObjects.Should().BeEquivalentTo(expectedPerformanceEvents);
}
private static void AssertLoadAllResults(TestWriterFactory testWriterFactory, PluginManager pluginManager, WritersStatistics writersStatistics)
{
// As number of plugins can chance over time, we're simply verifying that two known plugins are loaded correctly
var loadedPlugins = pluginManager.GetPlugins();
loadedPlugins.Count().Should().BeGreaterThan(1);
loadedPlugins.FirstOrDefault(plugin => plugin.Name == "Apache").Should().NotBe(null);
loadedPlugins.FirstOrDefault(plugin => plugin.Name == "Filestore").Should().NotBe(null);
testWriterFactory.Writers.Count.Should().BeGreaterThan(1);
testWriterFactory.AssertAllWritersDisposedState(false);
writersStatistics.DataSets.Should().ContainKey(new DataSetInfo("Apache", "ApacheRequests"));
writersStatistics.DataSets.Should().ContainKey(new DataSetInfo("Filestore", "Filestore"));
}
public void TestVersionOutput()
{
var testWriterFactory = new TestWriterFactory();
var processNotificationCollector = new ProcessingNotificationsCollector(10);
using (var plugin = new LogShark.Plugins.Config.ConfigPlugin())
{
plugin.Configure(testWriterFactory, null, processNotificationCollector, new NullLoggerFactory());
plugin.ProcessLogLine(_workgroupYamlWithVersionInfo.GetLogLine(), _workgroupYamlWithVersionInfo.LogType);
var pluginsExecutionResults = plugin.CompleteProcessing();
pluginsExecutionResults.HasAdditionalTags.Should().BeTrue();
pluginsExecutionResults.AdditionalTags.Count.Should().Be(2);
pluginsExecutionResults.AdditionalTags[0].Should().Be("9000.15.0427.2036");
pluginsExecutionResults.AdditionalTags[1].Should().Be("9.0.1");
}
}
public void ClusterControllerPluginErrorTest()
{
var testCases = new List <PluginTestCase>
{
new PluginTestCase
{
LogType = LogType.ClusterController,
LogContents = "2018-08-08 11:10:12.705 +1000 pool-18-thread-1 ERROR : com.tableausoftware.cluster.http.HttpServiceMonitor - IOException connecting to HTTP server at http://localhost:8000/favicon.ico",
LogFileInfo = new LogFileInfo("clustercontroller.log", @"folder1/clustercontroller.log", "worker1", new DateTime(2019, 04, 12, 13, 33, 31)),
LineNumber = 10,
ExpectedOutput = new
{
FileName = "clustercontroller.log",
FilePath = @"folder1/clustercontroller.log",
LineNumber = 10,
Timestamp = new DateTime(2018, 08, 08, 11, 10, 12, 705),
Worker = "worker1",
Class = "com.tableausoftware.cluster.http.HttpServiceMonitor",
Message = "IOException connecting to HTTP server at http://localhost:8000/favicon.ico",
Severity = "ERROR",
}
},
};
var testWriterFactory = new TestWriterFactory();
using (var plugin = new ClusterControllerPlugin())
{
plugin.Configure(testWriterFactory, null, null, new NullLoggerFactory());
foreach (var testCase in testCases)
{
var logLine = testCase.GetLogLine();
plugin.ProcessLogLine(logLine, testCase.LogType);
}
}
var expectedOutput = testCases.Select(testCase => testCase.ExpectedOutput).ToList();
var cceDs = new DataSetInfo("ClusterController", "ClusterControllerErrors");
var testWriter = testWriterFactory.Writers[cceDs] as TestWriter <ClusterControllerError>;
testWriterFactory.Writers.Count.Should().Be(_testWriterCount);
testWriter.WasDisposed.Should().Be(true);
testWriter.ReceivedObjects.Should().BeEquivalentTo(expectedOutput);
}
public void RunTestCases_ReplayPlugin()
{
var testWriterFactory = new TestWriterFactory();
foreach (var testCase in _testCases)
{
var jsonFile = $"Playback{testCase.LineNumber}.json";
var plugin = new ReplayerPlugin();
var configDict = new Dictionary <string, string>()
{
{ "ReplayerOutputDirectory", Path.Combine(Directory.GetCurrentDirectory(), "ReplayerOutput") },
{ "ReplayFileName", jsonFile }
};
var config = ConfigGenerator.GetConfigFromDictionary(configDict);
plugin.Configure(testWriterFactory, config, null, new NullLoggerFactory());
var testLines = testCase.LogContents.ToString();
var logLines = testLines.Split("\n");
// the first line will be the Apache event
var apacheLine = logLines[0];
LogLine l1 = new LogLine(new ReadLogLineResult(1, apacheLine), testCase.LogFileInfo);
plugin.ProcessLogLine(l1, LogType.Apache);
// the remaining lines will be the VizqlserverCpp events
for (var i = 1; i < logLines.Length; i++)
{
var l2 = new LogLine(new ReadLogLineResult(i + 1, JsonConvert.DeserializeObject <NativeJsonLogsBaseEvent>(logLines[i])), testCase.LogFileInfo);
plugin.ProcessLogLine(l2, LogType.VizqlserverCpp);
}
plugin.CompleteProcessing();
var fullJsonFile = Path.Combine("ReplayerOutput", jsonFile);
var output = File.ReadAllText(fullJsonFile);
var actualJson = JToken.Parse(output);
var expectedJson = JToken.Parse(testCase.ExpectedOutput.ToString());
actualJson.ToString().Should().Be(expectedJson.ToString());
if (File.Exists(fullJsonFile))
{
File.Delete(fullJsonFile);
}
}
}
public void ClusterControllerPluginZookeeperErrorTest()
{
var testCases = new List <PluginTestCase>
{
new PluginTestCase
{
LogType = LogType.Zookeeper,
LogContents = "2018-08-08 11:01:48.490 +1000 14754 main : ERROR org.apache.zookeeper.server.quorum.QuorumPeerConfig - Invalid configuration, only one server specified (ignoring)",
LogFileInfo = _testZookeeperLogFileInfo,
LineNumber = 1,
ExpectedOutput = new
{
FileName = _testZookeeperLogFileInfo.FileName,
FilePath = _testZookeeperLogFileInfo.FilePath,
LineNumber = 1,
Timestamp = new DateTime(2018, 08, 08, 11, 01, 48, 490),
Worker = _testZookeeperLogFileInfo.Worker,
Class = "org.apache.zookeeper.server.quorum.QuorumPeerConfig",
Message = "Invalid configuration, only one server specified (ignoring)",
Severity = "ERROR",
}
},
};
var testWriterFactory = new TestWriterFactory();
using (var plugin = new ClusterControllerPlugin())
{
plugin.Configure(testWriterFactory, null, null, new NullLoggerFactory());
foreach (var testCase in testCases)
{
var logLine = testCase.GetLogLine();
plugin.ProcessLogLine(logLine, testCase.LogType);
}
}
var expectedOutput = testCases.Select(testCase => testCase.ExpectedOutput).ToList();
var zkDs = new DataSetInfo("ClusterController", "ZookeeperErrors");
var testWriter = testWriterFactory.Writers[zkDs] as TestWriter <ZookeeperError>;
testWriterFactory.Writers.Count.Should().Be(_testWriterCount);
testWriter.WasDisposed.Should().Be(true);
testWriter.ReceivedObjects.Should().BeEquivalentTo(expectedOutput);
}
public void ClusterControllerPluginZookeeperFsyncLatencyTest()
{
var testCases = new List <PluginTestCase>
{
new PluginTestCase
{
LogType = LogType.Zookeeper,
LogContents = "2018-10-02 22:09:01.927 +0000 SyncThread:0 : WARN org.apache.zookeeper.server.persistence.FileTxnLog - fsync-ing the write ahead log in SyncThread:0 took 1013ms which will adversely effect operation latency. See the ZooKeeper troubleshooting guide",
LogFileInfo = _testZookeeperLogFileInfo,
LineNumber = 1725,
ExpectedOutput = new
{
FileName = _testZookeeperLogFileInfo.FileName,
FilePath = _testZookeeperLogFileInfo.FilePath,
LineNumber = 1725,
Timestamp = new DateTime(2018, 10, 02, 22, 09, 01, 927),
Worker = _testZookeeperLogFileInfo.Worker,
FsyncLatencyMs = 1013,
}
},
};
var testWriterFactory = new TestWriterFactory();
using (var plugin = new ClusterControllerPlugin())
{
plugin.Configure(testWriterFactory, null, null, new NullLoggerFactory());
foreach (var testCase in testCases)
{
var logLine = testCase.GetLogLine();
plugin.ProcessLogLine(logLine, testCase.LogType);
}
}
var expectedOutput = testCases.Select(testCase => testCase.ExpectedOutput).ToList();
var zkDs = new DataSetInfo("ClusterController", "ZookeeperFsyncLatencies");
var testWriter = testWriterFactory.Writers[zkDs] as TestWriter <ZookeeperFsyncLatency>;
testWriterFactory.Writers.Count.Should().Be(_testWriterCount);
testWriter.WasDisposed.Should().Be(true);
testWriter.ReceivedObjects.Should().BeEquivalentTo(expectedOutput);
}
public void ClusterControllerPluginPostgresActionTest()
{
var testCases = new List <PluginTestCase>
{
new PluginTestCase
{
LogType = LogType.ClusterController,
LogContents = "2018-08-08 15:04:51.901 +1000 Thread-6 INFO : com.tableausoftware.cluster.postgres.PostgresManager - PostgresManager stop",
LogFileInfo = _testClusterControllerLogFileInfo,
LineNumber = 101,
ExpectedOutput = new
{
FileName = _testClusterControllerLogFileInfo.FileName,
FilePath = _testClusterControllerLogFileInfo.FilePath,
LineNumber = 101,
Timestamp = new DateTime(2018, 08, 08, 15, 04, 51, 901),
Worker = _testClusterControllerLogFileInfo.Worker,
Action = "Stop",
}
},
};
var testWriterFactory = new TestWriterFactory();
using (var plugin = new ClusterControllerPlugin())
{
plugin.Configure(testWriterFactory, null, null, new NullLoggerFactory());
foreach (var testCase in testCases)
{
var logLine = testCase.GetLogLine();
plugin.ProcessLogLine(logLine, testCase.LogType);
}
}
var expectedOutput = testCases.Select(testCase => testCase.ExpectedOutput).ToList();
var ccpaDs = new DataSetInfo("ClusterController", "ClusterControllerPostgresActions");
var testWriter = testWriterFactory.Writers[ccpaDs] as TestWriter <ClusterControllerPostgresAction>;
testWriterFactory.Writers.Count.Should().Be(_testWriterCount);
testWriter.WasDisposed.Should().Be(true);
testWriter.ReceivedObjects.Should().BeEquivalentTo(expectedOutput);
}
public void BadInput()
{
var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
var testWriterFactory = new TestWriterFactory();
using (var plugin = new FilestorePlugin())
{
plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());
var wrongContentFormat = new LogLine(new ReadLogLineResult(123, 1234), TestLogFileInfo);
var nullContent = new LogLine(new ReadLogLineResult(123, null), TestLogFileInfo);
plugin.ProcessLogLine(wrongContentFormat, LogType.Apache);
plugin.ProcessLogLine(nullContent, LogType.Apache);
}
testWriterFactory.AssertAllWritersAreDisposedAndEmpty(1);
processingNotificationsCollector.TotalErrorsReported.Should().Be(2);
}
public void BadOrNoOpInput()
{
var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
var testWriterFactory = new TestWriterFactory();
using (var plugin = new HyperPlugin())
{
plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());
var wrongContentFormat = new LogLine(new ReadLogLineResult(123, "Hyper doesn't expect string"), TestLogFileInfo);
var nullContent = new LogLine(new ReadLogLineResult(123, null), TestLogFileInfo);
plugin.ProcessLogLine(wrongContentFormat, LogType.VizqlserverCpp);
plugin.ProcessLogLine(nullContent, LogType.VizqlserverCpp);
}
testWriterFactory.AssertAllWritersAreDisposedAndEmpty(2);
processingNotificationsCollector.TotalErrorsReported.Should().Be(2);
}
public void TestWithBothFiles()
{
var testWriterFactory = new TestWriterFactory();
using (var plugin = new LogShark.Plugins.Config.ConfigPlugin())
{
plugin.Configure(testWriterFactory, null, null, new NullLoggerFactory());
foreach (var testCase in _testCases)
{
plugin.ProcessLogLine(testCase.GetLogLine(), testCase.LogType);
}
var pluginsExecutionResults = plugin.CompleteProcessing();
pluginsExecutionResults.HasAdditionalTags.Should().BeFalse();
}
VerifyWritersState(testWriterFactory.Writers, _expectedConfigEntries, _expectedProcessInfoRecords);
}
public void SpecialCase_CompletelyIncompatibleJsonInArtData()
{
var testWriterFactory = new TestWriterFactory();
using (var plugin = new ArtPlugin())
{
plugin.Configure(testWriterFactory, null, null, new NullLoggerFactory());
var incorrectJsonInArtPayload = new LogLine(new ReadLogLineResult(123, new NativeJsonLogsBaseEvent {
EventType = "msg", ArtData = JToken.FromObject(new { SomeOtherKey = "test" })
}), TestLogFileInfo);
plugin.ProcessLogLine(incorrectJsonInArtPayload, LogType.VizqlserverCpp);
}
var testWriter = testWriterFactory.GetOneWriterAndVerifyOthersAreEmptyAndDisposed <FlattenedArtEvent>("Art", 2);
testWriter.ReceivedObjects.Count.Should().Be(1);
testWriter.ReceivedObjects.First().Should().NotBeNull(); // All props are null, but object still exists
}
public void TestWithTabsvcOnly()
{
var testWriterFactory = new TestWriterFactory();
using (var plugin = new LogShark.Plugins.Config.ConfigPlugin())
{
plugin.Configure(testWriterFactory, null, null, new NullLoggerFactory());
var filteredTestCases = _testCases.Where(testCase => testCase.LogType == LogType.TabsvcYml);
foreach (var testCase in filteredTestCases)
{
plugin.ProcessLogLine(testCase.GetLogLine(), testCase.LogType);
}
}
var expectedConfigEntries = _expectedConfigEntries.Where(entry => entry.FileName == "tabsvc.yml");
VerifyWritersState(testWriterFactory.Writers, expectedConfigEntries, new List <ConfigProcessInfo>());
testWriterFactory.Writers.Count.Should().Be(2);
}
public void UnsupportedLogType()
{
var testWriterFactory = new TestWriterFactory();
using (var plugin = new LogShark.Plugins.TabadminController.TabadminControllerPlugin())
{
plugin.Configure(testWriterFactory, null, _processingNotificationsCollectorMock.Object, new NullLoggerFactory());
var someLogLine = new LogLine(
new ReadLogLineResult(123, "2020-09-28 17:47:01.730 -0500 pool-20-thread-1 : INFO com.tableausoftware.tabadmin.webapp.asyncjobs.AsyncJobService - Running job 117 of type StartServerJob"),
TestLogFileInfo);
Action wrongLogTypeAction = () => { plugin.ProcessLogLine(someLogLine, LogType.Apache); };
wrongLogTypeAction.Should().Throw <LogSharkProgramLogicException>();
plugin.CompleteProcessing();
}
testWriterFactory.AssertAllWritersAreDisposedAndEmpty(1);
_processingNotificationsCollectorMock.VerifyNoOtherCalls();
}
public void NotAJavaLine()
{
var testWriterFactory = new TestWriterFactory();
var logLine = new LogLine(new ReadLogLineResult(123, "Not a Java Log Line"), TestLogFileInfo);
using (var plugin = new LogShark.Plugins.TabadminController.TabadminControllerPlugin())
{
plugin.Configure(testWriterFactory, null, _processingNotificationsCollectorMock.Object, new NullLoggerFactory());
plugin.ProcessLogLine(logLine, LogType.TabadminControllerJava);
plugin.CompleteProcessing();
}
testWriterFactory.AssertAllWritersAreDisposedAndEmpty(1);
_processingNotificationsCollectorMock.Verify(m => m.ReportError(
It.IsAny <string>(),
logLine,
nameof(LogShark.Plugins.TabadminController.TabadminControllerPlugin)),
Times.Once);
_processingNotificationsCollectorMock.VerifyNoOtherCalls();
}
public void TestWithWorkgroupOnly()
{
var testWriterFactory = new TestWriterFactory();
using (var plugin = new LogShark.Plugins.Config.ConfigPlugin())
{
plugin.Configure(testWriterFactory, null, null, new NullLoggerFactory());
var filteredTestCases = _testCases.Where(testCase => testCase.LogType == LogType.WorkgroupYml);
foreach (var testCase in filteredTestCases)
{
plugin.ProcessLogLine(testCase.GetLogLine(), testCase.LogType);
}
plugin.CompleteProcessing();
}
var expectedConfigEntries = _expectedConfigEntries.Where(entry => entry.FileName == "workgroup.yml");
VerifyWritersState(testWriterFactory.Writers, expectedConfigEntries, _expectedProcessInfoRecords);
}
public void BadInput()
{
var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
var testWriterFactory = new TestWriterFactory();
using (var plugin = new NetstatPlugin())
{
plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());
var logFileInfo = new LogFileInfo("netstat-anp.txt", @"folder1/netstat-anp.txt", "worker1", new DateTime(2019, 04, 12, 13, 33, 31));
var wrongContentFormat = new LogLine(new ReadLogLineResult(123, 1234), logFileInfo);
var nullContent = new LogLine(new ReadLogLineResult(123, null), logFileInfo);
plugin.ProcessLogLine(wrongContentFormat, LogType.NetstatLinux);
plugin.ProcessLogLine(nullContent, LogType.NetstatLinux);
plugin.ProcessLogLine(nullContent, LogType.NetstatWindows);
}
testWriterFactory.AssertAllWritersAreDisposedAndEmpty(1);
processingNotificationsCollector.TotalErrorsReported.Should().Be(3);
}
public void BadInput()
{
var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
var testWriterFactory = new TestWriterFactory();
using (var plugin = new LogShark.Plugins.Config.ConfigPlugin())
{
plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());
var wrongContentFormat = new LogLine(new ReadLogLineResult(123, "stringIsNotWhatConfigPluginExpects"), TestWorkgroupYmlInfo);
var nullContent = new LogLine(new ReadLogLineResult(123, null), TestWorkgroupYmlInfo);
plugin.ProcessLogLine(wrongContentFormat, LogType.WorkgroupYml);
plugin.ProcessLogLine(nullContent, LogType.WorkgroupYml);
plugin.CompleteProcessing();
}
VerifyWritersState(testWriterFactory.Writers, new List <ConfigEntry>(), new List <ConfigProcessInfo>());
processingNotificationsCollector.TotalErrorsReported.Should().Be(2);
}
public void SearchServerPluginTest()
{
var testWriterFactory = new TestWriterFactory();
using (var plugin = new SearchServerPlugin())
{
plugin.Configure(testWriterFactory, null, null, new NullLoggerFactory());
foreach (var testCase in _testCases)
{
var logLine = testCase.GetLogLine();
plugin.ProcessLogLine(logLine, testCase.LogType);
}
}
var expectedOutput = _testCases.Select(testCase => testCase.ExpectedOutput).ToList();
var testWriter = testWriterFactory.Writers.Values.First() as TestWriter <SearchServerEvent>;
testWriterFactory.Writers.Count.Should().Be(1);
testWriter.WasDisposed.Should().Be(true);
testWriter.ReceivedObjects.Should().BeEquivalentTo(expectedOutput);
}
