public void TestNonGenericCreateEventSource()
{
var config = TestUtility.GetConfig("Sources", "JsonLog1");
string timetampFormat = config["TimestampFormat"];
string timestampField = config["TimestampField"];
IRecordParser parser = new SingleLineJsonParser(timestampField, timetampFormat, NullLogger.Instance);
PluginContext context = new PluginContext(config, null, null);
var source = DirectorySourceFactory.CreateEventSource(context, parser);
Assert.NotNull(source);
Assert.IsType <DirectorySource <JObject, LogContext> >(source);
}
public async Task IncludeSubdirectoriesWithIncludeDirectoryFilterWithMultipleLevelSubdirectoriesTest()
{
var testName = "IncludeSubdirectoriesWithIncludeDirectoryFilterWithMultipleLevelSubdirectoriesTest";
var filter = "*.*";
var subDir1 = "CPU";
var subDir2 = "Memory";
var subDir3 = "CPU-1";
var subDir4 = "Network";
var directory = new TestDirectory(testName)
{
SubDirectories = new TestDirectory[]
{
new TestDirectory(subDir1)
{
SubDirectories = new TestDirectory[] { new TestDirectory(subDir3) }
},
new TestDirectory(subDir2),
new TestDirectory(subDir4)
}
};
var config = TestUtility.GetConfig("Sources", "IncludeSubdirectories");
config["IncludeDirectoryFilter"] = $@"CPU;CPU{Path.DirectorySeparatorChar}CPU-1";
await CreateAndRunWatcher(testName, filter, config, async (logRecords) =>
{
var filePath1 = Path.Combine(TestUtility.GetTestHome(), testName, subDir1, "test");
var filePath2 = Path.Combine(TestUtility.GetTestHome(), testName, subDir2, "test");
var filePath3 = Path.Combine(Path.Combine(TestUtility.GetTestHome(), testName, subDir1), subDir3, "test");
var filePath4 = Path.Combine(TestUtility.GetTestHome(), testName, subDir4, "test");
this.WriteLog(filePath1, "this is a test 1");
this.WriteLog(filePath2, "this is a test 2");
this.WriteLog(filePath3, "this is a test 3");
this.WriteLog(filePath4, "this is a test 4");
await Task.Delay(2000);
Assert.Equal(2, logRecords.Count);
var env1 = (ILogEnvelope)logRecords[0];
Assert.Equal("test", env1.FileName);
Assert.Equal(filePath1, env1.FilePath);
var env2 = (ILogEnvelope)logRecords[1];
Assert.Equal("test", env2.FileName);
Assert.Equal(filePath3, env2.FilePath);
}, new SingleLineRecordParser(), directory);
}
public void TestJsonLog(string sourceId)
{
using (Stream stream = Utility.StringToStream(LOG))
using (StreamReader sr = new StreamReader(stream))
{
var config = TestUtility.GetConfig("Sources", sourceId);
var records = ParseRecords(sr, config);
Assert.Equal(2, records.Count);
var record0 = records[0];
Assert.Equal(new DateTime(2018, 9, 21, 8, 38, 50, 972), records[0].Timestamp);
Assert.Equal("UserProfile", record0.Data["ul-log-data"]["method_name"]);
Assert.Equal("INFO", record0.Data["ul-tag-status"]);
}
}
public void ConvertsPowerShellSource()
{
var records = new List <Envelope <JObject> >
{
new Envelope <JObject>(JObject.Parse("{\"ComputerName\":\"MyComputer\",\"Name\":\"TrustedInstaller\",\"Status\":\"Running\"}")),
new Envelope <JObject>(JObject.Parse("{\"ComputerName\":\"MyComputer\",\"Name\":\"WinRM\",\"Status\":\"Stopped\"}"))
};
var config = TestUtility.GetConfig("Pipes", "PSEMFTestPipe");
using (var logger = new MemoryLogger(nameof(EMFPipeTests)))
{
var context = new PluginContext(config, logger, null);
var source = new MockEventSource <JObject>(context);
var sink = new MockEventSink(context);
context.ContextData[PluginContext.SOURCE_TYPE] = source.GetType();
context.ContextData[PluginContext.SOURCE_OUTPUT_TYPE] = source.GetOutputType();
context.ContextData[PluginContext.SINK_TYPE] = sink.GetType();
var pipe = new EMFPipe <JObject>(context);
source.Subscribe(pipe);
pipe.Subscribe(sink);
foreach (var r in records)
{
source.MockEvent(r.Data);
}
Assert.Equal(2, sink.Records.Count);
var jo = JObject.Parse(sink.Records.First());
Assert.Equal("PSNamespace", jo["_aws"]["CloudWatchMetrics"][0]["Namespace"].ToString());
Assert.Equal("ServiceStatus", jo["_aws"]["CloudWatchMetrics"][0]["Metrics"][0]["Name"].ToString());
Assert.Equal(1, jo["ServiceStatus"].ToObject <int>());
Assert.Equal("Running", jo["Status"].ToString());
Assert.Equal("TrustedInstaller", jo["Name"].ToString());
var dims = jo["_aws"]["CloudWatchMetrics"][0]["Dimensions"][0].ToArray().Select(i => i.ToString()).ToList();
Assert.Equal(3, dims.Count);
Assert.Contains("Name", dims);
Assert.Contains("ComputerName", dims);
Assert.Contains("Status", dims);
jo = JObject.Parse(sink.Records.Last());
Assert.Equal("Stopped", jo["Status"].ToString());
Assert.Equal("WinRM", jo["Name"].ToString());
}
}
public void ConvertsIISLogs()
{
var logs = new string[]
{
"2017-05-31 06:00:30 W3SVC1 EC2AMAZ-HCNHA1G 10.10.10.10 POST /DoWork - 443 EXAMPLE\\jonsmith 11.11.11.11 HTTP/1.1 SEA-HDFEHW23455/1.0.9/jonsmith - - localhost 500 2 0 1950 348 158",
"2017-05-31 06:00:30 W3SVC1 EC2AMAZ-HCNHA1G ::1 GET / - 80 - ::1 HTTP/1.1 Mozilla/5.0+(Windows+NT+10.0;+WOW64;+Trident/7.0;+rv:11.0)+like+Gecko - - localhost 200 0 0 950 348 128",
"2017-05-31 06:00:30 W3SVC1 EC2AMAZ-HCNHA1G ::1 GET / - 80 - ::1 HTTP/1.1 Mozilla/5.0+(Windows+NT+10.0;+WOW64;+Trident/7.0;+rv:11.0)+like+Gecko - - localhost 401 1 0 50 348 150",
"2017-05-31 06:00:30 W3SVC1 EC2AMAZ-HCNHA1G ::1 GET / - 80 - ::1 HTTP/1.1 Mozilla/5.0+(Windows+NT+10.0;+WOW64;+Trident/7.0;+rv:11.0)+like+Gecko - - localhost 503 7 0 550 348 192",
"2017-05-31 06:00:30 W3SVC1 EC2AMAZ-HCNHA1G ::1 GET /iisstart.png - 80 - ::1 HTTP/1.1 Mozilla/5.0+(Windows+NT+10.0;+WOW64;+Trident/7.0;+rv:11.0)+like+Gecko - http://localhost/ localhost 200 0 0 99960 317 3"
};
var config = TestUtility.GetConfig("Pipes", "IISEMFTestPipe");
var context = new PluginContext(config, NullLogger.Instance, null);
var sink = new MockEventSink(context);
context.ContextData[PluginContext.SINK_TYPE] = sink.GetType();
var pipe = new EMFPipe <IDictionary <string, string> >(context);
pipe.Subscribe(sink);
var records = ParseW3SVCLogs(logs);
foreach (var record in records)
{
pipe.OnNext(new Envelope <IDictionary <string, string> >(record));
}
Assert.Equal(5, sink.Records.Count);
var jo = JObject.Parse(sink.Records.First());
Assert.Equal("10.10.10.10", jo["s-ip"].ToString());
Assert.Equal("POST", jo["cs-method"].ToString());
Assert.Equal("/DoWork", jo["cs-uri-stem"].ToString());
Assert.Equal("443", jo["s-port"].ToString());
Assert.Equal("11.11.11.11", jo["c-ip"].ToString());
Assert.Equal(@"EXAMPLE\jonsmith", jo["cs-username"].ToString());
Assert.Equal(@"SEA-HDFEHW23455/1.0.9/jonsmith", jo["cs-User-Agent"].ToString());
Assert.Equal("500", jo["sc-status"].ToString());
Assert.Equal("2", jo["sc-substatus"].ToString());
Assert.Equal("IISNamespace", jo["_aws"]["CloudWatchMetrics"][0]["Namespace"].ToString());
Assert.Equal("time-taken", jo["_aws"]["CloudWatchMetrics"][0]["Metrics"][0]["Name"].ToString());
}
public void TestTimestampWithIncludeSubdirectories()
{
string sourceId = "TestTimestampWithIncludeSubdirectories";
var subDir1 = "CPU";
var subDir2 = "Memory";
var subdirectories = new string[] { subDir1, subDir2 };
Setup(sourceId, subdirectories);
WriteLogs("A", 1, subDir1);
Thread.Sleep(200);
DateTime timestamp = DateTime.Now;
Thread.Sleep(1000);
WriteLogs("A", 2, subDir1);
WriteLogs("B", 3, subDir2);
ListEventSink logRecords = new ListEventSink();
var config = TestUtility.GetConfig("Sources", "IncludeSubdirectories");
DirectorySource <IDictionary <string, string>, LogContext> watcher = CreateDirectorySource(sourceId, "log_?.log", logRecords, config);
watcher.InitialPosition = InitialPositionEnum.Timestamp;
watcher.InitialPositionTimestamp = timestamp;
watcher.Start();
Thread.Sleep(2000);
watcher.Stop();
Assert.Equal(5, logRecords.Count);
logRecords.Clear();
WriteLogs("B", 1, subDir1);
WriteLogs("C", 5, subDir2);
watcher.Start();
WriteLogs("D", 7, subDir1);
Thread.Sleep(2000);
watcher.Stop();
Assert.Equal(13, logRecords.Count);
}
public void TestStringPipe()
{
var config = TestUtility.GetConfig("Pipes", "TestPipe");
var context = new PluginContext(config, null, null);
var source = new MockEventSource <string>(context);
var sink = new MockEventSink(context);
context.ContextData[PluginContext.SOURCE_TYPE] = source.GetType();
context.ContextData[PluginContext.SINK_TYPE] = sink.GetType();
var pipe = new PipeFactory().CreateInstance(PipeFactory.REGEX_FILTER_PIPE, context);
source.Subscribe(pipe);
pipe.Subscribe(sink);
string record1 = "24,09/29/17,00:00:04,Database Cleanup Begin,,,,,0,6,,,,,,,,,0";
source.MockEvent(record1);
source.MockEvent("25,09/29/17,00:00:04,0 leases expired and 0 leases deleted,,,,,0,6,,,,,,,,,0");
Assert.Single(sink.Records);
Assert.Equal(record1, sink.Records[0]);
}
public async Task IncludeMultipleLevelSubdirectoriesTest()
{
var testName = "IncludeMultipleLevelSubdirectoriesTest";
var filter = "*.*";
var subDir1 = "CPU";
var subDir2 = "Memory";
var subDir3 = "CPU-1";
var directory = new TestDirectory(testName)
{
SubDirectories = new TestDirectory[]
{
new TestDirectory(subDir1)
{
SubDirectories = new TestDirectory[] { new TestDirectory(subDir3) }
},
new TestDirectory(subDir2)
}
};
var config = TestUtility.GetConfig("Sources", "IncludeSubdirectories");
await CreateAndRunWatcher(testName, filter, config, async (logRecords) =>
{
var filePath1 = Path.Combine(Path.Combine(TestUtility.GetTestHome(), testName, subDir1), subDir3, "test");
var filePath2 = Path.Combine(TestUtility.GetTestHome(), testName, subDir2, "test");
this.WriteLog(filePath1, "test test");
this.WriteLog(filePath2, "test test test test test test test test test test test test test test");
await Task.Delay(2000);
Assert.Equal(2, logRecords.Count);
var env1 = (ILogEnvelope)logRecords[0];
Assert.Equal("test", env1.FileName);
Assert.Equal(filePath1, env1.FilePath);
var env2 = (ILogEnvelope)logRecords[1];
Assert.Equal("test", env2.FileName);
Assert.Equal(filePath2, env2.FilePath);
}, new SingeLineRecordParser(), directory);
}
public void TestNPSLog()
{
string log = @"""NPS-MASTER"",""IAS"",03/22/2018,23:07:55,1,""demouser"",""demodomain\demouser"",,,,,,,,0,""10.62.86.137"",""Nate - Test 1"",,,,,,,1,,0,""311 1 10.1.0.213 03/15/2018 08:14:29 1"",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,""Use Windows authentication for all users"",1,,,,
""NPS-MASTER"",""IAS"",03/22/2018,23:07:55,3,,""demodomain\demouser"",,,,,,,,0,""10.62.86.137"",""Nate - Test 1"",,,,,,,1,,16,""311 1 10.1.0.213 03/15/2018 08:14:29 1"",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,""Use Windows authentication for all users"",1,,,,
""NPS-MASTER"",""IAS"",03/22/2018,23:08:38,1,""demouser"",""demodomain\demouser"",,,,,,,,0,""10.62.86.137"",""Nate - Test 1"",,,,,,,1,,0,""311 1 10.1.0.213 03/15/2018 08:14:29 2"",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,""Use Windows authentication for all users"",1,,,,";
using (Stream stream = Utility.StringToStream(log))
using (StreamReader sr = new StreamReader(stream))
{
var config = TestUtility.GetConfig("Sources", "NPS");
var records = ParseRecords(sr, config);
var record0 = records[0];
Assert.Equal(new DateTime(2018, 3, 22, 23, 7, 55), records[0].Timestamp);
Assert.Equal("NPS-MASTER", record0.Data["ComputerName"]);
Assert.Equal("IAS", record0.Data["ServiceName"]);
var record1 = records[1];
Assert.Equal("demodomain\\demouser", record1.Data["Fully-Qualified-Distinguished-Name"]);
Assert.Equal("10.62.86.137", record1.Data["Client-IP-Address"]);
}
}
public async Task TestJsonLog(string sourceId)
{
var config = TestUtility.GetConfig("Sources", sourceId);
var dir = config["Directory"];
Directory.CreateDirectory(dir);
var logFile = Path.Combine(dir, $"{Guid.NewGuid()}.log");
File.WriteAllText(logFile, LOG);
try
{
var source = new DirectorySource <JObject, LogContext>(dir, config["FileNameFilter"],
1000, new PluginContext(config, NullLogger.Instance, null),
new SingleLineJsonParser(config["TimestampField"], config["TimestampFormat"], NullLogger.Instance))
{
InitialPosition = InitialPositionEnum.BOS
};
var sink = new ListEventSink();
source.Subscribe(sink);
source.Start();
await Task.Delay(2000);
Assert.Equal(2, sink.Count);
var record0 = sink[0] as LogEnvelope <JObject>;
Assert.Equal(new DateTime(2018, 9, 21, 8, 38, 50, 972), record0.Timestamp);
Assert.Equal("UserProfile", record0.Data["ul-log-data"]["method_name"]);
Assert.Equal("INFO", record0.Data["ul-tag-status"]);
}
finally
{
File.Delete(logFile);
}
}
private MockDependentEventSource <int> GetMockDependentEventSource()
{
var config = TestUtility.GetConfig("Sources", "JsonLog1");
return(new MockDependentEventSource <int>(new PluginContext(config, null, null)));
}
