public void TestEOSWithIncludeSubdirectories()
{
string sourceId = "TestEOSWithIncludeSubdirectories";
var subDir1 = "CPU";
var subDir2 = "Memory";
var subdirectories = new string[] { subDir1, subDir2 };
Setup(sourceId, subdirectories);
WriteLogs("A", 2, subDir1);
WriteLogs("B", 3, subDir2);
ListEventSink logRecords = new ListEventSink();
var config = TestUtility.GetConfig("Sources", "IncludeSubdirectories");
DirectorySource <IDictionary <string, string>, LogContext> watcher = CreateDirectorySource(sourceId, "log_?.log", logRecords, config);
watcher.InitialPosition = InitialPositionEnum.EOS;
watcher.Start();
Thread.Sleep(2000);
watcher.Stop();
Assert.Empty(logRecords);
watcher.Start();
WriteLogs("B", 1, subDir1);
WriteLogs("C", 5, subDir2);
Thread.Sleep(2000);
watcher.Stop();
Assert.Equal(6, logRecords.Count);
}
public void TestBookmark()
{
string sourceId = "TestDirectorySourceBookmark";
Setup(sourceId);
WriteLogs("A", 2);
WriteLogs("B", 3);
ListEventSink logRecords = new ListEventSink();
DirectorySource <IDictionary <string, string>, LogContext> watcher = CreateDirectorySource(sourceId, logRecords);
watcher.InitialPosition = InitialPositionEnum.Bookmark;
watcher.Start();
Thread.Sleep(2000);
watcher.Stop();
Assert.Empty(logRecords);
WriteLogs("B", 1);
WriteLogs("C", 5);
watcher.Start();
WriteLogs("D", 7);
Thread.Sleep(2000);
watcher.Stop();
Assert.Equal(13, logRecords.Count);
}
private async Task CreateAndRunWatcher <TData>(string testName, string filter, IConfiguration config, Func <List <IEnvelope>, Task> testBody, IRecordParser <TData, LogContext> recordParser, ILogger logger, TestDirectory directory = null)
{
//Create a distinct directory based on testName so that tests can run in parallel
string testDir = Path.Combine(TestUtility.GetTestHome(), testName);
if (directory == null)
{
//The following will creates all directories and subdirectories in the specified path unless they already exist.
Directory.CreateDirectory(testDir);
//Clean up before the test rather than after so that we can inspect the files
DeleteFiles(testDir, "*.*");
}
else
{
this.SetUpTestDirectory(directory, TestUtility.GetTestHome());
}
ListEventSink logRecords = new ListEventSink();
DirectorySource <TData, LogContext> watcher = new DirectorySource <TData, LogContext>
(testDir, filter, 1000, new PluginContext(config, logger, null), recordParser);
watcher.Subscribe(logRecords);
watcher.Start();
await testBody(logRecords);
watcher.Stop();
}
public void TestDirectorySourceMetricsStart()
{
IConfiguration config = GetConfig("directorySourceTest");
config[ConfigConstants.ID] = "TestDirectorySourceMetricsStart";
MemoryLogger logger = new MemoryLogger(null);
KinesisTapMetricsSource metrics = new KinesisTapMetricsSource(new PluginContext(null, null, null, _bookmarkManager, null, null));
DirectorySource <string, LogContext> source = new DirectorySource <string, LogContext>(
TestUtility.GetTestHome(),
"*.log",
1000,
new PluginContext(config, logger, metrics, _bookmarkManager),
new SingleLineRecordParser());
MockMetricsSink metricsSink = new MockMetricsSink(3600, new PluginContext(config, logger, metrics, _bookmarkManager));
metrics.Subscribe(metricsSink);
source.Start();
source.Stop();
metricsSink.Stop();
Assert.Equal(2, metricsSink.AccumlatedValues.Count);
Assert.Equal(0, TestUtility.GetMetricsCount(metricsSink.AccumlatedValues));
}
private async Task CreateAndRunWatcher <TData>(string filter, IConfiguration config, Func <List <IEnvelope>, Task> testBody, IRecordParser <TData, LogContext> recordParser)
{
ListEventSink logRecords = new ListEventSink();
DirectorySource <TData, LogContext> watcher = new DirectorySource <TData, LogContext>
(TestUtility.GetTestHome(), filter, 1000, new PluginContext(config, NullLogger.Instance, null), recordParser, DirectorySourceFactory.CreateLogSourceInfo);
watcher.Subscribe(logRecords);
watcher.Start();
await testBody(logRecords);
watcher.Stop();
}
public void TestTimestamp()
{
string sourceId = "TestDirectorySourceTimestamp";
Setup(sourceId);
WriteLogs("A", 1);
Thread.Sleep(200);
DateTime timestamp = DateTime.Now;
Thread.Sleep(1000);
WriteLogs("A", 2);
WriteLogs("B", 3);
ListEventSink logRecords = new ListEventSink();
DirectorySource <IDictionary <string, string>, LogContext> watcher = CreateDirectorySource(sourceId, logRecords);
watcher.InitialPosition = InitialPositionEnum.Timestamp;
watcher.InitialPositionTimestamp = timestamp;
watcher.Start();
Thread.Sleep(2000);
watcher.Stop();
Assert.Equal(5, logRecords.Count);
logRecords.Clear();
WriteLogs("B", 1);
WriteLogs("C", 5);
watcher.Start();
WriteLogs("D", 7);
Thread.Sleep(2000);
watcher.Stop();
Assert.Equal(13, logRecords.Count);
}
public void NullFileNameFilterTest()
{
using (MemoryLogger logger = new MemoryLogger("memory"))
{
DirectorySource <string, LogContext> watcher = new DirectorySource <string, LogContext>(
TestUtility.GetTestHome(),
null,
1000,
new PluginContext(null, logger, null),
new SingleLineRecordParser());
watcher.Start();
Assert.Equal($"DirectorySource id {null} watching directory {TestUtility.GetTestHome()} with filter started.", logger.LastEntry);
watcher.Stop();
Assert.Equal($"DirectorySource id {null} watching directory {TestUtility.GetTestHome()} with filter stopped.", logger.LastEntry);
}
}
public void StartStopTest()
{
string filter = "*.*";
using (MemoryLogger logger = new MemoryLogger("memory"))
{
DirectorySource <string, LogContext> watcher = new DirectorySource <string, LogContext>(
TestUtility.GetTestHome(),
filter,
1000,
new PluginContext(null, logger, null, _bookmarkManager),
new SingleLineRecordParser());
watcher.Start();
Assert.Equal($"DirectorySource id {null} watching directory {TestUtility.GetTestHome()} with filter {filter} started.", logger.LastEntry);
watcher.Stop();
Assert.Equal($"DirectorySource id {null} watching directory {TestUtility.GetTestHome()} with filter {filter} stopped.", logger.LastEntry);
}
}
public async Task TestJsonLog(string sourceId)
{
var config = TestUtility.GetConfig("Sources", sourceId);
var dir = config["Directory"];
Directory.CreateDirectory(dir);
var logFile = Path.Combine(dir, $"{Guid.NewGuid()}.log");
File.WriteAllText(logFile, LOG);
try
{
var source = new DirectorySource <JObject, LogContext>(dir, config["FileNameFilter"],
1000, new PluginContext(config, NullLogger.Instance, null),
new SingleLineJsonParser(config["TimestampField"], config["TimestampFormat"], NullLogger.Instance))
{
InitialPosition = InitialPositionEnum.BOS
};
var sink = new ListEventSink();
source.Subscribe(sink);
source.Start();
await Task.Delay(2000);
Assert.Equal(2, sink.Count);
var record0 = sink[0] as LogEnvelope <JObject>;
Assert.Equal(new DateTime(2018, 9, 21, 8, 38, 50, 972), record0.Timestamp);
Assert.Equal("UserProfile", record0.Data["ul-log-data"]["method_name"]);
Assert.Equal("INFO", record0.Data["ul-tag-status"]);
}
finally
{
File.Delete(logFile);
}
}