public void TestEOSWithIncludeSubdirectories() { string sourceId = "TestEOSWithIncludeSubdirectories"; var subDir1 = "CPU"; var subDir2 = "Memory"; var subdirectories = new string[] { subDir1, subDir2 }; Setup(sourceId, subdirectories); WriteLogs("A", 2, subDir1); WriteLogs("B", 3, subDir2); ListEventSink logRecords = new ListEventSink(); var config = TestUtility.GetConfig("Sources", "IncludeSubdirectories"); DirectorySource <IDictionary <string, string>, LogContext> watcher = CreateDirectorySource(sourceId, "log_?.log", logRecords, config); watcher.InitialPosition = InitialPositionEnum.EOS; watcher.Start(); Thread.Sleep(2000); watcher.Stop(); Assert.Empty(logRecords); watcher.Start(); WriteLogs("B", 1, subDir1); WriteLogs("C", 5, subDir2); Thread.Sleep(2000); watcher.Stop(); Assert.Equal(6, logRecords.Count); }
public void TestBookmark() { string sourceId = "TestDirectorySourceBookmark"; Setup(sourceId); WriteLogs("A", 2); WriteLogs("B", 3); ListEventSink logRecords = new ListEventSink(); DirectorySource <IDictionary <string, string>, LogContext> watcher = CreateDirectorySource(sourceId, logRecords); watcher.InitialPosition = InitialPositionEnum.Bookmark; watcher.Start(); Thread.Sleep(2000); watcher.Stop(); Assert.Empty(logRecords); WriteLogs("B", 1); WriteLogs("C", 5); watcher.Start(); WriteLogs("D", 7); Thread.Sleep(2000); watcher.Stop(); Assert.Equal(13, logRecords.Count); }
private async Task CreateAndRunWatcher <TData>(string testName, string filter, IConfiguration config, Func <List <IEnvelope>, Task> testBody, IRecordParser <TData, LogContext> recordParser, ILogger logger, TestDirectory directory = null) { //Create a distinct directory based on testName so that tests can run in parallel string testDir = Path.Combine(TestUtility.GetTestHome(), testName); if (directory == null) { //The following will creates all directories and subdirectories in the specified path unless they already exist. Directory.CreateDirectory(testDir); //Clean up before the test rather than after so that we can inspect the files DeleteFiles(testDir, "*.*"); } else { this.SetUpTestDirectory(directory, TestUtility.GetTestHome()); } ListEventSink logRecords = new ListEventSink(); DirectorySource <TData, LogContext> watcher = new DirectorySource <TData, LogContext> (testDir, filter, 1000, new PluginContext(config, logger, null), recordParser); watcher.Subscribe(logRecords); watcher.Start(); await testBody(logRecords); watcher.Stop(); }
public void TestDirectorySourceMetricsStart() { IConfiguration config = GetConfig("directorySourceTest"); config[ConfigConstants.ID] = "TestDirectorySourceMetricsStart"; MemoryLogger logger = new MemoryLogger(null); KinesisTapMetricsSource metrics = new KinesisTapMetricsSource(new PluginContext(null, null, null, _bookmarkManager, null, null)); DirectorySource <string, LogContext> source = new DirectorySource <string, LogContext>( TestUtility.GetTestHome(), "*.log", 1000, new PluginContext(config, logger, metrics, _bookmarkManager), new SingleLineRecordParser()); MockMetricsSink metricsSink = new MockMetricsSink(3600, new PluginContext(config, logger, metrics, _bookmarkManager)); metrics.Subscribe(metricsSink); source.Start(); source.Stop(); metricsSink.Stop(); Assert.Equal(2, metricsSink.AccumlatedValues.Count); Assert.Equal(0, TestUtility.GetMetricsCount(metricsSink.AccumlatedValues)); }
private async Task CreateAndRunWatcher <TData>(string filter, IConfiguration config, Func <List <IEnvelope>, Task> testBody, IRecordParser <TData, LogContext> recordParser) { ListEventSink logRecords = new ListEventSink(); DirectorySource <TData, LogContext> watcher = new DirectorySource <TData, LogContext> (TestUtility.GetTestHome(), filter, 1000, new PluginContext(config, NullLogger.Instance, null), recordParser, DirectorySourceFactory.CreateLogSourceInfo); watcher.Subscribe(logRecords); watcher.Start(); await testBody(logRecords); watcher.Stop(); }
public void TestTimestamp() { string sourceId = "TestDirectorySourceTimestamp"; Setup(sourceId); WriteLogs("A", 1); Thread.Sleep(200); DateTime timestamp = DateTime.Now; Thread.Sleep(1000); WriteLogs("A", 2); WriteLogs("B", 3); ListEventSink logRecords = new ListEventSink(); DirectorySource <IDictionary <string, string>, LogContext> watcher = CreateDirectorySource(sourceId, logRecords); watcher.InitialPosition = InitialPositionEnum.Timestamp; watcher.InitialPositionTimestamp = timestamp; watcher.Start(); Thread.Sleep(2000); watcher.Stop(); Assert.Equal(5, logRecords.Count); logRecords.Clear(); WriteLogs("B", 1); WriteLogs("C", 5); watcher.Start(); WriteLogs("D", 7); Thread.Sleep(2000); watcher.Stop(); Assert.Equal(13, logRecords.Count); }
public void NullFileNameFilterTest() { using (MemoryLogger logger = new MemoryLogger("memory")) { DirectorySource <string, LogContext> watcher = new DirectorySource <string, LogContext>( TestUtility.GetTestHome(), null, 1000, new PluginContext(null, logger, null), new SingleLineRecordParser()); watcher.Start(); Assert.Equal($"DirectorySource id {null} watching directory {TestUtility.GetTestHome()} with filter started.", logger.LastEntry); watcher.Stop(); Assert.Equal($"DirectorySource id {null} watching directory {TestUtility.GetTestHome()} with filter stopped.", logger.LastEntry); } }
public void StartStopTest() { string filter = "*.*"; using (MemoryLogger logger = new MemoryLogger("memory")) { DirectorySource <string, LogContext> watcher = new DirectorySource <string, LogContext>( TestUtility.GetTestHome(), filter, 1000, new PluginContext(null, logger, null, _bookmarkManager), new SingleLineRecordParser()); watcher.Start(); Assert.Equal($"DirectorySource id {null} watching directory {TestUtility.GetTestHome()} with filter {filter} started.", logger.LastEntry); watcher.Stop(); Assert.Equal($"DirectorySource id {null} watching directory {TestUtility.GetTestHome()} with filter {filter} stopped.", logger.LastEntry); } }
public async Task TestJsonLog(string sourceId) { var config = TestUtility.GetConfig("Sources", sourceId); var dir = config["Directory"]; Directory.CreateDirectory(dir); var logFile = Path.Combine(dir, $"{Guid.NewGuid()}.log"); File.WriteAllText(logFile, LOG); try { var source = new DirectorySource <JObject, LogContext>(dir, config["FileNameFilter"], 1000, new PluginContext(config, NullLogger.Instance, null), new SingleLineJsonParser(config["TimestampField"], config["TimestampFormat"], NullLogger.Instance)) { InitialPosition = InitialPositionEnum.BOS }; var sink = new ListEventSink(); source.Subscribe(sink); source.Start(); await Task.Delay(2000); Assert.Equal(2, sink.Count); var record0 = sink[0] as LogEnvelope <JObject>; Assert.Equal(new DateTime(2018, 9, 21, 8, 38, 50, 972), record0.Timestamp); Assert.Equal("UserProfile", record0.Data["ul-log-data"]["method_name"]); Assert.Equal("INFO", record0.Data["ul-tag-status"]); } finally { File.Delete(logFile); } }