public bool ProcessLogs() { LogConsumer.OnProgress -= LogConsumer_OnProgress; LogConsumer.OnProgress += LogConsumer_OnProgress; bool done = false; var logs = LogSource.GetLogSources(); if (logs.Length == 0) { OnLogsProcessingProgress(this.GetType().Name, "No logs to process since no LogSources were retrieved.", LogLevel.Verbose); } for (int k = 0; k < logs.Length; k++) { LogReader.Open(logs[k]); OnLogsProcessingProgress(this.GetType().Name, string.Format("Started processing log source {0} ({1}/{2}).", LogReader.SourceName, k + 1, logs.Length), LogLevel.Verbose); while (!done && LogReader.HasMoreLines) { string line = LogReader.ReadLine(); done = LogConsumer.ConsumeLogEntry(line, LogReader.LineNumber, LogReader.SourceName); } LogReader.Close(); if (done) { OnLogsProcessingProgress(this.GetType().Name, string.Format("LogConsumer is DONE. Terminated processing in log source {0} ({1}/{2}).", logs[k], k + 1, logs.Length), LogLevel.Verbose); break; } else { OnLogsProcessingProgress(this.GetType().Name, string.Format("LogConsumer is NOT DONE after processing log source {0} ({1}/{2}).", logs[k], k + 1, logs.Length), LogLevel.Verbose); } } return(done); }
public void ckmon_files_and_text_files_are_produced() { var secret = Guid.NewGuid().ToString(); var binSecret = Encoding.UTF8.GetBytes(secret); TestHelper.Monitor.Info($"This will appear in ckmon, text files and the console: {secret}"); GrandOutput.Default.Should().NotBeNull(); GrandOutput.Default.Dispose(); Directory.EnumerateFiles(TestHelper.LogFolder, "*.log", SearchOption.AllDirectories) .Select(f => File.ReadAllText(f)) .Count(text => text.Contains(secret)) .Should().Be(1); // ckmon files are now gzipped by default. int count = 0; foreach (var fName in Directory.EnumerateFiles(TestHelper.LogFolder, "*.ckmon", SearchOption.AllDirectories)) { using (var input = LogReader.Open(fName)) { while (input.MoveNext()) { if (input.Current.LogType != LogEntryType.CloseGroup && input.Current.Text.Contains(secret)) { ++count; } } } } count.Should().Be(1); // TestHelper.WithWeakAssemblyResolver(() => TestHelper.Monitor.Info("From WeakAssemblyResolver.")); TestHelper.Monitor.Info($"From WeakAssemblyResolver: {TestHelper.WithWeakAssemblyResolver( () => 3 )}"); }
public void CompressedReadWriteTests() { TestHelper.CleanupTestFolder(); string directoryPath = Path.Combine(TestHelper.TestFolder, "GzipCKMonWriterClientTest"); ActivityMonitor m = new ActivityMonitor(); var client = new CKMonWriterClient(directoryPath, 20000, LogFilter.Undefined, true); m.Output.RegisterClient(client); using (m.OpenWarn().Send("Group test")) { m.Info().Send("Line test"); } // This closes the client: the file is then compressed asynchronously // on a thread from the ThreadPool. Assert.That(client.IsOpened); m.Output.UnregisterClient(client); string ckmonPath = TestHelper.WaitForCkmonFilesInDirectory(directoryPath, 1)[0]; LogReader r = LogReader.Open(ckmonPath); r.MoveNext(); Assert.That(r.Current.LogType, Is.EqualTo(LogEntryType.OpenGroup)); Assert.That(r.Current.Text, Is.EqualTo("Group test")); r.MoveNext(); Assert.That(r.Current.LogType, Is.EqualTo(LogEntryType.Line)); Assert.That(r.Current.Text, Is.EqualTo("Line test")); r.MoveNext(); Assert.That(r.Current.LogType, Is.EqualTo(LogEntryType.CloseGroup)); bool hasRemainingEntries = r.MoveNext(); Assert.That(hasRemainingEntries, Is.False); }
public void BinaryGzip_reconfiguration() { string folder = TestHelper.PrepareLogFolder(nameof(BinaryGzip_reconfiguration)); var h = new Handlers.BinaryFileConfiguration() { Path = folder + @"\FirstPath", UseGzipCompression = false }; var c = new GrandOutputConfiguration().AddHandler(h); var m = new ActivityMonitor(applyAutoConfigurations: false); using (GrandOutput g = new GrandOutput(c)) { g.EnsureGrandOutputClient(m); m.Trace("No Compression."); // We must ensure that the log above will use the current configuration. // This is by design and is a good thing: there is no causality/ordering between log emission and sink reconfigurations. Thread.Sleep(100); h.UseGzipCompression = true; g.ApplyConfiguration(c, true); m.Trace("With Compression."); Thread.Sleep(100); h.Path = folder + @"\SecondPath"; g.ApplyConfiguration(c, true); m.Trace("With Compression (in second folder)."); Thread.Sleep(100); h.UseGzipCompression = false; g.ApplyConfiguration(c, true); m.Trace("No Compression (in second folder)."); } // First file is NOT compressed, the second one is. var fileNamesFirst = Directory.EnumerateFiles(folder + @"\FirstPath").ToList(); fileNamesFirst.Should().BeInAscendingOrder().And.HaveCount(2).And.NotContain(s => s.EndsWith(".tmp"), "Temporary files have been closed."); File.ReadAllText(fileNamesFirst[0]).Should().Contain("No Compression."); File.ReadAllText(fileNamesFirst[1]).Should().NotContain("With Compression.", "Cannot read it in clear text since it is compressed..."); using (var reader = LogReader.Open(fileNamesFirst[1])) { reader.MoveNext().Should().BeTrue(); reader.Current.Text.Should().Be("With Compression."); } // First file is compressed, not the second one. var fileNamesSecond = Directory.EnumerateFiles(folder + @"\SecondPath").ToList(); fileNamesSecond.Should().BeInAscendingOrder().And.HaveCount(2).And.NotContain(s => s.EndsWith(".tmp"), "Temporary files have been closed."); File.ReadAllText(fileNamesSecond[0]).Should().NotContain("With Compression (in second folder).", "The fist file is compressed..."); // We restrict the log entries to the one of our monitor: this filters out the logs from the DispatcherSink. using (var reader = LogReader.Open(fileNamesSecond[0], filter: new LogReader.MulticastFilter(m))) { reader.MoveNext().Should().BeTrue(); reader.Current.Text.Should().Be("With Compression (in second folder)."); } File.ReadAllText(fileNamesSecond[1]).Should().Contain("No Compression (in second folder)."); }
public IndexCkmon(LuceneConfiguration configuration, string pathCkmon) { using (var indexer = new LuceneIndexer(configuration)) using (LogReader reader = LogReader.Open(pathCkmon)) { reader.MoveNext(); for ( ; ;) { indexer.IndexLog(reader.CurrentMulticast, configuration.Directory); if (!reader.MoveNext()) { return; } } } }
public void LevelDbSearchLogTest() { // https://github.com/google/leveldb/blob/master/doc/log_format.md LogReader logReader = new LogReader(new FileInfo(@"TestWorld\000047.log")); logReader.Open(); MemCache memCache = new MemCache(); memCache.Load(logReader); var result = memCache.Get(new byte[] { 0xeb, 0xff, 0xff, 0xff, 0xf3, 0xff, 0xff, 0xff, 0x31 }); Assert.IsTrue(ReadOnlySpan <byte> .Empty != result.Data); Assert.AreEqual(new byte[] { 0xA, 0x00, 0x00, 0x02, 0x05 }, result.Data.Slice(0, 5).ToArray()); }
private static void CopyTo(MulticastLogEntryTextBuilder b, string filePath) { using (LogReader r = LogReader.Open(filePath, 0L, null)) { var o = Console.Out; { int i = 0; while (r.MoveNext()) { b.AppendEntry(r.CurrentMulticast); if (i++ == 20) { o.Write(b.Builder.ToString()); b.Builder.Clear(); i = 0; } } o.Write(b.Builder.ToString()); } } }
public void LevelDbWriteUserDataTest() { // Plan var operations = new KeyValuePair <byte[], MemCache.ResultCacheEntry> [3]; for (int i = 0; i < 3; i++) { byte[] key = TestUtils.FillArrayWithRandomBytes(20); var entry = new MemCache.ResultCacheEntry(); entry.ResultState = ResultState.Exist; entry.Sequence = 10; entry.Data = TestUtils.FillArrayWithRandomBytes(32768); // 32KB is maz size for a block, not that it matters for this operations[i] = new KeyValuePair <byte[], MemCache.ResultCacheEntry>(key, entry); } MemCache memCache = new MemCache(); // Do ReadOnlySpan <byte> result = memCache.EncodeBatch(operations); // Check SpanReader reader = new SpanReader(result); Assert.AreEqual(10, reader.ReadInt64(), "Sequence number"); Assert.AreEqual(3, reader.ReadInt32(), "Operations count"); for (int i = 0; i < 3; i++) { var expectedKey = operations[i].Key; var expectedData = operations[i].Value.Data; Assert.AreEqual(1, reader.ReadByte(), "Operations type PUT"); var keyLen = reader.ReadVarLong(); Assert.AreEqual(expectedKey.Length, keyLen, "Key len"); Assert.AreEqual(expectedKey, reader.Read(keyLen).ToArray(), "Key"); var dataLen = reader.ReadVarLong(); Assert.AreEqual(expectedData.Length, dataLen, "Data len"); Assert.AreEqual(expectedData, reader.Read(dataLen).ToArray(), "Data"); } // test encoding complete blocks var stream = new MemoryStream(); LogWriter writer = new LogWriter(stream); writer.WriteData(result); Assert.Less(0, stream.Length); stream.Position = 0; // Roundtrip test by making sure i can read blocks I've encoded myself. LogReader logReader = new LogReader(stream); logReader.Open(); MemCache memCache2 = new MemCache(); memCache2.Load(logReader); var cache = memCache2._resultCache; Assert.AreEqual(3, cache.Count); int j = 0; foreach (var entry in cache) { var expectedKey = operations[j].Key; var expectedData = operations[j].Value.Data; Assert.AreEqual(ResultState.Exist, entry.Value.ResultState, "Value exists"); Assert.AreEqual(expectedKey.Length, entry.Key.Length, "Key len"); Assert.AreEqual(expectedKey, entry.Key, "Key"); Assert.AreEqual(expectedData.Length, entry.Value.Data.Length, "Data len"); Assert.AreEqual(expectedData, entry.Value.Data, "Data"); j++; } }