public LogMonitor(ConfigInstance config, string logDir, string logPrefixFilter, ILogMonitorHelper <T> logMonitorHelper, string instanceName = null, string wmkFileDiretory = null) { ArgumentValidator.ThrowIfNull("config", config); ArgumentValidator.ThrowIfNull("logDir", logDir); ArgumentValidator.ThrowIfNull("logMonitorHelper", logMonitorHelper); ArgumentValidator.ThrowIfNullOrEmpty("logPrefix", logPrefixFilter); this.logPrefixToBeMonitored = logPrefixFilter; this.config = config; this.logDirectory = logDir; this.logMonitorHelper = logMonitorHelper; this.logsJustFinishedParsing = new ConcurrentDictionary <string, LogFileInfo>(StringComparer.InvariantCultureIgnoreCase); this.watermarkFileHelper = new WatermarkFileHelper(this.logDirectory, wmkFileDiretory); this.instance = (string.IsNullOrWhiteSpace(instanceName) ? this.logPrefixToBeMonitored : instanceName); this.batchQueue = new ThreadSafeQueue <T>(this.config.QueueCapacity); this.knownLogNameToLogFileMap = new ConcurrentDictionary <string, ILogFileInfo>(StringComparer.InvariantCultureIgnoreCase); this.logsNeedProcessing = new List <LogFileInfo>(); this.previousLogDirectories = new HashSet <string>(); this.reprocessingActiveFileWaitTime = Tools.RandomizeTimeSpan(this.config.WaitTimeToReprocessActiveFile, this.config.WaitTimeToReprocessActiveFileRandomRange); this.instanceInstantiateTime = DateTime.UtcNow; this.staleLogs = new List <ILogFileInfo>(); this.veryStaleLogs = new List <ILogFileInfo>(); this.workerThreads = new List <Thread>(); this.maxNumberOfWriterThreads = config.MaxNumOfWriters; this.maxNumberOfReaderThreads = config.MaxNumOfReaders; this.perfCounterInstance = PerfCountersInstanceCache.GetInstance(this.instance); this.perfCounterInstance.TotalIncompleteLogs.RawValue = 0L; this.perfCounterInstance.BatchQueueLength.RawValue = 0L; this.perfCounterInstance.InputBufferBatchCounts.RawValue = 0L; this.perfCounterInstance.InputBufferBackfilledLines.RawValue = 0L; this.perfCounterInstance.TotalLogLinesProcessed.RawValue = 0L; if (Tools.IsRawProcessingType <T>()) { this.perfCounterInstance.RawIncompleteBytes.RawValue = 0L; this.perfCounterInstance.RawTotalLogBytes.RawValue = 0L; this.perfCounterInstance.RawWrittenBytes.RawValue = 0L; this.perfCounterInstance.RawReaderParsedBytes.RawValue = 0L; return; } this.perfCounterInstance.IncompleteBytes.RawValue = 0L; this.perfCounterInstance.TotalLogBytes.RawValue = 0L; this.perfCounterInstance.TotalLogBytesProcessed.RawValue = 0L; this.perfCounterInstance.ReaderParsedBytes.RawValue = 0L; }
public InputBuffer(int batchSizeInBytes, long beginOffset, ILogFileInfo logFileInfoObj, ThreadSafeQueue <T> logDataBatchQueue, string prefix, ILogMonitorHelper <T> logMonitorHelper, int messageBatchFlushInterval, CancellationContext cancelContext, int maxBatchCount, string instanceName = null) { if (batchSizeInBytes <= 0) { throw new ArgumentOutOfRangeException("batchSizeInByte", "The batch size should be greater than 0."); } if (beginOffset < 0L) { throw new ArgumentOutOfRangeException("beginOffset", "The beginOffset should be equal or greater than 0."); } if (logDataBatchQueue == null) { throw new ArgumentNullException("logDataBatchQueue cannot be null."); } if (messageBatchFlushInterval < 0) { throw new ArgumentOutOfRangeException("messageBatchFlushInterval", "The messageBatchFlushInterval must not be a negative number."); } ArgumentValidator.ThrowIfNull("logFileInfoObj", logFileInfoObj); ArgumentValidator.ThrowIfNull("watermarkFileRef", logFileInfoObj.WatermarkFileObj); if (string.IsNullOrEmpty(logFileInfoObj.FullFileName)) { throw new ArgumentException("fullLogName cannot be null or emtpy."); } ArgumentValidator.ThrowIfNull("cancelContext", cancelContext); this.cancellationContext = cancelContext; this.messageBatchFlushInterval = messageBatchFlushInterval; this.batchSizeInBytes = batchSizeInBytes; this.logDataBatchQueue = logDataBatchQueue; this.watermarkFileRef = logFileInfoObj.WatermarkFileObj; this.maximumBatchCount = ((maxBatchCount <= 0) ? int.MaxValue : maxBatchCount); this.lastFluchCheckTime = DateTime.UtcNow; this.logMonitorHelper = logMonitorHelper; this.fullLogName = logFileInfoObj.FullFileName; this.instance = (string.IsNullOrEmpty(instanceName) ? prefix : instanceName); this.logPrefix = prefix; this.perfCounterInstance = PerfCountersInstanceCache.GetInstance(this.instance); this.shouldBufferBatches = this.ShouldBufferBatches(); this.CreateNewBatch(beginOffset); if (this.shouldBufferBatches) { MessageBatchBase messageBatchBase = this.activeBatch as MessageBatchBase; if (messageBatchBase != null) { messageBatchBase.MessageBatchFlushInterval = this.messageBatchFlushInterval; } } }
public LogReader(ThreadSafeQueue <T> batchQueue, int id, ILogManager logMonitor, ConfigInstance config, string prefix, ILogMonitorHelper <T> logMonitorHelper, string instanceName = null) { ArgumentValidator.ThrowIfNull("batchQueue", batchQueue); ArgumentValidator.ThrowIfNull("logMonitor", logMonitor); ArgumentValidator.ThrowIfNull("config", config); ArgumentValidator.ThrowIfNull("logMonitorHelper", logMonitorHelper); ArgumentValidator.ThrowIfNullOrEmpty("prefix", prefix); if (id < 0) { throw new ArgumentOutOfRangeException("id cannot be negative."); } this.batchQueue = batchQueue; this.id = id; this.logMonitor = logMonitor; this.stopped = false; this.logPrefix = prefix; this.config = config; this.instance = (string.IsNullOrEmpty(instanceName) ? prefix : instanceName); this.messageBatchFlushInterval = (int)this.config.BatchFlushInterval.TotalSeconds; this.logMonitorHelper = logMonitorHelper; this.lastTimeWhenQeueFull = DateTime.UtcNow.Ticks; this.perfCounterInstance = PerfCountersInstanceCache.GetInstance(this.instance); }