public InputBuffer(int batchSizeInBytes, long beginOffset, ILogFileInfo logFileInfoObj, ThreadSafeQueue <T> logDataBatchQueue, string prefix, ILogMonitorHelper <T> logMonitorHelper, int messageBatchFlushInterval, CancellationContext cancelContext, int maxBatchCount, string instanceName = null) { if (batchSizeInBytes <= 0) { throw new ArgumentOutOfRangeException("batchSizeInByte", "The batch size should be greater than 0."); } if (beginOffset < 0L) { throw new ArgumentOutOfRangeException("beginOffset", "The beginOffset should be equal or greater than 0."); } if (logDataBatchQueue == null) { throw new ArgumentNullException("logDataBatchQueue cannot be null."); } if (messageBatchFlushInterval < 0) { throw new ArgumentOutOfRangeException("messageBatchFlushInterval", "The messageBatchFlushInterval must not be a negative number."); } ArgumentValidator.ThrowIfNull("logFileInfoObj", logFileInfoObj); ArgumentValidator.ThrowIfNull("watermarkFileRef", logFileInfoObj.WatermarkFileObj); if (string.IsNullOrEmpty(logFileInfoObj.FullFileName)) { throw new ArgumentException("fullLogName cannot be null or emtpy."); } ArgumentValidator.ThrowIfNull("cancelContext", cancelContext); this.cancellationContext = cancelContext; this.messageBatchFlushInterval = messageBatchFlushInterval; this.batchSizeInBytes = batchSizeInBytes; this.logDataBatchQueue = logDataBatchQueue; this.watermarkFileRef = logFileInfoObj.WatermarkFileObj; this.maximumBatchCount = ((maxBatchCount <= 0) ? int.MaxValue : maxBatchCount); this.lastFluchCheckTime = DateTime.UtcNow; this.logMonitorHelper = logMonitorHelper; this.fullLogName = logFileInfoObj.FullFileName; this.instance = (string.IsNullOrEmpty(instanceName) ? prefix : instanceName); this.logPrefix = prefix; this.perfCounterInstance = PerfCountersInstanceCache.GetInstance(this.instance); this.shouldBufferBatches = this.ShouldBufferBatches(); this.CreateNewBatch(beginOffset); if (this.shouldBufferBatches) { MessageBatchBase messageBatchBase = this.activeBatch as MessageBatchBase; if (messageBatchBase != null) { messageBatchBase.MessageBatchFlushInterval = this.messageBatchFlushInterval; } } }
public LogMonitor(ConfigInstance config, string logDir, string logPrefixFilter, ILogMonitorHelper <T> logMonitorHelper, string instanceName = null, string wmkFileDiretory = null) { ArgumentValidator.ThrowIfNull("config", config); ArgumentValidator.ThrowIfNull("logDir", logDir); ArgumentValidator.ThrowIfNull("logMonitorHelper", logMonitorHelper); ArgumentValidator.ThrowIfNullOrEmpty("logPrefix", logPrefixFilter); this.logPrefixToBeMonitored = logPrefixFilter; this.config = config; this.logDirectory = logDir; this.logMonitorHelper = logMonitorHelper; this.logsJustFinishedParsing = new ConcurrentDictionary <string, LogFileInfo>(StringComparer.InvariantCultureIgnoreCase); this.watermarkFileHelper = new WatermarkFileHelper(this.logDirectory, wmkFileDiretory); this.instance = (string.IsNullOrWhiteSpace(instanceName) ? this.logPrefixToBeMonitored : instanceName); this.batchQueue = new ThreadSafeQueue <T>(this.config.QueueCapacity); this.knownLogNameToLogFileMap = new ConcurrentDictionary <string, ILogFileInfo>(StringComparer.InvariantCultureIgnoreCase); this.logsNeedProcessing = new List <LogFileInfo>(); this.previousLogDirectories = new HashSet <string>(); this.reprocessingActiveFileWaitTime = Tools.RandomizeTimeSpan(this.config.WaitTimeToReprocessActiveFile, this.config.WaitTimeToReprocessActiveFileRandomRange); this.instanceInstantiateTime = DateTime.UtcNow; this.staleLogs = new List <ILogFileInfo>(); this.veryStaleLogs = new List <ILogFileInfo>(); this.workerThreads = new List <Thread>(); this.maxNumberOfWriterThreads = config.MaxNumOfWriters; this.maxNumberOfReaderThreads = config.MaxNumOfReaders; this.perfCounterInstance = PerfCountersInstanceCache.GetInstance(this.instance); this.perfCounterInstance.TotalIncompleteLogs.RawValue = 0L; this.perfCounterInstance.BatchQueueLength.RawValue = 0L; this.perfCounterInstance.InputBufferBatchCounts.RawValue = 0L; this.perfCounterInstance.InputBufferBackfilledLines.RawValue = 0L; this.perfCounterInstance.TotalLogLinesProcessed.RawValue = 0L; if (Tools.IsRawProcessingType <T>()) { this.perfCounterInstance.RawIncompleteBytes.RawValue = 0L; this.perfCounterInstance.RawTotalLogBytes.RawValue = 0L; this.perfCounterInstance.RawWrittenBytes.RawValue = 0L; this.perfCounterInstance.RawReaderParsedBytes.RawValue = 0L; return; } this.perfCounterInstance.IncompleteBytes.RawValue = 0L; this.perfCounterInstance.TotalLogBytes.RawValue = 0L; this.perfCounterInstance.TotalLogBytesProcessed.RawValue = 0L; this.perfCounterInstance.ReaderParsedBytes.RawValue = 0L; }
internal void LogErrorAndUpdatePerfCounter(long rowStartOffset, long rowEndOffset, Exception exception, ExEventLog.EventTuple eventTuple, LogUploaderEventLogConstants.Message message, string component) { string text = string.Format("Failed to parse log {0} at row ({1}, {2}): \nException: {3}", new object[] { this.FullLogName, rowStartOffset, rowEndOffset, exception }); ExTraceGlobals.ParserTracer.TraceError((long)this.GetHashCode(), text); EventLogger.Logger.LogEvent(eventTuple, exception.Message, new object[] { text }); PerfCountersInstanceCache.GetInstance(this.Instance).TotalInvalidLogLineParseErrors.Increment(); EventNotificationItem.Publish(ExchangeComponent.Name, component, null, text, ResultSeverityLevel.Error, false); ServiceLogger.LogError(ServiceLogger.Component.LogDataBatch, message, text, this.Instance, this.FullLogName); PerfCountersInstanceCache.GetInstance(this.Instance).TotalParseErrors.Increment(); }
public LogReader(ThreadSafeQueue <T> batchQueue, int id, ILogManager logMonitor, ConfigInstance config, string prefix, ILogMonitorHelper <T> logMonitorHelper, string instanceName = null) { ArgumentValidator.ThrowIfNull("batchQueue", batchQueue); ArgumentValidator.ThrowIfNull("logMonitor", logMonitor); ArgumentValidator.ThrowIfNull("config", config); ArgumentValidator.ThrowIfNull("logMonitorHelper", logMonitorHelper); ArgumentValidator.ThrowIfNullOrEmpty("prefix", prefix); if (id < 0) { throw new ArgumentOutOfRangeException("id cannot be negative."); } this.batchQueue = batchQueue; this.id = id; this.logMonitor = logMonitor; this.stopped = false; this.logPrefix = prefix; this.config = config; this.instance = (string.IsNullOrEmpty(instanceName) ? prefix : instanceName); this.messageBatchFlushInterval = (int)this.config.BatchFlushInterval.TotalSeconds; this.logMonitorHelper = logMonitorHelper; this.lastTimeWhenQeueFull = DateTime.UtcNow.Ticks; this.perfCounterInstance = PerfCountersInstanceCache.GetInstance(this.instance); }
public static ILogUploaderPerformanceCounters GetInstance(string instanceName) { ILogUploaderPerformanceCounters logUploaderPerformanceCounters; if (!PerfCountersInstanceCache.instances.TryGetValue(instanceName, out logUploaderPerformanceCounters)) { lock (PerfCountersInstanceCache.InstancesMutex) { if (!PerfCountersInstanceCache.instances.TryGetValue(instanceName, out logUploaderPerformanceCounters)) { Tools.DebugAssert(PerfCountersInstanceCache.GetPerfCountersInstance != null, "Performance counters factory method expected."); logUploaderPerformanceCounters = PerfCountersInstanceCache.GetPerfCountersInstance(instanceName); PerfCountersInstanceCache.instances = new Dictionary <string, ILogUploaderPerformanceCounters>(PerfCountersInstanceCache.instances, StringComparer.OrdinalIgnoreCase) { { instanceName, logUploaderPerformanceCounters } }; } } } return(logUploaderPerformanceCounters); }
public DatabaseWriter(ThreadSafeQueue <T> queue, int id, ConfigInstance config, string instanceName) { this.InitializeDatabaseWriter(queue, id, config, instanceName); this.perfCounterInstance = PerfCountersInstanceCache.GetInstance(this.instance); }
internal void ProcessLog(LogFileInfo log) { FileStream fileStream = null; ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderStartedParsingLog, string.Format("reprocessing or begin to process, isActive {0} +++++++++", log.IsActive), this.instance, log.FullFileName); if (log.Status == ProcessingStatus.NeedProcessing) { log.Status = ProcessingStatus.InProcessing; PerfCountersInstanceCache.GetInstance(this.instance).TotalNewLogsBeginProcessing.Increment(); ExTraceGlobals.ReaderTracer.TraceDebug <string, string>((long)this.GetHashCode(), "{0} started parsing log {1}", Thread.CurrentThread.Name, log.FullFileName); EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderStartedParsingLog, log.FileName, new object[] { this.id, log.FullFileName }); ServiceLogger.LogInfo(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderStartedParsingLog, null, this.instance, log.FullFileName); } try { string defaultLogVersion; fileStream = this.OpenLogWithRetry(log.FullFileName, out defaultLogVersion); if (fileStream == null) { if (!this.isStopRequested) { string text = string.Format("Failed to open log file {0}", log.FullFileName); ExTraceGlobals.ReaderTracer.TraceError((long)this.GetHashCode(), text); EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderFileOpenFailed, log.FullFileName, new object[] { log.FullFileName }); EventNotificationItem.Publish(ExchangeComponent.Name, "FailToOpenLogFile", null, text, ResultSeverityLevel.Error, false); ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 3221229475U, text, this.instance, log.FullFileName); } } else { if (string.IsNullOrWhiteSpace(defaultLogVersion)) { defaultLogVersion = this.logMonitorHelper.GetDefaultLogVersion(); ExTraceGlobals.ReaderTracer.TraceWarning <string, string>((long)this.GetHashCode(), "Failed to figure out version of log file {0}. Use default version {1} instead.", log.FullFileName, defaultLogVersion); EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_FailedToGetVersionFromLogHeader, log.FullFileName, new object[] { log.FullFileName, defaultLogVersion }); ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 2147487660U, null, this.instance, log.FullFileName); } CsvTable logSchema = this.logMonitorHelper.GetLogSchema(new Version(defaultLogVersion)); CsvFieldCache csvFieldCache = new CsvFieldCache(logSchema, fileStream, 32768); int num = 0; this.logHeaderEndOffset = csvFieldCache.Position; ExTraceGlobals.ReaderTracer.TraceDebug <string, long>((long)this.GetHashCode(), "The end offset of the header of {0} is {1}.", log.FullFileName, this.logHeaderEndOffset); bool flag = false; while (!flag) { if (this.CheckServiceStopRequest("ProcessLog()")) { return; } LogFileRange logFileRange = log.WatermarkFileObj.GetBlockToReprocess(); if (logFileRange == null) { logFileRange = log.WatermarkFileObj.GetNewBlockToProcess(); flag = true; if (logFileRange == null) { break; } } if (num == 0) { long startOffset = logFileRange.StartOffset; this.inputBuffer = new InputBuffer <T>(this.config.BatchSizeInBytes, startOffset, log, this.batchQueue, this.logPrefix, this.logMonitorHelper, this.messageBatchFlushInterval, this.cancellationContext, this.config.InputBufferMaximumBatchCount, this.instance); } bool isFirstBlock = num == 0; this.inputBuffer.BeforeDataBlockIsProcessed(logFileRange, isFirstBlock); this.ProcessBlock(csvFieldCache, logFileRange, log); this.inputBuffer.AfterDataBlockIsProcessed(); if (this.isStopRequested) { return; } num++; } this.EnqueueLastBatchParsed(); log.LastProcessedTime = DateTime.UtcNow; if (!log.IsActive) { log.Status = ProcessingStatus.ReadyToWriteToDatabase; ExTraceGlobals.ReaderTracer.TraceInformation <string>(0, (long)this.GetHashCode(), "Finished parsing log {0}", log.FullFileName); EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderFinishedParsingLog, log.FullFileName, new object[] { log.FullFileName }); ServiceLogger.LogInfo(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderFinishedParsingLog, null, this.instance, log.FullFileName); } } } finally { ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderFinishedParsingLog, string.Format("Finished parsing for this round, isActive {0} +++++++++", log.IsActive), this.instance, log.FullFileName); this.logMonitor.ReaderCompletedProcessingLog(log); this.Cleanup(fileStream); } }