public InputBuffer(int batchSizeInBytes, long beginOffset, ILogFileInfo logFileInfoObj, ThreadSafeQueue <T> logDataBatchQueue, string prefix, ILogMonitorHelper <T> logMonitorHelper, int messageBatchFlushInterval, CancellationContext cancelContext, int maxBatchCount, string instanceName = null)
{
if (batchSizeInBytes <= 0)
{
throw new ArgumentOutOfRangeException("batchSizeInByte", "The batch size should be greater than 0.");
}
if (beginOffset < 0L)
{
throw new ArgumentOutOfRangeException("beginOffset", "The beginOffset should be equal or greater than 0.");
}
if (logDataBatchQueue == null)
{
throw new ArgumentNullException("logDataBatchQueue cannot be null.");
}
if (messageBatchFlushInterval < 0)
{
throw new ArgumentOutOfRangeException("messageBatchFlushInterval", "The messageBatchFlushInterval must not be a negative number.");
}
ArgumentValidator.ThrowIfNull("logFileInfoObj", logFileInfoObj);
ArgumentValidator.ThrowIfNull("watermarkFileRef", logFileInfoObj.WatermarkFileObj);
if (string.IsNullOrEmpty(logFileInfoObj.FullFileName))
{
throw new ArgumentException("fullLogName cannot be null or emtpy.");
}
ArgumentValidator.ThrowIfNull("cancelContext", cancelContext);
this.cancellationContext = cancelContext;
this.messageBatchFlushInterval = messageBatchFlushInterval;
this.batchSizeInBytes = batchSizeInBytes;
this.logDataBatchQueue = logDataBatchQueue;
this.watermarkFileRef = logFileInfoObj.WatermarkFileObj;
this.maximumBatchCount = ((maxBatchCount <= 0) ? int.MaxValue : maxBatchCount);
this.lastFluchCheckTime = DateTime.UtcNow;
this.logMonitorHelper = logMonitorHelper;
this.fullLogName = logFileInfoObj.FullFileName;
this.instance = (string.IsNullOrEmpty(instanceName) ? prefix : instanceName);
this.logPrefix = prefix;
this.perfCounterInstance = PerfCountersInstanceCache.GetInstance(this.instance);
this.shouldBufferBatches = this.ShouldBufferBatches();
this.CreateNewBatch(beginOffset);
if (this.shouldBufferBatches)
{
MessageBatchBase messageBatchBase = this.activeBatch as MessageBatchBase;
if (messageBatchBase != null)
{
messageBatchBase.MessageBatchFlushInterval = this.messageBatchFlushInterval;
}
}
}
public LogMonitor(ConfigInstance config, string logDir, string logPrefixFilter, ILogMonitorHelper <T> logMonitorHelper, string instanceName = null, string wmkFileDiretory = null)
{
ArgumentValidator.ThrowIfNull("config", config);
ArgumentValidator.ThrowIfNull("logDir", logDir);
ArgumentValidator.ThrowIfNull("logMonitorHelper", logMonitorHelper);
ArgumentValidator.ThrowIfNullOrEmpty("logPrefix", logPrefixFilter);
this.logPrefixToBeMonitored = logPrefixFilter;
this.config = config;
this.logDirectory = logDir;
this.logMonitorHelper = logMonitorHelper;
this.logsJustFinishedParsing = new ConcurrentDictionary <string, LogFileInfo>(StringComparer.InvariantCultureIgnoreCase);
this.watermarkFileHelper = new WatermarkFileHelper(this.logDirectory, wmkFileDiretory);
this.instance = (string.IsNullOrWhiteSpace(instanceName) ? this.logPrefixToBeMonitored : instanceName);
this.batchQueue = new ThreadSafeQueue <T>(this.config.QueueCapacity);
this.knownLogNameToLogFileMap = new ConcurrentDictionary <string, ILogFileInfo>(StringComparer.InvariantCultureIgnoreCase);
this.logsNeedProcessing = new List <LogFileInfo>();
this.previousLogDirectories = new HashSet <string>();
this.reprocessingActiveFileWaitTime = Tools.RandomizeTimeSpan(this.config.WaitTimeToReprocessActiveFile, this.config.WaitTimeToReprocessActiveFileRandomRange);
this.instanceInstantiateTime = DateTime.UtcNow;
this.staleLogs = new List <ILogFileInfo>();
this.veryStaleLogs = new List <ILogFileInfo>();
this.workerThreads = new List <Thread>();
this.maxNumberOfWriterThreads = config.MaxNumOfWriters;
this.maxNumberOfReaderThreads = config.MaxNumOfReaders;
this.perfCounterInstance = PerfCountersInstanceCache.GetInstance(this.instance);
this.perfCounterInstance.TotalIncompleteLogs.RawValue = 0L;
this.perfCounterInstance.BatchQueueLength.RawValue = 0L;
this.perfCounterInstance.InputBufferBatchCounts.RawValue = 0L;
this.perfCounterInstance.InputBufferBackfilledLines.RawValue = 0L;
this.perfCounterInstance.TotalLogLinesProcessed.RawValue = 0L;
if (Tools.IsRawProcessingType <T>())
{
this.perfCounterInstance.RawIncompleteBytes.RawValue = 0L;
this.perfCounterInstance.RawTotalLogBytes.RawValue = 0L;
this.perfCounterInstance.RawWrittenBytes.RawValue = 0L;
this.perfCounterInstance.RawReaderParsedBytes.RawValue = 0L;
return;
}
this.perfCounterInstance.IncompleteBytes.RawValue = 0L;
this.perfCounterInstance.TotalLogBytes.RawValue = 0L;
this.perfCounterInstance.TotalLogBytesProcessed.RawValue = 0L;
this.perfCounterInstance.ReaderParsedBytes.RawValue = 0L;
}
internal void LogErrorAndUpdatePerfCounter(long rowStartOffset, long rowEndOffset, Exception exception, ExEventLog.EventTuple eventTuple, LogUploaderEventLogConstants.Message message, string component)
{
string text = string.Format("Failed to parse log {0} at row ({1}, {2}): \nException: {3}", new object[]
{
this.FullLogName,
rowStartOffset,
rowEndOffset,
exception
});
ExTraceGlobals.ParserTracer.TraceError((long)this.GetHashCode(), text);
EventLogger.Logger.LogEvent(eventTuple, exception.Message, new object[]
{
text
});
PerfCountersInstanceCache.GetInstance(this.Instance).TotalInvalidLogLineParseErrors.Increment();
EventNotificationItem.Publish(ExchangeComponent.Name, component, null, text, ResultSeverityLevel.Error, false);
ServiceLogger.LogError(ServiceLogger.Component.LogDataBatch, message, text, this.Instance, this.FullLogName);
PerfCountersInstanceCache.GetInstance(this.Instance).TotalParseErrors.Increment();
}
public LogReader(ThreadSafeQueue <T> batchQueue, int id, ILogManager logMonitor, ConfigInstance config, string prefix, ILogMonitorHelper <T> logMonitorHelper, string instanceName = null)
{
ArgumentValidator.ThrowIfNull("batchQueue", batchQueue);
ArgumentValidator.ThrowIfNull("logMonitor", logMonitor);
ArgumentValidator.ThrowIfNull("config", config);
ArgumentValidator.ThrowIfNull("logMonitorHelper", logMonitorHelper);
ArgumentValidator.ThrowIfNullOrEmpty("prefix", prefix);
if (id < 0)
{
throw new ArgumentOutOfRangeException("id cannot be negative.");
}
this.batchQueue = batchQueue;
this.id = id;
this.logMonitor = logMonitor;
this.stopped = false;
this.logPrefix = prefix;
this.config = config;
this.instance = (string.IsNullOrEmpty(instanceName) ? prefix : instanceName);
this.messageBatchFlushInterval = (int)this.config.BatchFlushInterval.TotalSeconds;
this.logMonitorHelper = logMonitorHelper;
this.lastTimeWhenQeueFull = DateTime.UtcNow.Ticks;
this.perfCounterInstance = PerfCountersInstanceCache.GetInstance(this.instance);
}
public static ILogUploaderPerformanceCounters GetInstance(string instanceName)
{
ILogUploaderPerformanceCounters logUploaderPerformanceCounters;
if (!PerfCountersInstanceCache.instances.TryGetValue(instanceName, out logUploaderPerformanceCounters))
{
lock (PerfCountersInstanceCache.InstancesMutex)
{
if (!PerfCountersInstanceCache.instances.TryGetValue(instanceName, out logUploaderPerformanceCounters))
{
Tools.DebugAssert(PerfCountersInstanceCache.GetPerfCountersInstance != null, "Performance counters factory method expected.");
logUploaderPerformanceCounters = PerfCountersInstanceCache.GetPerfCountersInstance(instanceName);
PerfCountersInstanceCache.instances = new Dictionary <string, ILogUploaderPerformanceCounters>(PerfCountersInstanceCache.instances, StringComparer.OrdinalIgnoreCase)
{
{
instanceName,
logUploaderPerformanceCounters
}
};
}
}
}
return(logUploaderPerformanceCounters);
}
public DatabaseWriter(ThreadSafeQueue <T> queue, int id, ConfigInstance config, string instanceName)
{
this.InitializeDatabaseWriter(queue, id, config, instanceName);
this.perfCounterInstance = PerfCountersInstanceCache.GetInstance(this.instance);
}
internal void ProcessLog(LogFileInfo log)
{
FileStream fileStream = null;
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderStartedParsingLog, string.Format("reprocessing or begin to process, isActive {0} +++++++++", log.IsActive), this.instance, log.FullFileName);
if (log.Status == ProcessingStatus.NeedProcessing)
{
log.Status = ProcessingStatus.InProcessing;
PerfCountersInstanceCache.GetInstance(this.instance).TotalNewLogsBeginProcessing.Increment();
ExTraceGlobals.ReaderTracer.TraceDebug <string, string>((long)this.GetHashCode(), "{0} started parsing log {1}", Thread.CurrentThread.Name, log.FullFileName);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderStartedParsingLog, log.FileName, new object[]
{
this.id,
log.FullFileName
});
ServiceLogger.LogInfo(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderStartedParsingLog, null, this.instance, log.FullFileName);
}
try
{
string defaultLogVersion;
fileStream = this.OpenLogWithRetry(log.FullFileName, out defaultLogVersion);
if (fileStream == null)
{
if (!this.isStopRequested)
{
string text = string.Format("Failed to open log file {0}", log.FullFileName);
ExTraceGlobals.ReaderTracer.TraceError((long)this.GetHashCode(), text);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderFileOpenFailed, log.FullFileName, new object[]
{
log.FullFileName
});
EventNotificationItem.Publish(ExchangeComponent.Name, "FailToOpenLogFile", null, text, ResultSeverityLevel.Error, false);
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 3221229475U, text, this.instance, log.FullFileName);
}
}
else
{
if (string.IsNullOrWhiteSpace(defaultLogVersion))
{
defaultLogVersion = this.logMonitorHelper.GetDefaultLogVersion();
ExTraceGlobals.ReaderTracer.TraceWarning <string, string>((long)this.GetHashCode(), "Failed to figure out version of log file {0}. Use default version {1} instead.", log.FullFileName, defaultLogVersion);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_FailedToGetVersionFromLogHeader, log.FullFileName, new object[]
{
log.FullFileName,
defaultLogVersion
});
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 2147487660U, null, this.instance, log.FullFileName);
}
CsvTable logSchema = this.logMonitorHelper.GetLogSchema(new Version(defaultLogVersion));
CsvFieldCache csvFieldCache = new CsvFieldCache(logSchema, fileStream, 32768);
int num = 0;
this.logHeaderEndOffset = csvFieldCache.Position;
ExTraceGlobals.ReaderTracer.TraceDebug <string, long>((long)this.GetHashCode(), "The end offset of the header of {0} is {1}.", log.FullFileName, this.logHeaderEndOffset);
bool flag = false;
while (!flag)
{
if (this.CheckServiceStopRequest("ProcessLog()"))
{
return;
}
LogFileRange logFileRange = log.WatermarkFileObj.GetBlockToReprocess();
if (logFileRange == null)
{
logFileRange = log.WatermarkFileObj.GetNewBlockToProcess();
flag = true;
if (logFileRange == null)
{
break;
}
}
if (num == 0)
{
long startOffset = logFileRange.StartOffset;
this.inputBuffer = new InputBuffer <T>(this.config.BatchSizeInBytes, startOffset, log, this.batchQueue, this.logPrefix, this.logMonitorHelper, this.messageBatchFlushInterval, this.cancellationContext, this.config.InputBufferMaximumBatchCount, this.instance);
}
bool isFirstBlock = num == 0;
this.inputBuffer.BeforeDataBlockIsProcessed(logFileRange, isFirstBlock);
this.ProcessBlock(csvFieldCache, logFileRange, log);
this.inputBuffer.AfterDataBlockIsProcessed();
if (this.isStopRequested)
{
return;
}
num++;
}
this.EnqueueLastBatchParsed();
log.LastProcessedTime = DateTime.UtcNow;
if (!log.IsActive)
{
log.Status = ProcessingStatus.ReadyToWriteToDatabase;
ExTraceGlobals.ReaderTracer.TraceInformation <string>(0, (long)this.GetHashCode(), "Finished parsing log {0}", log.FullFileName);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderFinishedParsingLog, log.FullFileName, new object[]
{
log.FullFileName
});
ServiceLogger.LogInfo(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderFinishedParsingLog, null, this.instance, log.FullFileName);
}
}
}
finally
{
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderFinishedParsingLog, string.Format("Finished parsing for this round, isActive {0} +++++++++", log.IsActive), this.instance, log.FullFileName);
this.logMonitor.ReaderCompletedProcessingLog(log);
this.Cleanup(fileStream);
}
}