internal virtual void AddNewLogsToProcess(IEnumerable <FileInfo> fileInfoList)
{
this.ResetStaleLogCollection();
int num = 0;
int num2 = 0;
IOrderedEnumerable <FileInfo> orderedEnumerable = from f in fileInfoList
orderby f.CreationTimeUtc descending
select f;
bool flag = true;
bool flag2 = false;
List <LogFileInfo> list = new List <LogFileInfo>();
DateTime t = DateTime.UtcNow - this.Config.ActiveLogFileIdleTimeout;
foreach (FileInfo fileInfo in orderedEnumerable)
{
try
{
bool isActive = flag || (this.Config.EnableMultipleWriters && fileInfo.LastWriteTimeUtc > t);
ILogFileInfo logFileInfo;
LogFileInfo logFileInfo2;
if (!this.knownLogNameToLogFileMap.TryGetValue(fileInfo.Name, out logFileInfo))
{
logFileInfo2 = new LogFileInfo(fileInfo.Name, isActive, this.instance, this.watermarkFileHelper);
this.knownLogNameToLogFileMap.TryAdd(fileInfo.Name, logFileInfo2);
list.Add(logFileInfo2);
num++;
}
else
{
logFileInfo2 = (LogFileInfo)logFileInfo;
logFileInfo2.IsActive = isActive;
}
if (!flag2)
{
this.CalculateIncompleteBytes(logFileInfo2, out flag2);
}
this.UpdateStaleLogs(logFileInfo2);
if (logFileInfo2.Status == ProcessingStatus.NeedProcessing)
{
num2++;
}
}
catch (FailedToInstantiateLogFileInfoException ex)
{
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_FailedToInstantiateLogFileInfo, fileInfo.Name, new object[]
{
ex.Message
});
ServiceLogger.LogError(ServiceLogger.Component.LogMonitor, (LogUploaderEventLogConstants.Message) 2147486660U, ex.Message, this.instance, fileInfo.Name);
}
flag = false;
}
if (list.Count > 0)
{
this.AddLogToNeedProcessing(list);
}
this.perfCounterInstance.TotalIncomingLogs.IncrementBy((long)num);
this.perfCounterInstance.NumberOfIncomingLogs.RawValue = (long)num;
this.perfCounterInstance.LogsNeverProcessedBefore.RawValue = (long)num2;
}
public virtual void ReaderCompletedProcessingLog(LogFileInfo log)
{
if (log == null)
{
return;
}
this.logsJustFinishedParsing.TryAdd(log.FileName, log);
}
private void DisposeAllWatermarkFileObjects()
{
foreach (ILogFileInfo logFileInfo in this.knownLogNameToLogFileMap.Values)
{
LogFileInfo logFileInfo2 = (LogFileInfo)logFileInfo;
logFileInfo2.WatermarkFileObj.Dispose();
}
this.knownLogNameToLogFileMap.Clear();
}
protected void CalculateIncompleteBytes(LogFileInfo logFile, out bool skipRestOfFilesInTheList)
{
skipRestOfFilesInTheList = false;
if (this.checkDirectoryCount == 0)
{
long num = logFile.AddedLogSize();
this.totalLogBytes += Math.Max(0L, num - logFile.WatermarkFileObj.ProcessedSize);
return;
}
long num2 = logFile.AddedLogSize();
this.totalLogBytes += num2;
skipRestOfFilesInTheList = (!logFile.IsActive && num2 == 0L);
}
internal bool DueForReprocess(LogFileInfo log)
{
bool flag = log.WatermarkFileObj.ReaderHasBytesToParse();
if (flag)
{
bool flag2 = DateTime.UtcNow.Subtract(log.LastProcessedTime).TotalMilliseconds > (double)this.reprocessingActiveFileWaitTime;
if (flag2)
{
this.reprocessingActiveFileWaitTime = Tools.RandomizeTimeSpan(this.config.WaitTimeToReprocessActiveFile, this.config.WaitTimeToReprocessActiveFileRandomRange);
}
return(flag2);
}
return(false);
}
public virtual LogFileInfo GetLogForReaderToProcess()
{
if (this.checkDirectoryCount == 0)
{
return(null);
}
LogFileInfo logFileInfo = null;
bool flag = false;
do
{
logFileInfo = null;
flag = false;
lock (this.logsNeedProcessingSyncObject)
{
if (this.logsNeedProcessing.Count > 0)
{
logFileInfo = this.logsNeedProcessing[0];
this.logsNeedProcessing.RemoveAt(0);
}
}
if (logFileInfo != null)
{
try
{
if (logFileInfo.Size == 0L)
{
flag = true;
}
}
catch (IOException)
{
flag = true;
}
if (flag)
{
this.logsJustFinishedParsing.TryAdd(logFileInfo.FileName, logFileInfo);
}
}
}while (flag);
return(logFileInfo);
}
internal void ReadLog()
{
LogFileInfo logFileInfo = null;
int num = 0;
try
{
if (this.batchQueue.IsFull)
{
num = Tools.RandomizeTimeSpan(this.config.ReaderSleepTime, this.config.ReaderSleepTimeRandomRange);
string text = string.Format("The queue for log prefix {0} is full. Reader will not parse it until the queue is no longer full.", this.instance);
ExTraceGlobals.ReaderTracer.TraceWarning((long)this.GetHashCode(), text);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderQueueFull, this.instance, new object[]
{
this.instance
});
long num2 = DateTime.UtcNow.Ticks - this.lastTimeWhenQeueFull;
if (num2 > 1800000000L)
{
this.lastTimeWhenQeueFull = DateTime.UtcNow.Ticks;
ServiceLogger.LogWarning(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 2147487654U, text, this.instance, "");
}
}
else
{
logFileInfo = this.logMonitor.GetLogForReaderToProcess();
if (logFileInfo == null)
{
num = Tools.RandomizeTimeSpan(this.config.ReaderSleepTime, this.config.ReaderSleepTimeRandomRange);
}
else
{
this.ProcessLog(logFileInfo);
}
}
}
catch (FileNotFoundException ex)
{
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderLogMissing, ex.Message, new object[]
{
Thread.CurrentThread.Name,
ex
});
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 2147487659U, ex.Message, this.instance, "");
this.logMonitor.ReaderCompletedProcessingLog(logFileInfo);
return;
}
catch (IOException ex2)
{
string text2 = (logFileInfo == null) ? "unknown log" : logFileInfo.FullFileName;
string text3 = string.Format("Caught an IOException when reading log {0}. Exception: {1}", text2, ex2);
ExTraceGlobals.LogMonitorTracer.TraceError((long)this.GetHashCode(), text3);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_ReadLogCaughtIOException, logFileInfo.FullFileName, new object[]
{
logFileInfo.FullFileName,
ex2
});
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 3221229485U, text3, this.instance, text2);
if (!ex2.Message.Contains("Insufficient system resources exist to complete the requested service"))
{
throw;
}
this.logMonitor.ReaderCompletedProcessingLog(logFileInfo);
}
catch (NotSupportedException ex3)
{
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_UnsupportedLogVersion, "UnSupportedLogVersion", new object[]
{
logFileInfo.FullFileName,
ex3
});
ServiceLogger.LogError(ServiceLogger.Component.LogMonitor, (LogUploaderEventLogConstants.Message) 3221229488U, ex3.Message, this.instance, logFileInfo.FullFileName);
EventNotificationItem.Publish(ExchangeComponent.Name, "UnsupportedLogVersion", null, ex3.Message, ResultSeverityLevel.Error, false);
throw ex3;
}
catch (Exception ex4)
{
if (!(ex4 is ThreadAbortException) || !this.cancellationContext.StopToken.IsCancellationRequested)
{
string text4 = (logFileInfo == null) ? "unknown log" : logFileInfo.FullFileName;
string text5 = string.Format("{0} processing {1} catches an exception: {2}", Thread.CurrentThread.Name, text4, ex4);
ExTraceGlobals.ReaderTracer.TraceError((long)this.GetHashCode(), text5);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderUnknownError, null, new object[]
{
Thread.CurrentThread.Name,
text4,
ex4
});
EventNotificationItem.Publish(ExchangeComponent.Name, "LogReaderUnknownError", null, text5, ResultSeverityLevel.Error, false);
this.perfCounterInstance.TotalLogReaderUnknownErrors.Increment();
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 3221229474U, text5, this.instance, text4);
throw;
}
Thread.ResetAbort();
ServiceLogger.LogInfo(ServiceLogger.Component.DatabaseWriter, LogUploaderEventLogConstants.Message.LogMonitorRequestedStop, Thread.CurrentThread.Name + " received thread abort event", this.instance, "");
}
if (num > 0 && this.cancellationContext.StopToken.WaitHandle.WaitOne(num))
{
this.cancellationContext.StopWaitHandle.Set();
}
}
internal void ProcessLog(LogFileInfo log)
{
FileStream fileStream = null;
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderStartedParsingLog, string.Format("reprocessing or begin to process, isActive {0} +++++++++", log.IsActive), this.instance, log.FullFileName);
if (log.Status == ProcessingStatus.NeedProcessing)
{
log.Status = ProcessingStatus.InProcessing;
PerfCountersInstanceCache.GetInstance(this.instance).TotalNewLogsBeginProcessing.Increment();
ExTraceGlobals.ReaderTracer.TraceDebug <string, string>((long)this.GetHashCode(), "{0} started parsing log {1}", Thread.CurrentThread.Name, log.FullFileName);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderStartedParsingLog, log.FileName, new object[]
{
this.id,
log.FullFileName
});
ServiceLogger.LogInfo(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderStartedParsingLog, null, this.instance, log.FullFileName);
}
try
{
string defaultLogVersion;
fileStream = this.OpenLogWithRetry(log.FullFileName, out defaultLogVersion);
if (fileStream == null)
{
if (!this.isStopRequested)
{
string text = string.Format("Failed to open log file {0}", log.FullFileName);
ExTraceGlobals.ReaderTracer.TraceError((long)this.GetHashCode(), text);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderFileOpenFailed, log.FullFileName, new object[]
{
log.FullFileName
});
EventNotificationItem.Publish(ExchangeComponent.Name, "FailToOpenLogFile", null, text, ResultSeverityLevel.Error, false);
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 3221229475U, text, this.instance, log.FullFileName);
}
}
else
{
if (string.IsNullOrWhiteSpace(defaultLogVersion))
{
defaultLogVersion = this.logMonitorHelper.GetDefaultLogVersion();
ExTraceGlobals.ReaderTracer.TraceWarning <string, string>((long)this.GetHashCode(), "Failed to figure out version of log file {0}. Use default version {1} instead.", log.FullFileName, defaultLogVersion);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_FailedToGetVersionFromLogHeader, log.FullFileName, new object[]
{
log.FullFileName,
defaultLogVersion
});
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 2147487660U, null, this.instance, log.FullFileName);
}
CsvTable logSchema = this.logMonitorHelper.GetLogSchema(new Version(defaultLogVersion));
CsvFieldCache csvFieldCache = new CsvFieldCache(logSchema, fileStream, 32768);
int num = 0;
this.logHeaderEndOffset = csvFieldCache.Position;
ExTraceGlobals.ReaderTracer.TraceDebug <string, long>((long)this.GetHashCode(), "The end offset of the header of {0} is {1}.", log.FullFileName, this.logHeaderEndOffset);
bool flag = false;
while (!flag)
{
if (this.CheckServiceStopRequest("ProcessLog()"))
{
return;
}
LogFileRange logFileRange = log.WatermarkFileObj.GetBlockToReprocess();
if (logFileRange == null)
{
logFileRange = log.WatermarkFileObj.GetNewBlockToProcess();
flag = true;
if (logFileRange == null)
{
break;
}
}
if (num == 0)
{
long startOffset = logFileRange.StartOffset;
this.inputBuffer = new InputBuffer <T>(this.config.BatchSizeInBytes, startOffset, log, this.batchQueue, this.logPrefix, this.logMonitorHelper, this.messageBatchFlushInterval, this.cancellationContext, this.config.InputBufferMaximumBatchCount, this.instance);
}
bool isFirstBlock = num == 0;
this.inputBuffer.BeforeDataBlockIsProcessed(logFileRange, isFirstBlock);
this.ProcessBlock(csvFieldCache, logFileRange, log);
this.inputBuffer.AfterDataBlockIsProcessed();
if (this.isStopRequested)
{
return;
}
num++;
}
this.EnqueueLastBatchParsed();
log.LastProcessedTime = DateTime.UtcNow;
if (!log.IsActive)
{
log.Status = ProcessingStatus.ReadyToWriteToDatabase;
ExTraceGlobals.ReaderTracer.TraceInformation <string>(0, (long)this.GetHashCode(), "Finished parsing log {0}", log.FullFileName);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderFinishedParsingLog, log.FullFileName, new object[]
{
log.FullFileName
});
ServiceLogger.LogInfo(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderFinishedParsingLog, null, this.instance, log.FullFileName);
}
}
}
finally
{
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderFinishedParsingLog, string.Format("Finished parsing for this round, isActive {0} +++++++++", log.IsActive), this.instance, log.FullFileName);
this.logMonitor.ReaderCompletedProcessingLog(log);
this.Cleanup(fileStream);
}
}
internal void ProcessBlock(CsvFieldCache cursor, LogFileRange block, LogFileInfo log)
{
bool flag = false;
if (cursor.Position < block.StartOffset)
{
cursor.Seek(block.StartOffset);
}
long num = cursor.Position;
while (cursor.Position < block.EndOffset)
{
if (this.CheckServiceStopRequest("ProcessBlock()"))
{
return;
}
try
{
flag = cursor.MoveNext(true);
if (!flag)
{
if (cursor.AtEnd && !log.IsActive)
{
this.inputBuffer.AddInvalidRowToSkip(num, cursor.Position);
block.EndOffset = cursor.Position;
num = cursor.Position;
}
break;
}
ReadOnlyRow readOnlyRow = new ReadOnlyRow(cursor, num);
this.inputBuffer.LineReceived(readOnlyRow);
num = readOnlyRow.EndPosition;
}
catch (Exception ex)
{
if (RetryHelper.IsSystemFatal(ex))
{
throw;
}
string text = string.Format("Log={0} blockRange=({1},{2}) cursorOffset={3} rowEnd={4} logSize={5} \nException:{6}", new object[]
{
log.FullFileName,
block.StartOffset,
block.EndOffset,
cursor.Position,
num,
log.Size,
ex
});
string periodicKey = string.Format("{0}_{1}_{2}", log.FileName, block.StartOffset, block.EndOffset);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_CsvParserFailedToParseLogLine, periodicKey, new object[]
{
text
});
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 3221229487U, string.Format("Detail={0}", text), this.instance, log.FullFileName);
flag = false;
break;
}
}
if (cursor.Position == block.EndOffset)
{
block.ProcessingStatus = ProcessingStatus.ReadyToWriteToDatabase;
}
if (!flag)
{
if (cursor.AtEnd)
{
if (block.EndOffset == 9223372036854775807L)
{
block.EndOffset = cursor.Position;
}
}
else if (this.logHeaderEndOffset != block.EndOffset)
{
string text2 = string.Format("Failed to read line from file {0} at position {1}", log.FullFileName, num);
ExTraceGlobals.ReaderTracer.TraceError((long)this.GetHashCode(), text2);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderReadFailed, log.FullFileName + "_" + num.ToString(), new object[]
{
log.FullFileName,
num
});
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 3221229476U, text2, this.instance, log.FullFileName);
}
}
long incrementValue = num - block.StartOffset;
if (Tools.IsRawProcessingType <T>())
{
this.perfCounterInstance.RawReaderParsedBytes.IncrementBy(incrementValue);
}
else
{
this.perfCounterInstance.ReaderParsedBytes.IncrementBy(incrementValue);
}
log.WatermarkFileObj.UpdateLastReaderParsedEndOffset(num);
}
