private void RemoveLogFileInfoObjects(IEnumerable <LogFileInfo> toBeRemovedList)
{
foreach (LogFileInfo logFileInfo in toBeRemovedList)
{
LogFileInfo logFileInfo2;
this.logsJustFinishedParsing.TryRemove(logFileInfo.FileName, out logFileInfo2);
logFileInfo.Status = ProcessingStatus.CompletedProcessing;
logFileInfo.WatermarkFileObj.Dispose();
if (logFileInfo.FileExists)
{
logFileInfo.WatermarkFileObj.CreateDoneFile();
}
ILogFileInfo logFileInfo3;
if (this.knownLogNameToLogFileMap.TryRemove(logFileInfo.FileName, out logFileInfo3))
{
string text = string.Format("log file {0} is removed from KnownLogNameToLogFileMap.", logFileInfo.FileName);
ExTraceGlobals.LogMonitorTracer.TraceDebug((long)this.GetHashCode(), text);
ServiceLogger.LogInfo(ServiceLogger.Component.LogMonitor, LogUploaderEventLogConstants.Message.LogFileDeletedFromKnownLogNameToLogFileMap, text, "", logFileInfo.FileName);
}
else
{
string.Format("The log {0} disappeared from KnownLogNameToLogFileMap. This indicates a bug somewhere", logFileInfo.FullFileName);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogDisappearedFromKnownLogNameToLogFileMap, logFileInfo.FullFileName, new object[]
{
logFileInfo.FullFileName
});
ServiceLogger.LogError(ServiceLogger.Component.LogMonitor, (LogUploaderEventLogConstants.Message) 2147748808U, "", "", logFileInfo.FullFileName);
}
}
}
public LogFileInfo(string fileName, bool isActive, string instanceName, IWatermarkFileHelper wmkFileHelper)
{
ArgumentValidator.ThrowIfNullOrEmpty("fileName", fileName);
ArgumentValidator.ThrowIfNull("wmkFileMgr", wmkFileHelper);
ArgumentValidator.ThrowIfNullOrEmpty("LogFileDirectory", wmkFileHelper.LogFileDirectory);
this.fileName = Path.GetFileName(fileName);
ArgumentValidator.ThrowIfNullOrEmpty("this.fileName", this.fileName);
this.fullFileName = Path.Combine(wmkFileHelper.LogFileDirectory, this.fileName);
this.status = ProcessingStatus.NeedProcessing;
this.isActive = isActive;
this.syncObject = new object();
this.instance = instanceName;
if (!File.Exists(this.fullFileName))
{
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_FailedToInstantiateLogFileInfoFileNotExist, this.fullFileName, new object[]
{
this.fullFileName
});
ServiceLogger.LogError(ServiceLogger.Component.LogFileInfo, (LogUploaderEventLogConstants.Message) 3221226477U, string.Empty, this.instance, this.fullFileName);
throw new FailedToInstantiateLogFileInfoException(Strings.FailedToInstantiateLogFileInfoFileNotExist(this.fullFileName));
}
this.fileInfo = new FileInfo(this.fullFileName);
this.fileInfo.Refresh();
this.creationTimeUtc = this.fileInfo.CreationTimeUtc;
this.waterMarkFile = wmkFileHelper.CreateWaterMarkFileObj(this.fileName, instanceName);
Tools.DebugAssert(this.waterMarkFile != null, "this.waterMarkFile != null");
}
internal virtual void AddNewLogsToProcess(IEnumerable <FileInfo> fileInfoList)
{
this.ResetStaleLogCollection();
int num = 0;
int num2 = 0;
IOrderedEnumerable <FileInfo> orderedEnumerable = from f in fileInfoList
orderby f.CreationTimeUtc descending
select f;
bool flag = true;
bool flag2 = false;
List <LogFileInfo> list = new List <LogFileInfo>();
DateTime t = DateTime.UtcNow - this.Config.ActiveLogFileIdleTimeout;
foreach (FileInfo fileInfo in orderedEnumerable)
{
try
{
bool isActive = flag || (this.Config.EnableMultipleWriters && fileInfo.LastWriteTimeUtc > t);
ILogFileInfo logFileInfo;
LogFileInfo logFileInfo2;
if (!this.knownLogNameToLogFileMap.TryGetValue(fileInfo.Name, out logFileInfo))
{
logFileInfo2 = new LogFileInfo(fileInfo.Name, isActive, this.instance, this.watermarkFileHelper);
this.knownLogNameToLogFileMap.TryAdd(fileInfo.Name, logFileInfo2);
list.Add(logFileInfo2);
num++;
}
else
{
logFileInfo2 = (LogFileInfo)logFileInfo;
logFileInfo2.IsActive = isActive;
}
if (!flag2)
{
this.CalculateIncompleteBytes(logFileInfo2, out flag2);
}
this.UpdateStaleLogs(logFileInfo2);
if (logFileInfo2.Status == ProcessingStatus.NeedProcessing)
{
num2++;
}
}
catch (FailedToInstantiateLogFileInfoException ex)
{
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_FailedToInstantiateLogFileInfo, fileInfo.Name, new object[]
{
ex.Message
});
ServiceLogger.LogError(ServiceLogger.Component.LogMonitor, (LogUploaderEventLogConstants.Message) 2147486660U, ex.Message, this.instance, fileInfo.Name);
}
flag = false;
}
if (list.Count > 0)
{
this.AddLogToNeedProcessing(list);
}
this.perfCounterInstance.TotalIncomingLogs.IncrementBy((long)num);
this.perfCounterInstance.NumberOfIncomingLogs.RawValue = (long)num;
this.perfCounterInstance.LogsNeverProcessedBefore.RawValue = (long)num2;
}
public void WriteWatermark(List <LogFileRange> ranges)
{
ArgumentValidator.ThrowIfNull("ranges", ranges);
if (this.disposed > 0)
{
ServiceLogger.LogError(ServiceLogger.Component.WatermarkFile, (LogUploaderEventLogConstants.Message) 3221231496U, string.Empty, this.instance, this.logFileFullName);
return;
}
lock (this.watermarkFileLock)
{
if (this.disposed > 0)
{
ServiceLogger.LogError(ServiceLogger.Component.WatermarkFile, (LogUploaderEventLogConstants.Message) 3221231496U, string.Empty, this.instance, this.logFileFullName);
return;
}
if (this.streamWriter == null)
{
FileStream stream = File.Open(this.watermarkFullFileName, FileMode.Append, FileAccess.Write, FileShare.Read);
this.streamWriter = new StreamWriter(stream);
}
string arg = DateTime.UtcNow.ToString("yyyy-MM-ddTHH\\:mm\\:ss");
foreach (LogFileRange logFileRange in ranges)
{
this.streamWriter.WriteLine("{0},{1},{2}", logFileRange.StartOffset, logFileRange.EndOffset, arg);
this.UpdateBlocksProcessed(logFileRange.StartOffset, logFileRange.EndOffset);
}
this.streamWriter.Flush();
}
if (ExTraceGlobals.WriterTracer.IsTraceEnabled(TraceType.DebugTrace))
{
DateTime utcNow = DateTime.UtcNow;
string message = string.Format("Watermark update time is {0} for log {1}", utcNow, this.watermarkFullFileName);
ExTraceGlobals.WriterTracer.TraceDebug((long)this.GetHashCode(), message);
}
}
private bool RetryableStart()
{
try
{
this.CreateWriters();
}
catch (ConfigurationErrorsException ex)
{
if (ex.Message.Contains("Log Path is not set in Registry"))
{
int num = Math.Max(1, 300 / (int)this.config.LogDirCheckInterval.TotalSeconds);
if (this.checkDirectoryCount / num < 1)
{
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_FailedToGetLogPath, this.instance, new object[]
{
this.instance,
ex.Message
});
ServiceLogger.LogError(ServiceLogger.Component.LogMonitor, (LogUploaderEventLogConstants.Message) 3221488641U, ex.Message, this.instance, this.logDirectory);
return(false);
}
}
throw;
}
this.StartWriterThreads(this.stopTokenSource.Token);
this.StartReaderThreads(this.stopTokenSource.Token);
return(true);
}
internal LogFileRange ProcessOneWatermark(string line)
{
ArgumentValidator.ThrowIfNull("line", line);
try
{
string fileRangeFromWatermark = WatermarkFile.GetFileRangeFromWatermark(line);
LogFileRange logFileRange = LogFileRange.Parse(fileRangeFromWatermark);
logFileRange.ProcessingStatus = ProcessingStatus.CompletedProcessing;
lock (this.blocksProcessedLock)
{
if (!this.blocksProcessed.ContainsKey(logFileRange.StartOffset))
{
this.AddRangeToProcessed(logFileRange);
return(logFileRange);
}
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_OverlappingLogRangeInWatermarkFile, this.WatermarkFileFullName, new object[]
{
this.WatermarkFileFullName,
logFileRange.StartOffset,
logFileRange.EndOffset,
logFileRange.StartOffset,
this.blocksProcessed[logFileRange.StartOffset].EndOffset
});
string text = string.Format("There are overlapping log ranges in watermark file {0}: ({1}, {2}), ({3}, {4}).", new object[]
{
this.WatermarkFileFullName,
logFileRange.StartOffset,
logFileRange.EndOffset,
logFileRange.StartOffset,
this.blocksProcessed[logFileRange.StartOffset].EndOffset
});
if (Interlocked.CompareExchange(ref WatermarkFile.overlappingWatermarksInWatermarkFile, 1, 0) == 0)
{
EventNotificationItem.Publish(ExchangeComponent.Name, "OverlappingWatermarkRecordsInFile", null, text, ResultSeverityLevel.Error, false);
}
ServiceLogger.LogError(ServiceLogger.Component.WatermarkFile, (LogUploaderEventLogConstants.Message) 3221231476U, text, this.instance, this.WatermarkFileFullName);
}
}
catch (MalformedLogRangeLineException ex)
{
string text2 = string.Format("Failed to parse watermark from {0}: {1}", this.watermarkFullFileName, ex.Message);
ExTraceGlobals.ReaderTracer.TraceError((long)this.GetHashCode(), text2);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_WatermarkFileParseException, this.watermarkFullFileName, new object[]
{
this.watermarkFullFileName,
ex.Message
});
if (Interlocked.CompareExchange(ref WatermarkFile.watermarkParseError, 1, 0) == 0)
{
EventNotificationItem.Publish(ExchangeComponent.Name, "MalformedWatermarkRecordError", null, text2, ResultSeverityLevel.Warning, false);
}
ServiceLogger.LogError(ServiceLogger.Component.WatermarkFile, (LogUploaderEventLogConstants.Message) 3221231475U, ex.Message, this.instance, this.watermarkFullFileName);
}
return(null);
}
internal void UpdateBlocksProcessed(long startOffset, long endOffset)
{
LogFileRange logFileRange = new LogFileRange(startOffset, endOffset, ProcessingStatus.CompletedProcessing);
lock (this.blocksProcessedLock)
{
if (this.blocksProcessed.ContainsKey(startOffset))
{
string text;
if (this.blocksProcessed[startOffset].EndOffset == endOffset)
{
text = string.Format("Tried to add an existing block ({0}, {1}) when updating in-memory watermarks for log {2}.", startOffset, endOffset, this.logFileFullName);
ExTraceGlobals.WriterTracer.TraceError((long)this.GetHashCode(), text);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_WatermarkFileDuplicateBlock, this.logFileFullName, new object[]
{
startOffset.ToString(),
endOffset.ToString(),
this.logFileFullName
});
ServiceLogger.LogError(ServiceLogger.Component.WatermarkFile, (LogUploaderEventLogConstants.Message) 3221231474U, string.Format("startOffset={0};endOffset={1}", startOffset, endOffset), this.instance, this.logFileFullName);
}
else
{
text = string.Format("Tried to add an block ({0}, {1}) that overlaps with an existing block ({2}, {3}) in the in-memory watermarks for log {4}.", new object[]
{
startOffset,
endOffset,
startOffset,
this.blocksProcessed[startOffset].EndOffset,
this.logFileFullName
});
ExTraceGlobals.WriterTracer.TraceError((long)this.GetHashCode(), text);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_WatermarkFileOverlappingBlock, null, new object[]
{
startOffset,
endOffset,
startOffset,
this.blocksProcessed[startOffset].EndOffset,
this.logFileFullName
});
ServiceLogger.LogError(ServiceLogger.Component.WatermarkFile, (LogUploaderEventLogConstants.Message) 3221231479U, string.Format("startOffset={0};endOffset={1}", startOffset, endOffset), this.instance, this.logFileFullName);
}
if (Interlocked.CompareExchange(ref WatermarkFile.overlappingWatermarksInMemory, 1, 0) == 0)
{
EventNotificationItem.Publish(ExchangeComponent.Name, "OverlappingWatermarkRecordsInMemory", null, text, ResultSeverityLevel.Error, false);
}
}
else
{
this.AddRangeToProcessed(logFileRange);
}
}
}
public void CreateDoneFile()
{
string path = Path.ChangeExtension(this.watermarkFullFileName, "done");
try
{
File.Open(path, FileMode.OpenOrCreate).Close();
}
catch (IOException ex)
{
if (!ex.Message.Contains("There is not enough space on the disk."))
{
throw;
}
ServiceLogger.LogError(ServiceLogger.Component.WatermarkFile, (LogUploaderEventLogConstants.Message) 3221231487U, ex.Message, this.instance, this.logFileFullName);
}
}
internal void ClearnupWmkDoneFiles(string dir, string filter)
{
foreach (string text in Directory.EnumerateFiles(dir, filter))
{
try
{
string path = this.watermarkFileHelper.DeduceLogFullFileNameFromDoneOrWatermarkFileName(text);
if (!File.Exists(path))
{
if (this.knownLogNameToLogFileMap.ContainsKey(Path.GetFileName(path)))
{
ServiceLogger.LogInfo(ServiceLogger.Component.LogMonitor, LogUploaderEventLogConstants.Message.DeletingFile, "Skipped for it's still processed by this logMonitor", this.instance, text);
}
else
{
ServiceLogger.LogInfo(ServiceLogger.Component.LogMonitor, LogUploaderEventLogConstants.Message.DeletingFile, "Delete Done or watermark file", this.instance, text);
File.Delete(text);
}
}
}
catch (IOException ex)
{
string text2 = string.Format("Failed to clean up {0}. Exception: {1}.", text, ex.Message);
ExTraceGlobals.LogMonitorTracer.TraceError((long)this.GetHashCode(), text2);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogMonitorWatermarkCleanupFailed, text, new object[]
{
text,
ex.Message
});
ServiceLogger.LogError(ServiceLogger.Component.LogMonitor, (LogUploaderEventLogConstants.Message) 3221228476U, text2, "", text);
}
catch (UnauthorizedAccessException ex2)
{
string text3 = string.Format("Failed to clean up {0}. Exception: {1}.", text, ex2);
ExTraceGlobals.LogMonitorTracer.TraceError((long)this.GetHashCode(), text3);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogMonitorWatermarkCleanupFailed, text, new object[]
{
text,
ex2
});
ServiceLogger.LogError(ServiceLogger.Component.LogMonitor, (LogUploaderEventLogConstants.Message) 3221228476U, text3, "", text);
}
}
}
private long GetLogFileSize()
{
long result;
try
{
result = new FileInfo(this.LogFileFullName).Length;
}
catch (FileNotFoundException ex)
{
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_FileDeletedWhenCheckingItsCompletion, this.LogFileFullName, new object[]
{
this.LogFileFullName
});
ServiceLogger.LogError(ServiceLogger.Component.WatermarkFile, (LogUploaderEventLogConstants.Message) 2147486661U, ex.Message, this.instance, this.LogFileFullName);
result = 0L;
}
return(result);
}
private Action <object> DoWorkAction(Action <object> realDoWorkMethod)
{
if (!this.catchReaderWriterExceptionsForUnitTest)
{
return(realDoWorkMethod);
}
return(delegate(object stateObj)
{
try
{
realDoWorkMethod(stateObj);
}
catch (Exception ex)
{
ServiceLogger.LogError(ServiceLogger.Component.LogMonitor, (LogUploaderEventLogConstants.Message) 3221226485U, ex.ToString(), "", "");
EventNotificationItem.Publish(ExchangeComponent.Name, "ServiceStartUnknownException", null, ex.ToString(), ResultSeverityLevel.Error, false);
LogMonitor <T> < > 4__this = this;
< > 4__this.unitTestCatchedExceptionMessage += ex.ToString();
}
internal void LogErrorAndUpdatePerfCounter(long rowStartOffset, long rowEndOffset, Exception exception, ExEventLog.EventTuple eventTuple, LogUploaderEventLogConstants.Message message, string component)
{
string text = string.Format("Failed to parse log {0} at row ({1}, {2}): \nException: {3}", new object[]
{
this.FullLogName,
rowStartOffset,
rowEndOffset,
exception
});
ExTraceGlobals.ParserTracer.TraceError((long)this.GetHashCode(), text);
EventLogger.Logger.LogEvent(eventTuple, exception.Message, new object[]
{
text
});
PerfCountersInstanceCache.GetInstance(this.Instance).TotalInvalidLogLineParseErrors.Increment();
EventNotificationItem.Publish(ExchangeComponent.Name, component, null, text, ResultSeverityLevel.Error, false);
ServiceLogger.LogError(ServiceLogger.Component.LogDataBatch, message, text, this.Instance, this.FullLogName);
PerfCountersInstanceCache.GetInstance(this.Instance).TotalParseErrors.Increment();
}
private void ReportMissingLogs(List <LogFileInfo> missingLogs)
{
if (missingLogs.Count == 0)
{
return;
}
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.AppendLine("The following logs are not found while being processed:");
foreach (LogFileInfo logFileInfo in missingLogs)
{
ServiceLogger.LogError(ServiceLogger.Component.LogMonitor, (LogUploaderEventLogConstants.Message) 3221231489U, "CheckLogsAndMarkComplete", this.instance, logFileInfo.FullFileName);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_FileDeleted, logFileInfo.FullFileName, new object[]
{
logFileInfo.FullFileName
});
stringBuilder.AppendLine(logFileInfo.FullFileName);
}
EventNotificationItem.Publish(ExchangeComponent.Name, "LogFileNotFoundError", null, stringBuilder.ToString(), ResultSeverityLevel.Error, false);
}
internal void DequeueAndWriteOneItem()
{
string text = string.Empty;
IWatermarkFile watermarkFile = null;
try
{
T t = this.queue.Dequeue(this.cancellationContext);
if (t != null)
{
this.perfCounterInstance.BatchQueueLength.Decrement();
text = t.FullLogName;
watermarkFile = this.logManager.FindWatermarkFileObject(text);
if (ExTraceGlobals.WriterTracer.IsTraceEnabled(TraceType.DebugTrace))
{
StringBuilder stringBuilder = new StringBuilder();
foreach (LogFileRange logFileRange in t.LogRanges)
{
string value = string.Format("({0}, {1}) ", logFileRange.StartOffset, logFileRange.EndOffset);
stringBuilder.Append(value);
}
string message = string.Format("For log {0}, writer {1} get batch that has {2} ranges: {3}", new object[]
{
text,
this.id,
t.LogRanges.Count,
stringBuilder.ToString()
});
ExTraceGlobals.WriterTracer.TraceDebug((long)this.GetHashCode(), message);
}
this.WriteBatch(t);
}
else if (!this.CheckServiceStopRequest("DequeueAndWriteOneItem()"))
{
Tools.DebugAssert(this.cancellationContext.StopToken.IsCancellationRequested, "If Enqueue is not cancelled, it should always get a batch back");
}
}
catch (FaultException ex)
{
ExTraceGlobals.WriterTracer.TraceError <string, string>((long)this.GetHashCode(), "Writer failed to write data through web service DAL when processing log {0}. Check the event log on the FFO web service role. The exception is: {1}.", text, ex.Message);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_WebServiceWriteException, ex.Message, new object[]
{
text,
ex.ToString()
});
ServiceLogger.LogError(ServiceLogger.Component.DatabaseWriter, (LogUploaderEventLogConstants.Message) 3221489634U, ex.Message, this.instance, text);
}
catch (CommunicationException ex2)
{
ExTraceGlobals.WriterTracer.TraceError <string, string>((long)this.GetHashCode(), "Writer failed to write data through web service DAL when processing log {0}. The exception is: {1}.", text, ex2.Message);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_WebServiceWriteException, ex2.Message, new object[]
{
text,
ex2.ToString()
});
ServiceLogger.LogError(ServiceLogger.Component.DatabaseWriter, (LogUploaderEventLogConstants.Message) 3221489633U, ex2.Message, this.instance, text);
}
catch (ADTopologyEndpointNotFoundException ex3)
{
ExTraceGlobals.WriterTracer.TraceError <string, string>((long)this.GetHashCode(), "Writer caught an ADTopologyEndpointNotFoundException when processing log {0}. Details is: {1}", text, ex3.Message);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_ADTopologyEndpointNotFound, ex3.Message, new object[]
{
text,
ex3.Message
});
ServiceLogger.LogError(ServiceLogger.Component.DatabaseWriter, (LogUploaderEventLogConstants.Message) 3221489629U, ex3.Message, this.instance, text);
}
catch (Exception ex4)
{
if (!(ex4 is ThreadAbortException) || !this.cancellationContext.StopToken.IsCancellationRequested)
{
string text2 = string.Format("Writer caught an exception when processing log {0}: {1}", text, ex4);
ExTraceGlobals.WriterTracer.TraceError((long)this.GetHashCode(), text2);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_DatabaseWriterUnknownException, ex4.Message, new object[]
{
text,
ex4
});
EventNotificationItem.Publish(ExchangeComponent.Name, "UnexpectedWriterError", null, text2, ResultSeverityLevel.Error, false);
this.perfCounterInstance.TotalUnexpectedWriterErrors.Increment();
ServiceLogger.LogError(ServiceLogger.Component.DatabaseWriter, (LogUploaderEventLogConstants.Message) 3221489617U, text2, this.instance, text);
throw;
}
Thread.ResetAbort();
ServiceLogger.LogInfo(ServiceLogger.Component.DatabaseWriter, LogUploaderEventLogConstants.Message.LogMonitorRequestedStop, Thread.CurrentThread.Name + " received thread abort event", this.instance, "");
}
finally
{
if (watermarkFile != null)
{
watermarkFile.InMemoryCountDecrease();
}
}
}
internal void ProcessLog(LogFileInfo log)
{
FileStream fileStream = null;
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderStartedParsingLog, string.Format("reprocessing or begin to process, isActive {0} +++++++++", log.IsActive), this.instance, log.FullFileName);
if (log.Status == ProcessingStatus.NeedProcessing)
{
log.Status = ProcessingStatus.InProcessing;
PerfCountersInstanceCache.GetInstance(this.instance).TotalNewLogsBeginProcessing.Increment();
ExTraceGlobals.ReaderTracer.TraceDebug <string, string>((long)this.GetHashCode(), "{0} started parsing log {1}", Thread.CurrentThread.Name, log.FullFileName);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderStartedParsingLog, log.FileName, new object[]
{
this.id,
log.FullFileName
});
ServiceLogger.LogInfo(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderStartedParsingLog, null, this.instance, log.FullFileName);
}
try
{
string defaultLogVersion;
fileStream = this.OpenLogWithRetry(log.FullFileName, out defaultLogVersion);
if (fileStream == null)
{
if (!this.isStopRequested)
{
string text = string.Format("Failed to open log file {0}", log.FullFileName);
ExTraceGlobals.ReaderTracer.TraceError((long)this.GetHashCode(), text);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderFileOpenFailed, log.FullFileName, new object[]
{
log.FullFileName
});
EventNotificationItem.Publish(ExchangeComponent.Name, "FailToOpenLogFile", null, text, ResultSeverityLevel.Error, false);
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 3221229475U, text, this.instance, log.FullFileName);
}
}
else
{
if (string.IsNullOrWhiteSpace(defaultLogVersion))
{
defaultLogVersion = this.logMonitorHelper.GetDefaultLogVersion();
ExTraceGlobals.ReaderTracer.TraceWarning <string, string>((long)this.GetHashCode(), "Failed to figure out version of log file {0}. Use default version {1} instead.", log.FullFileName, defaultLogVersion);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_FailedToGetVersionFromLogHeader, log.FullFileName, new object[]
{
log.FullFileName,
defaultLogVersion
});
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 2147487660U, null, this.instance, log.FullFileName);
}
CsvTable logSchema = this.logMonitorHelper.GetLogSchema(new Version(defaultLogVersion));
CsvFieldCache csvFieldCache = new CsvFieldCache(logSchema, fileStream, 32768);
int num = 0;
this.logHeaderEndOffset = csvFieldCache.Position;
ExTraceGlobals.ReaderTracer.TraceDebug <string, long>((long)this.GetHashCode(), "The end offset of the header of {0} is {1}.", log.FullFileName, this.logHeaderEndOffset);
bool flag = false;
while (!flag)
{
if (this.CheckServiceStopRequest("ProcessLog()"))
{
return;
}
LogFileRange logFileRange = log.WatermarkFileObj.GetBlockToReprocess();
if (logFileRange == null)
{
logFileRange = log.WatermarkFileObj.GetNewBlockToProcess();
flag = true;
if (logFileRange == null)
{
break;
}
}
if (num == 0)
{
long startOffset = logFileRange.StartOffset;
this.inputBuffer = new InputBuffer <T>(this.config.BatchSizeInBytes, startOffset, log, this.batchQueue, this.logPrefix, this.logMonitorHelper, this.messageBatchFlushInterval, this.cancellationContext, this.config.InputBufferMaximumBatchCount, this.instance);
}
bool isFirstBlock = num == 0;
this.inputBuffer.BeforeDataBlockIsProcessed(logFileRange, isFirstBlock);
this.ProcessBlock(csvFieldCache, logFileRange, log);
this.inputBuffer.AfterDataBlockIsProcessed();
if (this.isStopRequested)
{
return;
}
num++;
}
this.EnqueueLastBatchParsed();
log.LastProcessedTime = DateTime.UtcNow;
if (!log.IsActive)
{
log.Status = ProcessingStatus.ReadyToWriteToDatabase;
ExTraceGlobals.ReaderTracer.TraceInformation <string>(0, (long)this.GetHashCode(), "Finished parsing log {0}", log.FullFileName);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderFinishedParsingLog, log.FullFileName, new object[]
{
log.FullFileName
});
ServiceLogger.LogInfo(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderFinishedParsingLog, null, this.instance, log.FullFileName);
}
}
}
finally
{
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderFinishedParsingLog, string.Format("Finished parsing for this round, isActive {0} +++++++++", log.IsActive), this.instance, log.FullFileName);
this.logMonitor.ReaderCompletedProcessingLog(log);
this.Cleanup(fileStream);
}
}
internal void CheckDirectory(object state)
{
if (Interlocked.CompareExchange(ref this.checkDirectoryDone, 0, 1) == 1)
{
try
{
if (this.logDirectory == null)
{
return;
}
if (this.needToRetryStart && (this.needToRetryStart = !this.RetryableStart()))
{
return;
}
if (!Directory.Exists(this.logDirectory))
{
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_NonexistentLogDirectory, this.logDirectory, new object[]
{
this.logDirectory
});
return;
}
IEnumerable <FileInfo> source = new DirectoryInfo(this.logDirectory).EnumerateFiles(this.logPrefixToBeMonitored + "*.log");
IEnumerable <FileInfo> enumerable = from f in source
where f.Length < 2147483647L && this.logMonitorHelper.ShouldProcessLogFile(this.logPrefixToBeMonitored, f.Name) && !this.HasDoneFile(f.Name)
select f;
this.DumpPendingProcessLogFilesInfo(enumerable);
this.AddNewLogsToProcess(enumerable);
this.CheckBackLogs();
this.CheckLogsAndMarkComplete();
this.CheckLogsDueToReprocess();
this.DeleteResidueFilesForRetiredLogs();
}
catch (Exception ex)
{
string message = string.Format("Caught an Exception when checking directory {0}. Exception: {1}", this.logDirectory, ex);
ExTraceGlobals.LogMonitorTracer.TraceError((long)this.GetHashCode(), message);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_CheckDirectoryCaughtException, this.logDirectory, new object[]
{
this.logDirectory,
ex
});
ServiceLogger.LogError(ServiceLogger.Component.LogMonitor, (LogUploaderEventLogConstants.Message) 3221228483U, ex.Message, this.instance, this.logDirectory);
if (!ex.Message.Contains("Insufficient system resources exist to complete the requested service"))
{
throw;
}
}
finally
{
this.checkDirectoryCount++;
this.checkDirectoryDone = 1;
}
this.UpdateIncompleteLogsPerfCounter();
if (Tools.IsRawProcessingType <T>())
{
this.perfCounterInstance.RawIncompleteBytes.RawValue = this.unprocessedBytes;
this.perfCounterInstance.RawTotalLogBytes.RawValue = this.totalLogBytes;
}
else
{
this.perfCounterInstance.IncompleteBytes.RawValue = this.unprocessedBytes;
this.perfCounterInstance.TotalLogBytes.RawValue = this.totalLogBytes;
}
this.perfCounterInstance.ThreadSafeQueueConsumerSemaphoreCount.RawValue = (long)this.batchQueue.ConsumerSemaphoreCount;
this.perfCounterInstance.ThreadSafeQueueProducerSemaphoreCount.RawValue = (long)this.batchQueue.ProducerSemaphoreCount;
}
}
internal void ReadLog()
{
LogFileInfo logFileInfo = null;
int num = 0;
try
{
if (this.batchQueue.IsFull)
{
num = Tools.RandomizeTimeSpan(this.config.ReaderSleepTime, this.config.ReaderSleepTimeRandomRange);
string text = string.Format("The queue for log prefix {0} is full. Reader will not parse it until the queue is no longer full.", this.instance);
ExTraceGlobals.ReaderTracer.TraceWarning((long)this.GetHashCode(), text);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderQueueFull, this.instance, new object[]
{
this.instance
});
long num2 = DateTime.UtcNow.Ticks - this.lastTimeWhenQeueFull;
if (num2 > 1800000000L)
{
this.lastTimeWhenQeueFull = DateTime.UtcNow.Ticks;
ServiceLogger.LogWarning(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 2147487654U, text, this.instance, "");
}
}
else
{
logFileInfo = this.logMonitor.GetLogForReaderToProcess();
if (logFileInfo == null)
{
num = Tools.RandomizeTimeSpan(this.config.ReaderSleepTime, this.config.ReaderSleepTimeRandomRange);
}
else
{
this.ProcessLog(logFileInfo);
}
}
}
catch (FileNotFoundException ex)
{
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderLogMissing, ex.Message, new object[]
{
Thread.CurrentThread.Name,
ex
});
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 2147487659U, ex.Message, this.instance, "");
this.logMonitor.ReaderCompletedProcessingLog(logFileInfo);
return;
}
catch (IOException ex2)
{
string text2 = (logFileInfo == null) ? "unknown log" : logFileInfo.FullFileName;
string text3 = string.Format("Caught an IOException when reading log {0}. Exception: {1}", text2, ex2);
ExTraceGlobals.LogMonitorTracer.TraceError((long)this.GetHashCode(), text3);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_ReadLogCaughtIOException, logFileInfo.FullFileName, new object[]
{
logFileInfo.FullFileName,
ex2
});
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 3221229485U, text3, this.instance, text2);
if (!ex2.Message.Contains("Insufficient system resources exist to complete the requested service"))
{
throw;
}
this.logMonitor.ReaderCompletedProcessingLog(logFileInfo);
}
catch (NotSupportedException ex3)
{
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_UnsupportedLogVersion, "UnSupportedLogVersion", new object[]
{
logFileInfo.FullFileName,
ex3
});
ServiceLogger.LogError(ServiceLogger.Component.LogMonitor, (LogUploaderEventLogConstants.Message) 3221229488U, ex3.Message, this.instance, logFileInfo.FullFileName);
EventNotificationItem.Publish(ExchangeComponent.Name, "UnsupportedLogVersion", null, ex3.Message, ResultSeverityLevel.Error, false);
throw ex3;
}
catch (Exception ex4)
{
if (!(ex4 is ThreadAbortException) || !this.cancellationContext.StopToken.IsCancellationRequested)
{
string text4 = (logFileInfo == null) ? "unknown log" : logFileInfo.FullFileName;
string text5 = string.Format("{0} processing {1} catches an exception: {2}", Thread.CurrentThread.Name, text4, ex4);
ExTraceGlobals.ReaderTracer.TraceError((long)this.GetHashCode(), text5);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderUnknownError, null, new object[]
{
Thread.CurrentThread.Name,
text4,
ex4
});
EventNotificationItem.Publish(ExchangeComponent.Name, "LogReaderUnknownError", null, text5, ResultSeverityLevel.Error, false);
this.perfCounterInstance.TotalLogReaderUnknownErrors.Increment();
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 3221229474U, text5, this.instance, text4);
throw;
}
Thread.ResetAbort();
ServiceLogger.LogInfo(ServiceLogger.Component.DatabaseWriter, LogUploaderEventLogConstants.Message.LogMonitorRequestedStop, Thread.CurrentThread.Name + " received thread abort event", this.instance, "");
}
if (num > 0 && this.cancellationContext.StopToken.WaitHandle.WaitOne(num))
{
this.cancellationContext.StopWaitHandle.Set();
}
}
private void ReportBacklogCondition(int staleLogCount, int veryStaleLogCount, string logDir, List <ILogFileInfo> logList, bool isUrgent)
{
int num = Math.Min(logList.Count, 10);
IEnumerable <ILogFileInfo> enumerable = logList.Take(num);
StringBuilder stringBuilder = new StringBuilder();
string value = (num > 1) ? string.Format("The first {0} logs are:", num) : "Here are the detailed info:";
stringBuilder.AppendLine(value);
foreach (ILogFileInfo logFileInfo in enumerable)
{
string text = this.BuildWatermarkFileInfo(logFileInfo);
string text2 = "unknown";
try
{
FileInfo fileInfo = new FileInfo(logFileInfo.FullFileName);
if (fileInfo.Exists)
{
text2 = string.Format("{0} bytes", fileInfo.Length);
}
}
catch (Exception ex)
{
if (RetryHelper.IsSystemFatal(ex))
{
throw;
}
}
stringBuilder.AppendLine(string.Format("The log file {0} is {1}, its size is {2}, created on {3}, last modified on {4}, {5}", new object[]
{
logFileInfo.FullFileName,
logFileInfo.IsActive ? "Active" : "Inactive",
text2,
logFileInfo.CreationTimeUtc,
logFileInfo.LastWriteTimeUtc,
text
}));
}
string text3 = string.Format("There are {0} logs in directory {1} that haven't been processed for {2} hours. {3} of them are over {4} hours.\n{5}", new object[]
{
staleLogCount,
logDir,
this.config.BacklogAlertNonUrgentThreshold.TotalHours,
veryStaleLogCount,
this.config.BacklogAlertUrgentThreshold.TotalHours,
stringBuilder.ToString()
});
if (isUrgent)
{
LogMonitor <T> .RaiseAlertIfHealthStateChange(ref this.veryStaleLogReportedBefore, "SeriousBacklogBuiltUp", text3);
}
else
{
LogMonitor <T> .RaiseAlertIfHealthStateChange(ref this.staleLogReportedBefore, "BacklogBuiltUp", text3);
}
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogMonitorDetectLogProcessingFallsBehind, this.instance, new object[]
{
staleLogCount,
logDir,
this.config.BacklogAlertNonUrgentThreshold.TotalHours,
veryStaleLogCount,
this.config.BacklogAlertUrgentThreshold.TotalHours,
stringBuilder.ToString()
});
ServiceLogger.LogError(ServiceLogger.Component.LogMonitor, (LogUploaderEventLogConstants.Message) 3221228478U, text3, this.instance, this.logDirectory);
}
internal void ProcessBlock(CsvFieldCache cursor, LogFileRange block, LogFileInfo log)
{
bool flag = false;
if (cursor.Position < block.StartOffset)
{
cursor.Seek(block.StartOffset);
}
long num = cursor.Position;
while (cursor.Position < block.EndOffset)
{
if (this.CheckServiceStopRequest("ProcessBlock()"))
{
return;
}
try
{
flag = cursor.MoveNext(true);
if (!flag)
{
if (cursor.AtEnd && !log.IsActive)
{
this.inputBuffer.AddInvalidRowToSkip(num, cursor.Position);
block.EndOffset = cursor.Position;
num = cursor.Position;
}
break;
}
ReadOnlyRow readOnlyRow = new ReadOnlyRow(cursor, num);
this.inputBuffer.LineReceived(readOnlyRow);
num = readOnlyRow.EndPosition;
}
catch (Exception ex)
{
if (RetryHelper.IsSystemFatal(ex))
{
throw;
}
string text = string.Format("Log={0} blockRange=({1},{2}) cursorOffset={3} rowEnd={4} logSize={5} \nException:{6}", new object[]
{
log.FullFileName,
block.StartOffset,
block.EndOffset,
cursor.Position,
num,
log.Size,
ex
});
string periodicKey = string.Format("{0}_{1}_{2}", log.FileName, block.StartOffset, block.EndOffset);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_CsvParserFailedToParseLogLine, periodicKey, new object[]
{
text
});
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 3221229487U, string.Format("Detail={0}", text), this.instance, log.FullFileName);
flag = false;
break;
}
}
if (cursor.Position == block.EndOffset)
{
block.ProcessingStatus = ProcessingStatus.ReadyToWriteToDatabase;
}
if (!flag)
{
if (cursor.AtEnd)
{
if (block.EndOffset == 9223372036854775807L)
{
block.EndOffset = cursor.Position;
}
}
else if (this.logHeaderEndOffset != block.EndOffset)
{
string text2 = string.Format("Failed to read line from file {0} at position {1}", log.FullFileName, num);
ExTraceGlobals.ReaderTracer.TraceError((long)this.GetHashCode(), text2);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderReadFailed, log.FullFileName + "_" + num.ToString(), new object[]
{
log.FullFileName,
num
});
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 3221229476U, text2, this.instance, log.FullFileName);
}
}
long incrementValue = num - block.StartOffset;
if (Tools.IsRawProcessingType <T>())
{
this.perfCounterInstance.RawReaderParsedBytes.IncrementBy(incrementValue);
}
else
{
this.perfCounterInstance.ReaderParsedBytes.IncrementBy(incrementValue);
}
log.WatermarkFileObj.UpdateLastReaderParsedEndOffset(num);
}
