internal virtual void AddNewLogsToProcess(IEnumerable <FileInfo> fileInfoList)
{
this.ResetStaleLogCollection();
int num = 0;
int num2 = 0;
IOrderedEnumerable <FileInfo> orderedEnumerable = from f in fileInfoList
orderby f.CreationTimeUtc descending
select f;
bool flag = true;
bool flag2 = false;
List <LogFileInfo> list = new List <LogFileInfo>();
DateTime t = DateTime.UtcNow - this.Config.ActiveLogFileIdleTimeout;
foreach (FileInfo fileInfo in orderedEnumerable)
{
try
{
bool isActive = flag || (this.Config.EnableMultipleWriters && fileInfo.LastWriteTimeUtc > t);
ILogFileInfo logFileInfo;
LogFileInfo logFileInfo2;
if (!this.knownLogNameToLogFileMap.TryGetValue(fileInfo.Name, out logFileInfo))
{
logFileInfo2 = new LogFileInfo(fileInfo.Name, isActive, this.instance, this.watermarkFileHelper);
this.knownLogNameToLogFileMap.TryAdd(fileInfo.Name, logFileInfo2);
list.Add(logFileInfo2);
num++;
}
else
{
logFileInfo2 = (LogFileInfo)logFileInfo;
logFileInfo2.IsActive = isActive;
}
if (!flag2)
{
this.CalculateIncompleteBytes(logFileInfo2, out flag2);
}
this.UpdateStaleLogs(logFileInfo2);
if (logFileInfo2.Status == ProcessingStatus.NeedProcessing)
{
num2++;
}
}
catch (FailedToInstantiateLogFileInfoException ex)
{
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_FailedToInstantiateLogFileInfo, fileInfo.Name, new object[]
{
ex.Message
});
ServiceLogger.LogError(ServiceLogger.Component.LogMonitor, (LogUploaderEventLogConstants.Message) 2147486660U, ex.Message, this.instance, fileInfo.Name);
}
flag = false;
}
if (list.Count > 0)
{
this.AddLogToNeedProcessing(list);
}
this.perfCounterInstance.TotalIncomingLogs.IncrementBy((long)num);
this.perfCounterInstance.NumberOfIncomingLogs.RawValue = (long)num;
this.perfCounterInstance.LogsNeverProcessedBefore.RawValue = (long)num2;
}
private T Dequeue()
{
T result = default(T);
if (!this.queue.TryDequeue(out result))
{
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("Enqueue: called : Type={0}, thread={1}, queueSize={2}, capacity={3} ?---InDequeuExp ", new object[]
{
typeof(T),
Thread.CurrentThread.ManagedThreadId,
this.queue.Count,
this.Capacity
}), "", "");
throw new Exception("TryDequeue should always return true");
}
int num = this.producerSemaphore.Release();
this.ProducerSemaphoreCount = num + 1;
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("Enqueue: called : Type={0}, thread={1}, queueSize={2}, capacity={3}, consumerSemaphore={4} ?---InDequeuDone ", new object[]
{
typeof(T),
Thread.CurrentThread.ManagedThreadId,
this.queue.Count,
this.Capacity,
num
}), "", "");
return(result);
}
public LogFileInfo(string fileName, bool isActive, string instanceName, IWatermarkFileHelper wmkFileHelper)
{
ArgumentValidator.ThrowIfNullOrEmpty("fileName", fileName);
ArgumentValidator.ThrowIfNull("wmkFileMgr", wmkFileHelper);
ArgumentValidator.ThrowIfNullOrEmpty("LogFileDirectory", wmkFileHelper.LogFileDirectory);
this.fileName = Path.GetFileName(fileName);
ArgumentValidator.ThrowIfNullOrEmpty("this.fileName", this.fileName);
this.fullFileName = Path.Combine(wmkFileHelper.LogFileDirectory, this.fileName);
this.status = ProcessingStatus.NeedProcessing;
this.isActive = isActive;
this.syncObject = new object();
this.instance = instanceName;
if (!File.Exists(this.fullFileName))
{
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_FailedToInstantiateLogFileInfoFileNotExist, this.fullFileName, new object[]
{
this.fullFileName
});
ServiceLogger.LogError(ServiceLogger.Component.LogFileInfo, (LogUploaderEventLogConstants.Message) 3221226477U, string.Empty, this.instance, this.fullFileName);
throw new FailedToInstantiateLogFileInfoException(Strings.FailedToInstantiateLogFileInfoFileNotExist(this.fullFileName));
}
this.fileInfo = new FileInfo(this.fullFileName);
this.fileInfo.Refresh();
this.creationTimeUtc = this.fileInfo.CreationTimeUtc;
this.waterMarkFile = wmkFileHelper.CreateWaterMarkFileObj(this.fileName, instanceName);
Tools.DebugAssert(this.waterMarkFile != null, "this.waterMarkFile != null");
}
public static void LogCommon(ServiceLogger.LogLevel logLevel, string message, string customData, ServiceLogger.Component componentName = ServiceLogger.Component.None, string logFileType = "", string logFilePath = "")
{
if (ServiceLogger.serviceLog == null)
{
ServiceLogger.InitializeMtrtLog();
}
if (ServiceLogger.ServiceLogLevel == ServiceLogger.LogLevel.None || logLevel < ServiceLogger.ServiceLogLevel)
{
return;
}
LogRowFormatter logRowFormatter = new LogRowFormatter(ServiceLogger.MessageTracingServiceLogSchema);
logRowFormatter[1] = logLevel.ToString();
if (!string.IsNullOrEmpty(logFileType))
{
string[] array = logFileType.Split(new char[]
{
'_'
});
logRowFormatter[2] = array[0];
}
if (!string.IsNullOrEmpty(logFilePath))
{
logRowFormatter[3] = logFilePath;
}
logRowFormatter[4] = componentName.ToString();
logRowFormatter[5] = message;
if (!string.IsNullOrEmpty(customData))
{
logRowFormatter[6] = customData;
}
ServiceLogger.serviceLog.Append(logRowFormatter, 0);
}
public void WriteWatermark(List <LogFileRange> ranges)
{
ArgumentValidator.ThrowIfNull("ranges", ranges);
if (this.disposed > 0)
{
ServiceLogger.LogError(ServiceLogger.Component.WatermarkFile, (LogUploaderEventLogConstants.Message) 3221231496U, string.Empty, this.instance, this.logFileFullName);
return;
}
lock (this.watermarkFileLock)
{
if (this.disposed > 0)
{
ServiceLogger.LogError(ServiceLogger.Component.WatermarkFile, (LogUploaderEventLogConstants.Message) 3221231496U, string.Empty, this.instance, this.logFileFullName);
return;
}
if (this.streamWriter == null)
{
FileStream stream = File.Open(this.watermarkFullFileName, FileMode.Append, FileAccess.Write, FileShare.Read);
this.streamWriter = new StreamWriter(stream);
}
string arg = DateTime.UtcNow.ToString("yyyy-MM-ddTHH\\:mm\\:ss");
foreach (LogFileRange logFileRange in ranges)
{
this.streamWriter.WriteLine("{0},{1},{2}", logFileRange.StartOffset, logFileRange.EndOffset, arg);
this.UpdateBlocksProcessed(logFileRange.StartOffset, logFileRange.EndOffset);
}
this.streamWriter.Flush();
}
if (ExTraceGlobals.WriterTracer.IsTraceEnabled(TraceType.DebugTrace))
{
DateTime utcNow = DateTime.UtcNow;
string message = string.Format("Watermark update time is {0} for log {1}", utcNow, this.watermarkFullFileName);
ExTraceGlobals.WriterTracer.TraceDebug((long)this.GetHashCode(), message);
}
}
private bool RetryableStart()
{
try
{
this.CreateWriters();
}
catch (ConfigurationErrorsException ex)
{
if (ex.Message.Contains("Log Path is not set in Registry"))
{
int num = Math.Max(1, 300 / (int)this.config.LogDirCheckInterval.TotalSeconds);
if (this.checkDirectoryCount / num < 1)
{
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_FailedToGetLogPath, this.instance, new object[]
{
this.instance,
ex.Message
});
ServiceLogger.LogError(ServiceLogger.Component.LogMonitor, (LogUploaderEventLogConstants.Message) 3221488641U, ex.Message, this.instance, this.logDirectory);
return(false);
}
}
throw;
}
this.StartWriterThreads(this.stopTokenSource.Token);
this.StartReaderThreads(this.stopTokenSource.Token);
return(true);
}
private void RemoveLogFileInfoObjects(IEnumerable <LogFileInfo> toBeRemovedList)
{
foreach (LogFileInfo logFileInfo in toBeRemovedList)
{
LogFileInfo logFileInfo2;
this.logsJustFinishedParsing.TryRemove(logFileInfo.FileName, out logFileInfo2);
logFileInfo.Status = ProcessingStatus.CompletedProcessing;
logFileInfo.WatermarkFileObj.Dispose();
if (logFileInfo.FileExists)
{
logFileInfo.WatermarkFileObj.CreateDoneFile();
}
ILogFileInfo logFileInfo3;
if (this.knownLogNameToLogFileMap.TryRemove(logFileInfo.FileName, out logFileInfo3))
{
string text = string.Format("log file {0} is removed from KnownLogNameToLogFileMap.", logFileInfo.FileName);
ExTraceGlobals.LogMonitorTracer.TraceDebug((long)this.GetHashCode(), text);
ServiceLogger.LogInfo(ServiceLogger.Component.LogMonitor, LogUploaderEventLogConstants.Message.LogFileDeletedFromKnownLogNameToLogFileMap, text, "", logFileInfo.FileName);
}
else
{
string.Format("The log {0} disappeared from KnownLogNameToLogFileMap. This indicates a bug somewhere", logFileInfo.FullFileName);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogDisappearedFromKnownLogNameToLogFileMap, logFileInfo.FullFileName, new object[]
{
logFileInfo.FullFileName
});
ServiceLogger.LogError(ServiceLogger.Component.LogMonitor, (LogUploaderEventLogConstants.Message) 2147748808U, "", "", logFileInfo.FullFileName);
}
}
}
public bool ReaderHasBytesToParse()
{
long logFileSize = this.GetLogFileSize();
if (logFileSize == 0L)
{
return(false);
}
if (this.lastReaderProcessedEndOffset < this.lastCheckedEndOffsertBeforeHoles)
{
ServiceLogger.LogCommon(ServiceLogger.LogLevel.Error, "lastReaderProcessedEndOffset", string.Format("this.lastReaderProcessedEndOffset {0}, lastCheckedEndOffsertBeforeHoles {1}", this.lastReaderProcessedEndOffset, this.lastCheckedEndOffsertBeforeHoles), ServiceLogger.Component.WatermarkFile, this.instance, this.logFileFullName);
Tools.DebugAssert(false, string.Format("this.lastReaderProcessedEndOffset {0}, lastCheckedEndOffsertBeforeHoles {1}", this.lastReaderProcessedEndOffset, this.lastCheckedEndOffsertBeforeHoles));
}
if (this.blocksNeedReprocessing.Count == 0 && this.lastCheckedEndOffsertBeforeHoles < this.lastReaderProcessedEndOffset)
{
if (this.inMemoryBatchCount == 0)
{
this.FindUnprocessedHoles();
}
else
{
ServiceLogger.LogCommon(ServiceLogger.LogLevel.Debug, "MemoryBatchCountNotZero", this.inMemoryBatchCount.ToString(), ServiceLogger.Component.WatermarkFile, this.instance, this.logFileFullName);
}
}
return(this.blocksNeedReprocessing.Count > 0 || logFileSize > this.lastReaderProcessedEndOffset);
}
private void Enqueue(T item)
{
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("Enqueue: called : Type={0}, thread={1}, queueSize={2}, capacity={3} ?---In ", new object[]
{
typeof(T),
Thread.CurrentThread.ManagedThreadId,
this.queue.Count,
this.Capacity
}), "", "");
if (this.queue.Count >= this.Capacity)
{
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("Enqueue: called : Type={0}, thread={1}, queueSize={2}, capacity={3} ?---InEnqueuFull ", new object[]
{
typeof(T),
Thread.CurrentThread.ManagedThreadId,
this.queue.Count,
this.Capacity
}), "", "");
throw new Exception("Queue should never be full here. The caller should have acquired writer quota");
}
this.queue.Enqueue(item);
int num = this.consumerSemaphore.Release();
this.ConsumerSemaphoreCount = num + 1;
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("Enqueue: called : Type={0}, thread={1}, queueSize={2}, capacity={3}, consumerSemaphore={4} ?---InDone ", new object[]
{
typeof(T),
Thread.CurrentThread.ManagedThreadId,
this.queue.Count,
this.Capacity,
num
}), "", "");
}
internal void FlushMessageBatchBufferToWriter(bool forceFlush)
{
if (!this.shouldBufferBatches)
{
return;
}
if (!LogDataBatchReflectionCache <T> .IsMessageBatch)
{
return;
}
List <T> list = this.messageBatchBuffer;
Tools.DebugAssert(list.Count <= this.maximumBatchCount, "Verify the buffer size will never goes above the maximumBatchCount");
if (!forceFlush && (DateTime.UtcNow - this.lastFluchCheckTime).TotalMilliseconds < 1000.0 && list.Count < this.maximumBatchCount)
{
return;
}
this.lastFluchCheckTime = DateTime.UtcNow;
foreach (T t in list)
{
MessageBatchBase messageBatchBase = t as MessageBatchBase;
Tools.DebugAssert(messageBatchBase != null, "Failed cast to MessageBatchBase.");
if (forceFlush || messageBatchBase.ReadyToFlush(this.newestLogLineTS))
{
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("FlushMessageBatchBufferToWriter: called : thread={0}, forceFlush={1}, this.shouldBufferBatches={2}, bufferCount={3}, this.maximumBatchCount={4} ?---BufEnqueBatch ", new object[]
{
Thread.CurrentThread.ManagedThreadId,
forceFlush,
this.shouldBufferBatches,
list.Count,
this.maximumBatchCount
}), "", "");
this.EnqueueBatch(t);
messageBatchBase.Flushed = true;
}
}
list.RemoveAll(delegate(T batch)
{
MessageBatchBase messageBatchBase3 = batch as MessageBatchBase;
Tools.DebugAssert(messageBatchBase3 != null, "Failed cast to MessageBatchBase.");
return(messageBatchBase3.Flushed);
});
while (list.Count >= this.maximumBatchCount)
{
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("FlushMessageBatchBufferToWriter: called : thread={0}, forceFlush={1}, this.shouldBufferBatches={2}, bufferCount={3}, this.maximumBatchCount={4} ?---BufEnqueBatch2 ", new object[]
{
Thread.CurrentThread.ManagedThreadId,
forceFlush,
this.shouldBufferBatches,
list.Count,
this.maximumBatchCount
}), "", "");
this.EnqueueBatch(list[0]);
MessageBatchBase messageBatchBase2 = list[0] as MessageBatchBase;
Tools.DebugAssert(messageBatchBase2 != null, "Failed cast to MessageBatchBase.");
messageBatchBase2.Flushed = true;
list.RemoveAt(0);
}
}
private void TraceBlockNeedProcessing(long startOffset, long endOffset)
{
if (ServiceLogger.ServiceLogLevel == ServiceLogger.LogLevel.Debug)
{
string message = string.Format("FindblocksNeedProcessing for {0} add ({1}, {2})", Path.GetFileName(this.logFileFullName), startOffset, endOffset);
ExTraceGlobals.ReaderTracer.TraceDebug((long)this.GetHashCode(), message);
ServiceLogger.LogCommon(ServiceLogger.LogLevel.Debug, "FindblocksNeedProcessing", string.Format("{0},{1}", startOffset, endOffset), ServiceLogger.Component.WatermarkFile, this.instance, this.LogFileFullName);
}
}
internal LogFileRange ProcessOneWatermark(string line)
{
ArgumentValidator.ThrowIfNull("line", line);
try
{
string fileRangeFromWatermark = WatermarkFile.GetFileRangeFromWatermark(line);
LogFileRange logFileRange = LogFileRange.Parse(fileRangeFromWatermark);
logFileRange.ProcessingStatus = ProcessingStatus.CompletedProcessing;
lock (this.blocksProcessedLock)
{
if (!this.blocksProcessed.ContainsKey(logFileRange.StartOffset))
{
this.AddRangeToProcessed(logFileRange);
return(logFileRange);
}
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_OverlappingLogRangeInWatermarkFile, this.WatermarkFileFullName, new object[]
{
this.WatermarkFileFullName,
logFileRange.StartOffset,
logFileRange.EndOffset,
logFileRange.StartOffset,
this.blocksProcessed[logFileRange.StartOffset].EndOffset
});
string text = string.Format("There are overlapping log ranges in watermark file {0}: ({1}, {2}), ({3}, {4}).", new object[]
{
this.WatermarkFileFullName,
logFileRange.StartOffset,
logFileRange.EndOffset,
logFileRange.StartOffset,
this.blocksProcessed[logFileRange.StartOffset].EndOffset
});
if (Interlocked.CompareExchange(ref WatermarkFile.overlappingWatermarksInWatermarkFile, 1, 0) == 0)
{
EventNotificationItem.Publish(ExchangeComponent.Name, "OverlappingWatermarkRecordsInFile", null, text, ResultSeverityLevel.Error, false);
}
ServiceLogger.LogError(ServiceLogger.Component.WatermarkFile, (LogUploaderEventLogConstants.Message) 3221231476U, text, this.instance, this.WatermarkFileFullName);
}
}
catch (MalformedLogRangeLineException ex)
{
string text2 = string.Format("Failed to parse watermark from {0}: {1}", this.watermarkFullFileName, ex.Message);
ExTraceGlobals.ReaderTracer.TraceError((long)this.GetHashCode(), text2);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_WatermarkFileParseException, this.watermarkFullFileName, new object[]
{
this.watermarkFullFileName,
ex.Message
});
if (Interlocked.CompareExchange(ref WatermarkFile.watermarkParseError, 1, 0) == 0)
{
EventNotificationItem.Publish(ExchangeComponent.Name, "MalformedWatermarkRecordError", null, text2, ResultSeverityLevel.Warning, false);
}
ServiceLogger.LogError(ServiceLogger.Component.WatermarkFile, (LogUploaderEventLogConstants.Message) 3221231475U, ex.Message, this.instance, this.watermarkFullFileName);
}
return(null);
}
private void BufferOrEnqueueToWriter()
{
if (!this.shouldBufferBatches)
{
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("BufferOrEnqueueToWriter: called : thread={0} ?---BufEnque2 ", Thread.CurrentThread.ManagedThreadId), "", "");
this.EnqueueBatch(this.activeBatch);
return;
}
this.messageBatchBuffer.Add(this.activeBatch);
Tools.DebugAssert(this.messageBatchBuffer.Count <= this.maximumBatchCount, "Verify the buffer size will never go above the maximumBatchCount");
}
internal void UpdateBlocksProcessed(long startOffset, long endOffset)
{
LogFileRange logFileRange = new LogFileRange(startOffset, endOffset, ProcessingStatus.CompletedProcessing);
lock (this.blocksProcessedLock)
{
if (this.blocksProcessed.ContainsKey(startOffset))
{
string text;
if (this.blocksProcessed[startOffset].EndOffset == endOffset)
{
text = string.Format("Tried to add an existing block ({0}, {1}) when updating in-memory watermarks for log {2}.", startOffset, endOffset, this.logFileFullName);
ExTraceGlobals.WriterTracer.TraceError((long)this.GetHashCode(), text);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_WatermarkFileDuplicateBlock, this.logFileFullName, new object[]
{
startOffset.ToString(),
endOffset.ToString(),
this.logFileFullName
});
ServiceLogger.LogError(ServiceLogger.Component.WatermarkFile, (LogUploaderEventLogConstants.Message) 3221231474U, string.Format("startOffset={0};endOffset={1}", startOffset, endOffset), this.instance, this.logFileFullName);
}
else
{
text = string.Format("Tried to add an block ({0}, {1}) that overlaps with an existing block ({2}, {3}) in the in-memory watermarks for log {4}.", new object[]
{
startOffset,
endOffset,
startOffset,
this.blocksProcessed[startOffset].EndOffset,
this.logFileFullName
});
ExTraceGlobals.WriterTracer.TraceError((long)this.GetHashCode(), text);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_WatermarkFileOverlappingBlock, null, new object[]
{
startOffset,
endOffset,
startOffset,
this.blocksProcessed[startOffset].EndOffset,
this.logFileFullName
});
ServiceLogger.LogError(ServiceLogger.Component.WatermarkFile, (LogUploaderEventLogConstants.Message) 3221231479U, string.Format("startOffset={0};endOffset={1}", startOffset, endOffset), this.instance, this.logFileFullName);
}
if (Interlocked.CompareExchange(ref WatermarkFile.overlappingWatermarksInMemory, 1, 0) == 0)
{
EventNotificationItem.Publish(ExchangeComponent.Name, "OverlappingWatermarkRecordsInMemory", null, text, ResultSeverityLevel.Error, false);
}
}
else
{
this.AddRangeToProcessed(logFileRange);
}
}
}
public bool IsLogCompleted()
{
bool flag = this.ReaderHasBytesToParse();
ServiceLogger.LogCommon(ServiceLogger.LogLevel.Debug, "LogIsNotCompleted", string.Format("{0}, {1}, {2}, {3} ", new object[]
{
this.blocksNeedReprocessing.Count,
this.lastCheckedEndOffsertBeforeHoles,
this.lastReaderProcessedEndOffset,
this.inMemoryBatchCount
}), ServiceLogger.Component.WatermarkFile, this.instance, this.logFileFullName);
return(!flag && this.lastCheckedEndOffsertBeforeHoles >= this.GetLogFileSize());
}
private static void InitializeMtrtLog()
{
try
{
ServiceLogger.ReadConfiguration();
}
finally
{
Log log = new Log("MessageTracingService_", new LogHeaderFormatter(ServiceLogger.MessageTracingServiceLogSchema), "Microsoft.ForeFront.Hygiene.MessageTracing");
log.Configure(ServiceLogger.FileLocation, ServiceLogger.MaximumLogAge, ServiceLogger.MaximumLogDirectorySize, ServiceLogger.MaximumLogFileSize, false, 0, TimeSpan.MaxValue, LogFileRollOver.Hourly);
ServiceLogger.Initialize(log);
}
}
private void ReportCompleteLogs(List <LogFileInfo> completeLogs)
{
foreach (LogFileInfo logFileInfo in completeLogs)
{
this.perfCounterInstance.TotalCompletedLogs.Increment();
string message = string.Format("The processing of log {0} is completed.", logFileInfo.FullFileName);
ExTraceGlobals.LogMonitorTracer.TraceDebug((long)this.GetHashCode(), message);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogMonitorLogCompleted, logFileInfo.FullFileName, new object[]
{
logFileInfo.FullFileName
});
ServiceLogger.LogInfo(ServiceLogger.Component.LogMonitor, LogUploaderEventLogConstants.Message.LogMonitorLogCompleted, "", "", logFileInfo.FullFileName);
}
}
internal void FindUnprocessedHoles()
{
if (ServiceLogger.ServiceLogLevel == ServiceLogger.LogLevel.Debug)
{
string message = string.Format("FindblocksNeedProcessing found {0} blocks processed for {1}", this.blocksProcessed.Count, this.LogFileFullName);
ExTraceGlobals.ReaderTracer.TraceDebug((long)this.GetHashCode(), message);
}
Tools.DebugAssert(this.blocksNeedReprocessing.Count == 0, "this.blocksNeedProcessing.Count == 0");
bool foundHole = false;
Action <long, long> action = delegate(long s, long e)
{
foundHole = true;
LogFileRange logFileRange2 = new LogFileRange(s, e, ProcessingStatus.NeedProcessing);
this.TraceBlockNeedProcessing(logFileRange2.StartOffset, logFileRange2.EndOffset);
this.blocksNeedReprocessing.Add(logFileRange2);
};
long num = 0L;
lock (this.blocksProcessedLock)
{
int count = this.blocksProcessed.Count;
this.lastCheckedEndOffsertBeforeHoles = 0L;
for (int i = 0; i < count; i++)
{
LogFileRange logFileRange = this.blocksProcessed.Values[i];
if (logFileRange.EndOffset < logFileRange.StartOffset)
{
ServiceLogger.LogCommon(ServiceLogger.LogLevel.Error, "Invalid watermark range", string.Format("{0},{1}", logFileRange.StartOffset, logFileRange.EndOffset), ServiceLogger.Component.WatermarkFile, "", "");
Tools.DebugAssert(false, string.Format("detected invalid range {0},{1}", logFileRange.StartOffset, logFileRange.EndOffset));
}
else
{
if (num < logFileRange.StartOffset)
{
action(num, logFileRange.StartOffset);
}
num = Math.Max(num, logFileRange.EndOffset);
if (!foundHole)
{
this.lastCheckedEndOffsertBeforeHoles = num;
}
}
}
}
if (this.lastReaderProcessedEndOffset > num)
{
action(num, this.lastReaderProcessedEndOffset);
}
}
private void DumpPendingProcessLogFilesInfo(IEnumerable <FileInfo> pendingProcessLogFiles)
{
if (ServiceLogger.ServiceLogLevel > ServiceLogger.LogLevel.Debug)
{
return;
}
StringBuilder stringBuilder = new StringBuilder();
foreach (FileInfo fileInfo in pendingProcessLogFiles)
{
stringBuilder.AppendLine(string.Format("file={0},created={1},modified={2}", fileInfo.FullName, fileInfo.CreationTimeUtc, fileInfo.LastWriteTimeUtc));
}
if (stringBuilder.Length > 0)
{
ServiceLogger.LogDebug(ServiceLogger.Component.LogMonitor, LogUploaderEventLogConstants.Message.PendingProcessLogFilesInfo, stringBuilder.ToString(), this.instance, this.logDirectory);
}
}
public void CreateDoneFile()
{
string path = Path.ChangeExtension(this.watermarkFullFileName, "done");
try
{
File.Open(path, FileMode.OpenOrCreate).Close();
}
catch (IOException ex)
{
if (!ex.Message.Contains("There is not enough space on the disk."))
{
throw;
}
ServiceLogger.LogError(ServiceLogger.Component.WatermarkFile, (LogUploaderEventLogConstants.Message) 3221231487U, ex.Message, this.instance, this.logFileFullName);
}
}
internal void ClearnupWmkDoneFiles(string dir, string filter)
{
foreach (string text in Directory.EnumerateFiles(dir, filter))
{
try
{
string path = this.watermarkFileHelper.DeduceLogFullFileNameFromDoneOrWatermarkFileName(text);
if (!File.Exists(path))
{
if (this.knownLogNameToLogFileMap.ContainsKey(Path.GetFileName(path)))
{
ServiceLogger.LogInfo(ServiceLogger.Component.LogMonitor, LogUploaderEventLogConstants.Message.DeletingFile, "Skipped for it's still processed by this logMonitor", this.instance, text);
}
else
{
ServiceLogger.LogInfo(ServiceLogger.Component.LogMonitor, LogUploaderEventLogConstants.Message.DeletingFile, "Delete Done or watermark file", this.instance, text);
File.Delete(text);
}
}
}
catch (IOException ex)
{
string text2 = string.Format("Failed to clean up {0}. Exception: {1}.", text, ex.Message);
ExTraceGlobals.LogMonitorTracer.TraceError((long)this.GetHashCode(), text2);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogMonitorWatermarkCleanupFailed, text, new object[]
{
text,
ex.Message
});
ServiceLogger.LogError(ServiceLogger.Component.LogMonitor, (LogUploaderEventLogConstants.Message) 3221228476U, text2, "", text);
}
catch (UnauthorizedAccessException ex2)
{
string text3 = string.Format("Failed to clean up {0}. Exception: {1}.", text, ex2);
ExTraceGlobals.LogMonitorTracer.TraceError((long)this.GetHashCode(), text3);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogMonitorWatermarkCleanupFailed, text, new object[]
{
text,
ex2
});
ServiceLogger.LogError(ServiceLogger.Component.LogMonitor, (LogUploaderEventLogConstants.Message) 3221228476U, text3, "", text);
}
}
}
private long GetLogFileSize()
{
long result;
try
{
result = new FileInfo(this.LogFileFullName).Length;
}
catch (FileNotFoundException ex)
{
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_FileDeletedWhenCheckingItsCompletion, this.LogFileFullName, new object[]
{
this.LogFileFullName
});
ServiceLogger.LogError(ServiceLogger.Component.WatermarkFile, (LogUploaderEventLogConstants.Message) 2147486661U, ex.Message, this.instance, this.LogFileFullName);
result = 0L;
}
return(result);
}
private Action <object> DoWorkAction(Action <object> realDoWorkMethod)
{
if (!this.catchReaderWriterExceptionsForUnitTest)
{
return(realDoWorkMethod);
}
return(delegate(object stateObj)
{
try
{
realDoWorkMethod(stateObj);
}
catch (Exception ex)
{
ServiceLogger.LogError(ServiceLogger.Component.LogMonitor, (LogUploaderEventLogConstants.Message) 3221226485U, ex.ToString(), "", "");
EventNotificationItem.Publish(ExchangeComponent.Name, "ServiceStartUnknownException", null, ex.ToString(), ResultSeverityLevel.Error, false);
LogMonitor <T> < > 4__this = this;
< > 4__this.unitTestCatchedExceptionMessage += ex.ToString();
}
public void DoWork(object obj)
{
ArgumentValidator.ThrowIfNull("obj", obj);
if (string.IsNullOrEmpty(Thread.CurrentThread.Name))
{
Thread.CurrentThread.Name = string.Format("Writer {0} for log type {1}", this.id, this.instance);
}
ServiceLogger.LogDebug(ServiceLogger.Component.DatabaseWriter, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("DoWork: called : thread={0}, log={1} ?---LogWriter", Thread.CurrentThread.ManagedThreadId, this.instance), "", "");
this.cancellationContext = (CancellationContext)obj;
while (!this.CheckServiceStopRequest("DoWork()"))
{
this.DequeueAndWriteOneItem();
}
string message = string.Format("Writer {0} for log type {1} stopped.", this.id, this.instance);
ExTraceGlobals.WriterTracer.TraceDebug((long)this.GetHashCode(), message);
ServiceLogger.LogInfo(ServiceLogger.Component.DatabaseWriter, LogUploaderEventLogConstants.Message.LogMonitorRequestedStop, Thread.CurrentThread.Name, this.instance, "");
this.stopped = true;
}
internal void LogErrorAndUpdatePerfCounter(long rowStartOffset, long rowEndOffset, Exception exception, ExEventLog.EventTuple eventTuple, LogUploaderEventLogConstants.Message message, string component)
{
string text = string.Format("Failed to parse log {0} at row ({1}, {2}): \nException: {3}", new object[]
{
this.FullLogName,
rowStartOffset,
rowEndOffset,
exception
});
ExTraceGlobals.ParserTracer.TraceError((long)this.GetHashCode(), text);
EventLogger.Logger.LogEvent(eventTuple, exception.Message, new object[]
{
text
});
PerfCountersInstanceCache.GetInstance(this.Instance).TotalInvalidLogLineParseErrors.Increment();
EventNotificationItem.Publish(ExchangeComponent.Name, component, null, text, ResultSeverityLevel.Error, false);
ServiceLogger.LogError(ServiceLogger.Component.LogDataBatch, message, text, this.Instance, this.FullLogName);
PerfCountersInstanceCache.GetInstance(this.Instance).TotalParseErrors.Increment();
}
private void ReportMissingLogs(List <LogFileInfo> missingLogs)
{
if (missingLogs.Count == 0)
{
return;
}
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.AppendLine("The following logs are not found while being processed:");
foreach (LogFileInfo logFileInfo in missingLogs)
{
ServiceLogger.LogError(ServiceLogger.Component.LogMonitor, (LogUploaderEventLogConstants.Message) 3221231489U, "CheckLogsAndMarkComplete", this.instance, logFileInfo.FullFileName);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_FileDeleted, logFileInfo.FullFileName, new object[]
{
logFileInfo.FullFileName
});
stringBuilder.AppendLine(logFileInfo.FullFileName);
}
EventNotificationItem.Publish(ExchangeComponent.Name, "LogFileNotFoundError", null, stringBuilder.ToString(), ResultSeverityLevel.Error, false);
}
public void EnqueueBatch(T batch)
{
if (ExTraceGlobals.ReaderTracer.IsTraceEnabled(TraceType.DebugTrace))
{
for (int i = 0; i < batch.LogRanges.Count; i++)
{
string message = string.Format("InputBuffer: enqueueBatch range {0}: ({1}, {2}) for log {3}", new object[]
{
i,
batch.LogRanges[i].StartOffset,
batch.LogRanges[i].EndOffset,
this.fullLogName
});
ExTraceGlobals.ReaderTracer.TraceDebug((long)this.GetHashCode(), message);
}
}
long size = batch.Size;
long num = (long)batch.NumberOfLinesInBatch;
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("before enqueue; batch size (in original log) {0}, lines {1} +++++++++ ", size, num), this.instance, this.fullLogName);
this.watermarkFileRef.InMemoryCountIncrease();
try
{
this.logDataBatchQueue.Enqueue(batch, this.cancellationContext);
}
catch
{
this.watermarkFileRef.InMemoryCountDecrease();
throw;
}
if (this.cancellationContext.StopToken.IsCancellationRequested)
{
return;
}
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("after enqueue; batch size (in original log) {0}, lines {1} +++++++++ ", size, num), this.instance, this.fullLogName);
this.perfCounterInstance.InputBufferBatchCounts.Decrement();
this.perfCounterInstance.BatchQueueLength.Increment();
this.perfCounterInstance.InputBufferBackfilledLines.IncrementBy(this.newBackfilledLogLines);
this.newBackfilledLogLines = 0L;
}
public T Dequeue(CancellationContext cancellationContext)
{
if (cancellationContext == null)
{
this.consumerSemaphore.WaitOne();
}
else if (WaitHandle.WaitAny(new WaitHandle[]
{
cancellationContext.StopToken.WaitHandle,
this.consumerSemaphore
}) != 1)
{
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("Enqueue: called : Type={0}, thread={1}, queueSize={2}, capacity={3} ?---InDequeueEmptySkip ", new object[]
{
typeof(T),
Thread.CurrentThread.ManagedThreadId,
this.queue.Count,
this.Capacity
}), "", "");
return(default(T));
}
return(this.Dequeue());
}
public void DoWork(object obj)
{
ArgumentValidator.ThrowIfNull("obj", obj);
this.cancellationContext = (CancellationContext)obj;
CancellationToken stopToken = this.cancellationContext.StopToken;
if (string.IsNullOrEmpty(Thread.CurrentThread.Name))
{
Thread.CurrentThread.Name = string.Format("Reader {0} for log prefix {1}", this.id, this.instance);
}
string text = string.Format("Reader {0} for log prefix {1} started", this.id, this.instance);
ExTraceGlobals.ReaderTracer.TraceInformation(0, (long)this.GetHashCode(), text);
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("DoWork: called : thread={0}, {1} ?---LogReader", Thread.CurrentThread.ManagedThreadId, text), "", "");
while (!this.CheckServiceStopRequest("DoWork"))
{
this.ReadLog();
}
string message = string.Format("Reader {0} for log prefix {1} stopped", this.id, this.instance);
ExTraceGlobals.ReaderTracer.TraceInformation(0, (long)this.GetHashCode(), message);
ServiceLogger.LogInfo(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogMonitorRequestedStop, Thread.CurrentThread.Name, this.instance, "");
this.stopped = true;
}
internal void DequeueAndWriteOneItem()
{
string text = string.Empty;
IWatermarkFile watermarkFile = null;
try
{
T t = this.queue.Dequeue(this.cancellationContext);
if (t != null)
{
this.perfCounterInstance.BatchQueueLength.Decrement();
text = t.FullLogName;
watermarkFile = this.logManager.FindWatermarkFileObject(text);
if (ExTraceGlobals.WriterTracer.IsTraceEnabled(TraceType.DebugTrace))
{
StringBuilder stringBuilder = new StringBuilder();
foreach (LogFileRange logFileRange in t.LogRanges)
{
string value = string.Format("({0}, {1}) ", logFileRange.StartOffset, logFileRange.EndOffset);
stringBuilder.Append(value);
}
string message = string.Format("For log {0}, writer {1} get batch that has {2} ranges: {3}", new object[]
{
text,
this.id,
t.LogRanges.Count,
stringBuilder.ToString()
});
ExTraceGlobals.WriterTracer.TraceDebug((long)this.GetHashCode(), message);
}
this.WriteBatch(t);
}
else if (!this.CheckServiceStopRequest("DequeueAndWriteOneItem()"))
{
Tools.DebugAssert(this.cancellationContext.StopToken.IsCancellationRequested, "If Enqueue is not cancelled, it should always get a batch back");
}
}
catch (FaultException ex)
{
ExTraceGlobals.WriterTracer.TraceError <string, string>((long)this.GetHashCode(), "Writer failed to write data through web service DAL when processing log {0}. Check the event log on the FFO web service role. The exception is: {1}.", text, ex.Message);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_WebServiceWriteException, ex.Message, new object[]
{
text,
ex.ToString()
});
ServiceLogger.LogError(ServiceLogger.Component.DatabaseWriter, (LogUploaderEventLogConstants.Message) 3221489634U, ex.Message, this.instance, text);
}
catch (CommunicationException ex2)
{
ExTraceGlobals.WriterTracer.TraceError <string, string>((long)this.GetHashCode(), "Writer failed to write data through web service DAL when processing log {0}. The exception is: {1}.", text, ex2.Message);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_WebServiceWriteException, ex2.Message, new object[]
{
text,
ex2.ToString()
});
ServiceLogger.LogError(ServiceLogger.Component.DatabaseWriter, (LogUploaderEventLogConstants.Message) 3221489633U, ex2.Message, this.instance, text);
}
catch (ADTopologyEndpointNotFoundException ex3)
{
ExTraceGlobals.WriterTracer.TraceError <string, string>((long)this.GetHashCode(), "Writer caught an ADTopologyEndpointNotFoundException when processing log {0}. Details is: {1}", text, ex3.Message);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_ADTopologyEndpointNotFound, ex3.Message, new object[]
{
text,
ex3.Message
});
ServiceLogger.LogError(ServiceLogger.Component.DatabaseWriter, (LogUploaderEventLogConstants.Message) 3221489629U, ex3.Message, this.instance, text);
}
catch (Exception ex4)
{
if (!(ex4 is ThreadAbortException) || !this.cancellationContext.StopToken.IsCancellationRequested)
{
string text2 = string.Format("Writer caught an exception when processing log {0}: {1}", text, ex4);
ExTraceGlobals.WriterTracer.TraceError((long)this.GetHashCode(), text2);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_DatabaseWriterUnknownException, ex4.Message, new object[]
{
text,
ex4
});
EventNotificationItem.Publish(ExchangeComponent.Name, "UnexpectedWriterError", null, text2, ResultSeverityLevel.Error, false);
this.perfCounterInstance.TotalUnexpectedWriterErrors.Increment();
ServiceLogger.LogError(ServiceLogger.Component.DatabaseWriter, (LogUploaderEventLogConstants.Message) 3221489617U, text2, this.instance, text);
throw;
}
Thread.ResetAbort();
ServiceLogger.LogInfo(ServiceLogger.Component.DatabaseWriter, LogUploaderEventLogConstants.Message.LogMonitorRequestedStop, Thread.CurrentThread.Name + " received thread abort event", this.instance, "");
}
finally
{
if (watermarkFile != null)
{
watermarkFile.InMemoryCountDecrease();
}
}
}
