private T Dequeue()
{
T result = default(T);
if (!this.queue.TryDequeue(out result))
{
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("Enqueue: called : Type={0}, thread={1}, queueSize={2}, capacity={3} ?---InDequeuExp ", new object[]
{
typeof(T),
Thread.CurrentThread.ManagedThreadId,
this.queue.Count,
this.Capacity
}), "", "");
throw new Exception("TryDequeue should always return true");
}
int num = this.producerSemaphore.Release();
this.ProducerSemaphoreCount = num + 1;
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("Enqueue: called : Type={0}, thread={1}, queueSize={2}, capacity={3}, consumerSemaphore={4} ?---InDequeuDone ", new object[]
{
typeof(T),
Thread.CurrentThread.ManagedThreadId,
this.queue.Count,
this.Capacity,
num
}), "", "");
return(result);
}
private void Enqueue(T item)
{
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("Enqueue: called : Type={0}, thread={1}, queueSize={2}, capacity={3} ?---In ", new object[]
{
typeof(T),
Thread.CurrentThread.ManagedThreadId,
this.queue.Count,
this.Capacity
}), "", "");
if (this.queue.Count >= this.Capacity)
{
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("Enqueue: called : Type={0}, thread={1}, queueSize={2}, capacity={3} ?---InEnqueuFull ", new object[]
{
typeof(T),
Thread.CurrentThread.ManagedThreadId,
this.queue.Count,
this.Capacity
}), "", "");
throw new Exception("Queue should never be full here. The caller should have acquired writer quota");
}
this.queue.Enqueue(item);
int num = this.consumerSemaphore.Release();
this.ConsumerSemaphoreCount = num + 1;
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("Enqueue: called : Type={0}, thread={1}, queueSize={2}, capacity={3}, consumerSemaphore={4} ?---InDone ", new object[]
{
typeof(T),
Thread.CurrentThread.ManagedThreadId,
this.queue.Count,
this.Capacity,
num
}), "", "");
}
internal void FlushMessageBatchBufferToWriter(bool forceFlush)
{
if (!this.shouldBufferBatches)
{
return;
}
if (!LogDataBatchReflectionCache <T> .IsMessageBatch)
{
return;
}
List <T> list = this.messageBatchBuffer;
Tools.DebugAssert(list.Count <= this.maximumBatchCount, "Verify the buffer size will never goes above the maximumBatchCount");
if (!forceFlush && (DateTime.UtcNow - this.lastFluchCheckTime).TotalMilliseconds < 1000.0 && list.Count < this.maximumBatchCount)
{
return;
}
this.lastFluchCheckTime = DateTime.UtcNow;
foreach (T t in list)
{
MessageBatchBase messageBatchBase = t as MessageBatchBase;
Tools.DebugAssert(messageBatchBase != null, "Failed cast to MessageBatchBase.");
if (forceFlush || messageBatchBase.ReadyToFlush(this.newestLogLineTS))
{
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("FlushMessageBatchBufferToWriter: called : thread={0}, forceFlush={1}, this.shouldBufferBatches={2}, bufferCount={3}, this.maximumBatchCount={4} ?---BufEnqueBatch ", new object[]
{
Thread.CurrentThread.ManagedThreadId,
forceFlush,
this.shouldBufferBatches,
list.Count,
this.maximumBatchCount
}), "", "");
this.EnqueueBatch(t);
messageBatchBase.Flushed = true;
}
}
list.RemoveAll(delegate(T batch)
{
MessageBatchBase messageBatchBase3 = batch as MessageBatchBase;
Tools.DebugAssert(messageBatchBase3 != null, "Failed cast to MessageBatchBase.");
return(messageBatchBase3.Flushed);
});
while (list.Count >= this.maximumBatchCount)
{
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("FlushMessageBatchBufferToWriter: called : thread={0}, forceFlush={1}, this.shouldBufferBatches={2}, bufferCount={3}, this.maximumBatchCount={4} ?---BufEnqueBatch2 ", new object[]
{
Thread.CurrentThread.ManagedThreadId,
forceFlush,
this.shouldBufferBatches,
list.Count,
this.maximumBatchCount
}), "", "");
this.EnqueueBatch(list[0]);
MessageBatchBase messageBatchBase2 = list[0] as MessageBatchBase;
Tools.DebugAssert(messageBatchBase2 != null, "Failed cast to MessageBatchBase.");
messageBatchBase2.Flushed = true;
list.RemoveAt(0);
}
}
private void BufferOrEnqueueToWriter()
{
if (!this.shouldBufferBatches)
{
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("BufferOrEnqueueToWriter: called : thread={0} ?---BufEnque2 ", Thread.CurrentThread.ManagedThreadId), "", "");
this.EnqueueBatch(this.activeBatch);
return;
}
this.messageBatchBuffer.Add(this.activeBatch);
Tools.DebugAssert(this.messageBatchBuffer.Count <= this.maximumBatchCount, "Verify the buffer size will never go above the maximumBatchCount");
}
private void DumpPendingProcessLogFilesInfo(IEnumerable <FileInfo> pendingProcessLogFiles)
{
if (ServiceLogger.ServiceLogLevel > ServiceLogger.LogLevel.Debug)
{
return;
}
StringBuilder stringBuilder = new StringBuilder();
foreach (FileInfo fileInfo in pendingProcessLogFiles)
{
stringBuilder.AppendLine(string.Format("file={0},created={1},modified={2}", fileInfo.FullName, fileInfo.CreationTimeUtc, fileInfo.LastWriteTimeUtc));
}
if (stringBuilder.Length > 0)
{
ServiceLogger.LogDebug(ServiceLogger.Component.LogMonitor, LogUploaderEventLogConstants.Message.PendingProcessLogFilesInfo, stringBuilder.ToString(), this.instance, this.logDirectory);
}
}
public void DoWork(object obj)
{
ArgumentValidator.ThrowIfNull("obj", obj);
if (string.IsNullOrEmpty(Thread.CurrentThread.Name))
{
Thread.CurrentThread.Name = string.Format("Writer {0} for log type {1}", this.id, this.instance);
}
ServiceLogger.LogDebug(ServiceLogger.Component.DatabaseWriter, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("DoWork: called : thread={0}, log={1} ?---LogWriter", Thread.CurrentThread.ManagedThreadId, this.instance), "", "");
this.cancellationContext = (CancellationContext)obj;
while (!this.CheckServiceStopRequest("DoWork()"))
{
this.DequeueAndWriteOneItem();
}
string message = string.Format("Writer {0} for log type {1} stopped.", this.id, this.instance);
ExTraceGlobals.WriterTracer.TraceDebug((long)this.GetHashCode(), message);
ServiceLogger.LogInfo(ServiceLogger.Component.DatabaseWriter, LogUploaderEventLogConstants.Message.LogMonitorRequestedStop, Thread.CurrentThread.Name, this.instance, "");
this.stopped = true;
}
public void EnqueueBatch(T batch)
{
if (ExTraceGlobals.ReaderTracer.IsTraceEnabled(TraceType.DebugTrace))
{
for (int i = 0; i < batch.LogRanges.Count; i++)
{
string message = string.Format("InputBuffer: enqueueBatch range {0}: ({1}, {2}) for log {3}", new object[]
{
i,
batch.LogRanges[i].StartOffset,
batch.LogRanges[i].EndOffset,
this.fullLogName
});
ExTraceGlobals.ReaderTracer.TraceDebug((long)this.GetHashCode(), message);
}
}
long size = batch.Size;
long num = (long)batch.NumberOfLinesInBatch;
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("before enqueue; batch size (in original log) {0}, lines {1} +++++++++ ", size, num), this.instance, this.fullLogName);
this.watermarkFileRef.InMemoryCountIncrease();
try
{
this.logDataBatchQueue.Enqueue(batch, this.cancellationContext);
}
catch
{
this.watermarkFileRef.InMemoryCountDecrease();
throw;
}
if (this.cancellationContext.StopToken.IsCancellationRequested)
{
return;
}
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("after enqueue; batch size (in original log) {0}, lines {1} +++++++++ ", size, num), this.instance, this.fullLogName);
this.perfCounterInstance.InputBufferBatchCounts.Decrement();
this.perfCounterInstance.BatchQueueLength.Increment();
this.perfCounterInstance.InputBufferBackfilledLines.IncrementBy(this.newBackfilledLogLines);
this.newBackfilledLogLines = 0L;
}
public T Dequeue(CancellationContext cancellationContext)
{
if (cancellationContext == null)
{
this.consumerSemaphore.WaitOne();
}
else if (WaitHandle.WaitAny(new WaitHandle[]
{
cancellationContext.StopToken.WaitHandle,
this.consumerSemaphore
}) != 1)
{
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("Enqueue: called : Type={0}, thread={1}, queueSize={2}, capacity={3} ?---InDequeueEmptySkip ", new object[]
{
typeof(T),
Thread.CurrentThread.ManagedThreadId,
this.queue.Count,
this.Capacity
}), "", "");
return(default(T));
}
return(this.Dequeue());
}
public void DoWork(object obj)
{
ArgumentValidator.ThrowIfNull("obj", obj);
this.cancellationContext = (CancellationContext)obj;
CancellationToken stopToken = this.cancellationContext.StopToken;
if (string.IsNullOrEmpty(Thread.CurrentThread.Name))
{
Thread.CurrentThread.Name = string.Format("Reader {0} for log prefix {1}", this.id, this.instance);
}
string text = string.Format("Reader {0} for log prefix {1} started", this.id, this.instance);
ExTraceGlobals.ReaderTracer.TraceInformation(0, (long)this.GetHashCode(), text);
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogBatchEnqueue, string.Format("DoWork: called : thread={0}, {1} ?---LogReader", Thread.CurrentThread.ManagedThreadId, text), "", "");
while (!this.CheckServiceStopRequest("DoWork"))
{
this.ReadLog();
}
string message = string.Format("Reader {0} for log prefix {1} stopped", this.id, this.instance);
ExTraceGlobals.ReaderTracer.TraceInformation(0, (long)this.GetHashCode(), message);
ServiceLogger.LogInfo(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogMonitorRequestedStop, Thread.CurrentThread.Name, this.instance, "");
this.stopped = true;
}
internal void ProcessLog(LogFileInfo log)
{
FileStream fileStream = null;
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderStartedParsingLog, string.Format("reprocessing or begin to process, isActive {0} +++++++++", log.IsActive), this.instance, log.FullFileName);
if (log.Status == ProcessingStatus.NeedProcessing)
{
log.Status = ProcessingStatus.InProcessing;
PerfCountersInstanceCache.GetInstance(this.instance).TotalNewLogsBeginProcessing.Increment();
ExTraceGlobals.ReaderTracer.TraceDebug <string, string>((long)this.GetHashCode(), "{0} started parsing log {1}", Thread.CurrentThread.Name, log.FullFileName);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderStartedParsingLog, log.FileName, new object[]
{
this.id,
log.FullFileName
});
ServiceLogger.LogInfo(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderStartedParsingLog, null, this.instance, log.FullFileName);
}
try
{
string defaultLogVersion;
fileStream = this.OpenLogWithRetry(log.FullFileName, out defaultLogVersion);
if (fileStream == null)
{
if (!this.isStopRequested)
{
string text = string.Format("Failed to open log file {0}", log.FullFileName);
ExTraceGlobals.ReaderTracer.TraceError((long)this.GetHashCode(), text);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderFileOpenFailed, log.FullFileName, new object[]
{
log.FullFileName
});
EventNotificationItem.Publish(ExchangeComponent.Name, "FailToOpenLogFile", null, text, ResultSeverityLevel.Error, false);
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 3221229475U, text, this.instance, log.FullFileName);
}
}
else
{
if (string.IsNullOrWhiteSpace(defaultLogVersion))
{
defaultLogVersion = this.logMonitorHelper.GetDefaultLogVersion();
ExTraceGlobals.ReaderTracer.TraceWarning <string, string>((long)this.GetHashCode(), "Failed to figure out version of log file {0}. Use default version {1} instead.", log.FullFileName, defaultLogVersion);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_FailedToGetVersionFromLogHeader, log.FullFileName, new object[]
{
log.FullFileName,
defaultLogVersion
});
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 2147487660U, null, this.instance, log.FullFileName);
}
CsvTable logSchema = this.logMonitorHelper.GetLogSchema(new Version(defaultLogVersion));
CsvFieldCache csvFieldCache = new CsvFieldCache(logSchema, fileStream, 32768);
int num = 0;
this.logHeaderEndOffset = csvFieldCache.Position;
ExTraceGlobals.ReaderTracer.TraceDebug <string, long>((long)this.GetHashCode(), "The end offset of the header of {0} is {1}.", log.FullFileName, this.logHeaderEndOffset);
bool flag = false;
while (!flag)
{
if (this.CheckServiceStopRequest("ProcessLog()"))
{
return;
}
LogFileRange logFileRange = log.WatermarkFileObj.GetBlockToReprocess();
if (logFileRange == null)
{
logFileRange = log.WatermarkFileObj.GetNewBlockToProcess();
flag = true;
if (logFileRange == null)
{
break;
}
}
if (num == 0)
{
long startOffset = logFileRange.StartOffset;
this.inputBuffer = new InputBuffer <T>(this.config.BatchSizeInBytes, startOffset, log, this.batchQueue, this.logPrefix, this.logMonitorHelper, this.messageBatchFlushInterval, this.cancellationContext, this.config.InputBufferMaximumBatchCount, this.instance);
}
bool isFirstBlock = num == 0;
this.inputBuffer.BeforeDataBlockIsProcessed(logFileRange, isFirstBlock);
this.ProcessBlock(csvFieldCache, logFileRange, log);
this.inputBuffer.AfterDataBlockIsProcessed();
if (this.isStopRequested)
{
return;
}
num++;
}
this.EnqueueLastBatchParsed();
log.LastProcessedTime = DateTime.UtcNow;
if (!log.IsActive)
{
log.Status = ProcessingStatus.ReadyToWriteToDatabase;
ExTraceGlobals.ReaderTracer.TraceInformation <string>(0, (long)this.GetHashCode(), "Finished parsing log {0}", log.FullFileName);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderFinishedParsingLog, log.FullFileName, new object[]
{
log.FullFileName
});
ServiceLogger.LogInfo(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderFinishedParsingLog, null, this.instance, log.FullFileName);
}
}
}
finally
{
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderFinishedParsingLog, string.Format("Finished parsing for this round, isActive {0} +++++++++", log.IsActive), this.instance, log.FullFileName);
this.logMonitor.ReaderCompletedProcessingLog(log);
this.Cleanup(fileStream);
}
}