public void Merge(LogFileRange other)
{
if (other == null)
{
throw new ArgumentNullException("other", "The other log file range must not be null.");
}
if (this.ProcessingStatus != other.ProcessingStatus)
{
throw new ArgumentException("The two log file ranges must have the same processing status.", "other");
}
string message = Strings.MergeLogRangesFailed(other.StartOffset, other.EndOffset, this.startOffset, this.EndOffset);
if (this.Overlaps(other))
{
throw new IllegalRangeMergeException(message);
}
if (!this.IsAdjacentTo(other))
{
throw new IllegalRangeMergeException(message);
}
if (this.EndOffset == other.StartOffset)
{
this.EndOffset = other.EndOffset;
return;
}
this.startOffset = other.StartOffset;
}
public void BeforeDataBlockIsProcessed(LogFileRange block, bool isFirstBlock)
{
if (!isFirstBlock && this.activeBatch != null)
{
this.activeBatch.CreateNewRange(block.StartOffset);
}
}
public LogFileRange GetBlockToReprocess()
{
if (this.blocksNeedReprocessing.Count > 0)
{
LogFileRange result = this.blocksNeedReprocessing[0];
this.blocksNeedReprocessing.RemoveAt(0);
return(result);
}
return(null);
}
internal LogFileRange ProcessOneWatermark(string line)
{
ArgumentValidator.ThrowIfNull("line", line);
try
{
string fileRangeFromWatermark = WatermarkFile.GetFileRangeFromWatermark(line);
LogFileRange logFileRange = LogFileRange.Parse(fileRangeFromWatermark);
logFileRange.ProcessingStatus = ProcessingStatus.CompletedProcessing;
lock (this.blocksProcessedLock)
{
if (!this.blocksProcessed.ContainsKey(logFileRange.StartOffset))
{
this.AddRangeToProcessed(logFileRange);
return(logFileRange);
}
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_OverlappingLogRangeInWatermarkFile, this.WatermarkFileFullName, new object[]
{
this.WatermarkFileFullName,
logFileRange.StartOffset,
logFileRange.EndOffset,
logFileRange.StartOffset,
this.blocksProcessed[logFileRange.StartOffset].EndOffset
});
string text = string.Format("There are overlapping log ranges in watermark file {0}: ({1}, {2}), ({3}, {4}).", new object[]
{
this.WatermarkFileFullName,
logFileRange.StartOffset,
logFileRange.EndOffset,
logFileRange.StartOffset,
this.blocksProcessed[logFileRange.StartOffset].EndOffset
});
if (Interlocked.CompareExchange(ref WatermarkFile.overlappingWatermarksInWatermarkFile, 1, 0) == 0)
{
EventNotificationItem.Publish(ExchangeComponent.Name, "OverlappingWatermarkRecordsInFile", null, text, ResultSeverityLevel.Error, false);
}
ServiceLogger.LogError(ServiceLogger.Component.WatermarkFile, (LogUploaderEventLogConstants.Message) 3221231476U, text, this.instance, this.WatermarkFileFullName);
}
}
catch (MalformedLogRangeLineException ex)
{
string text2 = string.Format("Failed to parse watermark from {0}: {1}", this.watermarkFullFileName, ex.Message);
ExTraceGlobals.ReaderTracer.TraceError((long)this.GetHashCode(), text2);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_WatermarkFileParseException, this.watermarkFullFileName, new object[]
{
this.watermarkFullFileName,
ex.Message
});
if (Interlocked.CompareExchange(ref WatermarkFile.watermarkParseError, 1, 0) == 0)
{
EventNotificationItem.Publish(ExchangeComponent.Name, "MalformedWatermarkRecordError", null, text2, ResultSeverityLevel.Warning, false);
}
ServiceLogger.LogError(ServiceLogger.Component.WatermarkFile, (LogUploaderEventLogConstants.Message) 3221231475U, ex.Message, this.instance, this.watermarkFullFileName);
}
return(null);
}
internal void UpdateBlocksProcessed(long startOffset, long endOffset)
{
LogFileRange logFileRange = new LogFileRange(startOffset, endOffset, ProcessingStatus.CompletedProcessing);
lock (this.blocksProcessedLock)
{
if (this.blocksProcessed.ContainsKey(startOffset))
{
string text;
if (this.blocksProcessed[startOffset].EndOffset == endOffset)
{
text = string.Format("Tried to add an existing block ({0}, {1}) when updating in-memory watermarks for log {2}.", startOffset, endOffset, this.logFileFullName);
ExTraceGlobals.WriterTracer.TraceError((long)this.GetHashCode(), text);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_WatermarkFileDuplicateBlock, this.logFileFullName, new object[]
{
startOffset.ToString(),
endOffset.ToString(),
this.logFileFullName
});
ServiceLogger.LogError(ServiceLogger.Component.WatermarkFile, (LogUploaderEventLogConstants.Message) 3221231474U, string.Format("startOffset={0};endOffset={1}", startOffset, endOffset), this.instance, this.logFileFullName);
}
else
{
text = string.Format("Tried to add an block ({0}, {1}) that overlaps with an existing block ({2}, {3}) in the in-memory watermarks for log {4}.", new object[]
{
startOffset,
endOffset,
startOffset,
this.blocksProcessed[startOffset].EndOffset,
this.logFileFullName
});
ExTraceGlobals.WriterTracer.TraceError((long)this.GetHashCode(), text);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_WatermarkFileOverlappingBlock, null, new object[]
{
startOffset,
endOffset,
startOffset,
this.blocksProcessed[startOffset].EndOffset,
this.logFileFullName
});
ServiceLogger.LogError(ServiceLogger.Component.WatermarkFile, (LogUploaderEventLogConstants.Message) 3221231479U, string.Format("startOffset={0};endOffset={1}", startOffset, endOffset), this.instance, this.logFileFullName);
}
if (Interlocked.CompareExchange(ref WatermarkFile.overlappingWatermarksInMemory, 1, 0) == 0)
{
EventNotificationItem.Publish(ExchangeComponent.Name, "OverlappingWatermarkRecordsInMemory", null, text, ResultSeverityLevel.Error, false);
}
}
else
{
this.AddRangeToProcessed(logFileRange);
}
}
}
internal void FindUnprocessedHoles()
{
if (ServiceLogger.ServiceLogLevel == ServiceLogger.LogLevel.Debug)
{
string message = string.Format("FindblocksNeedProcessing found {0} blocks processed for {1}", this.blocksProcessed.Count, this.LogFileFullName);
ExTraceGlobals.ReaderTracer.TraceDebug((long)this.GetHashCode(), message);
}
Tools.DebugAssert(this.blocksNeedReprocessing.Count == 0, "this.blocksNeedProcessing.Count == 0");
bool foundHole = false;
Action <long, long> action = delegate(long s, long e)
{
foundHole = true;
LogFileRange logFileRange2 = new LogFileRange(s, e, ProcessingStatus.NeedProcessing);
this.TraceBlockNeedProcessing(logFileRange2.StartOffset, logFileRange2.EndOffset);
this.blocksNeedReprocessing.Add(logFileRange2);
};
long num = 0L;
lock (this.blocksProcessedLock)
{
int count = this.blocksProcessed.Count;
this.lastCheckedEndOffsertBeforeHoles = 0L;
for (int i = 0; i < count; i++)
{
LogFileRange logFileRange = this.blocksProcessed.Values[i];
if (logFileRange.EndOffset < logFileRange.StartOffset)
{
ServiceLogger.LogCommon(ServiceLogger.LogLevel.Error, "Invalid watermark range", string.Format("{0},{1}", logFileRange.StartOffset, logFileRange.EndOffset), ServiceLogger.Component.WatermarkFile, "", "");
Tools.DebugAssert(false, string.Format("detected invalid range {0},{1}", logFileRange.StartOffset, logFileRange.EndOffset));
}
else
{
if (num < logFileRange.StartOffset)
{
action(num, logFileRange.StartOffset);
}
num = Math.Max(num, logFileRange.EndOffset);
if (!foundHole)
{
this.lastCheckedEndOffsertBeforeHoles = num;
}
}
}
}
if (this.lastReaderProcessedEndOffset > num)
{
action(num, this.lastReaderProcessedEndOffset);
}
}
public LogFileRange Split(long splitOffset)
{
if (this.startOffset >= splitOffset || splitOffset >= this.EndOffset)
{
string paramName = string.Format(CultureInfo.InvariantCulture, "Argument ({0}) must be greater than startOffset ({1}) and less than endOffset ({2})", new object[]
{
splitOffset,
this.startOffset,
this.EndOffset
});
throw new ArgumentOutOfRangeException(paramName, "other");
}
LogFileRange result = new LogFileRange(splitOffset, this.EndOffset, ProcessingStatus.NeedProcessing);
this.EndOffset = splitOffset;
return(result);
}
private void OpenFileAndReadWatermark()
{
if (!File.Exists(this.watermarkFullFileName))
{
return;
}
using (FileStream fileStream = File.Open(this.watermarkFullFileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
using (StreamReader streamReader = new StreamReader(fileStream))
{
string line;
while ((line = streamReader.ReadLine()) != null)
{
LogFileRange logFileRange = this.ProcessOneWatermark(line);
if (logFileRange != null)
{
this.lastReaderProcessedEndOffset = Math.Max(this.lastReaderProcessedEndOffset, logFileRange.EndOffset);
}
}
}
}
}
public static LogFileRange Parse(string line)
{
if (line == null)
{
throw new ArgumentNullException("line");
}
string[] array = line.Split(new char[]
{
','
});
if (array.Length != 2)
{
throw new MalformedLogRangeLineException(Strings.MalformedLogRangeLine(line));
}
LogFileRange result;
try
{
long num = long.Parse(array[0]);
long num2 = long.Parse(array[1]);
if (num < 0L || num2 < 0L || num > num2)
{
throw new MalformedLogRangeLineException(Strings.MalformedLogRangeLine(line));
}
result = new LogFileRange(num, num2, ProcessingStatus.CompletedProcessing);
}
catch (FormatException)
{
throw new MalformedLogRangeLineException(Strings.MalformedLogRangeLine(line));
}
catch (OverflowException)
{
throw new MalformedLogRangeLineException(Strings.MalformedLogRangeLine(line));
}
return(result);
}
private void AddRangeToProcessed(LogFileRange logFileRange)
{
this.blocksProcessed.Add(logFileRange.StartOffset, logFileRange);
Interlocked.Add(ref this.processedBytes, (long)logFileRange.Size);
}
public bool Equals(LogFileRange other)
{
return(this.startOffset == other.startOffset && this.endOffset == other.endOffset && this.processingStatus == other.processingStatus);
}
public bool Overlaps(LogFileRange other)
{
return((this.startOffset <= other.StartOffset && other.StartOffset < this.EndOffset) || (other.StartOffset <= this.startOffset && this.startOffset < other.EndOffset));
}
public bool IsAdjacentTo(LogFileRange other)
{
return(this.EndOffset == other.startOffset || other.EndOffset == this.startOffset);
}
internal void ProcessLog(LogFileInfo log)
{
FileStream fileStream = null;
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderStartedParsingLog, string.Format("reprocessing or begin to process, isActive {0} +++++++++", log.IsActive), this.instance, log.FullFileName);
if (log.Status == ProcessingStatus.NeedProcessing)
{
log.Status = ProcessingStatus.InProcessing;
PerfCountersInstanceCache.GetInstance(this.instance).TotalNewLogsBeginProcessing.Increment();
ExTraceGlobals.ReaderTracer.TraceDebug <string, string>((long)this.GetHashCode(), "{0} started parsing log {1}", Thread.CurrentThread.Name, log.FullFileName);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderStartedParsingLog, log.FileName, new object[]
{
this.id,
log.FullFileName
});
ServiceLogger.LogInfo(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderStartedParsingLog, null, this.instance, log.FullFileName);
}
try
{
string defaultLogVersion;
fileStream = this.OpenLogWithRetry(log.FullFileName, out defaultLogVersion);
if (fileStream == null)
{
if (!this.isStopRequested)
{
string text = string.Format("Failed to open log file {0}", log.FullFileName);
ExTraceGlobals.ReaderTracer.TraceError((long)this.GetHashCode(), text);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderFileOpenFailed, log.FullFileName, new object[]
{
log.FullFileName
});
EventNotificationItem.Publish(ExchangeComponent.Name, "FailToOpenLogFile", null, text, ResultSeverityLevel.Error, false);
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 3221229475U, text, this.instance, log.FullFileName);
}
}
else
{
if (string.IsNullOrWhiteSpace(defaultLogVersion))
{
defaultLogVersion = this.logMonitorHelper.GetDefaultLogVersion();
ExTraceGlobals.ReaderTracer.TraceWarning <string, string>((long)this.GetHashCode(), "Failed to figure out version of log file {0}. Use default version {1} instead.", log.FullFileName, defaultLogVersion);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_FailedToGetVersionFromLogHeader, log.FullFileName, new object[]
{
log.FullFileName,
defaultLogVersion
});
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 2147487660U, null, this.instance, log.FullFileName);
}
CsvTable logSchema = this.logMonitorHelper.GetLogSchema(new Version(defaultLogVersion));
CsvFieldCache csvFieldCache = new CsvFieldCache(logSchema, fileStream, 32768);
int num = 0;
this.logHeaderEndOffset = csvFieldCache.Position;
ExTraceGlobals.ReaderTracer.TraceDebug <string, long>((long)this.GetHashCode(), "The end offset of the header of {0} is {1}.", log.FullFileName, this.logHeaderEndOffset);
bool flag = false;
while (!flag)
{
if (this.CheckServiceStopRequest("ProcessLog()"))
{
return;
}
LogFileRange logFileRange = log.WatermarkFileObj.GetBlockToReprocess();
if (logFileRange == null)
{
logFileRange = log.WatermarkFileObj.GetNewBlockToProcess();
flag = true;
if (logFileRange == null)
{
break;
}
}
if (num == 0)
{
long startOffset = logFileRange.StartOffset;
this.inputBuffer = new InputBuffer <T>(this.config.BatchSizeInBytes, startOffset, log, this.batchQueue, this.logPrefix, this.logMonitorHelper, this.messageBatchFlushInterval, this.cancellationContext, this.config.InputBufferMaximumBatchCount, this.instance);
}
bool isFirstBlock = num == 0;
this.inputBuffer.BeforeDataBlockIsProcessed(logFileRange, isFirstBlock);
this.ProcessBlock(csvFieldCache, logFileRange, log);
this.inputBuffer.AfterDataBlockIsProcessed();
if (this.isStopRequested)
{
return;
}
num++;
}
this.EnqueueLastBatchParsed();
log.LastProcessedTime = DateTime.UtcNow;
if (!log.IsActive)
{
log.Status = ProcessingStatus.ReadyToWriteToDatabase;
ExTraceGlobals.ReaderTracer.TraceInformation <string>(0, (long)this.GetHashCode(), "Finished parsing log {0}", log.FullFileName);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderFinishedParsingLog, log.FullFileName, new object[]
{
log.FullFileName
});
ServiceLogger.LogInfo(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderFinishedParsingLog, null, this.instance, log.FullFileName);
}
}
}
finally
{
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderFinishedParsingLog, string.Format("Finished parsing for this round, isActive {0} +++++++++", log.IsActive), this.instance, log.FullFileName);
this.logMonitor.ReaderCompletedProcessingLog(log);
this.Cleanup(fileStream);
}
}
internal void ProcessBlock(CsvFieldCache cursor, LogFileRange block, LogFileInfo log)
{
bool flag = false;
if (cursor.Position < block.StartOffset)
{
cursor.Seek(block.StartOffset);
}
long num = cursor.Position;
while (cursor.Position < block.EndOffset)
{
if (this.CheckServiceStopRequest("ProcessBlock()"))
{
return;
}
try
{
flag = cursor.MoveNext(true);
if (!flag)
{
if (cursor.AtEnd && !log.IsActive)
{
this.inputBuffer.AddInvalidRowToSkip(num, cursor.Position);
block.EndOffset = cursor.Position;
num = cursor.Position;
}
break;
}
ReadOnlyRow readOnlyRow = new ReadOnlyRow(cursor, num);
this.inputBuffer.LineReceived(readOnlyRow);
num = readOnlyRow.EndPosition;
}
catch (Exception ex)
{
if (RetryHelper.IsSystemFatal(ex))
{
throw;
}
string text = string.Format("Log={0} blockRange=({1},{2}) cursorOffset={3} rowEnd={4} logSize={5} \nException:{6}", new object[]
{
log.FullFileName,
block.StartOffset,
block.EndOffset,
cursor.Position,
num,
log.Size,
ex
});
string periodicKey = string.Format("{0}_{1}_{2}", log.FileName, block.StartOffset, block.EndOffset);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_CsvParserFailedToParseLogLine, periodicKey, new object[]
{
text
});
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 3221229487U, string.Format("Detail={0}", text), this.instance, log.FullFileName);
flag = false;
break;
}
}
if (cursor.Position == block.EndOffset)
{
block.ProcessingStatus = ProcessingStatus.ReadyToWriteToDatabase;
}
if (!flag)
{
if (cursor.AtEnd)
{
if (block.EndOffset == 9223372036854775807L)
{
block.EndOffset = cursor.Position;
}
}
else if (this.logHeaderEndOffset != block.EndOffset)
{
string text2 = string.Format("Failed to read line from file {0} at position {1}", log.FullFileName, num);
ExTraceGlobals.ReaderTracer.TraceError((long)this.GetHashCode(), text2);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderReadFailed, log.FullFileName + "_" + num.ToString(), new object[]
{
log.FullFileName,
num
});
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 3221229476U, text2, this.instance, log.FullFileName);
}
}
long incrementValue = num - block.StartOffset;
if (Tools.IsRawProcessingType <T>())
{
this.perfCounterInstance.RawReaderParsedBytes.IncrementBy(incrementValue);
}
else
{
this.perfCounterInstance.ReaderParsedBytes.IncrementBy(incrementValue);
}
log.WatermarkFileObj.UpdateLastReaderParsedEndOffset(num);
}
