// Token: 0x06000023 RID: 35 RVA: 0x00003060 File Offset: 0x00001260
private void FindEnd(DateTime end)
{
ExTraceGlobals.ServiceTracer.TraceDebug((long)this.GetHashCode(), "MsExchangeLogSearch LogCursor FindFindEnd");
this.endIdx = this.files.Count - 1;
while (this.endIdx >= this.idx)
{
using (FileStream fileStream = CsvFieldCache.OpenLogFile(this.files[this.endIdx].FullName))
{
if (fileStream != null)
{
CsvFieldCache csvFieldCache = new CsvFieldCache(this.table, this.schemaVersion, fileStream, 32768);
this.table.ClusteredIndex.Find(end, csvFieldCache);
this.totalBytes = this.totalBytes - this.files[this.endIdx].Length + csvFieldCache.Position;
if (csvFieldCache.AtEnd)
{
this.endOffset = csvFieldCache.Position + 1L;
}
else
{
this.endOffset = csvFieldCache.Position;
}
break;
}
this.totalBytes -= this.files[this.endIdx].Length;
this.endIdx--;
}
}
}
// Token: 0x06000022 RID: 34 RVA: 0x00002EB0 File Offset: 0x000010B0
private void FindBeginning(DateTime begin)
{
ExTraceGlobals.ServiceTracer.TraceDebug((long)this.GetHashCode(), "MsExchangeLogSearch LogCursor FindBeginning");
this.idx = 0;
while (this.idx < this.files.Count)
{
this.data = CsvFieldCache.OpenLogFile(this.files[this.idx].FullName);
if (this.data == null)
{
this.totalBytes -= this.files[this.idx].Length;
this.idx++;
}
else
{
this.cursor = new CsvFieldCache(this.table, this.schemaVersion, this.data, 32768);
bool flag = this.table.ClusteredIndex.Find(begin, this.cursor);
this.totalBytes = this.totalBytes - this.files[this.idx].Length + this.cursor.Length - this.cursor.Position;
if (flag)
{
ExTraceGlobals.ServiceTracer.TraceError <string>((long)this.GetHashCode(), "MsExchangeLogSearch LogCursor FindBeginning found file {0} corrupted", this.data.Name);
this.LogFileCorrupted(this.data.Name);
}
if (!this.cursor.AtEnd)
{
return;
}
ExTraceGlobals.ServiceTracer.TraceDebug <string>((long)this.GetHashCode(), "MsExchangeLogSearch LogCursor FindBeginning close file {0}", this.data.Name);
this.data.Close();
this.data = null;
this.cursor = null;
this.idx++;
}
}
}
// Token: 0x0600007B RID: 123 RVA: 0x00004774 File Offset: 0x00002974
private bool TryOpenAndProcessLogFile(out bool isAlreadyProcessed)
{
isAlreadyProcessed = false;
FileStream fileStream = null;
bool result;
try
{
object obj;
if (!this.TryCheckAlreadyProcessed(out isAlreadyProcessed))
{
result = false;
}
else if (isAlreadyProcessed)
{
result = true;
}
else if (!IOHelper.TryIOOperation(new IOHelper.FileIOOperation(this.OpenLogFile), out obj))
{
result = false;
}
else if (obj == null)
{
result = true;
}
else
{
CsvFieldCache cursor = ((LogFileInfo.OpenLogFileResult)obj).Cursor;
fileStream = ((LogFileInfo.OpenLogFileResult)obj).FileStream;
if (this.isActive && this.activeOffset != 0L)
{
cursor.Seek(this.activeOffset);
}
long position = cursor.Position;
while (cursor.MoveNext(false))
{
if (this.ProcessRow != null)
{
ReadOnlyRow readOnlyRow = new ReadOnlyRow(cursor, position);
ProcessRowEventArgs args = new ProcessRowEventArgs(readOnlyRow);
this.ProcessRow(this, args);
}
position = cursor.Position;
}
this.activeOffset = cursor.Position;
this.length = cursor.Length;
result = true;
}
}
finally
{
if (fileStream != null)
{
fileStream.Close();
}
}
return(result);
}
// Token: 0x06000046 RID: 70 RVA: 0x00003BF4 File Offset: 0x00001DF4
public override object GetValue(LogCursor row)
{
object field = row.GetField(this.Index);
if (this.Index == 10)
{
string messageId = field as string;
return(CsvFieldCache.NormalizeMessageID(messageId));
}
return(field);
}
// Token: 0x0600007C RID: 124 RVA: 0x00004880 File Offset: 0x00002A80
private LogFileInfo.OpenLogFileResult OpenLogFile()
{
FileStream fileStream = CsvFieldCache.OpenLogFile(this.fullName);
if (fileStream == null)
{
return(null);
}
CsvFieldCache cursor = new CsvFieldCache(this.csvTable, fileStream, 32768);
return(new LogFileInfo.OpenLogFileResult(cursor, fileStream));
}
// Token: 0x06000020 RID: 32 RVA: 0x00002C64 File Offset: 0x00000E64
private bool MoveNextPrivate()
{
ExTraceGlobals.ServiceTracer.TraceDebug((long)this.GetHashCode(), "MsExchangeLogSearch LogCursor MoveNextPrivate");
while (this.idx <= this.endIdx)
{
if (this.cursor != null)
{
bool flag;
if (this.firstIsReady)
{
this.firstIsReady = false;
flag = true;
}
else
{
long position = this.cursor.Position;
flag = this.cursor.MoveNext(false);
this.currentBytes += this.cursor.Position - position;
}
if (flag && (this.idx < this.endIdx || this.cursor.Position < this.endOffset))
{
if (this.cursor.FieldCount != this.table.Fields.Length)
{
this.LogFileCorrupted(this.files[this.idx].FullName);
}
return(true);
}
ExTraceGlobals.ServiceTracer.TraceDebug <string>((long)this.GetHashCode(), "MsExchangeLogSearch LogCursor MoveNextPrivate close file {0}", this.data.Name);
this.data.Close();
this.data = null;
this.cursor = null;
this.idx++;
}
else
{
this.data = CsvFieldCache.OpenLogFile(this.files[this.idx].FullName);
if (this.data == null)
{
this.totalBytes -= this.files[this.idx].Length;
this.idx++;
}
else
{
this.cursor = new CsvFieldCache(this.table, this.schemaVersion, this.data, 32768);
this.totalBytes = this.totalBytes - this.files[this.idx].Length + this.cursor.Length;
}
}
}
return(false);
}
// Token: 0x06000024 RID: 36 RVA: 0x0000319C File Offset: 0x0000139C
public bool SearchNext(string column, string value)
{
if (this.fileEnumerator == null)
{
this.fileEnumerator = this.files.GetEnumerator();
if (!this.fileEnumerator.MoveNext())
{
return(false);
}
this.offsetEnumerator = this.fileEnumerator.Current.GetOffsetsForData(column.ToLower(), value);
}
while (!this.offsetEnumerator.MoveNext())
{
if (!this.fileEnumerator.MoveNext())
{
return(false);
}
this.searchFileIdx++;
this.offsetEnumerator = this.fileEnumerator.Current.GetOffsetsForData(column.ToLower(), value);
}
if ((ulong)this.offsetEnumerator.Current > (ulong)this.fileEnumerator.Current.Length)
{
return(false);
}
if (this.data != null)
{
this.data.Close();
this.cursor = null;
}
this.data = CsvFieldCache.OpenLogFile(this.fileEnumerator.Current.FullName);
if (this.data == null)
{
ExTraceGlobals.ServiceTracer.TraceError <string>((long)this.GetHashCode(), "Unable to open logfile: {0}", this.fileEnumerator.Current.FullName);
return(false);
}
this.cursor = new CsvFieldCache(this.table, this.schemaVersion, this.data, 32768);
this.cursor.Seek((long)((ulong)this.offsetEnumerator.Current));
if (!this.cursor.MoveNext(false))
{
return(false);
}
this.idx = this.searchFileIdx;
this.currentBytes += (long)((ulong)this.offsetEnumerator.Current);
this.totalBytes = this.totalBytes - this.files[this.searchFileIdx].Length + (long)((ulong)this.offsetEnumerator.Current);
return(true);
}
internal FileStream OpenLogWithRetry(string logFileName, out string logVersion)
{
FileStream fileStream = null;
logVersion = string.Empty;
for (int i = 0; i < 3; i++)
{
fileStream = CsvFieldCache.OpenLogFile(logFileName, out logVersion);
if (fileStream != null)
{
ExTraceGlobals.ReaderTracer.TraceDebug <string, string>((long)this.GetHashCode(), "Open Log file {0}, Version={1}", logFileName, logVersion);
break;
}
if (this.CheckServiceStopRequest("OpenLogWithRetry()") || new FileInfo(logFileName).Length == 0L)
{
break;
}
Thread.Sleep((int)this.config.OpenFileRetryInterval.TotalMilliseconds);
}
return(fileStream);
}
// Token: 0x0600009E RID: 158 RVA: 0x00005200 File Offset: 0x00003400
private List <uint> GetOffsetsForDataFast(string searchData)
{
uint hashCode = (uint)searchData.GetHashCode();
LogIndex.IndexRecord item = new LogIndex.IndexRecord(hashCode, 0U);
int num = this.indexData.BinarySearch(item, LogIndex.IndexRecordComparer.Default);
if (num < 0)
{
num = ~num;
}
List <uint> list = new List <uint>();
for (int i = num; i < this.indexData.Count; i++)
{
LogIndex.IndexRecord indexRecord = this.indexData[i];
uint offset = indexRecord.Offset;
uint hashCode2 = indexRecord.HashCode;
if (hashCode2 != hashCode)
{
break;
}
object obj;
if (!IOHelper.TryIOOperation(() => CsvFieldCache.OpenLogFile(this.logFilePath), out obj) || obj == null)
{
ExTraceGlobals.ServiceTracer.TraceError <string>((long)this.GetHashCode(), "Search failed, unable to open file: {0}", this.logFilePath);
throw new LogSearchException(LogSearchErrorCode.LOGSEARCH_E_RPC_SERVER_UNAVAILABLE);
}
FileStream fileStream = (FileStream)obj;
try
{
CsvFieldCache csvFieldCache = new CsvFieldCache(this.table, fileStream, 32768);
if (offset != 0U)
{
csvFieldCache.Seek((long)((ulong)offset));
}
long position = csvFieldCache.Position;
while (csvFieldCache.MoveNext(false) && position - (long)((ulong)offset) <= (long)LogIndex.ClusterRange)
{
object field = csvFieldCache.GetField(this.columnIdToIndex);
if (field == null)
{
break;
}
uint hashCode3;
if (this.normalizeMethod != null)
{
hashCode3 = (uint)this.normalizeMethod((string)field).GetHashCode();
}
else
{
hashCode3 = (uint)field.GetHashCode();
}
if (hashCode3 == hashCode)
{
if (position > (long)((ulong)-1))
{
break;
}
list.Add((uint)position);
}
position = csvFieldCache.Position;
}
}
finally
{
if (fileStream != null)
{
fileStream.Close();
}
}
}
return(list);
}
internal static LogSearchCursor GetCursor(string server, ServerVersion version, string logFile, string messageId, ProxyAddressCollection senderProxyAddresses, DateTime startTime, DateTime endTime)
{
LogQuery logQuery = new LogQuery();
LogAndCondition logAndCondition = new LogAndCondition();
if (!string.IsNullOrEmpty(messageId))
{
LogCondition fieldStringComparison = TrackingSearch.GetFieldStringComparison(MessageTrackingField.MessageId, CsvFieldCache.NormalizeMessageID(messageId));
logAndCondition.Conditions.Add(fieldStringComparison);
}
if (senderProxyAddresses != null)
{
LogOrCondition logOrCondition = new LogOrCondition();
foreach (ProxyAddress proxyAddress in senderProxyAddresses)
{
LogCondition fieldStringComparison2 = TrackingSearch.GetFieldStringComparison(MessageTrackingField.SenderAddress, proxyAddress.AddressString);
logOrCondition.Conditions.Add(fieldStringComparison2);
}
logAndCondition.Conditions.Add(logOrCondition);
}
logQuery.Filter = logAndCondition;
logQuery.Beginning = startTime;
logQuery.End = endTime;
return(new LogSearchCursor(MessageTrackingSchema.MessageTrackingEvent, server, version, logFile, logQuery, null));
}
// Token: 0x06000085 RID: 133 RVA: 0x00004D48 File Offset: 0x00002F48
public OpenLogFileResult(CsvFieldCache cursor, FileStream fileStream)
{
this.cursor = cursor;
this.fileStream = fileStream;
}
protected override LogCondition GetCondition()
{
LogAndCondition logAndCondition = new LogAndCondition();
if (this.eventId != null)
{
logAndCondition.Conditions.Add(this.GetFieldStringComparison(MessageTrackingField.EventId, this.eventId));
}
if (this.sender != null)
{
logAndCondition.Conditions.Add(this.GetSenderCondition());
}
if (this.recipients != null)
{
logAndCondition.Conditions.Add(this.GetRecipientsCondition());
}
if (this.messageId != null)
{
logAndCondition.Conditions.Add(this.GetFieldStringComparison(MessageTrackingField.MessageId, CsvFieldCache.NormalizeMessageID(this.messageId)));
}
if (this.internalMessageId != null)
{
logAndCondition.Conditions.Add(this.GetFieldStringComparison(MessageTrackingField.InternalMessageId, this.internalMessageId));
}
if (this.subject != null)
{
LogConditionField logConditionField = new LogConditionField();
logConditionField.Name = base.Table.Fields[18].Name;
LogConditionConstant logConditionConstant = new LogConditionConstant();
logConditionConstant.Value = this.subject;
LogStringContainsCondition logStringContainsCondition = new LogStringContainsCondition();
logStringContainsCondition.Left = logConditionField;
logStringContainsCondition.Right = logConditionConstant;
logStringContainsCondition.IgnoreCase = true;
logAndCondition.Conditions.Add(logStringContainsCondition);
}
if (this.reference != null)
{
logAndCondition.Conditions.Add(this.GetFieldForAnyCondition(MessageTrackingField.Reference, this.reference, "e"));
}
return(logAndCondition);
}
internal void ProcessLog(LogFileInfo log)
{
FileStream fileStream = null;
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderStartedParsingLog, string.Format("reprocessing or begin to process, isActive {0} +++++++++", log.IsActive), this.instance, log.FullFileName);
if (log.Status == ProcessingStatus.NeedProcessing)
{
log.Status = ProcessingStatus.InProcessing;
PerfCountersInstanceCache.GetInstance(this.instance).TotalNewLogsBeginProcessing.Increment();
ExTraceGlobals.ReaderTracer.TraceDebug <string, string>((long)this.GetHashCode(), "{0} started parsing log {1}", Thread.CurrentThread.Name, log.FullFileName);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderStartedParsingLog, log.FileName, new object[]
{
this.id,
log.FullFileName
});
ServiceLogger.LogInfo(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderStartedParsingLog, null, this.instance, log.FullFileName);
}
try
{
string defaultLogVersion;
fileStream = this.OpenLogWithRetry(log.FullFileName, out defaultLogVersion);
if (fileStream == null)
{
if (!this.isStopRequested)
{
string text = string.Format("Failed to open log file {0}", log.FullFileName);
ExTraceGlobals.ReaderTracer.TraceError((long)this.GetHashCode(), text);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderFileOpenFailed, log.FullFileName, new object[]
{
log.FullFileName
});
EventNotificationItem.Publish(ExchangeComponent.Name, "FailToOpenLogFile", null, text, ResultSeverityLevel.Error, false);
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 3221229475U, text, this.instance, log.FullFileName);
}
}
else
{
if (string.IsNullOrWhiteSpace(defaultLogVersion))
{
defaultLogVersion = this.logMonitorHelper.GetDefaultLogVersion();
ExTraceGlobals.ReaderTracer.TraceWarning <string, string>((long)this.GetHashCode(), "Failed to figure out version of log file {0}. Use default version {1} instead.", log.FullFileName, defaultLogVersion);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_FailedToGetVersionFromLogHeader, log.FullFileName, new object[]
{
log.FullFileName,
defaultLogVersion
});
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 2147487660U, null, this.instance, log.FullFileName);
}
CsvTable logSchema = this.logMonitorHelper.GetLogSchema(new Version(defaultLogVersion));
CsvFieldCache csvFieldCache = new CsvFieldCache(logSchema, fileStream, 32768);
int num = 0;
this.logHeaderEndOffset = csvFieldCache.Position;
ExTraceGlobals.ReaderTracer.TraceDebug <string, long>((long)this.GetHashCode(), "The end offset of the header of {0} is {1}.", log.FullFileName, this.logHeaderEndOffset);
bool flag = false;
while (!flag)
{
if (this.CheckServiceStopRequest("ProcessLog()"))
{
return;
}
LogFileRange logFileRange = log.WatermarkFileObj.GetBlockToReprocess();
if (logFileRange == null)
{
logFileRange = log.WatermarkFileObj.GetNewBlockToProcess();
flag = true;
if (logFileRange == null)
{
break;
}
}
if (num == 0)
{
long startOffset = logFileRange.StartOffset;
this.inputBuffer = new InputBuffer <T>(this.config.BatchSizeInBytes, startOffset, log, this.batchQueue, this.logPrefix, this.logMonitorHelper, this.messageBatchFlushInterval, this.cancellationContext, this.config.InputBufferMaximumBatchCount, this.instance);
}
bool isFirstBlock = num == 0;
this.inputBuffer.BeforeDataBlockIsProcessed(logFileRange, isFirstBlock);
this.ProcessBlock(csvFieldCache, logFileRange, log);
this.inputBuffer.AfterDataBlockIsProcessed();
if (this.isStopRequested)
{
return;
}
num++;
}
this.EnqueueLastBatchParsed();
log.LastProcessedTime = DateTime.UtcNow;
if (!log.IsActive)
{
log.Status = ProcessingStatus.ReadyToWriteToDatabase;
ExTraceGlobals.ReaderTracer.TraceInformation <string>(0, (long)this.GetHashCode(), "Finished parsing log {0}", log.FullFileName);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderFinishedParsingLog, log.FullFileName, new object[]
{
log.FullFileName
});
ServiceLogger.LogInfo(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderFinishedParsingLog, null, this.instance, log.FullFileName);
}
}
}
finally
{
ServiceLogger.LogDebug(ServiceLogger.Component.LogReader, LogUploaderEventLogConstants.Message.LogReaderFinishedParsingLog, string.Format("Finished parsing for this round, isActive {0} +++++++++", log.IsActive), this.instance, log.FullFileName);
this.logMonitor.ReaderCompletedProcessingLog(log);
this.Cleanup(fileStream);
}
}
internal void ProcessBlock(CsvFieldCache cursor, LogFileRange block, LogFileInfo log)
{
bool flag = false;
if (cursor.Position < block.StartOffset)
{
cursor.Seek(block.StartOffset);
}
long num = cursor.Position;
while (cursor.Position < block.EndOffset)
{
if (this.CheckServiceStopRequest("ProcessBlock()"))
{
return;
}
try
{
flag = cursor.MoveNext(true);
if (!flag)
{
if (cursor.AtEnd && !log.IsActive)
{
this.inputBuffer.AddInvalidRowToSkip(num, cursor.Position);
block.EndOffset = cursor.Position;
num = cursor.Position;
}
break;
}
ReadOnlyRow readOnlyRow = new ReadOnlyRow(cursor, num);
this.inputBuffer.LineReceived(readOnlyRow);
num = readOnlyRow.EndPosition;
}
catch (Exception ex)
{
if (RetryHelper.IsSystemFatal(ex))
{
throw;
}
string text = string.Format("Log={0} blockRange=({1},{2}) cursorOffset={3} rowEnd={4} logSize={5} \nException:{6}", new object[]
{
log.FullFileName,
block.StartOffset,
block.EndOffset,
cursor.Position,
num,
log.Size,
ex
});
string periodicKey = string.Format("{0}_{1}_{2}", log.FileName, block.StartOffset, block.EndOffset);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_CsvParserFailedToParseLogLine, periodicKey, new object[]
{
text
});
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 3221229487U, string.Format("Detail={0}", text), this.instance, log.FullFileName);
flag = false;
break;
}
}
if (cursor.Position == block.EndOffset)
{
block.ProcessingStatus = ProcessingStatus.ReadyToWriteToDatabase;
}
if (!flag)
{
if (cursor.AtEnd)
{
if (block.EndOffset == 9223372036854775807L)
{
block.EndOffset = cursor.Position;
}
}
else if (this.logHeaderEndOffset != block.EndOffset)
{
string text2 = string.Format("Failed to read line from file {0} at position {1}", log.FullFileName, num);
ExTraceGlobals.ReaderTracer.TraceError((long)this.GetHashCode(), text2);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderReadFailed, log.FullFileName + "_" + num.ToString(), new object[]
{
log.FullFileName,
num
});
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 3221229476U, text2, this.instance, log.FullFileName);
}
}
long incrementValue = num - block.StartOffset;
if (Tools.IsRawProcessingType <T>())
{
this.perfCounterInstance.RawReaderParsedBytes.IncrementBy(incrementValue);
}
else
{
this.perfCounterInstance.ReaderParsedBytes.IncrementBy(incrementValue);
}
log.WatermarkFileObj.UpdateLastReaderParsedEndOffset(num);
}
