// Token: 0x060001D6 RID: 470 RVA: 0x0000BD6C File Offset: 0x00009F6C
internal static bool TryGetEventData(ReadOnlyRow inputRow, out DateTime entryTimeStamp, out string eventId, out KeyValuePair <string, object>[] eventData, out Exception exception)
{
eventId = null;
entryTimeStamp = DateTime.MinValue;
eventData = null;
exception = null;
DateTime?field;
try
{
field = inputRow.GetField <DateTime?>(0);
eventId = inputRow.GetField <string>(1);
eventData = inputRow.GetField <KeyValuePair <string, object>[]>(2);
}
catch (InvalidCastException ex)
{
exception = ex;
return(false);
}
bool flag = field == null;
bool flag2 = eventId == null;
bool flag3 = eventData == null;
if (flag || flag2 || flag3)
{
exception = new SyncHealthLogInvalidDataException(flag, flag2, flag3);
return(false);
}
entryTimeStamp = field.Value;
eventId = eventId.ToLowerInvariant();
return(true);
}
// Token: 0x0600009C RID: 156 RVA: 0x000050F4 File Offset: 0x000032F4
private void IndexRow(ReadOnlyRow row)
{
ulong position = (ulong)row.Position;
if (position > (ulong)-1)
{
throw new ArgumentOutOfRangeException(string.Format("Message tracking log files cannot be > 4Gb. Current file offset is {0} bytes", position));
}
uint num = (uint)position;
string field = row.GetField <string>(this.columnIdToIndex);
if (field == null)
{
throw new InvalidOperationException(string.Format("Column {0} not found at offset {1}", this.columnIdToIndex, num));
}
string text;
if (this.normalizeMethod != null)
{
text = this.normalizeMethod(field);
}
else
{
text = field;
}
uint hashCode = (uint)text.GetHashCode();
this.indexData.Add(new LogIndex.IndexRecord(hashCode, num));
}
public bool LineReceived(ReadOnlyRow row)
{
ArgumentValidator.ThrowIfNull("row", row);
try
{
ParsedReadOnlyRow parsedReadOnlyRow = new ParsedReadOnlyRow(row);
if (this.ShouldProcessLogLine(parsedReadOnlyRow))
{
this.ProcessRowData(parsedReadOnlyRow);
this.numOfBytesRead += row.EndPosition - row.Position;
this.numberOfLinesInBatch++;
}
}
catch (InvalidLogLineException exception)
{
this.LogErrorAndUpdatePerfCounter(row.Position, row.EndPosition, exception, LogUploaderEventLogConstants.Tuple_LogLineParseError, (LogUploaderEventLogConstants.Message) 3221230473U, "InvalidLogLineInParse");
}
catch (InvalidPropertyValueException exception2)
{
this.LogErrorAndUpdatePerfCounter(row.Position, row.EndPosition, exception2, LogUploaderEventLogConstants.Tuple_InvalidPropertyValueInParse, (LogUploaderEventLogConstants.Message) 3221230481U, "InvalidPropertyValueInParse");
}
catch (MissingPropertyException exception3)
{
this.LogErrorAndUpdatePerfCounter(row.Position, row.EndPosition, exception3, LogUploaderEventLogConstants.Tuple_MissingPropertyInParse, (LogUploaderEventLogConstants.Message) 3221230480U, "MissingPropertyInParse");
}
catch (InvalidCastException exception4)
{
this.LogErrorAndUpdatePerfCounter(row.Position, row.EndPosition, exception4, LogUploaderEventLogConstants.Tuple_InvalidCastInParse, (LogUploaderEventLogConstants.Message) 3221230482U, "InvalidCastInParse");
}
catch (FailedToRetrieveRegionTagException)
{
return(false);
}
return(true);
}
public void LineReceived(ReadOnlyRow row)
{
if (this.activeBatch.IsBatchFull(row))
{
this.FinishAndCreateNewBatch(row);
}
if (this.cancellationContext.StopToken.IsCancellationRequested)
{
return;
}
if (this.shouldBufferBatches)
{
this.MsgtrkLineReceivedHelper(row);
}
else
{
bool flag = this.activeBatch.LineReceived(row);
if (flag)
{
this.activeBatch.UpdateCurrentRangeOffsets(row.Position, row.EndPosition);
}
}
this.FlushMessageBatchBufferToWriter(false);
Tools.DebugAssert(this.messageBatchBuffer.Count <= this.maximumBatchCount, "Verify the buffer size will never go above the maximumBatchCount");
}
private static IEnumerable <TestCaseData> GetRowTestCases()
{
var matrix = ReadOnlyMatrix4x4.FromSystem2DArray(
new float[, ] {
{ 1, 2, 3, 4 },
{ 9, 8, 7, 6 },
{ 3, 6, 12, 24 },
{ 5, 25, 125, 625 }
});
var firstRow = new ReadOnlyRow <float, ReadOnlyMatrix4x4>(
ReadOnlyMatrix4x4.FromSystem2DArray(
new float[, ] {
{ 1, 2, 3, 4 },
{ 0, 0, 0, 0 },
{ 0, 0, 0, 0 },
{ 0, 0, 0, 0 }
}), 0);
var lastRow = new ReadOnlyRow <float, ReadOnlyMatrix4x4>(
ReadOnlyMatrix4x4.FromSystem2DArray(
new float[, ] {
{ 5, 25, 125, 625 },
{ 0, 0, 0, 0 },
{ 0, 0, 0, 0 },
{ 0, 0, 0, 0 }
}), 0);
yield return(new TestCaseData(matrix, 0, firstRow));
yield return(new TestCaseData(matrix, 3, lastRow));
}
// Token: 0x0600007B RID: 123 RVA: 0x00004774 File Offset: 0x00002974
private bool TryOpenAndProcessLogFile(out bool isAlreadyProcessed)
{
isAlreadyProcessed = false;
FileStream fileStream = null;
bool result;
try
{
object obj;
if (!this.TryCheckAlreadyProcessed(out isAlreadyProcessed))
{
result = false;
}
else if (isAlreadyProcessed)
{
result = true;
}
else if (!IOHelper.TryIOOperation(new IOHelper.FileIOOperation(this.OpenLogFile), out obj))
{
result = false;
}
else if (obj == null)
{
result = true;
}
else
{
CsvFieldCache cursor = ((LogFileInfo.OpenLogFileResult)obj).Cursor;
fileStream = ((LogFileInfo.OpenLogFileResult)obj).FileStream;
if (this.isActive && this.activeOffset != 0L)
{
cursor.Seek(this.activeOffset);
}
long position = cursor.Position;
while (cursor.MoveNext(false))
{
if (this.ProcessRow != null)
{
ReadOnlyRow readOnlyRow = new ReadOnlyRow(cursor, position);
ProcessRowEventArgs args = new ProcessRowEventArgs(readOnlyRow);
this.ProcessRow(this, args);
}
position = cursor.Position;
}
this.activeOffset = cursor.Position;
this.length = cursor.Length;
result = true;
}
}
finally
{
if (fileStream != null)
{
fileStream.Close();
}
}
return(result);
}
public virtual bool IsBatchFull(ReadOnlyRow row)
{
if (row == null)
{
throw new ArgumentNullException("row", "row must not be null.");
}
long num = row.EndPosition - row.Position;
return(this.numOfBytesRead + num > (long)this.batchSizeInBytes && this.numberOfLinesInBatch != 0);
}
public void FinishAndCreateNewBatch(ReadOnlyRow row)
{
this.FinishBatch();
this.CreateNewBatch(row.Position);
MessageBatchBase messageBatchBase = this.activeBatch as MessageBatchBase;
if (messageBatchBase != null)
{
messageBatchBase.MessageBatchFlushInterval = this.messageBatchFlushInterval;
}
}
private void MsgtrkLineReceivedHelper(ReadOnlyRow row)
{
bool flag = false;
try
{
ParsedReadOnlyRow parsedReadOnlyRow = new ParsedReadOnlyRow(row);
this.newestLogLineTS = parsedReadOnlyRow.GetField <DateTime>("date-time");
if (this.TryBackfillBufferedMessage(parsedReadOnlyRow))
{
this.activeBatch.StartNewRangeForActiveBatch(row.EndPosition);
}
else
{
flag = this.activeBatch.LineReceived(row);
}
}
catch (InvalidLogLineException exception)
{
this.activeBatch.LogErrorAndUpdatePerfCounter(row.Position, row.EndPosition, exception, LogUploaderEventLogConstants.Tuple_LogLineParseError, (LogUploaderEventLogConstants.Message) 3221230473U, "InvalidLogLineInParse");
flag = true;
}
catch (InvalidPropertyValueException exception2)
{
this.activeBatch.LogErrorAndUpdatePerfCounter(row.Position, row.EndPosition, exception2, LogUploaderEventLogConstants.Tuple_InvalidPropertyValueInParse, (LogUploaderEventLogConstants.Message) 3221230481U, "InvalidPropertyValueInParse");
flag = true;
}
catch (MissingPropertyException exception3)
{
this.activeBatch.LogErrorAndUpdatePerfCounter(row.Position, row.EndPosition, exception3, LogUploaderEventLogConstants.Tuple_MissingPropertyInParse, (LogUploaderEventLogConstants.Message) 3221230480U, "MissingPropertyInParse");
flag = true;
}
catch (InvalidCastException exception4)
{
this.activeBatch.LogErrorAndUpdatePerfCounter(row.Position, row.EndPosition, exception4, LogUploaderEventLogConstants.Tuple_InvalidCastInParse, (LogUploaderEventLogConstants.Message) 3221230482U, "InvalidCastInParse");
flag = true;
}
if (flag)
{
this.activeBatch.UpdateCurrentRangeOffsets(row.Position, row.EndPosition);
}
}
protected override void ProcessRowData(ParsedReadOnlyRow rowData)
{
ReadOnlyRow unParsedRow = rowData.UnParsedRow;
AuditLogRecordType field = unParsedRow.GetField <AuditLogRecordType>(3);
string field2 = unParsedRow.GetField <string>(4);
AuditRecord auditRecord = this.RecordSerializer.Read <AuditRecord>(field2, field);
switch (AuditUploaderConfig.GetAction(auditRecord.RecordType.ToString(), auditRecord.OrganizationName, auditRecord.UserId, auditRecord.Operation))
{
case Actions.LetThrough:
this.records.Add(auditRecord);
break;
case Actions.Skip:
case Actions.SkipAndLogEvent:
break;
default:
return;
}
}
private bool TryBackfillBufferedMessage(ParsedReadOnlyRow parsedRow)
{
if (!this.shouldBufferBatches)
{
return(false);
}
ReadOnlyRow unParsedRow = parsedRow.UnParsedRow;
foreach (T t in this.messageBatchBuffer)
{
MessageBatchBase messageBatchBase = t as MessageBatchBase;
Tools.DebugAssert(messageBatchBase != null, "Failed cast to MessageBatchBase.");
if (!messageBatchBase.ReachedDalOptimizationLimit() && messageBatchBase.ContainsMessage(parsedRow))
{
if (messageBatchBase.LineReceived(unParsedRow))
{
messageBatchBase.AddRangeToBufferedBatch(unParsedRow.Position, unParsedRow.EndPosition);
}
this.newBackfilledLogLines += 1L;
return(true);
}
}
return(false);
}
// Token: 0x06000087 RID: 135 RVA: 0x00004D66 File Offset: 0x00002F66
public ProcessRowEventArgs(ReadOnlyRow readOnlyRow)
{
this.readOnlyRow = readOnlyRow;
}
public static void GetRowTest(
ReadOnlyMatrix4x4 matrix,
int index,
ReadOnlyRow <float, ReadOnlyMatrix4x4> expected)
=> CollectionAssert.AreEqual(
expected, matrix.GetRow <float, ReadOnlyMatrix4x4>(index));
public ParsedReadOnlyRow(ReadOnlyRow unparsedRow)
{
this.table = unparsedRow.Schema;
this.unparsedRow = unparsedRow;
this.parsedRow = new Dictionary <string, object>(StringComparer.InvariantCultureIgnoreCase);
}
internal void ProcessBlock(CsvFieldCache cursor, LogFileRange block, LogFileInfo log)
{
bool flag = false;
if (cursor.Position < block.StartOffset)
{
cursor.Seek(block.StartOffset);
}
long num = cursor.Position;
while (cursor.Position < block.EndOffset)
{
if (this.CheckServiceStopRequest("ProcessBlock()"))
{
return;
}
try
{
flag = cursor.MoveNext(true);
if (!flag)
{
if (cursor.AtEnd && !log.IsActive)
{
this.inputBuffer.AddInvalidRowToSkip(num, cursor.Position);
block.EndOffset = cursor.Position;
num = cursor.Position;
}
break;
}
ReadOnlyRow readOnlyRow = new ReadOnlyRow(cursor, num);
this.inputBuffer.LineReceived(readOnlyRow);
num = readOnlyRow.EndPosition;
}
catch (Exception ex)
{
if (RetryHelper.IsSystemFatal(ex))
{
throw;
}
string text = string.Format("Log={0} blockRange=({1},{2}) cursorOffset={3} rowEnd={4} logSize={5} \nException:{6}", new object[]
{
log.FullFileName,
block.StartOffset,
block.EndOffset,
cursor.Position,
num,
log.Size,
ex
});
string periodicKey = string.Format("{0}_{1}_{2}", log.FileName, block.StartOffset, block.EndOffset);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_CsvParserFailedToParseLogLine, periodicKey, new object[]
{
text
});
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 3221229487U, string.Format("Detail={0}", text), this.instance, log.FullFileName);
flag = false;
break;
}
}
if (cursor.Position == block.EndOffset)
{
block.ProcessingStatus = ProcessingStatus.ReadyToWriteToDatabase;
}
if (!flag)
{
if (cursor.AtEnd)
{
if (block.EndOffset == 9223372036854775807L)
{
block.EndOffset = cursor.Position;
}
}
else if (this.logHeaderEndOffset != block.EndOffset)
{
string text2 = string.Format("Failed to read line from file {0} at position {1}", log.FullFileName, num);
ExTraceGlobals.ReaderTracer.TraceError((long)this.GetHashCode(), text2);
EventLogger.Logger.LogEvent(LogUploaderEventLogConstants.Tuple_LogReaderReadFailed, log.FullFileName + "_" + num.ToString(), new object[]
{
log.FullFileName,
num
});
ServiceLogger.LogError(ServiceLogger.Component.LogReader, (LogUploaderEventLogConstants.Message) 3221229476U, text2, this.instance, log.FullFileName);
}
}
long incrementValue = num - block.StartOffset;
if (Tools.IsRawProcessingType <T>())
{
this.perfCounterInstance.RawReaderParsedBytes.IncrementBy(incrementValue);
}
else
{
this.perfCounterInstance.ReaderParsedBytes.IncrementBy(incrementValue);
}
log.WatermarkFileObj.UpdateLastReaderParsedEndOffset(num);
}