protected DelimitedLogRecordBase(string[] data, DelimitedLogContext context)
{
if (context?.Mapping is null)
{
throw new ArgumentNullException($"Field mapping for this record is not determined. This might be due to field mapping line not present in the log. " +
$"Consider specifying '{ConfigConstants.DEFAULT_FIELD_MAPPING}' in the source configuration");
}
_data = data;
_context = context;
}
public static DelimitedLogContext CreateDelimitedLogSourceInfo(string filePath, long position)
{
var context = new DelimitedLogContext()
{
FilePath = filePath, Position = position
};
if (position > 0)
{
context.LineNumber = GetLineCount(filePath, position);
}
return(context);
}
protected override bool ShouldStopReading(string line, StreamReader sr, DelimitedLogContext context)
{
//Sometimes the log writer may expand the log by writing a block of \x00
//In this case, we rewind the position and retry from the position again
if (line.StartsWith("\x00", StringComparison.Ordinal))
{
//Need to rewind the position
context.Position = sr.BaseStream.Position - line.Length;
return(true);
}
else
{
return(base.ShouldStopReading(line, sr, context));
}
}
/// <summary>
/// Get field index map from a stream up to the position
/// </summary>
/// <param name="fs"></param>
/// <param name="position"></param>
/// <returns></returns>
protected IDictionary <string, int> GetFieldIndexMap(StreamReader sr, DelimitedLogContext context)
{
IDictionary <string, int> fieldIndexMap = null;
while (!sr.EndOfStream && sr.BaseStream.Position < context.Position)
{
string line = sr.ReadLine();
context.LineNumber++;
if (IsHeader(line))
{
fieldIndexMap = GetFieldIndexMap(line);
}
}
return(fieldIndexMap);
}
protected override void AnalyzeMapping(DelimitedLogContext context)
{
base.AnalyzeMapping(context);
if (!string.IsNullOrWhiteSpace(this.TimeStampField))
{
context.TimeStampField = TimeStampField;
}
else if (context.Mapping.ContainsKey("date-time"))
{
context.TimeStampField = "date-time";
}
else if (context.Mapping.ContainsKey("DateTime"))
{
context.TimeStampField = "DateTime";
}
else
{
throw new Exception("Exchange log parser cannot determine date-time field");
}
}
public virtual IEnumerable <IEnvelope <TData> > ParseRecords(StreamReader sr, DelimitedLogContext context)
{
if (context.Position > 0)
{
if (context.Mapping == null)
{
//Need to get the fieldIndexMap
context.Mapping = GetFieldIndexMap(sr, context);
if (context.Mapping != null)
{
AnalyzeMapping(context);
}
}
else
{
sr.BaseStream.Position = context.Position;
}
}
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
context.LineNumber++;
if (IsHeader(line))
{
context.Mapping = GetFieldIndexMap(line);
AnalyzeMapping(context);
}
else if (IsComment(line))
{
continue;
}
else
{
string[] data;
if (_delimiter == ",")
{
data = Utility.ParseCSVLine(line, new StringBuilder()).ToArray();
}
else
{
data = SplitData(line);
}
TData record = _recordFactoryMethod(data, context);
yield return(new LogEnvelope <TData>(
record,
ToUniversalTime(record.TimeStamp),
line,
context.FilePath,
context.Position,
context.LineNumber));
}
}
}
protected virtual void AnalyzeMapping(DelimitedLogContext context)
{
}
public W3SVCLogRecord(string[] data, DelimitedLogContext context) : base(data, context)
{
}
public virtual IEnumerable <IEnvelope <TData> > ParseRecords(StreamReader sr, DelimitedLogContext context)
{
if (context.Position > 0)
{
if (context.Mapping == null)
{
//Need to get the fieldIndexMap
context.Mapping = GetFieldIndexMap(sr, context);
if (context.Mapping != null)
{
AnalyzeMapping(context);
}
}
else
{
sr.BaseStream.Position = context.Position;
}
}
if (context.Mapping == null && _defaultMapping != null)
{
context.Mapping = GetFieldIndexMap(_defaultMapping);
}
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
if (ShouldStopReading(line, sr, context))
{
break;
}
context.LineNumber++;
if (IsHeader(line))
{
context.Mapping = GetFieldIndexMap(line);
AnalyzeMapping(context);
}
else if (IsComment(line))
{
continue;
}
else
{
string[] data;
if (_delimiter == ",")
{
data = Utility.ParseCSVLine(line, new StringBuilder()).ToArray();
}
else
{
data = SplitData(line);
}
TData record = _recordFactoryMethod(data, context);
//If we failed to get the timestamp, we log the error and continue reading the log
DateTime?timestamp = null;
try
{
timestamp = record.TimeStamp;
}
catch (Exception ex)
{
_plugInContext?.Logger?.LogError($"Failed to get time stamp in {context.FilePath}, {context.LineNumber}: {ex.ToMinimized()}");
_plugInContext?.Logger?.LogError(line);
}
if (timestamp.HasValue)
{
yield return(new LogEnvelope <TData>(
record,
ToUniversalTime(timestamp.Value),
line,
context.FilePath,
context.Position,
context.LineNumber));
}
}
}
}
protected virtual bool ShouldStopReading(string line, StreamReader sr, DelimitedLogContext context)
{
return(false);
}
//Record layout
//"date-time,client-ip,client-hostname,server-ip,server-hostname,source-context,connector-id,source,event-id,internal-message-id,message-id,network-message-id,recipient-address,recipient-status,total-bytes,recipient-count,related-recipient-address,reference,message-subject,sender-address,return-path,message-info,directionality,tenant-id,original-client-ip,original-server-ip,custom-data"
//Or could be
//DateTime,RequestId,MajorVersion,MinorVersion,BuildVersion,RevisionVersion,ClientRequestId,AuthenticationType,IsAuthenticated,AuthenticatedUser,Organization,UserAgent,VersionInfo,ClientIpAddress,ServerHostName,FrontEndServer,SoapAction,HttpStatus,RequestSize,ResponseSize,ErrorCode,ImpersonatedUser,ProxyAsUser,ActAsUser,Cookie,CorrelationGuid,PrimaryOrProxyServer,TaskType,RemoteBackendCount,LocalMailboxCount,RemoteMailboxCount,LocalIdCount,RemoteIdCount,BeginBudgetConnections,EndBudgetConnections,BeginBudgetHangingConnections,EndBudgetHangingConnections,BeginBudgetAD,EndBudgetAD,BeginBudgetCAS,EndBudgetCAS,BeginBudgetRPC,EndBudgetRPC,BeginBudgetFindCount,EndBudgetFindCount,BeginBudgetSubscriptions,EndBudgetSubscriptions,MDBResource,MDBHealth,MDBHistoricalLoad,ThrottlingPolicy,ThrottlingDelay,ThrottlingRequestType,TotalDCRequestCount,TotalDCRequestLatency,TotalMBXRequestCount,TotalMBXRequestLatency,RecipientLookupLatency,ExchangePrincipalLatency,HttpPipelineLatency,CheckAccessCoreLatency,AuthModuleLatency,CallContextInitLatency,PreExecutionLatency,CoreExecutionLatency,TotalRequestTime,DetailedExchangePrincipalLatency,ClientStatistics,GenericInfo,AuthenticationErrors,GenericErrors,Puid
public ExchangeLogRecord(string[] data, DelimitedLogContext context) : base(data, context)
{
_context = context;
}
public DelimitedLogRecord(string[] data, DelimitedLogContext context, Func <DelimitedLogRecordBase, DateTime> getDateTime) : base(data, context)
{
_getDateTime = getDateTime;
}
protected DelimitedLogRecordBase(string[] data, DelimitedLogContext context)
{
_data = data;
_context = context;
}
public override IEnumerable <IEnvelope <DelimitedLogRecord> > ParseRecords(StreamReader sr, DelimitedLogContext context)
{
if (_headers != null && context.Mapping == null)
{
context.Mapping = GetFieldIndexMap(_headers);
}
return(base.ParseRecords(sr, context));
}