/// <summary>
/// Creates and initializes an instance of the class <c>RawMessage</c>.
/// </summary>
/// <param name="format">The format.</param>
/// <param name="data">The data.</param>
/// <param name="readLength">Length of the read.</param>
/// <param name="sessionId">The session id.</param>
public RawMessage(RawMessageFormat format, byte[] data, int readLength, int sessionId)
{
this.format = format;
this.data = data;
this.readLength = readLength;
this.sessionId = sessionId;
}
private static IGenericMessage ParseMessage(string textMsg, RawMessageFormat format)
{
IGenericMessage message;
if (format == RawMessageFormat.Binary)
{
byte[] binaryData = Convert.FromBase64String(textMsg);
if (binarySerializer == null)
{
binarySerializer = new BinaryMessageSerializer();
}
message = binarySerializer.DeserializeFromBytes(binaryData, null);
}
else
{
//{"Type":10,"RequestId":2,"Target":"BSAG.Xitaro.XBCI.Requester.Interface.Services.IRouterService","Name":"GetLatestSequenceNumbers()","Payload":[]}
message = new GenericMessage();
message.Type = (MessageType)short.Parse(GetJsonSimpleStringValue(textMsg, "Type"));
message.RequestId = long.Parse(GetJsonSimpleStringValue(textMsg, "RequestId"));
message.Name = GetJsonSimpleStringValue(textMsg, "Name");
message.Target = GetJsonSimpleStringValue(textMsg, "Target");
}
return(message);
}
/// <summary>
/// Inits the specified source.
/// </summary>
/// <param name="source">The source.</param>
/// <param name="loggerName">Name of the logger.</param>
/// <param name="configXml">The config XML.</param>
public void Init(IGenericCommunicationService source, string loggerName, XElement configXml)
{
this.source = source;
this.log = source.Logger;
this.name = loggerName;
this.messageFormat = source.Serializer.MessageFormat;
if (dataStreamQueue == null)
{
UnboundedChannelOptions channelOptions = new UnboundedChannelOptions();
channelOptions.SingleReader = true;
channelOptions.SingleWriter = false;
dataStreamQueue = Channel.CreateUnbounded <StreamLogItem>(channelOptions);
queueWriter = dataStreamQueue.Writer;
}
else
{
throw new InvalidOperationException("Logger \"" + name + "\" already initialized!");
}
// load config
XElement targetDirElement;
if (configXml != null &&
(targetDirElement = configXml.Element("TargetDir")) != null)
{
this.TargetDir = targetDirElement.Value;
}
else
{
if (!string.IsNullOrEmpty(this.TargetDir))
{
this.TargetDir = Path.GetFullPath(this.TargetDir);
}
else
{
this.TargetDir = Environment.CurrentDirectory;
}
}
XElement keepStreamLogsDaysElement;
if (configXml != null &&
(keepStreamLogsDaysElement = configXml.Element("KeepStreamLogsDays")) != null)
{
keepStreamLogsDays = int.Parse(keepStreamLogsDaysElement.Value);
}
// start logging
Task.Run(StoreLogItemsThread);
}
/// <summary>
/// Message format: 5, 70 , 10 + (2 Bytes message format - short) + (4 Bytes data length - int) + 3 + (message bytes) + 3
/// </summary>
/// <param name="messageFormat"></param>
/// <param name="data"></param>
/// <returns></returns>
public static byte[] CreateMessage(RawMessageFormat messageFormat, byte[] data)
{
byte[] target = new byte[data.Length + 11];
Array.Copy(StartMessageIdentifier, target, StartMessageIdentifier.Length);
int currentIndex = StartMessageIdentifier.Length;
// copy message type bytes
short msgType = (short)messageFormat;
byte[] msgTypeBytes = BitConverter.GetBytes(msgType);
if (msgTypeBytes.Length != 2)
{
throw new InvalidCastException("Unexpected message type bytes: " + msgTypeBytes.Length);
}
Array.Copy(msgTypeBytes, 0, target, currentIndex, msgTypeBytes.Length);
currentIndex += msgTypeBytes.Length;
// copy messgae length bytes
int dataLength = data.Length;
byte[] msgLengthBytes = BitConverter.GetBytes(dataLength);
if (msgLengthBytes.Length != 4)
{
throw new InvalidCastException("Unexpected message length bytes: " + msgLengthBytes.Length);
}
Array.Copy(msgLengthBytes, 0, target, currentIndex, msgLengthBytes.Length);
currentIndex += msgLengthBytes.Length;
// set 3 DataBorderControlByte separator
target[currentIndex] = DataBorderControlByte;
currentIndex++;
// copy data
Array.Copy(data, 0, target, currentIndex, data.Length);
currentIndex += data.Length;
// set 3 DataBorderControlByte end mark
target[currentIndex] = DataBorderControlByte;
return(target);
}
/// <summary>
/// Creates and initializes an instance of the class <c>StreamSession</c>.
/// </summary>
/// <param name="sessionId">The session id.</param>
/// <param name="sessionInfo">The session info.</param>
public StreamSession(int sessionId, string sessionInfo, TimeSpan createdAt, RawMessageFormat format)
{
this.SessionId = sessionId;
string descrPattern = "Description";
int descrIndex = sessionInfo.IndexOf(descrPattern);
if (descrIndex > 0)
{
string shortVersion = sessionInfo.Substring(descrIndex + descrPattern.Length + 1);
sessionInfo = sessionInfo.Substring(18, descrIndex - 18) + shortVersion;
}
this.SessionInfo = sessionInfo;
this.IncomingSyncCalls = new Dictionary <long, MethodInvokeRoundtrip>();
this.OutgoingSyncCalls = new Dictionary <long, MethodInvokeRoundtrip>();
this.FlowRates = new List <FlowRate>();
this.CreatedAt = createdAt;
this.Format = format;
}
/// <summary>
/// Reads the raw message.
/// </summary>
/// <param name="source">The source.</param>
/// <param name="startIndex">The start index.</param>
/// <param name="length">The length.</param>
/// <param name="unusedSharedMessage">The unused shared message.</param>
/// <param name="pendingMessage">The pending message.</param>
/// <returns></returns>
public IRawMessage ReadRawMessage(byte[] source, ref int startIndex, int length, IRawMessage unusedSharedMessage, ref IRawMessage pendingMessage)
{
try
{
if (pendingMessage != null)
{
if (pendingMessage.MessageFormat == RawMessageFormat.IncompleteControlDataSlice)
{
if (startIndex == 0)
{
// put separated control bytes in front
int startPartLength = pendingMessage.Length;
int targetLength = startPartLength + length;
byte[] targetData = new byte[targetLength];
pendingMessage.Data.CopyTo(targetData, 0);
Array.Copy(source, startIndex, targetData, startPartLength, length);
if (targetLength > StartMessageControlMinByteCount)
{
try
{
pendingMessage = null;
return(ReadRawMessage(targetData, ref startIndex, targetLength, unusedSharedMessage, ref pendingMessage));
}
finally
{
// reset start index to fit previous buffer
startIndex -= startPartLength;
}
}
}
else
{
// not expected -> corrupt data
Logger.Error(string.Format("Invalid raw data received! Expected start index = 0; actual: {0}", startIndex));
}
}
else
{
// append message data
int expectedRestBytes = pendingMessage.Data.Length - pendingMessage.Length;
expectedRestBytes = Math.Min(expectedRestBytes, length);
Array.Copy(source, startIndex, pendingMessage.Data, pendingMessage.Length, expectedRestBytes);
pendingMessage.Length += expectedRestBytes;
startIndex += expectedRestBytes;
if (pendingMessage.Data.Length == pendingMessage.Length)
{
// message data completed
var msg = pendingMessage;
pendingMessage = null;
// check data border byte
if (source[startIndex] != DataBorderControlByte)
{
ThrowDataEndMarkException("Append", source, startIndex, msg.Length);
}
return(msg);
}
}
}
else
{
int oldStartIndex = startIndex;
int index = startIndex;
if (index < length &&
(source[index] == StartMessageIdentifier1 ||
(index = Array.IndexOf(source, StartMessageIdentifier1, index)) >= 0))
{
if (index + StartMessageControlMinByteCount < length &&
source[index + 1] == StartMessageIdentifier[1] &&
source[index + 2] == StartMessageIdentifier[2])
{
// message start
index += 3;
short messageFormatShort = BitConverter.ToInt16(source, index);
RawMessageFormat msgFormat = (RawMessageFormat)messageFormatShort;
index += 2;
int dataLength = BitConverter.ToInt32(source, index);
byte[] data = new byte[dataLength];
index += 5;
int sourceDataLength = length - index;
sourceDataLength = Math.Min(sourceDataLength, dataLength);
Array.Copy(source, index, data, 0, sourceDataLength);
index += sourceDataLength;
startIndex = index;
if (dataLength > sourceDataLength ||
length == index) // received byte length ends exactly at payload end (separator byte still expected)
{
// first message part read
pendingMessage = unusedSharedMessage;
pendingMessage.MessageFormat = msgFormat;
pendingMessage.Data = data;
pendingMessage.Length = sourceDataLength;
}
else
{
// complete message
// check data border byte
if (source[startIndex] != DataBorderControlByte)
{
ThrowDataEndMarkException($"param length: {length}; data.Length: {data.Length}; source.Length: {source.Length}", source, startIndex, data.Length);
}
startIndex++;
pendingMessage = null;
var message = unusedSharedMessage;
// update message data
message.MessageFormat = msgFormat;
message.Data = data;
message.Length = dataLength;
return(message);
}
}
else
{
var restByteCount = length - index;
if (restByteCount > 0)
{
// incomplete encapsulation control data slice
pendingMessage = unusedSharedMessage;
pendingMessage.MessageFormat = RawMessageFormat.IncompleteControlDataSlice;
byte[] endRawControlData = new byte[restByteCount];
Array.Copy(source, index, endRawControlData, 0, restByteCount);
pendingMessage.Data = endRawControlData;
pendingMessage.Length = restByteCount;
}
}
}
}
}
catch (Exception ex)
{
Logger.Error(ex.ToString());
pendingMessage = null;
}
return(null);
}
public static byte[] CreateMessage(RawMessageFormat messageFormat, string stringData)
{
return(CreateMessage(messageFormat, System.Text.Encoding.UTF8.GetBytes(stringData)));
}
public IList <StreamSession> AnalyzeDataStreamSession(string fileName, TimeSpan?roundTripTimeFilter, int?flowRateFilter, Action <int> progressPercentage, FileExportFilter fileExort, out StringBuilder errors)
{
this.LastFilePath = fileName;
errors = new StringBuilder();
HashSet <int> ignoreSessions = new HashSet <int>();
Dictionary <string, StreamWriter> openFileExportStreams = null;
using (StreamReader streamReader = new StreamReader(fileName))
{
Dictionary <int, StreamSession> sessions = new Dictionary <int, StreamSession>();
StreamSession currentSession = null;
double totalFileSize = (double)streamReader.BaseStream.Length;
string line = null;
long lineNumber = 1;
while ((line = streamReader.ReadLine()) != null)
{
string[] parts = line.Split(FieldSeparator);
if (parts.Length > 4)
{
// message contains \t in data part
// merge last parts
string lastPart = string.Join("", parts, 3, parts.Length - 3);
parts = new string[] { parts[0], parts[1], parts[2], lastPart };
}
if (parts.Length == 4)
{
TimeSpan time;
int sessionId;
bool isReceive;
if (TryParseDataStreamLine(parts, out time, out sessionId, out isReceive))
{
string dataPart = parts[3];
if (dataPart.StartsWith("Session Created"))
{
var sessionInfoParts = dataPart.Split(';');
RawMessageFormat format = RawMessageFormat.JSON;
if (sessionInfoParts.Length >= 3)
{
string formatStr = sessionInfoParts[2];
string[] formatParts = formatStr.Split(':');
format = (RawMessageFormat)Enum.Parse(typeof(RawMessageFormat), formatParts[1].Trim());
}
currentSession = new StreamSession(sessionId, dataPart, time, format);
sessions.Add(sessionId, currentSession);
}
else if (dataPart.StartsWith("Session Terminated"))
{
currentSession.TerminatedAt = time;
}
else if (dataPart.StartsWith("[Dismiss invalid session message]"))
{
errors.AppendLine("Packet received without valid session!");
errors.AppendLine(line);
errors.AppendLine();
continue;
}
else if (!dataPart.StartsWith("[Logger Stopped]"))
{
if (currentSession == null || currentSession.SessionId != sessionId)
{
if (!sessions.TryGetValue(sessionId, out currentSession))
{
if (!ignoreSessions.Contains(sessionId))
{
errors.Append("Session ID ");
errors.Append(sessionId);
errors.AppendLine(" not found! All messages from this session are ignored!");
ignoreSessions.Add(sessionId);
}
continue;
}
}
IGenericMessage message = ParseMessage(dataPart, currentSession.Format);
// Flow rate
if (currentSession.PendingFlowRate == null)
{
currentSession.PendingFlowRate = new FlowRate(time.Hours, time.Minutes, time.Seconds);
currentSession.PendingFlowRate.StartLineNumber = lineNumber;
}
else if (currentSession.PendingFlowRate.Time.Hours != time.Hours ||
currentSession.PendingFlowRate.Time.Minutes != time.Minutes ||
currentSession.PendingFlowRate.Time.Seconds != time.Seconds)
{
// add pending flowrate and create new
if (!flowRateFilter.HasValue || flowRateFilter <= currentSession.PendingFlowRate.TotalCallCount)
{
currentSession.FlowRates.Add(currentSession.PendingFlowRate);
}
currentSession.PendingFlowRate = new FlowRate(time.Hours, time.Minutes, time.Seconds);
currentSession.PendingFlowRate.StartLineNumber = lineNumber;
}
var pendFlowRate = currentSession.PendingFlowRate;
pendFlowRate.TotalCallCount++;
pendFlowRate.PayloadByteCount += dataPart.Length;
// round trip determination
MethodInvokeRoundtrip methodInvokeReq = null;
MethodInvokeRoundtrip methodInvokeResp = null;
switch (message.Type)
{
case MessageType.AsyncMethodInvokeRequest:
if (isReceive)
{
currentSession.IncomingAsyncCallCount++;
pendFlowRate.IncomingAsyncCallCount++;
CheckReceiveRequestOrder(errors, currentSession, message);
}
else
{
currentSession.OutgoingAsyncCallCount++;
pendFlowRate.OutgoingAsyncCallCount++;
CheckSendRequestOrder(errors, currentSession, message);
}
break;
case MessageType.MethodInvokeRequest:
methodInvokeReq = new MethodInvokeRoundtrip();
methodInvokeReq.IsReceive = isReceive;
methodInvokeReq.Request = message;
methodInvokeReq.RequestTime = time;
if (isReceive)
{
currentSession.IncomingSyncCallCount++;
currentSession.IncomingSyncCalls.Add(message.RequestId, methodInvokeReq);
pendFlowRate.IncomingSyncCallCount++;
CheckReceiveRequestOrder(errors, currentSession, message);
}
else
{
currentSession.OutgoingSyncCallCount++;
currentSession.OutgoingSyncCalls.Add(message.RequestId, methodInvokeReq);
pendFlowRate.OutgoingSyncCallCount++;
CheckSendRequestOrder(errors, currentSession, message);
}
break;
case MessageType.MethodInvokeResponse:
if (isReceive)
{
if (currentSession.OutgoingSyncCalls.TryGetValue(message.RequestId, out methodInvokeResp))
{
methodInvokeResp.Response = message;
methodInvokeResp.ResponseTime = time;
if (roundTripTimeFilter.HasValue &&
methodInvokeResp.RoundTripTime < roundTripTimeFilter)
{
currentSession.OutgoingSyncCalls.Remove(message.RequestId);
}
}
else
{
errors.AppendLine("Incoming method invoke response could not be assigned to a request: ");
errors.AppendLine(line);
errors.AppendLine();
}
}
else
{
if (currentSession.IncomingSyncCalls.TryGetValue(message.RequestId, out methodInvokeResp))
{
methodInvokeResp.Response = message;
methodInvokeResp.ResponseTime = time;
if (roundTripTimeFilter.HasValue &&
methodInvokeResp.RoundTripTime < roundTripTimeFilter)
{
currentSession.IncomingSyncCalls.Remove(message.RequestId);
}
}
else
{
errors.AppendLine("Outgoing method invoke response could not be assigned to a incoming request: ");
errors.AppendLine(line);
errors.AppendLine();
}
}
break;
}
// handle file export
if (fileExort != null)
{
if (openFileExportStreams == null)
{
openFileExportStreams = new Dictionary <string, StreamWriter>();
}
if (fileExort.MessageNames.Where(mn => mn.Equals(message.Name)).Any())
{
bool isConditionMatching = true;
if (fileExort.Conditions != null)
{
foreach (var cond in fileExort.Conditions)
{
string condValue = GetJsonSimpleStringValue(dataPart, cond.Key);
if (cond.Value != condValue)
{
isConditionMatching = false;
break;
}
}
}
if (isConditionMatching)
{
string fileExportGroupValues = string.Concat(fileExort.GroupByKeys.Select(gk => GetJsonSimpleStringValue(dataPart, gk) + "_"));
//string fileExportGroupValue = GetJsonSimpleStringValue(dataPart, fileExort.GroupByKey);
StreamWriter currentExportWriter = null;
if (openFileExportStreams.TryGetValue(fileExportGroupValues, out currentExportWriter) == false)
{
if (Directory.Exists(fileExort.ExportDirectory) == false)
{
Directory.CreateDirectory(fileExort.ExportDirectory);
}
string conditions = string.Concat(fileExort.Conditions?.Select(cd => $"{cd.Key}-{cd.Value}_"));
string groupKeyValues = string.Concat(fileExort.GroupByKeys?.Select(gk => $"{gk}-{GetJsonSimpleStringValue(dataPart, gk)}_"));
string exportPath = Path.Combine(Path.GetFullPath(fileExort.ExportDirectory), $"{conditions}{groupKeyValues.TrimEnd('_')}.txt");
if (File.Exists(exportPath))
{
File.Delete(exportPath);
}
currentExportWriter = new StreamWriter(exportPath);
openFileExportStreams.Add(fileExportGroupValues, currentExportWriter);
}
if (fileExort.SeparateOnKey != null)
{
string separateOnKeyValue = GetJsonSimpleStringValue(dataPart, fileExort.SeparateOnKey);
if (separateOnKeyValue == fileExort.SeparateOnKeyValue)
{
currentExportWriter.WriteLine(Environment.NewLine);
currentExportWriter.WriteLine(Environment.NewLine);
}
}
ExportOnlyHandling(fileExort, line, isReceive, dataPart, currentExportWriter);
if (methodInvokeReq != null && methodInvokeReq.ExportFileStream == null)
{
methodInvokeReq.ExportFileStream = currentExportWriter;
}
}
}
else if (methodInvokeResp != null &&
methodInvokeResp.ExportFileStream != null)
{
// write response as well
ExportOnlyHandling(fileExort, line, isReceive, dataPart, methodInvokeResp.ExportFileStream);
methodInvokeResp.ExportFileStream = null;
}
}
}
}
else
{
errors.AppendLine("Invalid data row: " + line);
}
}
else
{
errors.AppendLine("Invalid data row! Could not be splitted into 4 parts: " + line);
}
lineNumber++;
if (progressPercentage != null &&
(lineNumber % 30000) == 0)
{
double dblPercentage = 100d / totalFileSize * (double)streamReader.BaseStream.Position;
progressPercentage((int)dblPercentage);
}
}
if (openFileExportStreams != null)
{
foreach (var fileStream in openFileExportStreams.Values)
{
fileStream.Close();
}
}
return(sessions.Values.ToList <StreamSession>());
}
}
