/**
* Reads the next token from the stream and returns it.
*
* @param BinaryStream Stream used to serialize from
* @param InNetworkStream Network stream this token belongs to
* @return Token serialized
*/
public static TokenBase ReadNextToken(BinaryReader BinaryStream, NetworkStream InNetworkStream)
{
TokenBase SerializedToken = null;
ETokenTypes TokenType = (ETokenTypes)BinaryStream.ReadByte();
// Handle token specific serialization.
switch (TokenType)
{
case ETokenTypes.FrameMarker:
SerializedToken = new TokenFrameMarker(BinaryStream);
break;
case ETokenTypes.SocketSendTo:
SerializedToken = new TokenSocketSendTo(BinaryStream);
break;
case ETokenTypes.SendBunch:
SerializedToken = new TokenSendBunch(BinaryStream);
break;
case ETokenTypes.SendRPC:
SerializedToken = new TokenSendRPC(BinaryStream);
break;
case ETokenTypes.ReplicateActor:
SerializedToken = new TokenReplicateActor(BinaryStream);
break;
case ETokenTypes.ReplicateProperty:
SerializedToken = new TokenReplicateProperty(BinaryStream);
break;
case ETokenTypes.EndOfStreamMarker:
SerializedToken = new TokenEndOfStreamMarker();
break;
case ETokenTypes.Event:
SerializedToken = new TokenEvent(BinaryStream);
break;
case ETokenTypes.RawSocketData:
SerializedToken = new TokenRawSocketData(BinaryStream);
break;
default:
throw new InvalidDataException();
}
TokenTypeStats[(int)TokenType]++;
SerializedToken.NetworkStream = InNetworkStream;
SerializedToken.TokenType = TokenType;
return(SerializedToken);
}
/**
* Reads the next token from the stream and returns it.
*
* @param BinaryStream Stream used to serialize from
* @param InNetworkStream Network stream this token belongs to
* @return Token serialized
*/
public static TokenBase ReadNextToken(BinaryReader BinaryStream, NetworkStream InNetworkStream)
{
TokenBase SerializedToken = null;
ETokenTypes TokenType = (ETokenTypes)BinaryStream.ReadByte();
// Handle token specific serialization.
switch (TokenType)
{
case ETokenTypes.FrameMarker:
SerializedToken = new TokenFrameMarker(BinaryStream);
break;
case ETokenTypes.SocketSendTo:
SerializedToken = new TokenSocketSendTo(BinaryStream);
break;
case ETokenTypes.SendBunch:
SerializedToken = new TokenSendBunch(BinaryStream);
break;
case ETokenTypes.SendRPC:
SerializedToken = new TokenSendRPC(BinaryStream);
break;
case ETokenTypes.ReplicateActor:
SerializedToken = new TokenReplicateActor(BinaryStream);
break;
case ETokenTypes.ReplicateProperty:
SerializedToken = new TokenReplicateProperty(BinaryStream);
break;
case ETokenTypes.EndOfStreamMarker:
SerializedToken = new TokenEndOfStreamMarker();
break;
case ETokenTypes.Event:
SerializedToken = new TokenEvent(BinaryStream);
break;
case ETokenTypes.RawSocketData:
SerializedToken = new TokenRawSocketData(BinaryStream);
break;
case ETokenTypes.SendAck:
SerializedToken = new TokenSendAck(BinaryStream);
break;
case ETokenTypes.WritePropertyHeader:
SerializedToken = new TokenWritePropertyHeader(BinaryStream);
break;
case ETokenTypes.ExportBunch:
SerializedToken = new TokenExportBunch(BinaryStream);
break;
case ETokenTypes.MustBeMappedGuids:
SerializedToken = new TokenMustBeMappedGuids(BinaryStream);
break;
case ETokenTypes.BeginContentBlock:
SerializedToken = new TokenBeginContentBlock(BinaryStream);
break;
case ETokenTypes.EndContentBlock:
SerializedToken = new TokenEndContentBlock(BinaryStream);
break;
case ETokenTypes.WritePropertyHandle:
SerializedToken = new TokenWritePropertyHandle(BinaryStream);
break;
case ETokenTypes.NameReference:
SerializedToken = new TokenNameReference(BinaryStream);
break;
case ETokenTypes.ConnectionReference:
SerializedToken = new TokenConnectionReference(BinaryStream);
break;
case ETokenTypes.ConnectionChange:
SerializedToken = new TokenConnectionChanged(BinaryStream);
break;
default:
throw new InvalidDataException();
}
TokenTypeStats[(int)TokenType]++;
SerializedToken.NetworkStream = InNetworkStream;
SerializedToken.TokenType = TokenType;
SerializedToken.ConnectionIndex = InNetworkStream.CurrentConnectionIndex;
return(SerializedToken);
}
protected void UpdateSummary(TokenBase Token, FilterValues InFilterValues)
{
switch (Token.TokenType)
{
case ETokenTypes.FrameMarker:
var TokenFrameMarker = (TokenFrameMarker)Token;
if (StartTime < 0)
{
StartTime = TokenFrameMarker.RelativeTime;
EndTime = TokenFrameMarker.RelativeTime;
}
else
{
EndTime = TokenFrameMarker.RelativeTime;
}
NumFrames++;
break;
case ETokenTypes.SocketSendTo:
var TokenSocketSendTo = (TokenSocketSendTo)Token;
// Unreal game socket
if (TokenSocketSendTo.SocketNameIndex == NameIndexUnreal)
{
UnrealSocketCount++;
UnrealSocketSize += TokenSocketSendTo.BytesSent;
}
else
{
OtherSocketCount++;
OtherSocketSize += TokenSocketSendTo.BytesSent;
}
break;
case ETokenTypes.SendBunch:
var TokenSendBunch = (TokenSendBunch)Token;
SendBunchCount++;
SendBunchSizeBits += TokenSendBunch.GetNumTotalBits();
SendBunchHeaderSizeBits += TokenSendBunch.NumHeaderBits;
int ChannelTypeIndex = TokenSendBunch.GetChannelTypeIndex();
if (SendBunchCountPerChannel.ContainsKey(ChannelTypeIndex))
{
SendBunchCountPerChannel[ChannelTypeIndex]++;
SendBunchSizeBitsPerChannel[ChannelTypeIndex] += TokenSendBunch.GetNumTotalBits();
SendBunchHeaderSizeBitsPerChannel[ChannelTypeIndex] += TokenSendBunch.NumHeaderBits;
SendBunchPayloadSizeBitsPerChannel[ChannelTypeIndex] += TokenSendBunch.NumPayloadBits;
}
else
{
SendBunchCountPerChannel.Add(ChannelTypeIndex, 1);
SendBunchSizeBitsPerChannel.Add(ChannelTypeIndex, TokenSendBunch.GetNumTotalBits());
SendBunchHeaderSizeBitsPerChannel.Add(ChannelTypeIndex, TokenSendBunch.NumHeaderBits);
SendBunchPayloadSizeBitsPerChannel.Add(ChannelTypeIndex, TokenSendBunch.NumPayloadBits);
}
break;
case ETokenTypes.SendRPC:
var TokenSendRPC = (TokenSendRPC)Token;
RPCCount++;
RPCSizeBits += TokenSendRPC.GetNumTotalBits();
break;
case ETokenTypes.ReplicateActor:
var TokenReplicateActor = (TokenReplicateActor)Token;
ActorCount++;
if (TokenReplicateActor.Properties.Count > 0)
{
ReplicatedActorCount++;
}
ActorReplicateTimeInMS += TokenReplicateActor.TimeInMS;
foreach (var Property in TokenReplicateActor.Properties)
{
if (Property.MatchesFilters(InFilterValues))
{
PropertyCount++;
ReplicatedSizeBits += Property.NumBits;
}
}
foreach (var PropertyHeader in TokenReplicateActor.PropertyHeaders)
{
if (PropertyHeader.MatchesFilters(InFilterValues))
{
ReplicatedSizeBits += PropertyHeader.NumBits;
}
}
UniqueActors.AddItem(TokenReplicateActor, TokenReplicateActor.GetClassNameIndex());
break;
case ETokenTypes.Event:
NumEvents++;
break;
case ETokenTypes.RawSocketData:
break;
case ETokenTypes.SendAck:
var TokenSendAck = (TokenSendAck)Token;
SendAckCount++;
SendAckSizeBits += TokenSendAck.NumBits;
break;
case ETokenTypes.ExportBunch:
var TokenExportBunch = (TokenExportBunch)Token;
ExportBunchCount++;
ExportBunchSizeBits += TokenExportBunch.NumBits;
break;
case ETokenTypes.MustBeMappedGuids:
var TokenMustBeMappedGuids = (TokenMustBeMappedGuids)Token;
MustBeMappedGuidCount += TokenMustBeMappedGuids.NumGuids;
MustBeMappedGuidSizeBits += TokenMustBeMappedGuids.NumBits;
break;
case ETokenTypes.BeginContentBlock:
var TokenBeginContentBlock = (TokenBeginContentBlock)Token;
ContentBlockHeaderCount++;
ContentBlockHeaderSizeBits += TokenBeginContentBlock.NumBits;
break;
case ETokenTypes.EndContentBlock:
var TokenEndContentBlock = (TokenEndContentBlock)Token;
ContentBlockFooterCount++;
ContentBlockFooterSizeBits += TokenEndContentBlock.NumBits;
break;
case ETokenTypes.WritePropertyHandle:
var TokenWritePropertyHandle = (TokenWritePropertyHandle)Token;
PropertyHandleCount++;
PropertyHandleSizeBits += TokenWritePropertyHandle.NumBits;
break;
default:
throw new System.IO.InvalidDataException();
}
}
/**
* Parses passed in data stream into a network stream container class
*
* @param ParserStream Raw data stream, needs to support seeking
* @return NetworkStream data was parsed into
*/
public static NetworkStream Parse(Stream ParserStream)
{
var StartTime = DateTime.UtcNow;
// Network stream the file is parsed into.
NetworkStream NetworkStream = new NetworkStream();
// Serialize the header. This will also return an endian-appropriate binary reader to
// be used for reading the data.
BinaryReader BinaryStream = null;
var Header = StreamHeader.ReadHeader(ParserStream, out BinaryStream);
// Keep track of token stream offset as name table is at end of file.
long TokenStreamOffset = ParserStream.Position;
// Seek to name table and serialize it.
ParserStream.Seek(Header.NameTableOffset, SeekOrigin.Begin);
for (int NameIndex = 0; NameIndex < Header.NameTableEntries; NameIndex++)
{
UInt32 Length = BinaryStream.ReadUInt32();
NetworkStream.NameArray.Add(new string(BinaryStream.ReadChars((int)Length)));
// Find "Unreal" name index used for misc socket parsing optimizations.
if (NetworkStream.NameArray[NameIndex] == "Unreal")
{
NetworkStream.NameIndexUnreal = NameIndex;
}
}
// Seek to beginning of token stream.
ParserStream.Seek(TokenStreamOffset, SeekOrigin.Begin);
// Scratch variables used for building stream. Required as we emit information in reverse
// order needed for parsing.
var CurrentFrameTokens = new List <TokenBase>();
TokenReplicateActor LastActorToken = null;
List <TokenReplicateProperty> LastProperties = new List <TokenReplicateProperty>();
List <TokenWritePropertyHeader> LastPropertyHeaders = new List <TokenWritePropertyHeader>();
TokenFrameMarker LastFrameMarker = null;
// Parse stream till we reach the end, marked by special token.
bool bHasReachedEndOfStream = false;
while (bHasReachedEndOfStream == false)
{
TokenBase Token = TokenBase.ReadNextToken(BinaryStream, NetworkStream);
// Convert current tokens to frame if we reach a frame boundary or the end of the stream.
if (((Token.TokenType == ETokenTypes.FrameMarker) || (Token.TokenType == ETokenTypes.EndOfStreamMarker))
// Nothing to do if we don't have any tokens, e.g. first frame.
&& (CurrentFrameTokens.Count > 0))
{
// Figure out delta time of previous frame. Needed as partial network stream lacks relative
// information for last frame. We assume 30Hz for last frame and for the first frame in case
// we receive network traffic before the first frame marker.
float DeltaTime = 1 / 30.0f;
if (Token.TokenType == ETokenTypes.FrameMarker && LastFrameMarker != null)
{
DeltaTime = ((TokenFrameMarker)Token).RelativeTime - LastFrameMarker.RelativeTime;
}
// Create per frame partial stream and add it to the full stream.
var FrameStream = new PartialNetworkStream(CurrentFrameTokens, NetworkStream.NameIndexUnreal, DeltaTime);
NetworkStream.Frames.Add(FrameStream);
CurrentFrameTokens.Clear();
Debug.Assert(LastProperties.Count == 0); // We shouldn't have any properties now
Debug.Assert(LastPropertyHeaders.Count == 0); // We shouldn't have any property headers now either
// Finish up actor summary of last pending actor before switching frames.
HandleActorSummary(NetworkStream, LastActorToken);
LastActorToken = null;
}
// Keep track of last frame marker.
if (Token.TokenType == ETokenTypes.FrameMarker)
{
LastFrameMarker = (TokenFrameMarker)Token;
}
// Bail out if we hit the end. We already flushed tokens above.
if (Token.TokenType == ETokenTypes.EndOfStreamMarker)
{
Debug.Assert(LastProperties.Count == 0); // We shouldn't have any properties now
Debug.Assert(LastPropertyHeaders.Count == 0); // We shouldn't have any property headers now either
bHasReachedEndOfStream = true;
// Finish up actor summary of last pending actor at end of stream
HandleActorSummary(NetworkStream, LastActorToken);
}
// Keep track of per frame tokens.
else
{
// Keep track of last actor context for property replication.
if (Token.TokenType == ETokenTypes.ReplicateActor)
{
// Encountered a new actor so we can finish up existing one for summary.
FinishActorProperties(Token as TokenReplicateActor, LastProperties, LastPropertyHeaders);
Debug.Assert(LastProperties.Count == 0); // We shouldn't have any properties now
Debug.Assert(LastPropertyHeaders.Count == 0); // We shouldn't have any property headers now either
HandleActorSummary(NetworkStream, LastActorToken);
LastActorToken = Token as TokenReplicateActor;
}
// Keep track of RPC summary
else if (Token.TokenType == ETokenTypes.SendRPC)
{
var TokenSendRPC = Token as TokenSendRPC;
NetworkStream.UpdateSummary(ref NetworkStream.RPCNameToSummary, TokenSendRPC.FunctionNameIndex, TokenSendRPC.GetNumTotalBits(), 0.0f);
}
// Add properties to the actor token instead of network stream and keep track of summary.
if (Token.TokenType == ETokenTypes.ReplicateProperty)
{
var TokenReplicateProperty = Token as TokenReplicateProperty;
NetworkStream.UpdateSummary(ref NetworkStream.PropertyNameToSummary, TokenReplicateProperty.PropertyNameIndex, TokenReplicateProperty.NumBits, 0);
//LastActorToken.Properties.Add(TokenReplicateProperty);
LastProperties.Add(TokenReplicateProperty);
}
else if (Token.TokenType == ETokenTypes.WritePropertyHeader)
{
var TokenWritePropertyHeader = Token as TokenWritePropertyHeader;
LastPropertyHeaders.Add(TokenWritePropertyHeader);
}
else
{
CurrentFrameTokens.Add(Token);
}
}
}
// Stats for profiling.
double ParseTime = (DateTime.UtcNow - StartTime).TotalSeconds;
Console.WriteLine("Parsing {0} MBytes in stream took {1} seconds", ParserStream.Length / 1024 / 1024, ParseTime);
// Empty stream will have 0 frames and proper name table. Shouldn't happen as we only
// write out stream in engine if there are any events.
return(NetworkStream);
}
/**
* Parses passed in data stream into a network stream container class
*
* @param ParserStream Raw data stream, needs to support seeking
* @return NetworkStream data was parsed into
*/
public static NetworkStream Parse(MainWindow InMainWindow, Stream ParserStream)
{
var StartTime = DateTime.UtcNow;
// Network stream the file is parsed into.
NetworkStream = new NetworkStream();
// Serialize the header. This will also return an endian-appropriate binary reader to
// be used for reading the data.
BinaryReader BinaryStream = null;
var Header = StreamHeader.ReadHeader(ParserStream, out BinaryStream);
// Scratch variables used for building stream. Required as we emit information in reverse
// order needed for parsing.
var CurrentFrameTokens = new List <TokenBase>();
TokenReplicateActor LastActorToken = null;
List <TokenReplicateProperty> LastProperties = new List <TokenReplicateProperty>();
List <TokenWritePropertyHeader> LastPropertyHeaders = new List <TokenWritePropertyHeader>();
TokenFrameMarker LastFrameMarker = null;
InMainWindow.ShowProgress(true);
int Count = 0;
var AllFrames = new PartialNetworkStream(NetworkStream.NameIndexUnreal, 1.0f / 30.0f);
int EarlyOutMinutes = InMainWindow.GetMaxProfileMinutes();
// Parse stream till we reach the end, marked by special token.
bool bHasReachedEndOfStream = false;
List <TokenBase> TokenList = new List <TokenBase>();
float FrameStartTime = -1.0f;
float FrameEndTime = -1.0f;
while (bHasReachedEndOfStream == false)
{
if (Count++ % 1000 == 0)
{
float Percent = ( float )ParserStream.Position / ( float )ParserStream.Length;
InMainWindow.UpdateProgress(( int )(Percent * 100));
}
if (ParserStream.Position == ParserStream.Length)
{
// We reached stream early (must not have been finalized properly, but we can still read it)
break;
}
TokenBase Token = null;
try
{
Token = TokenBase.ReadNextToken(BinaryStream, NetworkStream);
}
catch (System.IO.EndOfStreamException)
{
// We reached stream early (must not have been finalized properly, but we can still read it)
break;
}
if (Token.TokenType == ETokenTypes.NameReference)
{
NetworkStream.NameArray.Add((Token as TokenNameReference).Name);
// Find "Unreal" name index used for misc socket parsing optimizations.
if (NetworkStream.NameArray[NetworkStream.NameArray.Count - 1] == "Unreal")
{
NetworkStream.NameIndexUnreal = NetworkStream.NameArray.Count - 1;
}
continue;
}
if (Token.TokenType == ETokenTypes.ConnectionReference)
{
NetworkStream.AddressArray.Add((Token as TokenConnectionReference).Address);
continue;
}
if (Token.TokenType == ETokenTypes.ConnectionChange)
{
// We need to setup CurrentConnectionIndex, since it's used in ReadNextToken
NetworkStream.CurrentConnectionIndex = (Token as TokenConnectionChanged).AddressIndex;
continue;
}
TokenList.Add(Token);
// Track frame start/end times manually so we can bail out when we hit the amount of time we want to load
if (Token.TokenType == ETokenTypes.FrameMarker)
{
var TokenFrameMarker = ( TokenFrameMarker )Token;
if (FrameStartTime < 0)
{
FrameStartTime = TokenFrameMarker.RelativeTime;
FrameEndTime = TokenFrameMarker.RelativeTime;
}
else
{
FrameEndTime = TokenFrameMarker.RelativeTime;
}
}
if (EarlyOutMinutes > 0 && ((FrameEndTime - FrameStartTime) > 60 * EarlyOutMinutes))
{
break;
}
}
for (int i = 0; i < TokenList.Count; i++)
{
if (i % 1000 == 0)
{
float Percent = ( float )(i + 1) / ( float )(TokenList.Count);
InMainWindow.UpdateProgress(( int )(Percent * 100));
}
TokenBase Token = TokenList[i];
// Convert current tokens to frame if we reach a frame boundary or the end of the stream.
if (((Token.TokenType == ETokenTypes.FrameMarker) || (Token.TokenType == ETokenTypes.EndOfStreamMarker))
// Nothing to do if we don't have any tokens, e.g. first frame.
&& (CurrentFrameTokens.Count > 0))
{
// Figure out delta time of previous frame. Needed as partial network stream lacks relative
// information for last frame. We assume 30Hz for last frame and for the first frame in case
// we receive network traffic before the first frame marker.
float DeltaTime = 1 / 30.0f;
if (Token.TokenType == ETokenTypes.FrameMarker && LastFrameMarker != null)
{
DeltaTime = ((TokenFrameMarker)Token).RelativeTime - LastFrameMarker.RelativeTime;
}
// Create per frame partial stream and add it to the full stream.
var FrameStream = new PartialNetworkStream(CurrentFrameTokens, NetworkStream.NameIndexUnreal, DeltaTime);
AllFrames.AddStream(FrameStream);
NetworkStream.Frames.Add(FrameStream);
CurrentFrameTokens.Clear();
Debug.Assert(LastProperties.Count == 0); // We shouldn't have any properties now
Debug.Assert(LastPropertyHeaders.Count == 0); // We shouldn't have any property headers now either
// Finish up actor summary of last pending actor before switching frames.
HandleActorSummary(NetworkStream, LastActorToken);
LastActorToken = null;
}
// Keep track of last frame marker.
if (Token.TokenType == ETokenTypes.FrameMarker)
{
LastFrameMarker = (TokenFrameMarker)Token;
}
// Bail out if we hit the end. We already flushed tokens above.
if (Token.TokenType == ETokenTypes.EndOfStreamMarker)
{
Debug.Assert(LastProperties.Count == 0); // We shouldn't have any properties now
Debug.Assert(LastPropertyHeaders.Count == 0); // We shouldn't have any property headers now either
bHasReachedEndOfStream = true;
// Finish up actor summary of last pending actor at end of stream
HandleActorSummary(NetworkStream, LastActorToken);
}
// Keep track of per frame tokens.
else
{
// Keep track of last actor context for property replication.
if (Token.TokenType == ETokenTypes.ReplicateActor)
{
// Encountered a new actor so we can finish up existing one for summary.
FinishActorProperties(Token as TokenReplicateActor, LastProperties, LastPropertyHeaders);
Debug.Assert(LastProperties.Count == 0); // We shouldn't have any properties now
Debug.Assert(LastPropertyHeaders.Count == 0); // We shouldn't have any property headers now either
HandleActorSummary(NetworkStream, LastActorToken);
LastActorToken = Token as TokenReplicateActor;
}
// Keep track of RPC summary
else if (Token.TokenType == ETokenTypes.SendRPC)
{
var TokenSendRPC = Token as TokenSendRPC;
NetworkStream.UpdateSummary(ref NetworkStream.RPCNameToSummary, TokenSendRPC.FunctionNameIndex, TokenSendRPC.GetNumTotalBits(), 0.0f);
}
// Add properties to the actor token instead of network stream and keep track of summary.
if (Token.TokenType == ETokenTypes.ReplicateProperty)
{
var TokenReplicateProperty = Token as TokenReplicateProperty;
NetworkStream.UpdateSummary(ref NetworkStream.PropertyNameToSummary, TokenReplicateProperty.PropertyNameIndex, TokenReplicateProperty.NumBits, 0);
//LastActorToken.Properties.Add(TokenReplicateProperty);
LastProperties.Add(TokenReplicateProperty);
}
else if (Token.TokenType == ETokenTypes.WritePropertyHeader)
{
var TokenWritePropertyHeader = Token as TokenWritePropertyHeader;
LastPropertyHeaders.Add(TokenWritePropertyHeader);
}
else
{
CurrentFrameTokens.Add(Token);
}
}
}
InMainWindow.SetCurrentStreamSelection(NetworkStream, AllFrames, false);
InMainWindow.ShowProgress(false);
// Stats for profiling.
double ParseTime = (DateTime.UtcNow - StartTime).TotalSeconds;
Console.WriteLine("Parsing {0} MBytes in stream took {1} seconds", ParserStream.Length / 1024 / 1024, ParseTime);
// Empty stream will have 0 frames and proper name table. Shouldn't happen as we only
// write out stream in engine if there are any events.
return(NetworkStream);
}
