/** * Reads the next token from the stream and returns it. * * @param BinaryStream Stream used to serialize from * @param InNetworkStream Network stream this token belongs to * @return Token serialized */ public static TokenBase ReadNextToken(BinaryReader BinaryStream, NetworkStream InNetworkStream) { TokenBase SerializedToken = null; ETokenTypes TokenType = (ETokenTypes)BinaryStream.ReadByte(); // Handle token specific serialization. switch (TokenType) { case ETokenTypes.FrameMarker: SerializedToken = new TokenFrameMarker(BinaryStream); break; case ETokenTypes.SocketSendTo: SerializedToken = new TokenSocketSendTo(BinaryStream); break; case ETokenTypes.SendBunch: SerializedToken = new TokenSendBunch(BinaryStream); break; case ETokenTypes.SendRPC: SerializedToken = new TokenSendRPC(BinaryStream); break; case ETokenTypes.ReplicateActor: SerializedToken = new TokenReplicateActor(BinaryStream); break; case ETokenTypes.ReplicateProperty: SerializedToken = new TokenReplicateProperty(BinaryStream); break; case ETokenTypes.EndOfStreamMarker: SerializedToken = new TokenEndOfStreamMarker(); break; case ETokenTypes.Event: SerializedToken = new TokenEvent(BinaryStream); break; case ETokenTypes.RawSocketData: SerializedToken = new TokenRawSocketData(BinaryStream); break; default: throw new InvalidDataException(); } TokenTypeStats[(int)TokenType]++; SerializedToken.NetworkStream = InNetworkStream; SerializedToken.TokenType = TokenType; return(SerializedToken); }
/** * Reads the next token from the stream and returns it. * * @param BinaryStream Stream used to serialize from * @param InNetworkStream Network stream this token belongs to * @return Token serialized */ public static TokenBase ReadNextToken(BinaryReader BinaryStream, NetworkStream InNetworkStream) { TokenBase SerializedToken = null; ETokenTypes TokenType = (ETokenTypes)BinaryStream.ReadByte(); // Handle token specific serialization. switch (TokenType) { case ETokenTypes.FrameMarker: SerializedToken = new TokenFrameMarker(BinaryStream); break; case ETokenTypes.SocketSendTo: SerializedToken = new TokenSocketSendTo(BinaryStream); break; case ETokenTypes.SendBunch: SerializedToken = new TokenSendBunch(BinaryStream); break; case ETokenTypes.SendRPC: SerializedToken = new TokenSendRPC(BinaryStream); break; case ETokenTypes.ReplicateActor: SerializedToken = new TokenReplicateActor(BinaryStream); break; case ETokenTypes.ReplicateProperty: SerializedToken = new TokenReplicateProperty(BinaryStream); break; case ETokenTypes.EndOfStreamMarker: SerializedToken = new TokenEndOfStreamMarker(); break; case ETokenTypes.Event: SerializedToken = new TokenEvent(BinaryStream); break; case ETokenTypes.RawSocketData: SerializedToken = new TokenRawSocketData(BinaryStream); break; case ETokenTypes.SendAck: SerializedToken = new TokenSendAck(BinaryStream); break; case ETokenTypes.WritePropertyHeader: SerializedToken = new TokenWritePropertyHeader(BinaryStream); break; case ETokenTypes.ExportBunch: SerializedToken = new TokenExportBunch(BinaryStream); break; case ETokenTypes.MustBeMappedGuids: SerializedToken = new TokenMustBeMappedGuids(BinaryStream); break; case ETokenTypes.BeginContentBlock: SerializedToken = new TokenBeginContentBlock(BinaryStream); break; case ETokenTypes.EndContentBlock: SerializedToken = new TokenEndContentBlock(BinaryStream); break; case ETokenTypes.WritePropertyHandle: SerializedToken = new TokenWritePropertyHandle(BinaryStream); break; case ETokenTypes.NameReference: SerializedToken = new TokenNameReference(BinaryStream); break; case ETokenTypes.ConnectionReference: SerializedToken = new TokenConnectionReference(BinaryStream); break; case ETokenTypes.ConnectionChange: SerializedToken = new TokenConnectionChanged(BinaryStream); break; default: throw new InvalidDataException(); } TokenTypeStats[(int)TokenType]++; SerializedToken.NetworkStream = InNetworkStream; SerializedToken.TokenType = TokenType; SerializedToken.ConnectionIndex = InNetworkStream.CurrentConnectionIndex; return(SerializedToken); }
protected void UpdateSummary(TokenBase Token, FilterValues InFilterValues) { switch (Token.TokenType) { case ETokenTypes.FrameMarker: var TokenFrameMarker = (TokenFrameMarker)Token; if (StartTime < 0) { StartTime = TokenFrameMarker.RelativeTime; EndTime = TokenFrameMarker.RelativeTime; } else { EndTime = TokenFrameMarker.RelativeTime; } NumFrames++; break; case ETokenTypes.SocketSendTo: var TokenSocketSendTo = (TokenSocketSendTo)Token; // Unreal game socket if (TokenSocketSendTo.SocketNameIndex == NameIndexUnreal) { UnrealSocketCount++; UnrealSocketSize += TokenSocketSendTo.BytesSent; } else { OtherSocketCount++; OtherSocketSize += TokenSocketSendTo.BytesSent; } break; case ETokenTypes.SendBunch: var TokenSendBunch = (TokenSendBunch)Token; SendBunchCount++; SendBunchSizeBits += TokenSendBunch.GetNumTotalBits(); SendBunchHeaderSizeBits += TokenSendBunch.NumHeaderBits; int ChannelTypeIndex = TokenSendBunch.GetChannelTypeIndex(); if (SendBunchCountPerChannel.ContainsKey(ChannelTypeIndex)) { SendBunchCountPerChannel[ChannelTypeIndex]++; SendBunchSizeBitsPerChannel[ChannelTypeIndex] += TokenSendBunch.GetNumTotalBits(); SendBunchHeaderSizeBitsPerChannel[ChannelTypeIndex] += TokenSendBunch.NumHeaderBits; SendBunchPayloadSizeBitsPerChannel[ChannelTypeIndex] += TokenSendBunch.NumPayloadBits; } else { SendBunchCountPerChannel.Add(ChannelTypeIndex, 1); SendBunchSizeBitsPerChannel.Add(ChannelTypeIndex, TokenSendBunch.GetNumTotalBits()); SendBunchHeaderSizeBitsPerChannel.Add(ChannelTypeIndex, TokenSendBunch.NumHeaderBits); SendBunchPayloadSizeBitsPerChannel.Add(ChannelTypeIndex, TokenSendBunch.NumPayloadBits); } break; case ETokenTypes.SendRPC: var TokenSendRPC = (TokenSendRPC)Token; RPCCount++; RPCSizeBits += TokenSendRPC.GetNumTotalBits(); break; case ETokenTypes.ReplicateActor: var TokenReplicateActor = (TokenReplicateActor)Token; ActorCount++; if (TokenReplicateActor.Properties.Count > 0) { ReplicatedActorCount++; } ActorReplicateTimeInMS += TokenReplicateActor.TimeInMS; foreach (var Property in TokenReplicateActor.Properties) { if (Property.MatchesFilters(InFilterValues)) { PropertyCount++; ReplicatedSizeBits += Property.NumBits; } } foreach (var PropertyHeader in TokenReplicateActor.PropertyHeaders) { if (PropertyHeader.MatchesFilters(InFilterValues)) { ReplicatedSizeBits += PropertyHeader.NumBits; } } UniqueActors.AddItem(TokenReplicateActor, TokenReplicateActor.GetClassNameIndex()); break; case ETokenTypes.Event: NumEvents++; break; case ETokenTypes.RawSocketData: break; case ETokenTypes.SendAck: var TokenSendAck = (TokenSendAck)Token; SendAckCount++; SendAckSizeBits += TokenSendAck.NumBits; break; case ETokenTypes.ExportBunch: var TokenExportBunch = (TokenExportBunch)Token; ExportBunchCount++; ExportBunchSizeBits += TokenExportBunch.NumBits; break; case ETokenTypes.MustBeMappedGuids: var TokenMustBeMappedGuids = (TokenMustBeMappedGuids)Token; MustBeMappedGuidCount += TokenMustBeMappedGuids.NumGuids; MustBeMappedGuidSizeBits += TokenMustBeMappedGuids.NumBits; break; case ETokenTypes.BeginContentBlock: var TokenBeginContentBlock = (TokenBeginContentBlock)Token; ContentBlockHeaderCount++; ContentBlockHeaderSizeBits += TokenBeginContentBlock.NumBits; break; case ETokenTypes.EndContentBlock: var TokenEndContentBlock = (TokenEndContentBlock)Token; ContentBlockFooterCount++; ContentBlockFooterSizeBits += TokenEndContentBlock.NumBits; break; case ETokenTypes.WritePropertyHandle: var TokenWritePropertyHandle = (TokenWritePropertyHandle)Token; PropertyHandleCount++; PropertyHandleSizeBits += TokenWritePropertyHandle.NumBits; break; default: throw new System.IO.InvalidDataException(); } }
/** * Parses passed in data stream into a network stream container class * * @param ParserStream Raw data stream, needs to support seeking * @return NetworkStream data was parsed into */ public static NetworkStream Parse(Stream ParserStream) { var StartTime = DateTime.UtcNow; // Network stream the file is parsed into. NetworkStream NetworkStream = new NetworkStream(); // Serialize the header. This will also return an endian-appropriate binary reader to // be used for reading the data. BinaryReader BinaryStream = null; var Header = StreamHeader.ReadHeader(ParserStream, out BinaryStream); // Keep track of token stream offset as name table is at end of file. long TokenStreamOffset = ParserStream.Position; // Seek to name table and serialize it. ParserStream.Seek(Header.NameTableOffset, SeekOrigin.Begin); for (int NameIndex = 0; NameIndex < Header.NameTableEntries; NameIndex++) { UInt32 Length = BinaryStream.ReadUInt32(); NetworkStream.NameArray.Add(new string(BinaryStream.ReadChars((int)Length))); // Find "Unreal" name index used for misc socket parsing optimizations. if (NetworkStream.NameArray[NameIndex] == "Unreal") { NetworkStream.NameIndexUnreal = NameIndex; } } // Seek to beginning of token stream. ParserStream.Seek(TokenStreamOffset, SeekOrigin.Begin); // Scratch variables used for building stream. Required as we emit information in reverse // order needed for parsing. var CurrentFrameTokens = new List <TokenBase>(); TokenReplicateActor LastActorToken = null; List <TokenReplicateProperty> LastProperties = new List <TokenReplicateProperty>(); List <TokenWritePropertyHeader> LastPropertyHeaders = new List <TokenWritePropertyHeader>(); TokenFrameMarker LastFrameMarker = null; // Parse stream till we reach the end, marked by special token. bool bHasReachedEndOfStream = false; while (bHasReachedEndOfStream == false) { TokenBase Token = TokenBase.ReadNextToken(BinaryStream, NetworkStream); // Convert current tokens to frame if we reach a frame boundary or the end of the stream. if (((Token.TokenType == ETokenTypes.FrameMarker) || (Token.TokenType == ETokenTypes.EndOfStreamMarker)) // Nothing to do if we don't have any tokens, e.g. first frame. && (CurrentFrameTokens.Count > 0)) { // Figure out delta time of previous frame. Needed as partial network stream lacks relative // information for last frame. We assume 30Hz for last frame and for the first frame in case // we receive network traffic before the first frame marker. float DeltaTime = 1 / 30.0f; if (Token.TokenType == ETokenTypes.FrameMarker && LastFrameMarker != null) { DeltaTime = ((TokenFrameMarker)Token).RelativeTime - LastFrameMarker.RelativeTime; } // Create per frame partial stream and add it to the full stream. var FrameStream = new PartialNetworkStream(CurrentFrameTokens, NetworkStream.NameIndexUnreal, DeltaTime); NetworkStream.Frames.Add(FrameStream); CurrentFrameTokens.Clear(); Debug.Assert(LastProperties.Count == 0); // We shouldn't have any properties now Debug.Assert(LastPropertyHeaders.Count == 0); // We shouldn't have any property headers now either // Finish up actor summary of last pending actor before switching frames. HandleActorSummary(NetworkStream, LastActorToken); LastActorToken = null; } // Keep track of last frame marker. if (Token.TokenType == ETokenTypes.FrameMarker) { LastFrameMarker = (TokenFrameMarker)Token; } // Bail out if we hit the end. We already flushed tokens above. if (Token.TokenType == ETokenTypes.EndOfStreamMarker) { Debug.Assert(LastProperties.Count == 0); // We shouldn't have any properties now Debug.Assert(LastPropertyHeaders.Count == 0); // We shouldn't have any property headers now either bHasReachedEndOfStream = true; // Finish up actor summary of last pending actor at end of stream HandleActorSummary(NetworkStream, LastActorToken); } // Keep track of per frame tokens. else { // Keep track of last actor context for property replication. if (Token.TokenType == ETokenTypes.ReplicateActor) { // Encountered a new actor so we can finish up existing one for summary. FinishActorProperties(Token as TokenReplicateActor, LastProperties, LastPropertyHeaders); Debug.Assert(LastProperties.Count == 0); // We shouldn't have any properties now Debug.Assert(LastPropertyHeaders.Count == 0); // We shouldn't have any property headers now either HandleActorSummary(NetworkStream, LastActorToken); LastActorToken = Token as TokenReplicateActor; } // Keep track of RPC summary else if (Token.TokenType == ETokenTypes.SendRPC) { var TokenSendRPC = Token as TokenSendRPC; NetworkStream.UpdateSummary(ref NetworkStream.RPCNameToSummary, TokenSendRPC.FunctionNameIndex, TokenSendRPC.GetNumTotalBits(), 0.0f); } // Add properties to the actor token instead of network stream and keep track of summary. if (Token.TokenType == ETokenTypes.ReplicateProperty) { var TokenReplicateProperty = Token as TokenReplicateProperty; NetworkStream.UpdateSummary(ref NetworkStream.PropertyNameToSummary, TokenReplicateProperty.PropertyNameIndex, TokenReplicateProperty.NumBits, 0); //LastActorToken.Properties.Add(TokenReplicateProperty); LastProperties.Add(TokenReplicateProperty); } else if (Token.TokenType == ETokenTypes.WritePropertyHeader) { var TokenWritePropertyHeader = Token as TokenWritePropertyHeader; LastPropertyHeaders.Add(TokenWritePropertyHeader); } else { CurrentFrameTokens.Add(Token); } } } // Stats for profiling. double ParseTime = (DateTime.UtcNow - StartTime).TotalSeconds; Console.WriteLine("Parsing {0} MBytes in stream took {1} seconds", ParserStream.Length / 1024 / 1024, ParseTime); // Empty stream will have 0 frames and proper name table. Shouldn't happen as we only // write out stream in engine if there are any events. return(NetworkStream); }
/** * Parses passed in data stream into a network stream container class * * @param ParserStream Raw data stream, needs to support seeking * @return NetworkStream data was parsed into */ public static NetworkStream Parse(MainWindow InMainWindow, Stream ParserStream) { var StartTime = DateTime.UtcNow; // Network stream the file is parsed into. NetworkStream = new NetworkStream(); // Serialize the header. This will also return an endian-appropriate binary reader to // be used for reading the data. BinaryReader BinaryStream = null; var Header = StreamHeader.ReadHeader(ParserStream, out BinaryStream); // Scratch variables used for building stream. Required as we emit information in reverse // order needed for parsing. var CurrentFrameTokens = new List <TokenBase>(); TokenReplicateActor LastActorToken = null; List <TokenReplicateProperty> LastProperties = new List <TokenReplicateProperty>(); List <TokenWritePropertyHeader> LastPropertyHeaders = new List <TokenWritePropertyHeader>(); TokenFrameMarker LastFrameMarker = null; InMainWindow.ShowProgress(true); int Count = 0; var AllFrames = new PartialNetworkStream(NetworkStream.NameIndexUnreal, 1.0f / 30.0f); int EarlyOutMinutes = InMainWindow.GetMaxProfileMinutes(); // Parse stream till we reach the end, marked by special token. bool bHasReachedEndOfStream = false; List <TokenBase> TokenList = new List <TokenBase>(); float FrameStartTime = -1.0f; float FrameEndTime = -1.0f; while (bHasReachedEndOfStream == false) { if (Count++ % 1000 == 0) { float Percent = ( float )ParserStream.Position / ( float )ParserStream.Length; InMainWindow.UpdateProgress(( int )(Percent * 100)); } if (ParserStream.Position == ParserStream.Length) { // We reached stream early (must not have been finalized properly, but we can still read it) break; } TokenBase Token = null; try { Token = TokenBase.ReadNextToken(BinaryStream, NetworkStream); } catch (System.IO.EndOfStreamException) { // We reached stream early (must not have been finalized properly, but we can still read it) break; } if (Token.TokenType == ETokenTypes.NameReference) { NetworkStream.NameArray.Add((Token as TokenNameReference).Name); // Find "Unreal" name index used for misc socket parsing optimizations. if (NetworkStream.NameArray[NetworkStream.NameArray.Count - 1] == "Unreal") { NetworkStream.NameIndexUnreal = NetworkStream.NameArray.Count - 1; } continue; } if (Token.TokenType == ETokenTypes.ConnectionReference) { NetworkStream.AddressArray.Add((Token as TokenConnectionReference).Address); continue; } if (Token.TokenType == ETokenTypes.ConnectionChange) { // We need to setup CurrentConnectionIndex, since it's used in ReadNextToken NetworkStream.CurrentConnectionIndex = (Token as TokenConnectionChanged).AddressIndex; continue; } TokenList.Add(Token); // Track frame start/end times manually so we can bail out when we hit the amount of time we want to load if (Token.TokenType == ETokenTypes.FrameMarker) { var TokenFrameMarker = ( TokenFrameMarker )Token; if (FrameStartTime < 0) { FrameStartTime = TokenFrameMarker.RelativeTime; FrameEndTime = TokenFrameMarker.RelativeTime; } else { FrameEndTime = TokenFrameMarker.RelativeTime; } } if (EarlyOutMinutes > 0 && ((FrameEndTime - FrameStartTime) > 60 * EarlyOutMinutes)) { break; } } for (int i = 0; i < TokenList.Count; i++) { if (i % 1000 == 0) { float Percent = ( float )(i + 1) / ( float )(TokenList.Count); InMainWindow.UpdateProgress(( int )(Percent * 100)); } TokenBase Token = TokenList[i]; // Convert current tokens to frame if we reach a frame boundary or the end of the stream. if (((Token.TokenType == ETokenTypes.FrameMarker) || (Token.TokenType == ETokenTypes.EndOfStreamMarker)) // Nothing to do if we don't have any tokens, e.g. first frame. && (CurrentFrameTokens.Count > 0)) { // Figure out delta time of previous frame. Needed as partial network stream lacks relative // information for last frame. We assume 30Hz for last frame and for the first frame in case // we receive network traffic before the first frame marker. float DeltaTime = 1 / 30.0f; if (Token.TokenType == ETokenTypes.FrameMarker && LastFrameMarker != null) { DeltaTime = ((TokenFrameMarker)Token).RelativeTime - LastFrameMarker.RelativeTime; } // Create per frame partial stream and add it to the full stream. var FrameStream = new PartialNetworkStream(CurrentFrameTokens, NetworkStream.NameIndexUnreal, DeltaTime); AllFrames.AddStream(FrameStream); NetworkStream.Frames.Add(FrameStream); CurrentFrameTokens.Clear(); Debug.Assert(LastProperties.Count == 0); // We shouldn't have any properties now Debug.Assert(LastPropertyHeaders.Count == 0); // We shouldn't have any property headers now either // Finish up actor summary of last pending actor before switching frames. HandleActorSummary(NetworkStream, LastActorToken); LastActorToken = null; } // Keep track of last frame marker. if (Token.TokenType == ETokenTypes.FrameMarker) { LastFrameMarker = (TokenFrameMarker)Token; } // Bail out if we hit the end. We already flushed tokens above. if (Token.TokenType == ETokenTypes.EndOfStreamMarker) { Debug.Assert(LastProperties.Count == 0); // We shouldn't have any properties now Debug.Assert(LastPropertyHeaders.Count == 0); // We shouldn't have any property headers now either bHasReachedEndOfStream = true; // Finish up actor summary of last pending actor at end of stream HandleActorSummary(NetworkStream, LastActorToken); } // Keep track of per frame tokens. else { // Keep track of last actor context for property replication. if (Token.TokenType == ETokenTypes.ReplicateActor) { // Encountered a new actor so we can finish up existing one for summary. FinishActorProperties(Token as TokenReplicateActor, LastProperties, LastPropertyHeaders); Debug.Assert(LastProperties.Count == 0); // We shouldn't have any properties now Debug.Assert(LastPropertyHeaders.Count == 0); // We shouldn't have any property headers now either HandleActorSummary(NetworkStream, LastActorToken); LastActorToken = Token as TokenReplicateActor; } // Keep track of RPC summary else if (Token.TokenType == ETokenTypes.SendRPC) { var TokenSendRPC = Token as TokenSendRPC; NetworkStream.UpdateSummary(ref NetworkStream.RPCNameToSummary, TokenSendRPC.FunctionNameIndex, TokenSendRPC.GetNumTotalBits(), 0.0f); } // Add properties to the actor token instead of network stream and keep track of summary. if (Token.TokenType == ETokenTypes.ReplicateProperty) { var TokenReplicateProperty = Token as TokenReplicateProperty; NetworkStream.UpdateSummary(ref NetworkStream.PropertyNameToSummary, TokenReplicateProperty.PropertyNameIndex, TokenReplicateProperty.NumBits, 0); //LastActorToken.Properties.Add(TokenReplicateProperty); LastProperties.Add(TokenReplicateProperty); } else if (Token.TokenType == ETokenTypes.WritePropertyHeader) { var TokenWritePropertyHeader = Token as TokenWritePropertyHeader; LastPropertyHeaders.Add(TokenWritePropertyHeader); } else { CurrentFrameTokens.Add(Token); } } } InMainWindow.SetCurrentStreamSelection(NetworkStream, AllFrames, false); InMainWindow.ShowProgress(false); // Stats for profiling. double ParseTime = (DateTime.UtcNow - StartTime).TotalSeconds; Console.WriteLine("Parsing {0} MBytes in stream took {1} seconds", ParserStream.Length / 1024 / 1024, ParseTime); // Empty stream will have 0 frames and proper name table. Shouldn't happen as we only // write out stream in engine if there are any events. return(NetworkStream); }