/// <summary>
/// Takes one TCP packet as a byte array, filters it based on source/destination ports, and stores it for stream reassembly.
/// </summary>
/// <param name="buffer"></param>
public unsafe void FilterAndStoreData(byte[] buffer)
{
// There is one TCP packet per buffer.
if (buffer?.Length < sizeof(TCPHeader))
{
Trace.WriteLine("TCPDecoder: Buffer length smaller than TCP header: Length=[" + (buffer?.Length ?? 0).ToString() + "].", "DEBUG-MACHINA");
return;
}
fixed(byte *ptr = buffer)
{
TCPHeader header = *(TCPHeader *)(ptr);
if (_sourcePort != header.source_port ||
_destinationPort != header.destination_port)
{
return;
}
// if there is no data, we can discard the packet - this will likely be an ACK, but it shouldnt affect stream processing.
if (buffer.Length - header.DataOffset == 0)
{
if ((header.flags & (byte)TCPFlags.SYN) == 0)
{
return;
}
}
//TEMP debugging
//Trace.WriteLine("TCPDecoder: TCP Sequence # " + header.SequenceNumber.ToString() + " received with " + (buffer.Length - header.DataOffset).ToString() + " bytes.");
Packets.Add(buffer);
}
}
/// <summary>
/// Reassembles the TCP stream from available packets.
/// </summary>
/// <returns>byte array containing a portion of the TCP stream payload</returns>
public unsafe byte[] GetNextTCPDatagram()
{
if (Packets.Count == 0)
{
return(null);
}
byte[] buffer = null;
List <byte[]> packets;
if (Packets.Count == 1)
{
packets = Packets;
}
else
{
Packets.Sort((x, y) => Utility.ntohl(BitConverter.ToUInt32(x, 4))
.CompareTo(Utility.ntohl(BitConverter.ToUInt32(y, 4))));
packets = Packets;
}
foreach (byte[] packet in packets)
{
fixed(byte *ptr = packet)
{
TCPHeader header = *(TCPHeader *)ptr;
// failsafe - if starting, or just reset, start with next available packet.
if (_NextSequence == 0)
{
_NextSequence = header.SequenceNumber;
}
if (header.SequenceNumber <= _NextSequence)
{
LastPacketTimestamp = DateTime.UtcNow;
if ((header.flags & (byte)TCPFlags.SYN) > 0)
{
// filter out only when difference between sequence numbers is ~10k
if (_NextSequence == 0 || _NextSequence == header.SequenceNumber)
{
_NextSequence = header.SequenceNumber + 1;
}
else if (Math.Abs(_NextSequence - header.SequenceNumber) > 100000)
{
Trace.WriteLine("TCPDecoder: Updating sequence number from SYN packet. Current Sequence: [" + _NextSequence.ToString() + ", sent sequence: [" + header.SequenceNumber.ToString() + "].", "DEBUG-MACHINA");
_NextSequence = header.SequenceNumber + 1;
}
else
{
Trace.WriteLine("TCPDecoder: Ignoring SYN packet new sequence number. Current Sequence: [" + _NextSequence.ToString() + ", sent sequence: [" + header.SequenceNumber.ToString() + "].", "DEBUG-MACHINA");
}
continue; // do not process SYN packet, but set next sequence #.
}
// if this is a retransmit, only include the portion of data that is not already processed
uint packetOffset = 0;
if (header.SequenceNumber < _NextSequence)
{
packetOffset = _NextSequence - header.SequenceNumber;
}
// do not process this packet if it was previously fully processed, or has no data.
if (packetOffset >= packet.Length - header.DataOffset)
{
// this packet will get removed once we exit the loop.
Trace.WriteLine("TCPDecoder: packet data already processed, expected sequence [" + _NextSequence.ToString() + "], received [" + header.SequenceNumber + "], size [" + (packet.Length - header.DataOffset) + "]. Data: " + Utility.ByteArrayToHexString(packet, 0, 50), "DEBUG-MACHINA");
continue;
}
if (buffer == null)
{
buffer = new byte[packet.Length - header.DataOffset - packetOffset];
Array.Copy(packet, header.DataOffset + packetOffset, buffer, 0, packet.Length - header.DataOffset - packetOffset);
}
else
{
int oldSize = buffer.Length;
Array.Resize(ref buffer, buffer.Length + (packet.Length - header.DataOffset - (int)packetOffset));
Array.Copy(packet, header.DataOffset + packetOffset, buffer, oldSize, (packet.Length - header.DataOffset - packetOffset));
}
// NOTE: do not need to correct for packetOffset here.
_NextSequence = header.SequenceNumber + (uint)packet.Length - header.DataOffset;
// if PUSH flag is set, return data immedately.
// Note: data in the TCP stream can be processed without the PSH flag set, the application must interpret the stream data.
if ((header.flags & (byte)TCPFlags.PSH) > 0)
{
break;
}
}
else if (header.SequenceNumber > _NextSequence)
{
break;// if the current sequence # is after the last processed packet, stop processing - missing data. May need recovery in the future.
}
}
}
// remove any earlier packets from the primary array
for (int i = Packets.Count - 1; i >= 0; i--)
{
if (Utility.ntohl(BitConverter.ToUInt32(Packets[i], 4)) < _NextSequence)
{
Packets.RemoveAt(i);
}
}
if (Packets.Count > 0)
{
if (LastPacketTimestamp.AddMilliseconds(2000) < DateTime.UtcNow)
{
Trace.WriteLine("TCPDecoder: >2 sec since last processed packet, resetting stream.", "DEBUG-MACHINA");
for (int i = Packets.Count - 1; i >= 0; i--)
{
// todo: need to explore this logic, can we recover and get next highest sequence?
Trace.WriteLine("TCPDecoder: Missing Sequence # [" + _NextSequence.ToString() + "], Dropping packet with sequence # [" +
Utility.ntohl(BitConverter.ToUInt32(Packets[i], 4)).ToString() + "].", "DEBUG-MACHINA");
Packets.RemoveAt(i);
}
_NextSequence = 0;
}
}
return(buffer);
}
private void ParseNetworkData(SocketObject asyncState, byte[] byteData, int nReceived)
{
if (byteData == null || byteData[9] != 6)
{
return;
}
var startIndex = (byte)((byteData[0] & 15) * 4);
var lengthCheck = IPAddress.NetworkToHostOrder(BitConverter.ToInt16(byteData, 2));
if (nReceived < lengthCheck || startIndex > lengthCheck)
{
return;
}
var IP = new IPHeader(byteData, nReceived);
var TCP = new TCPHeader(byteData, nReceived);
var serverConnection = new ServerConnection
{
SourceAddress = (uint)BitConverter.ToInt32(byteData, 12),
DestinationAddress = (uint)BitConverter.ToInt32(byteData, 16),
SourcePort = (ushort)BitConverter.ToInt16(byteData, startIndex),
DestinationPort = (ushort)BitConverter.ToInt16(byteData, startIndex + 2),
TimeStamp = DateTime.Now
/*
* // these don't return the right ports for some reason
* DestinationAddress = BitConverter.ToUInt32(IP.DestinationAddress.GetAddressBytes(), 0),
* DestinationPort = Convert.ToUInt16(TCP.DestinationPort),
* SourcePort = Convert.ToUInt16(TCP.SourcePort),
* SourceAddress = BitConverter.ToUInt32(IP.SourceAddress.GetAddressBytes(), 0),
* TimeStamp = DateTime.Now
*/
};
lock (Lock)
{
var found = Enumerable.Contains(ServerConnections, serverConnection);
if (!found)
{
if (Enumerable.Contains(DroppedConnections, serverConnection))
{
return;
}
UpdateConnectionList();
if (!Enumerable.Contains(ServerConnections, serverConnection))
{
DroppedConnections.Add(serverConnection);
return;
}
}
}
if (startIndex + 12 > nReceived)
{
return;
}
var nextTCPSequence = (uint)IPAddress.NetworkToHostOrder(BitConverter.ToInt32(byteData, startIndex + 4));
var cut = (byte)(((byteData[startIndex + 12] & 240) >> 4) * 4);
var length = nReceived - startIndex - cut;
if (length < 0 || length > 0x10000)
{
return;
}
if (lengthCheck == startIndex + cut)
{
return;
}
lock (asyncState.SocketLock)
{
var connection = asyncState.Connections.FirstOrDefault(x => x.Equals(serverConnection));
if (connection == null)
{
connection = new NetworkConnection
{
SourceAddress = serverConnection.SourceAddress,
SourcePort = serverConnection.SourcePort,
DestinationAddress = serverConnection.DestinationAddress,
DestinationPort = serverConnection.DestinationPort
};
asyncState.Connections.Add(connection);
}
if (length == 0)
{
return;
}
var destinationBuffer = new byte[length];
Array.Copy(byteData, startIndex + cut, destinationBuffer, 0, length);
if (connection.StalePackets.ContainsKey(nextTCPSequence))
{
connection.StalePackets.Remove(nextTCPSequence);
}
var packet = new NetworkPacket
{
TCPSequence = nextTCPSequence,
Buffer = destinationBuffer,
Push = (byteData[startIndex + 13] & 8) != 0
};
connection.StalePackets.Add(nextTCPSequence, packet);
if (!connection.NextTCPSequence.HasValue)
{
connection.NextTCPSequence = nextTCPSequence;
}
if (connection.StalePackets.Count == 1)
{
connection.LastGoodNetworkPacketTime = DateTime.Now;
}
if (!connection.StalePackets.Any(x => x.Key <= connection.NextTCPSequence.Value))
{
if (DateTime.Now.Subtract(connection.LastGoodNetworkPacketTime)
.TotalSeconds <= 10.0)
{
return;
}
connection.NextTCPSequence = connection.StalePackets.Min(x => x.Key);
}
while (connection.StalePackets.Any(x => x.Key <= connection.NextTCPSequence.Value))
{
NetworkPacket stalePacket;
uint sequenceLength = 0;
if (connection.StalePackets.ContainsKey(connection.NextTCPSequence.Value))
{
stalePacket = connection.StalePackets[connection.NextTCPSequence.Value];
}
else
{
stalePacket = connection.StalePackets.Where(x => x.Key <= connection.NextTCPSequence.Value)
.OrderBy(x => x.Key)
.FirstOrDefault()
.Value;
sequenceLength = connection.NextTCPSequence.Value - stalePacket.TCPSequence;
}
connection.StalePackets.Remove(stalePacket.TCPSequence);
if (connection.NetworkBufferPosition == 0)
{
connection.LastNetworkBufferUpdate = DateTime.Now;
}
if (sequenceLength >= stalePacket.Buffer.Length)
{
continue;
}
connection.NextTCPSequence = stalePacket.TCPSequence + (uint)stalePacket.Buffer.Length;
Array.Copy(stalePacket.Buffer, sequenceLength, connection.NetworkBuffer, connection.NetworkBufferPosition, stalePacket.Buffer.Length - sequenceLength);
connection.NetworkBufferPosition += stalePacket.Buffer.Length - (int)sequenceLength;
if (stalePacket.Push)
{
ProcessNetworkBuffer(connection);
}
}
}
}
