private static void Main(string[] args)
{
_rng = new Random(34324);
int threadCount = 8;
_maxBlocks = 200;
ThreadPool.SetMinThreads(threadCount + 2, 4); // Kludge to prevent slow thread startup.
_numBlocks = _maxBlocks;
var stopwatch = Stopwatch.StartNew();
var processor = new ParallelWorkProcessor <byte[]>(read, process, write, threadCount);
processor.WaitForFinished(Timeout.Infinite);
Console.WriteLine("\n\nFinished in " + stopwatch.Elapsed + "\n\n");
}
private void ProcessFileImpl()
{
if (_compression != FileCompression.None)
{
_tempName = Decompress();
}
switch (_dumpFormat)
{
case DumpFormatType.StatisticsPreParse:
{
var packets = ReadPackets();
if (packets.Count == 0)
{
break;
}
var firstPacket = packets.First();
var lastPacket = packets.Last();
// CSV format
// ReSharper disable once UseStringInterpolation
Trace.WriteLine(string.Format("{0};{1};{2};{3};{4};{5};{6};{7};{8}",
FileName, // - sniff file name
firstPacket.Time, // - time of first packet
lastPacket.Time, // - time of last packet
(lastPacket.Time - firstPacket.Time).TotalSeconds, // - sniff duration (seconds)
packets.Count, // - packet count
packets.AsParallel().Sum(packet => packet.Length), // - total packets size (bytes)
packets.AsParallel().Average(packet => packet.Length), // - average packet size (bytes)
packets.AsParallel().Min(packet => packet.Length), // - smaller packet size (bytes)
packets.AsParallel().Max(packet => packet.Length))); // - larger packet size (bytes)
break;
}
case DumpFormatType.SniffDataOnly:
case DumpFormatType.SqlOnly:
case DumpFormatType.Text:
case DumpFormatType.HexOnly:
{
var outFileName = Path.ChangeExtension(FileName, null) + "_parsed.txt";
if (Utilities.FileIsInUse(outFileName) && Settings.DumpFormat != DumpFormatType.SqlOnly)
{
// If our dump format requires a .txt to be created,
// check if we can write to that .txt before starting parsing
Trace.WriteLine($"Save file {outFileName} is in use, parsing will not be done.");
break;
}
Store.Store.SQLEnabledFlags = Settings.SQLOutputFlag;
File.Delete(outFileName);
_stats.SetStartTime(DateTime.Now);
var threadCount = Settings.Threads;
if (threadCount == 0)
{
threadCount = Environment.ProcessorCount;
}
ThreadPool.SetMinThreads(threadCount + 2, 4);
var written = false;
using (var writer = (Settings.DumpFormatWithText() ? new StreamWriter(outFileName, true) : null))
{
var firstRead = true;
var firstWrite = true;
var reader = _compression != FileCompression.None ? new Reader(_tempName, _sniffType) : new Reader(FileName, _sniffType);
var pwp = new ParallelWorkProcessor <Packet>(() => // read
{
if (!reader.PacketReader.CanRead())
{
return(Tuple.Create <Packet, bool>(null, true));
}
Packet packet;
var b = reader.TryRead(out packet);
if (firstRead)
{
Trace.WriteLine(
$"{_logPrefix}: Parsing {Utilities.BytesToString(reader.PacketReader.GetTotalSize())} of packets. Detected version {ClientVersion.VersionString}");
firstRead = false;
}
return(Tuple.Create(packet, b));
}, packet => // parse
{
// Parse the packet, adding text to Writer and stuff to the stores
if (packet.Direction == Direction.BNClientToServer ||
packet.Direction == Direction.BNServerToClient)
{
BattlenetHandler.ParseBattlenet(packet);
}
else
{
Handler.Parse(packet);
}
// Update statistics
_stats.AddByStatus(packet.Status);
return(packet);
},
packet => // write
{
if (!Console.IsOutputRedirected)
{
ShowPercentProgress("Processing...", reader.PacketReader.GetCurrentSize(), reader.PacketReader.GetTotalSize());
}
if (!packet.Status.HasAnyFlag(Settings.OutputFlag) || !packet.WriteToFile)
{
packet.ClosePacket();
return;
}
written = true;
if (firstWrite)
{
// ReSharper disable AccessToDisposedClosure
writer?.WriteLine(GetHeader(FileName));
// ReSharper restore AccessToDisposedClosure
firstWrite = false;
}
// get packet header if necessary
if (Settings.LogPacketErrors)
{
switch (packet.Status)
{
case ParsedStatus.WithErrors:
_withErrorHeaders.Add(packet.GetHeader());
break;
case ParsedStatus.NotParsed:
_skippedHeaders.Add(packet.GetHeader());
break;
case ParsedStatus.NoStructure:
_noStructureHeaders.Add(packet.GetHeader());
break;
}
}
// ReSharper disable AccessToDisposedClosure
if (writer != null)
{
// Write to file
writer.WriteLine(packet.Writer);
writer.Flush();
}
// ReSharper restore AccessToDisposedClosure
// Close Writer, Stream - Dispose
packet.ClosePacket();
}, threadCount);
pwp.WaitForFinished(Timeout.Infinite);
reader.PacketReader.Dispose();
_stats.SetEndTime(DateTime.Now);
}
if (Settings.DumpFormatWithText())
{
if (written)
{
Trace.WriteLine($"{_logPrefix}: Saved file to '{outFileName}'");
}
else
{
Trace.WriteLine($"{_logPrefix}: No file produced");
File.Delete(outFileName);
}
}
Trace.WriteLine($"{_logPrefix}: {_stats}");
if (Settings.SQLOutputFlag != 0 || HotfixSettings.Instance.ShouldLog())
{
WriteSQLs();
}
if (Settings.LogPacketErrors)
{
WritePacketErrors();
}
GC.Collect(); // Force a GC collect after parsing a file. It seems to help.
break;
}
case DumpFormatType.Pkt:
{
var packets = ReadPackets();
if (packets.Count == 0)
{
break;
}
if (Settings.FilterPacketsNum < 0)
{
var packetsPerSplit = Math.Abs(Settings.FilterPacketsNum);
var totalPackets = packets.Count;
var numberOfSplits = (int)Math.Ceiling((double)totalPackets / packetsPerSplit);
for (var i = 0; i < numberOfSplits; ++i)
{
var fileNamePart = FileName + "_part_" + (i + 1) + ".pkt";
var packetsPart = packets.Take(packetsPerSplit).ToList();
packets.RemoveRange(0, packetsPart.Count);
BinaryDump(fileNamePart, packetsPart);
}
}
else
{
var fileNameExcerpt = Path.ChangeExtension(FileName, null) + "_excerpt.pkt";
BinaryDump(fileNameExcerpt, packets);
}
break;
}
case DumpFormatType.PktSplit:
{
var packets = ReadPackets();
if (packets.Count == 0)
{
break;
}
SplitBinaryDump(packets);
break;
}
case DumpFormatType.PktDirectionSplit:
{
var packets = ReadPackets();
if (packets.Count == 0)
{
break;
}
DirectionSplitBinaryDump(packets);
break;
}
case DumpFormatType.PktSessionSplit:
{
var packets = ReadPackets();
if (packets.Count == 0)
{
break;
}
SessionSplitBinaryDump(packets);
break;
}
case DumpFormatType.CompressSniff:
{
if (_compression != FileCompression.None)
{
Trace.WriteLine($"Skipped compressing file {FileName}");
break;
}
Compress();
break;
}
case DumpFormatType.SniffVersionSplit:
{
var reader = _compression != FileCompression.None ? new Reader(_tempName, _sniffType) : new Reader(FileName, _sniffType);
if (ClientVersion.IsUndefined() && reader.PacketReader.CanRead())
{
Packet packet;
reader.TryRead(out packet);
packet.ClosePacket();
}
reader.PacketReader.Dispose();
var version = ClientVersion.IsUndefined() ? "unknown" : ClientVersion.VersionString;
var realFileName = GetCompressedFileName();
var destPath = Path.Combine(Path.GetDirectoryName(realFileName), version,
Path.GetFileName(realFileName));
var destDir = Path.GetDirectoryName(destPath);
if (!Directory.Exists(destDir))
{
Directory.CreateDirectory(destDir);
}
File.Move(realFileName, destPath);
Trace.WriteLine("Moved " + realFileName + " to " + destPath);
break;
}
case DumpFormatType.ConnectionIndexes:
{
var packets = ReadPackets();
if (packets.Count == 0)
{
break;
}
using (var writer = new StreamWriter(Path.ChangeExtension(FileName, null) + "_connidx.txt"))
{
if (ClientVersion.Build <= ClientVersionBuild.V6_0_3_19342)
{
writer.WriteLine("# Warning: versions before 6.1 might not have proper ConnectionIndex values.");
}
IEnumerable <IGrouping <Tuple <int, Direction>, Packet> > groupsOpcode = packets
.GroupBy(packet => Tuple.Create(packet.Opcode, packet.Direction))
.OrderBy(grouping => grouping.Key.Item2);
foreach (var groupOpcode in groupsOpcode)
{
var groups = groupOpcode
.GroupBy(packet => packet.ConnectionIndex)
.OrderBy(grouping => grouping.Key)
.ToList();
writer.Write("{0} {1,-50}: ", groupOpcode.Key.Item2, Opcodes.GetOpcodeName(groupOpcode.Key.Item1, groupOpcode.Key.Item2));
for (var i = 0; i < groups.Count; i++)
{
var idx = groups[i].Key;
writer.Write("{0} ({1}{2})", idx, (idx & 1) != 0 ? "INSTANCE" : "REALM", (idx & 2) != 0 ? "_NEW" : "");
if (i != groups.Count - 1)
{
writer.Write(", ");
}
}
writer.WriteLine();
}
}
break;
}
case DumpFormatType.Fusion:
{
var packets = ReadPackets();
if (packets.Count == 0)
{
break;
}
FusionDump(packets);
break;
}
default:
{
Trace.WriteLine($"{_logPrefix}: Dump format is none, nothing will be processed.");
break;
}
}
}
private void ProcessFileImpl()
{
if (_compression != FileCompression.None)
_tempName = Decompress();
switch (_dumpFormat)
{
case DumpFormatType.StatisticsPreParse:
{
var packets = ReadPackets();
if (packets.Count == 0)
break;
var firstPacket = packets.First();
var lastPacket = packets.Last();
// CSV format
// ReSharper disable once UseStringInterpolation
Trace.WriteLine(string.Format("{0};{1};{2};{3};{4};{5};{6};{7};{8}",
FileName, // - sniff file name
firstPacket.Time, // - time of first packet
lastPacket.Time, // - time of last packet
(lastPacket.Time - firstPacket.Time).TotalSeconds, // - sniff duration (seconds)
packets.Count, // - packet count
packets.AsParallel().Sum(packet => packet.Length), // - total packets size (bytes)
packets.AsParallel().Average(packet => packet.Length), // - average packet size (bytes)
packets.AsParallel().Min(packet => packet.Length), // - smaller packet size (bytes)
packets.AsParallel().Max(packet => packet.Length))); // - larger packet size (bytes)
break;
}
case DumpFormatType.SniffDataOnly:
case DumpFormatType.SqlOnly:
case DumpFormatType.Text:
case DumpFormatType.HexOnly:
{
var outFileName = Path.ChangeExtension(FileName, null) + "_parsed.txt";
if (Utilities.FileIsInUse(outFileName) && Settings.DumpFormat != DumpFormatType.SqlOnly)
{
// If our dump format requires a .txt to be created,
// check if we can write to that .txt before starting parsing
Trace.WriteLine($"Save file {outFileName} is in use, parsing will not be done.");
break;
}
Store.Store.SQLEnabledFlags = Settings.SQLOutputFlag;
File.Delete(outFileName);
_stats.SetStartTime(DateTime.Now);
var threadCount = Settings.Threads;
if (threadCount == 0)
threadCount = Environment.ProcessorCount;
ThreadPool.SetMinThreads(threadCount + 2, 4);
var written = false;
using (var writer = (Settings.DumpFormatWithText() ? new StreamWriter(outFileName, true) : null))
{
var firstRead = true;
var firstWrite = true;
var reader = _compression != FileCompression.None ? new Reader(_tempName, _sniffType) : new Reader(FileName, _sniffType);
var pwp = new ParallelWorkProcessor<Packet>(() => // read
{
if (!reader.PacketReader.CanRead())
return Tuple.Create<Packet, bool>(null, true);
Packet packet;
var b = reader.TryRead(out packet);
if (firstRead)
{
Trace.WriteLine(
$"{_logPrefix}: Parsing {Utilities.BytesToString(reader.PacketReader.GetTotalSize())} of packets. Detected version {ClientVersion.VersionString}");
firstRead = false;
}
return Tuple.Create(packet, b);
}, packet => // parse
{
// Parse the packet, adding text to Writer and stuff to the stores
if (packet.Direction == Direction.BNClientToServer ||
packet.Direction == Direction.BNServerToClient)
BattlenetHandler.ParseBattlenet(packet);
else
Handler.Parse(packet);
// Update statistics
_stats.AddByStatus(packet.Status);
return packet;
},
packet => // write
{
ShowPercentProgress("Processing...", reader.PacketReader.GetCurrentSize(), reader.PacketReader.GetTotalSize());
if (!packet.Status.HasAnyFlag(Settings.OutputFlag) || !packet.WriteToFile)
{
packet.ClosePacket();
return;
}
written = true;
if (firstWrite)
{
// ReSharper disable AccessToDisposedClosure
writer?.WriteLine(GetHeader(FileName));
// ReSharper restore AccessToDisposedClosure
firstWrite = false;
}
// get packet header if necessary
if (Settings.LogPacketErrors)
{
switch (packet.Status)
{
case ParsedStatus.WithErrors:
_withErrorHeaders.Add(packet.GetHeader());
break;
case ParsedStatus.NotParsed:
_skippedHeaders.Add(packet.GetHeader());
break;
case ParsedStatus.NoStructure:
_noStructureHeaders.Add(packet.GetHeader());
break;
}
}
// ReSharper disable AccessToDisposedClosure
if (writer != null)
{
// Write to file
writer.WriteLine(packet.Writer);
writer.Flush();
}
// ReSharper restore AccessToDisposedClosure
// Close Writer, Stream - Dispose
packet.ClosePacket();
}, threadCount);
pwp.WaitForFinished(Timeout.Infinite);
reader.PacketReader.Dispose();
_stats.SetEndTime(DateTime.Now);
}
if (written)
Trace.WriteLine($"{_logPrefix}: Saved file to '{outFileName}'");
else
{
Trace.WriteLine($"{_logPrefix}: No file produced");
File.Delete(outFileName);
}
Trace.WriteLine($"{_logPrefix}: {_stats}");
if (Settings.SQLOutputFlag != 0 || HotfixSettings.Instance.ShouldLog())
WriteSQLs();
if (Settings.LogPacketErrors)
WritePacketErrors();
GC.Collect(); // Force a GC collect after parsing a file. It seems to help.
break;
}
case DumpFormatType.Pkt:
{
var packets = ReadPackets();
if (packets.Count == 0)
break;
if (Settings.FilterPacketsNum < 0)
{
var packetsPerSplit = Math.Abs(Settings.FilterPacketsNum);
var totalPackets = packets.Count;
var numberOfSplits = (int)Math.Ceiling((double)totalPackets/packetsPerSplit);
for (var i = 0; i < numberOfSplits; ++i)
{
var fileNamePart = FileName + "_part_" + (i + 1) + ".pkt";
var packetsPart = packets.Take(packetsPerSplit).ToList();
packets.RemoveRange(0, packetsPart.Count);
BinaryDump(fileNamePart, packetsPart);
}
}
else
{
var fileNameExcerpt = Path.ChangeExtension(FileName, null) + "_excerpt.pkt";
BinaryDump(fileNameExcerpt, packets);
}
break;
}
case DumpFormatType.PktSplit:
{
var packets = ReadPackets();
if (packets.Count == 0)
break;
SplitBinaryDump(packets);
break;
}
case DumpFormatType.PktDirectionSplit:
{
var packets = ReadPackets();
if (packets.Count == 0)
break;
DirectionSplitBinaryDump(packets);
break;
}
case DumpFormatType.PktSessionSplit:
{
var packets = ReadPackets();
if (packets.Count == 0)
break;
SessionSplitBinaryDump(packets);
break;
}
case DumpFormatType.CompressSniff:
{
if (_compression != FileCompression.None)
{
Trace.WriteLine($"Skipped compressing file {FileName}");
break;
}
Compress();
break;
}
case DumpFormatType.SniffVersionSplit:
{
var reader = _compression != FileCompression.None ? new Reader(_tempName, _sniffType) : new Reader(FileName, _sniffType);
if (ClientVersion.IsUndefined() && reader.PacketReader.CanRead())
{
Packet packet;
reader.TryRead(out packet);
packet.ClosePacket();
}
reader.PacketReader.Dispose();
var version = ClientVersion.IsUndefined() ? "unknown" : ClientVersion.VersionString;
var realFileName = GetCompressedFileName();
var destPath = Path.Combine(Path.GetDirectoryName(realFileName), version,
Path.GetFileName(realFileName));
var destDir = Path.GetDirectoryName(destPath);
if (!Directory.Exists(destDir))
Directory.CreateDirectory(destDir);
File.Move(realFileName, destPath);
Trace.WriteLine("Moved " + realFileName + " to " + destPath);
break;
}
case DumpFormatType.ConnectionIndexes:
{
var packets = ReadPackets();
if (packets.Count == 0)
break;
using (var writer = new StreamWriter(Path.ChangeExtension(FileName, null) + "_connidx.txt"))
{
if (ClientVersion.Build <= ClientVersionBuild.V6_0_3_19342)
writer.WriteLine("# Warning: versions before 6.1 might not have proper ConnectionIndex values.");
IEnumerable<IGrouping<Tuple<int, Direction>, Packet>> groupsOpcode = packets
.GroupBy(packet => Tuple.Create(packet.Opcode, packet.Direction))
.OrderBy(grouping => grouping.Key.Item2);
foreach (var groupOpcode in groupsOpcode)
{
var groups = groupOpcode
.GroupBy(packet => packet.ConnectionIndex)
.OrderBy(grouping => grouping.Key)
.ToList();
writer.Write("{0} {1,-50}: ", groupOpcode.Key.Item2, Opcodes.GetOpcodeName(groupOpcode.Key.Item1, groupOpcode.Key.Item2));
for (var i = 0; i < groups.Count; i++)
{
var idx = groups[i].Key;
writer.Write("{0} ({1}{2})", idx, (idx & 1) != 0 ? "INSTANCE" : "REALM", (idx & 2) != 0 ? "_NEW" : "");
if (i != groups.Count - 1)
writer.Write(", ");
}
writer.WriteLine();
}
}
break;
}
default:
{
Trace.WriteLine($"{_logPrefix}: Dump format is none, nothing will be processed.");
break;
}
}
}
private string ProcessFileImpl()
{
string tempFile = null;
var extension = Path.GetExtension(_fileName);
if (extension != null && extension.ToLower() == ".gz")
{
tempFile = Decompress(new FileInfo(_fileName));
_fileName = tempFile;
}
switch (_dumpFormat)
{
case DumpFormatType.StatisticsPreParse:
{
var packets = (LinkedList<Packet>)ReadPackets();
if (packets.Count == 0)
break;
// CSV format
Trace.WriteLine(String.Format("{0};{1};{2};{3};{4};{5};{6};{7};{8}",
_originalFileName, // - sniff file name
packets.First.Value.Time, // - time of first packet
packets.Last.Value.Time, // - time of last packet
(packets.Last.Value.Time - packets.First.Value.Time).TotalSeconds, // - sniff duration (seconds)
packets.Count, // - packet count
packets.AsParallel().Sum(packet => packet.Length), // - total packets size (bytes)
packets.AsParallel().Average(packet => packet.Length), // - average packet size (bytes)
packets.AsParallel().Min(packet => packet.Length), // - smaller packet size (bytes)
packets.AsParallel().Max(packet => packet.Length))); // - larger packet size (bytes)
break;
}
case DumpFormatType.SniffDataOnly:
case DumpFormatType.SqlOnly:
case DumpFormatType.Text:
{
var outFileName = Path.ChangeExtension(_originalFileName, null) + "_parsed.txt";
if (Utilities.FileIsInUse(outFileName) && Settings.DumpFormat != DumpFormatType.SqlOnly)
{
// If our dump format requires a .txt to be created,
// check if we can write to that .txt before starting parsing
Trace.WriteLine(string.Format("Save file {0} is in use, parsing will not be done.", outFileName));
break;
}
Store.Store.SQLEnabledFlags = Settings.SQLOutputFlag;
File.Delete(outFileName);
_stats.SetStartTime(DateTime.Now);
int threadCount = Settings.Threads;
if (threadCount == 0)
threadCount = Environment.ProcessorCount;
ThreadPool.SetMinThreads(threadCount + 2, 4);
using (var writer = (Settings.DumpFormatWithText() ? new StreamWriter(outFileName, true) : null))
{
bool first = true;
var reader = new Reader(_fileName, _originalFileName);
var pwp = new ParallelWorkProcessor<Packet>(() => // read
{
if (!reader.PacketReader.CanRead())
return Tuple.Create<Packet, bool>(null, true);
Packet packet;
var b = reader.TryRead(out packet);
if (first)
{
Trace.WriteLine(string.Format("{0}: Parsing {1} packets. Detected version {2}",
_logPrefix, Utilities.BytesToString(reader.PacketReader.GetTotalSize()), ClientVersion.VersionString));
// ReSharper disable AccessToDisposedClosure
if (writer != null)
writer.WriteLine(GetHeader(_originalFileName));
// ReSharper restore AccessToDisposedClosure
first = false;
}
return Tuple.Create(packet, b);
}, packet => // parse
{
// Parse the packet, adding text to Writer and stuff to the stores
if (packet.Direction == Direction.BNClientToServer ||
packet.Direction == Direction.BNServerToClient)
Handler.ParseBattlenet(packet);
else
Handler.Parse(packet);
// Update statistics
_stats.AddByStatus(packet.Status);
return packet;
},
packet => // write
{
ShowPercentProgress("Processing...", reader.PacketReader.GetCurrentSize(), reader.PacketReader.GetTotalSize());
// get packet header if necessary
if (Settings.LogPacketErrors)
{
if (packet.Status == ParsedStatus.WithErrors)
_withErrorHeaders.AddLast(packet.GetHeader());
else if (packet.Status == ParsedStatus.NotParsed)
_skippedHeaders.AddLast(packet.GetHeader());
}
// ReSharper disable AccessToDisposedClosure
if (writer != null)
{
// Write to file
writer.WriteLine(packet.Writer);
writer.Flush();
}
// ReSharper restore AccessToDisposedClosure
// Close Writer, Stream - Dispose
packet.ClosePacket();
}, threadCount);
pwp.WaitForFinished(Timeout.Infinite);
_stats.SetEndTime(DateTime.Now);
}
Trace.WriteLine(string.Format("{0}: Saved file to '{1}'", _logPrefix, outFileName));
Trace.WriteLine(string.Format("{0}: {1}", _logPrefix, _stats));
if (Settings.SQLOutputFlag != 0)
WriteSQLs();
if (Settings.LogPacketErrors)
WritePacketErrors();
GC.Collect(); // Force a GC collect after parsing a file. It seems to help.
break;
}
case DumpFormatType.Pkt:
{
var packets = (LinkedList<Packet>)ReadPackets();
if (packets.Count == 0)
break;
if (Settings.FilterPacketsNum < 0)
{
int packetsPerSplit = Math.Abs(Settings.FilterPacketsNum);
int totalPackets = packets.Count;
int numberOfSplits = totalPackets/packetsPerSplit;
for (int i = 0; i < numberOfSplits; ++i)
{
var fileNamePart = _originalFileName + "_part_" + (i + 1) + ".pkt";
var packetsPart = new LinkedList<Packet>();
for (int j = 0; j < packetsPerSplit; ++j)
{
if (packets.Count == 0)
break;
packetsPart.AddLast(packets.First.Value);
packets.RemoveFirst();
}
BinaryDump(fileNamePart, packetsPart);
}
}
else
{
var fileNameExcerpt = Path.ChangeExtension(_originalFileName, null) + "_excerpt.pkt";
BinaryDump(fileNameExcerpt, packets);
}
break;
}
case DumpFormatType.PktSplit:
{
var packets = ReadPackets();
if (packets.Count == 0)
break;
SplitBinaryDump(packets);
break;
}
case DumpFormatType.PktSessionSplit:
{
var packets = ReadPackets();
if (packets.Count == 0)
break;
SessionSplitBinaryDump(packets);
break;
}
case DumpFormatType.CompressSniff:
{
if (extension == null || extension.ToLower() == ".gz")
{
Trace.WriteLine("Skipped compressing file {0}", _fileName);
break;
}
var fi = new FileInfo(_fileName);
Compress(fi);
break;
}
case DumpFormatType.SniffVersionSplit:
{
var reader = new Reader(_fileName, _originalFileName);
if (ClientVersion.IsUndefined() && reader.PacketReader.CanRead())
{
Packet packet;
reader.TryRead(out packet);
packet.ClosePacket();
}
reader.PacketReader.Dispose();
string version = ClientVersion.IsUndefined() ? "unknown" : ClientVersion.VersionString;
string realFileName = _originalFileName + (_fileName != _originalFileName ? ".gz" : "");
string destPath = Path.Combine(Path.GetDirectoryName(realFileName), version,
Path.GetFileName(realFileName));
string destDir = Path.GetDirectoryName(destPath);
if (!Directory.Exists(destDir))
Directory.CreateDirectory(destDir);
File.Move(realFileName, destPath);
Trace.WriteLine("Moved " + realFileName + " to " + destPath);
break;
}
default:
{
Trace.WriteLine(string.Format("{0}: Dump format is none, nothing will be processed.", _logPrefix));
break;
}
}
return tempFile;
}
private string ProcessFileImpl()
{
string tempFile = null;
var extension = Path.GetExtension(_fileName);
if (extension != null && extension.ToLower() == ".gz")
{
tempFile = Decompress(new FileInfo(_fileName));
_fileName = tempFile;
}
switch (_dumpFormat)
{
case DumpFormatType.StatisticsPreParse:
{
var packets = (LinkedList <Packet>)ReadPackets();
if (packets.Count == 0)
{
break;
}
// CSV format
Trace.WriteLine(String.Format("{0};{1};{2};{3};{4};{5};{6};{7};{8}",
_originalFileName, // - sniff file name
packets.First.Value.Time, // - time of first packet
packets.Last.Value.Time, // - time of last packet
(packets.Last.Value.Time - packets.First.Value.Time).TotalSeconds, // - sniff duration (seconds)
packets.Count, // - packet count
packets.AsParallel().Sum(packet => packet.Length), // - total packets size (bytes)
packets.AsParallel().Average(packet => packet.Length), // - average packet size (bytes)
packets.AsParallel().Min(packet => packet.Length), // - smaller packet size (bytes)
packets.AsParallel().Max(packet => packet.Length))); // - larger packet size (bytes)
break;
}
case DumpFormatType.SniffDataOnly:
case DumpFormatType.SqlOnly:
case DumpFormatType.Text:
case DumpFormatType.HexOnly:
{
var outFileName = Path.ChangeExtension(_originalFileName, null) + "_parsed.txt";
if (Utilities.FileIsInUse(outFileName) && Settings.DumpFormat != DumpFormatType.SqlOnly)
{
// If our dump format requires a .txt to be created,
// check if we can write to that .txt before starting parsing
Trace.WriteLine(string.Format("Save file {0} is in use, parsing will not be done.", outFileName));
break;
}
Store.Store.SQLEnabledFlags = Settings.SQLOutputFlag;
File.Delete(outFileName);
_stats.SetStartTime(DateTime.Now);
int threadCount = Settings.Threads;
if (threadCount == 0)
{
threadCount = Environment.ProcessorCount;
}
ThreadPool.SetMinThreads(threadCount + 2, 4);
using (var writer = (Settings.DumpFormatWithText() ? new StreamWriter(outFileName, true) : null))
{
bool first = true;
var reader = new Reader(_fileName, _originalFileName);
var pwp = new ParallelWorkProcessor <Packet>(() => // read
{
if (!reader.PacketReader.CanRead())
{
return(Tuple.Create <Packet, bool>(null, true));
}
Packet packet;
var b = reader.TryRead(out packet);
if (first)
{
Trace.WriteLine(string.Format("{0}: Parsing {1} of packets. Detected version {2}",
_logPrefix, Utilities.BytesToString(reader.PacketReader.GetTotalSize()), ClientVersion.VersionString));
// ReSharper disable AccessToDisposedClosure
if (writer != null)
{
writer.WriteLine(GetHeader(_originalFileName));
}
// ReSharper restore AccessToDisposedClosure
first = false;
}
return(Tuple.Create(packet, b));
}, packet => // parse
{
// Parse the packet, adding text to Writer and stuff to the stores
if (packet.Direction == Direction.BNClientToServer ||
packet.Direction == Direction.BNServerToClient)
{
Handler.ParseBattlenet(packet);
}
else
{
Handler.Parse(packet);
}
// Update statistics
_stats.AddByStatus(packet.Status);
return(packet);
},
packet => // write
{
ShowPercentProgress("Processing...", reader.PacketReader.GetCurrentSize(), reader.PacketReader.GetTotalSize());
// get packet header if necessary
if (Settings.LogPacketErrors)
{
if (packet.Status == ParsedStatus.WithErrors)
{
_withErrorHeaders.AddLast(packet.GetHeader());
}
else if (packet.Status == ParsedStatus.NotParsed)
{
_skippedHeaders.AddLast(packet.GetHeader());
}
}
// ReSharper disable AccessToDisposedClosure
if (writer != null && packet.Status.HasAnyFlag(Settings.OutputFlag))
{
// Write to file
writer.WriteLine(packet.Writer);
writer.Flush();
}
// ReSharper restore AccessToDisposedClosure
// Close Writer, Stream - Dispose
packet.ClosePacket();
}, threadCount);
pwp.WaitForFinished(Timeout.Infinite);
_stats.SetEndTime(DateTime.Now);
}
Trace.WriteLine(string.Format("{0}: Saved file to '{1}'", _logPrefix, outFileName));
Trace.WriteLine(string.Format("{0}: {1}", _logPrefix, _stats));
if (Settings.SQLOutputFlag != 0)
{
WriteSQLs();
}
if (Settings.LogPacketErrors)
{
WritePacketErrors();
}
GC.Collect(); // Force a GC collect after parsing a file. It seems to help.
break;
}
case DumpFormatType.Pkt:
{
var packets = (LinkedList <Packet>)ReadPackets();
if (packets.Count == 0)
{
break;
}
if (Settings.FilterPacketsNum < 0)
{
int packetsPerSplit = Math.Abs(Settings.FilterPacketsNum);
int totalPackets = packets.Count;
int numberOfSplits = totalPackets / packetsPerSplit;
for (int i = 0; i < numberOfSplits; ++i)
{
var fileNamePart = _originalFileName + "_part_" + (i + 1) + ".pkt";
var packetsPart = new LinkedList <Packet>();
for (int j = 0; j < packetsPerSplit; ++j)
{
if (packets.Count == 0)
{
break;
}
packetsPart.AddLast(packets.First.Value);
packets.RemoveFirst();
}
BinaryDump(fileNamePart, packetsPart);
}
}
else
{
var fileNameExcerpt = Path.ChangeExtension(_originalFileName, null) + "_excerpt.pkt";
BinaryDump(fileNameExcerpt, packets);
}
break;
}
case DumpFormatType.PktSplit:
{
var packets = ReadPackets();
if (packets.Count == 0)
{
break;
}
SplitBinaryDump(packets);
break;
}
case DumpFormatType.PktSessionSplit:
{
var packets = ReadPackets();
if (packets.Count == 0)
{
break;
}
SessionSplitBinaryDump(packets);
break;
}
case DumpFormatType.CompressSniff:
{
if (extension == null || extension.ToLower() == ".gz")
{
Trace.WriteLine("Skipped compressing file {0}", _fileName);
break;
}
var fi = new FileInfo(_fileName);
Compress(fi);
break;
}
case DumpFormatType.SniffVersionSplit:
{
var reader = new Reader(_fileName, _originalFileName);
if (ClientVersion.IsUndefined() && reader.PacketReader.CanRead())
{
Packet packet;
reader.TryRead(out packet);
packet.ClosePacket();
}
reader.PacketReader.Dispose();
string version = ClientVersion.IsUndefined() ? "unknown" : ClientVersion.VersionString;
string realFileName = _originalFileName + (_fileName != _originalFileName ? ".gz" : "");
string destPath = Path.Combine(Path.GetDirectoryName(realFileName), version,
Path.GetFileName(realFileName));
string destDir = Path.GetDirectoryName(destPath);
if (!Directory.Exists(destDir))
{
Directory.CreateDirectory(destDir);
}
File.Move(realFileName, destPath);
Trace.WriteLine("Moved " + realFileName + " to " + destPath);
break;
}
default:
{
Trace.WriteLine(string.Format("{0}: Dump format is none, nothing will be processed.", _logPrefix));
break;
}
}
return(tempFile);
}
private string ProcessFileImpl()
{
string tempFile = null;
var extension = Path.GetExtension(_fileName);
if (extension != null && extension.ToLower() == ".gz")
{
tempFile = Decompress(new FileInfo(_fileName));
_fileName = tempFile;
}
switch (_dumpFormat)
{
case DumpFormatType.StatisticsPreParse:
{
ReadPackets();
if (_stats.GetCount() == 0)
{
break;
}
var firstPacket = _stats.GetFirstPacket();
var lastPacket = _stats.GetLastPacket();
// CSV format
Trace.WriteLine(String.Format("{0};{1};{2};{3};{4};{5};{6};{7};{8}",
_originalFileName, // - sniff file name
firstPacket.Time, // - time of first packet
lastPacket.Time, // - time of last packet
(lastPacket.Time - firstPacket.Time).TotalSeconds, // - sniff duration (seconds)
_stats.GetCount(), // - packet count
_stats.GetTotalLength(), // - total packets size (bytes)
_stats.GetTotalLength() / _stats.GetCount(), // - average packet size (bytes)
_stats.GetMinLength(), // - smaller packet size (bytes)
_stats.GetMaxLenth())); // - larger packet size (bytes)
Console.WriteLine("filename: {0}", _originalFileName);
Console.WriteLine("fist packet time: {0}", firstPacket.Time);
Console.WriteLine("last packet time: {0}", lastPacket.Time);
Console.WriteLine("sniff duration (seconds): {0}", (lastPacket.Time - firstPacket.Time).TotalSeconds);
Console.WriteLine("packet count: {0}", _stats.GetCount());
Console.WriteLine("total packet size (bytes): {0}", _stats.GetTotalLength());
Console.WriteLine("average packet size (bytes): {0}", _stats.GetTotalLength() / _stats.GetCount());
Console.WriteLine("minimum packet size (bytes): {0}", _stats.GetMinLength());
Console.WriteLine("maximum packet size (bytes): {0}", _stats.GetMaxLenth());
_stats.PrintOpcodeCount();
_stats.PrintDirectionCount();
break;
}
case DumpFormatType.SniffDataOnly:
case DumpFormatType.SqlOnly:
case DumpFormatType.Text:
case DumpFormatType.HexOnly:
{
var outFileName = Path.ChangeExtension(_originalFileName, null) + "_parsed.txt";
if (Utilities.FileIsInUse(outFileName) && Settings.DumpFormat != DumpFormatType.SqlOnly)
{
// If our dump format requires a .txt to be created,
// check if we can write to that .txt before starting parsing
Trace.WriteLine(string.Format("Save file {0} is in use, parsing will not be done.", outFileName));
break;
}
Store.Store.SQLEnabledFlags = Settings.SQLOutputFlag;
File.Delete(outFileName);
_stats.SetStartTime(DateTime.Now);
int threadCount = Settings.Threads;
if (threadCount == 0)
{
threadCount = Environment.ProcessorCount;
}
ThreadPool.SetMinThreads(threadCount + 2, 4);
var written = false;
using (var writer = (Settings.DumpFormatWithText() ? new StreamWriter(outFileName, true) : null))
{
var firstRead = true;
var firstWrite = true;
var reader = new Reader(_fileName, _originalFileName);
var pwp = new ParallelWorkProcessor <Packet>(() => // read
{
if (!reader.PacketReader.CanRead())
{
return(Tuple.Create <Packet, bool>(null, true));
}
Packet packet;
var b = reader.TryRead(out packet);
if (firstRead)
{
Trace.WriteLine(string.Format("{0}: Parsing {1} of packets. Detected version {2}",
_logPrefix, Utilities.BytesToString(reader.PacketReader.GetTotalSize()), ClientVersion.VersionString));
firstRead = false;
}
return(Tuple.Create(packet, b));
}, packet => // parse
{
// Parse the packet, adding text to Writer and stuff to the stores
if (packet.Direction == Direction.BNClientToServer ||
packet.Direction == Direction.BNServerToClient)
{
Handler.ParseBattlenet(packet);
}
else
{
Handler.Parse(packet);
}
// Update statistics
_stats.AddByStatus(packet.Status);
return(packet);
},
packet => // write
{
ShowPercentProgress("Processing...", reader.PacketReader.GetCurrentSize(), reader.PacketReader.GetTotalSize());
if (!packet.Status.HasAnyFlag(Settings.OutputFlag) || !packet.WriteToFile)
{
packet.ClosePacket();
return;
}
written = true;
if (firstWrite)
{
// ReSharper disable AccessToDisposedClosure
if (writer != null)
{
writer.WriteLine(GetHeader(_originalFileName));
}
// ReSharper restore AccessToDisposedClosure
firstWrite = false;
}
// get packet header if necessary
if (Settings.LogPacketErrors)
{
switch (packet.Status)
{
case ParsedStatus.WithErrors:
_withErrorHeaders.Add(packet.GetHeader());
break;
case ParsedStatus.NotParsed:
_skippedHeaders.Add(packet.GetHeader());
break;
case ParsedStatus.NoStructure:
_noStructureHeaders.Add(packet.GetHeader());
break;
}
}
// ReSharper disable AccessToDisposedClosure
if (writer != null)
{
// Write to file
writer.WriteLine(packet.Writer);
writer.Flush();
}
// ReSharper restore AccessToDisposedClosure
// Close Writer, Stream - Dispose
packet.ClosePacket();
}, threadCount);
pwp.WaitForFinished(Timeout.Infinite);
_stats.SetEndTime(DateTime.Now);
}
if (written)
{
Trace.WriteLine(string.Format("{0}: Saved file to '{1}'", _logPrefix, outFileName));
}
else
{
Trace.WriteLine(string.Format("{0}: No file produced", _logPrefix));
File.Delete(outFileName);
}
Trace.WriteLine(string.Format("{0}: {1}", _logPrefix, _stats));
if (Settings.SQLOutputFlag != 0)
{
WriteSQLs();
}
if (Settings.LogPacketErrors)
{
WritePacketErrors();
}
GC.Collect(); // Force a GC collect after parsing a file. It seems to help.
break;
}
/*case DumpFormatType.Pkt:
* {
* var packets = ReadPackets();
* if (packets.Count == 0)
* break;
*
* if (Settings.FilterPacketsNum < 0)
* {
* int packetsPerSplit = Math.Abs(Settings.FilterPacketsNum);
* int totalPackets = packets.Count;
*
* int numberOfSplits = (int)Math.Ceiling((double)totalPackets/packetsPerSplit);
*
* for (int i = 0; i < numberOfSplits; ++i)
* {
* var fileNamePart = _originalFileName + "_part_" + (i + 1) + ".pkt";
*
* var packetsPart = packets.Take(packetsPerSplit).ToList();
* packets.RemoveRange(0, packetsPart.Count);
*
* BinaryDump(fileNamePart, packetsPart);
* }
* }
* else
* {
* var fileNameExcerpt = Path.ChangeExtension(_originalFileName, null) + "_excerpt.pkt";
* BinaryDump(fileNameExcerpt, packets);
* }
*
* break;
* }
* case DumpFormatType.PktSplit:
* {
* var packets = ReadPackets();
* if (packets.Count == 0)
* break;
*
* SplitBinaryDump(packets);
* break;
* }
* case DumpFormatType.PktDirectionSplit:
* {
* var packets = ReadPackets();
* if (packets.Count == 0)
* break;
*
* DirectionSplitBinaryDump(packets);
* break;
* }
* case DumpFormatType.PktSessionSplit:
* {
* var packets = ReadPackets();
* if (packets.Count == 0)
* break;
*
* SessionSplitBinaryDump(packets);
* break;
* }*/
case DumpFormatType.CompressSniff:
{
if (extension == null || extension.ToLower() == ".gz")
{
Trace.WriteLine("Skipped compressing file {0}", _fileName);
break;
}
var fi = new FileInfo(_fileName);
Compress(fi);
break;
}
case DumpFormatType.SniffVersionSplit:
{
var reader = new Reader(_fileName, _originalFileName);
if (ClientVersion.IsUndefined() && reader.PacketReader.CanRead())
{
Packet packet;
reader.TryRead(out packet);
packet.ClosePacket();
}
reader.PacketReader.Dispose();
string version = ClientVersion.IsUndefined() ? "unknown" : ClientVersion.VersionString;
string realFileName = _originalFileName + (_fileName != _originalFileName ? ".gz" : "");
string destPath = Path.Combine(Path.GetDirectoryName(realFileName), version,
Path.GetFileName(realFileName));
string destDir = Path.GetDirectoryName(destPath);
if (!Directory.Exists(destDir))
{
Directory.CreateDirectory(destDir);
}
File.Move(realFileName, destPath);
Trace.WriteLine("Moved " + realFileName + " to " + destPath);
break;
}
/*case DumpFormatType.ConnectionIndexes:
* {
* var packets = ReadPackets();
* if (packets.Count == 0)
* break;
*
* using (var writer = new StreamWriter(Path.ChangeExtension(_originalFileName, null) + "_connidx.txt"))
* {
* if (ClientVersion.Build <= ClientVersionBuild.V6_0_3_19342)
* writer.WriteLine("# Warning: versions before 6.1 might not have proper ConnectionIndex values.");
*
* IEnumerable<IGrouping<Tuple<int, Direction>, Packet>> groupsOpcode = packets
* .GroupBy(packet => Tuple.Create(packet.Opcode, packet.Direction))
* .OrderBy(grouping => grouping.Key.Item2);
*
* foreach (IGrouping<Tuple<int, Direction>, Packet> groupOpcode in groupsOpcode)
* {
* List<IGrouping<int, Packet>> groups = groupOpcode
* .GroupBy(packet => packet.ConnectionIndex)
* .OrderBy(grouping => grouping.Key)
* .ToList();
*
* writer.Write("{0} {1,-50}: ", groupOpcode.Key.Item2, Opcodes.GetOpcodeName(groupOpcode.Key.Item1, groupOpcode.Key.Item2));
*
* for (int i = 0; i < groups.Count; i++)
* {
* int idx = groups[i].Key;
* writer.Write("{0} ({1}{2})", idx, (idx & 1) != 0 ? "INSTANCE" : "REALM", (idx & 2) != 0 ? "_NEW" : "");
*
* if (i != groups.Count - 1)
* writer.Write(", ");
* }
*
* writer.WriteLine();
* }
* }
*
* break;
* }*/
default:
{
Trace.WriteLine(string.Format("{0}: Dump format is none, nothing will be processed.", _logPrefix));
break;
}
}
return(tempFile);
}
private void launchProcessing(int threadCount, string outFileName)
{
var firstRead = true;
var reader = _compression != FileCompression.None ? new Reader(_tempName, _sniffType) : new Reader(FileName, _sniffType);
var written = false;
IDumpWriter writer = null;
// this is done to ensure disposal of IDumpWriter (extended form of using(var ...) {...}
try
{
if (Settings.DumpFormatWithText() && Settings.DumpTextFormat == TextFormatType.Txt)
{
writer = new TextDumpWriter(outFileName + ".txt");
}
//var writer = (Settings.DumpFormatWithText() ? new StreamWriter(outFileName, true) : null
else if (Settings.DumpFormatWithText() && Settings.DumpTextFormat == TextFormatType.Xml)
{
writer = new XmlDumpWriter(outFileName + ".xml");
}
var firstWrite = true;
var pwp = new ParallelWorkProcessor <Packet>(() => // read
{
if (!reader.PacketReader.CanRead())
{
return(Tuple.Create <Packet, bool>(null, true));
}
Packet packet;
var b = reader.TryRead(out packet);
if (firstRead)
{
Trace.WriteLine(
$"{_logPrefix}: Parsing {Utilities.BytesToString(reader.PacketReader.GetTotalSize())} of packets. Detected version {ClientVersion.VersionString}");
firstRead = false;
}
return(Tuple.Create(packet, b));
}, packet => // parse
{
// Parse the packet, adding text to Writer and stuff to the stores
if (packet.Direction == Direction.BNClientToServer ||
packet.Direction == Direction.BNServerToClient)
{
BattlenetHandler.ParseBattlenet(packet);
}
else
{
Handler.Parse(packet);
}
// Update statistics
_stats.AddByStatus(packet.Status);
return(packet);
}, packet => // write
{
ShowPercentProgress("Processing...", reader.PacketReader.GetCurrentSize(), reader.PacketReader.GetTotalSize());
if (!packet.Status.HasAnyFlag(Settings.OutputFlag) || !packet.WriteToFile)
{
packet.ClosePacket();
return;
}
written = true;
if (firstWrite)
{
//writer?.WriteHeader(GetHeader(FileName));
writer?.WriteHeader(FileName);
firstWrite = false;
}
if (Settings.LogPacketErrors)
{
switch (packet.Status)
{
case ParsedStatus.WithErrors:
_withErrorHeaders.Add(packet.GetHeader());
break;
case ParsedStatus.NotParsed:
_skippedHeaders.Add(packet.GetHeader());
break;
case ParsedStatus.NoStructure:
_noStructureHeaders.Add(packet.GetHeader());
break;
}
}
if (writer != null)
{
// Write to file
writer.WriteItem(packet);
}
// Close Writer, Stream - Dispose
packet.ClosePacket();
}, threadCount);
pwp.WaitForFinished(Timeout.Infinite);
reader.PacketReader.Dispose();
_stats.SetEndTime(DateTime.Now);
}
finally
{
if (writer != null)
{
writer.Dispose();
}
}
if (written)
{
Trace.WriteLine($"{_logPrefix}: Saved file to '{outFileName}'");
}
else
{
Trace.WriteLine($"{_logPrefix}: No file produced");
File.Delete(outFileName);
}
}