private string ProcessFileImpl()
{
string tempFile = null;
var extension = Path.GetExtension(_fileName);
if (extension != null && extension.ToLower() == ".gz")
{
tempFile = Decompress(new FileInfo(_fileName));
_fileName = tempFile;
}
switch (_dumpFormat)
{
case DumpFormatType.StatisticsPreParse:
{
var packets = (LinkedList<Packet>)ReadPackets();
if (packets.Count == 0)
break;
// CSV format
Trace.WriteLine(String.Format("{0};{1};{2};{3};{4};{5};{6};{7};{8}",
_originalFileName, // - sniff file name
packets.First.Value.Time, // - time of first packet
packets.Last.Value.Time, // - time of last packet
(packets.Last.Value.Time - packets.First.Value.Time).TotalSeconds, // - sniff duration (seconds)
packets.Count, // - packet count
packets.AsParallel().Sum(packet => packet.Length), // - total packets size (bytes)
packets.AsParallel().Average(packet => packet.Length), // - average packet size (bytes)
packets.AsParallel().Min(packet => packet.Length), // - smaller packet size (bytes)
packets.AsParallel().Max(packet => packet.Length))); // - larger packet size (bytes)
break;
}
case DumpFormatType.SniffDataOnly:
case DumpFormatType.SqlOnly:
case DumpFormatType.Text:
{
var outFileName = Path.ChangeExtension(_originalFileName, null) + "_parsed.txt";
if (Utilities.FileIsInUse(outFileName) && Settings.DumpFormat != DumpFormatType.SqlOnly)
{
// If our dump format requires a .txt to be created,
// check if we can write to that .txt before starting parsing
Trace.WriteLine(string.Format("Save file {0} is in use, parsing will not be done.", outFileName));
break;
}
Store.Store.SQLEnabledFlags = Settings.SQLOutputFlag;
File.Delete(outFileName);
_stats.SetStartTime(DateTime.Now);
int threadCount = Settings.Threads;
if (threadCount == 0)
threadCount = Environment.ProcessorCount;
ThreadPool.SetMinThreads(threadCount + 2, 4);
using (var writer = (Settings.DumpFormatWithText() ? new StreamWriter(outFileName, true) : null))
{
bool first = true;
var reader = new Reader(_fileName, _originalFileName);
var pwp = new ParallelWorkProcessor<Packet>(() => // read
{
if (!reader.PacketReader.CanRead())
return Tuple.Create<Packet, bool>(null, true);
Packet packet;
var b = reader.TryRead(out packet);
if (first)
{
Trace.WriteLine(string.Format("{0}: Parsing {1} packets. Detected version {2}",
_logPrefix, Utilities.BytesToString(reader.PacketReader.GetTotalSize()), ClientVersion.VersionString));
// ReSharper disable AccessToDisposedClosure
if (writer != null)
writer.WriteLine(GetHeader(_originalFileName));
// ReSharper restore AccessToDisposedClosure
first = false;
}
return Tuple.Create(packet, b);
}, packet => // parse
{
// Parse the packet, adding text to Writer and stuff to the stores
if (packet.Direction == Direction.BNClientToServer ||
packet.Direction == Direction.BNServerToClient)
Handler.ParseBattlenet(packet);
else
Handler.Parse(packet);
// Update statistics
_stats.AddByStatus(packet.Status);
return packet;
},
packet => // write
{
ShowPercentProgress("Processing...", reader.PacketReader.GetCurrentSize(), reader.PacketReader.GetTotalSize());
// get packet header if necessary
if (Settings.LogPacketErrors)
{
if (packet.Status == ParsedStatus.WithErrors)
_withErrorHeaders.AddLast(packet.GetHeader());
else if (packet.Status == ParsedStatus.NotParsed)
_skippedHeaders.AddLast(packet.GetHeader());
}
// ReSharper disable AccessToDisposedClosure
if (writer != null)
{
// Write to file
writer.WriteLine(packet.Writer);
writer.Flush();
}
// ReSharper restore AccessToDisposedClosure
// Close Writer, Stream - Dispose
packet.ClosePacket();
}, threadCount);
pwp.WaitForFinished(Timeout.Infinite);
_stats.SetEndTime(DateTime.Now);
}
Trace.WriteLine(string.Format("{0}: Saved file to '{1}'", _logPrefix, outFileName));
Trace.WriteLine(string.Format("{0}: {1}", _logPrefix, _stats));
if (Settings.SQLOutputFlag != 0)
WriteSQLs();
if (Settings.LogPacketErrors)
WritePacketErrors();
GC.Collect(); // Force a GC collect after parsing a file. It seems to help.
break;
}
case DumpFormatType.Pkt:
{
var packets = (LinkedList<Packet>)ReadPackets();
if (packets.Count == 0)
break;
if (Settings.FilterPacketsNum < 0)
{
int packetsPerSplit = Math.Abs(Settings.FilterPacketsNum);
int totalPackets = packets.Count;
int numberOfSplits = totalPackets/packetsPerSplit;
for (int i = 0; i < numberOfSplits; ++i)
{
var fileNamePart = _originalFileName + "_part_" + (i + 1) + ".pkt";
var packetsPart = new LinkedList<Packet>();
for (int j = 0; j < packetsPerSplit; ++j)
{
if (packets.Count == 0)
break;
packetsPart.AddLast(packets.First.Value);
packets.RemoveFirst();
}
BinaryDump(fileNamePart, packetsPart);
}
}
else
{
var fileNameExcerpt = Path.ChangeExtension(_originalFileName, null) + "_excerpt.pkt";
BinaryDump(fileNameExcerpt, packets);
}
break;
}
case DumpFormatType.PktSplit:
{
var packets = ReadPackets();
if (packets.Count == 0)
break;
SplitBinaryDump(packets);
break;
}
case DumpFormatType.PktSessionSplit:
{
var packets = ReadPackets();
if (packets.Count == 0)
break;
SessionSplitBinaryDump(packets);
break;
}
case DumpFormatType.CompressSniff:
{
if (extension == null || extension.ToLower() == ".gz")
{
Trace.WriteLine("Skipped compressing file {0}", _fileName);
break;
}
var fi = new FileInfo(_fileName);
Compress(fi);
break;
}
case DumpFormatType.SniffVersionSplit:
{
var reader = new Reader(_fileName, _originalFileName);
if (ClientVersion.IsUndefined() && reader.PacketReader.CanRead())
{
Packet packet;
reader.TryRead(out packet);
packet.ClosePacket();
}
reader.PacketReader.Dispose();
string version = ClientVersion.IsUndefined() ? "unknown" : ClientVersion.VersionString;
string realFileName = _originalFileName + (_fileName != _originalFileName ? ".gz" : "");
string destPath = Path.Combine(Path.GetDirectoryName(realFileName), version,
Path.GetFileName(realFileName));
string destDir = Path.GetDirectoryName(destPath);
if (!Directory.Exists(destDir))
Directory.CreateDirectory(destDir);
File.Move(realFileName, destPath);
Trace.WriteLine("Moved " + realFileName + " to " + destPath);
break;
}
default:
{
Trace.WriteLine(string.Format("{0}: Dump format is none, nothing will be processed.", _logPrefix));
break;
}
}
return tempFile;
}
private string ProcessFileImpl()
{
string tempFile = null;
var extension = Path.GetExtension(_fileName);
if (extension != null && extension.ToLower() == ".gz")
{
tempFile = Decompress(new FileInfo(_fileName));
_fileName = tempFile;
}
switch (_dumpFormat)
{
case DumpFormatType.StatisticsPreParse:
{
var packets = ReadPackets();
if (packets.Count == 0)
break;
var firstPacket = packets.First();
var lastPacket = packets.Last();
// CSV format
Trace.WriteLine(String.Format("{0};{1};{2};{3};{4};{5};{6};{7};{8}",
_originalFileName, // - sniff file name
firstPacket.Time, // - time of first packet
lastPacket.Time, // - time of last packet
(lastPacket.Time - firstPacket.Time).TotalSeconds, // - sniff duration (seconds)
packets.Count, // - packet count
packets.AsParallel().Sum(packet => packet.Length), // - total packets size (bytes)
packets.AsParallel().Average(packet => packet.Length), // - average packet size (bytes)
packets.AsParallel().Min(packet => packet.Length), // - smaller packet size (bytes)
packets.AsParallel().Max(packet => packet.Length))); // - larger packet size (bytes)
break;
}
case DumpFormatType.SniffDataOnly:
case DumpFormatType.SqlOnly:
case DumpFormatType.Text:
case DumpFormatType.HexOnly:
{
var outFileName = Path.ChangeExtension(_originalFileName, null) + "_parsed.txt";
if (Utilities.FileIsInUse(outFileName) && Settings.DumpFormat != DumpFormatType.SqlOnly)
{
// If our dump format requires a .txt to be created,
// check if we can write to that .txt before starting parsing
Trace.WriteLine(string.Format("Save file {0} is in use, parsing will not be done.", outFileName));
break;
}
Store.Store.SQLEnabledFlags = Settings.SQLOutputFlag;
File.Delete(outFileName);
_stats.SetStartTime(DateTime.Now);
int threadCount = Settings.Threads;
if (threadCount == 0)
threadCount = Environment.ProcessorCount;
ThreadPool.SetMinThreads(threadCount + 2, 4);
var written = false;
using (var writer = (Settings.DumpFormatWithText() ? new StreamWriter(outFileName, true) : null))
{
var firstRead = true;
var firstWrite = true;
var reader = new Reader(_fileName, _originalFileName);
var pwp = new ParallelWorkProcessor<Packet>(() => // read
{
if (!reader.PacketReader.CanRead())
return Tuple.Create<Packet, bool>(null, true);
Packet packet;
var b = reader.TryRead(out packet);
if (firstRead)
{
Trace.WriteLine(string.Format("{0}: Parsing {1} of packets. Detected version {2}",
_logPrefix, Utilities.BytesToString(reader.PacketReader.GetTotalSize()), ClientVersion.VersionString));
firstRead = false;
}
return Tuple.Create(packet, b);
}, packet => // parse
{
// Parse the packet, adding text to Writer and stuff to the stores
if (packet.Direction == Direction.BNClientToServer ||
packet.Direction == Direction.BNServerToClient)
Handler.ParseBattlenet(packet);
else
Handler.Parse(packet);
// Update statistics
_stats.AddByStatus(packet.Status);
return packet;
},
packet => // write
{
ShowPercentProgress("Processing...", reader.PacketReader.GetCurrentSize(), reader.PacketReader.GetTotalSize());
if (!packet.Status.HasAnyFlag(Settings.OutputFlag))
{
packet.ClosePacket();
return;
}
written = true;
if (firstWrite)
{
// ReSharper disable AccessToDisposedClosure
if (writer != null)
writer.WriteLine(GetHeader(_originalFileName));
// ReSharper restore AccessToDisposedClosure
firstWrite = false;
}
// get packet header if necessary
if (Settings.LogPacketErrors)
{
switch (packet.Status)
{
case ParsedStatus.WithErrors:
_withErrorHeaders.Add(packet.GetHeader());
break;
case ParsedStatus.NotParsed:
_skippedHeaders.Add(packet.GetHeader());
break;
case ParsedStatus.NoStructure:
_noStructureHeaders.Add(packet.GetHeader());
break;
}
}
// ReSharper disable AccessToDisposedClosure
if (writer != null)
{
// Write to file
writer.WriteLine(packet.Writer);
writer.Flush();
}
// ReSharper restore AccessToDisposedClosure
// Close Writer, Stream - Dispose
packet.ClosePacket();
}, threadCount);
pwp.WaitForFinished(Timeout.Infinite);
_stats.SetEndTime(DateTime.Now);
}
if (written)
Trace.WriteLine(string.Format("{0}: Saved file to '{1}'", _logPrefix, outFileName));
else
{
Trace.WriteLine(string.Format("{0}: No file produced", _logPrefix));
File.Delete(outFileName);
}
Trace.WriteLine(string.Format("{0}: {1}", _logPrefix, _stats));
if (Settings.SQLOutputFlag != 0)
WriteSQLs();
if (Settings.LogPacketErrors)
WritePacketErrors();
GC.Collect(); // Force a GC collect after parsing a file. It seems to help.
break;
}
case DumpFormatType.Pkt:
{
var packets = ReadPackets();
if (packets.Count == 0)
break;
if (Settings.FilterPacketsNum < 0)
{
int packetsPerSplit = Math.Abs(Settings.FilterPacketsNum);
int totalPackets = packets.Count;
int numberOfSplits = (int)Math.Ceiling((double)totalPackets/packetsPerSplit);
for (int i = 0; i < numberOfSplits; ++i)
{
var fileNamePart = _originalFileName + "_part_" + (i + 1) + ".pkt";
var packetsPart = packets.Take(packetsPerSplit).ToList();
packets.RemoveRange(0, packetsPart.Count);
BinaryDump(fileNamePart, packetsPart);
}
}
else
{
var fileNameExcerpt = Path.ChangeExtension(_originalFileName, null) + "_excerpt.pkt";
BinaryDump(fileNameExcerpt, packets);
}
break;
}
case DumpFormatType.PktSplit:
{
var packets = ReadPackets();
if (packets.Count == 0)
break;
SplitBinaryDump(packets);
break;
}
case DumpFormatType.PktDirectionSplit:
{
var packets = ReadPackets();
if (packets.Count == 0)
break;
DirectionSplitBinaryDump(packets);
break;
}
case DumpFormatType.PktSessionSplit:
{
var packets = ReadPackets();
if (packets.Count == 0)
break;
SessionSplitBinaryDump(packets);
break;
}
case DumpFormatType.CompressSniff:
{
if (extension == null || extension.ToLower() == ".gz")
{
Trace.WriteLine("Skipped compressing file {0}", _fileName);
break;
}
var fi = new FileInfo(_fileName);
Compress(fi);
break;
}
case DumpFormatType.SniffVersionSplit:
{
var reader = new Reader(_fileName, _originalFileName);
if (ClientVersion.IsUndefined() && reader.PacketReader.CanRead())
{
Packet packet;
reader.TryRead(out packet);
packet.ClosePacket();
}
reader.PacketReader.Dispose();
string version = ClientVersion.IsUndefined() ? "unknown" : ClientVersion.VersionString;
string realFileName = _originalFileName + (_fileName != _originalFileName ? ".gz" : "");
string destPath = Path.Combine(Path.GetDirectoryName(realFileName), version,
Path.GetFileName(realFileName));
string destDir = Path.GetDirectoryName(destPath);
if (!Directory.Exists(destDir))
Directory.CreateDirectory(destDir);
File.Move(realFileName, destPath);
Trace.WriteLine("Moved " + realFileName + " to " + destPath);
break;
}
case DumpFormatType.ConnectionIndexes:
{
var packets = ReadPackets();
if (packets.Count == 0)
break;
using (var writer = new StreamWriter(Path.ChangeExtension(_originalFileName, null) + "_connidx.txt"))
{
if (ClientVersion.Build <= ClientVersionBuild.V6_0_3_19342)
writer.WriteLine("# Warning: versions before 6.1 might not have proper ConnectionIndex values.");
IEnumerable<IGrouping<Tuple<int, Direction>, Packet>> groupsOpcode = packets
.GroupBy(packet => Tuple.Create(packet.Opcode, packet.Direction))
.OrderBy(grouping => grouping.Key.Item2);
foreach (IGrouping<Tuple<int, Direction>, Packet> groupOpcode in groupsOpcode)
{
List<IGrouping<int, Packet>> groups = groupOpcode
.GroupBy(packet => packet.ConnectionIndex)
.OrderBy(grouping => grouping.Key)
.ToList();
writer.Write("{0} {1,-50}: ", groupOpcode.Key.Item2, Opcodes.GetOpcodeName(groupOpcode.Key.Item1, groupOpcode.Key.Item2));
for (int i = 0; i < groups.Count; i++)
{
int idx = groups[i].Key;
writer.Write("{0} ({1}{2})", idx, (idx & 1) != 0 ? "INSTANCE" : "REALM", (idx & 2) != 0 ? "_NEW" : "");
if (i != groups.Count - 1)
writer.Write(", ");
}
writer.WriteLine();
}
}
break;
}
default:
{
Trace.WriteLine(string.Format("{0}: Dump format is none, nothing will be processed.", _logPrefix));
break;
}
}
return tempFile;
}
