private string ProcessFileImpl() { string tempFile = null; var extension = Path.GetExtension(_fileName); if (extension != null && extension.ToLower() == ".gz") { tempFile = Decompress(new FileInfo(_fileName)); _fileName = tempFile; } switch (_dumpFormat) { case DumpFormatType.StatisticsPreParse: { var packets = (LinkedList<Packet>)ReadPackets(); if (packets.Count == 0) break; // CSV format Trace.WriteLine(String.Format("{0};{1};{2};{3};{4};{5};{6};{7};{8}", _originalFileName, // - sniff file name packets.First.Value.Time, // - time of first packet packets.Last.Value.Time, // - time of last packet (packets.Last.Value.Time - packets.First.Value.Time).TotalSeconds, // - sniff duration (seconds) packets.Count, // - packet count packets.AsParallel().Sum(packet => packet.Length), // - total packets size (bytes) packets.AsParallel().Average(packet => packet.Length), // - average packet size (bytes) packets.AsParallel().Min(packet => packet.Length), // - smaller packet size (bytes) packets.AsParallel().Max(packet => packet.Length))); // - larger packet size (bytes) break; } case DumpFormatType.SniffDataOnly: case DumpFormatType.SqlOnly: case DumpFormatType.Text: { var outFileName = Path.ChangeExtension(_originalFileName, null) + "_parsed.txt"; if (Utilities.FileIsInUse(outFileName) && Settings.DumpFormat != DumpFormatType.SqlOnly) { // If our dump format requires a .txt to be created, // check if we can write to that .txt before starting parsing Trace.WriteLine(string.Format("Save file {0} is in use, parsing will not be done.", outFileName)); break; } Store.Store.SQLEnabledFlags = Settings.SQLOutputFlag; File.Delete(outFileName); _stats.SetStartTime(DateTime.Now); int threadCount = Settings.Threads; if (threadCount == 0) threadCount = Environment.ProcessorCount; ThreadPool.SetMinThreads(threadCount + 2, 4); using (var writer = (Settings.DumpFormatWithText() ? new StreamWriter(outFileName, true) : null)) { bool first = true; var reader = new Reader(_fileName, _originalFileName); var pwp = new ParallelWorkProcessor<Packet>(() => // read { if (!reader.PacketReader.CanRead()) return Tuple.Create<Packet, bool>(null, true); Packet packet; var b = reader.TryRead(out packet); if (first) { Trace.WriteLine(string.Format("{0}: Parsing {1} packets. Detected version {2}", _logPrefix, Utilities.BytesToString(reader.PacketReader.GetTotalSize()), ClientVersion.VersionString)); // ReSharper disable AccessToDisposedClosure if (writer != null) writer.WriteLine(GetHeader(_originalFileName)); // ReSharper restore AccessToDisposedClosure first = false; } return Tuple.Create(packet, b); }, packet => // parse { // Parse the packet, adding text to Writer and stuff to the stores if (packet.Direction == Direction.BNClientToServer || packet.Direction == Direction.BNServerToClient) Handler.ParseBattlenet(packet); else Handler.Parse(packet); // Update statistics _stats.AddByStatus(packet.Status); return packet; }, packet => // write { ShowPercentProgress("Processing...", reader.PacketReader.GetCurrentSize(), reader.PacketReader.GetTotalSize()); // get packet header if necessary if (Settings.LogPacketErrors) { if (packet.Status == ParsedStatus.WithErrors) _withErrorHeaders.AddLast(packet.GetHeader()); else if (packet.Status == ParsedStatus.NotParsed) _skippedHeaders.AddLast(packet.GetHeader()); } // ReSharper disable AccessToDisposedClosure if (writer != null) { // Write to file writer.WriteLine(packet.Writer); writer.Flush(); } // ReSharper restore AccessToDisposedClosure // Close Writer, Stream - Dispose packet.ClosePacket(); }, threadCount); pwp.WaitForFinished(Timeout.Infinite); _stats.SetEndTime(DateTime.Now); } Trace.WriteLine(string.Format("{0}: Saved file to '{1}'", _logPrefix, outFileName)); Trace.WriteLine(string.Format("{0}: {1}", _logPrefix, _stats)); if (Settings.SQLOutputFlag != 0) WriteSQLs(); if (Settings.LogPacketErrors) WritePacketErrors(); GC.Collect(); // Force a GC collect after parsing a file. It seems to help. break; } case DumpFormatType.Pkt: { var packets = (LinkedList<Packet>)ReadPackets(); if (packets.Count == 0) break; if (Settings.FilterPacketsNum < 0) { int packetsPerSplit = Math.Abs(Settings.FilterPacketsNum); int totalPackets = packets.Count; int numberOfSplits = totalPackets/packetsPerSplit; for (int i = 0; i < numberOfSplits; ++i) { var fileNamePart = _originalFileName + "_part_" + (i + 1) + ".pkt"; var packetsPart = new LinkedList<Packet>(); for (int j = 0; j < packetsPerSplit; ++j) { if (packets.Count == 0) break; packetsPart.AddLast(packets.First.Value); packets.RemoveFirst(); } BinaryDump(fileNamePart, packetsPart); } } else { var fileNameExcerpt = Path.ChangeExtension(_originalFileName, null) + "_excerpt.pkt"; BinaryDump(fileNameExcerpt, packets); } break; } case DumpFormatType.PktSplit: { var packets = ReadPackets(); if (packets.Count == 0) break; SplitBinaryDump(packets); break; } case DumpFormatType.PktSessionSplit: { var packets = ReadPackets(); if (packets.Count == 0) break; SessionSplitBinaryDump(packets); break; } case DumpFormatType.CompressSniff: { if (extension == null || extension.ToLower() == ".gz") { Trace.WriteLine("Skipped compressing file {0}", _fileName); break; } var fi = new FileInfo(_fileName); Compress(fi); break; } case DumpFormatType.SniffVersionSplit: { var reader = new Reader(_fileName, _originalFileName); if (ClientVersion.IsUndefined() && reader.PacketReader.CanRead()) { Packet packet; reader.TryRead(out packet); packet.ClosePacket(); } reader.PacketReader.Dispose(); string version = ClientVersion.IsUndefined() ? "unknown" : ClientVersion.VersionString; string realFileName = _originalFileName + (_fileName != _originalFileName ? ".gz" : ""); string destPath = Path.Combine(Path.GetDirectoryName(realFileName), version, Path.GetFileName(realFileName)); string destDir = Path.GetDirectoryName(destPath); if (!Directory.Exists(destDir)) Directory.CreateDirectory(destDir); File.Move(realFileName, destPath); Trace.WriteLine("Moved " + realFileName + " to " + destPath); break; } default: { Trace.WriteLine(string.Format("{0}: Dump format is none, nothing will be processed.", _logPrefix)); break; } } return tempFile; }
private string ProcessFileImpl() { string tempFile = null; var extension = Path.GetExtension(_fileName); if (extension != null && extension.ToLower() == ".gz") { tempFile = Decompress(new FileInfo(_fileName)); _fileName = tempFile; } switch (_dumpFormat) { case DumpFormatType.StatisticsPreParse: { var packets = ReadPackets(); if (packets.Count == 0) break; var firstPacket = packets.First(); var lastPacket = packets.Last(); // CSV format Trace.WriteLine(String.Format("{0};{1};{2};{3};{4};{5};{6};{7};{8}", _originalFileName, // - sniff file name firstPacket.Time, // - time of first packet lastPacket.Time, // - time of last packet (lastPacket.Time - firstPacket.Time).TotalSeconds, // - sniff duration (seconds) packets.Count, // - packet count packets.AsParallel().Sum(packet => packet.Length), // - total packets size (bytes) packets.AsParallel().Average(packet => packet.Length), // - average packet size (bytes) packets.AsParallel().Min(packet => packet.Length), // - smaller packet size (bytes) packets.AsParallel().Max(packet => packet.Length))); // - larger packet size (bytes) break; } case DumpFormatType.SniffDataOnly: case DumpFormatType.SqlOnly: case DumpFormatType.Text: case DumpFormatType.HexOnly: { var outFileName = Path.ChangeExtension(_originalFileName, null) + "_parsed.txt"; if (Utilities.FileIsInUse(outFileName) && Settings.DumpFormat != DumpFormatType.SqlOnly) { // If our dump format requires a .txt to be created, // check if we can write to that .txt before starting parsing Trace.WriteLine(string.Format("Save file {0} is in use, parsing will not be done.", outFileName)); break; } Store.Store.SQLEnabledFlags = Settings.SQLOutputFlag; File.Delete(outFileName); _stats.SetStartTime(DateTime.Now); int threadCount = Settings.Threads; if (threadCount == 0) threadCount = Environment.ProcessorCount; ThreadPool.SetMinThreads(threadCount + 2, 4); var written = false; using (var writer = (Settings.DumpFormatWithText() ? new StreamWriter(outFileName, true) : null)) { var firstRead = true; var firstWrite = true; var reader = new Reader(_fileName, _originalFileName); var pwp = new ParallelWorkProcessor<Packet>(() => // read { if (!reader.PacketReader.CanRead()) return Tuple.Create<Packet, bool>(null, true); Packet packet; var b = reader.TryRead(out packet); if (firstRead) { Trace.WriteLine(string.Format("{0}: Parsing {1} of packets. Detected version {2}", _logPrefix, Utilities.BytesToString(reader.PacketReader.GetTotalSize()), ClientVersion.VersionString)); firstRead = false; } return Tuple.Create(packet, b); }, packet => // parse { // Parse the packet, adding text to Writer and stuff to the stores if (packet.Direction == Direction.BNClientToServer || packet.Direction == Direction.BNServerToClient) Handler.ParseBattlenet(packet); else Handler.Parse(packet); // Update statistics _stats.AddByStatus(packet.Status); return packet; }, packet => // write { ShowPercentProgress("Processing...", reader.PacketReader.GetCurrentSize(), reader.PacketReader.GetTotalSize()); if (!packet.Status.HasAnyFlag(Settings.OutputFlag)) { packet.ClosePacket(); return; } written = true; if (firstWrite) { // ReSharper disable AccessToDisposedClosure if (writer != null) writer.WriteLine(GetHeader(_originalFileName)); // ReSharper restore AccessToDisposedClosure firstWrite = false; } // get packet header if necessary if (Settings.LogPacketErrors) { switch (packet.Status) { case ParsedStatus.WithErrors: _withErrorHeaders.Add(packet.GetHeader()); break; case ParsedStatus.NotParsed: _skippedHeaders.Add(packet.GetHeader()); break; case ParsedStatus.NoStructure: _noStructureHeaders.Add(packet.GetHeader()); break; } } // ReSharper disable AccessToDisposedClosure if (writer != null) { // Write to file writer.WriteLine(packet.Writer); writer.Flush(); } // ReSharper restore AccessToDisposedClosure // Close Writer, Stream - Dispose packet.ClosePacket(); }, threadCount); pwp.WaitForFinished(Timeout.Infinite); _stats.SetEndTime(DateTime.Now); } if (written) Trace.WriteLine(string.Format("{0}: Saved file to '{1}'", _logPrefix, outFileName)); else { Trace.WriteLine(string.Format("{0}: No file produced", _logPrefix)); File.Delete(outFileName); } Trace.WriteLine(string.Format("{0}: {1}", _logPrefix, _stats)); if (Settings.SQLOutputFlag != 0) WriteSQLs(); if (Settings.LogPacketErrors) WritePacketErrors(); GC.Collect(); // Force a GC collect after parsing a file. It seems to help. break; } case DumpFormatType.Pkt: { var packets = ReadPackets(); if (packets.Count == 0) break; if (Settings.FilterPacketsNum < 0) { int packetsPerSplit = Math.Abs(Settings.FilterPacketsNum); int totalPackets = packets.Count; int numberOfSplits = (int)Math.Ceiling((double)totalPackets/packetsPerSplit); for (int i = 0; i < numberOfSplits; ++i) { var fileNamePart = _originalFileName + "_part_" + (i + 1) + ".pkt"; var packetsPart = packets.Take(packetsPerSplit).ToList(); packets.RemoveRange(0, packetsPart.Count); BinaryDump(fileNamePart, packetsPart); } } else { var fileNameExcerpt = Path.ChangeExtension(_originalFileName, null) + "_excerpt.pkt"; BinaryDump(fileNameExcerpt, packets); } break; } case DumpFormatType.PktSplit: { var packets = ReadPackets(); if (packets.Count == 0) break; SplitBinaryDump(packets); break; } case DumpFormatType.PktDirectionSplit: { var packets = ReadPackets(); if (packets.Count == 0) break; DirectionSplitBinaryDump(packets); break; } case DumpFormatType.PktSessionSplit: { var packets = ReadPackets(); if (packets.Count == 0) break; SessionSplitBinaryDump(packets); break; } case DumpFormatType.CompressSniff: { if (extension == null || extension.ToLower() == ".gz") { Trace.WriteLine("Skipped compressing file {0}", _fileName); break; } var fi = new FileInfo(_fileName); Compress(fi); break; } case DumpFormatType.SniffVersionSplit: { var reader = new Reader(_fileName, _originalFileName); if (ClientVersion.IsUndefined() && reader.PacketReader.CanRead()) { Packet packet; reader.TryRead(out packet); packet.ClosePacket(); } reader.PacketReader.Dispose(); string version = ClientVersion.IsUndefined() ? "unknown" : ClientVersion.VersionString; string realFileName = _originalFileName + (_fileName != _originalFileName ? ".gz" : ""); string destPath = Path.Combine(Path.GetDirectoryName(realFileName), version, Path.GetFileName(realFileName)); string destDir = Path.GetDirectoryName(destPath); if (!Directory.Exists(destDir)) Directory.CreateDirectory(destDir); File.Move(realFileName, destPath); Trace.WriteLine("Moved " + realFileName + " to " + destPath); break; } case DumpFormatType.ConnectionIndexes: { var packets = ReadPackets(); if (packets.Count == 0) break; using (var writer = new StreamWriter(Path.ChangeExtension(_originalFileName, null) + "_connidx.txt")) { if (ClientVersion.Build <= ClientVersionBuild.V6_0_3_19342) writer.WriteLine("# Warning: versions before 6.1 might not have proper ConnectionIndex values."); IEnumerable<IGrouping<Tuple<int, Direction>, Packet>> groupsOpcode = packets .GroupBy(packet => Tuple.Create(packet.Opcode, packet.Direction)) .OrderBy(grouping => grouping.Key.Item2); foreach (IGrouping<Tuple<int, Direction>, Packet> groupOpcode in groupsOpcode) { List<IGrouping<int, Packet>> groups = groupOpcode .GroupBy(packet => packet.ConnectionIndex) .OrderBy(grouping => grouping.Key) .ToList(); writer.Write("{0} {1,-50}: ", groupOpcode.Key.Item2, Opcodes.GetOpcodeName(groupOpcode.Key.Item1, groupOpcode.Key.Item2)); for (int i = 0; i < groups.Count; i++) { int idx = groups[i].Key; writer.Write("{0} ({1}{2})", idx, (idx & 1) != 0 ? "INSTANCE" : "REALM", (idx & 2) != 0 ? "_NEW" : ""); if (i != groups.Count - 1) writer.Write(", "); } writer.WriteLine(); } } break; } default: { Trace.WriteLine(string.Format("{0}: Dump format is none, nothing will be processed.", _logPrefix)); break; } } return tempFile; }