private static void Main(string[] args) { _rng = new Random(34324); int threadCount = 8; _maxBlocks = 200; ThreadPool.SetMinThreads(threadCount + 2, 4); // Kludge to prevent slow thread startup. _numBlocks = _maxBlocks; var stopwatch = Stopwatch.StartNew(); var processor = new ParallelWorkProcessor <byte[]>(read, process, write, threadCount); processor.WaitForFinished(Timeout.Infinite); Console.WriteLine("\n\nFinished in " + stopwatch.Elapsed + "\n\n"); }
private void ProcessFileImpl() { if (_compression != FileCompression.None) { _tempName = Decompress(); } switch (_dumpFormat) { case DumpFormatType.StatisticsPreParse: { var packets = ReadPackets(); if (packets.Count == 0) { break; } var firstPacket = packets.First(); var lastPacket = packets.Last(); // CSV format // ReSharper disable once UseStringInterpolation Trace.WriteLine(string.Format("{0};{1};{2};{3};{4};{5};{6};{7};{8}", FileName, // - sniff file name firstPacket.Time, // - time of first packet lastPacket.Time, // - time of last packet (lastPacket.Time - firstPacket.Time).TotalSeconds, // - sniff duration (seconds) packets.Count, // - packet count packets.AsParallel().Sum(packet => packet.Length), // - total packets size (bytes) packets.AsParallel().Average(packet => packet.Length), // - average packet size (bytes) packets.AsParallel().Min(packet => packet.Length), // - smaller packet size (bytes) packets.AsParallel().Max(packet => packet.Length))); // - larger packet size (bytes) break; } case DumpFormatType.SniffDataOnly: case DumpFormatType.SqlOnly: case DumpFormatType.Text: case DumpFormatType.HexOnly: { var outFileName = Path.ChangeExtension(FileName, null) + "_parsed.txt"; if (Utilities.FileIsInUse(outFileName) && Settings.DumpFormat != DumpFormatType.SqlOnly) { // If our dump format requires a .txt to be created, // check if we can write to that .txt before starting parsing Trace.WriteLine($"Save file {outFileName} is in use, parsing will not be done."); break; } Store.Store.SQLEnabledFlags = Settings.SQLOutputFlag; File.Delete(outFileName); _stats.SetStartTime(DateTime.Now); var threadCount = Settings.Threads; if (threadCount == 0) { threadCount = Environment.ProcessorCount; } ThreadPool.SetMinThreads(threadCount + 2, 4); var written = false; using (var writer = (Settings.DumpFormatWithText() ? new StreamWriter(outFileName, true) : null)) { var firstRead = true; var firstWrite = true; var reader = _compression != FileCompression.None ? new Reader(_tempName, _sniffType) : new Reader(FileName, _sniffType); var pwp = new ParallelWorkProcessor <Packet>(() => // read { if (!reader.PacketReader.CanRead()) { return(Tuple.Create <Packet, bool>(null, true)); } Packet packet; var b = reader.TryRead(out packet); if (firstRead) { Trace.WriteLine( $"{_logPrefix}: Parsing {Utilities.BytesToString(reader.PacketReader.GetTotalSize())} of packets. Detected version {ClientVersion.VersionString}"); firstRead = false; } return(Tuple.Create(packet, b)); }, packet => // parse { // Parse the packet, adding text to Writer and stuff to the stores if (packet.Direction == Direction.BNClientToServer || packet.Direction == Direction.BNServerToClient) { BattlenetHandler.ParseBattlenet(packet); } else { Handler.Parse(packet); } // Update statistics _stats.AddByStatus(packet.Status); return(packet); }, packet => // write { if (!Console.IsOutputRedirected) { ShowPercentProgress("Processing...", reader.PacketReader.GetCurrentSize(), reader.PacketReader.GetTotalSize()); } if (!packet.Status.HasAnyFlag(Settings.OutputFlag) || !packet.WriteToFile) { packet.ClosePacket(); return; } written = true; if (firstWrite) { // ReSharper disable AccessToDisposedClosure writer?.WriteLine(GetHeader(FileName)); // ReSharper restore AccessToDisposedClosure firstWrite = false; } // get packet header if necessary if (Settings.LogPacketErrors) { switch (packet.Status) { case ParsedStatus.WithErrors: _withErrorHeaders.Add(packet.GetHeader()); break; case ParsedStatus.NotParsed: _skippedHeaders.Add(packet.GetHeader()); break; case ParsedStatus.NoStructure: _noStructureHeaders.Add(packet.GetHeader()); break; } } // ReSharper disable AccessToDisposedClosure if (writer != null) { // Write to file writer.WriteLine(packet.Writer); writer.Flush(); } // ReSharper restore AccessToDisposedClosure // Close Writer, Stream - Dispose packet.ClosePacket(); }, threadCount); pwp.WaitForFinished(Timeout.Infinite); reader.PacketReader.Dispose(); _stats.SetEndTime(DateTime.Now); } if (Settings.DumpFormatWithText()) { if (written) { Trace.WriteLine($"{_logPrefix}: Saved file to '{outFileName}'"); } else { Trace.WriteLine($"{_logPrefix}: No file produced"); File.Delete(outFileName); } } Trace.WriteLine($"{_logPrefix}: {_stats}"); if (Settings.SQLOutputFlag != 0 || HotfixSettings.Instance.ShouldLog()) { WriteSQLs(); } if (Settings.LogPacketErrors) { WritePacketErrors(); } GC.Collect(); // Force a GC collect after parsing a file. It seems to help. break; } case DumpFormatType.Pkt: { var packets = ReadPackets(); if (packets.Count == 0) { break; } if (Settings.FilterPacketsNum < 0) { var packetsPerSplit = Math.Abs(Settings.FilterPacketsNum); var totalPackets = packets.Count; var numberOfSplits = (int)Math.Ceiling((double)totalPackets / packetsPerSplit); for (var i = 0; i < numberOfSplits; ++i) { var fileNamePart = FileName + "_part_" + (i + 1) + ".pkt"; var packetsPart = packets.Take(packetsPerSplit).ToList(); packets.RemoveRange(0, packetsPart.Count); BinaryDump(fileNamePart, packetsPart); } } else { var fileNameExcerpt = Path.ChangeExtension(FileName, null) + "_excerpt.pkt"; BinaryDump(fileNameExcerpt, packets); } break; } case DumpFormatType.PktSplit: { var packets = ReadPackets(); if (packets.Count == 0) { break; } SplitBinaryDump(packets); break; } case DumpFormatType.PktDirectionSplit: { var packets = ReadPackets(); if (packets.Count == 0) { break; } DirectionSplitBinaryDump(packets); break; } case DumpFormatType.PktSessionSplit: { var packets = ReadPackets(); if (packets.Count == 0) { break; } SessionSplitBinaryDump(packets); break; } case DumpFormatType.CompressSniff: { if (_compression != FileCompression.None) { Trace.WriteLine($"Skipped compressing file {FileName}"); break; } Compress(); break; } case DumpFormatType.SniffVersionSplit: { var reader = _compression != FileCompression.None ? new Reader(_tempName, _sniffType) : new Reader(FileName, _sniffType); if (ClientVersion.IsUndefined() && reader.PacketReader.CanRead()) { Packet packet; reader.TryRead(out packet); packet.ClosePacket(); } reader.PacketReader.Dispose(); var version = ClientVersion.IsUndefined() ? "unknown" : ClientVersion.VersionString; var realFileName = GetCompressedFileName(); var destPath = Path.Combine(Path.GetDirectoryName(realFileName), version, Path.GetFileName(realFileName)); var destDir = Path.GetDirectoryName(destPath); if (!Directory.Exists(destDir)) { Directory.CreateDirectory(destDir); } File.Move(realFileName, destPath); Trace.WriteLine("Moved " + realFileName + " to " + destPath); break; } case DumpFormatType.ConnectionIndexes: { var packets = ReadPackets(); if (packets.Count == 0) { break; } using (var writer = new StreamWriter(Path.ChangeExtension(FileName, null) + "_connidx.txt")) { if (ClientVersion.Build <= ClientVersionBuild.V6_0_3_19342) { writer.WriteLine("# Warning: versions before 6.1 might not have proper ConnectionIndex values."); } IEnumerable <IGrouping <Tuple <int, Direction>, Packet> > groupsOpcode = packets .GroupBy(packet => Tuple.Create(packet.Opcode, packet.Direction)) .OrderBy(grouping => grouping.Key.Item2); foreach (var groupOpcode in groupsOpcode) { var groups = groupOpcode .GroupBy(packet => packet.ConnectionIndex) .OrderBy(grouping => grouping.Key) .ToList(); writer.Write("{0} {1,-50}: ", groupOpcode.Key.Item2, Opcodes.GetOpcodeName(groupOpcode.Key.Item1, groupOpcode.Key.Item2)); for (var i = 0; i < groups.Count; i++) { var idx = groups[i].Key; writer.Write("{0} ({1}{2})", idx, (idx & 1) != 0 ? "INSTANCE" : "REALM", (idx & 2) != 0 ? "_NEW" : ""); if (i != groups.Count - 1) { writer.Write(", "); } } writer.WriteLine(); } } break; } case DumpFormatType.Fusion: { var packets = ReadPackets(); if (packets.Count == 0) { break; } FusionDump(packets); break; } default: { Trace.WriteLine($"{_logPrefix}: Dump format is none, nothing will be processed."); break; } } }
private void ProcessFileImpl() { if (_compression != FileCompression.None) _tempName = Decompress(); switch (_dumpFormat) { case DumpFormatType.StatisticsPreParse: { var packets = ReadPackets(); if (packets.Count == 0) break; var firstPacket = packets.First(); var lastPacket = packets.Last(); // CSV format // ReSharper disable once UseStringInterpolation Trace.WriteLine(string.Format("{0};{1};{2};{3};{4};{5};{6};{7};{8}", FileName, // - sniff file name firstPacket.Time, // - time of first packet lastPacket.Time, // - time of last packet (lastPacket.Time - firstPacket.Time).TotalSeconds, // - sniff duration (seconds) packets.Count, // - packet count packets.AsParallel().Sum(packet => packet.Length), // - total packets size (bytes) packets.AsParallel().Average(packet => packet.Length), // - average packet size (bytes) packets.AsParallel().Min(packet => packet.Length), // - smaller packet size (bytes) packets.AsParallel().Max(packet => packet.Length))); // - larger packet size (bytes) break; } case DumpFormatType.SniffDataOnly: case DumpFormatType.SqlOnly: case DumpFormatType.Text: case DumpFormatType.HexOnly: { var outFileName = Path.ChangeExtension(FileName, null) + "_parsed.txt"; if (Utilities.FileIsInUse(outFileName) && Settings.DumpFormat != DumpFormatType.SqlOnly) { // If our dump format requires a .txt to be created, // check if we can write to that .txt before starting parsing Trace.WriteLine($"Save file {outFileName} is in use, parsing will not be done."); break; } Store.Store.SQLEnabledFlags = Settings.SQLOutputFlag; File.Delete(outFileName); _stats.SetStartTime(DateTime.Now); var threadCount = Settings.Threads; if (threadCount == 0) threadCount = Environment.ProcessorCount; ThreadPool.SetMinThreads(threadCount + 2, 4); var written = false; using (var writer = (Settings.DumpFormatWithText() ? new StreamWriter(outFileName, true) : null)) { var firstRead = true; var firstWrite = true; var reader = _compression != FileCompression.None ? new Reader(_tempName, _sniffType) : new Reader(FileName, _sniffType); var pwp = new ParallelWorkProcessor<Packet>(() => // read { if (!reader.PacketReader.CanRead()) return Tuple.Create<Packet, bool>(null, true); Packet packet; var b = reader.TryRead(out packet); if (firstRead) { Trace.WriteLine( $"{_logPrefix}: Parsing {Utilities.BytesToString(reader.PacketReader.GetTotalSize())} of packets. Detected version {ClientVersion.VersionString}"); firstRead = false; } return Tuple.Create(packet, b); }, packet => // parse { // Parse the packet, adding text to Writer and stuff to the stores if (packet.Direction == Direction.BNClientToServer || packet.Direction == Direction.BNServerToClient) BattlenetHandler.ParseBattlenet(packet); else Handler.Parse(packet); // Update statistics _stats.AddByStatus(packet.Status); return packet; }, packet => // write { ShowPercentProgress("Processing...", reader.PacketReader.GetCurrentSize(), reader.PacketReader.GetTotalSize()); if (!packet.Status.HasAnyFlag(Settings.OutputFlag) || !packet.WriteToFile) { packet.ClosePacket(); return; } written = true; if (firstWrite) { // ReSharper disable AccessToDisposedClosure writer?.WriteLine(GetHeader(FileName)); // ReSharper restore AccessToDisposedClosure firstWrite = false; } // get packet header if necessary if (Settings.LogPacketErrors) { switch (packet.Status) { case ParsedStatus.WithErrors: _withErrorHeaders.Add(packet.GetHeader()); break; case ParsedStatus.NotParsed: _skippedHeaders.Add(packet.GetHeader()); break; case ParsedStatus.NoStructure: _noStructureHeaders.Add(packet.GetHeader()); break; } } // ReSharper disable AccessToDisposedClosure if (writer != null) { // Write to file writer.WriteLine(packet.Writer); writer.Flush(); } // ReSharper restore AccessToDisposedClosure // Close Writer, Stream - Dispose packet.ClosePacket(); }, threadCount); pwp.WaitForFinished(Timeout.Infinite); reader.PacketReader.Dispose(); _stats.SetEndTime(DateTime.Now); } if (written) Trace.WriteLine($"{_logPrefix}: Saved file to '{outFileName}'"); else { Trace.WriteLine($"{_logPrefix}: No file produced"); File.Delete(outFileName); } Trace.WriteLine($"{_logPrefix}: {_stats}"); if (Settings.SQLOutputFlag != 0 || HotfixSettings.Instance.ShouldLog()) WriteSQLs(); if (Settings.LogPacketErrors) WritePacketErrors(); GC.Collect(); // Force a GC collect after parsing a file. It seems to help. break; } case DumpFormatType.Pkt: { var packets = ReadPackets(); if (packets.Count == 0) break; if (Settings.FilterPacketsNum < 0) { var packetsPerSplit = Math.Abs(Settings.FilterPacketsNum); var totalPackets = packets.Count; var numberOfSplits = (int)Math.Ceiling((double)totalPackets/packetsPerSplit); for (var i = 0; i < numberOfSplits; ++i) { var fileNamePart = FileName + "_part_" + (i + 1) + ".pkt"; var packetsPart = packets.Take(packetsPerSplit).ToList(); packets.RemoveRange(0, packetsPart.Count); BinaryDump(fileNamePart, packetsPart); } } else { var fileNameExcerpt = Path.ChangeExtension(FileName, null) + "_excerpt.pkt"; BinaryDump(fileNameExcerpt, packets); } break; } case DumpFormatType.PktSplit: { var packets = ReadPackets(); if (packets.Count == 0) break; SplitBinaryDump(packets); break; } case DumpFormatType.PktDirectionSplit: { var packets = ReadPackets(); if (packets.Count == 0) break; DirectionSplitBinaryDump(packets); break; } case DumpFormatType.PktSessionSplit: { var packets = ReadPackets(); if (packets.Count == 0) break; SessionSplitBinaryDump(packets); break; } case DumpFormatType.CompressSniff: { if (_compression != FileCompression.None) { Trace.WriteLine($"Skipped compressing file {FileName}"); break; } Compress(); break; } case DumpFormatType.SniffVersionSplit: { var reader = _compression != FileCompression.None ? new Reader(_tempName, _sniffType) : new Reader(FileName, _sniffType); if (ClientVersion.IsUndefined() && reader.PacketReader.CanRead()) { Packet packet; reader.TryRead(out packet); packet.ClosePacket(); } reader.PacketReader.Dispose(); var version = ClientVersion.IsUndefined() ? "unknown" : ClientVersion.VersionString; var realFileName = GetCompressedFileName(); var destPath = Path.Combine(Path.GetDirectoryName(realFileName), version, Path.GetFileName(realFileName)); var destDir = Path.GetDirectoryName(destPath); if (!Directory.Exists(destDir)) Directory.CreateDirectory(destDir); File.Move(realFileName, destPath); Trace.WriteLine("Moved " + realFileName + " to " + destPath); break; } case DumpFormatType.ConnectionIndexes: { var packets = ReadPackets(); if (packets.Count == 0) break; using (var writer = new StreamWriter(Path.ChangeExtension(FileName, null) + "_connidx.txt")) { if (ClientVersion.Build <= ClientVersionBuild.V6_0_3_19342) writer.WriteLine("# Warning: versions before 6.1 might not have proper ConnectionIndex values."); IEnumerable<IGrouping<Tuple<int, Direction>, Packet>> groupsOpcode = packets .GroupBy(packet => Tuple.Create(packet.Opcode, packet.Direction)) .OrderBy(grouping => grouping.Key.Item2); foreach (var groupOpcode in groupsOpcode) { var groups = groupOpcode .GroupBy(packet => packet.ConnectionIndex) .OrderBy(grouping => grouping.Key) .ToList(); writer.Write("{0} {1,-50}: ", groupOpcode.Key.Item2, Opcodes.GetOpcodeName(groupOpcode.Key.Item1, groupOpcode.Key.Item2)); for (var i = 0; i < groups.Count; i++) { var idx = groups[i].Key; writer.Write("{0} ({1}{2})", idx, (idx & 1) != 0 ? "INSTANCE" : "REALM", (idx & 2) != 0 ? "_NEW" : ""); if (i != groups.Count - 1) writer.Write(", "); } writer.WriteLine(); } } break; } default: { Trace.WriteLine($"{_logPrefix}: Dump format is none, nothing will be processed."); break; } } }
private string ProcessFileImpl() { string tempFile = null; var extension = Path.GetExtension(_fileName); if (extension != null && extension.ToLower() == ".gz") { tempFile = Decompress(new FileInfo(_fileName)); _fileName = tempFile; } switch (_dumpFormat) { case DumpFormatType.StatisticsPreParse: { var packets = (LinkedList<Packet>)ReadPackets(); if (packets.Count == 0) break; // CSV format Trace.WriteLine(String.Format("{0};{1};{2};{3};{4};{5};{6};{7};{8}", _originalFileName, // - sniff file name packets.First.Value.Time, // - time of first packet packets.Last.Value.Time, // - time of last packet (packets.Last.Value.Time - packets.First.Value.Time).TotalSeconds, // - sniff duration (seconds) packets.Count, // - packet count packets.AsParallel().Sum(packet => packet.Length), // - total packets size (bytes) packets.AsParallel().Average(packet => packet.Length), // - average packet size (bytes) packets.AsParallel().Min(packet => packet.Length), // - smaller packet size (bytes) packets.AsParallel().Max(packet => packet.Length))); // - larger packet size (bytes) break; } case DumpFormatType.SniffDataOnly: case DumpFormatType.SqlOnly: case DumpFormatType.Text: { var outFileName = Path.ChangeExtension(_originalFileName, null) + "_parsed.txt"; if (Utilities.FileIsInUse(outFileName) && Settings.DumpFormat != DumpFormatType.SqlOnly) { // If our dump format requires a .txt to be created, // check if we can write to that .txt before starting parsing Trace.WriteLine(string.Format("Save file {0} is in use, parsing will not be done.", outFileName)); break; } Store.Store.SQLEnabledFlags = Settings.SQLOutputFlag; File.Delete(outFileName); _stats.SetStartTime(DateTime.Now); int threadCount = Settings.Threads; if (threadCount == 0) threadCount = Environment.ProcessorCount; ThreadPool.SetMinThreads(threadCount + 2, 4); using (var writer = (Settings.DumpFormatWithText() ? new StreamWriter(outFileName, true) : null)) { bool first = true; var reader = new Reader(_fileName, _originalFileName); var pwp = new ParallelWorkProcessor<Packet>(() => // read { if (!reader.PacketReader.CanRead()) return Tuple.Create<Packet, bool>(null, true); Packet packet; var b = reader.TryRead(out packet); if (first) { Trace.WriteLine(string.Format("{0}: Parsing {1} packets. Detected version {2}", _logPrefix, Utilities.BytesToString(reader.PacketReader.GetTotalSize()), ClientVersion.VersionString)); // ReSharper disable AccessToDisposedClosure if (writer != null) writer.WriteLine(GetHeader(_originalFileName)); // ReSharper restore AccessToDisposedClosure first = false; } return Tuple.Create(packet, b); }, packet => // parse { // Parse the packet, adding text to Writer and stuff to the stores if (packet.Direction == Direction.BNClientToServer || packet.Direction == Direction.BNServerToClient) Handler.ParseBattlenet(packet); else Handler.Parse(packet); // Update statistics _stats.AddByStatus(packet.Status); return packet; }, packet => // write { ShowPercentProgress("Processing...", reader.PacketReader.GetCurrentSize(), reader.PacketReader.GetTotalSize()); // get packet header if necessary if (Settings.LogPacketErrors) { if (packet.Status == ParsedStatus.WithErrors) _withErrorHeaders.AddLast(packet.GetHeader()); else if (packet.Status == ParsedStatus.NotParsed) _skippedHeaders.AddLast(packet.GetHeader()); } // ReSharper disable AccessToDisposedClosure if (writer != null) { // Write to file writer.WriteLine(packet.Writer); writer.Flush(); } // ReSharper restore AccessToDisposedClosure // Close Writer, Stream - Dispose packet.ClosePacket(); }, threadCount); pwp.WaitForFinished(Timeout.Infinite); _stats.SetEndTime(DateTime.Now); } Trace.WriteLine(string.Format("{0}: Saved file to '{1}'", _logPrefix, outFileName)); Trace.WriteLine(string.Format("{0}: {1}", _logPrefix, _stats)); if (Settings.SQLOutputFlag != 0) WriteSQLs(); if (Settings.LogPacketErrors) WritePacketErrors(); GC.Collect(); // Force a GC collect after parsing a file. It seems to help. break; } case DumpFormatType.Pkt: { var packets = (LinkedList<Packet>)ReadPackets(); if (packets.Count == 0) break; if (Settings.FilterPacketsNum < 0) { int packetsPerSplit = Math.Abs(Settings.FilterPacketsNum); int totalPackets = packets.Count; int numberOfSplits = totalPackets/packetsPerSplit; for (int i = 0; i < numberOfSplits; ++i) { var fileNamePart = _originalFileName + "_part_" + (i + 1) + ".pkt"; var packetsPart = new LinkedList<Packet>(); for (int j = 0; j < packetsPerSplit; ++j) { if (packets.Count == 0) break; packetsPart.AddLast(packets.First.Value); packets.RemoveFirst(); } BinaryDump(fileNamePart, packetsPart); } } else { var fileNameExcerpt = Path.ChangeExtension(_originalFileName, null) + "_excerpt.pkt"; BinaryDump(fileNameExcerpt, packets); } break; } case DumpFormatType.PktSplit: { var packets = ReadPackets(); if (packets.Count == 0) break; SplitBinaryDump(packets); break; } case DumpFormatType.PktSessionSplit: { var packets = ReadPackets(); if (packets.Count == 0) break; SessionSplitBinaryDump(packets); break; } case DumpFormatType.CompressSniff: { if (extension == null || extension.ToLower() == ".gz") { Trace.WriteLine("Skipped compressing file {0}", _fileName); break; } var fi = new FileInfo(_fileName); Compress(fi); break; } case DumpFormatType.SniffVersionSplit: { var reader = new Reader(_fileName, _originalFileName); if (ClientVersion.IsUndefined() && reader.PacketReader.CanRead()) { Packet packet; reader.TryRead(out packet); packet.ClosePacket(); } reader.PacketReader.Dispose(); string version = ClientVersion.IsUndefined() ? "unknown" : ClientVersion.VersionString; string realFileName = _originalFileName + (_fileName != _originalFileName ? ".gz" : ""); string destPath = Path.Combine(Path.GetDirectoryName(realFileName), version, Path.GetFileName(realFileName)); string destDir = Path.GetDirectoryName(destPath); if (!Directory.Exists(destDir)) Directory.CreateDirectory(destDir); File.Move(realFileName, destPath); Trace.WriteLine("Moved " + realFileName + " to " + destPath); break; } default: { Trace.WriteLine(string.Format("{0}: Dump format is none, nothing will be processed.", _logPrefix)); break; } } return tempFile; }
private string ProcessFileImpl() { string tempFile = null; var extension = Path.GetExtension(_fileName); if (extension != null && extension.ToLower() == ".gz") { tempFile = Decompress(new FileInfo(_fileName)); _fileName = tempFile; } switch (_dumpFormat) { case DumpFormatType.StatisticsPreParse: { var packets = (LinkedList <Packet>)ReadPackets(); if (packets.Count == 0) { break; } // CSV format Trace.WriteLine(String.Format("{0};{1};{2};{3};{4};{5};{6};{7};{8}", _originalFileName, // - sniff file name packets.First.Value.Time, // - time of first packet packets.Last.Value.Time, // - time of last packet (packets.Last.Value.Time - packets.First.Value.Time).TotalSeconds, // - sniff duration (seconds) packets.Count, // - packet count packets.AsParallel().Sum(packet => packet.Length), // - total packets size (bytes) packets.AsParallel().Average(packet => packet.Length), // - average packet size (bytes) packets.AsParallel().Min(packet => packet.Length), // - smaller packet size (bytes) packets.AsParallel().Max(packet => packet.Length))); // - larger packet size (bytes) break; } case DumpFormatType.SniffDataOnly: case DumpFormatType.SqlOnly: case DumpFormatType.Text: case DumpFormatType.HexOnly: { var outFileName = Path.ChangeExtension(_originalFileName, null) + "_parsed.txt"; if (Utilities.FileIsInUse(outFileName) && Settings.DumpFormat != DumpFormatType.SqlOnly) { // If our dump format requires a .txt to be created, // check if we can write to that .txt before starting parsing Trace.WriteLine(string.Format("Save file {0} is in use, parsing will not be done.", outFileName)); break; } Store.Store.SQLEnabledFlags = Settings.SQLOutputFlag; File.Delete(outFileName); _stats.SetStartTime(DateTime.Now); int threadCount = Settings.Threads; if (threadCount == 0) { threadCount = Environment.ProcessorCount; } ThreadPool.SetMinThreads(threadCount + 2, 4); using (var writer = (Settings.DumpFormatWithText() ? new StreamWriter(outFileName, true) : null)) { bool first = true; var reader = new Reader(_fileName, _originalFileName); var pwp = new ParallelWorkProcessor <Packet>(() => // read { if (!reader.PacketReader.CanRead()) { return(Tuple.Create <Packet, bool>(null, true)); } Packet packet; var b = reader.TryRead(out packet); if (first) { Trace.WriteLine(string.Format("{0}: Parsing {1} of packets. Detected version {2}", _logPrefix, Utilities.BytesToString(reader.PacketReader.GetTotalSize()), ClientVersion.VersionString)); // ReSharper disable AccessToDisposedClosure if (writer != null) { writer.WriteLine(GetHeader(_originalFileName)); } // ReSharper restore AccessToDisposedClosure first = false; } return(Tuple.Create(packet, b)); }, packet => // parse { // Parse the packet, adding text to Writer and stuff to the stores if (packet.Direction == Direction.BNClientToServer || packet.Direction == Direction.BNServerToClient) { Handler.ParseBattlenet(packet); } else { Handler.Parse(packet); } // Update statistics _stats.AddByStatus(packet.Status); return(packet); }, packet => // write { ShowPercentProgress("Processing...", reader.PacketReader.GetCurrentSize(), reader.PacketReader.GetTotalSize()); // get packet header if necessary if (Settings.LogPacketErrors) { if (packet.Status == ParsedStatus.WithErrors) { _withErrorHeaders.AddLast(packet.GetHeader()); } else if (packet.Status == ParsedStatus.NotParsed) { _skippedHeaders.AddLast(packet.GetHeader()); } } // ReSharper disable AccessToDisposedClosure if (writer != null && packet.Status.HasAnyFlag(Settings.OutputFlag)) { // Write to file writer.WriteLine(packet.Writer); writer.Flush(); } // ReSharper restore AccessToDisposedClosure // Close Writer, Stream - Dispose packet.ClosePacket(); }, threadCount); pwp.WaitForFinished(Timeout.Infinite); _stats.SetEndTime(DateTime.Now); } Trace.WriteLine(string.Format("{0}: Saved file to '{1}'", _logPrefix, outFileName)); Trace.WriteLine(string.Format("{0}: {1}", _logPrefix, _stats)); if (Settings.SQLOutputFlag != 0) { WriteSQLs(); } if (Settings.LogPacketErrors) { WritePacketErrors(); } GC.Collect(); // Force a GC collect after parsing a file. It seems to help. break; } case DumpFormatType.Pkt: { var packets = (LinkedList <Packet>)ReadPackets(); if (packets.Count == 0) { break; } if (Settings.FilterPacketsNum < 0) { int packetsPerSplit = Math.Abs(Settings.FilterPacketsNum); int totalPackets = packets.Count; int numberOfSplits = totalPackets / packetsPerSplit; for (int i = 0; i < numberOfSplits; ++i) { var fileNamePart = _originalFileName + "_part_" + (i + 1) + ".pkt"; var packetsPart = new LinkedList <Packet>(); for (int j = 0; j < packetsPerSplit; ++j) { if (packets.Count == 0) { break; } packetsPart.AddLast(packets.First.Value); packets.RemoveFirst(); } BinaryDump(fileNamePart, packetsPart); } } else { var fileNameExcerpt = Path.ChangeExtension(_originalFileName, null) + "_excerpt.pkt"; BinaryDump(fileNameExcerpt, packets); } break; } case DumpFormatType.PktSplit: { var packets = ReadPackets(); if (packets.Count == 0) { break; } SplitBinaryDump(packets); break; } case DumpFormatType.PktSessionSplit: { var packets = ReadPackets(); if (packets.Count == 0) { break; } SessionSplitBinaryDump(packets); break; } case DumpFormatType.CompressSniff: { if (extension == null || extension.ToLower() == ".gz") { Trace.WriteLine("Skipped compressing file {0}", _fileName); break; } var fi = new FileInfo(_fileName); Compress(fi); break; } case DumpFormatType.SniffVersionSplit: { var reader = new Reader(_fileName, _originalFileName); if (ClientVersion.IsUndefined() && reader.PacketReader.CanRead()) { Packet packet; reader.TryRead(out packet); packet.ClosePacket(); } reader.PacketReader.Dispose(); string version = ClientVersion.IsUndefined() ? "unknown" : ClientVersion.VersionString; string realFileName = _originalFileName + (_fileName != _originalFileName ? ".gz" : ""); string destPath = Path.Combine(Path.GetDirectoryName(realFileName), version, Path.GetFileName(realFileName)); string destDir = Path.GetDirectoryName(destPath); if (!Directory.Exists(destDir)) { Directory.CreateDirectory(destDir); } File.Move(realFileName, destPath); Trace.WriteLine("Moved " + realFileName + " to " + destPath); break; } default: { Trace.WriteLine(string.Format("{0}: Dump format is none, nothing will be processed.", _logPrefix)); break; } } return(tempFile); }
private string ProcessFileImpl() { string tempFile = null; var extension = Path.GetExtension(_fileName); if (extension != null && extension.ToLower() == ".gz") { tempFile = Decompress(new FileInfo(_fileName)); _fileName = tempFile; } switch (_dumpFormat) { case DumpFormatType.StatisticsPreParse: { ReadPackets(); if (_stats.GetCount() == 0) { break; } var firstPacket = _stats.GetFirstPacket(); var lastPacket = _stats.GetLastPacket(); // CSV format Trace.WriteLine(String.Format("{0};{1};{2};{3};{4};{5};{6};{7};{8}", _originalFileName, // - sniff file name firstPacket.Time, // - time of first packet lastPacket.Time, // - time of last packet (lastPacket.Time - firstPacket.Time).TotalSeconds, // - sniff duration (seconds) _stats.GetCount(), // - packet count _stats.GetTotalLength(), // - total packets size (bytes) _stats.GetTotalLength() / _stats.GetCount(), // - average packet size (bytes) _stats.GetMinLength(), // - smaller packet size (bytes) _stats.GetMaxLenth())); // - larger packet size (bytes) Console.WriteLine("filename: {0}", _originalFileName); Console.WriteLine("fist packet time: {0}", firstPacket.Time); Console.WriteLine("last packet time: {0}", lastPacket.Time); Console.WriteLine("sniff duration (seconds): {0}", (lastPacket.Time - firstPacket.Time).TotalSeconds); Console.WriteLine("packet count: {0}", _stats.GetCount()); Console.WriteLine("total packet size (bytes): {0}", _stats.GetTotalLength()); Console.WriteLine("average packet size (bytes): {0}", _stats.GetTotalLength() / _stats.GetCount()); Console.WriteLine("minimum packet size (bytes): {0}", _stats.GetMinLength()); Console.WriteLine("maximum packet size (bytes): {0}", _stats.GetMaxLenth()); _stats.PrintOpcodeCount(); _stats.PrintDirectionCount(); break; } case DumpFormatType.SniffDataOnly: case DumpFormatType.SqlOnly: case DumpFormatType.Text: case DumpFormatType.HexOnly: { var outFileName = Path.ChangeExtension(_originalFileName, null) + "_parsed.txt"; if (Utilities.FileIsInUse(outFileName) && Settings.DumpFormat != DumpFormatType.SqlOnly) { // If our dump format requires a .txt to be created, // check if we can write to that .txt before starting parsing Trace.WriteLine(string.Format("Save file {0} is in use, parsing will not be done.", outFileName)); break; } Store.Store.SQLEnabledFlags = Settings.SQLOutputFlag; File.Delete(outFileName); _stats.SetStartTime(DateTime.Now); int threadCount = Settings.Threads; if (threadCount == 0) { threadCount = Environment.ProcessorCount; } ThreadPool.SetMinThreads(threadCount + 2, 4); var written = false; using (var writer = (Settings.DumpFormatWithText() ? new StreamWriter(outFileName, true) : null)) { var firstRead = true; var firstWrite = true; var reader = new Reader(_fileName, _originalFileName); var pwp = new ParallelWorkProcessor <Packet>(() => // read { if (!reader.PacketReader.CanRead()) { return(Tuple.Create <Packet, bool>(null, true)); } Packet packet; var b = reader.TryRead(out packet); if (firstRead) { Trace.WriteLine(string.Format("{0}: Parsing {1} of packets. Detected version {2}", _logPrefix, Utilities.BytesToString(reader.PacketReader.GetTotalSize()), ClientVersion.VersionString)); firstRead = false; } return(Tuple.Create(packet, b)); }, packet => // parse { // Parse the packet, adding text to Writer and stuff to the stores if (packet.Direction == Direction.BNClientToServer || packet.Direction == Direction.BNServerToClient) { Handler.ParseBattlenet(packet); } else { Handler.Parse(packet); } // Update statistics _stats.AddByStatus(packet.Status); return(packet); }, packet => // write { ShowPercentProgress("Processing...", reader.PacketReader.GetCurrentSize(), reader.PacketReader.GetTotalSize()); if (!packet.Status.HasAnyFlag(Settings.OutputFlag) || !packet.WriteToFile) { packet.ClosePacket(); return; } written = true; if (firstWrite) { // ReSharper disable AccessToDisposedClosure if (writer != null) { writer.WriteLine(GetHeader(_originalFileName)); } // ReSharper restore AccessToDisposedClosure firstWrite = false; } // get packet header if necessary if (Settings.LogPacketErrors) { switch (packet.Status) { case ParsedStatus.WithErrors: _withErrorHeaders.Add(packet.GetHeader()); break; case ParsedStatus.NotParsed: _skippedHeaders.Add(packet.GetHeader()); break; case ParsedStatus.NoStructure: _noStructureHeaders.Add(packet.GetHeader()); break; } } // ReSharper disable AccessToDisposedClosure if (writer != null) { // Write to file writer.WriteLine(packet.Writer); writer.Flush(); } // ReSharper restore AccessToDisposedClosure // Close Writer, Stream - Dispose packet.ClosePacket(); }, threadCount); pwp.WaitForFinished(Timeout.Infinite); _stats.SetEndTime(DateTime.Now); } if (written) { Trace.WriteLine(string.Format("{0}: Saved file to '{1}'", _logPrefix, outFileName)); } else { Trace.WriteLine(string.Format("{0}: No file produced", _logPrefix)); File.Delete(outFileName); } Trace.WriteLine(string.Format("{0}: {1}", _logPrefix, _stats)); if (Settings.SQLOutputFlag != 0) { WriteSQLs(); } if (Settings.LogPacketErrors) { WritePacketErrors(); } GC.Collect(); // Force a GC collect after parsing a file. It seems to help. break; } /*case DumpFormatType.Pkt: * { * var packets = ReadPackets(); * if (packets.Count == 0) * break; * * if (Settings.FilterPacketsNum < 0) * { * int packetsPerSplit = Math.Abs(Settings.FilterPacketsNum); * int totalPackets = packets.Count; * * int numberOfSplits = (int)Math.Ceiling((double)totalPackets/packetsPerSplit); * * for (int i = 0; i < numberOfSplits; ++i) * { * var fileNamePart = _originalFileName + "_part_" + (i + 1) + ".pkt"; * * var packetsPart = packets.Take(packetsPerSplit).ToList(); * packets.RemoveRange(0, packetsPart.Count); * * BinaryDump(fileNamePart, packetsPart); * } * } * else * { * var fileNameExcerpt = Path.ChangeExtension(_originalFileName, null) + "_excerpt.pkt"; * BinaryDump(fileNameExcerpt, packets); * } * * break; * } * case DumpFormatType.PktSplit: * { * var packets = ReadPackets(); * if (packets.Count == 0) * break; * * SplitBinaryDump(packets); * break; * } * case DumpFormatType.PktDirectionSplit: * { * var packets = ReadPackets(); * if (packets.Count == 0) * break; * * DirectionSplitBinaryDump(packets); * break; * } * case DumpFormatType.PktSessionSplit: * { * var packets = ReadPackets(); * if (packets.Count == 0) * break; * * SessionSplitBinaryDump(packets); * break; * }*/ case DumpFormatType.CompressSniff: { if (extension == null || extension.ToLower() == ".gz") { Trace.WriteLine("Skipped compressing file {0}", _fileName); break; } var fi = new FileInfo(_fileName); Compress(fi); break; } case DumpFormatType.SniffVersionSplit: { var reader = new Reader(_fileName, _originalFileName); if (ClientVersion.IsUndefined() && reader.PacketReader.CanRead()) { Packet packet; reader.TryRead(out packet); packet.ClosePacket(); } reader.PacketReader.Dispose(); string version = ClientVersion.IsUndefined() ? "unknown" : ClientVersion.VersionString; string realFileName = _originalFileName + (_fileName != _originalFileName ? ".gz" : ""); string destPath = Path.Combine(Path.GetDirectoryName(realFileName), version, Path.GetFileName(realFileName)); string destDir = Path.GetDirectoryName(destPath); if (!Directory.Exists(destDir)) { Directory.CreateDirectory(destDir); } File.Move(realFileName, destPath); Trace.WriteLine("Moved " + realFileName + " to " + destPath); break; } /*case DumpFormatType.ConnectionIndexes: * { * var packets = ReadPackets(); * if (packets.Count == 0) * break; * * using (var writer = new StreamWriter(Path.ChangeExtension(_originalFileName, null) + "_connidx.txt")) * { * if (ClientVersion.Build <= ClientVersionBuild.V6_0_3_19342) * writer.WriteLine("# Warning: versions before 6.1 might not have proper ConnectionIndex values."); * * IEnumerable<IGrouping<Tuple<int, Direction>, Packet>> groupsOpcode = packets * .GroupBy(packet => Tuple.Create(packet.Opcode, packet.Direction)) * .OrderBy(grouping => grouping.Key.Item2); * * foreach (IGrouping<Tuple<int, Direction>, Packet> groupOpcode in groupsOpcode) * { * List<IGrouping<int, Packet>> groups = groupOpcode * .GroupBy(packet => packet.ConnectionIndex) * .OrderBy(grouping => grouping.Key) * .ToList(); * * writer.Write("{0} {1,-50}: ", groupOpcode.Key.Item2, Opcodes.GetOpcodeName(groupOpcode.Key.Item1, groupOpcode.Key.Item2)); * * for (int i = 0; i < groups.Count; i++) * { * int idx = groups[i].Key; * writer.Write("{0} ({1}{2})", idx, (idx & 1) != 0 ? "INSTANCE" : "REALM", (idx & 2) != 0 ? "_NEW" : ""); * * if (i != groups.Count - 1) * writer.Write(", "); * } * * writer.WriteLine(); * } * } * * break; * }*/ default: { Trace.WriteLine(string.Format("{0}: Dump format is none, nothing will be processed.", _logPrefix)); break; } } return(tempFile); }
private void launchProcessing(int threadCount, string outFileName) { var firstRead = true; var reader = _compression != FileCompression.None ? new Reader(_tempName, _sniffType) : new Reader(FileName, _sniffType); var written = false; IDumpWriter writer = null; // this is done to ensure disposal of IDumpWriter (extended form of using(var ...) {...} try { if (Settings.DumpFormatWithText() && Settings.DumpTextFormat == TextFormatType.Txt) { writer = new TextDumpWriter(outFileName + ".txt"); } //var writer = (Settings.DumpFormatWithText() ? new StreamWriter(outFileName, true) : null else if (Settings.DumpFormatWithText() && Settings.DumpTextFormat == TextFormatType.Xml) { writer = new XmlDumpWriter(outFileName + ".xml"); } var firstWrite = true; var pwp = new ParallelWorkProcessor <Packet>(() => // read { if (!reader.PacketReader.CanRead()) { return(Tuple.Create <Packet, bool>(null, true)); } Packet packet; var b = reader.TryRead(out packet); if (firstRead) { Trace.WriteLine( $"{_logPrefix}: Parsing {Utilities.BytesToString(reader.PacketReader.GetTotalSize())} of packets. Detected version {ClientVersion.VersionString}"); firstRead = false; } return(Tuple.Create(packet, b)); }, packet => // parse { // Parse the packet, adding text to Writer and stuff to the stores if (packet.Direction == Direction.BNClientToServer || packet.Direction == Direction.BNServerToClient) { BattlenetHandler.ParseBattlenet(packet); } else { Handler.Parse(packet); } // Update statistics _stats.AddByStatus(packet.Status); return(packet); }, packet => // write { ShowPercentProgress("Processing...", reader.PacketReader.GetCurrentSize(), reader.PacketReader.GetTotalSize()); if (!packet.Status.HasAnyFlag(Settings.OutputFlag) || !packet.WriteToFile) { packet.ClosePacket(); return; } written = true; if (firstWrite) { //writer?.WriteHeader(GetHeader(FileName)); writer?.WriteHeader(FileName); firstWrite = false; } if (Settings.LogPacketErrors) { switch (packet.Status) { case ParsedStatus.WithErrors: _withErrorHeaders.Add(packet.GetHeader()); break; case ParsedStatus.NotParsed: _skippedHeaders.Add(packet.GetHeader()); break; case ParsedStatus.NoStructure: _noStructureHeaders.Add(packet.GetHeader()); break; } } if (writer != null) { // Write to file writer.WriteItem(packet); } // Close Writer, Stream - Dispose packet.ClosePacket(); }, threadCount); pwp.WaitForFinished(Timeout.Infinite); reader.PacketReader.Dispose(); _stats.SetEndTime(DateTime.Now); } finally { if (writer != null) { writer.Dispose(); } } if (written) { Trace.WriteLine($"{_logPrefix}: Saved file to '{outFileName}'"); } else { Trace.WriteLine($"{_logPrefix}: No file produced"); File.Delete(outFileName); } }