public void ReadPage(FileReader rdr, HexDumperConsole console) { for (int i = 0; i < 1000; i++) { rdr.SetMilestone(); if (PageHeader.AllocationBitmap[i]) #pragma warning disable CS0642 // Possible mistaken empty statement { ; //console.ColorSpans.Add( new ColorSpan(ConsoleColor.Green, rdr.Position, rdr.Position + 63)); } #pragma warning restore CS0642 // Possible mistaken empty statement else { console.ColorSpans.Add(new ColorSpan(ConsoleColor.Gray, rdr.Position, rdr.Position + 63)); } PageHeader.Slot.Read(rdr); var ix = PageHeader.Slot.IndexOf(new byte[] { 0x23, 0xA3, 0xDB }); // Normal message types if (ix < 0) { ix = PageHeader.Slot.IndexOf(new byte[] { /*0x50,*/ 0x3B, 0xC1, 0x5C }); // Long message } if (ix >= 0) { byte type = PageHeader.Slot.Value[ix - 1]; UInt32 len = BitConverter.ToUInt32(PageHeader.Slot.Value, ix - 13); if (PageHeader.AllocationBitmap[i]) { Parser.Dumper.OnInfo($"Found a signature: {type:X2}, length: {len} ({len:X4})"); } else { Parser.Dumper.OnInfo($"Found a signature IN UNALLOCATED SPACE: {type:X2}, length: {len} ({len:X4})"); } rdr.GoToMilestone(); console.ColorSpans.Add(new ColorSpan(ConsoleColor.White, rdr.Position + ix - 13, rdr.Position + ix - 13 + len)); // +20 PageHeader.Slot.Read(rdr); /// TODO: Bufferten måste vara lika stor som chunken, inte bara en slot var memrdr = new MemoryReader(); memrdr.Open(PageHeader.Slot.Value); PolyChunk.Read(memrdr); /// } else { Parser.Dumper.OnInfo("."); } } }
public void Parsefiles(string mailboxname) { Console.WriteLine($"Parsing mailbox {mailboxname}"); var tocfilename = mailboxname + ".toc"; if (!File.Exists(tocfilename)) { throw new ApplicationException("File not found: " + tocfilename); } var mbxfilename = mailboxname + ".mbx"; if (!File.Exists(mbxfilename)) { throw new ApplicationException("File not found: " + mbxfilename); } var tocFile = new TocFile(); var rdr = new FileReader(); Parser.DefaultDumpFormat = DumpFormat.Hex; //Parser.Debug = true; HexDumperConsole console = new HexDumperConsole(); Parser.Dumper.Console = console; rdr.Open(tocfilename); tocFile.Header.Read(rdr); Console.WriteLine("\nNumber of messages: " + tocFile.Header.MessageCount.Value); tocFile.Message.Read(rdr); tocFile.Message.Read(rdr); tocFile.Message.Read(rdr); tocFile.Message.Read(rdr); }
/// <summary> /// Open a DAT file and try to parse it according to the spec /// </summary> static void ParseDat() { var datFile = new DatFile(); var rdr = new FileReader(); Parser.DefaultDumpFormat = DumpFormat.Hex; //Parser.Debug = true; HexDumperConsole console = new HexDumperConsole(); Parser.Dumper.Console = console; console.DefaultColor = ConsoleColor.Green; rdr.Open("2003_790171.dat"); datFile.MainHeader.Read(rdr); try { while (true) { datFile.PageHeader.Read(rdr); datFile.ReadPage(rdr, console); if (rdr.Position != datFile.PageHeader.Next.Value) { Console.WriteLine("\n...\n--- JUMPING ---\n"); rdr.GoTo(datFile.PageHeader.Next.Value); } } } catch (ParserEOFException) { Parser.Dumper.OnInfo("End of file."); } // Try parsing a page ProbeEntry entry = new ProbeEntry(); bool lastSigValid = false; while (true) { if (!lastSigValid) { (long start, long length) = FindSig(rdr); console.ColorSpans.Add(new ColorSpan(ConsoleColor.Green, start, start + length)); } try { entry.BeforeAutomaticRead(rdr); entry.ReadAutomatic(rdr); entry.AfterAutomaticRead(rdr); lastSigValid = true; } catch (Exception ex) { Debug.WriteLine(ex.Message); lastSigValid = false; var delta = entry.startPos + entry.length.Value - 4 - rdr.Position; if (delta < 0) { Parser.Dumper.OnInfo($"Negative jump on length: {delta} bytes."); } else if (delta > 100) { Parser.Dumper.OnInfo($"Jump on length too long: {delta} bytes."); } else { entry.diffdata.Length = (uint)delta; entry.diffdata.Read(rdr); } } } }
/// <summary> /// Scan a DAT file and look for signatures /// </summary> static void SearchDat(string filename) { Console.WriteLine($"Searching {filename} for valid entries..."); // Statistics: HashSet <string> unhandled = new HashSet <string>(); Dictionary <string, long> entryTypes = new Dictionary <string, long>(); Dictionary <int, long> messageTypes = new Dictionary <int, long>(); DateTime earliest = DateTime.MaxValue; DateTime latest = DateTime.MinValue; HashSet <string> properties = new HashSet <string>(); int duplicateContacts = 0; int duplicateMessages = 0; var datFile = new DatFile(); var rdr = new FileReader(); Parser.DefaultDumpFormat = DumpFormat.Hex; //Parser.Debug = true; HexDumperConsole console = new HexDumperConsole(); Parser.Dumper.Console = console; rdr.Open(filename); datFile.MainHeader.Read(rdr); FileImport fi = new FileImport() { Filename = filename, ImportDate = DateTime.Now }; _context.FileImports.Add(fi); _context.SaveChanges(); var fileImportId = fi.Id; byte prev0 = 0; byte prev1 = 0; long ix = 0; //rdr.GoTo(0x265280); //console.ColorSpan = new ColorSpan(ConsoleColor.Green, 0x265290, 0x265290 + 4 + 146); try { while (true) { Data8 d = new Data8(); d.Read(rdr); if (prev0 == 0x23 && d.Value == 0xA3) { string code = prev1.ToString("X2"); Parser.Dumper.OnInfo($"Found sig with code: {code}"); entryTypes[code] = entryTypes.GetValueOrDefault(code) + 1; if (datFile.ValidSigs.Contains(prev1)) { var pos = rdr.Position; rdr.GoTo(pos - (4 + 4 + 4 + 3)); datFile.PolyChunk.Read(rdr); if (datFile.PolyChunk.CurrentType == typeof(E0Entry)) { E0Entry entry = ((E0Entry)datFile.PolyChunk.CurrentChunk); int mtype = entry.entrySubtype.Value; messageTypes[mtype] = messageTypes.GetValueOrDefault(mtype) + 1; if (entry.timestamp.Value < earliest) { earliest = entry.timestamp.Value; } if (entry.timestamp.Value > latest) { latest = entry.timestamp.Value; } var msg = ToMessage((E0Entry)datFile.PolyChunk.CurrentChunk); if (_context.Messages.Any(m => m.Hash == msg.Hash)) { duplicateMessages++; } else { msg.FileImportId = fileImportId; _context.Messages.Add(msg); _context.SaveChanges(); } } if (datFile.PolyChunk.CurrentType == typeof(E5Entry)) { E5Entry e = (E5Entry)datFile.PolyChunk.CurrentChunk; e.Properties.Keys.ToList().ForEach(p => properties.Add(p)); var ct = ToContact(e); if (_context.Contacts.Any(c => c.Hash == ct.Hash)) { duplicateContacts++; } else { foreach (var pv in e.Properties) { var ctp = new ContactProperty() { UIN = e.UIN, Name = pv.Key, Value = pv.Value, }; _context.ContactProperties.Add(ctp); } ct.FileImportId = fileImportId; _context.Contacts.Add(ct); _context.SaveChanges(); } } prev0 = prev1 = 0; d.StartNew(); } else { if (!unhandled.Contains(code)) { unhandled.Add(code); Parser.Dumper.OnInfo("Unhandled signatures: " + string.Join(", ", unhandled.ToArray())); } } } if (prev1 == 0x50 && prev0 == 0x3B && d.Value == 0xC1) { // Long message format var pos = rdr.Position; rdr.GoTo(pos - (4 + 4 + 4 + 3)); datFile.LongMessage.Read(rdr); string code = "Long"; entryTypes[code] = entryTypes.GetValueOrDefault(code) + 1; int mtype = datFile.LongMessage.entrySubtype.Value; messageTypes[mtype] = messageTypes.GetValueOrDefault(mtype) + 1; if (datFile.LongMessage.timestamp.Value < earliest) { earliest = datFile.LongMessage.timestamp.Value; } if (datFile.LongMessage.timestamp.Value > latest) { latest = datFile.LongMessage.timestamp.Value; } var msg = ToMessage(datFile.LongMessage); if (_context.Messages.Any(m => m.Hash == msg.Hash)) { duplicateMessages++; } else { msg.FileImportId = fileImportId; _context.Messages.Add(msg); _context.SaveChanges(); } prev0 = prev1 = 0; d.StartNew(); } // Go on scanning prev1 = prev0; prev0 = d.Value; ix++; if (ix % 256 * 16 == 0) { Parser.Dumper.OnInfo(""); } } } catch (ParserEOFException) { Parser.Dumper.OnInfo("End of file."); } catch (Exception ex) { Parser.Dumper.OnInfo("Unhandled Exception: " + ex.Message); } StringBuilder sb = new StringBuilder(); Console.WriteLine(""); Console.WriteLine("Statistics for file: " + filename); sb.Append("Statistics for file: " + filename + Environment.NewLine); Console.WriteLine(""); Console.WriteLine("Unhandled signatures: " + string.Join(", ", unhandled.ToArray())); sb.Append("Unhandled signatures: " + string.Join(", ", unhandled.ToArray())); Console.WriteLine(""); Console.WriteLine("Duplicated Contacts: " + duplicateContacts); sb.Append("Duplicated Contacts: " + duplicateContacts); Console.WriteLine(""); Console.WriteLine("Duplicated Messages: " + duplicateMessages); sb.Append("Duplicated Messages: " + duplicateMessages); Console.WriteLine(""); Console.WriteLine("Earliest message: " + earliest.ToString("yyyy-MM-dd HH:mm:ss")); Console.WriteLine("Latest message : " + latest.ToString("yyyy-MM-dd HH:mm:ss")); sb.Append("Latest message : " + latest.ToString("yyyy-MM-dd HH:mm:ss") + "\n"); sb.Append("Earliest message: " + earliest.ToString("yyyy-MM-dd HH:mm:ss") + "\n"); Console.WriteLine("Entry types:\n" + entryTypes.Dump()); sb.Append("Entry types:\n" + entryTypes.Dump()); Console.WriteLine(""); Console.WriteLine("Message types:\n" + messageTypes.Dump()); sb.Append("Message types:\n" + messageTypes.Dump()); Console.WriteLine(""); Console.WriteLine("Contact properties: " + string.Join(", ", properties.ToArray())); fi.Statistics = sb.ToString(); _context.Entry(fi).State = EntityState.Modified; _context.SaveChanges(); }