Example #1
0
        /// <summary>
        /// Open a DAT file and try to parse it according to the spec
        /// </summary>
        static void ParseDat()
        {
            var datFile = new DatFile();
            var rdr     = new FileReader();

            Parser.DefaultDumpFormat = DumpFormat.Hex;
            //Parser.Debug = true;

            HexDumperConsole console = new HexDumperConsole();

            Parser.Dumper.Console = console;
            console.DefaultColor  = ConsoleColor.Green;

            rdr.Open("2003_790171.dat");

            datFile.MainHeader.Read(rdr);

            try
            {
                while (true)
                {
                    datFile.PageHeader.Read(rdr);
                    datFile.ReadPage(rdr, console);

                    if (rdr.Position != datFile.PageHeader.Next.Value)
                    {
                        Console.WriteLine("\n...\n--- JUMPING ---\n");
                        rdr.GoTo(datFile.PageHeader.Next.Value);
                    }
                }
            }
            catch (ParserEOFException)
            {
                Parser.Dumper.OnInfo("End of file.");
            }


            // Try parsing a page
            ProbeEntry entry        = new ProbeEntry();
            bool       lastSigValid = false;

            while (true)
            {
                if (!lastSigValid)
                {
                    (long start, long length) = FindSig(rdr);
                    console.ColorSpans.Add(new ColorSpan(ConsoleColor.Green, start, start + length));
                }

                try
                {
                    entry.BeforeAutomaticRead(rdr);
                    entry.ReadAutomatic(rdr);
                    entry.AfterAutomaticRead(rdr);
                    lastSigValid = true;
                }
                catch (Exception ex)
                {
                    Debug.WriteLine(ex.Message);
                    lastSigValid = false;
                    var delta = entry.startPos + entry.length.Value - 4 - rdr.Position;
                    if (delta < 0)
                    {
                        Parser.Dumper.OnInfo($"Negative jump on length: {delta} bytes.");
                    }
                    else if (delta > 100)
                    {
                        Parser.Dumper.OnInfo($"Jump on length too long: {delta} bytes.");
                    }
                    else
                    {
                        entry.diffdata.Length = (uint)delta;
                        entry.diffdata.Read(rdr);
                    }
                }
            }
        }
Example #2
0
        /// <summary>
        /// Scan a DAT file and look for signatures
        /// </summary>
        static void SearchDat(string filename)
        {
            Console.WriteLine($"Searching {filename} for valid entries...");

            // Statistics:
            HashSet <string>          unhandled    = new HashSet <string>();
            Dictionary <string, long> entryTypes   = new Dictionary <string, long>();
            Dictionary <int, long>    messageTypes = new Dictionary <int, long>();
            DateTime         earliest   = DateTime.MaxValue;
            DateTime         latest     = DateTime.MinValue;
            HashSet <string> properties = new HashSet <string>();
            int duplicateContacts       = 0;
            int duplicateMessages       = 0;

            var datFile = new DatFile();
            var rdr     = new FileReader();

            Parser.DefaultDumpFormat = DumpFormat.Hex;
            //Parser.Debug = true;

            HexDumperConsole console = new HexDumperConsole();

            Parser.Dumper.Console = console;

            rdr.Open(filename);

            datFile.MainHeader.Read(rdr);

            FileImport fi = new FileImport()
            {
                Filename = filename, ImportDate = DateTime.Now
            };

            _context.FileImports.Add(fi);
            _context.SaveChanges();

            var fileImportId = fi.Id;

            byte prev0 = 0;
            byte prev1 = 0;
            long ix    = 0;

            //rdr.GoTo(0x265280);
            //console.ColorSpan = new ColorSpan(ConsoleColor.Green, 0x265290, 0x265290 + 4 + 146);

            try
            {
                while (true)
                {
                    Data8 d = new Data8();
                    d.Read(rdr);

                    if (prev0 == 0x23 && d.Value == 0xA3)
                    {
                        string code = prev1.ToString("X2");
                        Parser.Dumper.OnInfo($"Found sig with code: {code}");
                        entryTypes[code] = entryTypes.GetValueOrDefault(code) + 1;

                        if (datFile.ValidSigs.Contains(prev1))
                        {
                            var pos = rdr.Position;
                            rdr.GoTo(pos - (4 + 4 + 4 + 3));

                            datFile.PolyChunk.Read(rdr);

                            if (datFile.PolyChunk.CurrentType == typeof(E0Entry))
                            {
                                E0Entry entry = ((E0Entry)datFile.PolyChunk.CurrentChunk);
                                int     mtype = entry.entrySubtype.Value;
                                messageTypes[mtype] = messageTypes.GetValueOrDefault(mtype) + 1;
                                if (entry.timestamp.Value < earliest)
                                {
                                    earliest = entry.timestamp.Value;
                                }
                                if (entry.timestamp.Value > latest)
                                {
                                    latest = entry.timestamp.Value;
                                }

                                var msg = ToMessage((E0Entry)datFile.PolyChunk.CurrentChunk);

                                if (_context.Messages.Any(m => m.Hash == msg.Hash))
                                {
                                    duplicateMessages++;
                                }
                                else
                                {
                                    msg.FileImportId = fileImportId;
                                    _context.Messages.Add(msg);
                                    _context.SaveChanges();
                                }
                            }

                            if (datFile.PolyChunk.CurrentType == typeof(E5Entry))
                            {
                                E5Entry e = (E5Entry)datFile.PolyChunk.CurrentChunk;

                                e.Properties.Keys.ToList().ForEach(p => properties.Add(p));

                                var ct = ToContact(e);

                                if (_context.Contacts.Any(c => c.Hash == ct.Hash))
                                {
                                    duplicateContacts++;
                                }
                                else
                                {
                                    foreach (var pv in e.Properties)
                                    {
                                        var ctp = new ContactProperty()
                                        {
                                            UIN   = e.UIN,
                                            Name  = pv.Key,
                                            Value = pv.Value,
                                        };
                                        _context.ContactProperties.Add(ctp);
                                    }

                                    ct.FileImportId = fileImportId;
                                    _context.Contacts.Add(ct);
                                    _context.SaveChanges();
                                }
                            }

                            prev0 = prev1 = 0;
                            d.StartNew();
                        }
                        else
                        {
                            if (!unhandled.Contains(code))
                            {
                                unhandled.Add(code);
                                Parser.Dumper.OnInfo("Unhandled signatures: " + string.Join(", ", unhandled.ToArray()));
                            }
                        }
                    }

                    if (prev1 == 0x50 && prev0 == 0x3B && d.Value == 0xC1)
                    {
                        // Long message format
                        var pos = rdr.Position;
                        rdr.GoTo(pos - (4 + 4 + 4 + 3));

                        datFile.LongMessage.Read(rdr);

                        string code = "Long";
                        entryTypes[code] = entryTypes.GetValueOrDefault(code) + 1;

                        int mtype = datFile.LongMessage.entrySubtype.Value;
                        messageTypes[mtype] = messageTypes.GetValueOrDefault(mtype) + 1;

                        if (datFile.LongMessage.timestamp.Value < earliest)
                        {
                            earliest = datFile.LongMessage.timestamp.Value;
                        }
                        if (datFile.LongMessage.timestamp.Value > latest)
                        {
                            latest = datFile.LongMessage.timestamp.Value;
                        }

                        var msg = ToMessage(datFile.LongMessage);
                        if (_context.Messages.Any(m => m.Hash == msg.Hash))
                        {
                            duplicateMessages++;
                        }
                        else
                        {
                            msg.FileImportId = fileImportId;
                            _context.Messages.Add(msg);
                            _context.SaveChanges();
                        }

                        prev0 = prev1 = 0;
                        d.StartNew();
                    }

                    // Go on scanning
                    prev1 = prev0;
                    prev0 = d.Value;
                    ix++;

                    if (ix % 256 * 16 == 0)
                    {
                        Parser.Dumper.OnInfo("");
                    }
                }
            }
            catch (ParserEOFException)
            {
                Parser.Dumper.OnInfo("End of file.");
            }
            catch (Exception ex)
            {
                Parser.Dumper.OnInfo("Unhandled Exception: " + ex.Message);
            }

            StringBuilder sb = new StringBuilder();

            Console.WriteLine("");
            Console.WriteLine("Statistics for file: " + filename);
            sb.Append("Statistics for file: " + filename + Environment.NewLine);

            Console.WriteLine("");
            Console.WriteLine("Unhandled signatures: " + string.Join(", ", unhandled.ToArray()));
            sb.Append("Unhandled signatures: " + string.Join(", ", unhandled.ToArray()));

            Console.WriteLine("");
            Console.WriteLine("Duplicated Contacts: " + duplicateContacts);
            sb.Append("Duplicated Contacts: " + duplicateContacts);

            Console.WriteLine("");
            Console.WriteLine("Duplicated Messages: " + duplicateMessages);
            sb.Append("Duplicated Messages: " + duplicateMessages);

            Console.WriteLine("");
            Console.WriteLine("Earliest message: " + earliest.ToString("yyyy-MM-dd HH:mm:ss"));
            Console.WriteLine("Latest message  : " + latest.ToString("yyyy-MM-dd HH:mm:ss"));
            sb.Append("Latest message  : " + latest.ToString("yyyy-MM-dd HH:mm:ss") + "\n");
            sb.Append("Earliest message: " + earliest.ToString("yyyy-MM-dd HH:mm:ss") + "\n");

            Console.WriteLine("Entry types:\n" + entryTypes.Dump());
            sb.Append("Entry types:\n" + entryTypes.Dump());

            Console.WriteLine("");
            Console.WriteLine("Message types:\n" + messageTypes.Dump());
            sb.Append("Message types:\n" + messageTypes.Dump());

            Console.WriteLine("");
            Console.WriteLine("Contact properties: " + string.Join(", ", properties.ToArray()));

            fi.Statistics            = sb.ToString();
            _context.Entry(fi).State = EntityState.Modified;
            _context.SaveChanges();
        }