/// <summary>
/// Open a DAT file and try to parse it according to the spec
/// </summary>
static void ParseDat()
{
var datFile = new DatFile();
var rdr = new FileReader();
Parser.DefaultDumpFormat = DumpFormat.Hex;
//Parser.Debug = true;
HexDumperConsole console = new HexDumperConsole();
Parser.Dumper.Console = console;
console.DefaultColor = ConsoleColor.Green;
rdr.Open("2003_790171.dat");
datFile.MainHeader.Read(rdr);
try
{
while (true)
{
datFile.PageHeader.Read(rdr);
datFile.ReadPage(rdr, console);
if (rdr.Position != datFile.PageHeader.Next.Value)
{
Console.WriteLine("\n...\n--- JUMPING ---\n");
rdr.GoTo(datFile.PageHeader.Next.Value);
}
}
}
catch (ParserEOFException)
{
Parser.Dumper.OnInfo("End of file.");
}
// Try parsing a page
ProbeEntry entry = new ProbeEntry();
bool lastSigValid = false;
while (true)
{
if (!lastSigValid)
{
(long start, long length) = FindSig(rdr);
console.ColorSpans.Add(new ColorSpan(ConsoleColor.Green, start, start + length));
}
try
{
entry.BeforeAutomaticRead(rdr);
entry.ReadAutomatic(rdr);
entry.AfterAutomaticRead(rdr);
lastSigValid = true;
}
catch (Exception ex)
{
Debug.WriteLine(ex.Message);
lastSigValid = false;
var delta = entry.startPos + entry.length.Value - 4 - rdr.Position;
if (delta < 0)
{
Parser.Dumper.OnInfo($"Negative jump on length: {delta} bytes.");
}
else if (delta > 100)
{
Parser.Dumper.OnInfo($"Jump on length too long: {delta} bytes.");
}
else
{
entry.diffdata.Length = (uint)delta;
entry.diffdata.Read(rdr);
}
}
}
}
/// <summary>
/// Scan a DAT file and look for signatures
/// </summary>
static void SearchDat(string filename)
{
Console.WriteLine($"Searching {filename} for valid entries...");
// Statistics:
HashSet <string> unhandled = new HashSet <string>();
Dictionary <string, long> entryTypes = new Dictionary <string, long>();
Dictionary <int, long> messageTypes = new Dictionary <int, long>();
DateTime earliest = DateTime.MaxValue;
DateTime latest = DateTime.MinValue;
HashSet <string> properties = new HashSet <string>();
int duplicateContacts = 0;
int duplicateMessages = 0;
var datFile = new DatFile();
var rdr = new FileReader();
Parser.DefaultDumpFormat = DumpFormat.Hex;
//Parser.Debug = true;
HexDumperConsole console = new HexDumperConsole();
Parser.Dumper.Console = console;
rdr.Open(filename);
datFile.MainHeader.Read(rdr);
FileImport fi = new FileImport()
{
Filename = filename, ImportDate = DateTime.Now
};
_context.FileImports.Add(fi);
_context.SaveChanges();
var fileImportId = fi.Id;
byte prev0 = 0;
byte prev1 = 0;
long ix = 0;
//rdr.GoTo(0x265280);
//console.ColorSpan = new ColorSpan(ConsoleColor.Green, 0x265290, 0x265290 + 4 + 146);
try
{
while (true)
{
Data8 d = new Data8();
d.Read(rdr);
if (prev0 == 0x23 && d.Value == 0xA3)
{
string code = prev1.ToString("X2");
Parser.Dumper.OnInfo($"Found sig with code: {code}");
entryTypes[code] = entryTypes.GetValueOrDefault(code) + 1;
if (datFile.ValidSigs.Contains(prev1))
{
var pos = rdr.Position;
rdr.GoTo(pos - (4 + 4 + 4 + 3));
datFile.PolyChunk.Read(rdr);
if (datFile.PolyChunk.CurrentType == typeof(E0Entry))
{
E0Entry entry = ((E0Entry)datFile.PolyChunk.CurrentChunk);
int mtype = entry.entrySubtype.Value;
messageTypes[mtype] = messageTypes.GetValueOrDefault(mtype) + 1;
if (entry.timestamp.Value < earliest)
{
earliest = entry.timestamp.Value;
}
if (entry.timestamp.Value > latest)
{
latest = entry.timestamp.Value;
}
var msg = ToMessage((E0Entry)datFile.PolyChunk.CurrentChunk);
if (_context.Messages.Any(m => m.Hash == msg.Hash))
{
duplicateMessages++;
}
else
{
msg.FileImportId = fileImportId;
_context.Messages.Add(msg);
_context.SaveChanges();
}
}
if (datFile.PolyChunk.CurrentType == typeof(E5Entry))
{
E5Entry e = (E5Entry)datFile.PolyChunk.CurrentChunk;
e.Properties.Keys.ToList().ForEach(p => properties.Add(p));
var ct = ToContact(e);
if (_context.Contacts.Any(c => c.Hash == ct.Hash))
{
duplicateContacts++;
}
else
{
foreach (var pv in e.Properties)
{
var ctp = new ContactProperty()
{
UIN = e.UIN,
Name = pv.Key,
Value = pv.Value,
};
_context.ContactProperties.Add(ctp);
}
ct.FileImportId = fileImportId;
_context.Contacts.Add(ct);
_context.SaveChanges();
}
}
prev0 = prev1 = 0;
d.StartNew();
}
else
{
if (!unhandled.Contains(code))
{
unhandled.Add(code);
Parser.Dumper.OnInfo("Unhandled signatures: " + string.Join(", ", unhandled.ToArray()));
}
}
}
if (prev1 == 0x50 && prev0 == 0x3B && d.Value == 0xC1)
{
// Long message format
var pos = rdr.Position;
rdr.GoTo(pos - (4 + 4 + 4 + 3));
datFile.LongMessage.Read(rdr);
string code = "Long";
entryTypes[code] = entryTypes.GetValueOrDefault(code) + 1;
int mtype = datFile.LongMessage.entrySubtype.Value;
messageTypes[mtype] = messageTypes.GetValueOrDefault(mtype) + 1;
if (datFile.LongMessage.timestamp.Value < earliest)
{
earliest = datFile.LongMessage.timestamp.Value;
}
if (datFile.LongMessage.timestamp.Value > latest)
{
latest = datFile.LongMessage.timestamp.Value;
}
var msg = ToMessage(datFile.LongMessage);
if (_context.Messages.Any(m => m.Hash == msg.Hash))
{
duplicateMessages++;
}
else
{
msg.FileImportId = fileImportId;
_context.Messages.Add(msg);
_context.SaveChanges();
}
prev0 = prev1 = 0;
d.StartNew();
}
// Go on scanning
prev1 = prev0;
prev0 = d.Value;
ix++;
if (ix % 256 * 16 == 0)
{
Parser.Dumper.OnInfo("");
}
}
}
catch (ParserEOFException)
{
Parser.Dumper.OnInfo("End of file.");
}
catch (Exception ex)
{
Parser.Dumper.OnInfo("Unhandled Exception: " + ex.Message);
}
StringBuilder sb = new StringBuilder();
Console.WriteLine("");
Console.WriteLine("Statistics for file: " + filename);
sb.Append("Statistics for file: " + filename + Environment.NewLine);
Console.WriteLine("");
Console.WriteLine("Unhandled signatures: " + string.Join(", ", unhandled.ToArray()));
sb.Append("Unhandled signatures: " + string.Join(", ", unhandled.ToArray()));
Console.WriteLine("");
Console.WriteLine("Duplicated Contacts: " + duplicateContacts);
sb.Append("Duplicated Contacts: " + duplicateContacts);
Console.WriteLine("");
Console.WriteLine("Duplicated Messages: " + duplicateMessages);
sb.Append("Duplicated Messages: " + duplicateMessages);
Console.WriteLine("");
Console.WriteLine("Earliest message: " + earliest.ToString("yyyy-MM-dd HH:mm:ss"));
Console.WriteLine("Latest message : " + latest.ToString("yyyy-MM-dd HH:mm:ss"));
sb.Append("Latest message : " + latest.ToString("yyyy-MM-dd HH:mm:ss") + "\n");
sb.Append("Earliest message: " + earliest.ToString("yyyy-MM-dd HH:mm:ss") + "\n");
Console.WriteLine("Entry types:\n" + entryTypes.Dump());
sb.Append("Entry types:\n" + entryTypes.Dump());
Console.WriteLine("");
Console.WriteLine("Message types:\n" + messageTypes.Dump());
sb.Append("Message types:\n" + messageTypes.Dump());
Console.WriteLine("");
Console.WriteLine("Contact properties: " + string.Join(", ", properties.ToArray()));
fi.Statistics = sb.ToString();
_context.Entry(fi).State = EntityState.Modified;
_context.SaveChanges();
}
