private static DbWriter.Contact ToContact(E5Entry entry)
{
/* About, Authorize, FirstName, GroupText, GroupText1, GroupText2, HomeAddress, HomeCell, HomeCity, HomeCountry, HomeFax, HomeHomepage, HomePhone, HomeState, HomeZipCode,
* InterestText, InterestText1, InterestText2, InterestText3, LastName, MyDefinedHandle, NickName, OfflineReminderDaysCounter, OfflineReminderLastTimeCounter,
* PastText, PastText1, PastText2, PrimaryEmail, TimeZone, UIN, WorkAddress, WorkCity, WorkCompany, WorkDepartment, WorkFax, WorkHomepage, WorkJobTitle, WorkPhone, WorkState, WorkZipCode,
* BirthDayReminderDays, ClientFeatures, TrueFeatures, OnlineAlert, Age, Gender, HomepageDescription, OldEmail, SecondEmail, IP, LastKnownUpdateDate,
* BirthDay, BirthMonth, BirthYear, InterestCategory, Language, Language1, PersistentUserShowIP, InterestCategory1, InterestCategory2, InterestCategory3, Language2, Occupation,
* GroupCategory, GroupCategory1, GroupCategory2, UserKind, LastOnlineTime, LastUpdateDate, AlreadyRemindedBirthday, AcceptInAway, OnlineAlertDisable, OnlineAlertFlg,
* UserSendByServer, WorkCountry, AdditionalEmails, LastMoreInfoUpdateDate, LastMoreStatusUpdateDate, PastCategory */
var ct = new Contact()
{
UIN = entry.UIN,
Nickname = entry.Nickname
};
ct.Hash = GetLongHash(ct.UIN, ct.Nickname);
return(ct);
}
/// <summary>
/// Scan a DAT file and look for signatures
/// </summary>
static void SearchDat(string filename)
{
Console.WriteLine($"Searching {filename} for valid entries...");
// Statistics:
HashSet <string> unhandled = new HashSet <string>();
Dictionary <string, long> entryTypes = new Dictionary <string, long>();
Dictionary <int, long> messageTypes = new Dictionary <int, long>();
DateTime earliest = DateTime.MaxValue;
DateTime latest = DateTime.MinValue;
HashSet <string> properties = new HashSet <string>();
int duplicateContacts = 0;
int duplicateMessages = 0;
var datFile = new DatFile();
var rdr = new FileReader();
Parser.DefaultDumpFormat = DumpFormat.Hex;
//Parser.Debug = true;
HexDumperConsole console = new HexDumperConsole();
Parser.Dumper.Console = console;
rdr.Open(filename);
datFile.MainHeader.Read(rdr);
FileImport fi = new FileImport()
{
Filename = filename, ImportDate = DateTime.Now
};
_context.FileImports.Add(fi);
_context.SaveChanges();
var fileImportId = fi.Id;
byte prev0 = 0;
byte prev1 = 0;
long ix = 0;
//rdr.GoTo(0x265280);
//console.ColorSpan = new ColorSpan(ConsoleColor.Green, 0x265290, 0x265290 + 4 + 146);
try
{
while (true)
{
Data8 d = new Data8();
d.Read(rdr);
if (prev0 == 0x23 && d.Value == 0xA3)
{
string code = prev1.ToString("X2");
Parser.Dumper.OnInfo($"Found sig with code: {code}");
entryTypes[code] = entryTypes.GetValueOrDefault(code) + 1;
if (datFile.ValidSigs.Contains(prev1))
{
var pos = rdr.Position;
rdr.GoTo(pos - (4 + 4 + 4 + 3));
datFile.PolyChunk.Read(rdr);
if (datFile.PolyChunk.CurrentType == typeof(E0Entry))
{
E0Entry entry = ((E0Entry)datFile.PolyChunk.CurrentChunk);
int mtype = entry.entrySubtype.Value;
messageTypes[mtype] = messageTypes.GetValueOrDefault(mtype) + 1;
if (entry.timestamp.Value < earliest)
{
earliest = entry.timestamp.Value;
}
if (entry.timestamp.Value > latest)
{
latest = entry.timestamp.Value;
}
var msg = ToMessage((E0Entry)datFile.PolyChunk.CurrentChunk);
if (_context.Messages.Any(m => m.Hash == msg.Hash))
{
duplicateMessages++;
}
else
{
msg.FileImportId = fileImportId;
_context.Messages.Add(msg);
_context.SaveChanges();
}
}
if (datFile.PolyChunk.CurrentType == typeof(E5Entry))
{
E5Entry e = (E5Entry)datFile.PolyChunk.CurrentChunk;
e.Properties.Keys.ToList().ForEach(p => properties.Add(p));
var ct = ToContact(e);
if (_context.Contacts.Any(c => c.Hash == ct.Hash))
{
duplicateContacts++;
}
else
{
foreach (var pv in e.Properties)
{
var ctp = new ContactProperty()
{
UIN = e.UIN,
Name = pv.Key,
Value = pv.Value,
};
_context.ContactProperties.Add(ctp);
}
ct.FileImportId = fileImportId;
_context.Contacts.Add(ct);
_context.SaveChanges();
}
}
prev0 = prev1 = 0;
d.StartNew();
}
else
{
if (!unhandled.Contains(code))
{
unhandled.Add(code);
Parser.Dumper.OnInfo("Unhandled signatures: " + string.Join(", ", unhandled.ToArray()));
}
}
}
if (prev1 == 0x50 && prev0 == 0x3B && d.Value == 0xC1)
{
// Long message format
var pos = rdr.Position;
rdr.GoTo(pos - (4 + 4 + 4 + 3));
datFile.LongMessage.Read(rdr);
string code = "Long";
entryTypes[code] = entryTypes.GetValueOrDefault(code) + 1;
int mtype = datFile.LongMessage.entrySubtype.Value;
messageTypes[mtype] = messageTypes.GetValueOrDefault(mtype) + 1;
if (datFile.LongMessage.timestamp.Value < earliest)
{
earliest = datFile.LongMessage.timestamp.Value;
}
if (datFile.LongMessage.timestamp.Value > latest)
{
latest = datFile.LongMessage.timestamp.Value;
}
var msg = ToMessage(datFile.LongMessage);
if (_context.Messages.Any(m => m.Hash == msg.Hash))
{
duplicateMessages++;
}
else
{
msg.FileImportId = fileImportId;
_context.Messages.Add(msg);
_context.SaveChanges();
}
prev0 = prev1 = 0;
d.StartNew();
}
// Go on scanning
prev1 = prev0;
prev0 = d.Value;
ix++;
if (ix % 256 * 16 == 0)
{
Parser.Dumper.OnInfo("");
}
}
}
catch (ParserEOFException)
{
Parser.Dumper.OnInfo("End of file.");
}
catch (Exception ex)
{
Parser.Dumper.OnInfo("Unhandled Exception: " + ex.Message);
}
StringBuilder sb = new StringBuilder();
Console.WriteLine("");
Console.WriteLine("Statistics for file: " + filename);
sb.Append("Statistics for file: " + filename + Environment.NewLine);
Console.WriteLine("");
Console.WriteLine("Unhandled signatures: " + string.Join(", ", unhandled.ToArray()));
sb.Append("Unhandled signatures: " + string.Join(", ", unhandled.ToArray()));
Console.WriteLine("");
Console.WriteLine("Duplicated Contacts: " + duplicateContacts);
sb.Append("Duplicated Contacts: " + duplicateContacts);
Console.WriteLine("");
Console.WriteLine("Duplicated Messages: " + duplicateMessages);
sb.Append("Duplicated Messages: " + duplicateMessages);
Console.WriteLine("");
Console.WriteLine("Earliest message: " + earliest.ToString("yyyy-MM-dd HH:mm:ss"));
Console.WriteLine("Latest message : " + latest.ToString("yyyy-MM-dd HH:mm:ss"));
sb.Append("Latest message : " + latest.ToString("yyyy-MM-dd HH:mm:ss") + "\n");
sb.Append("Earliest message: " + earliest.ToString("yyyy-MM-dd HH:mm:ss") + "\n");
Console.WriteLine("Entry types:\n" + entryTypes.Dump());
sb.Append("Entry types:\n" + entryTypes.Dump());
Console.WriteLine("");
Console.WriteLine("Message types:\n" + messageTypes.Dump());
sb.Append("Message types:\n" + messageTypes.Dump());
Console.WriteLine("");
Console.WriteLine("Contact properties: " + string.Join(", ", properties.ToArray()));
fi.Statistics = sb.ToString();
_context.Entry(fi).State = EntityState.Modified;
_context.SaveChanges();
}