/// <summary>
/// Checks a search string to see if any of the data objects exclude it.
/// </summary>
public bool DataExcludes(string search)
{
if (Data1.Contains(search) || Data2.Contains(search) || Data3.Contains(search) || Data4.Contains(search) || Data5.Contains(search) ||
Data6.Contains(search) || Data7.Contains(search) || Data8.Contains(search) || Data9.Contains(search) || Data10.Contains(search))
{
return(false);
}
return(true);
}
public static (long, long) FindSig(IReader rdr)
{
byte prev0 = 0;
byte prev1 = 0;
long ix = 0;
long start;
Data32LE length = new Data32LE();
while (true)
{
Data8 d = new Data8();
d.Read(rdr);
if (prev0 == 0x23 && d.Value == 0xA3)
{
string code = prev1.ToString("X2");
Parser.Dumper.OnInfo($"Found sig with code: {code}");
var pos = rdr.Position;
start = pos - (4 + 4 + 4 + 3); // Lenght, Filing, Type + 3 bytes from signature
rdr.GoTo(start);
length.Read(rdr);
rdr.GoTo(start);
return(start, length.Value);
}
if (prev1 == 0x50 && prev0 == 0x3B && d.Value == 0xC1)
{
Parser.Dumper.OnInfo($"Found sig for Long Message format");
var pos = rdr.Position;
start = pos - (4 + 4 + 4 + 3); // Lenght, Filing, Type + 3 bytes from signature
rdr.GoTo(start);
length.Read(rdr);
rdr.GoTo(start);
return(start, length.Value);
}
// Go on scanning
prev1 = prev0;
prev0 = d.Value;
ix++;
if (ix % 256 * 16 == 0)
{
Parser.Dumper.OnInfo("");
}
}
}
public ComWithS71200DB()
{
TypeList.Add(Data0.GetType().ToString());
TypeList.Add(Data1.GetType().ToString());
TypeList.Add(Data2.GetType().ToString());
TypeList.Add(Data3.GetType().ToString());
TypeList.Add(Data4.GetType().ToString());
TypeList.Add(Data5.GetType().ToString());
TypeList.Add(Data6.GetType().ToString());
TypeList.Add(Data7.GetType().ToString());
TypeList.Add(Data8.GetType().ToString());
TypeList.Add(Data9.GetType().ToString());
TypeList.Add(Data10.GetType().ToString());
TypeList.Add(Data11.GetType().ToString());
TypeList.Add(Data12.GetType().ToString());
TypeList.Add(Data13.GetType().ToString());
TypeList.Add(Data14.GetType().ToString());
TypeList.Add(Data15.GetType().ToString());
TypeList.Add(Data16.GetType().ToString());
TypeList.Add(Data17.GetType().ToString());
TypeList.Add(Data18.GetType().ToString());
TypeList.Add(Data19.GetType().ToString());
TypeList.Add(Data20.GetType().ToString());
TypeList.Add(Data21.GetType().ToString());
TypeList.Add(Data22.GetType().ToString());
TypeList.Add(Data23.GetType().ToString());
TypeList.Add(Data24.GetType().ToString());
TypeList.Add(Data25.GetType().ToString());
TypeList.Add(Data26.GetType().ToString());
TypeList.Add(Data27.GetType().ToString());
TypeList.Add(Data28.GetType().ToString());
TypeList.Add(Data29.GetType().ToString());
TypeList.Add(Data30.GetType().ToString());
TypeList.Add(Data31.GetType().ToString());
TypeList.Add(Data32.GetType().ToString());
TypeList.Add(Data33.GetType().ToString());
TypeList.Add(Data34.GetType().ToString());
TypeList.Add(Data35.GetType().ToString());
TypeList.Add(Data36.GetType().ToString());
TypeList.Add(Data37.GetType().ToString());
TypeList.Add(Data38.GetType().ToString());
TypeList.Add(Data39.GetType().ToString());
TypeList.Add(Data40.GetType().ToString());
TypeList.Add(Data41.GetType().ToString());
TypeList.Add(Data42.GetType().ToString());
TypeList.Add(Data43.GetType().ToString());
TypeList.Add(Data44.GetType().ToString());
TypeList.Add(Data45.GetType().ToString());
TypeList.Add(Data46.GetType().ToString());
TypeList.Add(Data47.GetType().ToString());
TypeList.Add(Data48.GetType().ToString());
TypeList.Add(Data49.GetType().ToString());
TypeList.Add(Data50.GetType().ToString());
TypeList.Add(Data51.GetType().ToString());
TypeList.Add(Data52.GetType().ToString());
TypeList.Add(Data53.GetType().ToString());
TypeList.Add(Data54.GetType().ToString());
TypeList.Add(Data55.GetType().ToString());
TypeList.Add(Data56.GetType().ToString());
TypeList.Add(Data57.GetType().ToString());
TypeList.Add(Data58.GetType().ToString());
TypeList.Add(Data59.GetType().ToString());
TypeList.Add(Data60.GetType().ToString());
TypeList.Add(Data61.GetType().ToString());
TypeList.Add(Data62.GetType().ToString());
TypeList.Add(Data63.GetType().ToString());
TypeList.Add(Data64.GetType().ToString());
TypeList.Add(Data65.GetType().ToString());
TypeList.Add(Data66.GetType().ToString());
TypeList.Add(Data67.GetType().ToString());
TypeList.Add(Data68.GetType().ToString());
TypeList.Add(Data69.GetType().ToString());
TypeList.Add(Data70.GetType().ToString());
TypeList.Add(Data71.GetType().ToString());
TypeList.Add(Data72.GetType().ToString());
TypeList.Add(Data73.GetType().ToString());
TypeList.Add(Data74.GetType().ToString());
TypeList.Add(Data75.GetType().ToString());
TypeList.Add(Data76.GetType().ToString());
TypeList.Add(Data77.GetType().ToString());
TypeList.Add(Data78.GetType().ToString());
TypeList.Add(Data79.GetType().ToString());
TypeList.Add(Data80.GetType().ToString());
TypeList.Add(Data81.GetType().ToString());
TypeList.Add(Data82.GetType().ToString());
TypeList.Add(Data83.GetType().ToString());
TypeList.Add(Data84.GetType().ToString());
TypeList.Add(Data85.GetType().ToString());
TypeList.Add(Data86.GetType().ToString());
TypeList.Add(Data87.GetType().ToString());
TypeList.Add(Data88.GetType().ToString());
TypeList.Add(Data89.GetType().ToString());
TypeList.Add(Data90.GetType().ToString());
TypeList.Add(Data91.GetType().ToString());
TypeList.Add(Data92.GetType().ToString());
TypeList.Add(Data93.GetType().ToString());
TypeList.Add(Data94.GetType().ToString());
TypeList.Add(Data95.GetType().ToString());
TypeList.Add(Data96.GetType().ToString());
TypeList.Add(Data97.GetType().ToString());
TypeList.Add(Data98.GetType().ToString());
TypeList.Add(Data99.GetType().ToString());
TypeList.Add(Data100.GetType().ToString());
DataList.Add(Data0);
DataList.Add(Data1);
DataList.Add(Data2);
DataList.Add(Data3);
DataList.Add(Data4);
DataList.Add(Data5);
DataList.Add(Data6);
DataList.Add(Data7);
DataList.Add(Data8);
DataList.Add(Data9);
DataList.Add(Data10);
DataList.Add(Data11);
DataList.Add(Data12);
DataList.Add(Data13);
DataList.Add(Data14);
DataList.Add(Data15);
DataList.Add(Data16);
DataList.Add(Data17);
DataList.Add(Data18);
DataList.Add(Data19);
DataList.Add(Data20);
DataList.Add(Data21);
DataList.Add(Data22);
DataList.Add(Data23);
DataList.Add(Data24);
DataList.Add(Data25);
DataList.Add(Data26);
DataList.Add(Data27);
DataList.Add(Data28);
DataList.Add(Data29);
DataList.Add(Data30);
DataList.Add(Data31);
DataList.Add(Data32);
DataList.Add(Data33);
DataList.Add(Data34);
DataList.Add(Data35);
DataList.Add(Data36);
DataList.Add(Data37);
DataList.Add(Data38);
DataList.Add(Data39);
DataList.Add(Data40);
DataList.Add(Data41);
DataList.Add(Data42);
DataList.Add(Data43);
DataList.Add(Data44);
DataList.Add(Data45);
DataList.Add(Data46);
DataList.Add(Data47);
DataList.Add(Data48);
DataList.Add(Data49);
DataList.Add(Data50);
DataList.Add(Data51);
DataList.Add(Data52);
DataList.Add(Data53);
DataList.Add(Data54);
DataList.Add(Data55);
DataList.Add(Data56);
DataList.Add(Data57);
DataList.Add(Data58);
DataList.Add(Data59);
DataList.Add(Data60);
DataList.Add(Data61);
DataList.Add(Data62);
DataList.Add(Data63);
DataList.Add(Data64);
DataList.Add(Data65);
DataList.Add(Data66);
DataList.Add(Data67);
DataList.Add(Data68);
DataList.Add(Data69);
DataList.Add(Data70);
DataList.Add(Data71);
DataList.Add(Data72);
DataList.Add(Data73);
DataList.Add(Data74);
DataList.Add(Data75);
DataList.Add(Data76);
DataList.Add(Data77);
DataList.Add(Data78);
DataList.Add(Data79);
DataList.Add(Data80);
DataList.Add(Data81);
DataList.Add(Data82);
DataList.Add(Data83);
DataList.Add(Data84);
DataList.Add(Data85);
DataList.Add(Data86);
DataList.Add(Data87);
DataList.Add(Data88);
DataList.Add(Data89);
DataList.Add(Data90);
DataList.Add(Data91);
DataList.Add(Data92);
DataList.Add(Data93);
DataList.Add(Data94);
DataList.Add(Data95);
DataList.Add(Data96);
DataList.Add(Data97);
DataList.Add(Data98);
DataList.Add(Data99);
DataList.Add(Data100);
}
public override void AfterAutomaticRead(IReader rdr)
{
// Repeat
Data16LE separator4 = new Data16LE();;
Data32LE properties = new Data32LE();;
// Repeat-repeat
AsciiZ name = new AsciiZ();
Data8 propType = new Data8();
Data8 data8 = new Data8();
Data16LE data16 = new Data16LE();
Data32LE data32 = new Data32LE();
AsciiZ textData = new AsciiZ();
ByteArray binData = new ByteArray();
for (int block = 0; block < propertyBlocks.Value; block++)
{
separator4.Read(rdr);
properties.Read(rdr);
for (int i = 0; i < properties.Value; i++)
{
name.Read(rdr);
var propName = name.Value;
propType.Read(rdr);
string value;
switch (propType.Value)
{
case 0x64:
case 0x65:
data8.Read(rdr);
value = data8.Value.ToString("X2");
break;
case 0x66:
case 0x67:
data16.Read(rdr);
value = data16.Value.ToString("D");
break;
case 0x68:
case 0x69:
data32.Read(rdr);
if (propName == "UIN")
{
UIN = data32.Value;
}
value = data32.Value.ToString("D");
break;
case 0x6B:
textData.Read(rdr);
value = textData.Value;
if (propName == "NickName" || propName == "MyDefinedHandle")
{
Nickname = value;
}
break;
case 0x6D:
data32.Read(rdr); // Count
data8.Read(rdr); // Type
value = "(sublist) : ";
if (data8.Value == 0x6B)
{
for (int j = 0; j < data32.Value; j++)
{
textData.Read(rdr);
value += textData.Value + ", ";
}
}
else
{
if (Parser.Debug)
{
Parser.Dumper.OnInfo($"Property type sublist, subtype 6D-{data8.Value.ToString("X2")} is Not Yet Implemented");
}
// throw new NotImplementedException($"Property type sublist, subtype 6D-{data8.Value.ToString("X2")} is Not Yet Implemented");
}
break;
case 0x6F:
data32.Read(rdr); // Length
binData.Length = data32.Value;
binData.Read(rdr);
value = "(blob)";
break;
default:
value = "";
break;
}
if (string.IsNullOrEmpty(propName))
{
if (Parser.Debug)
{
Parser.Dumper.OnInfo($"(empty propname)");
}
continue;
}
Properties.Add(propName, value);
if (Parser.Debug)
{
Parser.Dumper.OnInfo($"'{propName}' : " + propType.Value.ToString("X2") + $" Value: {value}");
}
}
}
Parser.Dumper.OnInfo($"Contact UIN: {UIN} Nick: {Nickname}");
base.AfterAutomaticRead(rdr);
}
public virtual void ReadChildData(BinaryReader reader)
{
int x = 0;
_particlePhysics.ReadString(reader);
for (x = 0; (x < _data.Count); x = (x + 1))
{
Data.Add(new ByteBlockBlock());
Data[x].Read(reader);
}
for (x = 0; (x < _data.Count); x = (x + 1))
{
Data[x].ReadChildData(reader);
}
for (x = 0; (x < _data2.Count); x = (x + 1))
{
Data2.Add(new ByteBlockBlock());
Data2[x].Read(reader);
}
for (x = 0; (x < _data2.Count); x = (x + 1))
{
Data2[x].ReadChildData(reader);
}
for (x = 0; (x < _data3.Count); x = (x + 1))
{
Data3.Add(new ByteBlockBlock());
Data3[x].Read(reader);
}
for (x = 0; (x < _data3.Count); x = (x + 1))
{
Data3[x].ReadChildData(reader);
}
for (x = 0; (x < _data4.Count); x = (x + 1))
{
Data4.Add(new ByteBlockBlock());
Data4[x].Read(reader);
}
for (x = 0; (x < _data4.Count); x = (x + 1))
{
Data4[x].ReadChildData(reader);
}
for (x = 0; (x < _data5.Count); x = (x + 1))
{
Data5.Add(new ByteBlockBlock());
Data5[x].Read(reader);
}
for (x = 0; (x < _data5.Count); x = (x + 1))
{
Data5[x].ReadChildData(reader);
}
for (x = 0; (x < _data6.Count); x = (x + 1))
{
Data6.Add(new ByteBlockBlock());
Data6[x].Read(reader);
}
for (x = 0; (x < _data6.Count); x = (x + 1))
{
Data6[x].ReadChildData(reader);
}
for (x = 0; (x < _data7.Count); x = (x + 1))
{
Data7.Add(new ByteBlockBlock());
Data7[x].Read(reader);
}
for (x = 0; (x < _data7.Count); x = (x + 1))
{
Data7[x].ReadChildData(reader);
}
for (x = 0; (x < _data8.Count); x = (x + 1))
{
Data8.Add(new ByteBlockBlock());
Data8[x].Read(reader);
}
for (x = 0; (x < _data8.Count); x = (x + 1))
{
Data8[x].ReadChildData(reader);
}
for (x = 0; (x < _data9.Count); x = (x + 1))
{
Data9.Add(new ByteBlockBlock());
Data9[x].Read(reader);
}
for (x = 0; (x < _data9.Count); x = (x + 1))
{
Data9[x].ReadChildData(reader);
}
}
/// <summary>
/// Scan a DAT file and look for signatures
/// </summary>
static void SearchDat(string filename)
{
Console.WriteLine($"Searching {filename} for valid entries...");
// Statistics:
HashSet <string> unhandled = new HashSet <string>();
Dictionary <string, long> entryTypes = new Dictionary <string, long>();
Dictionary <int, long> messageTypes = new Dictionary <int, long>();
DateTime earliest = DateTime.MaxValue;
DateTime latest = DateTime.MinValue;
HashSet <string> properties = new HashSet <string>();
int duplicateContacts = 0;
int duplicateMessages = 0;
var datFile = new DatFile();
var rdr = new FileReader();
Parser.DefaultDumpFormat = DumpFormat.Hex;
//Parser.Debug = true;
HexDumperConsole console = new HexDumperConsole();
Parser.Dumper.Console = console;
rdr.Open(filename);
datFile.MainHeader.Read(rdr);
FileImport fi = new FileImport()
{
Filename = filename, ImportDate = DateTime.Now
};
_context.FileImports.Add(fi);
_context.SaveChanges();
var fileImportId = fi.Id;
byte prev0 = 0;
byte prev1 = 0;
long ix = 0;
//rdr.GoTo(0x265280);
//console.ColorSpan = new ColorSpan(ConsoleColor.Green, 0x265290, 0x265290 + 4 + 146);
try
{
while (true)
{
Data8 d = new Data8();
d.Read(rdr);
if (prev0 == 0x23 && d.Value == 0xA3)
{
string code = prev1.ToString("X2");
Parser.Dumper.OnInfo($"Found sig with code: {code}");
entryTypes[code] = entryTypes.GetValueOrDefault(code) + 1;
if (datFile.ValidSigs.Contains(prev1))
{
var pos = rdr.Position;
rdr.GoTo(pos - (4 + 4 + 4 + 3));
datFile.PolyChunk.Read(rdr);
if (datFile.PolyChunk.CurrentType == typeof(E0Entry))
{
E0Entry entry = ((E0Entry)datFile.PolyChunk.CurrentChunk);
int mtype = entry.entrySubtype.Value;
messageTypes[mtype] = messageTypes.GetValueOrDefault(mtype) + 1;
if (entry.timestamp.Value < earliest)
{
earliest = entry.timestamp.Value;
}
if (entry.timestamp.Value > latest)
{
latest = entry.timestamp.Value;
}
var msg = ToMessage((E0Entry)datFile.PolyChunk.CurrentChunk);
if (_context.Messages.Any(m => m.Hash == msg.Hash))
{
duplicateMessages++;
}
else
{
msg.FileImportId = fileImportId;
_context.Messages.Add(msg);
_context.SaveChanges();
}
}
if (datFile.PolyChunk.CurrentType == typeof(E5Entry))
{
E5Entry e = (E5Entry)datFile.PolyChunk.CurrentChunk;
e.Properties.Keys.ToList().ForEach(p => properties.Add(p));
var ct = ToContact(e);
if (_context.Contacts.Any(c => c.Hash == ct.Hash))
{
duplicateContacts++;
}
else
{
foreach (var pv in e.Properties)
{
var ctp = new ContactProperty()
{
UIN = e.UIN,
Name = pv.Key,
Value = pv.Value,
};
_context.ContactProperties.Add(ctp);
}
ct.FileImportId = fileImportId;
_context.Contacts.Add(ct);
_context.SaveChanges();
}
}
prev0 = prev1 = 0;
d.StartNew();
}
else
{
if (!unhandled.Contains(code))
{
unhandled.Add(code);
Parser.Dumper.OnInfo("Unhandled signatures: " + string.Join(", ", unhandled.ToArray()));
}
}
}
if (prev1 == 0x50 && prev0 == 0x3B && d.Value == 0xC1)
{
// Long message format
var pos = rdr.Position;
rdr.GoTo(pos - (4 + 4 + 4 + 3));
datFile.LongMessage.Read(rdr);
string code = "Long";
entryTypes[code] = entryTypes.GetValueOrDefault(code) + 1;
int mtype = datFile.LongMessage.entrySubtype.Value;
messageTypes[mtype] = messageTypes.GetValueOrDefault(mtype) + 1;
if (datFile.LongMessage.timestamp.Value < earliest)
{
earliest = datFile.LongMessage.timestamp.Value;
}
if (datFile.LongMessage.timestamp.Value > latest)
{
latest = datFile.LongMessage.timestamp.Value;
}
var msg = ToMessage(datFile.LongMessage);
if (_context.Messages.Any(m => m.Hash == msg.Hash))
{
duplicateMessages++;
}
else
{
msg.FileImportId = fileImportId;
_context.Messages.Add(msg);
_context.SaveChanges();
}
prev0 = prev1 = 0;
d.StartNew();
}
// Go on scanning
prev1 = prev0;
prev0 = d.Value;
ix++;
if (ix % 256 * 16 == 0)
{
Parser.Dumper.OnInfo("");
}
}
}
catch (ParserEOFException)
{
Parser.Dumper.OnInfo("End of file.");
}
catch (Exception ex)
{
Parser.Dumper.OnInfo("Unhandled Exception: " + ex.Message);
}
StringBuilder sb = new StringBuilder();
Console.WriteLine("");
Console.WriteLine("Statistics for file: " + filename);
sb.Append("Statistics for file: " + filename + Environment.NewLine);
Console.WriteLine("");
Console.WriteLine("Unhandled signatures: " + string.Join(", ", unhandled.ToArray()));
sb.Append("Unhandled signatures: " + string.Join(", ", unhandled.ToArray()));
Console.WriteLine("");
Console.WriteLine("Duplicated Contacts: " + duplicateContacts);
sb.Append("Duplicated Contacts: " + duplicateContacts);
Console.WriteLine("");
Console.WriteLine("Duplicated Messages: " + duplicateMessages);
sb.Append("Duplicated Messages: " + duplicateMessages);
Console.WriteLine("");
Console.WriteLine("Earliest message: " + earliest.ToString("yyyy-MM-dd HH:mm:ss"));
Console.WriteLine("Latest message : " + latest.ToString("yyyy-MM-dd HH:mm:ss"));
sb.Append("Latest message : " + latest.ToString("yyyy-MM-dd HH:mm:ss") + "\n");
sb.Append("Earliest message: " + earliest.ToString("yyyy-MM-dd HH:mm:ss") + "\n");
Console.WriteLine("Entry types:\n" + entryTypes.Dump());
sb.Append("Entry types:\n" + entryTypes.Dump());
Console.WriteLine("");
Console.WriteLine("Message types:\n" + messageTypes.Dump());
sb.Append("Message types:\n" + messageTypes.Dump());
Console.WriteLine("");
Console.WriteLine("Contact properties: " + string.Join(", ", properties.ToArray()));
fi.Statistics = sb.ToString();
_context.Entry(fi).State = EntityState.Modified;
_context.SaveChanges();
}
