public CacheFileReader(CacheFileReader source)
{
Buffer = source.Buffer;
FileName = source.FileName;
Limit = source.Limit;
Position = source.Position;
}
public CacheFileParser(CacheFileReader reader)
{
Reader = reader;
ShareCount = 0;
ShareCursor = 0;
ShareObj = null;
ShareMap = null;
Streams = new List <SNode>();
}
public CacheFileParser(CacheFileReader reader)
{
Reader = reader;
ShareCount = 0;
ShareCursor = 0;
ShareObj = null;
ShareMap = null;
Streams = new List<SNode>();
}
private void InitWithFile(string fileName)
{
CacheFileReader cfReader = new CacheFileReader(fileName);
CacheFileParser parser = new CacheFileParser(cfReader);
parser.Parse();
SNode sNode = parser.Streams[0];
Stream = sNode.Members;
Parse();
Stream = null;
Valid = true;
}
static int Main(string[] args)
{
if (args.Length < 1)
{
Console.WriteLine("Syntax: " + Path.GetFileName(Environment.CommandLine) + " [options] [filenames+]");
Console.WriteLine("Options: --market Digest a market order file, converting it to a .CSV file");
Console.WriteLine(" --structure Print an AST of the cache file");
return -1;
}
bool dumpStructure = false;
bool dumpMarket = false;
int argsconsumed = 0;
// Parse options in simple mode
if (args.Length > 1)
{
for (int i = 0; i < args.Length; i++)
{
if (args[i] == "--market")
{
dumpMarket = true;
argsconsumed++;
}
if (args[i] == "--structure")
{
dumpStructure = true;
argsconsumed++;
}
}
}
else
dumpStructure = true; // default
for (int filelen = argsconsumed; filelen < args.Length; filelen++)
{
string fileName = args[filelen];
CacheFileReader cfr;
try { cfr = new CacheFileReader(fileName); }
catch (System.IO.FileNotFoundException) { continue; }
CacheFileParser parser = new CacheFileParser(cfr);
try
{
Console.Write("Processing {0}... ", fileName);
parser.Parse();
Console.WriteLine("SUCCESS");
}
catch (ParseException e)
{
Console.WriteLine("FAILED: " + e.Message);
}
if (dumpStructure)
SNode.DumpNodes(Path.ChangeExtension(fileName, ".structure"));
if (dumpMarket)
{
for (int i = 0; i < parser.Streams.Count; i++)
{
SNode snode = parser.Streams[i];
string csv = market(snode);
if (csv != null && csv.Length > 0)
File.WriteAllText(Path.ChangeExtension(fileName, ".market"), csv);
csv = marketHistory(snode);
if (csv != null && csv.Length > 0)
File.WriteAllText(Path.ChangeExtension(fileName, ".history"), csv);
}
}
}
return 1;
}
private void InitWithFile(string fileName)
{
CacheFileReader cfReader = new CacheFileReader(fileName);
CacheFileParser parser = new CacheFileParser(cfReader);
parser.Parse();
SNode sNode = parser.Streams[0];
Stream = sNode.Members;
Parse();
Stream = null;
Valid = true;
}
public CacheFileReader(CacheFileReader source)
{
Buffer = source.Buffer;
FileName = source.FileName;
Limit = source.Limit;
Position = source.Position;
}
public CacheFileReader(CacheFileReader source, int limit) : this(source)
{
Limit = limit + source.Position;
Position = source.Position;
}
protected SNode ParseOne()
{
EStreamCode check;
byte isshared = 0;
SNode thisobj = null;
SDBRow lastDbRow = null;
try
{
byte type = Reader.ReadByte();
check = (EStreamCode)(type & 0x3f);
isshared = (byte)(type & 0x40);
}
catch (EndOfFileException)
{
return(null);
}
#region EStreamCode Switch
switch (check)
{
case EStreamCode.EStreamStart:
break;
case EStreamCode.ENone:
thisobj = new SNone();
break;
case EStreamCode.EString:
thisobj = new SString(Reader.ReadString(Reader.ReadByte()));
break;
case EStreamCode.ELong:
thisobj = new SLong(Reader.ReadLong());
break;
case EStreamCode.EInteger:
thisobj = new SInt(Reader.ReadInt());
break;
case EStreamCode.EShort:
thisobj = new SInt(Reader.ReadShort());
break;
case EStreamCode.EByte:
thisobj = new SInt(Reader.ReadByte());
break;
case EStreamCode.ENeg1Integer:
thisobj = new SInt(-1);
break;
case EStreamCode.E0Integer:
thisobj = new SInt(0);
break;
case EStreamCode.E1Integer:
thisobj = new SInt(1);
break;
case EStreamCode.EReal:
thisobj = new SReal(Reader.ReadDouble());
break;
case EStreamCode.E0Real:
thisobj = new SReal(0);
break;
case EStreamCode.E0String:
thisobj = new SString(null);
break;
case EStreamCode.EString3:
thisobj = new SString(Reader.ReadString(1));
break;
case EStreamCode.EString4:
goto case EStreamCode.EString;
case EStreamCode.EMarker:
thisobj = new SMarker((byte)GetLength());
break;
case EStreamCode.EUnicodeString:
goto case EStreamCode.EString;
case EStreamCode.EIdent:
thisobj = new SIdent(Reader.ReadString(GetLength()));
break;
case EStreamCode.ETuple:
{
int length = GetLength();
thisobj = new STuple((uint)length);
Parse(thisobj, length);
break;
}
case EStreamCode.ETuple2:
goto case EStreamCode.ETuple;
case EStreamCode.EDict:
{
int len = (GetLength() * 2);
SDict dict = new SDict((uint)len);
thisobj = dict;
Parse(dict, len);
break;
}
case EStreamCode.EObject:
thisobj = new SObject();
Parse(thisobj, 2);
break;
case EStreamCode.ESharedObj:
thisobj = ShareGet(GetLength());
break;
case EStreamCode.EChecksum:
thisobj = new SString("checksum");
Reader.ReadInt();
break;
case EStreamCode.EBoolTrue:
thisobj = new SInt(1);
break;
case EStreamCode.EBoolFalse:
thisobj = new SInt(0);
break;
case EStreamCode.EObject22:
{
SObject obj = new SObject();
thisobj = obj;
Parse(thisobj, 1);
string oclass = obj.Name;
if (oclass == "dbutil.RowList")
{
SNode row;
while ((row = ParseOne()) != null)
{
obj.AddMember(row);
}
}
break;
}
case EStreamCode.EObject23:
goto case EStreamCode.EObject22;
case EStreamCode.E0Tuple:
thisobj = new STuple(0);
break;
case EStreamCode.E1Tuple:
thisobj = new STuple(1);
Parse(thisobj, 1);
break;
case EStreamCode.E0Tuple2:
goto case EStreamCode.E0Tuple;
case EStreamCode.E1Tuple2:
goto case EStreamCode.E1Tuple;
case EStreamCode.EEmptyString:
thisobj = new SString(string.Empty);
break;
case EStreamCode.EUnicodeString2:
/* Single unicode character */
thisobj = new SString(Reader.ReadString(2));
break;
case EStreamCode.ECompressedRow:
thisobj = GetDBRow();
break;
case EStreamCode.ESubstream:
{
int len = GetLength();
CacheFileReader readerSub = new CacheFileReader(Reader, len);
SSubStream ss = new SSubStream(len);
thisobj = ss;
CacheFileParser sp = new CacheFileParser(readerSub);
sp.Parse();
for (int i = 0; i < sp.Streams.Count; i++)
{
ss.AddMember(sp.Streams[i].Clone());
}
Reader.Seek(readerSub.Position, SeekOrigin.Begin);
break;
}
case EStreamCode.E2Tuple:
thisobj = new STuple(2);
Parse(thisobj, 2);
break;
case EStreamCode.EString2:
goto case EStreamCode.EString;
case EStreamCode.ESizedInt:
switch (Reader.ReadByte())
{
case 8:
thisobj = new SLong(Reader.ReadLong());
break;
case 4:
thisobj = new SInt(Reader.ReadInt());
break;
case 3:
// The following seems more correct than the forumla used.
// int value = (Reader.Char() << 16) + (Reader.ReadByte());
thisobj = new SInt((Reader.ReadByte()) + (Reader.ReadByte() << 16));
break;
case 2:
thisobj = new SInt(Reader.ReadShort());
break;
}
break;
case (EStreamCode)0x2d:
if (Reader.ReadByte() != (byte)0x2d)
{
throw new ParseException("Didn't encounter a double 0x2d where one was expected at " + (Reader.Position - 2));
}
else if (lastDbRow != null)
{
lastDbRow.IsLast = true;
}
return(null);
case 0:
break;
default:
throw new ParseException("Can't identify type " + String.Format("{0:x2}", (int)check) +
" at position " + String.Format("{0:x2}", Reader.Position) + " limit " + Reader.Length);
}
#endregion
if (thisobj == null)
{
throw new ParseException("no thisobj in parseone");
}
if (isshared != 0)
{
if (thisobj == null)
{
throw new ParseException("shared flag but no obj");
}
ShareAdd(thisobj);
}
return(thisobj);
}
protected SNode GetDBRow()
{
SNode nhead = ParseOne();
// get header
SObject head = nhead as SObject;
if (head == null)
{
throw new ParseException("The DBRow header isn't present...");
}
if (head.Name != "blue.DBRowDescriptor")
{
throw new ParseException("Bad descriptor name");
}
STuple fields = head.Members[0].Members[1].Members[0] as STuple;
int len = GetLength();
byte[] olddata = Reader.ReadBytes(len);
List <byte> newdata = new List <byte>();
rle_unpack(olddata, newdata);
SNode body = new SDBRow(17, newdata);
CacheFileReader blob = new CacheFileReader(newdata.ToArray());
SDict dict = new SDict(999999); // TODO: need dynamic sized dict
int step = 1;
while (step < 6)
{
foreach (SNode field in fields.Members)
{
SNode fieldName = field.Members[0];
SInt fieldType = field.Members[1] as SInt;
int fieldTypeInt = fieldType.Value;
byte boolcount = 0;
byte boolbuf = 0;
SNode obj = null;
switch (fieldTypeInt)
{
case 2: // 16bit int
if (step == 3)
{
obj = new SInt(blob.ReadShort());
}
break;
case 3: // 32bit int
if (step == 2)
{
obj = new SInt(blob.ReadInt());
}
break;
case 4:
obj = new SReal(blob.ReadFloat());
break;
case 5: // double
if (step == 1)
{
obj = new SReal(blob.ReadDouble());
}
break;
case 6: // currency
if (step == 1)
{
obj = new SLong(blob.ReadLong());
}
break;
case 11: // boolean
if (step == 5)
{
if (boolcount == 0)
{
boolbuf = blob.ReadByte();
boolcount = 0x1;
}
if (boolbuf != 0 && boolcount != 0)
{
obj = new SInt(1);
}
else
{
obj = new SInt(0);
}
boolcount <<= 1;
}
break;
case 16:
obj = new SInt(blob.ReadByte());
break;
case 17:
goto case 16;
case 18: // 16bit int
goto case 2;
case 19: // 32bit int
goto case 3;
case 20: // 64bit int
goto case 6;
case 21: // 64bit int
goto case 6;
case 64: // timestamp
goto case 6;
case 128: // string types
case 129:
case 130:
obj = new SString("I can't parse strings yet - be patient");
break;
default:
throw new ParseException("Unhandled ADO type " + fieldTypeInt);
}
if (obj != null)
{
dict.AddMember(obj);
dict.AddMember(fieldName.Clone());
}
}
step++;
}
SNode fakerow = new STuple(3);
fakerow.AddMember(head);
fakerow.AddMember(body);
fakerow.AddMember(dict);
return(fakerow);
}
public CacheFileReader(CacheFileReader source, int limit) : this(source)
{
Limit = limit + source.Position;
Position = source.Position;
}
protected SNode ParseOne()
{
EStreamCode check;
byte isshared = 0;
SNode thisobj = null;
SDBRow lastDbRow = null;
try
{
byte type = Reader.ReadByte();
check = (EStreamCode)(type & 0x3f);
isshared = (byte)(type & 0x40);
}
catch (EndOfFileException)
{
return null;
}
#region EStreamCode Switch
switch (check)
{
case EStreamCode.EStreamStart:
break;
case EStreamCode.ENone:
thisobj = new SNone();
break;
case EStreamCode.EString:
thisobj = new SString(Reader.ReadString(Reader.ReadByte()));
break;
case EStreamCode.ELong:
thisobj = new SLong(Reader.ReadLong());
break;
case EStreamCode.EInteger:
thisobj = new SInt(Reader.ReadInt());
break;
case EStreamCode.EShort:
thisobj = new SInt(Reader.ReadShort());
break;
case EStreamCode.EByte:
thisobj = new SInt(Reader.ReadByte());
break;
case EStreamCode.ENeg1Integer:
thisobj = new SInt(-1);
break;
case EStreamCode.E0Integer:
thisobj = new SInt(0);
break;
case EStreamCode.E1Integer:
thisobj = new SInt(1);
break;
case EStreamCode.EReal:
thisobj = new SReal(Reader.ReadDouble());
break;
case EStreamCode.E0Real:
thisobj = new SReal(0);
break;
case EStreamCode.E0String:
thisobj = new SString(null);
break;
case EStreamCode.EString3:
thisobj = new SString(Reader.ReadString(1));
break;
case EStreamCode.EString4:
goto case EStreamCode.EString;
case EStreamCode.EMarker:
thisobj = new SMarker((byte)GetLength());
break;
case EStreamCode.EUnicodeString:
goto case EStreamCode.EString;
case EStreamCode.EIdent:
thisobj = new SIdent(Reader.ReadString(GetLength()));
break;
case EStreamCode.ETuple:
{
int length = GetLength();
thisobj = new STuple((uint)length);
Parse(thisobj, length);
break;
}
case EStreamCode.ETuple2:
goto case EStreamCode.ETuple;
case EStreamCode.EDict:
{
int len = (GetLength() * 2);
SDict dict = new SDict((uint)len);
thisobj = dict;
Parse(dict, len);
break;
}
case EStreamCode.EObject:
thisobj = new SObject();
Parse(thisobj, 2);
break;
case EStreamCode.ESharedObj:
thisobj = ShareGet(GetLength());
break;
case EStreamCode.EChecksum:
thisobj = new SString("checksum");
Reader.ReadInt();
break;
case EStreamCode.EBoolTrue:
thisobj = new SInt(1);
break;
case EStreamCode.EBoolFalse:
thisobj = new SInt(0);
break;
case EStreamCode.EObject22:
{
SObject obj = new SObject();
thisobj = obj;
Parse(thisobj, 1);
string oclass = obj.Name;
if (oclass == "dbutil.RowList")
{
SNode row;
while ((row = ParseOne()) != null)
obj.AddMember(row);
}
break;
}
case EStreamCode.EObject23:
goto case EStreamCode.EObject22;
case EStreamCode.E0Tuple:
thisobj = new STuple(0);
break;
case EStreamCode.E1Tuple:
thisobj = new STuple(1);
Parse(thisobj, 1);
break;
case EStreamCode.E0Tuple2:
goto case EStreamCode.E0Tuple;
case EStreamCode.E1Tuple2:
goto case EStreamCode.E1Tuple;
case EStreamCode.EEmptyString:
thisobj = new SString(string.Empty);
break;
case EStreamCode.EUnicodeString2:
/* Single unicode character */
thisobj = new SString(Reader.ReadString(2));
break;
case EStreamCode.ECompressedRow:
thisobj = GetDBRow();
break;
case EStreamCode.ESubstream:
{
int len = GetLength();
CacheFileReader readerSub = new CacheFileReader(Reader, len);
SSubStream ss = new SSubStream(len);
thisobj = ss;
CacheFileParser sp = new CacheFileParser(readerSub);
sp.Parse();
for (int i = 0; i < sp.Streams.Count; i++)
ss.AddMember(sp.Streams[i].Clone());
Reader.Seek(readerSub.Position, SeekOrigin.Begin);
break;
}
case EStreamCode.E2Tuple:
thisobj = new STuple(2);
Parse(thisobj, 2);
break;
case EStreamCode.EString2:
goto case EStreamCode.EString;
case EStreamCode.ESizedInt:
switch (Reader.ReadByte())
{
case 8:
thisobj = new SLong(Reader.ReadLong());
break;
case 4:
thisobj = new SInt(Reader.ReadInt());
break;
case 3:
// The following seems more correct than the forumla used.
// int value = (Reader.Char() << 16) + (Reader.ReadByte());
thisobj = new SInt((Reader.ReadByte()) + (Reader.ReadByte() << 16));
break;
case 2:
thisobj = new SInt(Reader.ReadShort());
break;
}
break;
case (EStreamCode)0x2d:
if (Reader.ReadByte() != (byte)0x2d)
throw new ParseException("Didn't encounter a double 0x2d where one was expected at " + (Reader.Position - 2));
else if (lastDbRow != null)
lastDbRow.IsLast = true;
return null;
case 0:
break;
default:
throw new ParseException("Can't identify type " + String.Format("{0:x2}", (int)check) +
" at position " + String.Format("{0:x2}", Reader.Position) + " limit " + Reader.Length);
}
#endregion
if (thisobj == null)
throw new ParseException("no thisobj in parseone");
if (isshared != 0)
{
if (thisobj == null)
throw new ParseException("shared flag but no obj");
ShareAdd(thisobj);
}
return thisobj;
}
protected SNode GetDBRow()
{
SNode nhead = ParseOne();
// get header
SObject head = nhead as SObject;
if (head == null)
throw new ParseException("The DBRow header isn't present...");
if (head.Name != "blue.DBRowDescriptor")
throw new ParseException("Bad descriptor name");
STuple fields = head.Members[0].Members[1].Members[0] as STuple;
int len = GetLength();
byte[] olddata = Reader.ReadBytes(len);
List<byte> newdata = new List<byte>();
rle_unpack(olddata, newdata);
SNode body = new SDBRow(17, newdata);
CacheFileReader blob = new CacheFileReader(newdata.ToArray());
SDict dict = new SDict(999999); // TODO: need dynamic sized dict
int step = 1;
while (step < 6)
{
foreach (SNode field in fields.Members)
{
SNode fieldName = field.Members[0];
SInt fieldType = field.Members[1] as SInt;
int fieldTypeInt = fieldType.Value;
byte boolcount = 0;
byte boolbuf = 0;
SNode obj = null;
switch (fieldTypeInt)
{
case 2: // 16bit int
if(step == 3)
obj = new SInt(blob.ReadShort());
break;
case 3: // 32bit int
if (step == 2)
obj = new SInt(blob.ReadInt());
break;
case 4:
obj = new SReal(blob.ReadFloat());
break;
case 5: // double
if(step == 1)
obj = new SReal(blob.ReadDouble());
break;
case 6: // currency
if (step == 1)
obj = new SLong(blob.ReadLong());
break;
case 11: // boolean
if (step == 5)
{
if (boolcount == 0)
{
boolbuf = blob.ReadByte();
boolcount = 0x1;
}
if (boolbuf != 0 && boolcount != 0)
obj = new SInt(1);
else
obj = new SInt(0);
boolcount <<= 1;
}
break;
case 16:
obj = new SInt(blob.ReadByte());
break;
case 17:
goto case 16;
case 18: // 16bit int
goto case 2;
case 19: // 32bit int
goto case 3;
case 20: // 64bit int
goto case 6;
case 21: // 64bit int
goto case 6;
case 64: // timestamp
goto case 6;
case 128: // string types
case 129:
case 130:
obj = new SString("I can't parse strings yet - be patient");
break;
default:
throw new ParseException("Unhandled ADO type " + fieldTypeInt);
}
if (obj != null)
{
dict.AddMember(obj);
dict.AddMember(fieldName.Clone());
}
}
step++;
}
SNode fakerow = new STuple(3);
fakerow.AddMember(head);
fakerow.AddMember(body);
fakerow.AddMember(dict);
return fakerow;
}
