private bool HandleTick(int expectedCount, Action <TickIO, TickIO, ulong> assertTick, SymbolInfo symbol)
{
try {
while (!tickQueue.TryDequeue(ref tickBinary))
{
if (propagateException != null)
{
throw propagateException;
}
Thread.Sleep(1);
}
tick.Inject(tickBinary);
if (debug && countLog < 5)
{
log.Debug("Received a tick " + tick);
countLog++;
}
startTime = Environment.TickCount;
count++;
if (count > 0)
{
assertTick(tick, lastTick, symbol.BinaryIdentifier);
}
lastTick.Copy(tick);
if (propagateException != null)
{
throw propagateException;
}
if (count >= expectedCount)
{
return(true);
}
} catch (QueueException ex) {
switch (ex.EntryType)
{
case EventType.EndHistorical:
case EventType.StartRealTime:
case EventType.EndRealTime:
break;
case EventType.Terminate:
return(true);
default:
throw new ApplicationException("Unexpected QueueException: " + (EventType)ex.EntryType);
}
}
return(false);
}
public void SignatureResult()
{
string pair = "USD_JPY_Volume";
TickReader tickReader = new TickReader();
tickReader.LogProgress = true;
tickReader.Initialize("Test\\DataCache", pair);
int totalBytes = 0;
SignatureCompressor compressor = new SignatureCompressor();
ByteMemory compressed = new ByteMemory("compressed");
int compressedLength = 0;
try {
TickBinary tick = new TickBinary();
TickImpl tickImpl = new TickImpl();
for (int i = 0; i < 101; i++)
{
tickReader.ReadQueue.Dequeue(ref tick);
tickImpl.Inject(tick);
compressed.Reset();
compressor.CompressTick(tick, compressed.Memory);
totalBytes += compressedLength;
}
log.Debug(compressor.Signature.ToString());
Assert.AreEqual(TestSignature, compressor.Signature);
for (int i = 0; i < 200; i++)
{
tickReader.ReadQueue.Dequeue(ref tick);
tickImpl.Inject(tick);
compressed.Reset();
compressor.CompressTick(tick, compressed.Memory);
totalBytes += compressedLength;
int length = compressed.Bytes[0];
int diffBytes = length / 8 + 1;
log.Debug(compressedLength + ": " +
compressed.Bytes[0] + " " +
ByteArrayToString(compressor.Difference, 0, compressor.DiffLength));
}
} catch (CollectionTerminatedException) {
}
log.Debug("Total Compressed Bytes: " + totalBytes);
compressor.LogCounts();
}
public void BinaryDiffTest()
{
// string pair = "USD_JPY_Huge";
// string pair = "USD_JPY_YEARS";
string pair = "USD_JPY";
TickReader tickReader = new TickReader();
tickReader.LogProgress = true;
tickReader.Initialize("Test\\DataCache", pair);
byte[] previous = new byte[1024];
MemoryStream stream = new MemoryStream();
diffBits = new BitArray(1024);
long fileSize = 0;
long compressSize = 0;
int count = 0;
try {
TickBinary tick = new TickBinary();
TickIO tickIO = new TickImpl();
while (true)
{
tickReader.ReadQueue.Dequeue(ref tick);
stream.Seek(0, SeekOrigin.Begin);
tickIO.Inject(tick);
tickIO.ToWriter(stream);
CompareSignature(stream.GetBuffer(), previous, stream.Position);
fileSize += stream.Position;
int totDiffLength = 1 + diffLength + diffBits.ByteCount;
compressSize += totDiffLength;
if (count > 1000 && count < 2000)
{
// log.WriteFile(ByteArrayToString(memory,stream.Position));
if (logging)
{
log.Debug(count + ": " + totDiffLength + " " + diffLength + " byte " + ByteArrayToString(diffBits.Bytes, diffBits.ByteCount) + " " + ByteArrayToString(diff, diffLength));
}
}
count++;
Array.Copy(stream.GetBuffer(), previous, stream.Position);
}
} catch (QueueException ex) {
Assert.AreEqual(EventType.EndHistorical, ex.EntryType);
} catch (CollectionTerminatedException) {
}
TickReader.CloseAll();
log.Debug("File Size = " + fileSize + ", Compressed Size = " + compressSize);
}
// [Test]
public void CompressTickTest()
{
string pair = "USD_JPY_Volume";
TickReader tickReader = new TickReader();
tickReader.LogProgress = true;
tickReader.Initialize("Test\\DataCache", pair);
int totalBytes = 0;
SignatureCompressor compressor = new SignatureCompressor();
SignatureCompressor decompressor = new SignatureCompressor();
ByteMemory output = new ByteMemory("compressed");
int length = 0;
try {
TickBinary tick = new TickBinary();
TickImpl tickImpl = new TickImpl();
for (int i = 0; i < 101; i++)
{
tickReader.ReadQueue.Dequeue(ref tick);
tickImpl.Inject(tick);
compressor.CompareTick(tickImpl);
if (compressor.Count < 100)
{
compressor.CopyMemory(output.Bytes, out length);
}
compressor.SwapBuffers();
compressor.Count++;
if (compressor.Count % 100 == 0)
{
compressor.CreateSignature();
compressor.ResetCounters();
}
}
string temp = "";
for (int i = 0; i < compressor.Signature.Length; i++)
{
if (i != 0)
{
temp += ", ";
}
temp += compressor.Signature[i];
}
log.Notice("signature = " + temp);
Assert.AreEqual(TestSignature, compressor.Signature);
byte[] buffer = new byte[1024];
for (int i = 0; i < 200; i++)
{
tickReader.ReadQueue.Dequeue(ref tick);
tickImpl.Inject(tick);
compressor.CompareTick(tickImpl);
compressor.CalculateDifference(compressor.Current.Bytes, compressor.Previous.Bytes, compressor.Current.Length);
Array.Copy(compressor.Current.Bytes, buffer, compressor.Current.Length);
compressor.ReverseDifference(compressor.Current.Bytes, compressor.Previous.Bytes, compressor.Previous.Length);
Assert.AreEqual(buffer, compressor.Current.Bytes);
compressor.WriteMemory(output.Bytes, out length);
compressor.SwapBuffers();
compressor.Count++;
if (compressor.Count % 100 == 0)
{
compressor.CreateSignature();
compressor.ResetCounters();
}
}
} catch (CollectionTerminatedException) {
}
log.Debug("Total Compressed Bytes: " + totalBytes);
compressor.LogCounts();
}