public void OnOpen(Tick tick)
{
if (trace)
{
log.Trace("OnOpen(" + tick + ")");
}
isOpenTick = true;
openTime = tick.Time;
currentTick.Inject(tick.Extract());
}
private bool DequeueTick()
{
LatencyManager.IncrementSymbolHandler();
var result = false;
if (tickPool.AllocatedCount >= tickPool.Capacity / 2)
{
return(false);
}
while (!quoteSimulatorSupport.QuotePacketQueue.IsFull)
{
if (!reader.TryReadTick(temporaryTick))
{
if (onEndTick != null)
{
onEndTick(id);
}
else
{
throw new ApplicationException("OnEndTick was null");
}
queueTask.Stop();
return(result);
}
tickCounter++;
if (isFirstTick)
{
currentTick.Inject(temporaryTick.Extract());
}
else
{
currentTick.Inject(nextTick.Extract());
}
isFirstTick = false;
FillSimulator.StartTick(currentTick);
nextTick.Inject(temporaryTick.Extract());
if (FillSimulator.IsChanged)
{
FillSimulator.ProcessOrders();
}
if (trace)
{
log.Trace("Dequeue tick " + nextTick.UtcTime + "." + nextTick.UtcTime.Microsecond);
}
ProcessOnTickCallBack();
result = true;
}
return(result);
}
public void CompressTick(TickBinary tick, MemoryStream memory)
{
tickIO.Inject(tick);
byte[] output = memory.GetBuffer();
int length = (int)memory.Length;
CompareTick(tickIO);
if (count < 100)
{
CopyMemory(output, out length);
}
else
{
CalculateDifference(current.Bytes, previous.Bytes, current.Length);
WriteMemory(output, out length);
if (shortValueCompression)
{
// Frequency of individual short values
for (int i = 0; i < length - 1; i++)
{
int intValue = (int)(output[i] | (output[i + 1] << 8));
ByteCount bc = shortValueCounts[intValue];
shortValueCounts[intValue] = new ByteCount(intValue, bc.Count + 1);
}
}
}
SwapBuffers();
count++;
if (count % 100 == 0)
{
CreateSignature(); ResetCounters();
}
}
public void OnOpen(Tick tick)
{
if (trace)
{
log.Trace("OnOpen(" + tick + ")");
}
isOpenTick = true;
openTime = tick.Time;
if (!tick.IsQuote && !tick.IsTrade)
{
throw new ApplicationException("tick w/o either trade or quote data? " + tick);
}
currentTick.Inject(tick.Extract());
hasCurrentTick = true;
IsChanged = true;
}
private void DequeueTick()
{
LatencyManager.IncrementSymbolHandler();
if (!reader.TryReadTick(temporaryTick))
{
if (onEndTick == null)
{
throw new ApplicationException("OnEndTick was null");
}
onEndTick(id);
endOfTickData = true;
queueTask.Resume();
if (debug)
{
log.Debug("End Of Tick Data.");
}
return;
}
tickCounter++;
if (isFirstTick)
{
currentTick.Inject(temporaryTick.Extract());
}
else
{
currentTick.Inject(nextTick.Extract());
}
isFirstTick = false;
FillSimulator.StartTick(currentTick);
nextTick.Inject(temporaryTick.Extract());
tickSync.AddTick(nextTick);
if (trace)
{
log.Trace("Dequeue tick " + nextTick.UtcTime + "." + nextTick.UtcTime.Microsecond);
}
ProcessOnTickCallBack();
}
private Yield DequeueTick()
{
var result = Yield.NoWork.Repeat;
var binary = new TickBinary();
try {
if (reader.ReadQueue.TryDequeue(ref binary))
{
if (isPlayBack)
{
if (isFirstTick)
{
playbackOffset = fixSimulatorSupport.GetRealTimeOffset(binary.UtcTime);
}
binary.UtcTime += playbackOffset;
}
nextTick.Inject(binary);
tickSync.AddTick();
if (isFirstTick)
{
fillSimulator.StartTick(nextTick);
isFirstTick = false;
}
else
{
fillSimulator.ProcessOrders();
}
result = Yield.DidWork.Invoke(ProcessTick);
}
} catch (QueueException ex) {
if (ex.EntryType != EventType.EndHistorical)
{
throw;
}
}
return(result);
}
private void MigrateFile(string file, string symbol)
{
if (File.Exists(file + ".back"))
{
Console.WriteLine("A backup file already exists. Please delete it first at: " + file + ".back");
return;
}
using (var reader = Factory.TickUtil.TickReader())
using (var writer = Factory.TickUtil.TickWriter(true)) {
reader.Initialize(file, symbol);
writer.KeepFileOpen = true;
writer.Initialize(file + ".temp", symbol);
TickIO firstTick = Factory.TickUtil.TickIO();
TickIO tickIO = Factory.TickUtil.TickIO();
TickBinary tickBinary = new TickBinary();
int count = 0;
bool first = false;
try {
while (true)
{
while (!reader.ReadQueue.TryDequeue(ref tickBinary))
{
Thread.Sleep(1);
}
tickIO.Inject(tickBinary);
while (!writer.TryAdd(tickIO))
{
Thread.Sleep(1);
}
if (first)
{
firstTick.Copy(tickIO);
first = false;
}
count++;
}
} catch (QueueException ex) {
if (ex.EntryType != EventType.EndHistorical)
{
throw new ApplicationException("Unexpected QueueException: " + ex);
}
}
Console.WriteLine(reader.Symbol + ": Migrated " + count + " ticks from " + firstTick.Time + " to " + tickIO.Time);
}
File.Move(file, file + ".back");
File.Move(file + ".temp", file);
}
private Yield DequeueTick()
{
LatencyManager.IncrementSymbolHandler();
var result = Yield.NoWork.Repeat;
if (isFirstTick)
{
if (!reader.TryReadTick(currentTick))
{
return(result);
}
}
else
{
currentTick.Inject(nextTick.Extract());
if (!reader.TryReadTick(nextTick))
{
return(result);
}
}
tickCounter++;
if (isFirstTick)
{
playbackOffset = fixSimulatorSupport.GetRealTimeOffset(currentTick.UtcTime.Internal);
prevTickTime = TimeStamp.UtcNow.Internal + 5000000;
}
currentTick.SetTime(new TimeStamp(GetNextUtcTime(currentTick.lUtcTime)));
prevTickTime = currentTick.UtcTime.Internal;
if (tickCounter > 10)
{
intervalTime = 1000;
}
isFirstTick = false;
FillSimulator.StartTick(currentTick);
if (trace)
{
log.Trace("Dequeue tick " + nextTick.UtcTime + "." + nextTick.UtcTime.Microsecond);
}
return(Yield.DidWork.Invoke(ProcessTick));
}
private unsafe int FromFileVersion11(ref TickBinary binary, byte *fptr, int length)
{
if (verbose)
{
log.Verbose("Before Dx " + binary);
}
if (pricePrecision == 0L)
{
SetPricePrecision(ref binary);
}
length--;
byte *ptr = fptr;
var checksum = *ptr; ptr++;
while ((ptr - fptr) < length)
{
var field = (BinaryField)(*ptr >> 3);
switch (field)
{
case BinaryField.Precision:
if (debug)
{
log.Debug("Processing decimal place precision during tick de-compression.");
}
pricePrecision = ReadField2(&ptr);
break;
case BinaryField.Reset:
if (trace)
{
log.Trace("Processing Reset during tick de-compression.");
}
ReadField2(&ptr);
var symbol = binary.Symbol;
binary = default(TickBinary);
binary.Symbol = symbol;
if (verbose)
{
log.Verbose("Reset Dx " + binary);
}
break;
case BinaryField.ContentMask:
binary.contentMask += (byte)ReadField2(&ptr);
break;
case BinaryField.Time:
binary.UtcTime += ReadField2(&ptr);
break;
case BinaryField.Strike:
binary.Strike += ReadField2(&ptr) * pricePrecision;
break;
case BinaryField.OptionExpiration:
var readField = ReadField2(&ptr);
var currentExpiration = binary.UtcOptionExpiration + readField;
binary.UtcOptionExpiration = currentExpiration;
break;
case BinaryField.Bid:
binary.Bid += ReadField2(&ptr) * pricePrecision;
break;
case BinaryField.Ask:
binary.Ask += ReadField2(&ptr) * pricePrecision;
break;
case BinaryField.Price:
binary.Price += ReadField2(&ptr) * pricePrecision;
break;
case BinaryField.Size:
binary.Size += (int)ReadField2(&ptr);
break;
case BinaryField.BidSize:
ReadBidSize(ref binary, &ptr);
//fixed (ushort* usptr = binary.DepthBidLevels)
// for (int i = 0; i < TickBinary.DomLevels; i++)
// {
// var size = *(usptr + i);
// }
break;
case BinaryField.AskSize:
ReadAskSize(ref binary, &ptr);
//fixed (ushort* usptr = binary.DepthAskLevels)
// for (int i = 0; i < TickBinary.DomLevels; i++)
// {
// var size = *(usptr + i);
// }
break;
default:
throw new ApplicationException("Unknown tick field type: " + field);
}
}
if (verbose)
{
fromFileVerboseTickIO.Inject(binary);
log.Verbose("Dx tick: " + fromFileVerboseTickIO);
}
var expectedChecksum = CalcChecksum(ref binary, "Dx");
if (expectedChecksum != checksum)
{
fromFileVerboseTickIO.Inject(binary);
throw new ApplicationException("Checksum mismatch " + checksum + " vs. " + expectedChecksum + ". This means integrity checking of tick compression failed. The tick which failed checksum: " + fromFileVerboseTickIO);
}
var len = (int)(ptr - fptr);
return(len);
}
private unsafe void ToWriterVersion11(ref TickBinary binary, MemoryStream writer)
{
if (verbose)
{
log.Verbose("Before Cx " + lastBinary);
}
//var tempBinary = lastBinary;
var dataVersion = (byte)11;
writer.SetLength(writer.Position + minTickSize);
byte[] buffer = writer.GetBuffer();
fixed(byte *fptr = &buffer[writer.Position])
{
byte *ptr = fptr;
ptr++; // Save space for size header.
*(ptr) = dataVersion; ptr++;
ptr++; // Save space for checksum.
if (pricePrecision == 0L)
{
SetPricePrecision(ref binary);
if (trace)
{
log.Trace("Writing decimal places used in price compression.");
}
WriteField2(BinaryField.Precision, &ptr, pricePrecision);
}
if (!isCompressStarted)
{
if (trace)
{
log.Trace("Writing Reset token during tick compression.");
}
lastBinary = default(TickBinary);
WriteField2(BinaryField.Reset, &ptr, 1);
isCompressStarted = true;
if (verbose)
{
log.Verbose("Reset Dx " + lastBinary);
}
}
WriteField2(BinaryField.ContentMask, &ptr, binary.contentMask - lastBinary.contentMask);
lastBinary.contentMask = binary.contentMask;
var diff = (binary.UtcTime - lastBinary.UtcTime);
WriteField2(BinaryField.Time, &ptr, diff);
lastBinary.UtcTime = binary.UtcTime;
if (binary.IsQuote)
{
WriteField2(BinaryField.Bid, &ptr, binary.Bid / pricePrecision - lastBinary.Bid);
WriteField2(BinaryField.Ask, &ptr, binary.Ask / pricePrecision - lastBinary.Ask);
lastBinary.Bid = binary.Bid / pricePrecision;
lastBinary.Ask = binary.Ask / pricePrecision;
}
if (binary.IsTrade)
{
WriteField2(BinaryField.Price, &ptr, binary.Price / pricePrecision - lastBinary.Price);
WriteField2(BinaryField.Size, &ptr, binary.Size - lastBinary.Size);
lastBinary.Price = binary.Price / pricePrecision;
lastBinary.Size = binary.Size;
}
if (binary.IsOption)
{
WriteField2(BinaryField.Strike, &ptr, binary.Strike / pricePrecision - lastBinary.Strike);
lastBinary.Strike = binary.Strike / pricePrecision;
diff = (binary.UtcOptionExpiration - lastBinary.UtcOptionExpiration);
WriteField2(BinaryField.OptionExpiration, &ptr, diff);
lastBinary.UtcOptionExpiration = binary.UtcOptionExpiration;
}
if (binary.HasDepthOfMarket)
{
var field = (byte)((byte)BinaryField.BidSize << 3);
fixed(ushort *usptr = binary.DepthBidLevels)
for (int i = 0; i < TickBinary.DomLevels; i++)
{
WriteBidSize(ref binary, field, i, &ptr);
}
field = (byte)((byte)BinaryField.AskSize << 3);
fixed(ushort *usptr = binary.DepthAskLevels)
for (int i = 0; i < TickBinary.DomLevels; i++)
{
WriteAskSize(ref binary, field, i, &ptr);
}
}
int length = (int)(ptr - fptr);
writer.Position += length;
writer.SetLength(writer.Position);
* fptr = (byte)(ptr - fptr);
var checkSum = CalcChecksum(ref binary, "Cx");
*(fptr + 2) = checkSum;
//lastBinary = binary;
if (verbose)
{
toWriterTickIO.Inject(binary);
log.Verbose("Cx tick: " + toWriterTickIO);
}
//FromFileVersion11(ref tempBinary, fptr + 2, length-2);
}
}
private void FilterFile(string symbol, string inputPath, string outputPath, TimeStamp startTime, TimeStamp endTime)
{
TickReader reader = Factory.TickUtil.TickReader();
TickWriter writer = Factory.TickUtil.TickWriter(true);
writer.KeepFileOpen = true;
writer.Initialize(outputPath, symbol);
reader.Initialize(inputPath, symbol);
TickQueue inputQueue = reader.ReadQueue;
TickIO firstTick = Factory.TickUtil.TickIO();
TickIO lastTick = Factory.TickUtil.TickIO();
TickIO prevTick = Factory.TickUtil.TickIO();
long count = 0;
long fast = 0;
long dups = 0;
TickIO tickIO = Factory.TickUtil.TickIO();
TickBinary tickBinary = new TickBinary();
inputQueue.Dequeue(ref tickBinary);
tickIO.Inject(tickBinary);
count++;
firstTick.Copy(tickIO);
firstTick.IsSimulateTicks = true;
prevTick.Copy(tickIO);
prevTick.IsSimulateTicks = true;
if (tickIO.Time >= startTime)
{
writer.Add(firstTick);
}
try {
while (true)
{
while (!inputQueue.TryDequeue(ref tickBinary))
{
Thread.Sleep(1);
}
tickIO.Inject(tickBinary);
count++;
if (tickIO.Time >= startTime)
{
if (tickIO.Time > endTime)
{
break;
}
// if( tickIO.Bid == prevTick.Bid && tickIO.Ask == prevTick.Ask) {
// dups++;
// } else {
// Elapsed elapsed = tickIO.Time - prevTick.Time;
prevTick.Copy(tickIO);
prevTick.IsSimulateTicks = true;
// if( elapsed.TotalMilliseconds < 5000) {
// fast++;
// } else {
while (!writer.TryAdd(prevTick))
{
Thread.Sleep(1);
}
// }
// }
}
}
} catch (QueueException ex) {
if (ex.EntryType != EventType.EndHistorical)
{
throw new ApplicationException("Unexpected QueueException: " + ex);
}
}
lastTick.Copy(tickIO);
Console.WriteLine(reader.Symbol + ": " + count + " ticks from " + firstTick.Time + " to " + lastTick.Time + " " + dups + " duplicates, " + fast + " less than 50 ms");
Factory.TickUtil.TickReader().CloseAll();
writer.Close();
}
protected virtual Yield Invoke()
{
EventItem eventItem;
if (filter.Receive(out eventItem))
{
switch ((EventType)eventItem.EventType)
{
case EventType.Shutdown:
Dispose();
filter.Pop();
break;
default:
throw new ApplicationException("Unexpected event: " + eventItem);
}
}
var result = Yield.NoWork.Repeat;
try {
if (writeQueue.Count == 0)
{
return(result);
}
while (writeQueue.Count > 0)
{
if (!writeQueue.TryPeek(ref tick))
{
break;
}
tickIO.Inject(tick);
if (tickFile.TryWriteTick(tickIO))
{
writeQueue.TryDequeue(ref tick);
}
result = Yield.DidWork.Repeat;
}
return(result);
} catch (QueueException ex) {
if (ex.EntryType == EventType.Terminate)
{
log.Notice("Last tick written: " + tickIO);
if (debug)
{
log.Debug("Exiting, queue terminated.");
}
Finalize();
return(Yield.Terminate);
}
else
{
Exception exception = new ApplicationException("Queue returned unexpected: " + ex.EntryType);
writeQueue.SetException(exception);
writeQueue.Dispose();
Dispose();
throw ex;
}
} catch (Exception ex) {
writeQueue.SetException(ex);
writeQueue.Dispose();
Dispose();
throw;
}
}
public long Verify(int expectedCount, Action <TickIO, TickIO, long> assertTick, SymbolInfo symbol, int timeout, Action action)
{
if (debug)
{
log.Debug("Verify");
}
if (SyncTicks.Enabled)
{
tickSync = SyncTicks.GetTickSync(symbol.BinaryIdentifier);
}
long endTime = Factory.Parallel.TickCount + timeout * 1000;
count = 0;
while (Factory.Parallel.TickCount < endTime)
{
if (propagateException != null)
{
throw propagateException;
}
try {
if (tickQueue.TryDequeue(ref tickBinary))
{
tickIO.Inject(tickBinary);
if (debug && countLog < 5)
{
log.Debug("Received a tick " + tickIO);
countLog++;
}
startTime = Factory.TickCount;
count++;
if (count > 0 && assertTick != null)
{
assertTick(tickIO, lastTick, symbol.BinaryIdentifier);
}
lastTick.Copy(tickIO);
if (!actionAlreadyRun && action != null)
{
actionAlreadyRun = true;
action();
}
if (SyncTicks.Enabled)
{
tickSync.RemoveTick();
}
if (count >= expectedCount)
{
break;
}
}
else
{
Thread.Sleep(100);
}
} catch (QueueException ex) {
if (HandleQueueException(ex))
{
break;
}
}
}
return(count);
}
public long Verify(int expectedCount, Action <TickIO, TickIO, long> assertTick, int timeout, Action action)
{
if (debug)
{
log.Debug("Verify");
}
long endTime = Factory.Parallel.TickCount + timeout * 1000;
count = 0;
do
{
if (propagateException != null)
{
throw propagateException;
}
try
{
if (TryDequeueTick(ref tickBinary))
{
tickIO.Inject(tickBinary);
if (debug && countLog < 5)
{
log.Debug("Received a tick " + tickIO + " UTC " + tickIO.UtcTime);
countLog++;
}
else if (trace)
{
log.Trace("Received a tick " + tickIO + " UTC " + tickIO.UtcTime);
}
startTime = Factory.TickCount;
count++;
if (count > 0)
{
if (assertTick != null)
{
assertTick(tickIO, lastTick, symbol.BinaryIdentifier);
}
if (count % 10000 == 0)
{
log.Info("Read " + count + " ticks");
}
}
lastTick.Copy(tickIO);
if (!actionAlreadyRun && action != null)
{
actionAlreadyRun = true;
action();
}
if (SyncTicks.Enabled && symbolState == SymbolState.RealTime)
{
tickSync.RemoveTick(ref tickBinary);
}
if (count >= expectedCount)
{
break;
}
}
else
{
Thread.Sleep(100);
//if (queue.Count == 0 && SyncTicks.Enabled && SymbolState == SymbolState.RealTime)
//{
// tickSync.RemoveTick();
//}
}
}
catch (QueueException ex)
{
if (HandleQueueException(ex))
{
break;
}
}
} while (Factory.Parallel.TickCount < endTime);
return(count);
}
protected virtual Yield AppendData()
{
try {
if (writeQueue.Count == 0)
{
return(Yield.NoWork.Repeat);
}
if (!keepFileOpen)
{
fs = new FileStream(fileName, FileMode.Append, FileAccess.Write, FileShare.Read);
if (trace)
{
log.Trace("!keepFileOpen - Open()");
}
memory = new MemoryStream();
}
while (writeQueue.TryDequeue(ref tick))
{
tickIO.Inject(tick);
if (trace)
{
log.Trace("Writing to file: " + tickIO);
}
WriteToFile(memory, tickIO);
}
if (!keepFileOpen)
{
fs.Close();
if (trace)
{
log.Trace("!keepFileOpen - Close()");
}
fs = null;
}
return(Yield.DidWork.Repeat);
} catch (QueueException ex) {
if (ex.EntryType == EventType.Terminate)
{
log.Debug("Exiting, queue terminated.");
if (fs != null)
{
fs.Close();
log.Debug("Terminate - Close()");
}
return(Yield.Terminate);
}
else
{
Exception exception = new ApplicationException("Queue returned unexpected: " + ex.EntryType);
writeQueue.Terminate(exception);
throw ex;
}
} catch (Exception ex) {
writeQueue.Terminate(ex);
if (fs != null)
{
fs.Close();
}
throw;
}
}
public void ReadFile()
{
TickQueue queue = reader.ReadQueue;
TickIO firstTick = Factory.TickUtil.TickIO();
TickIO lastTick = Factory.TickUtil.TickIO();
TickIO prevTick = Factory.TickUtil.TickIO();
long count = 0;
long dups = 0;
long quotes = 0;
long trades = 0;
long quotesAndTrades = 0;
TickIO tickIO = Factory.TickUtil.TickIO();
TickBinary tickBinary = new TickBinary();
try {
while (true)
{
if (!TryGetNextTick(queue, ref tickBinary))
{
break;
}
tickIO.Inject(tickBinary);
if (count == 0)
{
firstTick.Copy(tickIO);
}
if (tickIO.IsQuote && tickIO.IsTrade)
{
quotesAndTrades++;
}
else if (tickIO.IsQuote)
{
quotes++;
}
else
{
trades++;
}
if (count > 0)
{
bool quoteDup = tickIO.IsQuote && prevTick.IsQuote && tickIO.Bid == prevTick.Bid && tickIO.Ask == prevTick.Ask;
bool tradeDup = tickIO.IsTrade && prevTick.IsTrade && tickIO.Price == prevTick.Price;
if (tickIO.IsQuote && tickIO.IsTrade)
{
if (quoteDup && tradeDup)
{
dups++;
}
}
else if (tickIO.IsQuote)
{
if (quoteDup)
{
dups++;
}
}
else
{
if (tradeDup)
{
dups++;
}
}
}
count++;
prevTick.Copy(tickIO);
}
} catch (QueueException) {
// Terminated.
}
lastTick.Copy(tickIO);
stringBuilder.AppendLine("Symbol: " + reader.Symbol);
stringBuilder.AppendLine("Version: " + reader.DataVersion);
stringBuilder.AppendLine("Ticks: " + count);
if (quotes > 0)
{
stringBuilder.AppendLine("Quote Only: " + quotes);
}
if (trades > 0)
{
stringBuilder.AppendLine("Trade Only: " + trades);
}
if (quotesAndTrades > 0)
{
stringBuilder.AppendLine("Quote and Trade: " + quotesAndTrades);
}
stringBuilder.AppendLine("From: " + firstTick.Time + "." + firstTick.Time.Microsecond);
stringBuilder.AppendLine("To: " + lastTick.Time + "." + lastTick.Time.Microsecond);
if (dups > 0)
{
stringBuilder.AppendLine("Prices duplicates: " + dups);
}
Factory.TickUtil.TickReader().CloseAll();
}