public bool TryReadTick(TickIO tickIO)
{
if (!isInitialized)
{
return(false);
}
if (dataIn == null || tickCount > MaxCount || endOfData)
{
return(false);
}
try
{
do
{
tickIO.SetSymbol(lSymbol);
byte size = dataIn.ReadByte();
// Check for old style prior to version 8 where
// single byte version # was first.
if (dataVersion < 8 && size < 8)
{
dataVersion = tickIO.FromReader((byte)size, dataIn);
}
else
{
// Subtract the size byte.
//if (dataIn.BaseStream.Position + size - 1 > length) {
// return false;
//}
int count = 1;
memory.SetLength(size);
memory.GetBuffer()[0] = size;
while (count < size)
{
var bytesRead = dataIn.Read(buffer, count, size - count);
if (bytesRead == 0)
{
return(false);
}
count += bytesRead;
}
memory.Position = 0;
dataVersion = tickIO.FromReader(memory);
}
var utcTime = new TimeStamp(tickIO.lUtcTime);
if (utcTime > EndTime)
{
ReportEndOfData();
return(false);
}
tickIO.SetTime(utcTime);
tickCount++;
} while (tickIO.UtcTime < StartTime);
return(true);
}
catch (EndOfStreamException ex)
{
ReportEndOfData();
return(false);
}
}
public virtual void AssertTick(TickIO tick, TickIO lastTick, ulong symbol)
{
Assert.Greater(tick.Price, 0);
Assert.Greater(tick.Size, 0);
Assert.IsTrue(tick.Time >= lastTick.Time, "tick.Time > lastTick.Time");
Assert.AreEqual(symbol, tick.lSymbol);
}
public void StartSymbol(Receiver receiver, SymbolInfo symbol, object eventDetail)
{
TickIO tickIO = Factory.TickUtil.TickIO();
tickIO.Initialize();
tickIO.SetSymbol(symbol.BinaryIdentifier);
tickIO.SetTime(new TimeStamp(2000, 1, 1));
tickIO.SetQuote(100D, 100D);
while (!receiver.OnEvent(symbol, (int)EventType.StartHistorical, symbol))
{
Factory.Parallel.Yield();
}
var binaryBox = tickPool.Create();
binaryBox.TickBinary = tickIO.Extract();
while (!receiver.OnEvent(symbol, (int)EventType.Tick, binaryBox))
{
Factory.Parallel.Yield();
}
tickIO.Initialize();
tickIO.SetSymbol(symbol.BinaryIdentifier);
tickIO.SetTime(new TimeStamp(2000, 1, 2));
tickIO.SetQuote(101D, 101D);
binaryBox = tickPool.Create();
binaryBox.TickBinary = tickIO.Extract();
while (!receiver.OnEvent(symbol, (int)EventType.Tick, binaryBox))
{
Factory.Parallel.Yield();
}
while (!receiver.OnEvent(symbol, (int)EventType.EndHistorical, symbol))
{
Factory.Parallel.Yield();
}
}
public void Add(TickIO tick)
{
while (!TryAdd(tick))
{
Thread.Sleep(1);
}
}
public void GetLastTick(TickIO lastTickIO)
{
if (!isInitialized)
{
throw new InvalidStateException("Please call one of the Initialize() methods first.");
}
Stream stream;
stream = new FileStream(FileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
dataIn = new BinaryReader(stream, Encoding.Unicode);
int count = 0;
try
{
while (stream.Position < stream.Length)
{
if (!TryReadTick(lastTickIO))
{
break;
}
count++;
}
}
catch (ObjectDisposedException)
{
// Only partial tick was read at the end of the file.
// Another writer must not have completed.
log.Warn("ObjectDisposedException returned from tickIO.FromReader(). Incomplete last tick. Ignoring.");
}
catch
{
log.Error("Error reading tick #" + count);
}
}
public bool TryReadTick(TickIO tickIO)
{
try
{
if (memory.Position >= tickBlockHeader.blockHeader.length)
{
return(false);
}
tickIO.SetSymbol(tickIO.lSymbol);
var size = memory.GetBuffer()[memory.Position];
dataVersion = tickIO.FromReader(memory);
var utcTime = new TimeStamp(tickIO.lUtcTime);
tickIO.SetTime(utcTime);
return(true);
}
catch (EndOfStreamException ex)
{
return(false);
}
catch (IndexOutOfRangeException)
{
int x = 0;
return(false);
}
}
public virtual void TestPositionSyncAndStopExits()
{
using (VerifyFeed verify = Factory.Utility.VerifyFeed(symbol))
using (Agent provider = ProviderFactory()) {
verify.PauseSeconds = secondsDelay;
provider.SendEvent(new EventItem(verify, null, EventType.Connect, null));
provider.SendEvent(new EventItem(verify, symbol, EventType.StartSymbol, new StartSymbolDetail(TimeStamp.MinValue)));
VerifyConnected(verify);
ClearOrders(0);
ClearPosition(provider, verify, secondsDelay);
TickIO lastTick = verify.LastTick;
double bid = lastTick.IsTrade ? lastTick.Price : lastTick.Bid;
double ask = lastTick.IsTrade ? lastTick.Price : lastTick.Ask;
int strategyPosition = 0;
CreateExit(strategy, OrderType.SellStop, bid - 180 * symbol.MinimumTick, strategyPosition);
LogicalOrder exitBuyStop = CreateExit(strategy, OrderType.BuyStop, ask + 540 * symbol.MinimumTick, strategyPosition);
var count = verify.Verify(2, assertTick, secondsDelay, () => {
SendOrders(provider, verify, 0, secondsDelay);
});
Assert.GreaterOrEqual(count, 2, "tick count");
ClearOrders(0);
ClearPosition(provider, verify, secondsDelay);
var expectedTicks = 1;
count = verify.Wait(expectedTicks, secondsDelay);
}
}
public void ZAutoReconnection()
{
using (VerifyFeed verify = Factory.Utility.VerifyFeed())
using (Provider provider = ProviderFactory()) {
verify.PauseSeconds = secondsDelay;
if (debug)
{
log.Debug("===ZAutoReconnection===");
}
provider.SendEvent(verify, null, EventType.Connect, null);
provider.SendEvent(verify, symbol, EventType.StartSymbol, new StartSymbolDetail(TimeStamp.MinValue));
VerifyConnected(verify);
if (debug)
{
log.Debug("===ClearOrders===");
}
ClearOrders(0);
ClearPosition(provider, verify, secondsDelay);
TickIO lastTick = verify.LastTick;
double bid = lastTick.IsTrade ? lastTick.Price : lastTick.Bid;
double ask = lastTick.IsTrade ? lastTick.Price : lastTick.Ask;
Assert.GreaterOrEqual(bid, 0, "Bid");
Assert.GreaterOrEqual(ask, 0, "Ask");
if (debug)
{
log.Debug("===Create a Buy and Sell Limit ===");
}
CreateLogicalEntry(OrderType.BuyLimit, bid - 280 * symbol.MinimumTick, 1);
CreateLogicalEntry(OrderType.SellLimit, ask + 340 * symbol.MinimumTick, 1);
SendOrders(provider, verify, secondsDelay);
ClearOrders(0);
ClearPosition(provider, verify, secondsDelay);
}
}
public void ReadFile()
{
TickIO tickIO = Factory.TickUtil.TickIO();
try
{
while (reader.TryReadTick(tickIO))
{
if (tickIO.UtcTime > endTime)
{
break;
}
if (tickIO.UtcTime > startTime)
{
Output(tickIO.ToString());
}
}
}
catch (QueueException ex)
{
if (ex.EntryType != EventType.EndHistorical)
{
throw;
}
}
}
public void GetLastTick(TickIO lastTickIO)
{
if (!isInitialized)
{
throw new InvalidStateException("Please call one of the Initialize() methods first.");
}
if (isLegacy)
{
legacy.GetLastTick(lastTickIO);
return;
}
OpenFileForReading();
var length = fs.Length;
if (length % fileHeader.blockSize != 0)
{
throw new InvalidOperationException("File size " + length + " isn't not an even multiple of block size " + fileHeader.blockSize);
}
fs.Seek(-fileHeader.blockSize, SeekOrigin.End);
ReadNextTickBlock();
while (TryReadTick(lastTickIO))
{
// Read till last tick in the last block.
}
}
public void Copy(TickIO tick, byte contentMask)
{
bool dom = (contentMask & ContentBit.DepthOfMarket) != 0;
bool simulateTicks = (contentMask & ContentBit.SimulateTicks) != 0;
bool quote = (contentMask & ContentBit.Quote) != 0;
bool trade = (contentMask & ContentBit.TimeAndSales) != 0;
Initialize();
SetSymbol(tick.lSymbol);
SetTime(tick.UtcTime);
IsSimulateTicks = simulateTicks;
if (quote)
{
SetQuote(tick.lBid, tick.lAsk);
}
if (trade)
{
SetTrade(tick.Side, tick.lPrice, tick.Size);
}
if (dom)
{
fixed(ushort *b = binary.DepthBidLevels)
fixed(ushort *a = binary.DepthAskLevels)
for (int i = 0; i < TickBinary.DomLevels; i++)
{
*(b + i) = (ushort)tick.BidLevel(i);
*(a + i) = (ushort)tick.AskLevel(i);
}
}
binary.ContentMask = contentMask;
dataVersion = tick.DataVersion;
}
public void Copy(TickIO tick, byte contentMask)
{
Initialize();
SetSymbol(tick.lSymbol);
SetTime(tick.UtcTime);
binary.contentMask = contentMask;
if (binary.IsQuote)
{
SetQuote(tick.lBid, tick.lAsk);
}
if (binary.IsTrade)
{
SetTrade(tick.Side, tick.lPrice, tick.Size);
}
if (binary.IsOption)
{
var type = binary.OptionType;
SetOption(type, tick.Strike, tick.UtcOptionExpiration);
}
if (binary.HasDepthOfMarket)
{
fixed(ushort *b = binary.DepthBidLevels)
fixed(ushort *a = binary.DepthAskLevels)
for (int i = 0; i < TickBinary.DomLevels; i++)
{
*(b + i) = (ushort)tick.BidLevel(i);
*(a + i) = (ushort)tick.AskLevel(i);
}
}
dataVersion = tick.DataVersion;
}
public void Initialize()
{
tick = Factory.TickUtil.TickIO();
lastTick = Factory.TickUtil.TickIO();
tick.SetSymbol(symbol.BinaryIdentifier);
this.level2Bids = new Level2Collection(this, symbol, enumMarketSide.msBid);
this.level2Asks = new Level2Collection(this, symbol, enumMarketSide.msAsk);
}
public void CompareTick(TickIO tick)
{
tick.ToWriter(current.Memory);
if (count > 0)
{
CompareMemory(current.Bytes, previous.Bytes, current.Length);
}
}
public void WriteTick(TickIO tickIO)
{
if (!isInitialized)
{
throw new InvalidOperationException("Please call one of the Initialize() methods first.");
}
TryWriteTick(tickIO);
}
public bool TryAdd(TickIO tickIO)
{
if (!isInitialized)
{
throw new ApplicationException("Please initialized TickWriter first.");
}
TickBinary tick = tickIO.Extract();
return(writeQueue.TryEnqueue(ref tick));
}
public void Copy(TickIO tick)
{
if (tick is TickImpl)
{
this = (TickImpl)tick;
}
else
{
Copy(tick, tick.ContentMask);
}
}
public long Wait(int expectedTicks, int timeout)
{
if (debug)
{
log.Debug("Wait");
}
long startTime = Factory.Parallel.TickCount;
lastTick = Factory.TickUtil.TickIO();
count = 0;
while (Factory.Parallel.TickCount - startTime < timeout * 1000)
{
if (propagateException != null)
{
throw propagateException;
}
try {
if (TryDequeueTick(ref tickBinary))
{
tickIO.Inject(tickBinary);
if (debug && count < 5)
{
log.Debug("Received a tick " + tickIO + " UTC " + tickIO.UtcTime);
countLog++;
}
else if (trace)
{
log.Trace("Received a tick " + tickIO + " UTC " + tickIO.UtcTime);
}
count++;
lastTick.Copy(tickIO);
if (SyncTicks.Enabled && symbolState == SymbolState.RealTime)
{
tickSync.RemoveTick(ref tickBinary);
}
if (count >= expectedTicks)
{
break;
}
}
else
{
Thread.Sleep(100);
}
} catch (QueueException ex) {
if (HandleQueueException(ex))
{
break;
}
}
}
return(count);
}
public void WriteTick(TickIO tickIO)
{
if (!isInitialized)
{
throw new InvalidStateException("Please call one of the Initialize() methods first.");
}
tickIO.ToWriter(memory);
if (memory.Position > 5000)
{
WriteToFile();
}
}
private void MigrateFile(string file, string symbol)
{
if (File.Exists(file + ".back"))
{
Console.WriteLine("A backup file already exists. Please delete it first at: " + file + ".back");
return;
}
using (var reader = Factory.TickUtil.TickReader())
using (var writer = Factory.TickUtil.TickWriter(true)) {
reader.Initialize(file, symbol);
writer.KeepFileOpen = true;
writer.Initialize(file + ".temp", symbol);
TickIO firstTick = Factory.TickUtil.TickIO();
TickIO tickIO = Factory.TickUtil.TickIO();
TickBinary tickBinary = new TickBinary();
int count = 0;
bool first = false;
try {
while (true)
{
while (!reader.ReadQueue.TryDequeue(ref tickBinary))
{
Thread.Sleep(1);
}
tickIO.Inject(tickBinary);
while (!writer.TryAdd(tickIO))
{
Thread.Sleep(1);
}
if (first)
{
firstTick.Copy(tickIO);
first = false;
}
count++;
}
} catch (QueueException ex) {
if (ex.EntryType != EventType.EndHistorical)
{
throw new ApplicationException("Unexpected QueueException: " + ex);
}
}
Console.WriteLine(reader.Symbol + ": Migrated " + count + " ticks from " + firstTick.Time + " to " + tickIO.Time);
}
File.Move(file, file + ".back");
File.Move(file + ".temp", file);
}
private void FilterFile(string inputPath, string outputPath, TimeStamp startTime, TimeStamp endTime)
{
using (var reader = Factory.TickUtil.TickFile())
using (var writer = Factory.TickUtil.TickFile())
{
writer.Initialize(outputPath, TickFileMode.Write);
reader.Initialize(inputPath, TickFileMode.Read);
TickIO firstTick = Factory.TickUtil.TickIO();
TickIO lastTick = Factory.TickUtil.TickIO();
TickIO prevTick = Factory.TickUtil.TickIO();
long count = 0;
long dups = 0;
TickIO tickIO = Factory.TickUtil.TickIO();
try
{
while (reader.TryReadTick(tickIO))
{
if (tickIO.Time >= startTime)
{
if (tickIO.Time > endTime)
{
break;
}
if (count == 0)
{
prevTick.Copy(tickIO);
prevTick.IsSimulateTicks = true;
firstTick.Copy(tickIO);
firstTick.IsSimulateTicks = true;
}
count++;
prevTick.Copy(tickIO);
writer.WriteTick(prevTick);
}
}
}
catch (QueueException ex)
{
if (ex.EntryType != EventType.EndHistorical)
{
throw new ApplicationException("Unexpected QueueException: " + ex);
}
}
lastTick.Copy(tickIO);
Output(reader.Symbol + ": " + count + " ticks.");
Output("From " + firstTick.Time + " to " + lastTick.Time);
Output(dups + " duplicates elimated.");
}
}
public void AssertLevel1(TickIO tick, TickIO lastTick, long symbol)
{
Assert.IsTrue(tick.IsQuote || tick.IsTrade);
if (tick.IsQuote)
{
Assert.Greater(tick.Bid, 0);
Assert.Greater(tick.Ask, 0);
}
if (tick.IsTrade)
{
Assert.Greater(tick.Price, 0);
Assert.Greater(tick.Size, 0);
}
Assert.AreEqual(symbol, tick.lSymbol);
}
public bool TryAdd(TickIO tickIO)
{
if (!isInitialized)
{
throw new ApplicationException("Please initialized TickWriter first.");
}
TickBinary tick = tickIO.Extract();
var result = writeQueue.TryEnqueue(ref tick);
if (result)
{
Interlocked.Increment(ref appendCounter);
}
return(result);
}
private void MigrateFile(string file)
{
if (File.Exists(file + ".back"))
{
Output("A backup file already exists. Please delete it first at: " + file + ".back");
return;
}
TickFile reader;
TickFile writer;
int count = 0;
TickIO firstTick = Factory.TickUtil.TickIO();
TickIO tickIO = Factory.TickUtil.TickIO();
try
{
using (reader = Factory.TickUtil.TickFile())
{
using (writer = Factory.TickUtil.TickFile())
{
reader.Initialize(file, TickFileMode.Read);
writer.EraseFileToStart = true;
writer.Initialize(file + ".temp", TickFileMode.Write);
bool first = true;
while (reader.TryReadTick(tickIO))
{
writer.WriteTick(tickIO);
if (first)
{
firstTick.Copy(tickIO);
first = false;
}
count++;
}
}
}
Output(reader.Symbol + ": Migrated " + count + " ticks from " + firstTick.Time + " to " + tickIO.Time);
Alter.MoveFile(file, file + ".back");
Alter.MoveFile(file + ".temp", file);
}
catch (Exception ex)
{
Output("Error: " + ex.Message);
}
}
public override void AssertTick(TickIO tick, TickIO lastTick, ulong symbol)
{
Assert.IsTrue(tick.IsQuote);
if (tick.IsQuote)
{
Assert.Greater(tick.Bid, 0);
Assert.Greater(tick.BidLevel(0), 0);
Assert.Greater(tick.Ask, 0);
Assert.Greater(tick.AskLevel(0), 0);
}
if (tick.IsTrade)
{
Assert.Greater(tick.Price, 0);
Assert.Greater(tick.Size, 0);
}
Assert.IsTrue(tick.Time >= lastTick.Time, "tick.Time > lastTick.Time");
Assert.AreEqual(symbol, tick.lSymbol);
}
public void AssertTimeAndSales(TickIO tick, TickIO lastTick, long symbol)
{
Assert.IsFalse(tick.IsQuote);
if (tick.IsQuote)
{
Assert.Greater(tick.Bid, 0);
Assert.Greater(tick.BidLevel(0), 0);
Assert.Greater(tick.Ask, 0);
Assert.Greater(tick.AskLevel(0), 0);
}
Assert.IsTrue(tick.IsTrade);
if (tick.IsTrade)
{
Assert.Greater(tick.Price, 0);
Assert.Greater(tick.Size, 0);
}
Assert.AreEqual(symbol, tick.lSymbol);
}
public bool TryReadTick(TickIO tickIO)
{
if (!isInitialized)
{
return(false);
}
if (isLegacy)
{
return(legacy.TryReadTick(tickIO));
}
if (tickCount > MaxCount || endOfData)
{
return(false);
}
try
{
do
{
tickIO.SetSymbol(lSymbol);
if (!fileBlock.TryReadTick(tickIO))
{
ReadNextTickBlock();
if (!fileBlock.TryReadTick(tickIO))
{
throw new InvalidOperationException("Unable to write the first tick in a new block.");
}
}
dataVersion = fileBlock.DataVersion;
if (tickIO.lUtcTime > EndTime.Internal)
{
ReportEndOfData();
return(false);
}
tickCount++;
} while (tickIO.UtcTime < StartTime);
return(true);
}
catch (EndOfStreamException ex)
{
ReportEndOfData();
return(false);
}
}
public void TestLogicalStopOrders()
{
using (VerifyFeed verify = Factory.Utility.VerifyFeed())
using (Provider provider = ProviderFactory()) {
verify.PauseSeconds = secondsDelay;
provider.SendEvent(verify, null, (int)EventType.Connect, null);
provider.SendEvent(verify, symbol, (int)EventType.StartSymbol, new StartSymbolDetail(TimeStamp.MinValue));
VerifyConnected(verify);
ClearOrders(0);
ClearPosition(provider, verify, secondsDelay);
// var expectedTicks = 1;
// var count = verify.Wait(symbol,expectedTicks,secondsDelay);
// Assert.GreaterOrEqual(count,expectedTicks,"at least one tick");
TickIO lastTick = verify.LastTick;
double bid = lastTick.IsTrade ? lastTick.Price : lastTick.Bid;
double ask = lastTick.IsTrade ? lastTick.Price : lastTick.Ask;
int strategyPosition = 0;
LogicalOrder enterBuyStop = CreateEntry(strategy, OrderType.BuyStop, bid + 420 * symbol.MinimumTick, 2, strategyPosition);
LogicalOrder enterSellStop = CreateEntry(strategy, OrderType.SellStop, bid - 400 * symbol.MinimumTick, 2, strategyPosition);
CreateExit(strategy, OrderType.SellStop, bid - 180 * symbol.MinimumTick, strategyPosition);
LogicalOrder exitBuyStop = CreateExit(strategy, OrderType.BuyStop, ask + 540 * symbol.MinimumTick, strategyPosition);
var count = verify.Verify(2, assertTick, symbol, secondsDelay, () => {
SendOrders(provider, verify, 0, secondsDelay);
});
Assert.GreaterOrEqual(count, 2, "tick count");
ClearOrders(0);
enterSellStop.Price = bid - 360 * symbol.MinimumTick;
enterBuyStop.Price = ask + 380 * symbol.MinimumTick;
orders.AddLast(enterBuyStop);
orders.AddLast(enterSellStop);
orders.AddLast(exitBuyStop);
count = verify.Verify(2, assertTick, symbol, secondsDelay, () => {
SendOrders(provider, verify, 0, secondsDelay);
});
Assert.GreaterOrEqual(count, 2, "tick count");
// ClearOrders(0);
// ClearPosition(provider,verify,secondsDelay);
}
}
public void TestLogicalLimitOrders()
{
using (VerifyFeed verify = Factory.Utility.VerifyFeed(symbol))
using (Agent provider = ProviderFactory()) {
verify.PauseSeconds = secondsDelay;
provider.SendEvent(new EventItem(verify, null, EventType.Connect, null));
provider.SendEvent(new EventItem(verify, symbol, EventType.StartSymbol, new StartSymbolDetail(TimeStamp.MinValue)));
VerifyConnected(verify);
ClearOrders(0);
ClearPosition(provider, verify, secondsDelay);
TickIO lastTick = verify.LastTick;
double bid = lastTick.IsTrade ? lastTick.Price : lastTick.Bid;
double ask = lastTick.IsTrade ? lastTick.Price : lastTick.Ask;
int strategyPosition = 0;
LogicalOrder enterBuyLimit = CreateEntry(strategy, OrderType.BuyLimit, bid - 280 * symbol.MinimumTick, 2, strategyPosition);
LogicalOrder enterSellLimit = CreateEntry(strategy, OrderType.SellLimit, ask + 340 * symbol.MinimumTick, 2, strategyPosition);
LogicalOrder exitSellLimit = CreateExit(strategy, OrderType.SellLimit, ask + 380 * symbol.MinimumTick, strategyPosition);
CreateExit(strategy, OrderType.SellLimit, ask + 400 * symbol.MinimumTick, strategyPosition);
CreateExit(strategy, OrderType.BuyLimit, bid - 150 * symbol.MinimumTick, strategyPosition);
LogicalOrder exitBuyStop = CreateExit(strategy, OrderType.BuyStop, ask + 540 * symbol.MinimumTick, strategyPosition);
var count = verify.Verify(2, assertTick, secondsDelay, () => {
SendOrders(provider, verify, 0, secondsDelay);
});
Assert.GreaterOrEqual(count, 2, "tick count");
ClearOrders(0);
enterBuyLimit.Price = bid - 260 * symbol.MinimumTick;
enterSellLimit.Price = ask + 280 * symbol.MinimumTick;
orders.AddLast(enterBuyLimit);
orders.AddLast(enterSellLimit);
orders.AddLast(exitSellLimit);
count = verify.Verify(2, assertTick, secondsDelay, () => {
SendOrders(provider, verify, 0, secondsDelay);
});
Assert.GreaterOrEqual(count, 2, "tick count");
ClearOrders(0);
//ClearPosition(provider,verify,secondsDelay);
count = verify.Wait(1, secondsDelay);
}
}
protected override void TrySendTick(SymbolInfo symbol, TickIO tick)
{
SendSide(symbol, tick, true);
if (tick.IsQuote)
{
#if NOTUSED
bool result;
if (!isFirstTick.TryGetValue(symbol.BinaryIdentifier, out result))
{
isFirstTick.Add(symbol.BinaryIdentifier, true);
}
else
{
var tickSync = SyncTicks.GetTickSync(symbol.BinaryIdentifier);
tickSync.AddTick(tick);
}
#endif
SendSide(symbol, tick, false);
}
var lastTick = lastTicks[symbol.BinaryIdentifier];
lastTick.Inject(tick.Extract());
}
