private void FilterFile(string inputPath, string outputPath, TimeStamp startTime, TimeStamp endTime)
{
using (var reader = Factory.TickUtil.TickFile())
using (var writer = Factory.TickUtil.TickFile())
{
writer.Initialize(outputPath, TickFileMode.Write);
reader.Initialize(inputPath, TickFileMode.Read);
TickIO firstTick = Factory.TickUtil.TickIO();
TickIO lastTick = Factory.TickUtil.TickIO();
TickIO prevTick = Factory.TickUtil.TickIO();
long count = 0;
long dups = 0;
TickIO tickIO = Factory.TickUtil.TickIO();
try
{
while (reader.TryReadTick(tickIO))
{
if (tickIO.Time >= startTime)
{
if (tickIO.Time > endTime)
{
break;
}
if (count == 0)
{
prevTick.Copy(tickIO);
prevTick.IsSimulateTicks = true;
firstTick.Copy(tickIO);
firstTick.IsSimulateTicks = true;
}
count++;
prevTick.Copy(tickIO);
writer.WriteTick(prevTick);
}
}
}
catch (QueueException ex)
{
if (ex.EntryType != EventType.EndHistorical)
{
throw new ApplicationException("Unexpected QueueException: " + ex);
}
}
lastTick.Copy(tickIO);
Output(reader.Symbol + ": " + count + " ticks.");
Output("From " + firstTick.Time + " to " + lastTick.Time);
Output(dups + " duplicates elimated.");
}
}
public long Wait(int expectedTicks, int timeout)
{
if (debug)
{
log.Debug("Wait");
}
long startTime = Factory.Parallel.TickCount;
lastTick = Factory.TickUtil.TickIO();
count = 0;
while (Factory.Parallel.TickCount - startTime < timeout * 1000)
{
if (propagateException != null)
{
throw propagateException;
}
try {
if (TryDequeueTick(ref tickBinary))
{
tickIO.Inject(tickBinary);
if (debug && count < 5)
{
log.Debug("Received a tick " + tickIO + " UTC " + tickIO.UtcTime);
countLog++;
}
else if (trace)
{
log.Trace("Received a tick " + tickIO + " UTC " + tickIO.UtcTime);
}
count++;
lastTick.Copy(tickIO);
if (SyncTicks.Enabled && symbolState == SymbolState.RealTime)
{
tickSync.RemoveTick(ref tickBinary);
}
if (count >= expectedTicks)
{
break;
}
}
else
{
Thread.Sleep(100);
}
} catch (QueueException ex) {
if (HandleQueueException(ex))
{
break;
}
}
}
return(count);
}
private void SendTick()
{
lastTick.Copy(tick, tick.ContentMask);
TickBinary binary = new TickBinary();
binary = tick.Extract();
lastChangeTime = Environment.TickCount;
receiver.OnEvent(symbol, (int)EventType.Tick, binary);
if (debug)
{
log.Debug("Sent Tick: " + tick);
}
}
private void MigrateFile(string file, string symbol)
{
if (File.Exists(file + ".back"))
{
Console.WriteLine("A backup file already exists. Please delete it first at: " + file + ".back");
return;
}
using (var reader = Factory.TickUtil.TickReader())
using (var writer = Factory.TickUtil.TickWriter(true)) {
reader.Initialize(file, symbol);
writer.KeepFileOpen = true;
writer.Initialize(file + ".temp", symbol);
TickIO firstTick = Factory.TickUtil.TickIO();
TickIO tickIO = Factory.TickUtil.TickIO();
TickBinary tickBinary = new TickBinary();
int count = 0;
bool first = false;
try {
while (true)
{
while (!reader.ReadQueue.TryDequeue(ref tickBinary))
{
Thread.Sleep(1);
}
tickIO.Inject(tickBinary);
while (!writer.TryAdd(tickIO))
{
Thread.Sleep(1);
}
if (first)
{
firstTick.Copy(tickIO);
first = false;
}
count++;
}
} catch (QueueException ex) {
if (ex.EntryType != EventType.EndHistorical)
{
throw new ApplicationException("Unexpected QueueException: " + ex);
}
}
Console.WriteLine(reader.Symbol + ": Migrated " + count + " ticks from " + firstTick.Time + " to " + tickIO.Time);
}
File.Move(file, file + ".back");
File.Move(file + ".temp", file);
}
private void MigrateFile(string file)
{
if (File.Exists(file + ".back"))
{
Output("A backup file already exists. Please delete it first at: " + file + ".back");
return;
}
TickFile reader;
TickFile writer;
int count = 0;
TickIO firstTick = Factory.TickUtil.TickIO();
TickIO tickIO = Factory.TickUtil.TickIO();
try
{
using (reader = Factory.TickUtil.TickFile())
{
using (writer = Factory.TickUtil.TickFile())
{
reader.Initialize(file, TickFileMode.Read);
writer.EraseFileToStart = true;
writer.Initialize(file + ".temp", TickFileMode.Write);
bool first = true;
while (reader.TryReadTick(tickIO))
{
writer.WriteTick(tickIO);
if (first)
{
firstTick.Copy(tickIO);
first = false;
}
count++;
}
}
}
Output(reader.Symbol + ": Migrated " + count + " ticks from " + firstTick.Time + " to " + tickIO.Time);
Alter.MoveFile(file, file + ".back");
Alter.MoveFile(file + ".temp", file);
}
catch (Exception ex)
{
Output("Error: " + ex.Message);
}
}
private void FilterFile(string symbol, string inputPath, string outputPath, TimeStamp startTime, TimeStamp endTime)
{
TickReader reader = Factory.TickUtil.TickReader();
TickWriter writer = Factory.TickUtil.TickWriter(true);
writer.KeepFileOpen = true;
writer.Initialize(outputPath, symbol);
reader.Initialize(inputPath, symbol);
TickQueue inputQueue = reader.ReadQueue;
TickIO firstTick = Factory.TickUtil.TickIO();
TickIO lastTick = Factory.TickUtil.TickIO();
TickIO prevTick = Factory.TickUtil.TickIO();
long count = 0;
long fast = 0;
long dups = 0;
TickIO tickIO = Factory.TickUtil.TickIO();
TickBinary tickBinary = new TickBinary();
inputQueue.Dequeue(ref tickBinary);
tickIO.Inject(tickBinary);
count++;
firstTick.Copy(tickIO);
firstTick.IsSimulateTicks = true;
prevTick.Copy(tickIO);
prevTick.IsSimulateTicks = true;
if (tickIO.Time >= startTime)
{
writer.Add(firstTick);
}
try {
while (true)
{
while (!inputQueue.TryDequeue(ref tickBinary))
{
Thread.Sleep(1);
}
tickIO.Inject(tickBinary);
count++;
if (tickIO.Time >= startTime)
{
if (tickIO.Time > endTime)
{
break;
}
// if( tickIO.Bid == prevTick.Bid && tickIO.Ask == prevTick.Ask) {
// dups++;
// } else {
// Elapsed elapsed = tickIO.Time - prevTick.Time;
prevTick.Copy(tickIO);
prevTick.IsSimulateTicks = true;
// if( elapsed.TotalMilliseconds < 5000) {
// fast++;
// } else {
while (!writer.TryAdd(prevTick))
{
Thread.Sleep(1);
}
// }
// }
}
}
} catch (QueueException ex) {
if (ex.EntryType != EventType.EndHistorical)
{
throw new ApplicationException("Unexpected QueueException: " + ex);
}
}
lastTick.Copy(tickIO);
Console.WriteLine(reader.Symbol + ": " + count + " ticks from " + firstTick.Time + " to " + lastTick.Time + " " + dups + " duplicates, " + fast + " less than 50 ms");
Factory.TickUtil.TickReader().CloseAll();
writer.Close();
}
public void ReadFile()
{
try
{
TickIO firstTick = Factory.TickUtil.TickIO();
TickIO lastTick = Factory.TickUtil.TickIO();
TickIO prevTick = Factory.TickUtil.TickIO();
long count = 0;
long dups = 0;
long quotes = 0;
long trades = 0;
long quotesAndTrades = 0;
var tickIO = Factory.TickUtil.TickIO();
try
{
while (reader.TryReadTick(tickIO))
{
if (count == 0)
{
firstTick.Copy(tickIO);
}
if (tickIO.IsQuote && tickIO.IsTrade)
{
quotesAndTrades++;
}
else if (tickIO.IsQuote)
{
quotes++;
}
else
{
trades++;
}
if (count > 0)
{
bool quoteDup = tickIO.IsQuote && prevTick.IsQuote && tickIO.Bid == prevTick.Bid && tickIO.Ask == prevTick.Ask;
bool tradeDup = tickIO.IsTrade && prevTick.IsTrade && tickIO.Price == prevTick.Price;
if (tickIO.IsQuote && tickIO.IsTrade)
{
if (quoteDup && tradeDup)
{
dups++;
}
}
else if (tickIO.IsQuote)
{
if (quoteDup)
{
dups++;
}
}
else
{
if (tradeDup)
{
dups++;
}
}
}
count++;
prevTick.Copy(tickIO);
}
}
catch (QueueException)
{
// Terminated.
}
lastTick.Copy(tickIO);
stringBuilder.AppendLine("Symbol: " + reader.Symbol);
stringBuilder.AppendLine("Version: " + reader.DataVersion);
stringBuilder.AppendLine("Ticks: " + count);
if (quotes > 0)
{
stringBuilder.AppendLine("Quote Only: " + quotes);
}
if (trades > 0)
{
stringBuilder.AppendLine("Trade Only: " + trades);
}
if (quotesAndTrades > 0)
{
stringBuilder.AppendLine("Quote and Trade: " + quotesAndTrades);
}
var time = firstTick.Time.ToString();
var utcTime = firstTick.UtcTime.ToString();
stringBuilder.AppendLine("From: " + time + " (local), " + utcTime + " (UTC)");
time = lastTick.Time.ToString();
utcTime = lastTick.UtcTime.ToString();
stringBuilder.AppendLine(" To: " + time + " (local), " + utcTime + " (UTC)");
if (dups > 0)
{
stringBuilder.AppendLine("Prices duplicates: " + dups);
}
}
finally
{
reader.Dispose();
}
}
public long Verify(int expectedCount, Action <TickIO, TickIO, long> assertTick, SymbolInfo symbol, int timeout, Action action)
{
if (debug)
{
log.Debug("Verify");
}
if (SyncTicks.Enabled)
{
tickSync = SyncTicks.GetTickSync(symbol.BinaryIdentifier);
}
long endTime = Factory.Parallel.TickCount + timeout * 1000;
count = 0;
while (Factory.Parallel.TickCount < endTime)
{
if (propagateException != null)
{
throw propagateException;
}
try {
if (tickQueue.TryDequeue(ref tickBinary))
{
tickIO.Inject(tickBinary);
if (debug && countLog < 5)
{
log.Debug("Received a tick " + tickIO);
countLog++;
}
startTime = Factory.TickCount;
count++;
if (count > 0 && assertTick != null)
{
assertTick(tickIO, lastTick, symbol.BinaryIdentifier);
}
lastTick.Copy(tickIO);
if (!actionAlreadyRun && action != null)
{
actionAlreadyRun = true;
action();
}
if (SyncTicks.Enabled)
{
tickSync.RemoveTick();
}
if (count >= expectedCount)
{
break;
}
}
else
{
Thread.Sleep(100);
}
} catch (QueueException ex) {
if (HandleQueueException(ex))
{
break;
}
}
}
return(count);
}
public long Verify(int expectedCount, Action <TickIO, TickIO, long> assertTick, int timeout, Action action)
{
if (debug)
{
log.Debug("Verify");
}
long endTime = Factory.Parallel.TickCount + timeout * 1000;
count = 0;
do
{
if (propagateException != null)
{
throw propagateException;
}
try
{
if (TryDequeueTick(ref tickBinary))
{
tickIO.Inject(tickBinary);
if (debug && countLog < 5)
{
log.Debug("Received a tick " + tickIO + " UTC " + tickIO.UtcTime);
countLog++;
}
else if (trace)
{
log.Trace("Received a tick " + tickIO + " UTC " + tickIO.UtcTime);
}
startTime = Factory.TickCount;
count++;
if (count > 0)
{
if (assertTick != null)
{
assertTick(tickIO, lastTick, symbol.BinaryIdentifier);
}
if (count % 10000 == 0)
{
log.Info("Read " + count + " ticks");
}
}
lastTick.Copy(tickIO);
if (!actionAlreadyRun && action != null)
{
actionAlreadyRun = true;
action();
}
if (SyncTicks.Enabled && symbolState == SymbolState.RealTime)
{
tickSync.RemoveTick(ref tickBinary);
}
if (count >= expectedCount)
{
break;
}
}
else
{
Thread.Sleep(100);
//if (queue.Count == 0 && SyncTicks.Enabled && SymbolState == SymbolState.RealTime)
//{
// tickSync.RemoveTick();
//}
}
}
catch (QueueException ex)
{
if (HandleQueueException(ex))
{
break;
}
}
} while (Factory.Parallel.TickCount < endTime);
return(count);
}
public void ReadFile()
{
TickQueue queue = reader.ReadQueue;
TickIO firstTick = Factory.TickUtil.TickIO();
TickIO lastTick = Factory.TickUtil.TickIO();
TickIO prevTick = Factory.TickUtil.TickIO();
long count = 0;
long dups = 0;
long quotes = 0;
long trades = 0;
long quotesAndTrades = 0;
TickIO tickIO = Factory.TickUtil.TickIO();
TickBinary tickBinary = new TickBinary();
try {
while (true)
{
if (!TryGetNextTick(queue, ref tickBinary))
{
break;
}
tickIO.Inject(tickBinary);
if (count == 0)
{
firstTick.Copy(tickIO);
}
if (tickIO.IsQuote && tickIO.IsTrade)
{
quotesAndTrades++;
}
else if (tickIO.IsQuote)
{
quotes++;
}
else
{
trades++;
}
if (count > 0)
{
bool quoteDup = tickIO.IsQuote && prevTick.IsQuote && tickIO.Bid == prevTick.Bid && tickIO.Ask == prevTick.Ask;
bool tradeDup = tickIO.IsTrade && prevTick.IsTrade && tickIO.Price == prevTick.Price;
if (tickIO.IsQuote && tickIO.IsTrade)
{
if (quoteDup && tradeDup)
{
dups++;
}
}
else if (tickIO.IsQuote)
{
if (quoteDup)
{
dups++;
}
}
else
{
if (tradeDup)
{
dups++;
}
}
}
count++;
prevTick.Copy(tickIO);
}
} catch (QueueException) {
// Terminated.
}
lastTick.Copy(tickIO);
stringBuilder.AppendLine("Symbol: " + reader.Symbol);
stringBuilder.AppendLine("Version: " + reader.DataVersion);
stringBuilder.AppendLine("Ticks: " + count);
if (quotes > 0)
{
stringBuilder.AppendLine("Quote Only: " + quotes);
}
if (trades > 0)
{
stringBuilder.AppendLine("Trade Only: " + trades);
}
if (quotesAndTrades > 0)
{
stringBuilder.AppendLine("Quote and Trade: " + quotesAndTrades);
}
stringBuilder.AppendLine("From: " + firstTick.Time + "." + firstTick.Time.Microsecond);
stringBuilder.AppendLine("To: " + lastTick.Time + "." + lastTick.Time.Microsecond);
if (dups > 0)
{
stringBuilder.AppendLine("Prices duplicates: " + dups);
}
Factory.TickUtil.TickReader().CloseAll();
}
