private void FilterFile(string inputPath, string outputPath, TimeStamp startTime, TimeStamp endTime) { using (var reader = Factory.TickUtil.TickFile()) using (var writer = Factory.TickUtil.TickFile()) { writer.Initialize(outputPath, TickFileMode.Write); reader.Initialize(inputPath, TickFileMode.Read); TickIO firstTick = Factory.TickUtil.TickIO(); TickIO lastTick = Factory.TickUtil.TickIO(); TickIO prevTick = Factory.TickUtil.TickIO(); long count = 0; long dups = 0; TickIO tickIO = Factory.TickUtil.TickIO(); try { while (reader.TryReadTick(tickIO)) { if (tickIO.Time >= startTime) { if (tickIO.Time > endTime) { break; } if (count == 0) { prevTick.Copy(tickIO); prevTick.IsSimulateTicks = true; firstTick.Copy(tickIO); firstTick.IsSimulateTicks = true; } count++; prevTick.Copy(tickIO); writer.WriteTick(prevTick); } } } catch (QueueException ex) { if (ex.EntryType != EventType.EndHistorical) { throw new ApplicationException("Unexpected QueueException: " + ex); } } lastTick.Copy(tickIO); Output(reader.Symbol + ": " + count + " ticks."); Output("From " + firstTick.Time + " to " + lastTick.Time); Output(dups + " duplicates elimated."); } }
public long Wait(int expectedTicks, int timeout) { if (debug) { log.Debug("Wait"); } long startTime = Factory.Parallel.TickCount; lastTick = Factory.TickUtil.TickIO(); count = 0; while (Factory.Parallel.TickCount - startTime < timeout * 1000) { if (propagateException != null) { throw propagateException; } try { if (TryDequeueTick(ref tickBinary)) { tickIO.Inject(tickBinary); if (debug && count < 5) { log.Debug("Received a tick " + tickIO + " UTC " + tickIO.UtcTime); countLog++; } else if (trace) { log.Trace("Received a tick " + tickIO + " UTC " + tickIO.UtcTime); } count++; lastTick.Copy(tickIO); if (SyncTicks.Enabled && symbolState == SymbolState.RealTime) { tickSync.RemoveTick(ref tickBinary); } if (count >= expectedTicks) { break; } } else { Thread.Sleep(100); } } catch (QueueException ex) { if (HandleQueueException(ex)) { break; } } } return(count); }
private void SendTick() { lastTick.Copy(tick, tick.ContentMask); TickBinary binary = new TickBinary(); binary = tick.Extract(); lastChangeTime = Environment.TickCount; receiver.OnEvent(symbol, (int)EventType.Tick, binary); if (debug) { log.Debug("Sent Tick: " + tick); } }
private void MigrateFile(string file, string symbol) { if (File.Exists(file + ".back")) { Console.WriteLine("A backup file already exists. Please delete it first at: " + file + ".back"); return; } using (var reader = Factory.TickUtil.TickReader()) using (var writer = Factory.TickUtil.TickWriter(true)) { reader.Initialize(file, symbol); writer.KeepFileOpen = true; writer.Initialize(file + ".temp", symbol); TickIO firstTick = Factory.TickUtil.TickIO(); TickIO tickIO = Factory.TickUtil.TickIO(); TickBinary tickBinary = new TickBinary(); int count = 0; bool first = false; try { while (true) { while (!reader.ReadQueue.TryDequeue(ref tickBinary)) { Thread.Sleep(1); } tickIO.Inject(tickBinary); while (!writer.TryAdd(tickIO)) { Thread.Sleep(1); } if (first) { firstTick.Copy(tickIO); first = false; } count++; } } catch (QueueException ex) { if (ex.EntryType != EventType.EndHistorical) { throw new ApplicationException("Unexpected QueueException: " + ex); } } Console.WriteLine(reader.Symbol + ": Migrated " + count + " ticks from " + firstTick.Time + " to " + tickIO.Time); } File.Move(file, file + ".back"); File.Move(file + ".temp", file); }
private void MigrateFile(string file) { if (File.Exists(file + ".back")) { Output("A backup file already exists. Please delete it first at: " + file + ".back"); return; } TickFile reader; TickFile writer; int count = 0; TickIO firstTick = Factory.TickUtil.TickIO(); TickIO tickIO = Factory.TickUtil.TickIO(); try { using (reader = Factory.TickUtil.TickFile()) { using (writer = Factory.TickUtil.TickFile()) { reader.Initialize(file, TickFileMode.Read); writer.EraseFileToStart = true; writer.Initialize(file + ".temp", TickFileMode.Write); bool first = true; while (reader.TryReadTick(tickIO)) { writer.WriteTick(tickIO); if (first) { firstTick.Copy(tickIO); first = false; } count++; } } } Output(reader.Symbol + ": Migrated " + count + " ticks from " + firstTick.Time + " to " + tickIO.Time); Alter.MoveFile(file, file + ".back"); Alter.MoveFile(file + ".temp", file); } catch (Exception ex) { Output("Error: " + ex.Message); } }
private void FilterFile(string symbol, string inputPath, string outputPath, TimeStamp startTime, TimeStamp endTime) { TickReader reader = Factory.TickUtil.TickReader(); TickWriter writer = Factory.TickUtil.TickWriter(true); writer.KeepFileOpen = true; writer.Initialize(outputPath, symbol); reader.Initialize(inputPath, symbol); TickQueue inputQueue = reader.ReadQueue; TickIO firstTick = Factory.TickUtil.TickIO(); TickIO lastTick = Factory.TickUtil.TickIO(); TickIO prevTick = Factory.TickUtil.TickIO(); long count = 0; long fast = 0; long dups = 0; TickIO tickIO = Factory.TickUtil.TickIO(); TickBinary tickBinary = new TickBinary(); inputQueue.Dequeue(ref tickBinary); tickIO.Inject(tickBinary); count++; firstTick.Copy(tickIO); firstTick.IsSimulateTicks = true; prevTick.Copy(tickIO); prevTick.IsSimulateTicks = true; if (tickIO.Time >= startTime) { writer.Add(firstTick); } try { while (true) { while (!inputQueue.TryDequeue(ref tickBinary)) { Thread.Sleep(1); } tickIO.Inject(tickBinary); count++; if (tickIO.Time >= startTime) { if (tickIO.Time > endTime) { break; } // if( tickIO.Bid == prevTick.Bid && tickIO.Ask == prevTick.Ask) { // dups++; // } else { // Elapsed elapsed = tickIO.Time - prevTick.Time; prevTick.Copy(tickIO); prevTick.IsSimulateTicks = true; // if( elapsed.TotalMilliseconds < 5000) { // fast++; // } else { while (!writer.TryAdd(prevTick)) { Thread.Sleep(1); } // } // } } } } catch (QueueException ex) { if (ex.EntryType != EventType.EndHistorical) { throw new ApplicationException("Unexpected QueueException: " + ex); } } lastTick.Copy(tickIO); Console.WriteLine(reader.Symbol + ": " + count + " ticks from " + firstTick.Time + " to " + lastTick.Time + " " + dups + " duplicates, " + fast + " less than 50 ms"); Factory.TickUtil.TickReader().CloseAll(); writer.Close(); }
public void ReadFile() { try { TickIO firstTick = Factory.TickUtil.TickIO(); TickIO lastTick = Factory.TickUtil.TickIO(); TickIO prevTick = Factory.TickUtil.TickIO(); long count = 0; long dups = 0; long quotes = 0; long trades = 0; long quotesAndTrades = 0; var tickIO = Factory.TickUtil.TickIO(); try { while (reader.TryReadTick(tickIO)) { if (count == 0) { firstTick.Copy(tickIO); } if (tickIO.IsQuote && tickIO.IsTrade) { quotesAndTrades++; } else if (tickIO.IsQuote) { quotes++; } else { trades++; } if (count > 0) { bool quoteDup = tickIO.IsQuote && prevTick.IsQuote && tickIO.Bid == prevTick.Bid && tickIO.Ask == prevTick.Ask; bool tradeDup = tickIO.IsTrade && prevTick.IsTrade && tickIO.Price == prevTick.Price; if (tickIO.IsQuote && tickIO.IsTrade) { if (quoteDup && tradeDup) { dups++; } } else if (tickIO.IsQuote) { if (quoteDup) { dups++; } } else { if (tradeDup) { dups++; } } } count++; prevTick.Copy(tickIO); } } catch (QueueException) { // Terminated. } lastTick.Copy(tickIO); stringBuilder.AppendLine("Symbol: " + reader.Symbol); stringBuilder.AppendLine("Version: " + reader.DataVersion); stringBuilder.AppendLine("Ticks: " + count); if (quotes > 0) { stringBuilder.AppendLine("Quote Only: " + quotes); } if (trades > 0) { stringBuilder.AppendLine("Trade Only: " + trades); } if (quotesAndTrades > 0) { stringBuilder.AppendLine("Quote and Trade: " + quotesAndTrades); } var time = firstTick.Time.ToString(); var utcTime = firstTick.UtcTime.ToString(); stringBuilder.AppendLine("From: " + time + " (local), " + utcTime + " (UTC)"); time = lastTick.Time.ToString(); utcTime = lastTick.UtcTime.ToString(); stringBuilder.AppendLine(" To: " + time + " (local), " + utcTime + " (UTC)"); if (dups > 0) { stringBuilder.AppendLine("Prices duplicates: " + dups); } } finally { reader.Dispose(); } }
public long Verify(int expectedCount, Action <TickIO, TickIO, long> assertTick, SymbolInfo symbol, int timeout, Action action) { if (debug) { log.Debug("Verify"); } if (SyncTicks.Enabled) { tickSync = SyncTicks.GetTickSync(symbol.BinaryIdentifier); } long endTime = Factory.Parallel.TickCount + timeout * 1000; count = 0; while (Factory.Parallel.TickCount < endTime) { if (propagateException != null) { throw propagateException; } try { if (tickQueue.TryDequeue(ref tickBinary)) { tickIO.Inject(tickBinary); if (debug && countLog < 5) { log.Debug("Received a tick " + tickIO); countLog++; } startTime = Factory.TickCount; count++; if (count > 0 && assertTick != null) { assertTick(tickIO, lastTick, symbol.BinaryIdentifier); } lastTick.Copy(tickIO); if (!actionAlreadyRun && action != null) { actionAlreadyRun = true; action(); } if (SyncTicks.Enabled) { tickSync.RemoveTick(); } if (count >= expectedCount) { break; } } else { Thread.Sleep(100); } } catch (QueueException ex) { if (HandleQueueException(ex)) { break; } } } return(count); }
public long Verify(int expectedCount, Action <TickIO, TickIO, long> assertTick, int timeout, Action action) { if (debug) { log.Debug("Verify"); } long endTime = Factory.Parallel.TickCount + timeout * 1000; count = 0; do { if (propagateException != null) { throw propagateException; } try { if (TryDequeueTick(ref tickBinary)) { tickIO.Inject(tickBinary); if (debug && countLog < 5) { log.Debug("Received a tick " + tickIO + " UTC " + tickIO.UtcTime); countLog++; } else if (trace) { log.Trace("Received a tick " + tickIO + " UTC " + tickIO.UtcTime); } startTime = Factory.TickCount; count++; if (count > 0) { if (assertTick != null) { assertTick(tickIO, lastTick, symbol.BinaryIdentifier); } if (count % 10000 == 0) { log.Info("Read " + count + " ticks"); } } lastTick.Copy(tickIO); if (!actionAlreadyRun && action != null) { actionAlreadyRun = true; action(); } if (SyncTicks.Enabled && symbolState == SymbolState.RealTime) { tickSync.RemoveTick(ref tickBinary); } if (count >= expectedCount) { break; } } else { Thread.Sleep(100); //if (queue.Count == 0 && SyncTicks.Enabled && SymbolState == SymbolState.RealTime) //{ // tickSync.RemoveTick(); //} } } catch (QueueException ex) { if (HandleQueueException(ex)) { break; } } } while (Factory.Parallel.TickCount < endTime); return(count); }
public void ReadFile() { TickQueue queue = reader.ReadQueue; TickIO firstTick = Factory.TickUtil.TickIO(); TickIO lastTick = Factory.TickUtil.TickIO(); TickIO prevTick = Factory.TickUtil.TickIO(); long count = 0; long dups = 0; long quotes = 0; long trades = 0; long quotesAndTrades = 0; TickIO tickIO = Factory.TickUtil.TickIO(); TickBinary tickBinary = new TickBinary(); try { while (true) { if (!TryGetNextTick(queue, ref tickBinary)) { break; } tickIO.Inject(tickBinary); if (count == 0) { firstTick.Copy(tickIO); } if (tickIO.IsQuote && tickIO.IsTrade) { quotesAndTrades++; } else if (tickIO.IsQuote) { quotes++; } else { trades++; } if (count > 0) { bool quoteDup = tickIO.IsQuote && prevTick.IsQuote && tickIO.Bid == prevTick.Bid && tickIO.Ask == prevTick.Ask; bool tradeDup = tickIO.IsTrade && prevTick.IsTrade && tickIO.Price == prevTick.Price; if (tickIO.IsQuote && tickIO.IsTrade) { if (quoteDup && tradeDup) { dups++; } } else if (tickIO.IsQuote) { if (quoteDup) { dups++; } } else { if (tradeDup) { dups++; } } } count++; prevTick.Copy(tickIO); } } catch (QueueException) { // Terminated. } lastTick.Copy(tickIO); stringBuilder.AppendLine("Symbol: " + reader.Symbol); stringBuilder.AppendLine("Version: " + reader.DataVersion); stringBuilder.AppendLine("Ticks: " + count); if (quotes > 0) { stringBuilder.AppendLine("Quote Only: " + quotes); } if (trades > 0) { stringBuilder.AppendLine("Trade Only: " + trades); } if (quotesAndTrades > 0) { stringBuilder.AppendLine("Quote and Trade: " + quotesAndTrades); } stringBuilder.AppendLine("From: " + firstTick.Time + "." + firstTick.Time.Microsecond); stringBuilder.AppendLine("To: " + lastTick.Time + "." + lastTick.Time.Microsecond); if (dups > 0) { stringBuilder.AppendLine("Prices duplicates: " + dups); } Factory.TickUtil.TickReader().CloseAll(); }