// [TestFixtureSetUp] public void Init() { tickArray = new TickReader(); // tickArray.TimeFrames.Add( Period.Minute10); // tickArray.TimeFrames.Add( Period.Hour4); tickArray.Initialize("UnitTestData", "USD_JPY"); }
public virtual Provider[] SetupProviders(bool quietMode, bool singleLoad) { List <Provider> senderList = new List <Provider>(); SymbolInfo[] symbols = ProjectProperties.Starter.SymbolProperties; string[] symbolFiles = projectProperties.Starter.SymbolArray; for (int i = 0; i < symbols.Length; i++) { TickReader tickReader = Factory.TickUtil.TickReader(); tickReader.MaxCount = EndCount; tickReader.StartTime = ProjectProperties.Starter.StartTime; tickReader.EndTime = ProjectProperties.Starter.EndTime; tickReader.BackgroundWorker = BackgroundWorker; tickReader.LogProgress = true; tickReader.BulkFileLoad = singleLoad; tickReader.QuietMode = quietMode; try { tickReader.Initialize(DataFolder, symbolFiles[i]); senderList.Add(tickReader); } catch (System.IO.FileNotFoundException ex) { throw new ApplicationException("Error: " + ex.Message); } } return(senderList.ToArray()); }
public void TZConverter(string file, string symbol) { TickReader reader = Factory.TickUtil.TickReader(); reader.Initialize(file, symbol); TickQueue queue = reader.ReadQueue; TickBinary tickBinary = new TickBinary(); if (!TryGetNextTick(queue, ref tickBinary)) { } }
public void BinaryDiffTest() { // string pair = "USD_JPY_Huge"; // string pair = "USD_JPY_YEARS"; string pair = "USD_JPY"; TickReader tickReader = new TickReader(); tickReader.LogProgress = true; tickReader.Initialize("Test\\DataCache", pair); byte[] previous = new byte[1024]; MemoryStream stream = new MemoryStream(); diffBits = new BitArray(1024); long fileSize = 0; long compressSize = 0; int count = 0; try { TickBinary tick = new TickBinary(); TickIO tickIO = new TickImpl(); while (true) { tickReader.ReadQueue.Dequeue(ref tick); stream.Seek(0, SeekOrigin.Begin); tickIO.Inject(tick); tickIO.ToWriter(stream); CompareSignature(stream.GetBuffer(), previous, stream.Position); fileSize += stream.Position; int totDiffLength = 1 + diffLength + diffBits.ByteCount; compressSize += totDiffLength; if (count > 1000 && count < 2000) { // log.WriteFile(ByteArrayToString(memory,stream.Position)); if (logging) { log.Debug(count + ": " + totDiffLength + " " + diffLength + " byte " + ByteArrayToString(diffBits.Bytes, diffBits.ByteCount) + " " + ByteArrayToString(diff, diffLength)); } } count++; Array.Copy(stream.GetBuffer(), previous, stream.Position); } } catch (QueueException ex) { Assert.AreEqual(EventType.EndHistorical, ex.EntryType); } catch (CollectionTerminatedException) { } TickReader.CloseAll(); log.Debug("File Size = " + fileSize + ", Compressed Size = " + compressSize); }
public void SignatureResult() { string pair = "USD_JPY_Volume"; TickReader tickReader = new TickReader(); tickReader.LogProgress = true; tickReader.Initialize("Test\\DataCache", pair); int totalBytes = 0; SignatureCompressor compressor = new SignatureCompressor(); ByteMemory compressed = new ByteMemory("compressed"); int compressedLength = 0; try { TickBinary tick = new TickBinary(); TickImpl tickImpl = new TickImpl(); for (int i = 0; i < 101; i++) { tickReader.ReadQueue.Dequeue(ref tick); tickImpl.Inject(tick); compressed.Reset(); compressor.CompressTick(tick, compressed.Memory); totalBytes += compressedLength; } log.Debug(compressor.Signature.ToString()); Assert.AreEqual(TestSignature, compressor.Signature); for (int i = 0; i < 200; i++) { tickReader.ReadQueue.Dequeue(ref tick); tickImpl.Inject(tick); compressed.Reset(); compressor.CompressTick(tick, compressed.Memory); totalBytes += compressedLength; int length = compressed.Bytes[0]; int diffBytes = length / 8 + 1; log.Debug(compressedLength + ": " + compressed.Bytes[0] + " " + ByteArrayToString(compressor.Difference, 0, compressor.DiffLength)); } } catch (CollectionTerminatedException) { } log.Debug("Total Compressed Bytes: " + totalBytes); compressor.LogCounts(); }
public FIXServerSymbolHandler(FIXSimulatorSupport fixSimulatorSupport, bool isPlayBack, string symbolString, Func <Yield> onHeartbeat, Func <SymbolInfo, Tick, Yield> onTick, Action <PhysicalFill, int, int, int> onPhysicalFill, Action <PhysicalOrder, string> onRejectOrder) { this.fixSimulatorSupport = fixSimulatorSupport; this.isPlayBack = isPlayBack; this.onHeartbeat = onHeartbeat; this.onTick = onTick; this.symbol = Factory.Symbol.LookupSymbol(symbolString); reader = Factory.TickUtil.TickReader(); reader.Initialize("Test\\MockProviderData", symbolString); fillSimulator = Factory.Utility.FillSimulator("FIX", symbol, false); fillSimulator.OnPhysicalFill = onPhysicalFill; fillSimulator.OnRejectOrder = onRejectOrder; tickSync = SyncTicks.GetTickSync(symbol.BinaryIdentifier); tickSync.ForceClear(); queueTask = Factory.Parallel.Loop("FIXServerSymbol-" + symbolString, OnException, ProcessQueue); firstHearbeat = true; }
public void CompressionTest() { // string pair = "USD_JPY_Huge"; string pair = "USD_JPY_YEARS"; TickReader tickReader = new TickReader(); tickReader.LogProgress = true; tickReader.Initialize("TestData", pair); TickCompressor compressor = new TickCompressor(true); compressor.LogProgress = true; compressor.Initialize("Compressed", pair); long fileSize = 0; long compressSize = 0; TickIO tick = new TickImpl(); TickImpl tickImpl = new TickImpl(); TickBinary tickBinary = new TickBinary(); try { for (int i = 0;; i++) { if (!tickReader.ReadQueue.Dequeue(ref tickBinary)) { continue; } tickImpl.init(tickBinary); compressor.Add(tickImpl); } } catch (QueueException ex) { Assert.AreEqual(EntryType.EndHistorical, ex.EntryType); } catch (CollectionTerminatedException) { } tickReader.Stop(); compressor.Close(); log.WriteFile("File Size = " + fileSize + ", Compressed Size = " + compressSize); }
public void TearDown() { TickReader.CloseAll(); }
// [Test] public void CompressTickTest() { string pair = "USD_JPY_Volume"; TickReader tickReader = new TickReader(); tickReader.LogProgress = true; tickReader.Initialize("Test\\DataCache", pair); int totalBytes = 0; SignatureCompressor compressor = new SignatureCompressor(); SignatureCompressor decompressor = new SignatureCompressor(); ByteMemory output = new ByteMemory("compressed"); int length = 0; try { TickBinary tick = new TickBinary(); TickImpl tickImpl = new TickImpl(); for (int i = 0; i < 101; i++) { tickReader.ReadQueue.Dequeue(ref tick); tickImpl.Inject(tick); compressor.CompareTick(tickImpl); if (compressor.Count < 100) { compressor.CopyMemory(output.Bytes, out length); } compressor.SwapBuffers(); compressor.Count++; if (compressor.Count % 100 == 0) { compressor.CreateSignature(); compressor.ResetCounters(); } } string temp = ""; for (int i = 0; i < compressor.Signature.Length; i++) { if (i != 0) { temp += ", "; } temp += compressor.Signature[i]; } log.Notice("signature = " + temp); Assert.AreEqual(TestSignature, compressor.Signature); byte[] buffer = new byte[1024]; for (int i = 0; i < 200; i++) { tickReader.ReadQueue.Dequeue(ref tick); tickImpl.Inject(tick); compressor.CompareTick(tickImpl); compressor.CalculateDifference(compressor.Current.Bytes, compressor.Previous.Bytes, compressor.Current.Length); Array.Copy(compressor.Current.Bytes, buffer, compressor.Current.Length); compressor.ReverseDifference(compressor.Current.Bytes, compressor.Previous.Bytes, compressor.Previous.Length); Assert.AreEqual(buffer, compressor.Current.Bytes); compressor.WriteMemory(output.Bytes, out length); compressor.SwapBuffers(); compressor.Count++; if (compressor.Count % 100 == 0) { compressor.CreateSignature(); compressor.ResetCounters(); } } } catch (CollectionTerminatedException) { } log.Debug("Total Compressed Bytes: " + totalBytes); compressor.LogCounts(); }
public void Dispose() { TickReader.CloseAll(); }
private void FilterFile(string symbol, string inputPath, string outputPath, TimeStamp startTime, TimeStamp endTime) { TickReader reader = Factory.TickUtil.TickReader(); TickWriter writer = Factory.TickUtil.TickWriter(true); writer.KeepFileOpen = true; writer.Initialize(outputPath, symbol); reader.Initialize(inputPath, symbol); TickQueue inputQueue = reader.ReadQueue; TickIO firstTick = Factory.TickUtil.TickIO(); TickIO lastTick = Factory.TickUtil.TickIO(); TickIO prevTick = Factory.TickUtil.TickIO(); long count = 0; long fast = 0; long dups = 0; TickIO tickIO = Factory.TickUtil.TickIO(); TickBinary tickBinary = new TickBinary(); inputQueue.Dequeue(ref tickBinary); tickIO.Inject(tickBinary); count++; firstTick.Copy(tickIO); firstTick.IsSimulateTicks = true; prevTick.Copy(tickIO); prevTick.IsSimulateTicks = true; if (tickIO.Time >= startTime) { writer.Add(firstTick); } try { while (true) { while (!inputQueue.TryDequeue(ref tickBinary)) { Thread.Sleep(1); } tickIO.Inject(tickBinary); count++; if (tickIO.Time >= startTime) { if (tickIO.Time > endTime) { break; } // if( tickIO.Bid == prevTick.Bid && tickIO.Ask == prevTick.Ask) { // dups++; // } else { // Elapsed elapsed = tickIO.Time - prevTick.Time; prevTick.Copy(tickIO); prevTick.IsSimulateTicks = true; // if( elapsed.TotalMilliseconds < 5000) { // fast++; // } else { while (!writer.TryAdd(prevTick)) { Thread.Sleep(1); } // } // } } } } catch (QueueException ex) { if (ex.EntryType != EventType.EndHistorical) { throw new ApplicationException("Unexpected QueueException: " + ex); } } lastTick.Copy(tickIO); Console.WriteLine(reader.Symbol + ": " + count + " ticks from " + firstTick.Time + " to " + lastTick.Time + " " + dups + " duplicates, " + fast + " less than 50 ms"); Factory.TickUtil.TickReader().CloseAll(); writer.Close(); }
// [TestFixtureTearDown] public void Dispose() { tickArray = null; // ModelContext.Context.Reset(); }
public void TestCapturedDataMatchesProvider() { try { using (var config = CreateSimulateConfig()) { config.SymbolList = "/ESZ9"; config.DefaultPeriod = 1; config.DefaultBarUnit = BarUnit.Minute.ToString(); config.EndDateTime = DateTime.UtcNow; config.ModelLoader = "Example: Reversal Multi-Symbol"; config.StarterName = "TestRealTimeStarter"; config.Start(); config.WaitComplete(10); config.Stop(); config.WaitComplete(120, () => { return(!config.CommandWorker.IsBusy); }); Assert.IsFalse(config.CommandWorker.IsBusy, "ProcessWorker.Busy"); string appData = Factory.Settings["AppDataFolder"]; string compareFile1 = appData + @"\Test\MockProviderData\ESZ9.tck"; string compareFile2 = appData + @"\Test\ServerCache\ESZ9.tck"; using (TickReader reader1 = Factory.TickUtil.TickReader()) { reader1.Initialize(compareFile1, config.SymbolList); TickBinary tick1 = new TickBinary(); try { int count = 0; while (true) { while (!reader1.ReadQueue.TryDequeue(ref tick1)) { Thread.Sleep(1); } TimeStamp ts1 = new TimeStamp(tick1.UtcTime); count++; } } catch (QueueException ex) { Assert.AreEqual(ex.EntryType, EventType.EndHistorical); } } using (TickReader reader1 = Factory.TickUtil.TickReader()) using (TickReader reader2 = Factory.TickUtil.TickReader()) { reader1.Initialize(compareFile1, config.SymbolList); reader2.Initialize(compareFile2, config.SymbolList); TickBinary tick1 = new TickBinary(); TickBinary tick2 = new TickBinary(); bool result = true; try { int count = 0; while (true) { while (!reader1.ReadQueue.TryDequeue(ref tick1)) { Thread.Sleep(1); } while (!reader2.ReadQueue.TryDequeue(ref tick2)) { Thread.Sleep(1); } TimeStamp ts1 = new TimeStamp(tick1.UtcTime); TimeStamp ts2 = new TimeStamp(tick2.UtcTime); if (!ts1.Equals(ts2)) { result = false; log.Error("Tick# " + count + " failed. Expected: " + ts1 + ", But was:" + ts2); } count++; } } catch (QueueException ex) { Assert.AreEqual(ex.EntryType, EventType.EndHistorical); } Assert.IsTrue(result, "Tick mismatch errors. See log file."); } } } catch (Exception ex) { log.Error("Test failed with error: " + ex.Message, ex); Environment.Exit(1); } }