// [TestFixtureSetUp]
public void Init()
{
tickArray = new TickReader();
// tickArray.TimeFrames.Add( Period.Minute10);
// tickArray.TimeFrames.Add( Period.Hour4);
tickArray.Initialize("UnitTestData", "USD_JPY");
}
public virtual Provider[] SetupProviders(bool quietMode, bool singleLoad)
{
List <Provider> senderList = new List <Provider>();
SymbolInfo[] symbols = ProjectProperties.Starter.SymbolProperties;
string[] symbolFiles = projectProperties.Starter.SymbolArray;
for (int i = 0; i < symbols.Length; i++)
{
TickReader tickReader = Factory.TickUtil.TickReader();
tickReader.MaxCount = EndCount;
tickReader.StartTime = ProjectProperties.Starter.StartTime;
tickReader.EndTime = ProjectProperties.Starter.EndTime;
tickReader.BackgroundWorker = BackgroundWorker;
tickReader.LogProgress = true;
tickReader.BulkFileLoad = singleLoad;
tickReader.QuietMode = quietMode;
try {
tickReader.Initialize(DataFolder, symbolFiles[i]);
senderList.Add(tickReader);
} catch (System.IO.FileNotFoundException ex) {
throw new ApplicationException("Error: " + ex.Message);
}
}
return(senderList.ToArray());
}
public void TZConverter(string file, string symbol)
{
TickReader reader = Factory.TickUtil.TickReader();
reader.Initialize(file, symbol);
TickQueue queue = reader.ReadQueue;
TickBinary tickBinary = new TickBinary();
if (!TryGetNextTick(queue, ref tickBinary))
{
}
}
public void BinaryDiffTest()
{
// string pair = "USD_JPY_Huge";
// string pair = "USD_JPY_YEARS";
string pair = "USD_JPY";
TickReader tickReader = new TickReader();
tickReader.LogProgress = true;
tickReader.Initialize("Test\\DataCache", pair);
byte[] previous = new byte[1024];
MemoryStream stream = new MemoryStream();
diffBits = new BitArray(1024);
long fileSize = 0;
long compressSize = 0;
int count = 0;
try {
TickBinary tick = new TickBinary();
TickIO tickIO = new TickImpl();
while (true)
{
tickReader.ReadQueue.Dequeue(ref tick);
stream.Seek(0, SeekOrigin.Begin);
tickIO.Inject(tick);
tickIO.ToWriter(stream);
CompareSignature(stream.GetBuffer(), previous, stream.Position);
fileSize += stream.Position;
int totDiffLength = 1 + diffLength + diffBits.ByteCount;
compressSize += totDiffLength;
if (count > 1000 && count < 2000)
{
// log.WriteFile(ByteArrayToString(memory,stream.Position));
if (logging)
{
log.Debug(count + ": " + totDiffLength + " " + diffLength + " byte " + ByteArrayToString(diffBits.Bytes, diffBits.ByteCount) + " " + ByteArrayToString(diff, diffLength));
}
}
count++;
Array.Copy(stream.GetBuffer(), previous, stream.Position);
}
} catch (QueueException ex) {
Assert.AreEqual(EventType.EndHistorical, ex.EntryType);
} catch (CollectionTerminatedException) {
}
TickReader.CloseAll();
log.Debug("File Size = " + fileSize + ", Compressed Size = " + compressSize);
}
public void SignatureResult()
{
string pair = "USD_JPY_Volume";
TickReader tickReader = new TickReader();
tickReader.LogProgress = true;
tickReader.Initialize("Test\\DataCache", pair);
int totalBytes = 0;
SignatureCompressor compressor = new SignatureCompressor();
ByteMemory compressed = new ByteMemory("compressed");
int compressedLength = 0;
try {
TickBinary tick = new TickBinary();
TickImpl tickImpl = new TickImpl();
for (int i = 0; i < 101; i++)
{
tickReader.ReadQueue.Dequeue(ref tick);
tickImpl.Inject(tick);
compressed.Reset();
compressor.CompressTick(tick, compressed.Memory);
totalBytes += compressedLength;
}
log.Debug(compressor.Signature.ToString());
Assert.AreEqual(TestSignature, compressor.Signature);
for (int i = 0; i < 200; i++)
{
tickReader.ReadQueue.Dequeue(ref tick);
tickImpl.Inject(tick);
compressed.Reset();
compressor.CompressTick(tick, compressed.Memory);
totalBytes += compressedLength;
int length = compressed.Bytes[0];
int diffBytes = length / 8 + 1;
log.Debug(compressedLength + ": " +
compressed.Bytes[0] + " " +
ByteArrayToString(compressor.Difference, 0, compressor.DiffLength));
}
} catch (CollectionTerminatedException) {
}
log.Debug("Total Compressed Bytes: " + totalBytes);
compressor.LogCounts();
}
public FIXServerSymbolHandler(FIXSimulatorSupport fixSimulatorSupport,
bool isPlayBack, string symbolString,
Func <Yield> onHeartbeat, Func <SymbolInfo, Tick, Yield> onTick,
Action <PhysicalFill, int, int, int> onPhysicalFill,
Action <PhysicalOrder, string> onRejectOrder)
{
this.fixSimulatorSupport = fixSimulatorSupport;
this.isPlayBack = isPlayBack;
this.onHeartbeat = onHeartbeat;
this.onTick = onTick;
this.symbol = Factory.Symbol.LookupSymbol(symbolString);
reader = Factory.TickUtil.TickReader();
reader.Initialize("Test\\MockProviderData", symbolString);
fillSimulator = Factory.Utility.FillSimulator("FIX", symbol, false);
fillSimulator.OnPhysicalFill = onPhysicalFill;
fillSimulator.OnRejectOrder = onRejectOrder;
tickSync = SyncTicks.GetTickSync(symbol.BinaryIdentifier);
tickSync.ForceClear();
queueTask = Factory.Parallel.Loop("FIXServerSymbol-" + symbolString, OnException, ProcessQueue);
firstHearbeat = true;
}
public void CompressionTest()
{
// string pair = "USD_JPY_Huge";
string pair = "USD_JPY_YEARS";
TickReader tickReader = new TickReader();
tickReader.LogProgress = true;
tickReader.Initialize("TestData", pair);
TickCompressor compressor = new TickCompressor(true);
compressor.LogProgress = true;
compressor.Initialize("Compressed", pair);
long fileSize = 0;
long compressSize = 0;
TickIO tick = new TickImpl();
TickImpl tickImpl = new TickImpl();
TickBinary tickBinary = new TickBinary();
try {
for (int i = 0;; i++)
{
if (!tickReader.ReadQueue.Dequeue(ref tickBinary))
{
continue;
}
tickImpl.init(tickBinary);
compressor.Add(tickImpl);
}
} catch (QueueException ex) {
Assert.AreEqual(EntryType.EndHistorical, ex.EntryType);
} catch (CollectionTerminatedException) {
}
tickReader.Stop();
compressor.Close();
log.WriteFile("File Size = " + fileSize + ", Compressed Size = " + compressSize);
}
public void TearDown()
{
TickReader.CloseAll();
}
// [Test]
public void CompressTickTest()
{
string pair = "USD_JPY_Volume";
TickReader tickReader = new TickReader();
tickReader.LogProgress = true;
tickReader.Initialize("Test\\DataCache", pair);
int totalBytes = 0;
SignatureCompressor compressor = new SignatureCompressor();
SignatureCompressor decompressor = new SignatureCompressor();
ByteMemory output = new ByteMemory("compressed");
int length = 0;
try {
TickBinary tick = new TickBinary();
TickImpl tickImpl = new TickImpl();
for (int i = 0; i < 101; i++)
{
tickReader.ReadQueue.Dequeue(ref tick);
tickImpl.Inject(tick);
compressor.CompareTick(tickImpl);
if (compressor.Count < 100)
{
compressor.CopyMemory(output.Bytes, out length);
}
compressor.SwapBuffers();
compressor.Count++;
if (compressor.Count % 100 == 0)
{
compressor.CreateSignature();
compressor.ResetCounters();
}
}
string temp = "";
for (int i = 0; i < compressor.Signature.Length; i++)
{
if (i != 0)
{
temp += ", ";
}
temp += compressor.Signature[i];
}
log.Notice("signature = " + temp);
Assert.AreEqual(TestSignature, compressor.Signature);
byte[] buffer = new byte[1024];
for (int i = 0; i < 200; i++)
{
tickReader.ReadQueue.Dequeue(ref tick);
tickImpl.Inject(tick);
compressor.CompareTick(tickImpl);
compressor.CalculateDifference(compressor.Current.Bytes, compressor.Previous.Bytes, compressor.Current.Length);
Array.Copy(compressor.Current.Bytes, buffer, compressor.Current.Length);
compressor.ReverseDifference(compressor.Current.Bytes, compressor.Previous.Bytes, compressor.Previous.Length);
Assert.AreEqual(buffer, compressor.Current.Bytes);
compressor.WriteMemory(output.Bytes, out length);
compressor.SwapBuffers();
compressor.Count++;
if (compressor.Count % 100 == 0)
{
compressor.CreateSignature();
compressor.ResetCounters();
}
}
} catch (CollectionTerminatedException) {
}
log.Debug("Total Compressed Bytes: " + totalBytes);
compressor.LogCounts();
}
public void Dispose()
{
TickReader.CloseAll();
}
private void FilterFile(string symbol, string inputPath, string outputPath, TimeStamp startTime, TimeStamp endTime)
{
TickReader reader = Factory.TickUtil.TickReader();
TickWriter writer = Factory.TickUtil.TickWriter(true);
writer.KeepFileOpen = true;
writer.Initialize(outputPath, symbol);
reader.Initialize(inputPath, symbol);
TickQueue inputQueue = reader.ReadQueue;
TickIO firstTick = Factory.TickUtil.TickIO();
TickIO lastTick = Factory.TickUtil.TickIO();
TickIO prevTick = Factory.TickUtil.TickIO();
long count = 0;
long fast = 0;
long dups = 0;
TickIO tickIO = Factory.TickUtil.TickIO();
TickBinary tickBinary = new TickBinary();
inputQueue.Dequeue(ref tickBinary);
tickIO.Inject(tickBinary);
count++;
firstTick.Copy(tickIO);
firstTick.IsSimulateTicks = true;
prevTick.Copy(tickIO);
prevTick.IsSimulateTicks = true;
if (tickIO.Time >= startTime)
{
writer.Add(firstTick);
}
try {
while (true)
{
while (!inputQueue.TryDequeue(ref tickBinary))
{
Thread.Sleep(1);
}
tickIO.Inject(tickBinary);
count++;
if (tickIO.Time >= startTime)
{
if (tickIO.Time > endTime)
{
break;
}
// if( tickIO.Bid == prevTick.Bid && tickIO.Ask == prevTick.Ask) {
// dups++;
// } else {
// Elapsed elapsed = tickIO.Time - prevTick.Time;
prevTick.Copy(tickIO);
prevTick.IsSimulateTicks = true;
// if( elapsed.TotalMilliseconds < 5000) {
// fast++;
// } else {
while (!writer.TryAdd(prevTick))
{
Thread.Sleep(1);
}
// }
// }
}
}
} catch (QueueException ex) {
if (ex.EntryType != EventType.EndHistorical)
{
throw new ApplicationException("Unexpected QueueException: " + ex);
}
}
lastTick.Copy(tickIO);
Console.WriteLine(reader.Symbol + ": " + count + " ticks from " + firstTick.Time + " to " + lastTick.Time + " " + dups + " duplicates, " + fast + " less than 50 ms");
Factory.TickUtil.TickReader().CloseAll();
writer.Close();
}
// [TestFixtureTearDown]
public void Dispose()
{
tickArray = null;
// ModelContext.Context.Reset();
}
public void TestCapturedDataMatchesProvider()
{
try {
using (var config = CreateSimulateConfig()) {
config.SymbolList = "/ESZ9";
config.DefaultPeriod = 1;
config.DefaultBarUnit = BarUnit.Minute.ToString();
config.EndDateTime = DateTime.UtcNow;
config.ModelLoader = "Example: Reversal Multi-Symbol";
config.StarterName = "TestRealTimeStarter";
config.Start();
config.WaitComplete(10);
config.Stop();
config.WaitComplete(120, () => { return(!config.CommandWorker.IsBusy); });
Assert.IsFalse(config.CommandWorker.IsBusy, "ProcessWorker.Busy");
string appData = Factory.Settings["AppDataFolder"];
string compareFile1 = appData + @"\Test\MockProviderData\ESZ9.tck";
string compareFile2 = appData + @"\Test\ServerCache\ESZ9.tck";
using (TickReader reader1 = Factory.TickUtil.TickReader()) {
reader1.Initialize(compareFile1, config.SymbolList);
TickBinary tick1 = new TickBinary();
try {
int count = 0;
while (true)
{
while (!reader1.ReadQueue.TryDequeue(ref tick1))
{
Thread.Sleep(1);
}
TimeStamp ts1 = new TimeStamp(tick1.UtcTime);
count++;
}
} catch (QueueException ex) {
Assert.AreEqual(ex.EntryType, EventType.EndHistorical);
}
}
using (TickReader reader1 = Factory.TickUtil.TickReader())
using (TickReader reader2 = Factory.TickUtil.TickReader()) {
reader1.Initialize(compareFile1, config.SymbolList);
reader2.Initialize(compareFile2, config.SymbolList);
TickBinary tick1 = new TickBinary();
TickBinary tick2 = new TickBinary();
bool result = true;
try {
int count = 0;
while (true)
{
while (!reader1.ReadQueue.TryDequeue(ref tick1))
{
Thread.Sleep(1);
}
while (!reader2.ReadQueue.TryDequeue(ref tick2))
{
Thread.Sleep(1);
}
TimeStamp ts1 = new TimeStamp(tick1.UtcTime);
TimeStamp ts2 = new TimeStamp(tick2.UtcTime);
if (!ts1.Equals(ts2))
{
result = false;
log.Error("Tick# " + count + " failed. Expected: " + ts1 + ", But was:" + ts2);
}
count++;
}
} catch (QueueException ex) {
Assert.AreEqual(ex.EntryType, EventType.EndHistorical);
}
Assert.IsTrue(result, "Tick mismatch errors. See log file.");
}
}
} catch (Exception ex) {
log.Error("Test failed with error: " + ex.Message, ex);
Environment.Exit(1);
}
}
