public void DoesNotSetRunTimeErrorWhenReconnectMessageComesThrough()
{
var algorithm = new AlgorithmStub(equities: new List<string> { "SPY" });
var referenceTime = DateTime.UtcNow;
algorithm.SetDateTime(referenceTime);
var localReferencTime = referenceTime.ConvertFromUtc(TimeZones.NewYork);
algorithm.Securities["SPY"].Exchange.SetMarketHours(localReferencTime.AddSeconds(1).TimeOfDay, TimeSpan.FromDays(1), localReferencTime.DayOfWeek);
var job = new LiveNodePacket();
var results = new TestResultHandler();//packet => Console.WriteLine(FieldsToString(packet)));
var api = new Api.Api();
var handler = new DefaultBrokerageMessageHandler(algorithm, job, results, api, TimeSpan.FromMinutes(15), TimeSpan.FromSeconds(.25));
Assert.IsNull(algorithm.RunTimeError);
handler.Handle(BrokerageMessageEvent.Disconnected("Disconnection!"));
Thread.Sleep(100);
handler.Handle(BrokerageMessageEvent.Reconnected("Reconnected!"));
Thread.Sleep(500);
Assert.IsNull(algorithm.RunTimeError);
results.Exit();
}
public void FastExitsDoNotThrowUnhandledExceptions()
{
var algorithm = new AlgorithmStub();
// job is used to send into DataQueueHandler
var job = new LiveNodePacket();
// result handler is used due to dependency in SubscriptionDataReader
var resultHandler = new BacktestingResultHandler();
var feed = new TestableLiveTradingDataFeed();
var dataManager = new DataManager(feed, algorithm);
algorithm.SubscriptionManager.SetDataManager(dataManager);
algorithm.AddSecurities(Resolution.Tick, Enumerable.Range(0, 20).Select(x => x.ToString()).ToList());
var getNextTicksFunction = Enumerable.Range(0, 20).Select(x => new Tick {
Symbol = SymbolCache.GetSymbol(x.ToString())
}).ToList();
feed.DataQueueHandler = new FuncDataQueueHandler(handler => getNextTicksFunction);
var mapFileProvider = new LocalDiskMapFileProvider();
var fileProvider = new DefaultDataProvider();
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), fileProvider, dataManager);
var feedThreadStarted = new ManualResetEvent(false);
var unhandledExceptionWasThrown = false;
Task.Run(() =>
{
try
{
feedThreadStarted.Set();
feed.Run();
}
catch (Exception ex)
{
QuantConnect.Logging.Log.Error(ex.ToString());
unhandledExceptionWasThrown = true;
}
});
feedThreadStarted.WaitOne();
feed.Exit();
Thread.Sleep(1000);
Assert.IsFalse(unhandledExceptionWasThrown);
}
public void RemovalFromUniverseAndDataFeedMakesSecurityNotTradable()
{
SymbolCache.Clear();
var algorithm = new AlgorithmStub(new MockDataFeedWithSubscription());
var orderProcessorMock = new Mock <IOrderProcessor>();
orderProcessorMock.Setup(m => m.GetOpenOrders(It.IsAny <Func <Order, bool> >())).Returns(new List <Order>());
algorithm.Transactions.SetOrderProcessor(orderProcessorMock.Object);
algorithm.SetStartDate(2012, 3, 27);
algorithm.SetEndDate(2012, 3, 30);
algorithm.AddUniverse("my-custom-universe", dt => dt.Day < 30 ? new List <string> {
"CPRT"
} : Enumerable.Empty <string>());
var universe = algorithm.UniverseManager.Values.First();
var securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection(
universe,
algorithm.EndDate.ConvertToUtc(algorithm.TimeZone).Subtract(TimeSpan.FromDays(2)),
new BaseDataCollection(
algorithm.UtcTime,
Symbol.Create("CPRT", SecurityType.Equity, Market.USA),
new List <BaseData>()
)
);
Assert.AreEqual(1, securityChanges.AddedSecurities.Count);
Assert.AreEqual(0, securityChanges.RemovedSecurities.Count);
var security = securityChanges.AddedSecurities.First();
Assert.IsTrue(security.IsTradable);
securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection(
universe,
algorithm.EndDate.ConvertToUtc(algorithm.TimeZone),
new BaseDataCollection(
algorithm.UtcTime,
Symbol.Create("CPRT", SecurityType.Equity, Market.USA),
new List <BaseData>()
)
);
Assert.AreEqual(0, securityChanges.AddedSecurities.Count);
Assert.AreEqual(1, securityChanges.RemovedSecurities.Count);
Assert.AreEqual(security.Symbol, securityChanges.RemovedSecurities.First().Symbol);
Assert.IsFalse(security.IsTradable);
}
public void HandlesManyCustomDataSubscriptions()
{
DataManager dataManager;
var resolution = Resolution.Second;
var algorithm = new AlgorithmStub(out dataManager);
for (int i = 0; i < 100; i++)
{
algorithm.AddData <CustomMockedFileBaseData>((100 + i).ToString(), resolution, fillDataForward: false);
}
var feed = RunDataFeed(algorithm, dataManager);
int count = 0;
var emittedData = false;
var stopwatch = Stopwatch.StartNew();
var previousTime = DateTime.Now;
Console.WriteLine("start: " + previousTime.ToString("o"));
ConsumeBridge(algorithm, feed, TimeSpan.FromSeconds(5), false, ts =>
{
// because this is a remote file we may skip data points while the newest
// version of the file is downloading [internet speed] and also we decide
// not to emit old data
stopwatch.Stop();
if (ts.Slice.Count == 0)
{
return;
}
emittedData = true;
count++;
// make sure within 2 seconds
var delta = DateTime.Now.Subtract(previousTime);
previousTime = DateTime.Now;
Assert.IsTrue(delta <= TimeSpan.FromSeconds(2), delta.ToString());
ConsoleWriteLine("TimeProvider now: " + _manualTimeProvider.GetUtcNow() + " Count: "
+ ts.Slice.Count + ". Delta (ms): "
+ ((decimal)delta.TotalMilliseconds).SmartRounding() + Environment.NewLine);
});
Console.WriteLine("Count: " + count);
Console.WriteLine("Spool up time: " + stopwatch.Elapsed);
Assert.That(count, Is.GreaterThan(20));
Assert.IsTrue(emittedData);
}
public void HandlesManyCustomDataSubscriptions()
{
var resolution = Resolution.Second;
var algorithm = new AlgorithmStub();
for (int i = 0; i < 5; i++)
{
algorithm.AddData <RemoteFileBaseData>((100 + i).ToString(), resolution, fillDataForward: false);
}
var feed = RunDataFeed(algorithm);
int count = 0;
bool receivedData = false;
var stopwatch = Stopwatch.StartNew();
Console.WriteLine("start: " + DateTime.UtcNow.ToString("o"));
ConsumeBridge(feed, TimeSpan.FromSeconds(5), ts =>
{
// because this is a remote file we may skip data points while the newest
// version of the file is downloading [internet speed] and also we decide
// not to emit old data
stopwatch.Stop();
if (ts.Slice.Count == 0)
{
return;
}
count++;
receivedData = true;
var time = ts.Slice.Min(x => x.Value.EndTime).ConvertToUtc(TimeZones.NewYork);
// make sure within 2 seconds
var delta = DateTime.UtcNow.Subtract(time);
//Assert.IsTrue(delta <= TimeSpan.FromSeconds(2), delta.ToString());
Console.WriteLine("Count: " + ts.Slice.Count + "Data time: " + time.ConvertFromUtc(TimeZones.NewYork) + " Delta (ms): "
+ ((decimal)delta.TotalMilliseconds).SmartRounding() + Environment.NewLine);
});
Console.WriteLine("end: " + DateTime.UtcNow.ToString("o"));
Console.WriteLine("Spool up time: " + stopwatch.Elapsed);
// even though we're doing 20 seconds, give a little
// leeway for slow internet traffic
//Assert.That(count, Is.GreaterThan(17));
//Assert.IsTrue(receivedData);
}
public void EmitsData()
{
var algorithm = new AlgorithmStub(forex: new List<string> {"EURUSD"});
// job is used to send into DataQueueHandler
var job = new LiveNodePacket();
// result handler is used due to dependency in SubscriptionDataReader
var resultHandler = new BacktestingResultHandler();
var dataFileProvider = new DefaultDataFileProvider();
var lastTime = DateTime.MinValue;
var timeProvider = new RealTimeProvider();
var dataQueueHandler = new FuncDataQueueHandler(fdqh =>
{
var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.EasternStandard);
if (time == lastTime) return Enumerable.Empty<BaseData>();
lastTime = time;
return Enumerable.Range(0, 9).Select(x => new Tick(time.AddMilliseconds(x*100), Symbols.EURUSD, 1.3m, 1.2m, 1.3m));
});
var feed = new TestableLiveTradingDataFeed(dataQueueHandler, timeProvider);
var mapFileProvider = new LocalDiskMapFileProvider();
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), dataFileProvider);
var feedThreadStarted = new ManualResetEvent(false);
Task.Factory.StartNew(() =>
{
feedThreadStarted.Set();
feed.Run();
});
// wait for feed.Run to actually begin
feedThreadStarted.WaitOne();
var emittedData = false;
ConsumeBridge(feed, TimeSpan.FromSeconds(10), true, ts =>
{
if (ts.Slice.Count != 0)
{
emittedData = true;
Console.WriteLine("HasData: " + ts.Slice.Bars[Symbols.EURUSD].EndTime);
Console.WriteLine();
}
});
Assert.IsTrue(emittedData);
}
public void HandlesRestApi()
{
var resolution = Resolution.Second;
var algorithm = new AlgorithmStub();
algorithm.AddData <RestApiBaseData>("RestApi", resolution);
var symbol = SymbolCache.GetSymbol("RestApi");
FuncDataQueueHandler dqgh;
var timeProvider = new ManualTimeProvider(new DateTime(2015, 10, 10, 16, 36, 0));
var feed = RunDataFeed(algorithm, out dqgh, null);
var count = 0;
var receivedData = false;
var timeZone = algorithm.Securities[symbol].Exchange.TimeZone;
RestApiBaseData last = null;
var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
foreach (var ts in feed)
{
//timeProvider.AdvanceSeconds(0.5);
if (!ts.Slice.ContainsKey(symbol))
{
return;
}
count++;
receivedData = true;
var data = (RestApiBaseData)ts.Slice[symbol];
var time = data.EndTime.ConvertToUtc(timeZone);
Console.WriteLine(DateTime.UtcNow + ": Data time: " + time.ConvertFromUtc(TimeZones.NewYork) + Environment.NewLine);
if (last != null)
{
Assert.AreEqual(last.EndTime, data.EndTime.Subtract(resolution.ToTimeSpan()));
}
last = data;
}
// even though we're doing 10 seconds, give a little
// leeway for slow internet traffic
Assert.That(count, Is.GreaterThanOrEqualTo(8));
Assert.IsTrue(receivedData);
Assert.That(RestApiBaseData.ReaderCount, Is.LessThanOrEqualTo(30)); // we poll at 10x frequency
Console.WriteLine("Count: " + count + " ReaderCount: " + RestApiBaseData.ReaderCount);
}
public void DoesNotSubscribeToCustomData()
{
// Current implementation only sends equity/forex subscriptions to the queue handler,
// new impl sends all, the restriction shouldn't live in the feed, but rather in the
// queue handler impl
var algorithm = new AlgorithmStub(equities: new List<string> { "SPY" }, forex: new List<string> { "EURUSD" });
algorithm.AddData<RemoteFileBaseData>("RemoteFile");
FuncDataQueueHandler dataQueueHandler;
RunDataFeed(algorithm, out dataQueueHandler);
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(new SymbolSecurityType("SPY", SecurityType.Equity)));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(new SymbolSecurityType("EURUSD", SecurityType.Forex)));
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(new SymbolSecurityType("REMOTEFILE", SecurityType.Base)));
Assert.AreEqual(2, dataQueueHandler.Subscriptions.Count);
}
public void HandlesMultipleSecurities()
{
var algorithm = new AlgorithmStub(
equities: new List <string> {
"SPY", "IBM", "AAPL", "GOOG", "MSFT", "BAC", "GS"
},
forex: new List <string> {
"EURUSD", "USDJPY", "GBPJPY", "AUDUSD", "NZDUSD"
}
);
var feed = RunDataFeed(algorithm);
ConsumeBridge(feed, TimeSpan.FromSeconds(5), ts =>
{
var delta = (DateTime.UtcNow - ts.Time).TotalMilliseconds;
Console.WriteLine(((decimal)delta).SmartRounding() + "ms : " + string.Join(",", ts.Slice.Keys.Select(x => x.Value)));
});
}
public void RemoveSecurity()
{
DataManager dataManager;
var algorithm = new AlgorithmStub(out dataManager, equities: new List <string> {
"SPY"
}, forex: new List <string> {
"EURUSD"
});
algorithm.SetFinishedWarmingUp();
algorithm.Transactions.SetOrderProcessor(new FakeOrderProcessor());
algorithm.AddData <CustomMockedFileBaseData>("CustomMockedFileBaseData");
var customMockedFileBaseData = SymbolCache.GetSymbol("CustomMockedFileBaseData");
FuncDataQueueHandler dataQueueHandler;
var feed = RunDataFeed(algorithm, out dataQueueHandler, dataManager);
var emittedData = false;
var currentSubscriptionCount = 0;
var securityChanges = 0;
ConsumeBridge(algorithm, feed, TimeSpan.FromSeconds(5), true, ts =>
{
securityChanges += ts.SecurityChanges.Count;
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(customMockedFileBaseData));
if (!emittedData)
{
currentSubscriptionCount = dataQueueHandler.Subscriptions.Count;
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.SPY));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.EURUSD));
algorithm.RemoveSecurity(Symbols.SPY);
emittedData = true;
}
else
{
Assert.AreEqual(currentSubscriptionCount - 1, dataQueueHandler.Subscriptions.Count);
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(Symbols.SPY));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.EURUSD));
}
});
Assert.IsTrue(emittedData);
Assert.AreEqual(4, securityChanges + algorithm.SecurityChangesRecord.Count);
Assert.AreEqual(Symbols.SPY, algorithm.SecurityChangesRecord.First().RemovedSecurities.First().Symbol);
}
public void SetsAlgorithmRunTimeErrorOnDisconnectIfNonCustomSecurityIsOpen()
{
var algorithm = new AlgorithmStub(equities: new List<string> { "SPY" });
algorithm.Securities[Symbols.SPY].Exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork));
var job = new LiveNodePacket();
var results = new TestResultHandler();//packet => Console.WriteLine(FieldsToString(packet)));
var api = new Api.Api();
var handler = new DefaultBrokerageMessageHandler(algorithm, job, results, api, TimeSpan.Zero, TimeSpan.Zero);
Assert.IsNull(algorithm.RunTimeError);
handler.Handle(BrokerageMessageEvent.Disconnected("Disconnection!"));
Thread.Sleep(100);
Assert.IsNotNull(algorithm.RunTimeError);
results.Exit();
}
public void DoesNotSetAlgorithmRunTimeErrorOnDisconnectIfAllSecuritiesClosed()
{
var referenceTime = DateTime.UtcNow;
var algorithm = new AlgorithmStub(equities: new List<string> { "SPY" });
algorithm.SetDateTime(referenceTime);
algorithm.Securities["SPY"].Exchange.SetMarketHours(TimeSpan.Zero, TimeSpan.Zero, referenceTime.ConvertFromUtc(TimeZones.NewYork).DayOfWeek);
var job = new LiveNodePacket();
var results = new TestResultHandler();//packet => Console.WriteLine(FieldsToString(packet)));
var api = new Api.Api();
var handler = new DefaultBrokerageMessageHandler(algorithm, job, results, api, TimeSpan.FromMinutes(15));
Assert.IsNull(algorithm.RunTimeError);
handler.Handle(BrokerageMessageEvent.Disconnected("Disconnection!"));
Assert.IsNull(algorithm.RunTimeError);
results.Exit();
}
public void HandlesRestApi()
{
DataManager dataManager;
var resolution = Resolution.Second;
var algorithm = new AlgorithmStub(out dataManager);
algorithm.AddData <RestApiBaseData>("RestApi", resolution);
var symbol = SymbolCache.GetSymbol("RestApi");
FuncDataQueueHandler dqgh;
var feed = RunDataFeed(algorithm, out dqgh, dataManager);
var count = 0;
var receivedData = false;
var timeZone = algorithm.Securities[symbol].Exchange.TimeZone;
RestApiBaseData last = null;
foreach (var ts in feed)
{
if (!ts.Slice.ContainsKey(symbol))
{
return;
}
count++;
receivedData = true;
var data = (RestApiBaseData)ts.Slice[symbol];
var time = data.EndTime.ConvertToUtc(timeZone);
ConsoleWriteLine(DateTime.UtcNow + ": Data time: " + time.ConvertFromUtc(TimeZones.NewYork) + Environment.NewLine);
if (last != null)
{
Assert.AreEqual(last.EndTime, data.EndTime.Subtract(resolution.ToTimeSpan()));
}
last = data;
}
feed.Exit();
Assert.That(count, Is.GreaterThanOrEqualTo(8));
Assert.IsTrue(receivedData);
Assert.That(RestApiBaseData.ReaderCount, Is.LessThanOrEqualTo(30)); // we poll at 10x frequency
Console.WriteLine("Count: " + count + " ReaderCount: " + RestApiBaseData.ReaderCount);
}
public void PerformanceBenchmark()
{
DataManager dataManager;
var symbolCount = 600;
var algorithm = new AlgorithmStub(out dataManager, Resolution.Tick,
equities: Enumerable.Range(0, symbolCount).Select(x => "E" + x.ToString()).ToList()
);
var securitiesCount = algorithm.Securities.Count;
var expected = algorithm.Securities.Keys.ToHashSet();
Console.WriteLine("Securities.Count: " + securitiesCount);
FuncDataQueueHandler queue;
var count = new Count();
var stopwatch = Stopwatch.StartNew();
var feed = RunDataFeed(algorithm, out queue, dataManager, fdqh => ProduceBenchmarkTicks(fdqh, count));
ConsumeBridge(algorithm, feed, TimeSpan.FromSeconds(5), ts =>
{
ConsoleWriteLine("Count: " + ts.Slice.Keys.Count + " " + DateTime.UtcNow.ToString("o"));
if (ts.Slice.Keys.Count != securitiesCount)
{
var included = ts.Slice.Keys.ToHashSet();
expected.ExceptWith(included);
ConsoleWriteLine("Missing: " + string.Join(",", expected.OrderBy(x => x.Value)));
}
});
stopwatch.Stop();
Console.WriteLine("Total ticks: " + count.Value);
Assert.GreaterOrEqual(count.Value, 700000);
Console.WriteLine("Elapsed time: " + stopwatch.Elapsed);
var ticksPerSec = count.Value / stopwatch.Elapsed.TotalSeconds;
Console.WriteLine("Ticks/sec: " + ticksPerSec);
Assert.GreaterOrEqual(ticksPerSec, 70000);
var ticksPerSecPerSymbol = (count.Value / stopwatch.Elapsed.TotalSeconds) / symbolCount;
Console.WriteLine("Ticks/sec/symbol: " + ticksPerSecPerSymbol);
Assert.GreaterOrEqual(ticksPerSecPerSymbol, 100);
}
public void Unsubscribes()
{
DataManager dataManager;
var algorithm = new AlgorithmStub(out dataManager, equities: new List <string> {
"SPY"
}, forex: new List <string> {
"EURUSD"
});
algorithm.AddData <CustomMockedFileBaseData>("CustomMockedFileBaseData");
var customMockedFileBaseData = SymbolCache.GetSymbol("CustomMockedFileBaseData");
FuncDataQueueHandler dataQueueHandler;
var feed = RunDataFeed(algorithm, out dataQueueHandler, dataManager);
var emittedData = false;
var currentSubscriptionCount = 0;
ConsumeBridge(algorithm, feed, TimeSpan.FromSeconds(5), false, ts =>
{
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(customMockedFileBaseData));
if (!emittedData)
{
currentSubscriptionCount = dataQueueHandler.Subscriptions.Count;
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(_equityUsaUserUniverse));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(_forexFxcmUserUniverse));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.SPY));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.EURUSD));
feed.RemoveSubscription(feed.Subscriptions.Single(sub => sub.Configuration.Symbol == Symbols.SPY).Configuration);
emittedData = true;
}
else
{
Assert.AreEqual(currentSubscriptionCount - 1, dataQueueHandler.Subscriptions.Count);
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(Symbols.SPY));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.EURUSD));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(_equityUsaUserUniverse));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(_forexFxcmUserUniverse));
}
});
Assert.IsTrue(emittedData);
}
public void Unsubscribes()
{
var algorithm = new AlgorithmStub(equities: new List <string> {
"SPY"
}, forex: new List <string> {
"EURUSD"
});
algorithm.AddData <RemoteFileBaseData>("RemoteFile");
FuncDataQueueHandler dataQueueHandler;
var feed = RunDataFeed(algorithm, out dataQueueHandler);
feed.RemoveSubscription(feed.Subscriptions.Single(x => x.Configuration.Symbol == "SPY"));
Assert.AreEqual(1, dataQueueHandler.Subscriptions.Count);
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(new SymbolSecurityType("SPY", SecurityType.Equity)));
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(new SymbolSecurityType("RemoteFile", SecurityType.Base)));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(new SymbolSecurityType("EURUSD", SecurityType.Forex)));
}
public void Unsubscribes()
{
var algorithm = new AlgorithmStub(equities: new List <string> {
"SPY"
}, forex: new List <string> {
"EURUSD"
});
algorithm.AddData <RemoteFileBaseData>("RemoteFile");
var remoteFile = SymbolCache.GetSymbol("RemoteFile");
FuncDataQueueHandler dataQueueHandler;
var feed = RunDataFeed(algorithm, out dataQueueHandler);
feed.RemoveSubscription(feed.Subscriptions.Single(x => x.Configuration.Symbol == Symbols.SPY));
Assert.AreEqual(1, dataQueueHandler.Subscriptions.Count);
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(Symbols.SPY));
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(remoteFile));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.EURUSD));
}
public void CoarseFundamentalHasFundamentalDataFalseExcludedInFineUniverseSelection()
{
var algorithm = new AlgorithmStub(new MockDataFeed());
algorithm.SetEndDate(Time.EndOfTime);
algorithm.SetStartDate(DateTime.UtcNow.Subtract(TimeSpan.FromDays(10)));
algorithm.AddUniverse(
coarse => coarse.Select(c => c.Symbol),
fine => fine.Select(f => f.Symbol)
);
// OnEndOfTimeStep will add all pending universe additions
algorithm.OnEndOfTimeStep();
var universe = algorithm.UniverseManager.Values.First();
var securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection(
universe,
algorithm.EndDate.ConvertToUtc(algorithm.TimeZone).Subtract(TimeSpan.FromDays(1)),
new BaseDataCollection(
DateTime.UtcNow,
Symbols.AAPL,
new[]
{
new CoarseFundamental
{
Symbol = Symbols.AAPL,
HasFundamentalData = true
},
new CoarseFundamental
{
Symbol = Symbols.SPY,
HasFundamentalData = false
}
}
)
);
Assert.AreEqual(1, securityChanges.Count);
Assert.AreEqual(Symbols.AAPL, securityChanges.AddedSecurities.First().Symbol);
}
public void CreatedEquityIsNotAddedToSymbolCache()
{
SymbolCache.Clear();
var algorithm = new AlgorithmStub(new MockDataFeed());
algorithm.SetEndDate(Time.EndOfTime);
algorithm.SetStartDate(DateTime.UtcNow.Subtract(TimeSpan.FromDays(10)));
algorithm.AddUniverse(CoarseSelectionFunction, FineSelectionFunction);
// OnEndOfTimeStep will add all pending universe additions
algorithm.OnEndOfTimeStep();
var universe = algorithm.UniverseManager.Values.First();
var securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection(
universe,
algorithm.EndDate.ConvertToUtc(algorithm.TimeZone).Subtract(TimeSpan.FromDays(1)),
new BaseDataCollection(
DateTime.UtcNow,
Symbols.AAPL,
new[]
{
new CoarseFundamental
{
Symbol = Symbols.AAPL,
HasFundamentalData = true
},
new CoarseFundamental
{
Symbol = Symbols.SPY,
HasFundamentalData = false
}
}
)
);
Symbol symbol;
Assert.AreEqual(1, securityChanges.AddedSecurities.Count);
Assert.AreEqual(Symbols.AAPL, securityChanges.AddedSecurities.First().Symbol);
Assert.IsFalse(SymbolCache.TryGetSymbol("AAPL", out symbol));
Assert.IsFalse(SymbolCache.TryGetSymbol("SPY", out symbol));
}
public void DoesNotSubscribeToCustomData()
{
// Current implementation only sends equity/forex subscriptions to the queue handler,
// new impl sends all, the restriction shouldn't live in the feed, but rather in the
// queue handler impl
var algorithm = new AlgorithmStub(equities: new List <string> {
"SPY"
}, forex: new List <string> {
"EURUSD"
});
algorithm.AddData <RemoteFileBaseData>("RemoteFile");
var remoteFile = SymbolCache.GetSymbol("RemoteFile");
FuncDataQueueHandler dataQueueHandler;
RunDataFeed(algorithm, out dataQueueHandler);
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.SPY));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.EURUSD));
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(remoteFile));
Assert.AreEqual(2, dataQueueHandler.Subscriptions.Count);
}
public void EmitsData()
{
DataManager dataManager;
var algorithm = new AlgorithmStub(out dataManager, forex: new List <string> {
Symbols.EURUSD
});
var feed = RunDataFeed(algorithm, dataManager);
var emittedData = false;
ConsumeBridge(algorithm, feed, TimeSpan.FromSeconds(10), true, ts =>
{
if (ts.Slice.HasData)
{
emittedData = true;
var data = ts.Slice[Symbols.EURUSD];
ConsoleWriteLine("HasData: " + data);
ConsoleWriteLine();
}
});
Assert.IsTrue(emittedData);
}
public void HandlesDaylightSavingTimeChange_InifinteLoop()
{
var dailyBarsEmitted = 0;
var fillForwardBars = new List <BaseData>();
var data = new BaseData[]
{
new QuoteBar {
Value = 0, Time = new DateTime(2019, 10, 4, 10, 0, 0), Period = Time.OneDay
},
new QuoteBar {
Value = 1, Time = new DateTime(2019, 10, 8, 11, 0, 0), Period = Time.OneDay
}
}.ToList();
var enumerator = data.GetEnumerator();
var algo = new AlgorithmStub();
var market = Market.Oanda;
var security = algo.AddCfd("AU200AUD", Resolution.Daily, market);
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, security.Exchange, Ref.Create(TimeSpan.FromDays(1)), false, data.Last().EndTime, Time.OneDay, TimeZones.Utc, data.First().EndTime);
while (fillForwardEnumerator.MoveNext())
{
fillForwardBars.Add(fillForwardEnumerator.Current);
Console.WriteLine(fillForwardEnumerator.Current.Time.DayOfWeek + " " + fillForwardEnumerator.Current.Time + " - " + fillForwardEnumerator.Current.EndTime.DayOfWeek + " " + fillForwardEnumerator.Current.EndTime + " " + fillForwardEnumerator.Current.IsFillForward);
dailyBarsEmitted++;
}
Assert.AreEqual(4, dailyBarsEmitted);
Assert.AreEqual(new DateTime(2019, 10, 4, 10, 0, 0), fillForwardBars[0].Time);
Assert.AreEqual(new DateTime(2019, 10, 6, 11, 0, 0), fillForwardBars[1].Time);
Assert.AreEqual(new DateTime(2019, 10, 7, 11, 0, 0), fillForwardBars[2].Time);
Assert.AreEqual(new DateTime(2019, 10, 8, 11, 0, 0), fillForwardBars[3].Time);
fillForwardEnumerator.Dispose();
}
public void AddsSubscription_NewUserUniverse()
{
DataManager dataManager;
var algorithm = new AlgorithmStub(out dataManager, equities: new List <string> {
"SPY"
});
FuncDataQueueHandler dataQueueHandler;
var feed = RunDataFeed(algorithm, out dataQueueHandler, dataManager);
var emittedData = false;
var newDataCount = 0;
ConsumeBridge(algorithm, feed, TimeSpan.FromSeconds(5), false, ts =>
{
if (!emittedData)
{
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.SPY));
Assert.IsTrue(ts.Slice.Keys.Contains(Symbols.SPY));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(_equityUsaUserUniverse));
Assert.AreEqual(2, dataQueueHandler.Subscriptions.Count);
algorithm.AddSecurities(forex: new List <string> {
"EURUSD"
});
emittedData = true;
}
else
{
if (dataQueueHandler.Subscriptions.Count == 3)
{
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.SPY));
Assert.IsTrue(ts.Slice.Keys.Contains(Symbols.SPY));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(_equityUsaUserUniverse));
// first it will add the universe
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(_forexFxcmUserUniverse));
}
else if (dataQueueHandler.Subscriptions.Count == 4 &&
ts.Slice.HasData) // there could be some slices with no data
{
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.SPY));
Assert.IsTrue(ts.Slice.Keys.Contains(Symbols.SPY));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(_equityUsaUserUniverse));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.EURUSD));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(_forexFxcmUserUniverse));
// Might delay a couple of Slices to send over the data, so we will count them
// and assert a minimum amount
if (ts.Slice.Keys.Contains(Symbols.EURUSD))
{
newDataCount++;
}
}
else
{
Assert.Fail($"Subscriptions.Count: {dataQueueHandler.Subscriptions.Count}");
}
}
});
Console.WriteLine("newDataCount: " + newDataCount);
Assert.GreaterOrEqual(newDataCount, 1000);
Assert.IsTrue(emittedData);
}
public void HandlesMultipleSecurities()
{
var algorithm = new AlgorithmStub(
equities: new List<string> {"SPY", "IBM", "AAPL", "GOOG", "MSFT", "BAC", "GS"},
forex: new List<string> {"EURUSD", "USDJPY", "GBPJPY", "AUDUSD", "NZDUSD"}
);
var feed = RunDataFeed(algorithm);
ConsumeBridge(feed, TimeSpan.FromSeconds(5), ts =>
{
var delta = (DateTime.UtcNow - ts.Time).TotalMilliseconds;
Console.WriteLine(((decimal)delta).SmartRounding() + "ms : " + string.Join(",", ts.Slice.Keys.Select(x => x.Value)));
});
}
public void AddsSubscription_NewUserUniverse()
{
DataManager dataManager;
var algorithm = new AlgorithmStub(out dataManager, equities: new List <string> {
"SPY"
});
FuncDataQueueHandler dataQueueHandler;
var feed = RunDataFeed(algorithm, out dataQueueHandler, dataManager);
var forexFxcmUserUniverse = UserDefinedUniverse.CreateSymbol(SecurityType.Forex, Market.FXCM);
var emittedData = false;
var newDataCount = 0;
var securityChanges = 0;
ConsumeBridge(algorithm, feed, TimeSpan.FromSeconds(5), true, ts =>
{
securityChanges += ts.SecurityChanges.Count;
if (!emittedData)
{
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.SPY));
if (ts.Data.Count > 0)
{
Assert.IsTrue(ts.Slice.Keys.Contains(Symbols.SPY));
}
Assert.AreEqual(1, dataQueueHandler.Subscriptions.Count);
algorithm.AddSecurities(forex: new List <string> {
"EURUSD"
});
emittedData = true;
}
else
{
if (dataQueueHandler.Subscriptions.Count == 2) // there could be some slices with no data
{
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.SPY));
if (ts.Data.Count > 0)
{
Assert.IsTrue(ts.Slice.Keys.Contains(Symbols.SPY));
}
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.EURUSD) ||
dataQueueHandler.Subscriptions.Contains(forexFxcmUserUniverse));
// Might delay a couple of Slices to send over the data, so we will count them
// and assert a minimum amount
if (ts.Slice.Keys.Contains(Symbols.EURUSD))
{
newDataCount++;
}
}
else
{
Assert.Fail($"Subscriptions.Count: {dataQueueHandler.Subscriptions.Count}");
}
}
});
Console.WriteLine("newDataCount: " + newDataCount);
Assert.AreEqual(2, securityChanges);
Assert.GreaterOrEqual(newDataCount, 1000);
Assert.IsTrue(emittedData);
}
public void HandlesRestApi()
{
var resolution = Resolution.Second;
var algorithm = new AlgorithmStub();
algorithm.AddData<RestApiBaseData>("RestApi", resolution);
var symbol = SymbolCache.GetSymbol("RestApi");
FuncDataQueueHandler dqgh;
var timeProvider = new ManualTimeProvider(new DateTime(2015, 10, 10, 16, 36, 0));
var feed = RunDataFeed(algorithm, out dqgh, null);
var count = 0;
var receivedData = false;
var timeZone = algorithm.Securities[symbol].Exchange.TimeZone;
RestApiBaseData last = null;
var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
foreach (var ts in feed)
{
//timeProvider.AdvanceSeconds(0.5);
if (!ts.Slice.ContainsKey(symbol)) return;
count++;
receivedData = true;
var data = (RestApiBaseData)ts.Slice[symbol];
var time = data.EndTime.ConvertToUtc(timeZone);
Console.WriteLine(DateTime.UtcNow + ": Data time: " + time.ConvertFromUtc(TimeZones.NewYork) + Environment.NewLine);
if (last != null)
{
Assert.AreEqual(last.EndTime, data.EndTime.Subtract(resolution.ToTimeSpan()));
}
last = data;
}
// even though we're doing 10 seconds, give a little
// leeway for slow internet traffic
Assert.That(count, Is.GreaterThanOrEqualTo(8));
Assert.IsTrue(receivedData);
Assert.That(RestApiBaseData.ReaderCount, Is.LessThanOrEqualTo(30)); // we poll at 10x frequency
Console.WriteLine("Count: " + count + " ReaderCount: " + RestApiBaseData.ReaderCount);
}
public void HandlesCoarseFundamentalData()
{
var algorithm = new AlgorithmStub();
Symbol symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA);
algorithm.AddUniverse(new FuncUniverse(
new SubscriptionDataConfig(typeof(CoarseFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false),
new UniverseSettings(Resolution.Second, 1, true, false, TimeSpan.Zero),
coarse => coarse.Take(10).Select(x => x.Symbol)
));
var lck = new object();
BaseDataCollection list = null;
const int coarseDataPointCount = 100000;
var timer = new Timer(state =>
{
var currentTime = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
Console.WriteLine(currentTime + ": timer.Elapsed");
lock (state)
{
list = new BaseDataCollection {Symbol = symbol};
list.Data.AddRange(Enumerable.Range(0, coarseDataPointCount).Select(x => new CoarseFundamental
{
Symbol = SymbolCache.GetSymbol(x.ToString()),
Time = currentTime - Time.OneDay, // hard-coded coarse period of one day
}));
}
}, lck, TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(500));
bool yieldedUniverseData = false;
var feed = RunDataFeed(algorithm, fdqh =>
{
lock (lck)
{
if (list != null)
try
{
var tmp = list;
return new List<BaseData> { tmp };
}
finally
{
list = null;
yieldedUniverseData = true;
}
}
return Enumerable.Empty<BaseData>();
});
Assert.IsTrue(feed.Subscriptions.Any(x => x.IsUniverseSelectionSubscription));
var universeSelectionHadAllData = false;
ConsumeBridge(feed, TimeSpan.FromSeconds(5), ts =>
{
});
Assert.IsTrue(yieldedUniverseData);
Assert.IsTrue(universeSelectionHadAllData);
}
public void EmitsForexDataWithRoundedUtcTimes()
{
var algorithm = new AlgorithmStub(forex: new List<string> { "EURUSD" });
var feed = RunDataFeed(algorithm);
var emittedData = false;
var lastTime = DateTime.UtcNow;
ConsumeBridge(feed, TimeSpan.FromSeconds(5), ts =>
{
if (!emittedData)
{
emittedData = true;
lastTime = ts.Time;
return;
}
var delta = (DateTime.UtcNow - ts.Time).TotalMilliseconds;
Console.WriteLine(((decimal)delta).SmartRounding() + "ms : " + string.Join(", ", ts.Slice.Keys.Select(x => x.Value + ": " + ts.Slice[x].Volume)));
Assert.AreEqual(lastTime.Add(Time.OneSecond), ts.Time);
Assert.AreEqual(1, ts.Slice.Bars.Count);
lastTime = ts.Time;
});
Assert.IsTrue(emittedData);
}
public void HandlesManyCustomDataSubscriptions()
{
var resolution = Resolution.Second;
var algorithm = new AlgorithmStub();
for (int i = 0; i < 5; i++)
{
algorithm.AddData<RemoteFileBaseData>((100+ i).ToString(), resolution, fillDataForward: false);
}
var feed = RunDataFeed(algorithm);
int count = 0;
bool receivedData = false;
var stopwatch = Stopwatch.StartNew();
Console.WriteLine("start: " + DateTime.UtcNow.ToString("o"));
ConsumeBridge(feed, TimeSpan.FromSeconds(5), ts =>
{
// because this is a remote file we may skip data points while the newest
// version of the file is downloading [internet speed] and also we decide
// not to emit old data
stopwatch.Stop();
if (ts.Slice.Count == 0) return;
count++;
receivedData = true;
var time = ts.Slice.Min(x => x.Value.EndTime).ConvertToUtc(TimeZones.NewYork);
// make sure within 2 seconds
var delta = DateTime.UtcNow.Subtract(time);
//Assert.IsTrue(delta <= TimeSpan.FromSeconds(2), delta.ToString());
Console.WriteLine("Count: " + ts.Slice.Count + "Data time: " + time.ConvertFromUtc(TimeZones.NewYork) + " Delta (ms): "
+ ((decimal) delta.TotalMilliseconds).SmartRounding() + Environment.NewLine);
});
Console.WriteLine("end: " + DateTime.UtcNow.ToString("o"));
Console.WriteLine("Spool up time: " + stopwatch.Elapsed);
// even though we're doing 20 seconds, give a little
// leeway for slow internet traffic
//Assert.That(count, Is.GreaterThan(17));
//Assert.IsTrue(receivedData);
}
public void Unsubscribes()
{
var algorithm = new AlgorithmStub(equities: new List<string> { "SPY" }, forex: new List<string> { "EURUSD" });
algorithm.AddData<RemoteFileBaseData>("RemoteFile");
var remoteFile = SymbolCache.GetSymbol("RemoteFile");
FuncDataQueueHandler dataQueueHandler;
var feed = RunDataFeed(algorithm, out dataQueueHandler);
feed.RemoveSubscription(feed.Subscriptions.Single(x => x.Configuration.Symbol == Symbols.SPY));
Assert.AreEqual(1, dataQueueHandler.Subscriptions.Count);
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(Symbols.SPY));
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(remoteFile));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.EURUSD));
}
public void HandlesAtLeast10kTicksPerSecondWithTwentySymbols()
{
// this ran at ~25k ticks/per symbol for 20 symbols
var algorithm = new AlgorithmStub(Resolution.Tick, Enumerable.Range(0, 20).Select(x => x.ToString()).ToList());
var t = Enumerable.Range(0, 20).Select(x => new Tick {Symbol = SymbolCache.GetSymbol(x.ToString())}).ToList();
var feed = RunDataFeed(algorithm, handler => t);
var flag = false;
int ticks = 0;
var averages = new List<decimal>();
var timer = new Timer(state =>
{
var avg = ticks/20m;
Interlocked.Exchange(ref ticks, 0);
Console.WriteLine("Average ticks per symbol: " + avg.SmartRounding());
if (flag) flag = false;
averages.Add(avg);
}, null, Time.OneSecond, Time.OneSecond);
ConsumeBridge(feed, TimeSpan.FromSeconds(5), false, ts =>
{
Interlocked.Add(ref ticks, ts.Slice.Ticks.Sum(x => x.Value.Count));
}, true);
var average = averages.Average();
Console.WriteLine("\r\nAverage ticks per symbol per second: " + average);
Assert.That(average, Is.GreaterThan(10000));
}
public void CoarseUniverseRotatesActiveSecurity()
{
var startDate = new DateTime(2014, 3, 24);
var endDate = new DateTime(2014, 3, 28);
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
timeProvider.SetCurrentTime(startDate);
var coarseTimes = new List <DateTime>
{
new DateTime(2014, 3, 25),
new DateTime(2014, 3, 25, 23, 0, 0),
new DateTime(2014, 3, 27, 1, 0, 0)
}.ToHashSet();
var coarseSymbols = new List <Symbol> {
Symbols.SPY, Symbols.AAPL, Symbols.MSFT
};
var coarseUsaSymbol = CoarseFundamental.CreateUniverseSymbol(Market.USA, false);
var coarseDataEmittedCount = 0;
var lastTime = DateTime.MinValue;
var dataQueueHandler = new FuncDataQueueHandler(fdqh =>
{
var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (time != lastTime)
{
lastTime = time;
if (coarseTimes.Contains(time))
{
// emit coarse data at selected times
var coarseData = new BaseDataCollection {
Symbol = coarseUsaSymbol
};
foreach (var symbol in coarseSymbols)
{
coarseData.Data.Add(
new CoarseFundamental
{
Symbol = symbol,
Time = time,
Market = Market.USA,
Value = 100
});
}
coarseDataEmittedCount++;
return(new List <BaseData> {
coarseData
});
}
}
return(Enumerable.Empty <BaseData>());
});
var feed = new TestableLiveTradingDataFeed(dataQueueHandler);
var algorithm = new AlgorithmStub(feed);
algorithm.SetLiveMode(true);
var mock = new Mock <ITransactionHandler>();
mock.Setup(m => m.GetOpenOrders(It.IsAny <Func <Order, bool> >())).Returns(new List <Order>());
algorithm.Transactions.SetOrderProcessor(mock.Object);
var synchronizer = new TestableLiveSynchronizer(timeProvider);
synchronizer.Initialize(algorithm, algorithm.DataManager);
var mapFileProvider = new LocalDiskMapFileProvider();
feed.Initialize(algorithm, new LiveNodePacket(), new BacktestingResultHandler(),
mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), new DefaultDataProvider(), algorithm.DataManager, synchronizer);
var symbolIndex = 0;
var coarseUniverseSelectionCount = 0;
algorithm.AddUniverse(
coarse =>
{
coarseUniverseSelectionCount++;
// rotate single symbol in universe
if (symbolIndex == coarseSymbols.Count)
{
symbolIndex = 0;
}
return(new[] { coarseSymbols[symbolIndex++] });
});
algorithm.PostInitialize();
var cancellationTokenSource = new CancellationTokenSource();
Exception exceptionThrown = null;
// create a timer to advance time much faster than realtime
var timerInterval = TimeSpan.FromMilliseconds(50);
var timer = Ref.Create <Timer>(null);
timer.Value = new Timer(state =>
{
try
{
// stop the timer to prevent reentrancy
timer.Value.Change(Timeout.Infinite, Timeout.Infinite);
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (currentTime.Date > endDate.Date)
{
feed.Exit();
cancellationTokenSource.Cancel();
return;
}
timeProvider.Advance(TimeSpan.FromHours(1));
var activeSecuritiesCount = algorithm.ActiveSecurities.Count;
Assert.That(activeSecuritiesCount <= 1);
// restart the timer
timer.Value.Change(timerInterval, timerInterval);
}
catch (Exception exception)
{
Log.Error(exception);
exceptionThrown = exception;
feed.Exit();
cancellationTokenSource.Cancel();
}
}, null, TimeSpan.FromSeconds(1), timerInterval);
foreach (var _ in synchronizer.StreamData(cancellationTokenSource.Token))
{
}
timer.Value.Dispose();
if (exceptionThrown != null)
{
throw new Exception("Exception in timer: ", exceptionThrown);
}
Assert.AreEqual(coarseTimes.Count, coarseDataEmittedCount);
Assert.AreEqual(coarseTimes.Count, coarseUniverseSelectionCount);
}
public void PerformanceBenchmark()
{
var symbolCount = 600;
var algorithm = new AlgorithmStub(Resolution.Tick,
equities: Enumerable.Range(0, symbolCount).Select(x => "E"+x.ToString()).ToList()
);
var securitiesCount = algorithm.Securities.Count;
var expected = algorithm.Securities.Keys.ToHashSet();
Console.WriteLine("Securities.Count: " + securitiesCount);
FuncDataQueueHandler queue;
var count = new Count();
var stopwatch = Stopwatch.StartNew();
var feed = RunDataFeed(algorithm, out queue, null, fdqh => ProduceBenchmarkTicks(fdqh, count));
ConsumeBridge(feed, TimeSpan.FromSeconds(5), ts =>
{
Console.WriteLine("Count: " + ts.Slice.Keys.Count + " " + DateTime.UtcNow.ToString("o"));
if (ts.Slice.Keys.Count != securitiesCount)
{
var included = ts.Slice.Keys.ToHashSet();
expected.ExceptWith(included);
Console.WriteLine("Missing: " + string.Join(",", expected.OrderBy(x => x.Value)));
}
});
stopwatch.Stop();
Console.WriteLine("Total ticks: " + count.Value);
Console.WriteLine("Elapsed time: " + stopwatch.Elapsed);
Console.WriteLine("Ticks/sec: " + (count.Value/stopwatch.Elapsed.TotalSeconds));
Console.WriteLine("Ticks/sec/symbol: " + (count.Value/stopwatch.Elapsed.TotalSeconds)/symbolCount);
}
public void FutureChainEnumerator(bool fillForward)
{
var job = new BacktestNodePacket();
var resultHandler = new BacktestingResultHandler();
var feed = new FileSystemDataFeed();
var algorithm = new AlgorithmStub(feed);
algorithm.Transactions.SetOrderProcessor(new FakeOrderProcessor());
algorithm.SetStartDate(new DateTime(2013, 10, 07));
algorithm.SetEndDate(new DateTime(2013, 10, 08));
algorithm.SetFutureChainProvider(new BacktestingFutureChainProvider(TestGlobals.DataCacheProvider));
var dataPermissionManager = new DataPermissionManager();
using var synchronizer = new Synchronizer();
synchronizer.Initialize(algorithm, algorithm.DataManager);
feed.Initialize(algorithm, job, resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, algorithm.DataManager, synchronizer, dataPermissionManager.DataChannelProvider);
var future = algorithm.AddFuture("ES", fillDataForward: fillForward);
future.SetFilter(0, 300);
algorithm.PostInitialize();
using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
var count = 0L;
var lastMonth = algorithm.StartDate.Month;
foreach (var timeSlice in synchronizer.StreamData(cancellationTokenSource.Token))
{
if (!timeSlice.IsTimePulse && timeSlice.UniverseData?.Count > 0)
{
var nyTime = timeSlice.Time.ConvertFromUtc(algorithm.TimeZone);
var currentExpectedTime = new TimeSpan(0, 0, 0).Add(TimeSpan.FromMinutes(count % (24 * 60)));
while (!future.Exchange.DateTimeIsOpen(nyTime.Date.Add(currentExpectedTime).AddMinutes(-1)))
{
// skip closed market times
currentExpectedTime = new TimeSpan(0, 0, 0).Add(TimeSpan.FromMinutes(++count % (24 * 60)));
}
var universeData = timeSlice.UniverseData.OrderBy(kvp => kvp.Key.Configuration.Symbol).ToList();
var chainData = universeData[0].Value;
Log.Trace($"{nyTime}. Count: {count}. Universe Data Count {universeData.Count}");
Assert.AreEqual(currentExpectedTime, nyTime.TimeOfDay, $"Failed on: {nyTime}. Count: {count}");
Assert.IsTrue(timeSlice.UniverseData.All(kvp => kvp.Value.EndTime.ConvertFromUtc(algorithm.TimeZone).TimeOfDay == nyTime.TimeOfDay));
if (chainData.FilteredContracts.IsNullOrEmpty())
{
Assert.AreEqual(new DateTime(2013, 10, 09), nyTime, $"Unexpected chain FilteredContracts was empty on {nyTime}");
}
if (universeData.Count == 1)
{
// the chain
Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol == future.Symbol));
}
else
{
// we have 2 universe data, the chain and the continuous future
Assert.AreEqual(2, universeData.Count);
Assert.IsTrue(universeData.All(kvp => kvp.Key.Configuration.Symbol.SecurityType == SecurityType.Future));
Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol == future.Symbol));
Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol.ID.Symbol.Contains("CONTINUOUS", StringComparison.InvariantCultureIgnoreCase)));
var continuousData = universeData[1].Value;
Assert.AreEqual(currentExpectedTime, nyTime.TimeOfDay, $"Failed on: {nyTime}");
Assert.IsTrue(!chainData.FilteredContracts.IsNullOrEmpty());
}
count++;
}
}
feed.Exit();
algorithm.DataManager.RemoveAllSubscriptions();
// 2 days worth of minute data
Assert.AreEqual(24 * 2 * 60 + 1, count);
}
public void CoarseUniverseRotatesActiveSecurity()
{
var startDate = new DateTime(2014, 3, 24);
var endDate = new DateTime(2014, 3, 29);
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
timeProvider.SetCurrentTime(startDate);
var coarseTimes = new List <DateTime>
{
new DateTime(2014, 3, 25, 5, 0, 0, 0),
new DateTime(2014, 3, 26, 5, 0, 0, 0),
new DateTime(2014, 3, 27, 5, 0, 0, 0),
new DateTime(2014, 3, 28, 5, 0, 0, 0),
new DateTime(2014, 3, 29, 5, 0, 0, 0)
}.ToHashSet();
var coarseSymbols = new List <Symbol> {
Symbols.SPY, Symbols.AAPL, Symbols.MSFT
};
var emitted = new AutoResetEvent(false);
var dataQueueHandler = new FuncDataQueueHandler(fdqh => Enumerable.Empty <BaseData>(), timeProvider);
var feed = new TestableLiveTradingDataFeed(dataQueueHandler);
var algorithm = new AlgorithmStub(feed);
algorithm.SetLiveMode(true);
var mock = new Mock <ITransactionHandler>();
mock.Setup(m => m.GetOpenOrders(It.IsAny <Func <Order, bool> >())).Returns(new List <Order>());
algorithm.Transactions.SetOrderProcessor(mock.Object);
var synchronizer = new TestableLiveSynchronizer(timeProvider);
synchronizer.Initialize(algorithm, algorithm.DataManager);
var mapFileProvider = new LocalDiskMapFileProvider();
feed.Initialize(algorithm, new LiveNodePacket(), new BacktestingResultHandler(),
mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), new DefaultDataProvider(), algorithm.DataManager, synchronizer, new DataChannelProvider());
var symbolIndex = 0;
var coarseUniverseSelectionCount = 0;
algorithm.AddUniverse(
coarse =>
{
Log.Trace($"Emitted at {algorithm.Time}. Coarse {coarse.First().Time} to {coarse.First().EndTime}");
Interlocked.Increment(ref coarseUniverseSelectionCount);
emitted.Set();
// rotate single symbol in universe
if (symbolIndex == coarseSymbols.Count)
{
symbolIndex = 0;
}
return(new[] { coarseSymbols[symbolIndex++] });
});
algorithm.PostInitialize();
var cancellationTokenSource = new CancellationTokenSource();
Exception exceptionThrown = null;
// create a timer to advance time much faster than realtime
var timerInterval = TimeSpan.FromMilliseconds(5);
var timer = Ref.Create <Timer>(null);
timer.Value = new Timer(state =>
{
try
{
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (currentTime.Date > endDate.Date)
{
feed.Exit();
cancellationTokenSource.Cancel();
return;
}
timeProvider.Advance(TimeSpan.FromHours(1));
var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
algorithm.SetDateTime(timeProvider.GetUtcNow());
if (coarseTimes.Contains(time))
{
// lets wait for coarse to emit
if (!emitted.WaitOne(TimeSpan.FromMilliseconds(15000)))
{
throw new TimeoutException($"Timeout waiting for coarse to emit at {time}");
}
}
var activeSecuritiesCount = algorithm.ActiveSecurities.Count;
Assert.That(activeSecuritiesCount <= 1);
// restart the timer
timer.Value.Change(timerInterval, Timeout.InfiniteTimeSpan);
}
catch (Exception exception)
{
Log.Error(exception);
exceptionThrown = exception;
feed.Exit();
cancellationTokenSource.Cancel();
}
}, null, timerInterval, Timeout.InfiniteTimeSpan);
foreach (var _ in synchronizer.StreamData(cancellationTokenSource.Token))
{
}
timer.Value.DisposeSafely();
algorithm.DataManager.RemoveAllSubscriptions();
dataQueueHandler.DisposeSafely();
synchronizer.DisposeSafely();
emitted.DisposeSafely();
if (exceptionThrown != null)
{
throw new Exception("Exception in timer: ", exceptionThrown);
}
Assert.AreEqual(coarseTimes.Count, coarseUniverseSelectionCount, message: "coarseUniverseSelectionCount");
}
public void FastExitsDoNotThrowUnhandledExceptions()
{
var algorithm = new AlgorithmStub();
// job is used to send into DataQueueHandler
var job = new LiveNodePacket();
// result handler is used due to dependency in SubscriptionDataReader
var resultHandler = new BacktestingResultHandler();
var feed = new TestableLiveTradingDataFeed();
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var securityService = new SecurityService(
algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDataBase,
algorithm);
algorithm.Securities.SetSecurityService(securityService);
var dataManager = new DataManager(feed,
new UniverseSelection(feed, algorithm, securityService),
algorithm.Settings,
algorithm.TimeKeeper,
marketHoursDatabase);
algorithm.SubscriptionManager.SetDataManager(dataManager);
var synchronizer = new TestableSynchronizer(_algorithm, dataManager, feed, true);
algorithm.AddSecurities(Resolution.Tick, Enumerable.Range(0, 20).Select(x => x.ToString()).ToList());
var getNextTicksFunction = Enumerable.Range(0, 20).Select(x => new Tick {
Symbol = SymbolCache.GetSymbol(x.ToString())
}).ToList();
feed.DataQueueHandler = new FuncDataQueueHandler(handler => getNextTicksFunction);
var mapFileProvider = new LocalDiskMapFileProvider();
var fileProvider = new DefaultDataProvider();
feed.Initialize(
algorithm,
job,
resultHandler,
mapFileProvider,
new LocalDiskFactorFileProvider(mapFileProvider),
fileProvider,
dataManager,
synchronizer);
var unhandledExceptionWasThrown = false;
try
{
feed.Exit();
}
catch (Exception ex)
{
QuantConnect.Logging.Log.Error(ex.ToString());
unhandledExceptionWasThrown = true;
}
Thread.Sleep(500);
Assert.IsFalse(unhandledExceptionWasThrown);
}
public void Unsubscribes()
{
var algorithm = new AlgorithmStub(equities: new List<string> { "SPY" }, forex: new List<string> { "EURUSD" });
algorithm.AddData<RemoteFileBaseData>("RemoteFile");
FuncDataQueueHandler dataQueueHandler;
var feed = RunDataFeed(algorithm, out dataQueueHandler);
feed.RemoveSubscription(algorithm.Securities["SPY"]);
Assert.AreEqual(1, dataQueueHandler.Subscriptions.Count);
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(new SymbolSecurityType("SPY", SecurityType.Equity)));
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(new SymbolSecurityType("RemoteFile", SecurityType.Base)));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(new SymbolSecurityType("EURUSD", SecurityType.Forex)));
}
public void HandlesCoarseFundamentalData()
{
var algorithm = new AlgorithmStub();
Symbol symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA);
algorithm.AddUniverse(new FuncUniverse(
new SubscriptionDataConfig(typeof(CoarseFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false),
new UniverseSettings(Resolution.Second, 1, true, false, TimeSpan.Zero),
coarse => coarse.Take(10).Select(x => x.Symbol)
));
var lck = new object();
BaseDataCollection list = null;
const int coarseDataPointCount = 100000;
var timer = new Timer(state =>
{
var currentTime = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
Console.WriteLine(currentTime + ": timer.Elapsed");
lock (state)
{
list = new BaseDataCollection {
Symbol = symbol
};
list.Data.AddRange(Enumerable.Range(0, coarseDataPointCount).Select(x => new CoarseFundamental
{
Symbol = SymbolCache.GetSymbol(x.ToString()),
Time = currentTime - Time.OneDay, // hard-coded coarse period of one day
}));
}
}, lck, TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(500));
bool yieldedUniverseData = false;
var feed = RunDataFeed(algorithm, fdqh =>
{
lock (lck)
{
if (list != null)
{
try
{
var tmp = list;
return(new List <BaseData> {
tmp
});
}
finally
{
list = null;
yieldedUniverseData = true;
}
}
}
return(Enumerable.Empty <BaseData>());
});
Assert.IsTrue(feed.Subscriptions.Any(x => x.IsUniverseSelectionSubscription));
var universeSelectionHadAllData = false;
ConsumeBridge(feed, TimeSpan.FromSeconds(5), ts =>
{
});
Assert.IsTrue(yieldedUniverseData);
Assert.IsTrue(universeSelectionHadAllData);
}
