public void DoesNotSubscribeToCustomData()
{
// Current implementation only sends equity/forex subscriptions to the queue handler,
// new impl sends all, the restriction shouldn't live in the feed, but rather in the
// queue handler impl
FuncDataQueueHandler dataQueueHandler;
var feed = RunDataFeed(out dataQueueHandler, equities: new List <string> {
"SPY"
}, forex: new List <string> {
"EURUSD"
});
_algorithm.AddData <CustomMockedFileBaseData>("CustomMockedFileBaseData");
var customMockedFileBaseData = SymbolCache.GetSymbol("CustomMockedFileBaseData");
var emittedData = false;
ConsumeBridge(feed, TimeSpan.FromSeconds(2), ts =>
{
ConsoleWriteLine("Count: " + ts.Slice.Keys.Count + " " + DateTime.UtcNow.ToString("o"));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.SPY));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.EURUSD));
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(customMockedFileBaseData));
emittedData = true;
});
Assert.IsTrue(emittedData);
}
public void HandlesManyCustomDataSubscriptions()
{
DataManager dataManager;
var resolution = Resolution.Second;
var algorithm = new AlgorithmStub(out dataManager);
for (int i = 0; i < 100; i++)
{
algorithm.AddData <CustomMockedFileBaseData>((100 + i).ToString(), resolution, fillDataForward: false);
}
var feed = RunDataFeed(algorithm, dataManager);
int count = 0;
var emittedData = false;
var stopwatch = Stopwatch.StartNew();
var previousTime = DateTime.Now;
Console.WriteLine("start: " + previousTime.ToString("o"));
ConsumeBridge(algorithm, feed, TimeSpan.FromSeconds(5), false, ts =>
{
// because this is a remote file we may skip data points while the newest
// version of the file is downloading [internet speed] and also we decide
// not to emit old data
stopwatch.Stop();
if (ts.Slice.Count == 0)
{
return;
}
emittedData = true;
count++;
// make sure within 2 seconds
var delta = DateTime.Now.Subtract(previousTime);
previousTime = DateTime.Now;
Assert.IsTrue(delta <= TimeSpan.FromSeconds(2), delta.ToString());
ConsoleWriteLine("TimeProvider now: " + _manualTimeProvider.GetUtcNow() + " Count: "
+ ts.Slice.Count + ". Delta (ms): "
+ ((decimal)delta.TotalMilliseconds).SmartRounding() + Environment.NewLine);
});
Console.WriteLine("Count: " + count);
Console.WriteLine("Spool up time: " + stopwatch.Elapsed);
Assert.That(count, Is.GreaterThan(20));
Assert.IsTrue(emittedData);
}
public void HandlesRestApi()
{
var resolution = Resolution.Second;
var algorithm = new AlgorithmStub();
algorithm.AddData <RestApiBaseData>("RestApi", resolution);
var symbol = SymbolCache.GetSymbol("RestApi");
FuncDataQueueHandler dqgh;
var timeProvider = new ManualTimeProvider(new DateTime(2015, 10, 10, 16, 36, 0));
var feed = RunDataFeed(algorithm, out dqgh, null);
var count = 0;
var receivedData = false;
var timeZone = algorithm.Securities[symbol].Exchange.TimeZone;
RestApiBaseData last = null;
var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
foreach (var ts in feed)
{
//timeProvider.AdvanceSeconds(0.5);
if (!ts.Slice.ContainsKey(symbol))
{
return;
}
count++;
receivedData = true;
var data = (RestApiBaseData)ts.Slice[symbol];
var time = data.EndTime.ConvertToUtc(timeZone);
Console.WriteLine(DateTime.UtcNow + ": Data time: " + time.ConvertFromUtc(TimeZones.NewYork) + Environment.NewLine);
if (last != null)
{
Assert.AreEqual(last.EndTime, data.EndTime.Subtract(resolution.ToTimeSpan()));
}
last = data;
}
// even though we're doing 10 seconds, give a little
// leeway for slow internet traffic
Assert.That(count, Is.GreaterThanOrEqualTo(8));
Assert.IsTrue(receivedData);
Assert.That(RestApiBaseData.ReaderCount, Is.LessThanOrEqualTo(30)); // we poll at 10x frequency
Console.WriteLine("Count: " + count + " ReaderCount: " + RestApiBaseData.ReaderCount);
}
public void HandlesManyCustomDataSubscriptions()
{
var resolution = Resolution.Second;
var algorithm = new AlgorithmStub();
for (int i = 0; i < 5; i++)
{
algorithm.AddData <RemoteFileBaseData>((100 + i).ToString(), resolution, fillDataForward: false);
}
var feed = RunDataFeed(algorithm);
int count = 0;
bool receivedData = false;
var stopwatch = Stopwatch.StartNew();
Console.WriteLine("start: " + DateTime.UtcNow.ToString("o"));
ConsumeBridge(feed, TimeSpan.FromSeconds(5), ts =>
{
// because this is a remote file we may skip data points while the newest
// version of the file is downloading [internet speed] and also we decide
// not to emit old data
stopwatch.Stop();
if (ts.Slice.Count == 0)
{
return;
}
count++;
receivedData = true;
var time = ts.Slice.Min(x => x.Value.EndTime).ConvertToUtc(TimeZones.NewYork);
// make sure within 2 seconds
var delta = DateTime.UtcNow.Subtract(time);
//Assert.IsTrue(delta <= TimeSpan.FromSeconds(2), delta.ToString());
Console.WriteLine("Count: " + ts.Slice.Count + "Data time: " + time.ConvertFromUtc(TimeZones.NewYork) + " Delta (ms): "
+ ((decimal)delta.TotalMilliseconds).SmartRounding() + Environment.NewLine);
});
Console.WriteLine("end: " + DateTime.UtcNow.ToString("o"));
Console.WriteLine("Spool up time: " + stopwatch.Elapsed);
// even though we're doing 20 seconds, give a little
// leeway for slow internet traffic
//Assert.That(count, Is.GreaterThan(17));
//Assert.IsTrue(receivedData);
}
public void DoesNotSubscribeToCustomData()
{
// Current implementation only sends equity/forex subscriptions to the queue handler,
// new impl sends all, the restriction shouldn't live in the feed, but rather in the
// queue handler impl
var algorithm = new AlgorithmStub(equities: new List<string> { "SPY" }, forex: new List<string> { "EURUSD" });
algorithm.AddData<RemoteFileBaseData>("RemoteFile");
FuncDataQueueHandler dataQueueHandler;
RunDataFeed(algorithm, out dataQueueHandler);
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(new SymbolSecurityType("SPY", SecurityType.Equity)));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(new SymbolSecurityType("EURUSD", SecurityType.Forex)));
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(new SymbolSecurityType("REMOTEFILE", SecurityType.Base)));
Assert.AreEqual(2, dataQueueHandler.Subscriptions.Count);
}
public void RemoveSecurity()
{
DataManager dataManager;
var algorithm = new AlgorithmStub(out dataManager, equities: new List <string> {
"SPY"
}, forex: new List <string> {
"EURUSD"
});
algorithm.SetFinishedWarmingUp();
algorithm.Transactions.SetOrderProcessor(new FakeOrderProcessor());
algorithm.AddData <CustomMockedFileBaseData>("CustomMockedFileBaseData");
var customMockedFileBaseData = SymbolCache.GetSymbol("CustomMockedFileBaseData");
FuncDataQueueHandler dataQueueHandler;
var feed = RunDataFeed(algorithm, out dataQueueHandler, dataManager);
var emittedData = false;
var currentSubscriptionCount = 0;
var securityChanges = 0;
ConsumeBridge(algorithm, feed, TimeSpan.FromSeconds(5), true, ts =>
{
securityChanges += ts.SecurityChanges.Count;
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(customMockedFileBaseData));
if (!emittedData)
{
currentSubscriptionCount = dataQueueHandler.Subscriptions.Count;
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.SPY));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.EURUSD));
algorithm.RemoveSecurity(Symbols.SPY);
emittedData = true;
}
else
{
Assert.AreEqual(currentSubscriptionCount - 1, dataQueueHandler.Subscriptions.Count);
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(Symbols.SPY));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.EURUSD));
}
});
Assert.IsTrue(emittedData);
Assert.AreEqual(4, securityChanges + algorithm.SecurityChangesRecord.Count);
Assert.AreEqual(Symbols.SPY, algorithm.SecurityChangesRecord.First().RemovedSecurities.First().Symbol);
}
public void HandlesRestApi()
{
DataManager dataManager;
var resolution = Resolution.Second;
var algorithm = new AlgorithmStub(out dataManager);
algorithm.AddData <RestApiBaseData>("RestApi", resolution);
var symbol = SymbolCache.GetSymbol("RestApi");
FuncDataQueueHandler dqgh;
var feed = RunDataFeed(algorithm, out dqgh, dataManager);
var count = 0;
var receivedData = false;
var timeZone = algorithm.Securities[symbol].Exchange.TimeZone;
RestApiBaseData last = null;
foreach (var ts in feed)
{
if (!ts.Slice.ContainsKey(symbol))
{
return;
}
count++;
receivedData = true;
var data = (RestApiBaseData)ts.Slice[symbol];
var time = data.EndTime.ConvertToUtc(timeZone);
ConsoleWriteLine(DateTime.UtcNow + ": Data time: " + time.ConvertFromUtc(TimeZones.NewYork) + Environment.NewLine);
if (last != null)
{
Assert.AreEqual(last.EndTime, data.EndTime.Subtract(resolution.ToTimeSpan()));
}
last = data;
}
feed.Exit();
Assert.That(count, Is.GreaterThanOrEqualTo(8));
Assert.IsTrue(receivedData);
Assert.That(RestApiBaseData.ReaderCount, Is.LessThanOrEqualTo(30)); // we poll at 10x frequency
Console.WriteLine("Count: " + count + " ReaderCount: " + RestApiBaseData.ReaderCount);
}
public void Unsubscribes()
{
DataManager dataManager;
var algorithm = new AlgorithmStub(out dataManager, equities: new List <string> {
"SPY"
}, forex: new List <string> {
"EURUSD"
});
algorithm.AddData <CustomMockedFileBaseData>("CustomMockedFileBaseData");
var customMockedFileBaseData = SymbolCache.GetSymbol("CustomMockedFileBaseData");
FuncDataQueueHandler dataQueueHandler;
var feed = RunDataFeed(algorithm, out dataQueueHandler, dataManager);
var emittedData = false;
var currentSubscriptionCount = 0;
ConsumeBridge(algorithm, feed, TimeSpan.FromSeconds(5), false, ts =>
{
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(customMockedFileBaseData));
if (!emittedData)
{
currentSubscriptionCount = dataQueueHandler.Subscriptions.Count;
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(_equityUsaUserUniverse));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(_forexFxcmUserUniverse));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.SPY));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.EURUSD));
feed.RemoveSubscription(feed.Subscriptions.Single(sub => sub.Configuration.Symbol == Symbols.SPY).Configuration);
emittedData = true;
}
else
{
Assert.AreEqual(currentSubscriptionCount - 1, dataQueueHandler.Subscriptions.Count);
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(Symbols.SPY));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.EURUSD));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(_equityUsaUserUniverse));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(_forexFxcmUserUniverse));
}
});
Assert.IsTrue(emittedData);
}
public void Unsubscribes()
{
var algorithm = new AlgorithmStub(equities: new List <string> {
"SPY"
}, forex: new List <string> {
"EURUSD"
});
algorithm.AddData <RemoteFileBaseData>("RemoteFile");
var remoteFile = SymbolCache.GetSymbol("RemoteFile");
FuncDataQueueHandler dataQueueHandler;
var feed = RunDataFeed(algorithm, out dataQueueHandler);
feed.RemoveSubscription(feed.Subscriptions.Single(x => x.Configuration.Symbol == Symbols.SPY));
Assert.AreEqual(1, dataQueueHandler.Subscriptions.Count);
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(Symbols.SPY));
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(remoteFile));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.EURUSD));
}
public void Unsubscribes()
{
var algorithm = new AlgorithmStub(equities: new List <string> {
"SPY"
}, forex: new List <string> {
"EURUSD"
});
algorithm.AddData <RemoteFileBaseData>("RemoteFile");
FuncDataQueueHandler dataQueueHandler;
var feed = RunDataFeed(algorithm, out dataQueueHandler);
feed.RemoveSubscription(feed.Subscriptions.Single(x => x.Configuration.Symbol == "SPY"));
Assert.AreEqual(1, dataQueueHandler.Subscriptions.Count);
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(new SymbolSecurityType("SPY", SecurityType.Equity)));
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(new SymbolSecurityType("RemoteFile", SecurityType.Base)));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(new SymbolSecurityType("EURUSD", SecurityType.Forex)));
}
public void DoesNotSubscribeToCustomData()
{
// Current implementation only sends equity/forex subscriptions to the queue handler,
// new impl sends all, the restriction shouldn't live in the feed, but rather in the
// queue handler impl
var algorithm = new AlgorithmStub(equities: new List <string> {
"SPY"
}, forex: new List <string> {
"EURUSD"
});
algorithm.AddData <RemoteFileBaseData>("RemoteFile");
var remoteFile = SymbolCache.GetSymbol("RemoteFile");
FuncDataQueueHandler dataQueueHandler;
RunDataFeed(algorithm, out dataQueueHandler);
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.SPY));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.EURUSD));
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(remoteFile));
Assert.AreEqual(2, dataQueueHandler.Subscriptions.Count);
}
public void HandlesRestApi()
{
var resolution = Resolution.Second;
var algorithm = new AlgorithmStub();
algorithm.AddData<RestApiBaseData>("RestApi", resolution);
var symbol = SymbolCache.GetSymbol("RestApi");
FuncDataQueueHandler dqgh;
var timeProvider = new ManualTimeProvider(new DateTime(2015, 10, 10, 16, 36, 0));
var feed = RunDataFeed(algorithm, out dqgh, null);
var count = 0;
var receivedData = false;
var timeZone = algorithm.Securities[symbol].Exchange.TimeZone;
RestApiBaseData last = null;
var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
foreach (var ts in feed)
{
//timeProvider.AdvanceSeconds(0.5);
if (!ts.Slice.ContainsKey(symbol)) return;
count++;
receivedData = true;
var data = (RestApiBaseData)ts.Slice[symbol];
var time = data.EndTime.ConvertToUtc(timeZone);
Console.WriteLine(DateTime.UtcNow + ": Data time: " + time.ConvertFromUtc(TimeZones.NewYork) + Environment.NewLine);
if (last != null)
{
Assert.AreEqual(last.EndTime, data.EndTime.Subtract(resolution.ToTimeSpan()));
}
last = data;
}
// even though we're doing 10 seconds, give a little
// leeway for slow internet traffic
Assert.That(count, Is.GreaterThanOrEqualTo(8));
Assert.IsTrue(receivedData);
Assert.That(RestApiBaseData.ReaderCount, Is.LessThanOrEqualTo(30)); // we poll at 10x frequency
Console.WriteLine("Count: " + count + " ReaderCount: " + RestApiBaseData.ReaderCount);
}
public void HandlesManyCustomDataSubscriptions()
{
var resolution = Resolution.Second;
var algorithm = new AlgorithmStub();
for (int i = 0; i < 5; i++)
{
algorithm.AddData<RemoteFileBaseData>((100+ i).ToString(), resolution, fillDataForward: false);
}
var feed = RunDataFeed(algorithm);
int count = 0;
bool receivedData = false;
var stopwatch = Stopwatch.StartNew();
Console.WriteLine("start: " + DateTime.UtcNow.ToString("o"));
ConsumeBridge(feed, TimeSpan.FromSeconds(5), ts =>
{
// because this is a remote file we may skip data points while the newest
// version of the file is downloading [internet speed] and also we decide
// not to emit old data
stopwatch.Stop();
if (ts.Slice.Count == 0) return;
count++;
receivedData = true;
var time = ts.Slice.Min(x => x.Value.EndTime).ConvertToUtc(TimeZones.NewYork);
// make sure within 2 seconds
var delta = DateTime.UtcNow.Subtract(time);
//Assert.IsTrue(delta <= TimeSpan.FromSeconds(2), delta.ToString());
Console.WriteLine("Count: " + ts.Slice.Count + "Data time: " + time.ConvertFromUtc(TimeZones.NewYork) + " Delta (ms): "
+ ((decimal) delta.TotalMilliseconds).SmartRounding() + Environment.NewLine);
});
Console.WriteLine("end: " + DateTime.UtcNow.ToString("o"));
Console.WriteLine("Spool up time: " + stopwatch.Elapsed);
// even though we're doing 20 seconds, give a little
// leeway for slow internet traffic
//Assert.That(count, Is.GreaterThan(17));
//Assert.IsTrue(receivedData);
}
public void Unsubscribes()
{
var algorithm = new AlgorithmStub(equities: new List<string> { "SPY" }, forex: new List<string> { "EURUSD" });
algorithm.AddData<RemoteFileBaseData>("RemoteFile");
var remoteFile = SymbolCache.GetSymbol("RemoteFile");
FuncDataQueueHandler dataQueueHandler;
var feed = RunDataFeed(algorithm, out dataQueueHandler);
feed.RemoveSubscription(feed.Subscriptions.Single(x => x.Configuration.Symbol == Symbols.SPY));
Assert.AreEqual(1, dataQueueHandler.Subscriptions.Count);
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(Symbols.SPY));
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(remoteFile));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.EURUSD));
}
public void Unsubscribes()
{
var algorithm = new AlgorithmStub(equities: new List<string> { "SPY" }, forex: new List<string> { "EURUSD" });
algorithm.AddData<RemoteFileBaseData>("RemoteFile");
FuncDataQueueHandler dataQueueHandler;
var feed = RunDataFeed(algorithm, out dataQueueHandler);
feed.RemoveSubscription(algorithm.Securities["SPY"]);
Assert.AreEqual(1, dataQueueHandler.Subscriptions.Count);
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(new SymbolSecurityType("SPY", SecurityType.Equity)));
Assert.IsFalse(dataQueueHandler.Subscriptions.Contains(new SymbolSecurityType("RemoteFile", SecurityType.Base)));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(new SymbolSecurityType("EURUSD", SecurityType.Forex)));
}