public void AddsNewUniverse()
{
_algorithm.UniverseSettings.Resolution = Resolution.Second; // Default is Minute and we need something faster
_algorithm.UniverseSettings.ExtendedMarketHours = true; // Current _startDate is at extended market hours
FuncDataQueueHandler dataQueueHandler;
var feed = RunDataFeed(out dataQueueHandler, forex: new List <string> {
"EURUSD"
});
var firstTime = false;
var securityChanges = 0;
var newDataCount = 0;
ConsumeBridge(feed, TimeSpan.FromSeconds(5), true, ts =>
{
securityChanges += ts.SecurityChanges.Count;
if (!firstTime)
{
Assert.AreEqual(1, dataQueueHandler.Subscriptions.Count);
_algorithm.AddUniverse("TestUniverse", time => new List <string> {
"AAPL", "SPY"
});
firstTime = true;
}
else
{
if (dataQueueHandler.Subscriptions.Count == 2)
{
Assert.AreEqual(1, dataQueueHandler.Subscriptions.Count(x => x.Value.Contains("TESTUNIVERSE")));
}
else if (dataQueueHandler.Subscriptions.Count == 4)
{
Assert.AreEqual(1, dataQueueHandler.Subscriptions.Count(x => x.Value.Contains("TESTUNIVERSE")));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.SPY));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.AAPL));
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.EURUSD));
// Might delay a couple of Slices to send over the data, so we will count them and assert a minimum amount
if (ts.Slice.Keys.Contains(Symbols.AAPL) &&
ts.Slice.Keys.Contains(Symbols.SPY))
{
newDataCount++;
}
}
else
{
Assert.Fail($"Subscriptions.Count: {dataQueueHandler.Subscriptions.Count}");
}
}
});
Console.WriteLine("newDataCount: " + newDataCount);
Assert.AreEqual(3, securityChanges);
Assert.GreaterOrEqual(newDataCount, 490);
Assert.IsTrue(firstTime);
}
public void CreatedEquityIsNotAddedToSymbolCache()
{
SymbolCache.Clear();
var algorithm = new AlgorithmStub(new MockDataFeed());
algorithm.SetEndDate(Time.EndOfTime);
algorithm.SetStartDate(DateTime.UtcNow.Subtract(TimeSpan.FromDays(10)));
algorithm.AddUniverse(CoarseSelectionFunction, FineSelectionFunction);
var universe = algorithm.UniverseManager.Values.First();
var securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection(
universe,
algorithm.EndDate.ConvertToUtc(algorithm.TimeZone).Subtract(TimeSpan.FromDays(1)),
new BaseDataCollection(
DateTime.UtcNow,
Symbol.Create("GOOG", SecurityType.Equity, Market.USA),
new List <BaseData>()
)
);
Symbol symbol;
Assert.AreEqual(1, securityChanges.AddedSecurities.Count);
Assert.AreEqual(Symbol.Create("SPY", SecurityType.Equity, Market.USA),
securityChanges.AddedSecurities.First().Symbol);
Assert.IsFalse(SymbolCache.TryGetSymbol("GOOG", out symbol));
Assert.IsFalse(SymbolCache.TryGetSymbol("SPY", out symbol));
}
public void RemovalFromUniverseAndDataFeedMakesSecurityNotTradable()
{
SymbolCache.Clear();
var algorithm = new AlgorithmStub(new MockDataFeedWithSubscription());
var orderProcessorMock = new Mock <IOrderProcessor>();
orderProcessorMock.Setup(m => m.GetOpenOrders(It.IsAny <Func <Order, bool> >())).Returns(new List <Order>());
algorithm.Transactions.SetOrderProcessor(orderProcessorMock.Object);
algorithm.SetStartDate(2012, 3, 27);
algorithm.SetEndDate(2012, 3, 30);
algorithm.AddUniverse("my-custom-universe", dt => dt.Day < 30 ? new List <string> {
"CPRT"
} : Enumerable.Empty <string>());
// OnEndOfTimeStep will add all pending universe additions
algorithm.OnEndOfTimeStep();
var universe = algorithm.UniverseManager.Values.First();
var securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection(
universe,
algorithm.EndDate.ConvertToUtc(algorithm.TimeZone).Subtract(TimeSpan.FromDays(2)),
new BaseDataCollection(
algorithm.UtcTime,
Symbol.Create("CPRT", SecurityType.Equity, Market.USA),
new List <BaseData>()
)
);
Assert.AreEqual(1, securityChanges.AddedSecurities.Count);
Assert.AreEqual(0, securityChanges.RemovedSecurities.Count);
var security = securityChanges.AddedSecurities.First();
Assert.IsTrue(security.IsTradable);
securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection(
universe,
algorithm.EndDate.ConvertToUtc(algorithm.TimeZone),
new BaseDataCollection(
algorithm.UtcTime,
Symbol.Create("CPRT", SecurityType.Equity, Market.USA),
new List <BaseData>()
)
);
Assert.AreEqual(0, securityChanges.AddedSecurities.Count);
Assert.AreEqual(1, securityChanges.RemovedSecurities.Count);
Assert.AreEqual(security.Symbol, securityChanges.RemovedSecurities.First().Symbol);
Assert.IsFalse(security.IsTradable);
}
public void CoarseFundamentalHasFundamentalDataFalseExcludedInFineUniverseSelection()
{
var algorithm = new AlgorithmStub(new MockDataFeed());
algorithm.SetEndDate(Time.EndOfTime);
algorithm.SetStartDate(DateTime.UtcNow.Subtract(TimeSpan.FromDays(10)));
algorithm.AddUniverse(
coarse => coarse.Select(c => c.Symbol),
fine => fine.Select(f => f.Symbol)
);
// OnEndOfTimeStep will add all pending universe additions
algorithm.OnEndOfTimeStep();
var universe = algorithm.UniverseManager.Values.First();
var securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection(
universe,
algorithm.EndDate.ConvertToUtc(algorithm.TimeZone).Subtract(TimeSpan.FromDays(1)),
new BaseDataCollection(
DateTime.UtcNow,
Symbols.AAPL,
new[]
{
new CoarseFundamental
{
Symbol = Symbols.AAPL,
HasFundamentalData = true
},
new CoarseFundamental
{
Symbol = Symbols.SPY,
HasFundamentalData = false
}
}
)
);
Assert.AreEqual(1, securityChanges.Count);
Assert.AreEqual(Symbols.AAPL, securityChanges.AddedSecurities.First().Symbol);
}
public void CreatedEquityIsNotAddedToSymbolCache()
{
SymbolCache.Clear();
var algorithm = new AlgorithmStub(new MockDataFeed());
algorithm.SetEndDate(Time.EndOfTime);
algorithm.SetStartDate(DateTime.UtcNow.Subtract(TimeSpan.FromDays(10)));
algorithm.AddUniverse(CoarseSelectionFunction, FineSelectionFunction);
// OnEndOfTimeStep will add all pending universe additions
algorithm.OnEndOfTimeStep();
var universe = algorithm.UniverseManager.Values.First();
var securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection(
universe,
algorithm.EndDate.ConvertToUtc(algorithm.TimeZone).Subtract(TimeSpan.FromDays(1)),
new BaseDataCollection(
DateTime.UtcNow,
Symbols.AAPL,
new[]
{
new CoarseFundamental
{
Symbol = Symbols.AAPL,
HasFundamentalData = true
},
new CoarseFundamental
{
Symbol = Symbols.SPY,
HasFundamentalData = false
}
}
)
);
Symbol symbol;
Assert.AreEqual(1, securityChanges.AddedSecurities.Count);
Assert.AreEqual(Symbols.AAPL, securityChanges.AddedSecurities.First().Symbol);
Assert.IsFalse(SymbolCache.TryGetSymbol("AAPL", out symbol));
Assert.IsFalse(SymbolCache.TryGetSymbol("SPY", out symbol));
}
public void HandlesCoarseFundamentalData()
{
var algorithm = new AlgorithmStub();
Symbol symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA);
algorithm.AddUniverse(new FuncUniverse(
new SubscriptionDataConfig(typeof(CoarseFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false),
new UniverseSettings(Resolution.Second, 1, true, false, TimeSpan.Zero),
coarse => coarse.Take(10).Select(x => x.Symbol)
));
var lck = new object();
BaseDataCollection list = null;
const int coarseDataPointCount = 100000;
var timer = new Timer(state =>
{
var currentTime = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
Console.WriteLine(currentTime + ": timer.Elapsed");
lock (state)
{
list = new BaseDataCollection {Symbol = symbol};
list.Data.AddRange(Enumerable.Range(0, coarseDataPointCount).Select(x => new CoarseFundamental
{
Symbol = SymbolCache.GetSymbol(x.ToString()),
Time = currentTime - Time.OneDay, // hard-coded coarse period of one day
}));
}
}, lck, TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(500));
bool yieldedUniverseData = false;
var feed = RunDataFeed(algorithm, fdqh =>
{
lock (lck)
{
if (list != null)
try
{
var tmp = list;
return new List<BaseData> { tmp };
}
finally
{
list = null;
yieldedUniverseData = true;
}
}
return Enumerable.Empty<BaseData>();
});
Assert.IsTrue(feed.Subscriptions.Any(x => x.IsUniverseSelectionSubscription));
var universeSelectionHadAllData = false;
ConsumeBridge(feed, TimeSpan.FromSeconds(5), ts =>
{
});
Assert.IsTrue(yieldedUniverseData);
Assert.IsTrue(universeSelectionHadAllData);
}
public void CoarseUniverseRotatesActiveSecurity()
{
var startDate = new DateTime(2014, 3, 24);
var endDate = new DateTime(2014, 3, 29);
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
timeProvider.SetCurrentTime(startDate);
var coarseTimes = new List <DateTime>
{
new DateTime(2014, 3, 25, 5, 0, 0, 0),
new DateTime(2014, 3, 26, 5, 0, 0, 0),
new DateTime(2014, 3, 27, 5, 0, 0, 0),
new DateTime(2014, 3, 28, 5, 0, 0, 0),
new DateTime(2014, 3, 29, 5, 0, 0, 0)
}.ToHashSet();
var coarseSymbols = new List <Symbol> {
Symbols.SPY, Symbols.AAPL, Symbols.MSFT
};
var emitted = new AutoResetEvent(false);
var dataQueueHandler = new FuncDataQueueHandler(fdqh => Enumerable.Empty <BaseData>(), timeProvider);
var feed = new TestableLiveTradingDataFeed(dataQueueHandler);
var algorithm = new AlgorithmStub(feed);
algorithm.SetLiveMode(true);
var mock = new Mock <ITransactionHandler>();
mock.Setup(m => m.GetOpenOrders(It.IsAny <Func <Order, bool> >())).Returns(new List <Order>());
algorithm.Transactions.SetOrderProcessor(mock.Object);
var synchronizer = new TestableLiveSynchronizer(timeProvider);
synchronizer.Initialize(algorithm, algorithm.DataManager);
var mapFileProvider = new LocalDiskMapFileProvider();
feed.Initialize(algorithm, new LiveNodePacket(), new BacktestingResultHandler(),
mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), new DefaultDataProvider(), algorithm.DataManager, synchronizer, new DataChannelProvider());
var symbolIndex = 0;
var coarseUniverseSelectionCount = 0;
algorithm.AddUniverse(
coarse =>
{
Log.Trace($"Emitted at {algorithm.Time}. Coarse {coarse.First().Time} to {coarse.First().EndTime}");
Interlocked.Increment(ref coarseUniverseSelectionCount);
emitted.Set();
// rotate single symbol in universe
if (symbolIndex == coarseSymbols.Count)
{
symbolIndex = 0;
}
return(new[] { coarseSymbols[symbolIndex++] });
});
algorithm.PostInitialize();
var cancellationTokenSource = new CancellationTokenSource();
Exception exceptionThrown = null;
// create a timer to advance time much faster than realtime
var timerInterval = TimeSpan.FromMilliseconds(5);
var timer = Ref.Create <Timer>(null);
timer.Value = new Timer(state =>
{
try
{
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (currentTime.Date > endDate.Date)
{
feed.Exit();
cancellationTokenSource.Cancel();
return;
}
timeProvider.Advance(TimeSpan.FromHours(1));
var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
algorithm.SetDateTime(timeProvider.GetUtcNow());
if (coarseTimes.Contains(time))
{
// lets wait for coarse to emit
if (!emitted.WaitOne(TimeSpan.FromMilliseconds(15000)))
{
throw new TimeoutException($"Timeout waiting for coarse to emit at {time}");
}
}
var activeSecuritiesCount = algorithm.ActiveSecurities.Count;
Assert.That(activeSecuritiesCount <= 1);
// restart the timer
timer.Value.Change(timerInterval, Timeout.InfiniteTimeSpan);
}
catch (Exception exception)
{
Log.Error(exception);
exceptionThrown = exception;
feed.Exit();
cancellationTokenSource.Cancel();
}
}, null, timerInterval, Timeout.InfiniteTimeSpan);
foreach (var _ in synchronizer.StreamData(cancellationTokenSource.Token))
{
}
timer.Value.DisposeSafely();
algorithm.DataManager.RemoveAllSubscriptions();
dataQueueHandler.DisposeSafely();
synchronizer.DisposeSafely();
emitted.DisposeSafely();
if (exceptionThrown != null)
{
throw new Exception("Exception in timer: ", exceptionThrown);
}
Assert.AreEqual(coarseTimes.Count, coarseUniverseSelectionCount, message: "coarseUniverseSelectionCount");
}
public void HandlesCoarseFundamentalData()
{
var algorithm = new AlgorithmStub();
Symbol symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA);
algorithm.AddUniverse(new FuncUniverse(
new SubscriptionDataConfig(typeof(CoarseFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false),
new UniverseSettings(Resolution.Second, 1, true, false, TimeSpan.Zero),
coarse => coarse.Take(10).Select(x => x.Symbol)
));
var lck = new object();
BaseDataCollection list = null;
const int coarseDataPointCount = 100000;
var timer = new Timer(state =>
{
var currentTime = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
Console.WriteLine(currentTime + ": timer.Elapsed");
lock (state)
{
list = new BaseDataCollection {
Symbol = symbol
};
list.Data.AddRange(Enumerable.Range(0, coarseDataPointCount).Select(x => new CoarseFundamental
{
Symbol = SymbolCache.GetSymbol(x.ToString()),
Time = currentTime - Time.OneDay, // hard-coded coarse period of one day
}));
}
}, lck, TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(500));
bool yieldedUniverseData = false;
var feed = RunDataFeed(algorithm, fdqh =>
{
lock (lck)
{
if (list != null)
{
try
{
var tmp = list;
return(new List <BaseData> {
tmp
});
}
finally
{
list = null;
yieldedUniverseData = true;
}
}
}
return(Enumerable.Empty <BaseData>());
});
Assert.IsTrue(feed.Subscriptions.Any(x => x.IsUniverseSelectionSubscription));
var universeSelectionHadAllData = false;
ConsumeBridge(feed, TimeSpan.FromSeconds(5), ts =>
{
});
Assert.IsTrue(yieldedUniverseData);
Assert.IsTrue(universeSelectionHadAllData);
}
public void CoarseUniverseRotatesActiveSecurity()
{
var startDate = new DateTime(2014, 3, 24);
var endDate = new DateTime(2014, 3, 28);
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
timeProvider.SetCurrentTime(startDate);
var coarseTimes = new List <DateTime>
{
new DateTime(2014, 3, 25),
new DateTime(2014, 3, 25, 23, 0, 0),
new DateTime(2014, 3, 27, 1, 0, 0)
}.ToHashSet();
var coarseSymbols = new List <Symbol> {
Symbols.SPY, Symbols.AAPL, Symbols.MSFT
};
var coarseUsaSymbol = CoarseFundamental.CreateUniverseSymbol(Market.USA, false);
var coarseDataEmittedCount = 0;
var lastTime = DateTime.MinValue;
var dataQueueHandler = new FuncDataQueueHandler(fdqh =>
{
var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (time != lastTime)
{
lastTime = time;
if (coarseTimes.Contains(time))
{
// emit coarse data at selected times
var coarseData = new BaseDataCollection {
Symbol = coarseUsaSymbol
};
foreach (var symbol in coarseSymbols)
{
coarseData.Data.Add(
new CoarseFundamental
{
Symbol = symbol,
Time = time,
Market = Market.USA,
Value = 100
});
}
coarseDataEmittedCount++;
return(new List <BaseData> {
coarseData
});
}
}
return(Enumerable.Empty <BaseData>());
});
var feed = new TestableLiveTradingDataFeed(dataQueueHandler);
var algorithm = new AlgorithmStub(feed);
algorithm.SetLiveMode(true);
var mock = new Mock <ITransactionHandler>();
mock.Setup(m => m.GetOpenOrders(It.IsAny <Func <Order, bool> >())).Returns(new List <Order>());
algorithm.Transactions.SetOrderProcessor(mock.Object);
var synchronizer = new TestableLiveSynchronizer(timeProvider);
synchronizer.Initialize(algorithm, algorithm.DataManager);
var mapFileProvider = new LocalDiskMapFileProvider();
feed.Initialize(algorithm, new LiveNodePacket(), new BacktestingResultHandler(),
mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), new DefaultDataProvider(), algorithm.DataManager, synchronizer);
var symbolIndex = 0;
var coarseUniverseSelectionCount = 0;
algorithm.AddUniverse(
coarse =>
{
coarseUniverseSelectionCount++;
// rotate single symbol in universe
if (symbolIndex == coarseSymbols.Count)
{
symbolIndex = 0;
}
return(new[] { coarseSymbols[symbolIndex++] });
});
algorithm.PostInitialize();
var cancellationTokenSource = new CancellationTokenSource();
Exception exceptionThrown = null;
// create a timer to advance time much faster than realtime
var timerInterval = TimeSpan.FromMilliseconds(50);
var timer = Ref.Create <Timer>(null);
timer.Value = new Timer(state =>
{
try
{
// stop the timer to prevent reentrancy
timer.Value.Change(Timeout.Infinite, Timeout.Infinite);
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (currentTime.Date > endDate.Date)
{
feed.Exit();
cancellationTokenSource.Cancel();
return;
}
timeProvider.Advance(TimeSpan.FromHours(1));
var activeSecuritiesCount = algorithm.ActiveSecurities.Count;
Assert.That(activeSecuritiesCount <= 1);
// restart the timer
timer.Value.Change(timerInterval, timerInterval);
}
catch (Exception exception)
{
Log.Error(exception);
exceptionThrown = exception;
feed.Exit();
cancellationTokenSource.Cancel();
}
}, null, TimeSpan.FromSeconds(1), timerInterval);
foreach (var _ in synchronizer.StreamData(cancellationTokenSource.Token))
{
}
timer.Value.Dispose();
if (exceptionThrown != null)
{
throw new Exception("Exception in timer: ", exceptionThrown);
}
Assert.AreEqual(coarseTimes.Count, coarseDataEmittedCount);
Assert.AreEqual(coarseTimes.Count, coarseUniverseSelectionCount);
}