public void EmitsData()
{
var algorithm = new AlgorithmStub(forex: new List<string> {"EURUSD"});
// job is used to send into DataQueueHandler
var job = new LiveNodePacket();
// result handler is used due to dependency in SubscriptionDataReader
var resultHandler = new BacktestingResultHandler();
var dataFileProvider = new DefaultDataFileProvider();
var lastTime = DateTime.MinValue;
var timeProvider = new RealTimeProvider();
var dataQueueHandler = new FuncDataQueueHandler(fdqh =>
{
var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.EasternStandard);
if (time == lastTime) return Enumerable.Empty<BaseData>();
lastTime = time;
return Enumerable.Range(0, 9).Select(x => new Tick(time.AddMilliseconds(x*100), Symbols.EURUSD, 1.3m, 1.2m, 1.3m));
});
var feed = new TestableLiveTradingDataFeed(dataQueueHandler, timeProvider);
var mapFileProvider = new LocalDiskMapFileProvider();
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), dataFileProvider);
var feedThreadStarted = new ManualResetEvent(false);
Task.Factory.StartNew(() =>
{
feedThreadStarted.Set();
feed.Run();
});
// wait for feed.Run to actually begin
feedThreadStarted.WaitOne();
var emittedData = false;
ConsumeBridge(feed, TimeSpan.FromSeconds(10), true, ts =>
{
if (ts.Slice.Count != 0)
{
emittedData = true;
Console.WriteLine("HasData: " + ts.Slice.Bars[Symbols.EURUSD].EndTime);
Console.WriteLine();
}
});
Assert.IsTrue(emittedData);
}
public void ReadsZipEntryNames()
{
var time = new DateTime(2016, 03, 03, 12, 48, 15);
var source = Path.Combine("TestData", "20151224_quote_american.zip");
var config = new SubscriptionDataConfig(typeof (ZipEntryName), Symbol.Create("XLRE", SecurityType.Option, Market.USA), Resolution.Tick,
TimeZones.NewYork, TimeZones.NewYork, false, false, false);
var dataFileProvider = new DefaultDataFileProvider();
var factory = new ZipEntryNameSubscriptionDataSourceReader(dataFileProvider, config, time, false);
var expected = new[]
{
Symbol.CreateOption("XLRE", Market.USA, OptionStyle.American, OptionRight.Call, 21m, new DateTime(2016, 08, 19)),
Symbol.CreateOption("XLRE", Market.USA, OptionStyle.American, OptionRight.Call, 22m, new DateTime(2016, 08, 19)),
Symbol.CreateOption("XLRE", Market.USA, OptionStyle.American, OptionRight.Put, 37m, new DateTime(2016, 08, 19)),
};
var actual = factory.Read(new SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.ZipEntryName)).ToList();
// we only really care about the symbols
CollectionAssert.AreEqual(expected, actual.Select(x => x.Symbol));
Assert.IsTrue(actual.All(x => x is ZipEntryName));
}
public void ReadsFineFundamental(FineFundamentalTestParameters parameters)
{
var stopwatch = Stopwatch.StartNew();
var rows = new List<FineFundamental>();
var config = new SubscriptionDataConfig(typeof(FineFundamental), parameters.Symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, TickType.Trade, false);
var security = new Security(SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork), config, new Cash(CashBook.AccountCurrency, 0, 1), SymbolProperties.GetDefault(CashBook.AccountCurrency));
var request = new SubscriptionRequest(false, null, security, config, parameters.StartDate, parameters.EndDate);
var fileProvider = new DefaultDataFileProvider();
var factory = new FineFundamentalSubscriptionEnumeratorFactory();
var enumerator = factory.CreateEnumerator(request, fileProvider);
while (enumerator.MoveNext())
{
var current = enumerator.Current as FineFundamental;
rows.Add(current);
}
stopwatch.Stop();
Console.WriteLine("Total rows: {0}, elapsed time: {1}", rows.Count, stopwatch.Elapsed);
Assert.AreEqual(parameters.RowCount, rows.Count);
if (parameters.RowCount != 1) return;
var row = rows[0];
Assert.AreEqual(parameters.CompanyShortName, row.CompanyReference.ShortName);
Assert.AreEqual(parameters.Symbol, row.Symbol);
Assert.AreEqual(parameters.Symbol != Symbol.Empty ? parameters.Symbol.Value : null, row.CompanyReference.PrimarySymbol);
Assert.AreEqual(parameters.Symbol != Symbol.Empty ? parameters.Symbol.Value : null, row.SecurityReference.SecuritySymbol);
Assert.AreEqual(parameters.Ebitda3M, row.FinancialStatements.IncomeStatement.EBITDA.ThreeMonths);
Assert.AreEqual(parameters.Ebitda12M, row.FinancialStatements.IncomeStatement.EBITDA.TwelveMonths);
Assert.AreEqual(parameters.Ebitda12M, row.FinancialStatements.IncomeStatement.EBITDA);
Assert.AreEqual(parameters.CostOfRevenue3M, row.FinancialStatements.IncomeStatement.CostOfRevenue.ThreeMonths);
Assert.AreEqual(parameters.CostOfRevenue12M, row.FinancialStatements.IncomeStatement.CostOfRevenue.TwelveMonths);
Assert.AreEqual(parameters.CostOfRevenue12M, row.FinancialStatements.IncomeStatement.CostOfRevenue);
Assert.AreEqual(parameters.EquityPerShareGrowth1Y, row.EarningRatios.EquityPerShareGrowth.OneYear);
Assert.AreEqual(parameters.EquityPerShareGrowth1Y, row.EarningRatios.EquityPerShareGrowth);
Assert.AreEqual(parameters.PeRatio, row.ValuationRatios.PERatio);
}
public void TestsFileSystemDataFeedSpeed()
{
var job = new BacktestNodePacket();
var resultHandler = new BacktestingResultHandler();
var mapFileProvider = new LocalDiskMapFileProvider();
var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
var dataFileProvider = new DefaultDataFileProvider();
var algorithm = new BenchmarkTest();
var feed = new FileSystemDataFeed();
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataFileProvider);
algorithm.Initialize();
var feedThreadStarted = new ManualResetEvent(false);
Task.Factory.StartNew(() =>
{
feedThreadStarted.Set();
feed.Run();
});
feedThreadStarted.WaitOne();
var stopwatch = Stopwatch.StartNew();
var lastMonth = -1;
var count = 0;
foreach (var timeSlice in feed)
{
if (timeSlice.Time.Month != lastMonth)
{
Console.WriteLine(DateTime.Now + " - Time: " + timeSlice.Time);
lastMonth = timeSlice.Time.Month;
}
count++;
}
Console.WriteLine("Count: " + count);
stopwatch.Stop();
Console.WriteLine("Elapsed time: " + stopwatch.Elapsed);
}
private IDataFeed RunDataFeed(IAlgorithm algorithm, out FuncDataQueueHandler dataQueueHandler, ITimeProvider timeProvider = null, Func<FuncDataQueueHandler, IEnumerable<BaseData>> getNextTicksFunction = null)
{
getNextTicksFunction = getNextTicksFunction ?? (fdqh => fdqh.Subscriptions.Select(symbol => new Tick(DateTime.Now, symbol, 1, 2){Quantity = 1}));
// job is used to send into DataQueueHandler
var job = new LiveNodePacket();
// result handler is used due to dependency in SubscriptionDataReader
var resultHandler = new BacktestingResultHandler(); // new ResultHandlerStub();
dataQueueHandler = new FuncDataQueueHandler(getNextTicksFunction);
var feed = new TestableLiveTradingDataFeed(dataQueueHandler, timeProvider);
var mapFileProvider = new LocalDiskMapFileProvider();
var fileProvider = new DefaultDataFileProvider();
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), fileProvider);
var feedThreadStarted = new ManualResetEvent(false);
Task.Factory.StartNew(() =>
{
feedThreadStarted.Set();
feed.Run();
});
// wait for feed.Run to actually begin
feedThreadStarted.WaitOne();
return feed;
}
/// <summary>
/// Applies universe selection the the data feed and algorithm
/// </summary>
/// <param name="universe">The universe to perform selection on</param>
/// <param name="dateTimeUtc">The current date time in utc</param>
/// <param name="universeData">The data provided to perform selection with</param>
public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
{
IEnumerable <Symbol> selectSymbolsResult;
// check if this universe must be filtered with fine fundamental data
var fineFiltered = universe as FineFundamentalFilteredUniverse;
if (fineFiltered != null)
{
// perform initial filtering and limit the result
selectSymbolsResult = universe.SelectSymbols(dateTimeUtc, universeData);
// prepare a BaseDataCollection of FineFundamental instances
var fineCollection = new BaseDataCollection();
var dataFileProvider = new DefaultDataFileProvider();
foreach (var symbol in selectSymbolsResult)
{
var factory = new FineFundamentalSubscriptionEnumeratorFactory(_algorithm.LiveMode, x => new[] { dateTimeUtc });
var config = FineFundamentalUniverse.CreateConfiguration(symbol);
Security security;
if (!_algorithm.Securities.TryGetValue(symbol, out security))
{
security = universe.CreateSecurity(symbol, _algorithm, _marketHoursDatabase, _symbolPropertiesDatabase);
_algorithm.Securities.Add(security);
}
var request = new SubscriptionRequest(true, universe, security, config, dateTimeUtc, dateTimeUtc);
var enumerator = factory.CreateEnumerator(request, dataFileProvider);
if (enumerator.MoveNext())
{
fineCollection.Data.Add(enumerator.Current);
}
}
// perform the fine fundamental universe selection
selectSymbolsResult = fineFiltered.FineFundamentalUniverse.PerformSelection(dateTimeUtc, fineCollection);
}
else
{
// perform initial filtering and limit the result
selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData);
}
// check for no changes first
if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
return(SecurityChanges.None);
}
// materialize the enumerable into a set for processing
var selections = selectSymbolsResult.ToHashSet();
var additions = new List <Security>();
var removals = new List <Security>();
var algorithmEndDateUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
// remove previously deselected members which were kept in the universe because of holdings or open orders
foreach (var member in _pendingRemovals.ToList())
{
var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == member.Symbol);
if (!member.HoldStock && !openOrders.Any())
{
RemoveSecurityFromUniverse(universe, member, removals, dateTimeUtc, algorithmEndDateUtc);
_pendingRemovals.Remove(member);
}
}
// determine which data subscriptions need to be removed from this universe
foreach (var member in universe.Members.Values)
{
// if we've selected this subscription again, keep it
if (selections.Contains(member.Symbol))
{
continue;
}
// don't remove if the universe wants to keep him in
if (!universe.CanRemoveMember(dateTimeUtc, member))
{
continue;
}
// remove the member - this marks this member as not being
// selected by the universe, but it may remain in the universe
// until open orders are closed and the security is liquidated
removals.Add(member);
// but don't physically remove it from the algorithm if we hold stock or have open orders against it
var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == member.Symbol);
if (!member.HoldStock && !openOrders.Any())
{
RemoveSecurityFromUniverse(universe, member, removals, dateTimeUtc, algorithmEndDateUtc);
}
else
{
_pendingRemovals.Add(member);
}
}
// find new selections and add them to the algorithm
foreach (var symbol in selections)
{
// create the new security, the algorithm thread will add this at the appropriate time
Security security;
if (!_algorithm.Securities.TryGetValue(symbol, out security))
{
security = universe.CreateSecurity(symbol, _algorithm, _marketHoursDatabase, _symbolPropertiesDatabase);
}
var addedSubscription = false;
foreach (var request in universe.GetSubscriptionRequests(security, dateTimeUtc, algorithmEndDateUtc))
{
// ask the limiter if we can add another subscription at that resolution
string reason;
if (!_limiter.CanAddSubscription(request.Configuration.Resolution, out reason))
{
// should we be counting universe subscriptions against user subscriptions limits?
_algorithm.Error(reason);
Log.Trace("UniverseSelection.ApplyUniverseSelection(): Skipping adding subscription: " + request.Configuration.Symbol.ToString() + ": " + reason);
continue;
}
// add the new subscriptions to the data feed
_dataFeed.AddSubscription(request);
// only update our security changes if we actually added data
if (!request.IsUniverseSubscription)
{
addedSubscription = true;
}
}
if (addedSubscription)
{
var addedMember = universe.AddMember(dateTimeUtc, security);
if (addedMember)
{
additions.Add(security);
}
}
}
// Add currency data feeds that weren't explicitly added in Initialize
if (additions.Count > 0)
{
var addedSecurities = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(_algorithm.Securities, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, _algorithm.BrokerageModel.DefaultMarkets);
foreach (var security in addedSecurities)
{
// assume currency feeds are always one subscription per, these are typically quote subscriptions
_dataFeed.AddSubscription(new SubscriptionRequest(false, universe, security, security.Subscriptions.First(), dateTimeUtc, algorithmEndDateUtc));
}
}
// return None if there's no changes, otherwise return what we've modified
var securityChanges = additions.Count + removals.Count != 0
? new SecurityChanges(additions, removals)
: SecurityChanges.None;
if (securityChanges != SecurityChanges.None)
{
Log.Debug("UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc + ": " + securityChanges);
}
return(securityChanges);
}
