/// <summary> /// Initializes this instance /// </summary> /// <param name="config">The <see cref="SubscriptionDataConfig"/></param> /// <param name="factorFile">The factor file to use</param> /// <param name="mapFile">The <see cref="MapFile"/> to use</param> /// <param name="startTime">Start date for the data request</param> public override void Initialize( SubscriptionDataConfig config, FactorFile factorFile, MapFile mapFile, DateTime startTime) { base.Initialize(config, factorFile, mapFile, startTime); _delistingEnumerator = _dataQueueHandler.Subscribe(_dataConfig, (sender, args) => { if (_delistingEnumerator != null && _delistingEnumerator.MoveNext()) { // live trading always returns true but could be null if (_delistingEnumerator.Current != null) { var delisting = _delistingEnumerator.Current as Delisting; if (delisting != null) { // we set the delisting date! DelistingDate = new ReferenceWrapper <DateTime>(delisting.Time); } else { Log.Error($"LiveDataBasedDelistingEventProvider(): Current is not a {nameof(Delisting)} event: {_delistingEnumerator.Current?.GetType()}"); } } } else { Log.Error("LiveDataBasedDelistingEventProvider(): new data available triggered with no data"); } }); }
public override void Initialize() { SetStartDate(2014, 3, 25); //Set Start Date SetEndDate(2014, 4, 7); //Set End Date SetCash(100000); //Set Strategy Cash // Set our DataNormalizationMode to raw UniverseSettings.DataNormalizationMode = DataNormalizationMode.Raw; _googl = AddEquity(Ticker, Resolution.Daily).Symbol; // Get our factor file for this regression var dataProvider = Composer.Instance.GetExportedValueByTypeName <IDataProvider>(Config.Get("data-provider", "DefaultDataProvider")); var mapFileProvider = new LocalDiskMapFileProvider(); mapFileProvider.Initialize(dataProvider); var factorFileProvider = new LocalDiskFactorFileProvider(); factorFileProvider.Initialize(mapFileProvider, dataProvider); _factorFile = factorFileProvider.Get(_googl); // Prime our expected values _expectedRawPrices.MoveNext(); }
// This unit tests reproduces GH 3885 where the consumer hanged forever public void ConsumerDoesNotHang() { for (var i = 0; i < 10000; i++) { var dataPoints = 10; var enumerator = new TestDataEnumerator { MoveNextTrueCount = dataPoints }; var factorFileProfider = new Mock <IFactorFileProvider>(); factorFileProfider.Setup(s => s.Get(It.IsAny <Symbol>())).Returns(FactorFile.Read(_security.Symbol.Value, _config.Market)); var subscription = SubscriptionUtils.CreateAndScheduleWorker( new SubscriptionRequest( false, null, _security, _config, DateTime.UtcNow, Time.EndOfTime ), enumerator, factorFileProfider.Object, false); for (var j = 0; j < dataPoints; j++) { Assert.IsTrue(subscription.MoveNext()); } Assert.IsFalse(subscription.MoveNext()); subscription.DisposeSafely(); } }
private static FactorFile GetFactorFileToUse( SubscriptionDataConfig config, IFactorFileProvider factorFileProvider) { var factorFileToUse = new FactorFile(config.Symbol.Value, new List <FactorFileRow>()); if (!config.IsCustomData && config.SecurityType == SecurityType.Equity) { try { var factorFile = factorFileProvider.Get(config.Symbol); if (factorFile != null) { factorFileToUse = factorFile; } } catch (Exception err) { Log.Error(err, "CorporateEventEnumeratorFactory.GetFactorFileToUse(): Factors File: " + config.Symbol.ID + ": "); } } return(factorFileToUse); }
public void ReadsOldFactorFileFormat() { var lines = new[] { "19980102,1.0000000,0.5", "20130828,1.0000000,0.5", "20501231,1.0000000,1" }; var factorFile = FactorFile.Parse("bno", lines); var firstRow = factorFile.SortedFactorFileData[new DateTime(1998, 01, 02)]; Assert.AreEqual(1m, firstRow.PriceFactor); Assert.AreEqual(0.5m, firstRow.SplitFactor); Assert.AreEqual(0m, firstRow.ReferencePrice); var secondRow = factorFile.SortedFactorFileData[new DateTime(2013, 08, 28)]; Assert.AreEqual(1m, secondRow.PriceFactor); Assert.AreEqual(0.5m, secondRow.SplitFactor); Assert.AreEqual(0m, firstRow.ReferencePrice); var thirdRow = factorFile.SortedFactorFileData[Time.EndOfTime]; Assert.AreEqual(1m, thirdRow.PriceFactor); Assert.AreEqual(1m, thirdRow.SplitFactor); Assert.AreEqual(0m, firstRow.ReferencePrice); }
/// <summary> /// Creates a new instance /// </summary> /// <param name="config">The <see cref="SubscriptionDataConfig"/></param> /// <param name="factorFile">The factor file to use</param> /// <param name="mapFile">The <see cref="MapFile"/> to use</param> /// <param name="tradableDateEventProviders">The tradable dates event providers</param> /// <param name="tradableDayNotifier">Tradable dates provider</param> /// <param name="includeAuxiliaryData">True to emit auxiliary data</param> public AuxiliaryDataEnumerator( SubscriptionDataConfig config, FactorFile factorFile, MapFile mapFile, ITradableDateEventProvider [] tradableDateEventProviders, ITradableDatesNotifier tradableDayNotifier, bool includeAuxiliaryData) { _auxiliaryData = new Queue <BaseData>(); foreach (var tradableDateEventProvider in tradableDateEventProviders) { tradableDateEventProvider.Initialize( config, factorFile, mapFile); } tradableDayNotifier.NewTradableDate += (sender, eventArgs) => { foreach (var tradableDateEventProvider in tradableDateEventProviders) { // Call implementation var newEvents = tradableDateEventProvider.GetEvents(eventArgs); if (includeAuxiliaryData) { foreach (var newEvent in newEvents) { _auxiliaryData.Enqueue(newEvent); } } } }; }
/// <summary> /// Initializes this instance /// </summary> /// <param name="config">The <see cref="SubscriptionDataConfig"/></param> /// <param name="factorFile">The factor file to use</param> /// <param name="mapFile">The <see cref="MapFile"/> to use</param> public void Initialize( SubscriptionDataConfig config, FactorFile factorFile, MapFile mapFile) { _mapFile = mapFile; _config = config; }
public void ReadsFactorFileWithoutInfValues() { var factorFile = FactorFile.Read("AAPL", "usa"); Assert.AreEqual(19, factorFile.SortedFactorFileData.Count); Assert.IsNull(factorFile.FactorFileMinimumDate); }
public void ReadsFactorFileWithoutInfValues() { var factorFile = FactorFile.Read("AAPL", "usa"); Assert.AreEqual(29, factorFile.SortedFactorFileData.Count); Assert.AreEqual(new DateTime(1998, 01, 01), factorFile.FactorFileMinimumDate.Value); }
public void EmptyFactorFileReturnsEmptyListForSplitsAndDividends(string contents) { var lines = contents.Split('\n').Where(l => !string.IsNullOrWhiteSpace(l)); var factorFile = FactorFile.Parse("bno", lines); Assert.IsEmpty(factorFile.GetSplitsAndDividends(Symbols.SPY, SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork))); }
/// <summary> /// Initializes this instance /// </summary> /// <param name="config">The <see cref="SubscriptionDataConfig"/></param> /// <param name="factorFile">The factor file to use</param> /// <param name="mapFile">The <see cref="MapFile"/> to use</param> /// <param name="startTime">Start date for the data request</param> public void Initialize( SubscriptionDataConfig config, FactorFile factorFile, MapFile mapFile, DateTime startTime) { _config = config; DelistingDate = new ReferenceWrapper <DateTime>(config.Symbol.GetDelistingDate(mapFile)); }
/// <summary> /// Initializes this instance /// </summary> /// <param name="config">The <see cref="SubscriptionDataConfig"/></param> /// <param name="factorFile">The factor file to use</param> /// <param name="mapFile">The <see cref="MapFile"/> to use</param> /// <param name="startTime">Start date for the data request</param> public void Initialize( SubscriptionDataConfig config, FactorFile factorFile, MapFile mapFile, DateTime startTime) { _config = config; _delistingDate = config.Symbol.GetDelistingDate(mapFile); }
public void ReadsFactorFileWithInfValues() { var factorFile = FactorFile.Read("GBSN", "usa"); Assert.AreEqual(3, factorFile.SortedFactorFileData.Count); Assert.IsNotNull(factorFile.FactorFileMinimumDate); Assert.AreEqual(new DateTime(2016, 3, 31).Ticks, factorFile.FactorFileMinimumDate.Value.Ticks); }
/// <summary> /// Creates a new instance of the <see cref="PriceScaleFactorEnumerator"/>. /// </summary> /// <param name="rawDataEnumerator">The underlying raw data enumerator</param> /// <param name="config">The <see cref="SubscriptionDataConfig"/> to enumerate for. /// Will determine the <see cref="DataNormalizationMode"/> to use.</param> /// <param name="factorFile">The <see cref="FactorFile"/> instance to use</param> public PriceScaleFactorEnumerator( IEnumerator <BaseData> rawDataEnumerator, SubscriptionDataConfig config, FactorFile factorFile) { _lastTradableDate = DateTime.MinValue; _config = config; _rawDataEnumerator = rawDataEnumerator; _factorFile = factorFile; }
public void ResolvesCorrectMostRecentFactorChangeDate() { var lines = new[] { "19980102,1.0000000,0.5", "20130828,1.0000000,0.5", "20501231,1.0000000,1" }; var factorFile = FactorFile.Parse("bno", lines); Assert.AreEqual(new DateTime(2013, 08, 28), factorFile.MostRecentFactorChange); }
public void PriceScaleDoesNotUpdateForFillForwardBar() { var referenceTime = new DateTime(2020, 08, 06); var point = new Tick(referenceTime, Symbols.SPY, 1, 2); var point2 = point.Clone(true); point2.Time = referenceTime.AddDays(1); var point3 = point.Clone(false); point3.Time = referenceTime.AddDays(2); ; var enumerator = new List <BaseData> { point, point2, point3 }.GetEnumerator(); var factorFileProfider = new Mock <IFactorFileProvider>(); var factorFile = new FactorFile(_security.Symbol.Value, new[] { new FactorFileRow(referenceTime, 0.5m, 1), new FactorFileRow(referenceTime.AddDays(1), 1m, 1) }, referenceTime); factorFileProfider.Setup(s => s.Get(It.IsAny <Symbol>())).Returns(factorFile); var subscription = SubscriptionUtils.CreateAndScheduleWorker( new SubscriptionRequest( false, null, _security, _config, referenceTime, Time.EndOfTime ), enumerator, factorFileProfider.Object, true); Assert.IsTrue(subscription.MoveNext()); Assert.AreEqual(1, (subscription.Current.Data as Tick).AskPrice); Assert.IsFalse((subscription.Current.Data as Tick).IsFillForward); Assert.IsTrue(subscription.MoveNext()); Assert.AreEqual(1, (subscription.Current.Data as Tick).AskPrice); Assert.IsTrue((subscription.Current.Data as Tick).IsFillForward); Assert.IsTrue(subscription.MoveNext()); Assert.AreEqual(2, (subscription.Current.Data as Tick).AskPrice); Assert.IsFalse((subscription.Current.Data as Tick).IsFillForward); subscription.DisposeSafely(); }
public void Setup() { _config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.SPY, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, true, true, false); _factorFile = TestGlobals.FactorFileProvider.Get(_config.Symbol); _rawDataEnumerator = new RawDataEnumerator(); }
/// <summary> /// Initializes this instance /// </summary> /// <param name="config">The <see cref="SubscriptionDataConfig"/></param> /// <param name="factorFile">The factor file to use</param> /// <param name="mapFile">The <see cref="MapFile"/> to use</param> /// <param name="startTime">Start date for the data request</param> public void Initialize( SubscriptionDataConfig config, FactorFile factorFile, MapFile mapFile, DateTime startTime) { _mapFile = mapFile; _config = config; if (_mapFile.HasData(startTime.Date)) { // initialize mapped symbol using request start date _config.MappedSymbol = _mapFile.GetMappedSymbol(startTime.Date, _config.MappedSymbol); } }
private static FactorFile GetTestFactorFile(string symbol, DateTime reference) { var file = new FactorFile(symbol, new List <FactorFileRow> { new FactorFileRow(reference, 1, 1), new FactorFileRow(reference.AddDays(-7), .9m, 1), // dividend new FactorFileRow(reference.AddDays(-14), .8m, 1), // dividend new FactorFileRow(reference.AddDays(-21), .8m, .5m), // split new FactorFileRow(reference.AddDays(-90), .8m, .25m), // split new FactorFileRow(reference.AddDays(-365), .7m, .125m) // split+dividend }); return(file); }
public void Setup() { _config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.SPY, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, true, true, false); _factorFile = FactorFile.Read( _config.Symbol.Value, _config.Symbol.ID.Market); _rawDataEnumerator = new RawDataEnumerator(); }
public void SubscriptionEmitsAuxData(Type typeOfConfig, bool shouldReceiveAuxData) { var factorFileProvider = new Mock <IFactorFileProvider>(); var config = new SubscriptionDataConfig(typeOfConfig, _security.Symbol, Resolution.Hour, TimeZones.NewYork, TimeZones.NewYork, true, true, false); factorFileProvider.Setup(s => s.Get(It.IsAny <Symbol>())).Returns(FactorFile.Read(_security.Symbol.Value, config.Market)); var totalPoints = 8; var time = new DateTime(2010, 1, 1); var enumerator = Enumerable.Range(0, totalPoints).Select(x => new Delisting { Time = time.AddHours(x) }).GetEnumerator(); var subscription = SubscriptionUtils.CreateAndScheduleWorker( new SubscriptionRequest( false, null, _security, config, DateTime.UtcNow, Time.EndOfTime ), enumerator, factorFileProvider.Object, false); // Test our subscription stream to see if it emits the aux data it should be filtered // by the SubscriptionUtils produce function if the config isn't for a TradeBar int dataReceivedCount = 0; while (subscription.MoveNext()) { dataReceivedCount++; if (subscription.Current != null && subscription.Current.Data.DataType == MarketDataType.Auxiliary) { Assert.IsTrue(shouldReceiveAuxData); } } // If it should receive aux data it should have emitted all points // otherwise none should have been emitted if (shouldReceiveAuxData) { Assert.AreEqual(totalPoints, dataReceivedCount); } else { Assert.AreEqual(0, dataReceivedCount); } }
public void FactorFiles_CanBeGenerated_Accurately() { // Arrange var yahooEvents = _yahooDataDownloader.DownloadSplitAndDividendData(_symbol, Parse.DateTime("01/01/1970"), DateTime.MaxValue); var filePath = LeanData.GenerateRelativeFactorFilePath(_symbol); var tolerance = 0.00001m; if (!File.Exists(filePath)) { throw new ArgumentException("This test requires an already calculated factor file." + "Try using one of the pre-existing factor files "); } var originalFactorFileInstance = FactorFile.Read(PermTick, Market); // we limit events to the penultimate time in our factor file (last one is 2050) var lastValidRow = originalFactorFileInstance.SortedFactorFileData.Reverse().Skip(1).First(); // Act var newFactorFileInstance = _factorFileGenerator.CreateFactorFile(yahooEvents.Where(data => data.Time.AddDays(-1) <= lastValidRow.Key).ToList()); var earliestDate = originalFactorFileInstance.SortedFactorFileData.First().Key; var latestDate = originalFactorFileInstance.SortedFactorFileData.Last().Key; // Assert Assert.AreEqual(originalFactorFileInstance.SortedFactorFileData.Count, newFactorFileInstance.SortedFactorFileData.Count); for (var i = earliestDate; i < latestDate; i = i.AddDays(1)) { FactorFileRow expected = null; FactorFileRow actual = null; originalFactorFileInstance.SortedFactorFileData.TryGetValue(i, out expected); newFactorFileInstance.SortedFactorFileData.TryGetValue(i, out actual); if (expected == null || actual == null) { Assert.IsTrue(actual == null); Assert.IsTrue(expected == null); } else { Assert.IsTrue(Math.Abs(expected.PriceFactor - actual.PriceFactor) < tolerance); Assert.IsTrue(Math.Abs(expected.SplitFactor - actual.SplitFactor) < tolerance); } } }
public void ThrowingEnumeratorStackDisposesOfSubscription() { var enumerator = new TestDataEnumerator { MoveNextTrueCount = 10, ThrowException = true }; var factorFileProfider = new Mock <IFactorFileProvider>(); factorFileProfider.Setup(s => s.Get(It.IsAny <Symbol>())).Returns(FactorFile.Read(_security.Symbol.Value, _config.Market)); var subscription = SubscriptionUtils.CreateAndScheduleWorker( new SubscriptionRequest( false, null, _security, _config, DateTime.UtcNow, Time.EndOfTime ), enumerator, factorFileProfider.Object, false); var count = 0; while (enumerator.MoveNextTrueCount != 9) { if (count++ > 100) { Assert.Fail("Timeout waiting for producer"); } Thread.Sleep(1); } Assert.IsFalse(subscription.MoveNext()); Assert.IsTrue(subscription.EndOfStream); // enumerator is disposed by the producer count = 0; while (!enumerator.Disposed) { if (count++ > 100) { Assert.Fail("Timeout waiting for producer"); } Thread.Sleep(1); } }
private static decimal GetScaleFactor(FactorFile factorFile, DataNormalizationMode mode, DateTime date) { switch (mode) { case DataNormalizationMode.Raw: return(1); case DataNormalizationMode.TotalReturn: case DataNormalizationMode.SplitAdjusted: return(factorFile.GetSplitFactor(date)); case DataNormalizationMode.Adjusted: return(factorFile.GetPriceScaleFactor(date)); default: throw new ArgumentOutOfRangeException(); } }
public override void Initialize() { SetStartDate(2014, 6, 5); //Set Start Date SetEndDate(2014, 6, 5); //Set End Date UniverseSettings.DataNormalizationMode = DataNormalizationMode.SplitAdjusted; _aapl = AddEquity(Ticker, Resolution.Minute).Symbol; var dataProvider = Composer.Instance.GetExportedValueByTypeName <IDataProvider>(Config.Get("data-provider", "DefaultDataProvider")); var mapFileProvider = new LocalDiskMapFileProvider(); mapFileProvider.Initialize(dataProvider); var factorFileProvider = new LocalDiskFactorFileProvider(); factorFileProvider.Initialize(mapFileProvider, dataProvider); _factorFile = factorFileProvider.Get(_aapl); }
/// <summary> /// Initializes this instance /// </summary> /// <param name="config">The <see cref="SubscriptionDataConfig"/></param> /// <param name="factorFile">The factor file to use</param> /// <param name="mapFile">The <see cref="MapFile"/> to use</param> public void Initialize( SubscriptionDataConfig config, FactorFile factorFile, MapFile mapFile) { _config = config; // Estimate delisting date. switch (config.Symbol.ID.SecurityType) { case SecurityType.Future: _delistingDate = config.Symbol.ID.Date; break; case SecurityType.Option: _delistingDate = OptionSymbol.GetLastDayOfTrading( config.Symbol); break; default: _delistingDate = mapFile.DelistingDate; break; } }
public void ThrowsWhenEmptyReferencePrice() { var dividendProvider = new DividendEventProvider(); var config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.AAPL, Resolution.Second, TimeZones.NewYork, TimeZones.NewYork, false, false, false); var start = new DateTime(1998, 01, 02); var row1 = new DateTime(2000, 01, 02); var row2 = new DateTime(2001, 01, 02); var row3 = new DateTime(2002, 01, 02); var factorFile = new FactorFile("AAPL", new [] { new FactorFileRow(row1, 0.693m, 1), new FactorFileRow(row2, 0.77m, 1), new FactorFileRow(row3, 0.85555m, 1) }, start); var mapFile = new LocalDiskMapFileProvider(); dividendProvider.Initialize(config, factorFile, mapFile.Get(Market.USA).ResolveMapFile(Symbols.AAPL, start), start); foreach (var row in factorFile.Take(1)) { var lastRawPrice = 100; var events = dividendProvider .GetEvents(new NewTradableDateEventArgs(row.Date, null, Symbols.AAPL, lastRawPrice)) .ToList(); // ex dividend date does not emit anything Assert.AreEqual(0, events.Count); Assert.Throws <InvalidOperationException>(() => { dividendProvider .GetEvents(new NewTradableDateEventArgs(row.Date.AddDays(1), null, Symbols.AAPL, lastRawPrice)) .ToList(); }); } }
public void SubscriptionIsDisposed() { var dataPoints = 10; var enumerator = new TestDataEnumerator { MoveNextTrueCount = dataPoints }; var factorFileProfider = new Mock <IFactorFileProvider>(); factorFileProfider.Setup(s => s.Get(It.IsAny <Symbol>())).Returns(FactorFile.Read(_security.Symbol.Value, _config.Market)); var subscription = SubscriptionUtils.CreateAndScheduleWorker( new SubscriptionRequest( false, null, _security, _config, DateTime.UtcNow, Time.EndOfTime ), enumerator, factorFileProfider.Object, false); var count = 0; while (enumerator.MoveNextTrueCount > 8) { if (count++ > 100) { Assert.Fail($"Timeout waiting for producer. {enumerator.MoveNextTrueCount}"); } Thread.Sleep(1); } subscription.DisposeSafely(); Assert.IsFalse(subscription.MoveNext()); }
public void AppliesSplitAndDividendAtSameTime() { var reference = new DateTime(2018, 08, 01); var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(QuantConnect.Market.USA, Symbols.SPY, SecurityType.Equity); var expected = GetTestFactorFile("AAPL", reference); // remove the last entry that contains a split and dividend at the same time var factorFile = new FactorFile("AAPL", expected.SortedFactorFileData.Where(kvp => kvp.Value.PriceFactor >= .8m).Select(kvp => kvp.Value)); var actual = factorFile.Apply(new List <BaseData> { new Split(Symbols.SPY, reference.AddDays(-364), 100m, 1 / 2m, SplitType.SplitOccurred), new Dividend(Symbols.SPY, reference.AddDays(-364), 12.5m, 100m) }, exchangeHours); foreach (var item in actual.Reverse().Zip(expected.Reverse(), (a, e) => new { actual = a, expected = e })) { Console.WriteLine($"expected: {item.expected} actual: {item.actual} diff: {100 * (1 - item.actual.PriceFactor / item.expected.PriceFactor):0.0000}%"); Assert.AreEqual(item.expected.Date, item.actual.Date); Assert.AreEqual(item.expected.ReferencePrice, item.actual.ReferencePrice); Assert.AreEqual(item.expected.SplitFactor, item.actual.SplitFactor); Assert.AreEqual(item.expected.PriceFactor.RoundToSignificantDigits(4), item.actual.PriceFactor.RoundToSignificantDigits(4)); } }
/// <summary> /// Creates a new instance /// </summary> /// <param name="config">The <see cref="SubscriptionDataConfig"/></param> /// <param name="factorFile">The factor file to use</param> /// <param name="mapFile">The <see cref="MapFile"/> to use</param> /// <param name="tradableDateEventProviders">The tradable dates event providers</param> /// <param name="tradableDayNotifier">Tradable dates provider</param> /// <param name="includeAuxiliaryData">True to emit auxiliary data</param> public AuxiliaryDataEnumerator( SubscriptionDataConfig config, FactorFile factorFile, MapFile mapFile, ITradableDateEventProvider [] tradableDateEventProviders, ITradableDatesNotifier tradableDayNotifier, bool includeAuxiliaryData) { _auxiliaryData = new Queue <BaseData>(); foreach (var tradableDateEventProvider in tradableDateEventProviders) { tradableDateEventProvider.Initialize( config, factorFile, mapFile); } tradableDayNotifier.NewTradableDate += (sender, eventArgs) => { foreach (var tradableDateEventProvider in tradableDateEventProviders) { // Call implementation // and materialize list since we need symbol changes applied to the config // regardless of the includeAuxiliaryData argument var newEvents = tradableDateEventProvider.GetEvents(eventArgs).ToList(); if (includeAuxiliaryData) { foreach (var newEvent in newEvents) { _auxiliaryData.Enqueue(newEvent); } } } }; }