public void AppliesSplitAndDividendAtSameTime()
{
var reference = new DateTime(2018, 08, 01);
var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(QuantConnect.Market.USA, Symbols.SPY, SecurityType.Equity);
var expected = GetTestFactorFile("AAPL", reference);
// remove the last entry that contains a split and dividend at the same time
var factorFile = new CorporateFactorProvider("AAPL", expected.SortedFactorFileData.Where(kvp => kvp.Value.Single().PriceFactor >= .8m).Select(kvp => kvp.Value.Single()));
var actual = factorFile.Apply(new List <BaseData>
{
new Split(Symbols.AAPL, reference.AddDays(-364), 100m, 1 / 2m, SplitType.SplitOccurred),
new Dividend(Symbols.AAPL, reference.AddDays(-364), 12.5m, 100m)
}, exchangeHours);
foreach (var item in actual.Reverse().Zip(expected.Reverse(), (a, e) => new { actual = a, expected = e }))
{
var expectedRow = (CorporateFactorRow)item.expected;
var actualRow = (CorporateFactorRow)item.actual;
Log.Trace($"expected: {item.expected} actual: {item.actual} diff: {100 * (1 - actualRow.PriceFactor / expectedRow.PriceFactor):0.0000}%");
Assert.AreEqual(item.expected.Date, item.actual.Date);
Assert.AreEqual(expectedRow.ReferencePrice, actualRow.ReferencePrice);
Assert.AreEqual(expectedRow.SplitFactor, actualRow.SplitFactor);
Assert.AreEqual(expectedRow.PriceFactor.RoundToSignificantDigits(4), actualRow.PriceFactor.RoundToSignificantDigits(4));
}
}
public override void Initialize()
{
SetStartDate(2014, 3, 25); //Set Start Date
SetEndDate(2014, 4, 7); //Set End Date
SetCash(100000); //Set Strategy Cash
// Set our DataNormalizationMode to raw
UniverseSettings.DataNormalizationMode = DataNormalizationMode.Raw;
_googl = AddEquity(Ticker, Resolution.Daily).Symbol;
// Get our factor file for this regression
var dataProvider =
Composer.Instance.GetExportedValueByTypeName <IDataProvider>(Config.Get("data-provider",
"DefaultDataProvider"));
var mapFileProvider = new LocalDiskMapFileProvider();
mapFileProvider.Initialize(dataProvider);
var factorFileProvider = new LocalDiskFactorFileProvider();
factorFileProvider.Initialize(mapFileProvider, dataProvider);
_factorFile = factorFileProvider.Get(_googl) as CorporateFactorProvider;
// Prime our expected values
_expectedRawPrices.MoveNext();
}
/// <summary>
/// Initializes this instance
/// </summary>
/// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
/// <param name="factorFileProvider">The factor file provider to use</param>
/// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param>
/// <param name="startTime">Start date for the data request</param>
public void Initialize(
SubscriptionDataConfig config,
IFactorFileProvider factorFileProvider,
IMapFileProvider mapFileProvider,
DateTime startTime)
{
_config = config;
_mapFile = mapFileProvider.ResolveMapFile(_config);
_factorFile = factorFileProvider.Get(_config.Symbol) as CorporateFactorProvider;
}
public void PriceScaleDoesNotUpdateForFillForwardBar()
{
var referenceTime = new DateTime(2020, 08, 06);
var point = new Tick(referenceTime, Symbols.SPY, 1, 2);
var point2 = point.Clone(true);
point2.Time = referenceTime.AddDays(1);
var point3 = point.Clone(false);
point3.Time = referenceTime.AddDays(2);
;
var enumerator = new List <BaseData> {
point, point2, point3
}.GetEnumerator();
var factorFileProfider = new Mock <IFactorFileProvider>();
var factorFile = new CorporateFactorProvider(_security.Symbol.Value, new[]
{
new CorporateFactorRow(referenceTime, 0.5m, 1),
new CorporateFactorRow(referenceTime.AddDays(1), 1m, 1)
}, referenceTime);
factorFileProfider.Setup(s => s.Get(It.IsAny <Symbol>())).Returns(factorFile);
var subscription = SubscriptionUtils.CreateAndScheduleWorker(
new SubscriptionRequest(
false,
null,
_security,
_config,
referenceTime,
Time.EndOfTime
),
enumerator,
factorFileProfider.Object,
true);
Assert.IsTrue(subscription.MoveNext());
Assert.AreEqual(1, (subscription.Current.Data as Tick).AskPrice);
Assert.IsFalse((subscription.Current.Data as Tick).IsFillForward);
Assert.IsTrue(subscription.MoveNext());
Assert.AreEqual(1, (subscription.Current.Data as Tick).AskPrice);
Assert.IsTrue((subscription.Current.Data as Tick).IsFillForward);
Assert.IsTrue(subscription.MoveNext());
Assert.AreEqual(2, (subscription.Current.Data as Tick).AskPrice);
Assert.IsFalse((subscription.Current.Data as Tick).IsFillForward);
subscription.DisposeSafely();
}
private static CorporateFactorProvider GetTestFactorFile(string symbol, DateTime reference)
{
var file = new CorporateFactorProvider(symbol, new List <CorporateFactorRow>
{
new CorporateFactorRow(reference, 1, 1),
new CorporateFactorRow(reference.AddDays(-7), .9m, 1, 100m), // dividend
new CorporateFactorRow(reference.AddDays(-14), .8m, 1, 100m), // dividend
new CorporateFactorRow(reference.AddDays(-21), .8m, .5m, 100m), // split
new CorporateFactorRow(reference.AddDays(-90), .8m, .25m, 100m), // split
new CorporateFactorRow(reference.AddDays(-365), .7m, .125m, 100m) // split+dividend
});
return(file);
}
public override void Initialize()
{
SetStartDate(2014, 6, 5); //Set Start Date
SetEndDate(2014, 6, 5); //Set End Date
UniverseSettings.DataNormalizationMode = DataNormalizationMode.SplitAdjusted;
_aapl = AddEquity(Ticker, Resolution.Minute).Symbol;
var dataProvider =
Composer.Instance.GetExportedValueByTypeName <IDataProvider>(Config.Get("data-provider",
"DefaultDataProvider"));
var mapFileProvider = new LocalDiskMapFileProvider();
mapFileProvider.Initialize(dataProvider);
var factorFileProvider = new LocalDiskFactorFileProvider();
factorFileProvider.Initialize(mapFileProvider, dataProvider);
_factorFile = factorFileProvider.Get(_aapl) as CorporateFactorProvider;
}
public static void GenerateRandomData(RandomDataGeneratorSettings settings, ConsoleLeveledOutput output)
{
// can specify a seed value in this ctor if determinism is desired
var random = new Random();
var randomValueGenerator = new RandomValueGenerator();
if (settings.RandomSeedSet)
{
random = new Random(settings.RandomSeed);
randomValueGenerator = new RandomValueGenerator(settings.RandomSeed);
}
var maxSymbolCount = randomValueGenerator.GetAvailableSymbolCount(settings.SecurityType, settings.Market);
if (settings.SymbolCount > maxSymbolCount)
{
output.Warn.WriteLine($"Limiting symbol count to {maxSymbolCount}, we don't have more {settings.SecurityType} tickers for {settings.Market}");
settings.SymbolCount = maxSymbolCount;
}
var symbolGenerator = new SymbolGenerator(settings, randomValueGenerator);
var tickGenerator = new TickGenerator(settings, randomValueGenerator);
output.Warn.WriteLine($"Begin data generation of {settings.SymbolCount} randomly generated {settings.SecurityType} assets...");
// iterate over our randomly generated symbols
var count = 0;
var progress = 0d;
var previousMonth = -1;
Func <Tick, DateTime> tickDay = (tick => new DateTime(tick.Time.Year, tick.Time.Month, tick.Time.Day));
Func <Data.BaseData, DateTime> dataDay = (data => new DateTime(data.Time.Year, data.Time.Month, data.Time.Day));
foreach (var currentSymbol in symbolGenerator.GenerateRandomSymbols())
{
// This is done so that we can update the symbol in the case of a rename event
var delistDate = GetDelistingDate(settings.Start, settings.End, randomValueGenerator);
var symbol = currentSymbol;
var willBeDelisted = randomValueGenerator.NextBool(1.0);
var monthsTrading = 0;
// Keep track of renamed symbols and the time they were renamed.
var renamedSymbols = new Dictionary <Symbol, DateTime>();
output.Warn.WriteLine($"\tSymbol[{++count}]: {symbol} Progress: {progress:0.0}% - Generating data...");
// define aggregators via settings
var aggregators = settings.CreateAggregators().ToList();
var tickHistory = tickGenerator.GenerateTicks(symbol).ToList();
// Companies rarely IPO then disappear within 6 months
if (willBeDelisted && tickHistory.Select(tick => tick.Time.Month).Distinct().Count() <= 6)
{
willBeDelisted = false;
}
var dividendsSplitsMaps = new DividendSplitMapGenerator(
symbol,
settings,
randomValueGenerator,
random,
delistDate,
willBeDelisted);
if (settings.SecurityType == SecurityType.Equity)
{
dividendsSplitsMaps.GenerateSplitsDividends(tickHistory);
if (!willBeDelisted)
{
dividendsSplitsMaps.DividendsSplits.Add(new CorporateFactorRow(new DateTime(2050, 12, 31), 1m, 1m));
if (dividendsSplitsMaps.MapRows.Count > 1)
{
// Remove the last element if we're going to have a 20501231 entry
dividendsSplitsMaps.MapRows.RemoveAt(dividendsSplitsMaps.MapRows.Count - 1);
}
dividendsSplitsMaps.MapRows.Add(new MapFileRow(new DateTime(2050, 12, 31), dividendsSplitsMaps.CurrentSymbol.Value));
}
// If the symbol value has changed, update the current symbol
if (symbol != dividendsSplitsMaps.CurrentSymbol)
{
// Add all symbol rename events to dictionary
// We skip the first row as it contains the listing event instead of a rename event
foreach (var renameEvent in dividendsSplitsMaps.MapRows.Skip(1))
{
// Symbol.UpdateMappedSymbol does not update the underlying security ID symbol, which
// is used to create the hash code. Create a new equity symbol from scratch instead.
symbol = Symbol.Create(renameEvent.MappedSymbol, SecurityType.Equity, settings.Market);
renamedSymbols.Add(symbol, renameEvent.Date);
output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} will be renamed on {renameEvent.Date}");
}
}
else
{
// This ensures that ticks will be written for the current symbol up until 9999-12-31
renamedSymbols.Add(symbol, new DateTime(9999, 12, 31));
}
symbol = dividendsSplitsMaps.CurrentSymbol;
// Write Splits and Dividend events to directory factor_files
var factorFile = new CorporateFactorProvider(symbol.Value, dividendsSplitsMaps.DividendsSplits, settings.Start);
var mapFile = new MapFile(symbol.Value, dividendsSplitsMaps.MapRows);
factorFile.WriteToFile(symbol);
mapFile.WriteToCsv(settings.Market, symbol.SecurityType);
output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} Dividends, splits, and map files have been written to disk.");
}
else
{
// This ensures that ticks will be written for the current symbol up until 9999-12-31
renamedSymbols.Add(symbol, new DateTime(9999, 12, 31));
}
Symbol previousSymbol = null;
var currentCount = 0;
foreach (var renamed in renamedSymbols)
{
var previousRenameDate = previousSymbol == null ? new DateTime(1, 1, 1) : renamedSymbols[previousSymbol];
var previousRenameDateDay = new DateTime(previousRenameDate.Year, previousRenameDate.Month, previousRenameDate.Day);
var renameDate = renamed.Value;
var renameDateDay = new DateTime(renameDate.Year, renameDate.Month, renameDate.Day);
foreach (var tick in tickHistory.Where(tick => tick.Time >= previousRenameDate && previousRenameDateDay != tickDay(tick)))
{
// Prevents the aggregator from being updated with ticks after the rename event
if (tickDay(tick) > renameDateDay)
{
break;
}
if (tick.Time.Month != previousMonth)
{
output.Info.WriteLine($"\tSymbol[{count}]: Month: {tick.Time:MMMM}");
previousMonth = tick.Time.Month;
monthsTrading++;
}
foreach (var item in aggregators)
{
tick.Value = tick.Value / dividendsSplitsMaps.FinalSplitFactor;
item.Consolidator.Update(tick);
}
if (monthsTrading >= 6 && willBeDelisted && tick.Time > delistDate)
{
output.Warn.WriteLine($"\tSymbol[{count}]: {renamed.Key} delisted at {tick.Time:MMMM yyyy}");
break;
}
}
// count each stage as a point, so total points is 2*symbol-count
// and the current progress is twice the current, but less one because we haven't finished writing data yet
progress = 100 * (2 * count - 1) / (2.0 * settings.SymbolCount);
output.Warn.WriteLine($"\tSymbol[{count}]: {renamed.Key} Progress: {progress:0.0}% - Saving data in LEAN format");
// persist consolidated data to disk
foreach (var item in aggregators)
{
var writer = new LeanDataWriter(item.Resolution, renamed.Key, Globals.DataFolder, item.TickType);
// send the flushed data into the writer. pulling the flushed list is very important,
// lest we likely wouldn't get the last piece of data stuck in the consolidator
// Filter out the data we're going to write here because filtering them in the consolidator update phase
// makes it write all dates for some unknown reason
writer.Write(item.Flush().Where(data => data.Time > previousRenameDate && previousRenameDateDay != dataDay(data)));
}
// update progress
progress = 100 * (2 * count) / (2.0 * settings.SymbolCount);
output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} Progress: {progress:0.0}% - Symbol data generation and output completed");
previousSymbol = renamed.Key;
currentCount++;
}
}
output.Info.WriteLine("Random data generation has completed.");
}
/// <summary>
/// Starts data generation
/// </summary>
public void Run()
{
var tickTypesPerSecurityType = SubscriptionManager.DefaultDataTypes();
// can specify a seed value in this ctor if determinism is desired
var random = new Random();
var randomValueGenerator = new RandomValueGenerator();
if (_settings.RandomSeedSet)
{
random = new Random(_settings.RandomSeed);
randomValueGenerator = new RandomValueGenerator(_settings.RandomSeed);
}
var symbolGenerator = BaseSymbolGenerator.Create(_settings, randomValueGenerator);
var maxSymbolCount = symbolGenerator.GetAvailableSymbolCount();
if (_settings.SymbolCount > maxSymbolCount)
{
Log.Error($"RandomDataGenerator.Run(): Limiting Symbol count to {maxSymbolCount}, we don't have more {_settings.SecurityType} tickers for {_settings.Market}");
_settings.SymbolCount = maxSymbolCount;
}
Log.Trace($"RandomDataGenerator.Run(): Begin data generation of {_settings.SymbolCount} randomly generated {_settings.SecurityType} assets...");
// iterate over our randomly generated symbols
var count = 0;
var progress = 0d;
var previousMonth = -1;
foreach (var(symbolRef, currentSymbolGroup) in symbolGenerator.GenerateRandomSymbols()
.GroupBy(s => s.HasUnderlying ? s.Underlying : s)
.Select(g => (g.Key, g.OrderBy(s => s.HasUnderlying).ToList())))
{
Log.Trace($"RandomDataGenerator.Run(): Symbol[{++count}]: {symbolRef} Progress: {progress:0.0}% - Generating data...");
var tickGenerators = new List <IEnumerator <Tick> >();
var tickHistories = new Dictionary <Symbol, List <Tick> >();
Security underlyingSecurity = null;
foreach (var currentSymbol in currentSymbolGroup)
{
if (!_securityManager.TryGetValue(currentSymbol, out var security))
{
security = _securityManager.CreateSecurity(
currentSymbol,
new List <SubscriptionDataConfig>(),
underlying: underlyingSecurity);
_securityManager.Add(security);
}
underlyingSecurity ??= security;
tickGenerators.Add(
new TickGenerator(_settings, tickTypesPerSecurityType[currentSymbol.SecurityType].ToArray(), security, randomValueGenerator)
.GenerateTicks()
.GetEnumerator());
tickHistories.Add(
currentSymbol,
new List <Tick>());
}
using var sync = new SynchronizingBaseDataEnumerator(tickGenerators);
while (sync.MoveNext())
{
var dataPoint = sync.Current;
if (!_securityManager.TryGetValue(dataPoint.Symbol, out var security))
{
Log.Error($"RandomDataGenerator.Run(): Could not find security for symbol {sync.Current.Symbol}");
continue;
}
tickHistories[security.Symbol].Add(dataPoint as Tick);
security.Update(new List <BaseData> {
dataPoint
}, dataPoint.GetType(), false);
}
foreach (var(currentSymbol, tickHistory) in tickHistories)
{
var symbol = currentSymbol;
// This is done so that we can update the Symbol in the case of a rename event
var delistDate = GetDelistingDate(_settings.Start, _settings.End, randomValueGenerator);
var willBeDelisted = randomValueGenerator.NextBool(1.0);
// Companies rarely IPO then disappear within 6 months
if (willBeDelisted && tickHistory.Select(tick => tick.Time.Month).Distinct().Count() <= 6)
{
willBeDelisted = false;
}
var dividendsSplitsMaps = new DividendSplitMapGenerator(
symbol,
_settings,
randomValueGenerator,
symbolGenerator,
random,
delistDate,
willBeDelisted);
// Keep track of renamed symbols and the time they were renamed.
var renamedSymbols = new Dictionary <Symbol, DateTime>();
if (_settings.SecurityType == SecurityType.Equity)
{
dividendsSplitsMaps.GenerateSplitsDividends(tickHistory);
if (!willBeDelisted)
{
dividendsSplitsMaps.DividendsSplits.Add(new CorporateFactorRow(new DateTime(2050, 12, 31), 1m, 1m));
if (dividendsSplitsMaps.MapRows.Count > 1)
{
// Remove the last element if we're going to have a 20501231 entry
dividendsSplitsMaps.MapRows.RemoveAt(dividendsSplitsMaps.MapRows.Count - 1);
}
dividendsSplitsMaps.MapRows.Add(new MapFileRow(new DateTime(2050, 12, 31), dividendsSplitsMaps.CurrentSymbol.Value));
}
// If the Symbol value has changed, update the current Symbol
if (symbol != dividendsSplitsMaps.CurrentSymbol)
{
// Add all Symbol rename events to dictionary
// We skip the first row as it contains the listing event instead of a rename event
foreach (var renameEvent in dividendsSplitsMaps.MapRows.Skip(1))
{
// Symbol.UpdateMappedSymbol does not update the underlying security ID Symbol, which
// is used to create the hash code. Create a new equity Symbol from scratch instead.
symbol = Symbol.Create(renameEvent.MappedSymbol, SecurityType.Equity, _settings.Market);
renamedSymbols.Add(symbol, renameEvent.Date);
Log.Trace($"RandomDataGenerator.Run(): Symbol[{count}]: {symbol} will be renamed on {renameEvent.Date}");
}
}
else
{
// This ensures that ticks will be written for the current Symbol up until 9999-12-31
renamedSymbols.Add(symbol, new DateTime(9999, 12, 31));
}
symbol = dividendsSplitsMaps.CurrentSymbol;
// Write Splits and Dividend events to directory factor_files
var factorFile = new CorporateFactorProvider(symbol.Value, dividendsSplitsMaps.DividendsSplits, _settings.Start);
var mapFile = new MapFile(symbol.Value, dividendsSplitsMaps.MapRows);
factorFile.WriteToFile(symbol);
mapFile.WriteToCsv(_settings.Market, symbol.SecurityType);
Log.Trace($"RandomDataGenerator.Run(): Symbol[{count}]: {symbol} Dividends, splits, and map files have been written to disk.");
}
else
{
// This ensures that ticks will be written for the current Symbol up until 9999-12-31
renamedSymbols.Add(symbol, new DateTime(9999, 12, 31));
}
// define aggregators via settings
var aggregators = CreateAggregators(_settings, tickTypesPerSecurityType[currentSymbol.SecurityType].ToArray()).ToList();
Symbol previousSymbol = null;
var currentCount = 0;
var monthsTrading = 0;
foreach (var renamed in renamedSymbols)
{
var previousRenameDate = previousSymbol == null ? new DateTime(1, 1, 1) : renamedSymbols[previousSymbol];
var previousRenameDateDay = new DateTime(previousRenameDate.Year, previousRenameDate.Month, previousRenameDate.Day);
var renameDate = renamed.Value;
var renameDateDay = new DateTime(renameDate.Year, renameDate.Month, renameDate.Day);
foreach (var tick in tickHistory.Where(tick => tick.Time >= previousRenameDate && previousRenameDateDay != TickDay(tick)))
{
// Prevents the aggregator from being updated with ticks after the rename event
if (TickDay(tick) > renameDateDay)
{
break;
}
if (tick.Time.Month != previousMonth)
{
Log.Trace($"RandomDataGenerator.Run(): Symbol[{count}]: Month: {tick.Time:MMMM}");
previousMonth = tick.Time.Month;
monthsTrading++;
}
foreach (var item in aggregators)
{
tick.Value = tick.Value / dividendsSplitsMaps.FinalSplitFactor;
item.Consolidator.Update(tick);
}
if (monthsTrading >= 6 && willBeDelisted && tick.Time > delistDate)
{
Log.Trace($"RandomDataGenerator.Run(): Symbol[{count}]: {renamed.Key} delisted at {tick.Time:MMMM yyyy}");
break;
}
}
// count each stage as a point, so total points is 2*Symbol-count
// and the current progress is twice the current, but less one because we haven't finished writing data yet
progress = 100 * (2 * count - 1) / (2.0 * _settings.SymbolCount);
Log.Trace($"RandomDataGenerator.Run(): Symbol[{count}]: {renamed.Key} Progress: {progress:0.0}% - Saving data in LEAN format");
// persist consolidated data to disk
foreach (var item in aggregators)
{
var writer = new LeanDataWriter(item.Resolution, renamed.Key, Globals.DataFolder, item.TickType);
// send the flushed data into the writer. pulling the flushed list is very important,
// lest we likely wouldn't get the last piece of data stuck in the consolidator
// Filter out the data we're going to write here because filtering them in the consolidator update phase
// makes it write all dates for some unknown reason
writer.Write(item.Flush().Where(data => data.Time > previousRenameDate && previousRenameDateDay != DataDay(data)));
}
// update progress
progress = 100 * (2 * count) / (2.0 * _settings.SymbolCount);
Log.Trace($"RandomDataGenerator.Run(): Symbol[{count}]: {symbol} Progress: {progress:0.0}% - Symbol data generation and output completed");
previousSymbol = renamed.Key;
currentCount++;
}
}
}
Log.Trace("RandomDataGenerator.Run(): Random data generation has completed.");
DateTime TickDay(Tick tick) => new(tick.Time.Year, tick.Time.Month, tick.Time.Day);
DateTime DataDay(BaseData data) => new(data.Time.Year, data.Time.Month, data.Time.Day);
}
/// <summary>
/// Generates the factor file row.
/// </summary>
/// <param name="ticker">The ticker.</param>
/// <param name="sidContext">The sid context.</param>
/// <param name="factorFile">The factor file.</param>
/// <param name="tradeBar">The trade bar.</param>
/// <param name="fineAvailableDates">The fine available dates.</param>
/// <param name="fineFundamentalFolder">The fine fundamental folder.</param>
/// <returns></returns>
private static string GenerateFactorFileRow(string ticker, SecurityIdentifierContext sidContext, CorporateFactorProvider factorFile, TradeBar tradeBar, IEnumerable <DateTime> fineAvailableDates, DirectoryInfo fineFundamentalFolder)
{
var date = tradeBar.Time;
var factorFileRow = factorFile?.GetScalingFactors(date);
var dollarVolume = Math.Truncate(tradeBar.Close * tradeBar.Volume);
var priceFactor = factorFileRow?.PriceFactor.Normalize() ?? 1m;
var splitFactor = factorFileRow?.SplitFactor.Normalize() ?? 1m;
bool hasFundamentalData = CheckFundamentalData(date, sidContext.MapFile, fineAvailableDates, fineFundamentalFolder);
// sid,symbol,close,volume,dollar volume,has fundamental data,price factor,split factor
var coarseFileLine = $"{sidContext.SID},{ticker.ToUpperInvariant()},{tradeBar.Close.Normalize()},{tradeBar.Volume.Normalize()},{Math.Truncate(dollarVolume)},{hasFundamentalData},{priceFactor},{splitFactor}";
return(coarseFileLine);
}
