/// <summary>
/// Creates a new instance of the DataManager
/// </summary>
public DataManager(IDataFeed dataFeed, UniverseSelection universeSelection, IAlgorithmSettings algorithmSettings, ITimeKeeper timeKeeper)
{
_dataFeed = dataFeed;
UniverseSelection = universeSelection;
_algorithmSettings = algorithmSettings;
AvailableDataTypes = SubscriptionManager.DefaultDataTypes();
_timeKeeper = timeKeeper;
}
/// <summary>
/// Pumps a bunch of ticks into the queue
/// </summary>
private void PopulateQueue()
{
List <Symbol> symbols;
lock (_sync)
{
symbols = _symbols.ToList();
}
foreach (var symbol in symbols)
{
var offsetProvider = GetTimeZoneOffsetProvider(symbol);
var trades = SubscriptionManager.DefaultDataTypes()[symbol.SecurityType].Contains(TickType.Trade);
var quotes = SubscriptionManager.DefaultDataTypes()[symbol.SecurityType].Contains(TickType.Quote);
// emits 500k per second
for (var i = 0; i < 500000; i++)
{
var now = TimeProvider.GetUtcNow();
if (trades)
{
_count++;
_aggregator.Update(new Tick
{
Time = offsetProvider.ConvertFromUtc(now),
Symbol = symbol,
Value = 10 + (decimal)Math.Abs(Math.Sin(now.TimeOfDay.TotalMinutes)),
TickType = TickType.Trade,
Quantity = _random.Next(10, (int)_timer.Interval)
});
}
if (quotes)
{
_count++;
var bid = 10 + (decimal)Math.Abs(Math.Sin(now.TimeOfDay.TotalMinutes));
var bidSize = _random.Next(10, (int)_timer.Interval);
var askSize = _random.Next(10, (int)_timer.Interval);
var time = offsetProvider.ConvertFromUtc(now);
_aggregator.Update(new Tick(time, symbol, "", "", bid, bidSize, bid * 1.01m, askSize));
}
}
}
}
/// <summary>
/// Creates a new instance of the DataManager
/// </summary>
public DataManager(
IDataFeed dataFeed,
UniverseSelection universeSelection,
IAlgorithm algorithm,
ITimeKeeper timeKeeper,
MarketHoursDatabase marketHoursDatabase,
bool liveMode,
IRegisteredSecurityDataTypesProvider registeredTypesProvider,
IDataPermissionManager dataPermissionManager)
{
_dataFeed = dataFeed;
UniverseSelection = universeSelection;
UniverseSelection.SetDataManager(this);
_algorithmSettings = algorithm.Settings;
AvailableDataTypes = SubscriptionManager.DefaultDataTypes();
_timeKeeper = timeKeeper;
_marketHoursDatabase = marketHoursDatabase;
_liveMode = liveMode;
_registeredTypesProvider = registeredTypesProvider;
_dataPermissionManager = dataPermissionManager;
// wire ourselves up to receive notifications when universes are added/removed
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType <Universe>())
{
var config = universe.Configuration;
var start = algorithm.UtcTime;
var end = algorithm.LiveMode ? Time.EndOfTime
: algorithm.EndDate.ConvertToUtc(algorithm.TimeZone);
Security security;
if (!algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object if it doesn't exist
security = new Security(
_marketHoursDatabase.GetExchangeHours(config),
config,
algorithm.Portfolio.CashBook[algorithm.AccountCurrency],
SymbolProperties.GetDefault(algorithm.AccountCurrency),
algorithm.Portfolio.CashBook,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
}
AddSubscription(
new SubscriptionRequest(true,
universe,
security,
config,
start,
end));
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType <Universe>())
{
// removing the subscription will be handled by the SubscriptionSynchronizer
// in the next loop as well as executing a UniverseSelection one last time.
if (!universe.DisposeRequested)
{
universe.Dispose();
}
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
/// <summary>
/// Creates a new instance of the DataManager
/// </summary>
public DataManager(
IDataFeed dataFeed,
UniverseSelection universeSelection,
IAlgorithm algorithm,
ITimeKeeper timeKeeper,
MarketHoursDatabase marketHoursDatabase)
{
_dataFeed = dataFeed;
UniverseSelection = universeSelection;
UniverseSelection.SetDataManager(this);
_algorithmSettings = algorithm.Settings;
AvailableDataTypes = SubscriptionManager.DefaultDataTypes();
_timeKeeper = timeKeeper;
_marketHoursDatabase = marketHoursDatabase;
var liveStart = DateTime.UtcNow;
// wire ourselves up to receive notifications when universes are added/removed
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType <Universe>())
{
var config = universe.Configuration;
var start = algorithm.LiveMode ? liveStart : algorithm.UtcTime;
var end = algorithm.LiveMode ? Time.EndOfTime
: algorithm.EndDate.ConvertToUtc(algorithm.TimeZone);
Security security;
if (!algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object if it doesn't exist
security = new Security(
_marketHoursDatabase.GetExchangeHours(config),
config,
algorithm.Portfolio.CashBook[CashBook.AccountCurrency],
SymbolProperties.GetDefault(CashBook.AccountCurrency),
algorithm.Portfolio.CashBook
);
}
AddSubscription(
new SubscriptionRequest(true,
universe,
security,
config,
start,
end));
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType <Universe>())
{
RemoveSubscription(universe.Configuration);
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
/// <summary>
/// Starts data generation
/// </summary>
public void Run()
{
var tickTypesPerSecurityType = SubscriptionManager.DefaultDataTypes();
// can specify a seed value in this ctor if determinism is desired
var random = new Random();
var randomValueGenerator = new RandomValueGenerator();
if (_settings.RandomSeedSet)
{
random = new Random(_settings.RandomSeed);
randomValueGenerator = new RandomValueGenerator(_settings.RandomSeed);
}
var symbolGenerator = BaseSymbolGenerator.Create(_settings, randomValueGenerator);
var maxSymbolCount = symbolGenerator.GetAvailableSymbolCount();
if (_settings.SymbolCount > maxSymbolCount)
{
Log.Error($"RandomDataGenerator.Run(): Limiting Symbol count to {maxSymbolCount}, we don't have more {_settings.SecurityType} tickers for {_settings.Market}");
_settings.SymbolCount = maxSymbolCount;
}
Log.Trace($"RandomDataGenerator.Run(): Begin data generation of {_settings.SymbolCount} randomly generated {_settings.SecurityType} assets...");
// iterate over our randomly generated symbols
var count = 0;
var progress = 0d;
var previousMonth = -1;
foreach (var(symbolRef, currentSymbolGroup) in symbolGenerator.GenerateRandomSymbols()
.GroupBy(s => s.HasUnderlying ? s.Underlying : s)
.Select(g => (g.Key, g.OrderBy(s => s.HasUnderlying).ToList())))
{
Log.Trace($"RandomDataGenerator.Run(): Symbol[{++count}]: {symbolRef} Progress: {progress:0.0}% - Generating data...");
var tickGenerators = new List <IEnumerator <Tick> >();
var tickHistories = new Dictionary <Symbol, List <Tick> >();
Security underlyingSecurity = null;
foreach (var currentSymbol in currentSymbolGroup)
{
if (!_securityManager.TryGetValue(currentSymbol, out var security))
{
security = _securityManager.CreateSecurity(
currentSymbol,
new List <SubscriptionDataConfig>(),
underlying: underlyingSecurity);
_securityManager.Add(security);
}
underlyingSecurity ??= security;
tickGenerators.Add(
new TickGenerator(_settings, tickTypesPerSecurityType[currentSymbol.SecurityType].ToArray(), security, randomValueGenerator)
.GenerateTicks()
.GetEnumerator());
tickHistories.Add(
currentSymbol,
new List <Tick>());
}
using var sync = new SynchronizingBaseDataEnumerator(tickGenerators);
while (sync.MoveNext())
{
var dataPoint = sync.Current;
if (!_securityManager.TryGetValue(dataPoint.Symbol, out var security))
{
Log.Error($"RandomDataGenerator.Run(): Could not find security for symbol {sync.Current.Symbol}");
continue;
}
tickHistories[security.Symbol].Add(dataPoint as Tick);
security.Update(new List <BaseData> {
dataPoint
}, dataPoint.GetType(), false);
}
foreach (var(currentSymbol, tickHistory) in tickHistories)
{
var symbol = currentSymbol;
// This is done so that we can update the Symbol in the case of a rename event
var delistDate = GetDelistingDate(_settings.Start, _settings.End, randomValueGenerator);
var willBeDelisted = randomValueGenerator.NextBool(1.0);
// Companies rarely IPO then disappear within 6 months
if (willBeDelisted && tickHistory.Select(tick => tick.Time.Month).Distinct().Count() <= 6)
{
willBeDelisted = false;
}
var dividendsSplitsMaps = new DividendSplitMapGenerator(
symbol,
_settings,
randomValueGenerator,
symbolGenerator,
random,
delistDate,
willBeDelisted);
// Keep track of renamed symbols and the time they were renamed.
var renamedSymbols = new Dictionary <Symbol, DateTime>();
if (_settings.SecurityType == SecurityType.Equity)
{
dividendsSplitsMaps.GenerateSplitsDividends(tickHistory);
if (!willBeDelisted)
{
dividendsSplitsMaps.DividendsSplits.Add(new CorporateFactorRow(new DateTime(2050, 12, 31), 1m, 1m));
if (dividendsSplitsMaps.MapRows.Count > 1)
{
// Remove the last element if we're going to have a 20501231 entry
dividendsSplitsMaps.MapRows.RemoveAt(dividendsSplitsMaps.MapRows.Count - 1);
}
dividendsSplitsMaps.MapRows.Add(new MapFileRow(new DateTime(2050, 12, 31), dividendsSplitsMaps.CurrentSymbol.Value));
}
// If the Symbol value has changed, update the current Symbol
if (symbol != dividendsSplitsMaps.CurrentSymbol)
{
// Add all Symbol rename events to dictionary
// We skip the first row as it contains the listing event instead of a rename event
foreach (var renameEvent in dividendsSplitsMaps.MapRows.Skip(1))
{
// Symbol.UpdateMappedSymbol does not update the underlying security ID Symbol, which
// is used to create the hash code. Create a new equity Symbol from scratch instead.
symbol = Symbol.Create(renameEvent.MappedSymbol, SecurityType.Equity, _settings.Market);
renamedSymbols.Add(symbol, renameEvent.Date);
Log.Trace($"RandomDataGenerator.Run(): Symbol[{count}]: {symbol} will be renamed on {renameEvent.Date}");
}
}
else
{
// This ensures that ticks will be written for the current Symbol up until 9999-12-31
renamedSymbols.Add(symbol, new DateTime(9999, 12, 31));
}
symbol = dividendsSplitsMaps.CurrentSymbol;
// Write Splits and Dividend events to directory factor_files
var factorFile = new CorporateFactorProvider(symbol.Value, dividendsSplitsMaps.DividendsSplits, _settings.Start);
var mapFile = new MapFile(symbol.Value, dividendsSplitsMaps.MapRows);
factorFile.WriteToFile(symbol);
mapFile.WriteToCsv(_settings.Market, symbol.SecurityType);
Log.Trace($"RandomDataGenerator.Run(): Symbol[{count}]: {symbol} Dividends, splits, and map files have been written to disk.");
}
else
{
// This ensures that ticks will be written for the current Symbol up until 9999-12-31
renamedSymbols.Add(symbol, new DateTime(9999, 12, 31));
}
// define aggregators via settings
var aggregators = CreateAggregators(_settings, tickTypesPerSecurityType[currentSymbol.SecurityType].ToArray()).ToList();
Symbol previousSymbol = null;
var currentCount = 0;
var monthsTrading = 0;
foreach (var renamed in renamedSymbols)
{
var previousRenameDate = previousSymbol == null ? new DateTime(1, 1, 1) : renamedSymbols[previousSymbol];
var previousRenameDateDay = new DateTime(previousRenameDate.Year, previousRenameDate.Month, previousRenameDate.Day);
var renameDate = renamed.Value;
var renameDateDay = new DateTime(renameDate.Year, renameDate.Month, renameDate.Day);
foreach (var tick in tickHistory.Where(tick => tick.Time >= previousRenameDate && previousRenameDateDay != TickDay(tick)))
{
// Prevents the aggregator from being updated with ticks after the rename event
if (TickDay(tick) > renameDateDay)
{
break;
}
if (tick.Time.Month != previousMonth)
{
Log.Trace($"RandomDataGenerator.Run(): Symbol[{count}]: Month: {tick.Time:MMMM}");
previousMonth = tick.Time.Month;
monthsTrading++;
}
foreach (var item in aggregators)
{
tick.Value = tick.Value / dividendsSplitsMaps.FinalSplitFactor;
item.Consolidator.Update(tick);
}
if (monthsTrading >= 6 && willBeDelisted && tick.Time > delistDate)
{
Log.Trace($"RandomDataGenerator.Run(): Symbol[{count}]: {renamed.Key} delisted at {tick.Time:MMMM yyyy}");
break;
}
}
// count each stage as a point, so total points is 2*Symbol-count
// and the current progress is twice the current, but less one because we haven't finished writing data yet
progress = 100 * (2 * count - 1) / (2.0 * _settings.SymbolCount);
Log.Trace($"RandomDataGenerator.Run(): Symbol[{count}]: {renamed.Key} Progress: {progress:0.0}% - Saving data in LEAN format");
// persist consolidated data to disk
foreach (var item in aggregators)
{
var writer = new LeanDataWriter(item.Resolution, renamed.Key, Globals.DataFolder, item.TickType);
// send the flushed data into the writer. pulling the flushed list is very important,
// lest we likely wouldn't get the last piece of data stuck in the consolidator
// Filter out the data we're going to write here because filtering them in the consolidator update phase
// makes it write all dates for some unknown reason
writer.Write(item.Flush().Where(data => data.Time > previousRenameDate && previousRenameDateDay != DataDay(data)));
}
// update progress
progress = 100 * (2 * count) / (2.0 * _settings.SymbolCount);
Log.Trace($"RandomDataGenerator.Run(): Symbol[{count}]: {symbol} Progress: {progress:0.0}% - Symbol data generation and output completed");
previousSymbol = renamed.Key;
currentCount++;
}
}
}
Log.Trace("RandomDataGenerator.Run(): Random data generation has completed.");
DateTime TickDay(Tick tick) => new(tick.Time.Year, tick.Time.Month, tick.Time.Day);
DateTime DataDay(BaseData data) => new(data.Time.Year, data.Time.Month, data.Time.Day);
}
/// <summary>
/// Primary entry point to the program. This program only supports SecurityType.Equity
/// </summary>
public static void PolygonDownloader(IList <string> tickers, string securityTypeString, string market, string resolutionString, DateTime fromDate, DateTime toDate)
{
if (tickers.IsNullOrEmpty() || securityTypeString.IsNullOrEmpty() || market.IsNullOrEmpty() || resolutionString.IsNullOrEmpty())
{
Console.WriteLine("PolygonDownloader ERROR: '--tickers=' or '--security-type=' or '--market=' or '--resolution=' parameter is missing");
Console.WriteLine("--tickers=eg SPY,AAPL");
Console.WriteLine("--security-type=Equity");
Console.WriteLine("--market=usa");
Console.WriteLine("--resolution=Minute/Hour/Daily");
Environment.Exit(1);
}
try
{
// Load settings from command line
var resolution = (Resolution)Enum.Parse(typeof(Resolution), resolutionString);
var securityType = (SecurityType)Enum.Parse(typeof(SecurityType), securityTypeString);
// Polygon.io does not support Crypto historical quotes
var tickTypes = securityType == SecurityType.Crypto
? new List <TickType> {
TickType.Trade
}
: SubscriptionManager.DefaultDataTypes()[securityType];
// Load settings from config.json
var dataDirectory = Config.Get("data-directory", "../../../Data");
var startDate = fromDate.ConvertToUtc(TimeZones.NewYork);
var endDate = toDate.ConvertToUtc(TimeZones.NewYork);
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
// Create an instance of the downloader
using (var downloader = new PolygonDataDownloader())
{
foreach (var ticker in tickers)
{
var symbol = Symbol.Create(ticker, securityType, market);
var exchangeTimeZone = marketHoursDatabase.GetExchangeHours(market, symbol, securityType).TimeZone;
var dataTimeZone = marketHoursDatabase.GetDataTimeZone(market, symbol, securityType);
foreach (var tickType in tickTypes)
{
// Download the data
var data = downloader.Get(new DataDownloaderGetParameters(symbol, resolution, startDate, endDate, tickType))
.Select(x =>
{
x.Time = x.Time.ConvertTo(exchangeTimeZone, dataTimeZone);
return(x);
}
);
// Save the data
var writer = new LeanDataWriter(resolution, symbol, dataDirectory, tickType);
writer.Write(data);
}
}
}
}
catch (Exception err)
{
Log.Error(err);
}
}
