public FutureSymbolGenerator(RandomDataGeneratorSettings settings, IRandomValueGenerator random)
: base(settings, random)
{
_minExpiry = settings.Start;
_maxExpiry = settings.End;
_market = settings.Market;
}
public static void RandomDataGenerator(
string startDateString,
string endDateString,
string symbolCountString,
string market,
string securityTypeString,
string resolutionString,
string dataDensityString,
string includeCoarseString,
string quoteTradeRatioString,
string randomSeed,
string hasIpoPercentageString,
string hasRenamePercentageString,
string hasSplitsPercentageString,
string hasDividendsPercentageString,
string dividendEveryQuarterPercentageString
)
{
var output = new ConsoleLeveledOutput();
var settings = RandomDataGeneratorSettings.FromCommandLineArguments(
startDateString,
endDateString,
symbolCountString,
market,
securityTypeString,
resolutionString,
dataDensityString,
includeCoarseString,
quoteTradeRatioString,
randomSeed,
hasIpoPercentageString,
hasRenamePercentageString,
hasSplitsPercentageString,
hasDividendsPercentageString,
dividendEveryQuarterPercentageString,
output
);
if (settings.Start.Year < 1998)
{
output.Error.WriteLine($"Required parameter --start must be at least 19980101");
Environment.Exit(1);
}
GenerateRandomData(settings, output);
if (settings.IncludeCoarse && settings.SecurityType == SecurityType.Equity)
{
output.Info.WriteLine("Launching coarse data generator...");
CoarseUniverseGeneratorProgram.CoarseUniverseGenerator();
}
if (!Console.IsInputRedirected)
{
output.Info.WriteLine("Press any key to exit...");
Console.ReadKey();
}
}
/// <summary>
/// Base constructor implementation for Symbol generator
/// </summary>
/// <param name="settings">random data generation run settings</param>
/// <param name="random">produces random values for use in random data generation</param>
protected BaseSymbolGenerator(RandomDataGeneratorSettings settings, IRandomValueGenerator random)
{
Settings = settings;
Random = random;
_symbols = new FixedSizeHashQueue <Symbol>(1000);
SymbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder();
MarketHoursDatabase = MarketHoursDatabase.FromDataFolder();
}
public static void RandomDataGenerator(
string startDateString,
string endDateString,
string symbolCountString,
string market,
string securityTypeString,
string resolutionString,
string dataDensityString,
string includeCoarseString,
string quoteTradeRatioString,
string randomSeed,
string hasIpoPercentageString,
string hasRenamePercentageString,
string hasSplitsPercentageString,
string hasDividendsPercentageString,
string dividendEveryQuarterPercentageString
)
{
var output = new ConsoleLeveledOutput();
var settings = RandomDataGeneratorSettings.FromCommandLineArguments(
startDateString,
endDateString,
symbolCountString,
market,
securityTypeString,
resolutionString,
dataDensityString,
includeCoarseString,
quoteTradeRatioString,
randomSeed,
hasIpoPercentageString,
hasRenamePercentageString,
hasSplitsPercentageString,
hasDividendsPercentageString,
dividendEveryQuarterPercentageString,
output
);
GenerateRandomData(settings, output);
if (settings.IncludeCoarse && settings.SecurityType == SecurityType.Equity)
{
output.Info.WriteLine("Launching coarse data generator...");
var coarseFiles = CoarseUniverseGenerator.CoarseUniverseGeneratorProgram.ProcessEquityDirectories(
Globals.DataFolder,
false
);
output.Info.WriteLine("Coarse data generation completed. Produced the following files:");
foreach (var coarseFile in coarseFiles)
{
output.Info.WriteLine($"Generated coarse file: {coarseFile}");
}
}
output.Info.WriteLine("Press any key to exit...");
Console.ReadKey();
}
public static void GenerateRandomData(RandomDataGeneratorSettings settings, ConsoleLeveledOutput output)
{
// can specify a seed value in this ctor if determinism is desired
var randomValueGenerator = new RandomValueGenerator();
var symbolGenerator = new SymbolGenerator(settings, randomValueGenerator);
var tickGenerator = new TickGenerator(settings, randomValueGenerator);
output.Warn.WriteLine($"Begin data generation of {settings.SymbolCount} randomly generated {settings.SecurityType} assets...");
// iterate over our randomly generated symbols
var count = 0;
var progress = 0d;
var previousMonth = -1;
foreach (var symbol in symbolGenerator.GenerateRandomSymbols())
{
output.Warn.WriteLine($"\tSymbol[{++count}]: {symbol} Progress: {progress:0.0}% - Generating data...");
// define aggregators via settings
var aggregators = settings.CreateAggregators().ToList();
// generate and consolidate data
foreach (var tick in tickGenerator.GenerateTicks(symbol))
{
if (tick.Time.Month != previousMonth)
{
output.Info.WriteLine($"\tMonth: {tick.Time:MMMM}");
previousMonth = tick.Time.Month;
}
foreach (var item in aggregators)
{
item.Consolidator.Update(tick);
}
}
// count each stage as a point, so total points is 2*symbol-count
// and the current progress is twice the current, but less one because we haven't finished writing data yet
progress = 100 * (2 * count - 1) / (2.0 * settings.SymbolCount);
output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} Progress: {progress:0.0}% - Saving data in LEAN format");
// persist consolidated data to disk
foreach (var item in aggregators)
{
var writer = new LeanDataWriter(item.Resolution, symbol, Globals.DataFolder, item.TickType);
// send the flushed data into the writer. pulling the flushed list is very important,
// lest we likely wouldn't get the last piece of data stuck in the consolidator
writer.Write(item.Flush());
}
// update progress
progress = 100 * (2 * count) / (2.0 * settings.SymbolCount);
output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} Progress: {progress:0.0}% - Symbol data generation and output completed");
}
output.Info.WriteLine("Random data generation has completed.");
}
public OptionSymbolGenerator(RandomDataGeneratorSettings settings, IRandomValueGenerator random, decimal underlyingPrice, decimal maximumStrikePriceDeviation)
: base(settings, random)
{
_minExpiry = settings.Start;
_maxExpiry = settings.End;
_market = settings.Market;
_underlyingPrice = underlyingPrice;
_symbolChainSize = settings.ChainSymbolCount;
_maximumStrikePriceDeviation = maximumStrikePriceDeviation;
}
public static void RandomDataGenerator(
string startDateString,
string endDateString,
string symbolCountString,
string market,
string securityTypeString,
string resolutionString,
string dataDensityString,
string includeCoarseString,
string quoteTradeRatioString,
string randomSeed,
string hasIpoPercentageString,
string hasRenamePercentageString,
string hasSplitsPercentageString,
string hasDividendsPercentageString,
string dividendEveryQuarterPercentageString
)
{
var output = new ConsoleLeveledOutput();
var settings = RandomDataGeneratorSettings.FromCommandLineArguments(
startDateString,
endDateString,
symbolCountString,
market,
securityTypeString,
resolutionString,
dataDensityString,
includeCoarseString,
quoteTradeRatioString,
randomSeed,
hasIpoPercentageString,
hasRenamePercentageString,
hasSplitsPercentageString,
hasDividendsPercentageString,
dividendEveryQuarterPercentageString,
output
);
GenerateRandomData(settings, output);
if (settings.IncludeCoarse && settings.SecurityType == SecurityType.Equity)
{
output.Info.WriteLine("Launching coarse data generator...");
CoarseUniverseGeneratorProgram.CoarseUniverseGenerator();
}
output.Info.WriteLine("Press any key to exit...");
Console.ReadKey();
}
public DividendSplitMapGenerator(
Symbol symbol,
RandomDataGeneratorSettings settings,
RandomValueGenerator randomValueGenerator,
Random random,
DateTime delistDate,
bool willBeDelisted)
{
CurrentSymbol = symbol;
_settings = settings;
_randomValueGenerator = randomValueGenerator;
_random = random;
_delistDate = delistDate;
_willBeDelisted = willBeDelisted;
}
public TickGenerator(RandomDataGeneratorSettings settings, TickType[] tickTypes, Security security, IRandomValueGenerator random)
{
_random = random;
_settings = settings;
_tickTypes = tickTypes;
Security = security;
SymbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder();
MarketHoursDatabase = MarketHoursDatabase.FromDataFolder();
if (Symbol.SecurityType.IsOption())
{
_priceGenerator = new OptionPriceModelPriceGenerator(security);
}
else
{
_priceGenerator = new RandomPriceGenerator(security, random);
}
}
public static IEnumerable <TickAggregator> CreateAggregators(RandomDataGeneratorSettings settings, TickType[] tickTypes)
{
// create default aggregators for tick type/resolution
foreach (var tickAggregator in TickAggregator.ForTickTypes(settings.Resolution, tickTypes))
{
yield return(tickAggregator);
}
// ensure we have a daily consolidator when coarse is enabled
if (settings.IncludeCoarse && settings.Resolution != Resolution.Daily)
{
// prefer trades for coarse - in practice equity only does trades, but leaving this as configurable
if (tickTypes.Contains(TickType.Trade))
{
yield return(TickAggregator.ForTickTypes(Resolution.Daily, TickType.Trade).Single());
}
else
{
yield return(TickAggregator.ForTickTypes(Resolution.Daily, TickType.Quote).Single());
}
}
}
/// <summary>
/// Creates a ad-hoc symbol generator depending on settings
/// </summary>
/// <param name="settings">random data generator settings</param>
/// <param name="random">produces random values for use in random data generation</param>
/// <returns>New symbol generator</returns>
public static BaseSymbolGenerator Create(RandomDataGeneratorSettings settings, IRandomValueGenerator random)
{
if (settings is null)
{
throw new ArgumentNullException(nameof(settings), "Settings cannot be null or empty");
}
if (random is null)
{
throw new ArgumentNullException(nameof(random), "Randomizer cannot be null");
}
switch (settings.SecurityType)
{
case SecurityType.Option:
return(new OptionSymbolGenerator(settings, random, 100m, 75m));
case SecurityType.Future:
return(new FutureSymbolGenerator(settings, random));
default:
return(new DefaultSymbolGenerator(settings, random));
}
}
public static void RandomDataGenerator(
string startDateString,
string endDateString,
string symbolCountString,
string market,
string securityTypeString,
string resolutionString,
string dataDensityString,
string includeCoarseString,
string quoteTradeRatioString,
string randomSeed,
string hasIpoPercentageString,
string hasRenamePercentageString,
string hasSplitsPercentageString,
string hasDividendsPercentageString,
string dividendEveryQuarterPercentageString,
string optionPriceEngineName,
string volatilityModelResolutionString,
string chainSymbolCountString,
List <string> tickers
)
{
var settings = RandomDataGeneratorSettings.FromCommandLineArguments(
startDateString,
endDateString,
symbolCountString,
market,
securityTypeString,
resolutionString,
dataDensityString,
includeCoarseString,
quoteTradeRatioString,
randomSeed,
hasIpoPercentageString,
hasRenamePercentageString,
hasSplitsPercentageString,
hasDividendsPercentageString,
dividendEveryQuarterPercentageString,
optionPriceEngineName,
volatilityModelResolutionString,
chainSymbolCountString,
tickers
);
if (settings.Start.Year < 1998)
{
Log.Error($"RandomDataGeneratorProgram(): Required parameter --start must be at least 19980101");
Environment.Exit(1);
}
var securityManager = new SecurityManager(new TimeKeeper(settings.Start, new[] { TimeZones.Utc }));
var securityService = new SecurityService(
new CashBook(),
MarketHoursDatabase.FromDataFolder(),
SymbolPropertiesDatabase.FromDataFolder(),
new SecurityInitializerProvider(new FuncSecurityInitializer(security =>
{
// init price
security.SetMarketPrice(new Tick(settings.Start, security.Symbol, 100, 100));
security.SetMarketPrice(new OpenInterest(settings.Start, security.Symbol, 10000));
// from settings
security.VolatilityModel = new StandardDeviationOfReturnsVolatilityModel(settings.VolatilityModelResolution);
// from settings
if (security is Option option)
{
option.PriceModel = OptionPriceModels.Create(settings.OptionPriceEngineName, Statistics.PortfolioStatistics.GetRiskFreeRate());
}
})),
RegisteredSecurityDataTypesProvider.Null,
new SecurityCacheProvider(
new SecurityPortfolioManager(securityManager, new SecurityTransactionManager(null, securityManager))),
new MapFilePrimaryExchangeProvider(Composer.Instance.GetExportedValueByTypeName <IMapFileProvider>(Config.Get("map-file-provider", "LocalDiskMapFileProvider")))
);
securityManager.SetSecurityService(securityService);
var generator = new RandomDataGenerator();
generator.Init(settings, securityManager);
generator.Run();
if (settings.IncludeCoarse && settings.SecurityType == SecurityType.Equity)
{
Log.Trace("RandomDataGeneratorProgram(): Launching coarse data generator...");
CoarseUniverseGeneratorProgram.CoarseUniverseGenerator();
}
if (!Console.IsInputRedirected)
{
Log.Trace("RandomDataGeneratorProgram(): Press any key to exit...");
Console.ReadKey();
}
}
/// <summary>
/// Creates <see cref="DefaultSymbolGenerator"/> instance
/// </summary>
/// <param name="settings">random data generation run settings</param>
/// <param name="random">produces random values for use in random data generation</param>
public DefaultSymbolGenerator(RandomDataGeneratorSettings settings, IRandomValueGenerator random)
: base(settings, random)
{
_market = settings.Market;
_securityType = settings.SecurityType;
}
public static void GenerateRandomData(RandomDataGeneratorSettings settings, ConsoleLeveledOutput output)
{
// can specify a seed value in this ctor if determinism is desired
var random = new Random();
var randomValueGenerator = new RandomValueGenerator();
if (settings.RandomSeedSet)
{
random = new Random(settings.RandomSeed);
randomValueGenerator = new RandomValueGenerator(settings.RandomSeed);
}
var symbolGenerator = new SymbolGenerator(settings, randomValueGenerator);
var tickGenerator = new TickGenerator(settings, randomValueGenerator);
output.Warn.WriteLine($"Begin data generation of {settings.SymbolCount} randomly generated {settings.SecurityType} assets...");
// iterate over our randomly generated symbols
var count = 0;
var progress = 0d;
var previousMonth = -1;
Func <Tick, DateTime> tickDay = (tick => new DateTime(tick.Time.Year, tick.Time.Month, tick.Time.Day));
Func <Data.BaseData, DateTime> dataDay = (data => new DateTime(data.Time.Year, data.Time.Month, data.Time.Day));
foreach (var currentSymbol in symbolGenerator.GenerateRandomSymbols())
{
// This is done so that we can update the symbol in the case of a rename event
var delistDate = GetDelistingDate(settings.Start, settings.End, randomValueGenerator);
var symbol = currentSymbol;
var willBeDelisted = randomValueGenerator.NextBool(1.0);
var monthsTrading = 0;
// Keep track of renamed symbols and the time they were renamed.
var renamedSymbols = new Dictionary <Symbol, DateTime>();
output.Warn.WriteLine($"\tSymbol[{++count}]: {symbol} Progress: {progress:0.0}% - Generating data...");
// define aggregators via settings
var aggregators = settings.CreateAggregators().ToList();
var tickHistory = tickGenerator.GenerateTicks(symbol).ToList();
var dividendsSplitsMaps = new DividendSplitMapGenerator(
symbol,
settings,
randomValueGenerator,
random,
delistDate,
willBeDelisted);
if (settings.SecurityType == SecurityType.Equity)
{
dividendsSplitsMaps.GenerateSplitsDividends(tickHistory);
if (!willBeDelisted)
{
dividendsSplitsMaps.DividendsSplits.Add(new FactorFileRow(new DateTime(2050, 12, 31), 1m, 1m));
if (dividendsSplitsMaps.MapRows.Count > 1)
{
// Remove the last element if we're going to have a 20501231 entry
dividendsSplitsMaps.MapRows.RemoveAt(dividendsSplitsMaps.MapRows.Count - 1);
}
dividendsSplitsMaps.MapRows.Add(new MapFileRow(new DateTime(2050, 12, 31), dividendsSplitsMaps.CurrentSymbol.Value));
}
// If the symbol value has changed, update the current symbol
if (symbol != dividendsSplitsMaps.CurrentSymbol)
{
// Add all symbol rename events to dictionary
foreach (var renameEvent in dividendsSplitsMaps.MapRows)
{
// Symbol.UpdateMappedSymbol does not update the underlying security ID symbol, which
// is used to create the hash code. Create a new equity symbol from scratch instead.
symbol = Symbol.Create(renameEvent.MappedSymbol, SecurityType.Equity, settings.Market);
renamedSymbols.Add(symbol, renameEvent.Date);
output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} will be renamed on {renameEvent.Date}");
}
}
else
{
// This ensures that ticks will be written for the current symbol up until 9999-12-31
renamedSymbols.Add(symbol, new DateTime(9999, 12, 31));
}
symbol = dividendsSplitsMaps.CurrentSymbol;
// Write Splits and Dividend events to directory factor_files
var factorFile = new FactorFile(symbol.Value, dividendsSplitsMaps.DividendsSplits, settings.Start);
var mapFile = new MapFile(symbol.Value, dividendsSplitsMaps.MapRows);
factorFile.WriteToCsv(symbol);
mapFile.WriteToCsv(settings.Market);
output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} Dividends, splits, and map files have been written to disk.");
}
Symbol previousSymbol = null;
var currentCount = 0;
foreach (var renamed in renamedSymbols)
{
var previousRenameDate = previousSymbol == null ? new DateTime(1, 1, 1) : renamedSymbols[previousSymbol];
var previousRenameDateDay = new DateTime(previousRenameDate.Year, previousRenameDate.Month, previousRenameDate.Day);
var renameDate = renamed.Value;
var renameDateDay = new DateTime(renameDate.Year, renameDate.Month, renameDate.Day);
foreach (var tick in tickHistory.Where(tick => tick.Time >= previousRenameDate && previousRenameDateDay != tickDay(tick)))
{
// Prevents the aggregator from being updated with ticks after the rename event
if (tickDay(tick) > renameDateDay)
{
break;
}
if (tick.Time.Month != previousMonth)
{
output.Info.WriteLine($"\tSymbol[{count}]: Month: {tick.Time:MMMM}");
previousMonth = tick.Time.Month;
monthsTrading++;
}
foreach (var item in aggregators)
{
tick.Value = tick.Value / dividendsSplitsMaps.FinalSplitFactor;
item.Consolidator.Update(tick);
}
if (monthsTrading >= 6 && willBeDelisted && tick.Time > delistDate)
{
output.Warn.WriteLine($"\tSymbol[{count}]: {renamed.Key} delisted at {tick.Time:MMMM yyyy}");
break;
}
}
// count each stage as a point, so total points is 2*symbol-count
// and the current progress is twice the current, but less one because we haven't finished writing data yet
progress = 100 * (2 * count - 1) / (2.0 * settings.SymbolCount);
output.Warn.WriteLine($"\tSymbol[{count}]: {renamed.Key} Progress: {progress:0.0}% - Saving data in LEAN format");
// persist consolidated data to disk
foreach (var item in aggregators)
{
var writer = new LeanDataWriter(item.Resolution, renamed.Key, Globals.DataFolder, item.TickType);
// send the flushed data into the writer. pulling the flushed list is very important,
// lest we likely wouldn't get the last piece of data stuck in the consolidator
// Filter out the data we're going to write here because filtering them in the consolidator update phase
// makes it write all dates for some unknown reason
writer.Write(item.Flush().Where(data => data.Time > previousRenameDate && previousRenameDateDay != dataDay(data)));
}
// update progress
progress = 100 * (2 * count) / (2.0 * settings.SymbolCount);
output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} Progress: {progress:0.0}% - Symbol data generation and output completed");
previousSymbol = renamed.Key;
currentCount++;
}
}
output.Info.WriteLine("Random data generation has completed.");
}
public SymbolGenerator(RandomDataGeneratorSettings settings, IRandomValueGenerator random)
{
_settings = settings;
_random = random;
}
public TickGenerator(RandomDataGeneratorSettings settings, IRandomValueGenerator random)
{
_random = random;
_settings = settings;
}
public TickGenerator(RandomDataGeneratorSettings settings)
{
_settings = settings;
_random = new RandomValueGenerator();
}
/// <summary>
/// Initializes <see cref="RandomDataGenerator"/> instance fields
/// </summary>
/// <param name="settings">random data generation settings</param>
/// <param name="securityManager">security management</param>
public void Init(RandomDataGeneratorSettings settings, SecurityManager securityManager)
{
_settings = settings;
_securityManager = securityManager;
}