public static void RandomDataGenerator(
string startDateString,
string endDateString,
string symbolCountString,
string market,
string securityTypeString,
string resolutionString,
string dataDensityString,
string includeCoarseString,
string quoteTradeRatioString,
string randomSeed,
string hasIpoPercentageString,
string hasRenamePercentageString,
string hasSplitsPercentageString,
string hasDividendsPercentageString,
string dividendEveryQuarterPercentageString
)
{
var output = new ConsoleLeveledOutput();
var settings = RandomDataGeneratorSettings.FromCommandLineArguments(
startDateString,
endDateString,
symbolCountString,
market,
securityTypeString,
resolutionString,
dataDensityString,
includeCoarseString,
quoteTradeRatioString,
randomSeed,
hasIpoPercentageString,
hasRenamePercentageString,
hasSplitsPercentageString,
hasDividendsPercentageString,
dividendEveryQuarterPercentageString,
output
);
if (settings.Start.Year < 1998)
{
output.Error.WriteLine($"Required parameter --start must be at least 19980101");
Environment.Exit(1);
}
GenerateRandomData(settings, output);
if (settings.IncludeCoarse && settings.SecurityType == SecurityType.Equity)
{
output.Info.WriteLine("Launching coarse data generator...");
CoarseUniverseGeneratorProgram.CoarseUniverseGenerator();
}
if (!Console.IsInputRedirected)
{
output.Info.WriteLine("Press any key to exit...");
Console.ReadKey();
}
}
public static void GenerateRandomData(RandomDataGeneratorSettings settings, ConsoleLeveledOutput output)
{
// can specify a seed value in this ctor if determinism is desired
var randomValueGenerator = new RandomValueGenerator();
var symbolGenerator = new SymbolGenerator(settings, randomValueGenerator);
var tickGenerator = new TickGenerator(settings, randomValueGenerator);
output.Warn.WriteLine($"Begin data generation of {settings.SymbolCount} randomly generated {settings.SecurityType} assets...");
// iterate over our randomly generated symbols
var count = 0;
var progress = 0d;
var previousMonth = -1;
foreach (var symbol in symbolGenerator.GenerateRandomSymbols())
{
output.Warn.WriteLine($"\tSymbol[{++count}]: {symbol} Progress: {progress:0.0}% - Generating data...");
// define aggregators via settings
var aggregators = settings.CreateAggregators().ToList();
// generate and consolidate data
foreach (var tick in tickGenerator.GenerateTicks(symbol))
{
if (tick.Time.Month != previousMonth)
{
output.Info.WriteLine($"\tMonth: {tick.Time:MMMM}");
previousMonth = tick.Time.Month;
}
foreach (var item in aggregators)
{
item.Consolidator.Update(tick);
}
}
// count each stage as a point, so total points is 2*symbol-count
// and the current progress is twice the current, but less one because we haven't finished writing data yet
progress = 100 * (2 * count - 1) / (2.0 * settings.SymbolCount);
output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} Progress: {progress:0.0}% - Saving data in LEAN format");
// persist consolidated data to disk
foreach (var item in aggregators)
{
var writer = new LeanDataWriter(item.Resolution, symbol, Globals.DataFolder, item.TickType);
// send the flushed data into the writer. pulling the flushed list is very important,
// lest we likely wouldn't get the last piece of data stuck in the consolidator
writer.Write(item.Flush());
}
// update progress
progress = 100 * (2 * count) / (2.0 * settings.SymbolCount);
output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} Progress: {progress:0.0}% - Symbol data generation and output completed");
}
output.Info.WriteLine("Random data generation has completed.");
}
public static void RandomDataGenerator(
string startDateString,
string endDateString,
string symbolCountString,
string market,
string securityTypeString,
string resolutionString,
string dataDensityString,
string includeCoarseString,
string quoteTradeRatioString,
string randomSeed,
string hasIpoPercentageString,
string hasRenamePercentageString,
string hasSplitsPercentageString,
string hasDividendsPercentageString,
string dividendEveryQuarterPercentageString
)
{
var output = new ConsoleLeveledOutput();
var settings = RandomDataGeneratorSettings.FromCommandLineArguments(
startDateString,
endDateString,
symbolCountString,
market,
securityTypeString,
resolutionString,
dataDensityString,
includeCoarseString,
quoteTradeRatioString,
randomSeed,
hasIpoPercentageString,
hasRenamePercentageString,
hasSplitsPercentageString,
hasDividendsPercentageString,
dividendEveryQuarterPercentageString,
output
);
GenerateRandomData(settings, output);
if (settings.IncludeCoarse && settings.SecurityType == SecurityType.Equity)
{
output.Info.WriteLine("Launching coarse data generator...");
var coarseFiles = CoarseUniverseGenerator.CoarseUniverseGeneratorProgram.ProcessEquityDirectories(
Globals.DataFolder,
false
);
output.Info.WriteLine("Coarse data generation completed. Produced the following files:");
foreach (var coarseFile in coarseFiles)
{
output.Info.WriteLine($"Generated coarse file: {coarseFile}");
}
}
output.Info.WriteLine("Press any key to exit...");
Console.ReadKey();
}
public static void RandomDataGenerator(
string startDateString,
string endDateString,
string symbolCountString,
string market,
string securityTypeString,
string resolutionString,
string dataDensityString,
string includeCoarseString,
string quoteTradeRatioString,
string randomSeed,
string hasIpoPercentageString,
string hasRenamePercentageString,
string hasSplitsPercentageString,
string hasDividendsPercentageString,
string dividendEveryQuarterPercentageString
)
{
var output = new ConsoleLeveledOutput();
var settings = RandomDataGeneratorSettings.FromCommandLineArguments(
startDateString,
endDateString,
symbolCountString,
market,
securityTypeString,
resolutionString,
dataDensityString,
includeCoarseString,
quoteTradeRatioString,
randomSeed,
hasIpoPercentageString,
hasRenamePercentageString,
hasSplitsPercentageString,
hasDividendsPercentageString,
dividendEveryQuarterPercentageString,
output
);
GenerateRandomData(settings, output);
if (settings.IncludeCoarse && settings.SecurityType == SecurityType.Equity)
{
output.Info.WriteLine("Launching coarse data generator...");
CoarseUniverseGeneratorProgram.CoarseUniverseGenerator();
}
output.Info.WriteLine("Press any key to exit...");
Console.ReadKey();
}
public static void GenerateRandomData(RandomDataGeneratorSettings settings, ConsoleLeveledOutput output)
{
// can specify a seed value in this ctor if determinism is desired
var random = new Random();
var randomValueGenerator = new RandomValueGenerator();
if (settings.RandomSeedSet)
{
random = new Random(settings.RandomSeed);
randomValueGenerator = new RandomValueGenerator(settings.RandomSeed);
}
var symbolGenerator = new SymbolGenerator(settings, randomValueGenerator);
var tickGenerator = new TickGenerator(settings, randomValueGenerator);
output.Warn.WriteLine($"Begin data generation of {settings.SymbolCount} randomly generated {settings.SecurityType} assets...");
// iterate over our randomly generated symbols
var count = 0;
var progress = 0d;
var previousMonth = -1;
Func <Tick, DateTime> tickDay = (tick => new DateTime(tick.Time.Year, tick.Time.Month, tick.Time.Day));
Func <Data.BaseData, DateTime> dataDay = (data => new DateTime(data.Time.Year, data.Time.Month, data.Time.Day));
foreach (var currentSymbol in symbolGenerator.GenerateRandomSymbols())
{
// This is done so that we can update the symbol in the case of a rename event
var delistDate = GetDelistingDate(settings.Start, settings.End, randomValueGenerator);
var symbol = currentSymbol;
var willBeDelisted = randomValueGenerator.NextBool(1.0);
var monthsTrading = 0;
// Keep track of renamed symbols and the time they were renamed.
var renamedSymbols = new Dictionary <Symbol, DateTime>();
output.Warn.WriteLine($"\tSymbol[{++count}]: {symbol} Progress: {progress:0.0}% - Generating data...");
// define aggregators via settings
var aggregators = settings.CreateAggregators().ToList();
var tickHistory = tickGenerator.GenerateTicks(symbol).ToList();
var dividendsSplitsMaps = new DividendSplitMapGenerator(
symbol,
settings,
randomValueGenerator,
random,
delistDate,
willBeDelisted);
if (settings.SecurityType == SecurityType.Equity)
{
dividendsSplitsMaps.GenerateSplitsDividends(tickHistory);
if (!willBeDelisted)
{
dividendsSplitsMaps.DividendsSplits.Add(new FactorFileRow(new DateTime(2050, 12, 31), 1m, 1m));
if (dividendsSplitsMaps.MapRows.Count > 1)
{
// Remove the last element if we're going to have a 20501231 entry
dividendsSplitsMaps.MapRows.RemoveAt(dividendsSplitsMaps.MapRows.Count - 1);
}
dividendsSplitsMaps.MapRows.Add(new MapFileRow(new DateTime(2050, 12, 31), dividendsSplitsMaps.CurrentSymbol.Value));
}
// If the symbol value has changed, update the current symbol
if (symbol != dividendsSplitsMaps.CurrentSymbol)
{
// Add all symbol rename events to dictionary
foreach (var renameEvent in dividendsSplitsMaps.MapRows)
{
// Symbol.UpdateMappedSymbol does not update the underlying security ID symbol, which
// is used to create the hash code. Create a new equity symbol from scratch instead.
symbol = Symbol.Create(renameEvent.MappedSymbol, SecurityType.Equity, settings.Market);
renamedSymbols.Add(symbol, renameEvent.Date);
output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} will be renamed on {renameEvent.Date}");
}
}
else
{
// This ensures that ticks will be written for the current symbol up until 9999-12-31
renamedSymbols.Add(symbol, new DateTime(9999, 12, 31));
}
symbol = dividendsSplitsMaps.CurrentSymbol;
// Write Splits and Dividend events to directory factor_files
var factorFile = new FactorFile(symbol.Value, dividendsSplitsMaps.DividendsSplits, settings.Start);
var mapFile = new MapFile(symbol.Value, dividendsSplitsMaps.MapRows);
factorFile.WriteToCsv(symbol);
mapFile.WriteToCsv(settings.Market);
output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} Dividends, splits, and map files have been written to disk.");
}
Symbol previousSymbol = null;
var currentCount = 0;
foreach (var renamed in renamedSymbols)
{
var previousRenameDate = previousSymbol == null ? new DateTime(1, 1, 1) : renamedSymbols[previousSymbol];
var previousRenameDateDay = new DateTime(previousRenameDate.Year, previousRenameDate.Month, previousRenameDate.Day);
var renameDate = renamed.Value;
var renameDateDay = new DateTime(renameDate.Year, renameDate.Month, renameDate.Day);
foreach (var tick in tickHistory.Where(tick => tick.Time >= previousRenameDate && previousRenameDateDay != tickDay(tick)))
{
// Prevents the aggregator from being updated with ticks after the rename event
if (tickDay(tick) > renameDateDay)
{
break;
}
if (tick.Time.Month != previousMonth)
{
output.Info.WriteLine($"\tSymbol[{count}]: Month: {tick.Time:MMMM}");
previousMonth = tick.Time.Month;
monthsTrading++;
}
foreach (var item in aggregators)
{
tick.Value = tick.Value / dividendsSplitsMaps.FinalSplitFactor;
item.Consolidator.Update(tick);
}
if (monthsTrading >= 6 && willBeDelisted && tick.Time > delistDate)
{
output.Warn.WriteLine($"\tSymbol[{count}]: {renamed.Key} delisted at {tick.Time:MMMM yyyy}");
break;
}
}
// count each stage as a point, so total points is 2*symbol-count
// and the current progress is twice the current, but less one because we haven't finished writing data yet
progress = 100 * (2 * count - 1) / (2.0 * settings.SymbolCount);
output.Warn.WriteLine($"\tSymbol[{count}]: {renamed.Key} Progress: {progress:0.0}% - Saving data in LEAN format");
// persist consolidated data to disk
foreach (var item in aggregators)
{
var writer = new LeanDataWriter(item.Resolution, renamed.Key, Globals.DataFolder, item.TickType);
// send the flushed data into the writer. pulling the flushed list is very important,
// lest we likely wouldn't get the last piece of data stuck in the consolidator
// Filter out the data we're going to write here because filtering them in the consolidator update phase
// makes it write all dates for some unknown reason
writer.Write(item.Flush().Where(data => data.Time > previousRenameDate && previousRenameDateDay != dataDay(data)));
}
// update progress
progress = 100 * (2 * count) / (2.0 * settings.SymbolCount);
output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} Progress: {progress:0.0}% - Symbol data generation and output completed");
previousSymbol = renamed.Key;
currentCount++;
}
}
output.Info.WriteLine("Random data generation has completed.");
}
public static RandomDataGeneratorSettings FromCommandLineArguments(
string startDateString,
string endDateString,
string symbolCountString,
string market,
string securityTypeString,
string resolutionString,
string dataDensityString,
string includeCoarseString,
string quoteTradeRatioString,
string randomSeedString,
string hasIpoPercentageString,
string hasRenamePercentageString,
string hasSplitsPercentageString,
string hasDividendsPercentageString,
string dividendEveryQuarterPercentageString,
ConsoleLeveledOutput output
)
{
bool randomSeedSet = true;
int randomSeed;
int symbolCount;
bool includeCoarse;
TickType[] tickTypes;
Resolution resolution;
double quoteTradeRatio;
DataDensity dataDensity;
SecurityType securityType;
DateTime startDate, endDate;
double hasIpoPercentage;
double hasRenamePercentage;
double hasSplitsPercentage;
double hasDividendsPercentage;
double dividendEveryQuarterPercentage;
// --start
if (!DateTime.TryParseExact(startDateString, DateFormats, null, DateTimeStyles.None, out startDate))
{
output.Error.WriteLine($"Required parameter --from-date was incorrectly formatted. Please specify in yyyyMMdd format. Value provided: '{startDateString}'");
}
// --end
if (!DateTime.TryParseExact(endDateString, DateFormats, null, DateTimeStyles.None, out endDate))
{
output.Error.WriteLine($"Required parameter --to-date was incorrectly formatted. Please specify in yyyyMMdd format. Value provided: '{endDateString}'");
}
// --symbol-count
if (!int.TryParse(symbolCountString, out symbolCount) || symbolCount <= 0)
{
output.Error.WriteLine($"Required parameter --symbol-count was incorrectly formatted. Please specify a valid integer greater than zero. Value provided: '{symbolCountString}'");
}
// --resolution
if (string.IsNullOrEmpty(resolutionString))
{
resolution = Resolution.Minute;
output.Info.WriteLine($"Using default value of '{resolution}' for --resolution");
}
else if (!Enum.TryParse(resolutionString, true, out resolution))
{
var validValues = string.Join(", ", Enum.GetValues(typeof(Resolution)).Cast <Resolution>());
output.Error.WriteLine($"Optional parameter --resolution was incorrectly formatted. Default is Minute. Please specify a valid Resolution. Value provided: '{resolutionString}' Valid values: {validValues}");
}
// --security-type
if (string.IsNullOrEmpty(securityTypeString))
{
securityType = SecurityType.Equity;
output.Info.WriteLine($"Using default value of '{securityType}' for --security-type");
}
else if (!Enum.TryParse(securityTypeString, true, out securityType))
{
var validValues = string.Join(", ", Enum.GetValues(typeof(SecurityType)).Cast <SecurityType>());
output.Error.WriteLine($"Optional parameter --security-type is invalid. Default is Equity. Please specify a valid SecurityType. Value provided: '{securityTypeString}' Valid values: {validValues}");
}
if (securityType == SecurityType.Option && resolution != Resolution.Minute)
{
output.Error.WriteLine($"When using --security-type=Option you must specify --resolution=Minute");
}
// --market
if (string.IsNullOrEmpty(market))
{
market = DefaultBrokerageModel.DefaultMarketMap[securityType];
output.Info.WriteLine($"Using default value of '{market}' for --market and --security-type={securityType}");
}
else if (QuantConnect.Market.Encode(market) == null)
{
// be sure to add a reference to the unknown market, otherwise we won't be able to decode it coming out
QuantConnect.Market.Add(market, Interlocked.Increment(ref MarketCode));
output.Warn.WriteLine($"Please verify that the specified market value is correct: '{market}' This value is not known has been added to the market value map. If this is an error, stop the application immediately using Ctrl+C");
}
// --include-coarse
if (string.IsNullOrEmpty(includeCoarseString))
{
includeCoarse = securityType == SecurityType.Equity;
if (securityType != SecurityType.Equity)
{
output.Info.WriteLine($"Using default value of '{includeCoarse}' for --security-type={securityType}");
}
}
else if (!bool.TryParse(includeCoarseString, out includeCoarse))
{
output.Error.WriteLine($"Optional parameter --include-coarse was incorrectly formated. Please specify a valid boolean. Value provided: '{includeCoarseString}'. Valid values: 'true' or 'false'");
}
else if (includeCoarse && securityType != SecurityType.Equity)
{
output.Warn.WriteLine("Optional parameter --include-coarse will be ignored because it only applies to --security-type=Equity");
}
// --data-density
if (string.IsNullOrEmpty(dataDensityString))
{
dataDensity = DataDensity.Dense;
output.Info.WriteLine($"Using default value of '{dataDensity}' for --data-density");
}
else if (!Enum.TryParse(dataDensityString, true, out dataDensity))
{
var validValues = string.Join(", ", Enum.GetValues(typeof(DataDensity))).Cast <DataDensity>();
output.Error.WriteLine($"Optional parameter --data-density was incorrectly formated. Please specify a valid DataDensity. Value provided: '{dataDensityString}'. Valid values: {validValues}");
}
// --quote-trade-ratio
if (string.IsNullOrEmpty(quoteTradeRatioString))
{
quoteTradeRatio = 1;
}
else if (!double.TryParse(quoteTradeRatioString, out quoteTradeRatio))
{
output.Error.WriteLine($"Optional parameter --quote-trade-ratio was incorrectly formatted. Please specify a valid double greater than or equal to zero. Value provided: '{quoteTradeRatioString}'");
}
// --random-seed
if (string.IsNullOrEmpty(randomSeedString))
{
randomSeed = 0;
randomSeedSet = false;
}
else if (!int.TryParse(randomSeedString, out randomSeed))
{
output.Error.WriteLine($"Optional parameter --random-seed was incorrectly formatted. Please specify a valid integer");
}
// --ipo-percentage
if (string.IsNullOrEmpty(hasIpoPercentageString))
{
hasIpoPercentage = 5.0;
}
else if (!double.TryParse(hasIpoPercentageString, out hasIpoPercentage))
{
output.Error.WriteLine($"Optional parameter --ipo-percentage was incorrectly formatted. Please specify a valid double greater than or equal to zero. Value provided: '{hasIpoPercentageString}'");
}
// --rename-percentage
if (string.IsNullOrEmpty(hasRenamePercentageString))
{
hasRenamePercentage = 30.0;
}
else if (!double.TryParse(hasRenamePercentageString, out hasRenamePercentage))
{
output.Error.WriteLine($"Optional parameter --rename-percentage was incorrectly formatted. Please specify a valid double greater than or equal to zero. Value provided: '{hasRenamePercentageString}'");
}
// --splits-percentage
if (string.IsNullOrEmpty(hasSplitsPercentageString))
{
hasSplitsPercentage = 15.0;
}
else if (!double.TryParse(hasSplitsPercentageString, out hasSplitsPercentage))
{
output.Error.WriteLine($"Optional parameter --splits-percentage was incorrectly formatted. Please specify a valid double greater than or equal to zero. Value provided: '{hasSplitsPercentageString}'");
}
// --dividends-percentage
if (string.IsNullOrEmpty(hasDividendsPercentageString))
{
hasDividendsPercentage = 60.0;
}
else if (!double.TryParse(hasDividendsPercentageString, out hasDividendsPercentage))
{
output.Error.WriteLine($"Optional parameter --dividends-percentage was incorrectly formatted. Please specify a valid double greater than or equal to zero. Value provided: '{hasDividendsPercentageString}'");
}
// --dividend-every-quarter-percentage
if (string.IsNullOrEmpty(dividendEveryQuarterPercentageString))
{
dividendEveryQuarterPercentage = 30.0;
}
else if (!double.TryParse(dividendEveryQuarterPercentageString, out dividendEveryQuarterPercentage))
{
output.Error.WriteLine($"Optional parameter --dividend-ever-quarter-percentage was incorrectly formatted. Please specify a valid double greater than or equal to zero. Value provided: '{dividendEveryQuarterPercentageString}'");
}
if (output.ErrorMessageWritten)
{
output.Error.WriteLine("Please address the errors and run the application again.");
Environment.Exit(-1);
}
switch (securityType)
{
case SecurityType.Base:
case SecurityType.Equity:
tickTypes = new[] { TickType.Trade };
break;
case SecurityType.Forex:
case SecurityType.Cfd:
tickTypes = new[] { TickType.Quote };
break;
case SecurityType.Option:
case SecurityType.Future:
tickTypes = new[] { TickType.Trade, TickType.Quote, TickType.OpenInterest };
break;
case SecurityType.Crypto:
tickTypes = new[] { TickType.Trade, TickType.Quote };
break;
case SecurityType.Commodity:
default:
throw new ArgumentOutOfRangeException();
}
output.Info.WriteLine($"Selected tick types for {securityType}: {string.Join(", ", tickTypes)}");
return(new RandomDataGeneratorSettings
{
End = endDate,
Start = startDate,
Market = market,
SymbolCount = symbolCount,
SecurityType = securityType,
QuoteTradeRatio = quoteTradeRatio,
TickTypes = tickTypes,
Resolution = resolution,
DataDensity = dataDensity,
IncludeCoarse = includeCoarse,
RandomSeed = randomSeed,
RandomSeedSet = randomSeedSet,
HasIpoPercentage = hasIpoPercentage,
HasRenamePercentage = hasRenamePercentage,
HasSplitsPercentage = hasSplitsPercentage,
HasDividendsPercentage = hasDividendsPercentage,
DividendEveryQuarterPercentage = dividendEveryQuarterPercentage
});
}