public static void GenerateRandomData(RandomDataGeneratorSettings settings, ConsoleLeveledOutput output)
{
// can specify a seed value in this ctor if determinism is desired
var random = new Random();
var randomValueGenerator = new RandomValueGenerator();
if (settings.RandomSeedSet)
{
random = new Random(settings.RandomSeed);
randomValueGenerator = new RandomValueGenerator(settings.RandomSeed);
}
var symbolGenerator = new SymbolGenerator(settings, randomValueGenerator);
var tickGenerator = new TickGenerator(settings, randomValueGenerator);
output.Warn.WriteLine($"Begin data generation of {settings.SymbolCount} randomly generated {settings.SecurityType} assets...");
// iterate over our randomly generated symbols
var count = 0;
var progress = 0d;
var previousMonth = -1;
Func <Tick, DateTime> tickDay = (tick => new DateTime(tick.Time.Year, tick.Time.Month, tick.Time.Day));
Func <Data.BaseData, DateTime> dataDay = (data => new DateTime(data.Time.Year, data.Time.Month, data.Time.Day));
foreach (var currentSymbol in symbolGenerator.GenerateRandomSymbols())
{
// This is done so that we can update the symbol in the case of a rename event
var delistDate = GetDelistingDate(settings.Start, settings.End, randomValueGenerator);
var symbol = currentSymbol;
var willBeDelisted = randomValueGenerator.NextBool(1.0);
var monthsTrading = 0;
// Keep track of renamed symbols and the time they were renamed.
var renamedSymbols = new Dictionary <Symbol, DateTime>();
output.Warn.WriteLine($"\tSymbol[{++count}]: {symbol} Progress: {progress:0.0}% - Generating data...");
// define aggregators via settings
var aggregators = settings.CreateAggregators().ToList();
var tickHistory = tickGenerator.GenerateTicks(symbol).ToList();
var dividendsSplitsMaps = new DividendSplitMapGenerator(
symbol,
settings,
randomValueGenerator,
random,
delistDate,
willBeDelisted);
if (settings.SecurityType == SecurityType.Equity)
{
dividendsSplitsMaps.GenerateSplitsDividends(tickHistory);
if (!willBeDelisted)
{
dividendsSplitsMaps.DividendsSplits.Add(new FactorFileRow(new DateTime(2050, 12, 31), 1m, 1m));
if (dividendsSplitsMaps.MapRows.Count > 1)
{
// Remove the last element if we're going to have a 20501231 entry
dividendsSplitsMaps.MapRows.RemoveAt(dividendsSplitsMaps.MapRows.Count - 1);
}
dividendsSplitsMaps.MapRows.Add(new MapFileRow(new DateTime(2050, 12, 31), dividendsSplitsMaps.CurrentSymbol.Value));
}
// If the symbol value has changed, update the current symbol
if (symbol != dividendsSplitsMaps.CurrentSymbol)
{
// Add all symbol rename events to dictionary
foreach (var renameEvent in dividendsSplitsMaps.MapRows)
{
// Symbol.UpdateMappedSymbol does not update the underlying security ID symbol, which
// is used to create the hash code. Create a new equity symbol from scratch instead.
symbol = Symbol.Create(renameEvent.MappedSymbol, SecurityType.Equity, settings.Market);
renamedSymbols.Add(symbol, renameEvent.Date);
output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} will be renamed on {renameEvent.Date}");
}
}
else
{
// This ensures that ticks will be written for the current symbol up until 9999-12-31
renamedSymbols.Add(symbol, new DateTime(9999, 12, 31));
}
symbol = dividendsSplitsMaps.CurrentSymbol;
// Write Splits and Dividend events to directory factor_files
var factorFile = new FactorFile(symbol.Value, dividendsSplitsMaps.DividendsSplits, settings.Start);
var mapFile = new MapFile(symbol.Value, dividendsSplitsMaps.MapRows);
factorFile.WriteToCsv(symbol);
mapFile.WriteToCsv(settings.Market);
output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} Dividends, splits, and map files have been written to disk.");
}
Symbol previousSymbol = null;
var currentCount = 0;
foreach (var renamed in renamedSymbols)
{
var previousRenameDate = previousSymbol == null ? new DateTime(1, 1, 1) : renamedSymbols[previousSymbol];
var previousRenameDateDay = new DateTime(previousRenameDate.Year, previousRenameDate.Month, previousRenameDate.Day);
var renameDate = renamed.Value;
var renameDateDay = new DateTime(renameDate.Year, renameDate.Month, renameDate.Day);
foreach (var tick in tickHistory.Where(tick => tick.Time >= previousRenameDate && previousRenameDateDay != tickDay(tick)))
{
// Prevents the aggregator from being updated with ticks after the rename event
if (tickDay(tick) > renameDateDay)
{
break;
}
if (tick.Time.Month != previousMonth)
{
output.Info.WriteLine($"\tSymbol[{count}]: Month: {tick.Time:MMMM}");
previousMonth = tick.Time.Month;
monthsTrading++;
}
foreach (var item in aggregators)
{
tick.Value = tick.Value / dividendsSplitsMaps.FinalSplitFactor;
item.Consolidator.Update(tick);
}
if (monthsTrading >= 6 && willBeDelisted && tick.Time > delistDate)
{
output.Warn.WriteLine($"\tSymbol[{count}]: {renamed.Key} delisted at {tick.Time:MMMM yyyy}");
break;
}
}
// count each stage as a point, so total points is 2*symbol-count
// and the current progress is twice the current, but less one because we haven't finished writing data yet
progress = 100 * (2 * count - 1) / (2.0 * settings.SymbolCount);
output.Warn.WriteLine($"\tSymbol[{count}]: {renamed.Key} Progress: {progress:0.0}% - Saving data in LEAN format");
// persist consolidated data to disk
foreach (var item in aggregators)
{
var writer = new LeanDataWriter(item.Resolution, renamed.Key, Globals.DataFolder, item.TickType);
// send the flushed data into the writer. pulling the flushed list is very important,
// lest we likely wouldn't get the last piece of data stuck in the consolidator
// Filter out the data we're going to write here because filtering them in the consolidator update phase
// makes it write all dates for some unknown reason
writer.Write(item.Flush().Where(data => data.Time > previousRenameDate && previousRenameDateDay != dataDay(data)));
}
// update progress
progress = 100 * (2 * count) / (2.0 * settings.SymbolCount);
output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} Progress: {progress:0.0}% - Symbol data generation and output completed");
previousSymbol = renamed.Key;
currentCount++;
}
}
output.Info.WriteLine("Random data generation has completed.");
}
