Esempio n. 1
0
        public static void RandomDataGenerator(
            string startDateString,
            string endDateString,
            string symbolCountString,
            string market,
            string securityTypeString,
            string resolutionString,
            string dataDensityString,
            string includeCoarseString,
            string quoteTradeRatioString,
            string randomSeed,
            string hasIpoPercentageString,
            string hasRenamePercentageString,
            string hasSplitsPercentageString,
            string hasDividendsPercentageString,
            string dividendEveryQuarterPercentageString
            )
        {
            var output   = new ConsoleLeveledOutput();
            var settings = RandomDataGeneratorSettings.FromCommandLineArguments(
                startDateString,
                endDateString,
                symbolCountString,
                market,
                securityTypeString,
                resolutionString,
                dataDensityString,
                includeCoarseString,
                quoteTradeRatioString,
                randomSeed,
                hasIpoPercentageString,
                hasRenamePercentageString,
                hasSplitsPercentageString,
                hasDividendsPercentageString,
                dividendEveryQuarterPercentageString,

                output
                );

            if (settings.Start.Year < 1998)
            {
                output.Error.WriteLine($"Required parameter --start must be at least 19980101");
                Environment.Exit(1);
            }

            GenerateRandomData(settings, output);

            if (settings.IncludeCoarse && settings.SecurityType == SecurityType.Equity)
            {
                output.Info.WriteLine("Launching coarse data generator...");

                CoarseUniverseGeneratorProgram.CoarseUniverseGenerator();
            }

            if (!Console.IsInputRedirected)
            {
                output.Info.WriteLine("Press any key to exit...");
                Console.ReadKey();
            }
        }
Esempio n. 2
0
        public static void GenerateRandomData(RandomDataGeneratorSettings settings, ConsoleLeveledOutput output)
        {
            // can specify a seed value in this ctor if determinism is desired
            var randomValueGenerator = new RandomValueGenerator();
            var symbolGenerator      = new SymbolGenerator(settings, randomValueGenerator);
            var tickGenerator        = new TickGenerator(settings, randomValueGenerator);

            output.Warn.WriteLine($"Begin data generation of {settings.SymbolCount} randomly generated {settings.SecurityType} assets...");

            // iterate over our randomly generated symbols
            var count         = 0;
            var progress      = 0d;
            var previousMonth = -1;

            foreach (var symbol in symbolGenerator.GenerateRandomSymbols())
            {
                output.Warn.WriteLine($"\tSymbol[{++count}]: {symbol} Progress: {progress:0.0}% - Generating data...");

                // define aggregators via settings
                var aggregators = settings.CreateAggregators().ToList();

                // generate and consolidate data
                foreach (var tick in tickGenerator.GenerateTicks(symbol))
                {
                    if (tick.Time.Month != previousMonth)
                    {
                        output.Info.WriteLine($"\tMonth: {tick.Time:MMMM}");
                        previousMonth = tick.Time.Month;
                    }

                    foreach (var item in aggregators)
                    {
                        item.Consolidator.Update(tick);
                    }
                }

                // count each stage as a point, so total points is 2*symbol-count
                // and the current progress is twice the current, but less one because we haven't finished writing data yet
                progress = 100 * (2 * count - 1) / (2.0 * settings.SymbolCount);
                output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} Progress: {progress:0.0}% - Saving data in LEAN format");

                // persist consolidated data to disk
                foreach (var item in aggregators)
                {
                    var writer = new LeanDataWriter(item.Resolution, symbol, Globals.DataFolder, item.TickType);

                    // send the flushed data into the writer. pulling the flushed list is very important,
                    // lest we likely wouldn't get the last piece of data stuck in the consolidator
                    writer.Write(item.Flush());
                }

                // update progress
                progress = 100 * (2 * count) / (2.0 * settings.SymbolCount);
                output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} Progress: {progress:0.0}% - Symbol data generation and output completed");
            }

            output.Info.WriteLine("Random data generation has completed.");
        }
        public static void RandomDataGenerator(
            string startDateString,
            string endDateString,
            string symbolCountString,
            string market,
            string securityTypeString,
            string resolutionString,
            string dataDensityString,
            string includeCoarseString,
            string quoteTradeRatioString,
            string randomSeed,
            string hasIpoPercentageString,
            string hasRenamePercentageString,
            string hasSplitsPercentageString,
            string hasDividendsPercentageString,
            string dividendEveryQuarterPercentageString
            )
        {
            var output   = new ConsoleLeveledOutput();
            var settings = RandomDataGeneratorSettings.FromCommandLineArguments(
                startDateString,
                endDateString,
                symbolCountString,
                market,
                securityTypeString,
                resolutionString,
                dataDensityString,
                includeCoarseString,
                quoteTradeRatioString,
                randomSeed,
                hasIpoPercentageString,
                hasRenamePercentageString,
                hasSplitsPercentageString,
                hasDividendsPercentageString,
                dividendEveryQuarterPercentageString,

                output
                );

            GenerateRandomData(settings, output);

            if (settings.IncludeCoarse && settings.SecurityType == SecurityType.Equity)
            {
                output.Info.WriteLine("Launching coarse data generator...");
                var coarseFiles = CoarseUniverseGenerator.CoarseUniverseGeneratorProgram.ProcessEquityDirectories(
                    Globals.DataFolder,
                    false
                    );
                output.Info.WriteLine("Coarse data generation completed. Produced the following files:");
                foreach (var coarseFile in coarseFiles)
                {
                    output.Info.WriteLine($"Generated coarse file: {coarseFile}");
                }
            }

            output.Info.WriteLine("Press any key to exit...");
            Console.ReadKey();
        }
Esempio n. 4
0
        public static void RandomDataGenerator(
            string startDateString,
            string endDateString,
            string symbolCountString,
            string market,
            string securityTypeString,
            string resolutionString,
            string dataDensityString,
            string includeCoarseString,
            string quoteTradeRatioString,
            string randomSeed,
            string hasIpoPercentageString,
            string hasRenamePercentageString,
            string hasSplitsPercentageString,
            string hasDividendsPercentageString,
            string dividendEveryQuarterPercentageString
            )
        {
            var output   = new ConsoleLeveledOutput();
            var settings = RandomDataGeneratorSettings.FromCommandLineArguments(
                startDateString,
                endDateString,
                symbolCountString,
                market,
                securityTypeString,
                resolutionString,
                dataDensityString,
                includeCoarseString,
                quoteTradeRatioString,
                randomSeed,
                hasIpoPercentageString,
                hasRenamePercentageString,
                hasSplitsPercentageString,
                hasDividendsPercentageString,
                dividendEveryQuarterPercentageString,

                output
                );

            GenerateRandomData(settings, output);

            if (settings.IncludeCoarse && settings.SecurityType == SecurityType.Equity)
            {
                output.Info.WriteLine("Launching coarse data generator...");

                CoarseUniverseGeneratorProgram.CoarseUniverseGenerator();
            }

            output.Info.WriteLine("Press any key to exit...");
            Console.ReadKey();
        }
Esempio n. 5
0
        public static void GenerateRandomData(RandomDataGeneratorSettings settings, ConsoleLeveledOutput output)
        {
            // can specify a seed value in this ctor if determinism is desired
            var random = new Random();
            var randomValueGenerator = new RandomValueGenerator();

            if (settings.RandomSeedSet)
            {
                random = new Random(settings.RandomSeed);
                randomValueGenerator = new RandomValueGenerator(settings.RandomSeed);
            }
            var symbolGenerator = new SymbolGenerator(settings, randomValueGenerator);
            var tickGenerator   = new TickGenerator(settings, randomValueGenerator);

            output.Warn.WriteLine($"Begin data generation of {settings.SymbolCount} randomly generated {settings.SecurityType} assets...");

            // iterate over our randomly generated symbols
            var count         = 0;
            var progress      = 0d;
            var previousMonth = -1;

            Func <Tick, DateTime>          tickDay = (tick => new DateTime(tick.Time.Year, tick.Time.Month, tick.Time.Day));
            Func <Data.BaseData, DateTime> dataDay = (data => new DateTime(data.Time.Year, data.Time.Month, data.Time.Day));

            foreach (var currentSymbol in symbolGenerator.GenerateRandomSymbols())
            {
                // This is done so that we can update the symbol in the case of a rename event
                var delistDate     = GetDelistingDate(settings.Start, settings.End, randomValueGenerator);
                var symbol         = currentSymbol;
                var willBeDelisted = randomValueGenerator.NextBool(1.0);
                var monthsTrading  = 0;

                // Keep track of renamed symbols and the time they were renamed.
                var renamedSymbols = new Dictionary <Symbol, DateTime>();

                output.Warn.WriteLine($"\tSymbol[{++count}]: {symbol} Progress: {progress:0.0}% - Generating data...");

                // define aggregators via settings
                var aggregators = settings.CreateAggregators().ToList();
                var tickHistory = tickGenerator.GenerateTicks(symbol).ToList();

                var dividendsSplitsMaps = new DividendSplitMapGenerator(
                    symbol,
                    settings,
                    randomValueGenerator,
                    random,
                    delistDate,
                    willBeDelisted);

                if (settings.SecurityType == SecurityType.Equity)
                {
                    dividendsSplitsMaps.GenerateSplitsDividends(tickHistory);

                    if (!willBeDelisted)
                    {
                        dividendsSplitsMaps.DividendsSplits.Add(new FactorFileRow(new DateTime(2050, 12, 31), 1m, 1m));

                        if (dividendsSplitsMaps.MapRows.Count > 1)
                        {
                            // Remove the last element if we're going to have a 20501231 entry
                            dividendsSplitsMaps.MapRows.RemoveAt(dividendsSplitsMaps.MapRows.Count - 1);
                        }
                        dividendsSplitsMaps.MapRows.Add(new MapFileRow(new DateTime(2050, 12, 31), dividendsSplitsMaps.CurrentSymbol.Value));
                    }

                    // If the symbol value has changed, update the current symbol
                    if (symbol != dividendsSplitsMaps.CurrentSymbol)
                    {
                        // Add all symbol rename events to dictionary
                        foreach (var renameEvent in dividendsSplitsMaps.MapRows)
                        {
                            // Symbol.UpdateMappedSymbol does not update the underlying security ID symbol, which
                            // is used to create the hash code. Create a new equity symbol from scratch instead.
                            symbol = Symbol.Create(renameEvent.MappedSymbol, SecurityType.Equity, settings.Market);
                            renamedSymbols.Add(symbol, renameEvent.Date);

                            output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} will be renamed on {renameEvent.Date}");
                        }
                    }
                    else
                    {
                        // This ensures that ticks will be written for the current symbol up until 9999-12-31
                        renamedSymbols.Add(symbol, new DateTime(9999, 12, 31));
                    }

                    symbol = dividendsSplitsMaps.CurrentSymbol;

                    // Write Splits and Dividend events to directory factor_files
                    var factorFile = new FactorFile(symbol.Value, dividendsSplitsMaps.DividendsSplits, settings.Start);
                    var mapFile    = new MapFile(symbol.Value, dividendsSplitsMaps.MapRows);

                    factorFile.WriteToCsv(symbol);
                    mapFile.WriteToCsv(settings.Market);

                    output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} Dividends, splits, and map files have been written to disk.");
                }

                Symbol previousSymbol = null;
                var    currentCount   = 0;

                foreach (var renamed in renamedSymbols)
                {
                    var previousRenameDate    = previousSymbol == null ? new DateTime(1, 1, 1) : renamedSymbols[previousSymbol];
                    var previousRenameDateDay = new DateTime(previousRenameDate.Year, previousRenameDate.Month, previousRenameDate.Day);
                    var renameDate            = renamed.Value;
                    var renameDateDay         = new DateTime(renameDate.Year, renameDate.Month, renameDate.Day);

                    foreach (var tick in tickHistory.Where(tick => tick.Time >= previousRenameDate && previousRenameDateDay != tickDay(tick)))
                    {
                        // Prevents the aggregator from being updated with ticks after the rename event
                        if (tickDay(tick) > renameDateDay)
                        {
                            break;
                        }

                        if (tick.Time.Month != previousMonth)
                        {
                            output.Info.WriteLine($"\tSymbol[{count}]: Month: {tick.Time:MMMM}");
                            previousMonth = tick.Time.Month;
                            monthsTrading++;
                        }

                        foreach (var item in aggregators)
                        {
                            tick.Value = tick.Value / dividendsSplitsMaps.FinalSplitFactor;
                            item.Consolidator.Update(tick);
                        }

                        if (monthsTrading >= 6 && willBeDelisted && tick.Time > delistDate)
                        {
                            output.Warn.WriteLine($"\tSymbol[{count}]: {renamed.Key} delisted at {tick.Time:MMMM yyyy}");
                            break;
                        }
                    }

                    // count each stage as a point, so total points is 2*symbol-count
                    // and the current progress is twice the current, but less one because we haven't finished writing data yet
                    progress = 100 * (2 * count - 1) / (2.0 * settings.SymbolCount);

                    output.Warn.WriteLine($"\tSymbol[{count}]: {renamed.Key} Progress: {progress:0.0}% - Saving data in LEAN format");

                    // persist consolidated data to disk
                    foreach (var item in aggregators)
                    {
                        var writer = new LeanDataWriter(item.Resolution, renamed.Key, Globals.DataFolder, item.TickType);

                        // send the flushed data into the writer. pulling the flushed list is very important,
                        // lest we likely wouldn't get the last piece of data stuck in the consolidator
                        // Filter out the data we're going to write here because filtering them in the consolidator update phase
                        // makes it write all dates for some unknown reason
                        writer.Write(item.Flush().Where(data => data.Time > previousRenameDate && previousRenameDateDay != dataDay(data)));
                    }

                    // update progress
                    progress = 100 * (2 * count) / (2.0 * settings.SymbolCount);
                    output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} Progress: {progress:0.0}% - Symbol data generation and output completed");

                    previousSymbol = renamed.Key;
                    currentCount++;
                }
            }

            output.Info.WriteLine("Random data generation has completed.");
        }
        public static RandomDataGeneratorSettings FromCommandLineArguments(
            string startDateString,
            string endDateString,
            string symbolCountString,
            string market,
            string securityTypeString,
            string resolutionString,
            string dataDensityString,
            string includeCoarseString,
            string quoteTradeRatioString,
            string randomSeedString,
            string hasIpoPercentageString,
            string hasRenamePercentageString,
            string hasSplitsPercentageString,
            string hasDividendsPercentageString,
            string dividendEveryQuarterPercentageString,
            ConsoleLeveledOutput output
            )
        {
            bool randomSeedSet = true;

            int  randomSeed;
            int  symbolCount;
            bool includeCoarse;

            TickType[]   tickTypes;
            Resolution   resolution;
            double       quoteTradeRatio;
            DataDensity  dataDensity;
            SecurityType securityType;
            DateTime     startDate, endDate;
            double       hasIpoPercentage;
            double       hasRenamePercentage;
            double       hasSplitsPercentage;
            double       hasDividendsPercentage;
            double       dividendEveryQuarterPercentage;

            // --start
            if (!DateTime.TryParseExact(startDateString, DateFormats, null, DateTimeStyles.None, out startDate))
            {
                output.Error.WriteLine($"Required parameter --from-date was incorrectly formatted. Please specify in yyyyMMdd format. Value provided: '{startDateString}'");
            }

            // --end
            if (!DateTime.TryParseExact(endDateString, DateFormats, null, DateTimeStyles.None, out endDate))
            {
                output.Error.WriteLine($"Required parameter --to-date was incorrectly formatted. Please specify in yyyyMMdd format. Value provided: '{endDateString}'");
            }

            // --symbol-count
            if (!int.TryParse(symbolCountString, out symbolCount) || symbolCount <= 0)
            {
                output.Error.WriteLine($"Required parameter --symbol-count was incorrectly formatted. Please specify a valid integer greater than zero. Value provided: '{symbolCountString}'");
            }

            // --resolution
            if (string.IsNullOrEmpty(resolutionString))
            {
                resolution = Resolution.Minute;
                output.Info.WriteLine($"Using default value of '{resolution}' for --resolution");
            }
            else if (!Enum.TryParse(resolutionString, true, out resolution))
            {
                var validValues = string.Join(", ", Enum.GetValues(typeof(Resolution)).Cast <Resolution>());
                output.Error.WriteLine($"Optional parameter --resolution was incorrectly formatted. Default is Minute. Please specify a valid Resolution. Value provided: '{resolutionString}' Valid values: {validValues}");
            }

            // --security-type
            if (string.IsNullOrEmpty(securityTypeString))
            {
                securityType = SecurityType.Equity;
                output.Info.WriteLine($"Using default value of '{securityType}' for --security-type");
            }
            else if (!Enum.TryParse(securityTypeString, true, out securityType))
            {
                var validValues = string.Join(", ", Enum.GetValues(typeof(SecurityType)).Cast <SecurityType>());
                output.Error.WriteLine($"Optional parameter --security-type is invalid. Default is Equity. Please specify a valid SecurityType. Value provided: '{securityTypeString}' Valid values: {validValues}");
            }

            if (securityType == SecurityType.Option && resolution != Resolution.Minute)
            {
                output.Error.WriteLine($"When using --security-type=Option you must specify --resolution=Minute");
            }

            // --market
            if (string.IsNullOrEmpty(market))
            {
                market = DefaultBrokerageModel.DefaultMarketMap[securityType];
                output.Info.WriteLine($"Using default value of '{market}' for --market and --security-type={securityType}");
            }
            else if (QuantConnect.Market.Encode(market) == null)
            {
                // be sure to add a reference to the unknown market, otherwise we won't be able to decode it coming out
                QuantConnect.Market.Add(market, Interlocked.Increment(ref MarketCode));
                output.Warn.WriteLine($"Please verify that the specified market value is correct: '{market}'   This value is not known has been added to the market value map. If this is an error, stop the application immediately using Ctrl+C");
            }

            // --include-coarse
            if (string.IsNullOrEmpty(includeCoarseString))
            {
                includeCoarse = securityType == SecurityType.Equity;
                if (securityType != SecurityType.Equity)
                {
                    output.Info.WriteLine($"Using default value of '{includeCoarse}' for --security-type={securityType}");
                }
            }
            else if (!bool.TryParse(includeCoarseString, out includeCoarse))
            {
                output.Error.WriteLine($"Optional parameter --include-coarse was incorrectly formated. Please specify a valid boolean. Value provided: '{includeCoarseString}'. Valid values: 'true' or 'false'");
            }
            else if (includeCoarse && securityType != SecurityType.Equity)
            {
                output.Warn.WriteLine("Optional parameter --include-coarse will be ignored because it only applies to --security-type=Equity");
            }

            // --data-density
            if (string.IsNullOrEmpty(dataDensityString))
            {
                dataDensity = DataDensity.Dense;
                output.Info.WriteLine($"Using default value of '{dataDensity}' for --data-density");
            }
            else if (!Enum.TryParse(dataDensityString, true, out dataDensity))
            {
                var validValues = string.Join(", ", Enum.GetValues(typeof(DataDensity))).Cast <DataDensity>();
                output.Error.WriteLine($"Optional parameter --data-density was incorrectly formated. Please specify a valid DataDensity. Value provided: '{dataDensityString}'. Valid values: {validValues}");
            }

            // --quote-trade-ratio
            if (string.IsNullOrEmpty(quoteTradeRatioString))
            {
                quoteTradeRatio = 1;
            }
            else if (!double.TryParse(quoteTradeRatioString, out quoteTradeRatio))
            {
                output.Error.WriteLine($"Optional parameter --quote-trade-ratio was incorrectly formatted. Please specify a valid double greater than or equal to zero. Value provided: '{quoteTradeRatioString}'");
            }

            // --random-seed
            if (string.IsNullOrEmpty(randomSeedString))
            {
                randomSeed    = 0;
                randomSeedSet = false;
            }
            else if (!int.TryParse(randomSeedString, out randomSeed))
            {
                output.Error.WriteLine($"Optional parameter --random-seed was incorrectly formatted. Please specify a valid integer");
            }

            // --ipo-percentage
            if (string.IsNullOrEmpty(hasIpoPercentageString))
            {
                hasIpoPercentage = 5.0;
            }
            else if (!double.TryParse(hasIpoPercentageString, out hasIpoPercentage))
            {
                output.Error.WriteLine($"Optional parameter --ipo-percentage was incorrectly formatted. Please specify a valid double greater than or equal to zero. Value provided: '{hasIpoPercentageString}'");
            }

            // --rename-percentage
            if (string.IsNullOrEmpty(hasRenamePercentageString))
            {
                hasRenamePercentage = 30.0;
            }
            else if (!double.TryParse(hasRenamePercentageString, out hasRenamePercentage))
            {
                output.Error.WriteLine($"Optional parameter --rename-percentage was incorrectly formatted. Please specify a valid double greater than or equal to zero. Value provided: '{hasRenamePercentageString}'");
            }

            // --splits-percentage
            if (string.IsNullOrEmpty(hasSplitsPercentageString))
            {
                hasSplitsPercentage = 15.0;
            }
            else if (!double.TryParse(hasSplitsPercentageString, out hasSplitsPercentage))
            {
                output.Error.WriteLine($"Optional parameter --splits-percentage was incorrectly formatted. Please specify a valid double greater than or equal to zero. Value provided: '{hasSplitsPercentageString}'");
            }

            // --dividends-percentage
            if (string.IsNullOrEmpty(hasDividendsPercentageString))
            {
                hasDividendsPercentage = 60.0;
            }
            else if (!double.TryParse(hasDividendsPercentageString, out hasDividendsPercentage))
            {
                output.Error.WriteLine($"Optional parameter --dividends-percentage was incorrectly formatted. Please specify a valid double greater than or equal to zero. Value provided: '{hasDividendsPercentageString}'");
            }

            // --dividend-every-quarter-percentage
            if (string.IsNullOrEmpty(dividendEveryQuarterPercentageString))
            {
                dividendEveryQuarterPercentage = 30.0;
            }
            else if (!double.TryParse(dividendEveryQuarterPercentageString, out dividendEveryQuarterPercentage))
            {
                output.Error.WriteLine($"Optional parameter --dividend-ever-quarter-percentage was incorrectly formatted. Please specify a valid double greater than or equal to zero. Value provided: '{dividendEveryQuarterPercentageString}'");
            }

            if (output.ErrorMessageWritten)
            {
                output.Error.WriteLine("Please address the errors and run the application again.");
                Environment.Exit(-1);
            }

            switch (securityType)
            {
            case SecurityType.Base:
            case SecurityType.Equity:
                tickTypes = new[] { TickType.Trade };
                break;

            case SecurityType.Forex:
            case SecurityType.Cfd:
                tickTypes = new[] { TickType.Quote };
                break;

            case SecurityType.Option:
            case SecurityType.Future:
                tickTypes = new[] { TickType.Trade, TickType.Quote, TickType.OpenInterest };
                break;

            case SecurityType.Crypto:
                tickTypes = new[] { TickType.Trade, TickType.Quote };
                break;

            case SecurityType.Commodity:
            default:
                throw new ArgumentOutOfRangeException();
            }

            output.Info.WriteLine($"Selected tick types for {securityType}: {string.Join(", ", tickTypes)}");

            return(new RandomDataGeneratorSettings
            {
                End = endDate,
                Start = startDate,

                Market = market,
                SymbolCount = symbolCount,
                SecurityType = securityType,
                QuoteTradeRatio = quoteTradeRatio,

                TickTypes = tickTypes,
                Resolution = resolution,

                DataDensity = dataDensity,
                IncludeCoarse = includeCoarse,
                RandomSeed = randomSeed,
                RandomSeedSet = randomSeedSet,

                HasIpoPercentage = hasIpoPercentage,
                HasRenamePercentage = hasRenamePercentage,
                HasSplitsPercentage = hasSplitsPercentage,
                HasDividendsPercentage = hasDividendsPercentage,
                DividendEveryQuarterPercentage = dividendEveryQuarterPercentage
            });
        }