/// <summary> /// QuantConnect Google Downloader For LEAN Algorithmic Trading Engine. /// Original by @chrisdk2015, tidied by @jaredbroad /// </summary> public static void Main(string[] args) { if (args.Length != 3) { Console.WriteLine("Usage: GoogleDownloader SYMBOL RESOLUTION PERIOD"); Console.WriteLine("SYMBOL = eg SPY"); Console.WriteLine("RESOLUTION = Minute/Hour"); Console.WriteLine("PERIOD = 10 for 10 days intraday data"); Environment.Exit(1); } try { // Load settings from command line var symbol = args[0]; var resolution = (Resolution)Enum.Parse(typeof(Resolution), args[1]); var period = args[2].ToInt32(); // Load settings from config.json var dataDirectory = Config.Get("data-directory", "../../../Data"); // Download the data var downloader = new GoogleDataDownloader(); var data = downloader.Get(new Symbol(symbol), SecurityType.Equity, resolution, DateTime.UtcNow.AddDays(-period), DateTime.UtcNow); // Save the data var writer = new LeanDataWriter(SecurityType.Equity, resolution, symbol, dataDirectory, "usa"); writer.Write(data); } catch (Exception err) { Log.Error("GoogleDownloader(): Error: " + err.Message); } }
public void LeanDataWriter_MultipleDays() { var leanDataWriter = new LeanDataWriter(Resolution.Second, _forex, _dataDirectory, TickType.Quote); var sourceData = new List <QuoteBar> { new (Parse.DateTime("3/16/2021 12:00:00 PM"), _forex, new Bar(1m, 2m, 3m, 4m), 1, new Bar(5m, 6m, 7m, 8m), 2) }; for (var i = 1; i < 100; i++) { sourceData.Add(new QuoteBar(sourceData.Last().Time.AddDays(1), _forex, new Bar(1m, 2m, 3m, 4m), 1, new Bar(5m, 6m, 7m, 8m), 2)); } leanDataWriter.Write(sourceData); foreach (var bar in sourceData) { var filePath = LeanData.GenerateZipFilePath(_dataDirectory, _forex, bar.Time, Resolution.Second, TickType.Quote); Assert.IsTrue(File.Exists(filePath)); Assert.IsFalse(File.Exists(filePath + ".tmp")); var data = QuantConnect.Compression.Unzip(filePath).Single(); Assert.AreEqual(1, data.Value.Count()); Assert.IsTrue(data.Key.Contains(bar.Time.ToStringInvariant(DateFormat.EightCharacter)), $"Key {data.Key} BarTime: {bar.Time}"); } }
/// <summary> /// Create ticks from Raw Kaiko data /// </summary> /// <param name="fileLocation">Path to the raw Kaiko data</param> /// <param name="market">The exchange the data represents</param> /// <param name="tickType">The tick type being processed</param> /// <param name="aggregateFunction">Function that parses the Kaiko data file and returns the enumerable of ticks</param> private static void CreateCryptoTicks(string fileLocation, string market, TickType tickType, Func <Symbol, string, IEnumerable <Tick> > aggregateFunction) { foreach (var symbolFolder in Directory.EnumerateDirectories(fileLocation)) { var symbolDirectoryInfo = new DirectoryInfo(symbolFolder); // Create symbol from folder name var symbol = Symbol.Create(symbolDirectoryInfo.Name, SecurityType.Crypto, market); foreach (var symbolMonthDirectory in Directory.EnumerateDirectories(symbolDirectoryInfo.FullName)) { foreach (var tradeFile in Directory.EnumerateFiles(symbolMonthDirectory)) { // Unzip file var unzippedFile = Compression.UnGZip(tradeFile, symbolMonthDirectory); // Write the ticks var writer = new LeanDataWriter(Resolution.Tick, symbol, Globals.DataFolder, tickType); writer.Write(aggregateFunction(symbol, unzippedFile)); // Clean up unzipped file File.Delete(unzippedFile); } } } }
/// <summary> /// Kraken Downloader Toolbox Project For LEAN Algorithmic Trading Engine. /// By @matthewsedam /// </summary> public static void KrakenDownloader(IList <string> tickers, string resolution, DateTime startDate, DateTime endDate) { if (resolution.IsNullOrEmpty() || tickers.IsNullOrEmpty()) { Console.WriteLine("KrakenDownloader ERROR: '--tickers=' or '--resolution=' parameter is missing"); Console.WriteLine("--tickers=eg XXBTZUSD,XETHZUSD"); Console.WriteLine("--resolution=Minute/Hour/Daily/Tick"); Environment.Exit(1); } try { var castResolution = (Resolution)Enum.Parse(typeof(Resolution), resolution); // Load settings from config.json and create downloader var dataDirectory = Config.Get("data-directory", "../../../Data"); var downloader = new KrakenDataDownloader(); foreach (var pair in tickers) { // Download data var pairObject = Symbol.Create(pair, SecurityType.Crypto, Market.Kraken); var data = downloader.Get(pairObject, castResolution, startDate, endDate); // Write data var writer = new LeanDataWriter(castResolution, pairObject, dataDirectory); writer.Write(data); } } catch (Exception err) { Log.Error(err); } }
/// <summary> /// QuantConnect Google Downloader For LEAN Algorithmic Trading Engine. /// Original by @chrisdk2015, tidied by @jaredbroad /// </summary> public static void Main(string[] args) { if (args.Length != 3) { Console.WriteLine("Usage: GoogleDownloader SYMBOL RESOLUTION PERIOD"); Console.WriteLine("SYMBOL = eg SPY"); Console.WriteLine("RESOLUTION = Minute/Hour"); Console.WriteLine("PERIOD = 10 for 10 days intraday data"); Environment.Exit(1); } try { // Load settings from command line var symbol = args[0]; var resolution = (Resolution)Enum.Parse(typeof(Resolution), args[1]); var period = args[2].ToInt32(); // Load settings from config.json var dataDirectory = Config.Get("data-directory", "../../../Data"); // Download the data var downloader = new GoogleDataDownloader(); var data = downloader.Get(new Symbol(symbol), SecurityType.Equity, resolution, DateTime.UtcNow.AddDays(-period), DateTime.UtcNow); // Save the data var writer = new LeanDataWriter(SecurityType.Equity, resolution, symbol, dataDirectory, "usa"); writer.Write(data); } catch (Exception err) { Log.Error("GoogleDownloader(): Error: " + err.Message); } }
/// <summary> /// Quandl Bitfinex Toolbox Project For LEAN Algorithmic Trading Engine. /// </summary> public static void QuandlBitfinexDownloader(DateTime fromDate, string apiKey) { if (apiKey.IsNullOrEmpty()) { Console.WriteLine("QuandlBitfinexDownloader ERROR: '--api-key=' parameter is missing"); Environment.Exit(1); } try { // Load settings from config.json var dataDirectory = Config.Get("data-directory", "../../../Data"); // Create an instance of the downloader const string market = Market.Bitfinex; var downloader = new QuandlBitfinexDownloader(apiKey); // Download the data var symbol = Symbol.Create("BTCUSD", SecurityType.Forex, market); var data = downloader.Get(symbol, Resolution.Daily, fromDate, DateTime.UtcNow); // Save the data var writer = new LeanDataWriter(Resolution.Daily, symbol, dataDirectory, TickType.Quote); writer.Write(data); } catch (Exception err) { Log.Error(err); } }
/// <summary> /// Quandl Bitfinex Toolbox Project For LEAN Algorithmic Trading Engine. /// </summary> static void Main(string[] args) { if (args.Length != 2) { Console.WriteLine("Usage: Downloader FROMDATE APIKEY"); Console.WriteLine("FROMDATE = yyyymmdd"); Environment.Exit(1); } try { // Load settings from config.json var dataDirectory = Config.Get("data-directory", "../../../Data"); var scaleFactor = Config.GetInt("bitfinex-scale-factor", 100); // Create an instance of the downloader const string market = Market.Bitfinex; var downloader = new QuandlBitfinexDownloader(args[1], scaleFactor); // Download the data var symbol = Symbol.Create("BTCUSD", SecurityType.Forex, market); var data = downloader.Get(symbol, Resolution.Daily, DateTime.ParseExact(args[0], "yyyyMMdd", CultureInfo.CurrentCulture), DateTime.UtcNow); // Save the data var writer = new LeanDataWriter(Resolution.Daily, symbol, dataDirectory, TickType.Quote); writer.Write(data); } catch (Exception err) { Log.Error(err); } }
/// <summary> /// Create ticks from Raw Kaiko data /// </summary> /// <param name="fileLocation">Path to the raw Kaiko data</param> /// <param name="market">The exchange the data represents</param> /// <param name="tickType">The tick type being processed</param> /// <param name="aggregateFunction">Function that parses the Kaiko data file and returns the enumerable of ticks</param> private static void CreateCryptoTicks(string fileLocation, string market, TickType tickType, Func <Symbol, string, IEnumerable <Tick> > aggregateFunction) { foreach (var symbolFolder in Directory.EnumerateDirectories(fileLocation)) { var symbolDirectoryInfo = new DirectoryInfo(symbolFolder); // Create symbol from folder name var symbol = Symbol.Create(symbolDirectoryInfo.Name, SecurityType.Crypto, market); foreach (var symbolMonthDirectory in Directory.EnumerateDirectories(symbolDirectoryInfo.FullName)) { foreach (var tradeFile in Directory.EnumerateFiles(symbolMonthDirectory, "*.gz")) { string unzippedFile; // Unzip file try { unzippedFile = Compression.UnGZip(tradeFile, symbolMonthDirectory); } catch (Exception e) { Log.Error($"KaikoDataConverter.CreateCryptoTicks(): File {tradeFile} cannot be unzipped. Exception {e}"); continue; } // Write the ticks var writer = new LeanDataWriter(Resolution.Tick, symbol, Globals.DataFolder, tickType); writer.Write(aggregateFunction(symbol, unzippedFile)); // Clean up unzipped file File.Delete(unzippedFile); } } } }
/// <summary> /// Quandl Bitfinex Toolbox Project For LEAN Algorithmic Trading Engine. /// </summary> static void Main(string[] args) { if (args.Length != 2) { Console.WriteLine("Usage: Downloader FROMDATE APIKEY"); Console.WriteLine("FROMDATE = yyyymmdd"); Environment.Exit(1); } try { // Load settings from config.json var dataDirectory = Config.Get("data-directory", "../../../Data"); var scaleFactor = Config.GetInt("bitfinex-scale-factor", 100); // Create an instance of the downloader const string market = Market.Bitfinex; var downloader = new QuandlBitfinexDownloader(args[1], scaleFactor); // Download the data var symbol = Symbol.Create("BTCUSD", SecurityType.Forex, market); var data = downloader.Get(symbol, Resolution.Daily, DateTime.ParseExact(args[0], "yyyyMMdd", CultureInfo.CurrentCulture), DateTime.UtcNow); // Save the data var writer = new LeanDataWriter(Resolution.Daily, symbol, dataDirectory, TickType.Quote); writer.Write(data); } catch (Exception err) { Log.Error(err); } }
/// <summary> /// Yahoo Downloader Toolbox Project For LEAN Algorithmic Trading Engine. /// Original by @chrisdk2015, tidied by @jaredbroad /// </summary> static void Main(string[] args) { if (args.Length != 1) { Console.WriteLine("Usage: YahooDownloader SYMBOL"); Console.WriteLine("Usage: Place the data into your LEAN Data directory: /data/equity/usa/daily/SYMBOL.zip"); Console.WriteLine("SYMBOL = eg SPY"); Environment.Exit(1); } //Command line inputs: symbol var symbol = args[0]; var dataDirectory = Config.Get("data-directory", "../../../Data"); try { //Get Yahoo Downloader: var yahooDownloader = new YahooDataDownloader(); var enumerableYahoo = yahooDownloader.Get(new Symbol(symbol), SecurityType.Equity, Resolution.Daily, DateTime.MinValue, DateTime.UtcNow); //Get LEAN Data Writer: var writer = new LeanDataWriter(SecurityType.Equity, Resolution.Daily, symbol, dataDirectory, "usa"); writer.Write(enumerableYahoo); } catch (Exception err) { Log.Error("YahooDownloader(): Error: " + err.Message); } }
/// <summary> /// Yahoo Downloader Toolbox Project For LEAN Algorithmic Trading Engine. /// Original by @chrisdk2015, tidied by @jaredbroad /// </summary> static void Main(string[] args) { if (args.Length != 1) { Console.WriteLine("Usage: YahooDownloader SYMBOL"); Console.WriteLine("Usage: Place the data into your LEAN Data directory: /data/equity/usa/daily/SYMBOL.zip"); Console.WriteLine("SYMBOL = eg SPY"); Environment.Exit(1); } //Command line inputs: symbol var symbol = args[0]; var dataDirectory = Config.Get("data-directory", "../../../Data"); try { //Get Yahoo Downloader: var yahooDownloader = new YahooDataDownloader(); var enumerableYahoo = yahooDownloader.Get(new Symbol(symbol), SecurityType.Equity, Resolution.Daily, DateTime.MinValue, DateTime.UtcNow); //Get LEAN Data Writer: var writer = new LeanDataWriter(SecurityType.Equity, Resolution.Daily, symbol, dataDirectory, "usa"); writer.Write(enumerableYahoo); } catch (Exception err) { Log.Error("YahooDownloader(): Error: " + err.Message); } }
/// <summary> /// Cryptoiq Downloader Toolbox Project For LEAN Algorithmic Trading Engine. /// </summary> public static void CryptoiqDownloader(IList <string> tickers, string exchange, DateTime startDate, DateTime endDate) { if (exchange.IsNullOrEmpty() || tickers.IsNullOrEmpty()) { Console.WriteLine("CryptoiqDownloader ERROR: '--exchange=' or '--tickers=' parameter is missing"); Environment.Exit(1); } try { // Load settings from config.json var dataDirectory = Config.Get("data-directory", "../../../Data"); //todo: will download any exchange but always save as gdax // Create an instance of the downloader const string market = Market.GDAX; var downloader = new CryptoiqDownloader(exchange); foreach (var ticker in tickers) { // Download the data var symbolObject = Symbol.Create(ticker, SecurityType.Crypto, market); var data = downloader.Get(symbolObject, Resolution.Tick, startDate, endDate); // Save the data var writer = new LeanDataWriter(Resolution.Tick, symbolObject, dataDirectory, TickType.Quote); writer.Write(data); } } catch (Exception err) { Log.Error(err); } }
/// <summary> /// Processes the entry. /// </summary> /// <param name="coinapiDataReader">The coinapi data reader.</param> /// <param name="file">The file.</param> private void ProcessEntry(CoinApiDataReader coinapiDataReader, FileInfo file) { var entryData = coinapiDataReader.GetCoinApiEntryData(file, _processingDate); if (!SupportedMarkets.Contains(entryData.Symbol.ID.Market)) { // only convert data for supported exchanges return; } // materialize the enumerable into a list, since we need to enumerate over it twice var ticks = coinapiDataReader.ProcessCoinApiEntry(entryData, file).OrderBy(t => t.Time).ToList(); var writer = new LeanDataWriter(Resolution.Tick, entryData.Symbol, _destinationFolder.FullName, entryData.TickType); writer.Write(ticks); Log.Trace($"CoinApiDataConverter(): Starting consolidation for {entryData.Symbol.Value} {entryData.TickType}"); var consolidators = new List <TickAggregator>(); if (entryData.TickType == TickType.Trade) { consolidators.AddRange(new[] { new TradeTickAggregator(Resolution.Second), new TradeTickAggregator(Resolution.Minute) }); } else { consolidators.AddRange(new[] { new QuoteTickAggregator(Resolution.Second), new QuoteTickAggregator(Resolution.Minute) }); } foreach (var tick in ticks) { if (tick.Suspicious) { // When CoinAPI loses connectivity to the exchange, they indicate // it in the data by providing a value of `-1` for bid/ask price. // We will keep it in tick data, but will remove it from consolidated data. continue; } foreach (var consolidator in consolidators) { consolidator.Update(tick); } } foreach (var consolidator in consolidators) { writer = new LeanDataWriter(consolidator.Resolution, entryData.Symbol, _destinationFolder.FullName, entryData.TickType); writer.Write(consolidator.Flush()); } }
public void RespectsWritePolicy(WritePolicy?writePolicy, Resolution resolution) { var filePath = LeanData.GenerateZipFilePath(_dataDirectory, _crypto, _date, resolution, TickType.Quote); if (File.Exists(filePath)) { File.Delete(filePath); } var loopCount = 3; var dataPointsPerLoop = 2; for (var i = 0; i < loopCount; i++) { var leanDataWriter = new LeanDataWriter(resolution, _crypto, _dataDirectory, TickType.Quote, writePolicy: writePolicy); var quoteBar = new QuoteBar(Parse.DateTime("3/16/2017 12:00:00 PM").AddHours(i), _crypto, new Bar(1m, 2m, 3m, 4m), 1, new Bar(5m, 6m, 7m, 8m), 2); // same quote twice! it has the same time, so it will be dropped when merging leanDataWriter.Write(Enumerable.Repeat(quoteBar, dataPointsPerLoop)); Assert.IsTrue(File.Exists(filePath)); Assert.IsFalse(File.Exists(filePath + ".tmp")); } var data = QuantConnect.Compression.Unzip(filePath).First().Value; switch (writePolicy) { case WritePolicy.Overwrite: Assert.AreEqual(dataPointsPerLoop, data.Count); break; case WritePolicy.Merge: Assert.AreEqual(loopCount, data.Count); break; case WritePolicy.Append: Assert.AreEqual(dataPointsPerLoop * loopCount, data.Count); break; case null: if (resolution >= Resolution.Hour) { // will merge by default Assert.AreEqual(loopCount, data.Count); } else { // overwrite Assert.AreEqual(dataPointsPerLoop, data.Count); } break; default: throw new ArgumentOutOfRangeException(nameof(writePolicy), writePolicy, null); } }
public static void GenerateRandomData(RandomDataGeneratorSettings settings, ConsoleLeveledOutput output) { // can specify a seed value in this ctor if determinism is desired var randomValueGenerator = new RandomValueGenerator(); var symbolGenerator = new SymbolGenerator(settings, randomValueGenerator); var tickGenerator = new TickGenerator(settings, randomValueGenerator); output.Warn.WriteLine($"Begin data generation of {settings.SymbolCount} randomly generated {settings.SecurityType} assets..."); // iterate over our randomly generated symbols var count = 0; var progress = 0d; var previousMonth = -1; foreach (var symbol in symbolGenerator.GenerateRandomSymbols()) { output.Warn.WriteLine($"\tSymbol[{++count}]: {symbol} Progress: {progress:0.0}% - Generating data..."); // define aggregators via settings var aggregators = settings.CreateAggregators().ToList(); // generate and consolidate data foreach (var tick in tickGenerator.GenerateTicks(symbol)) { if (tick.Time.Month != previousMonth) { output.Info.WriteLine($"\tMonth: {tick.Time:MMMM}"); previousMonth = tick.Time.Month; } foreach (var item in aggregators) { item.Consolidator.Update(tick); } } // count each stage as a point, so total points is 2*symbol-count // and the current progress is twice the current, but less one because we haven't finished writing data yet progress = 100 * (2 * count - 1) / (2.0 * settings.SymbolCount); output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} Progress: {progress:0.0}% - Saving data in LEAN format"); // persist consolidated data to disk foreach (var item in aggregators) { var writer = new LeanDataWriter(item.Resolution, symbol, Globals.DataFolder, item.TickType); // send the flushed data into the writer. pulling the flushed list is very important, // lest we likely wouldn't get the last piece of data stuck in the consolidator writer.Write(item.Flush()); } // update progress progress = 100 * (2 * count) / (2.0 * settings.SymbolCount); output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} Progress: {progress:0.0}% - Symbol data generation and output completed"); } output.Info.WriteLine("Random data generation has completed."); }
/// <summary> /// Create ticks from Raw Kaiko data /// </summary> /// <param name="fileLocation">Path to the raw Kaiko data</param> /// <param name="market">The exchange the data represents</param> /// <param name="tickType">The tick type being processed</param> /// <param name="aggregateFunction">Function that parses the Kaiko data file and returns the enumerable of ticks</param> private static void CreateCryptoTicks(string fileLocation, string market, TickType tickType, Func <Symbol, string, IEnumerable <Tick> > aggregateFunction, string ticker) { // Create symbol from folder name var symbol = Symbol.Create(ticker, SecurityType.Crypto, market); foreach (var tradeFile in Directory.EnumerateFiles(fileLocation)) { // Write the ticks var writer = new LeanDataWriter(Resolution.Tick, symbol, Globals.DataFolder, tickType); writer.Write(aggregateFunction(symbol, tradeFile)); } }
/// <summary> /// GDAX Downloader Toolbox Project For LEAN Algorithmic Trading Engine. /// </summary> static void Main(string[] args) { Thread.CurrentThread.CurrentCulture = new CultureInfo("en-US"); if (args.Length < 3) { Console.WriteLine("Usage: GDAX Downloader SYMBOL RESOLUTION FROMDATE TODATE"); Console.WriteLine("SYMBOL = ETHUSD, ETHBTC, BTCUSD etc."); Console.WriteLine("RESOLUTION = Second/Minute/Hour/Daily"); Console.WriteLine("FROMDATE = yyyyMMdd HH:mm:ss"); Console.WriteLine("TODATE = yyyyMMdd HH:mm:ss"); Environment.Exit(1); } try { var resolution = (Resolution)Enum.Parse(typeof(Resolution), args[1]); // Load settings from command line var startDate = DateTime.ParseExact(args[2], "yyyyMMdd HH:mm:ss", CultureInfo.InvariantCulture); var endDate = DateTime.UtcNow; if (args[3] != null) { endDate = DateTime.ParseExact(args[3], "yyyyMMdd HH:mm:ss", CultureInfo.InvariantCulture); } // Load settings from config.json var dataDirectory = Config.Get("data-directory", "../../../Data"); //todo: will download any exchange but always save as gdax // Create an instance of the downloader const string market = Market.GDAX; var downloader = new GDAXDownloader(); // Download the data var symbolObject = Symbol.Create(args[0], SecurityType.Crypto, market); var data = downloader.Get(symbolObject, resolution, startDate, endDate); // Save the data var writer = new LeanDataWriter(resolution, symbolObject, dataDirectory, TickType.Trade); var distinctData = data.GroupBy(i => i.Time, (key, group) => group.First()).ToArray(); writer.Write(distinctData); Log.Trace("Finish data download. Press any key to continue.."); } catch (Exception err) { Log.Error(err); Log.Trace(err.Message); Log.Trace(err.StackTrace); } // Console.ReadLine(); }
/// <summary> /// Determines if it should downloads new data and retrieves data from disc /// </summary> /// <param name="key">A string representing where the data is stored</param> /// <returns>A <see cref="Stream"/> of the data requested</returns> public override Stream Fetch(string key) { return(DownloadOnce(key, s => { if (LeanData.TryParsePath(key, out var symbol, out var date, out var resolution, out var tickType, out var dataType)) { var dataTimeZone = MarketHoursDatabase.FromDataFolder().GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType); DateTime startTimeUtc; DateTime endTimeUtc; if (resolution < Resolution.Hour) { // we can get the date from the path startTimeUtc = date.ConvertToUtc(dataTimeZone); endTimeUtc = date.Add(resolution.ToTimeSpan()).ConvertToUtc(dataTimeZone); } else { try { startTimeUtc = symbol.ID.Date; } catch (InvalidOperationException) { startTimeUtc = Time.BeginningOfTime; } endTimeUtc = DateTime.UtcNow; } // Save the data var writer = new LeanDataWriter(resolution, symbol, Globals.DataFolder, tickType); try { var getParams = new DataDownloaderGetParameters(symbol, resolution, startTimeUtc, endTimeUtc, tickType); var data = _dataDownloader.Get(getParams) .Where(baseData => symbol.SecurityType == SecurityType.Base || baseData.GetType() == dataType) // for canonical symbols, downloader will return data for all of the chain .GroupBy(baseData => baseData.Symbol); foreach (var dataPerSymbol in data) { writer.Write(dataPerSymbol); } } catch (Exception e) { Log.Error(e); } } }));
/// <summary> /// Processes the entry. /// </summary> /// <param name="coinapiDataReader">The coinapi data reader.</param> /// <param name="file">The file.</param> private void ProcessEntry(CoinApiDataReader coinapiDataReader, FileInfo file) { var entryData = coinapiDataReader.GetCoinApiEntryData(file, _processingDate, _market); if (!SupportedMarkets.Contains(entryData.Symbol.ID.Market)) { // only convert data for supported exchanges return; } // materialize the enumerable into a list, since we need to enumerate over it twice var ticks = coinapiDataReader.ProcessCoinApiEntry(entryData, file).ToList(); var writer = new LeanDataWriter(Resolution.Tick, entryData.Symbol, _destinationFolder.FullName, entryData.TickType); writer.Write(ticks); Log.Trace($"CoinApiDataConverter(): Starting consolidation for {entryData.Symbol.Value} {entryData.TickType}"); var consolidators = new List <TickAggregator>(); if (entryData.TickType == TickType.Trade) { consolidators.AddRange(new[] { new TradeTickAggregator(Resolution.Second), new TradeTickAggregator(Resolution.Minute) }); } else { consolidators.AddRange(new[] { new QuoteTickAggregator(Resolution.Second), new QuoteTickAggregator(Resolution.Minute) }); } foreach (var tick in ticks) { foreach (var consolidator in consolidators) { consolidator.Update(tick); } } foreach (var consolidator in consolidators) { writer = new LeanDataWriter(consolidator.Resolution, entryData.Symbol, _destinationFolder.FullName, entryData.TickType); writer.Write(consolidator.Flush()); } }
/// <summary> /// Primary entry point to the program. /// </summary> public static void DataDownloader(IList <string> tickers, string resolution, DateTime fromDate, DateTime toDate) { if (resolution.IsNullOrEmpty() || tickers.IsNullOrEmpty()) { Console.WriteLine("BinanceDownloader ERROR: '--tickers=' or '--resolution=' parameter is missing"); Console.WriteLine("--tickers=eg BTCUSD"); Console.WriteLine("--resolution=Minute/Hour/Daily/All"); Environment.Exit(1); } try { var allResolutions = resolution.Equals("all", StringComparison.OrdinalIgnoreCase); var castResolution = allResolutions ? Resolution.Minute : (Resolution)Enum.Parse(typeof(Resolution), resolution); // Load settings from config.json var dataDirectory = Config.Get("data-folder", "../../../Data"); using (var downloader = new BinanceDataDownloader()) { foreach (var ticker in tickers) { // Download the data var startDate = fromDate; var symbol = downloader.GetSymbol(ticker); var data = downloader.Get(symbol, castResolution, fromDate, toDate); var bars = data.Cast <TradeBar>().ToList(); // Save the data (single resolution) var writer = new LeanDataWriter(castResolution, symbol, dataDirectory); writer.Write(bars); if (allResolutions) { // Save the data (other resolutions) foreach (var res in new[] { Resolution.Hour, Resolution.Daily }) { var resData = LeanData.AggregateTradeBars(bars, symbol, res.ToTimeSpan()); writer = new LeanDataWriter(res, symbol, dataDirectory); writer.Write(resData); } } } } } catch (Exception err) { Log.Error(err); } }
/// <summary> /// GDAX Downloader Toolbox Project For LEAN Algorithmic Trading Engine. /// </summary> static void Main(string[] args) { Thread.CurrentThread.CurrentCulture = new CultureInfo("en-US"); if (args.Length == 2) { args = new [] { args[0], DateTime.UtcNow.ToString("yyyyMMdd"), args[1] }; } else if (args.Length < 3) { Console.WriteLine("Usage: GDAX Downloader SYMBOL FROMDATE TODATE"); Console.WriteLine("FROMDATE = yyyymmdd"); Console.WriteLine("TODATE = yyyymmdd"); Environment.Exit(1); } try { // Load settings from command line var startDate = DateTime.ParseExact(args[1], "yyyyMMdd", CultureInfo.InvariantCulture); var endDate = DateTime.ParseExact(args[2], "yyyyMMdd", CultureInfo.InvariantCulture); // Load settings from config.json var dataDirectory = Config.Get("data-directory", "../../../Data"); //todo: will download any exchange but always save as gdax // Create an instance of the downloader const string market = Market.GDAX; var downloader = new GDAXDownloader(); // Download the data var symbolObject = Symbol.Create(args[0], SecurityType.Crypto, market); var data = downloader.Get(symbolObject, Resolution.Hour, startDate, endDate); // Save the data var writer = new LeanDataWriter(Resolution.Hour, symbolObject, dataDirectory, TickType.Trade); var distinctData = data.GroupBy(i => i.Time, (key, group) => group.First()).ToArray(); writer.Write(distinctData); Log.Trace("Finish data download"); } catch (Exception err) { Log.Error(err); Log.Trace(err.Message); Log.Trace(err.StackTrace); } Console.ReadLine(); }
/// <summary> /// Primary entry point to the program. This program only supports FOREX for now. /// </summary> public static void IEXDownloader(IList <string> tickers, string resolution, DateTime fromDate, DateTime toDate, string apiKey) { if (resolution.IsNullOrEmpty() || tickers.IsNullOrEmpty()) { Console.WriteLine("IEXDownloader ERROR: '--tickers=' or '--resolution=' parameter is missing"); Console.WriteLine("--tickers=eg SPY,AAPL"); Console.WriteLine("--resolution=Minute/Daily"); Environment.Exit(1); } if (apiKey.IsNullOrEmpty()) { Console.WriteLine("QuandlBitfinexDownloader ERROR: '--api-key=' parameter is missing"); Environment.Exit(1); } try { // Load settings from command line var castResolution = (Resolution)Enum.Parse(typeof(Resolution), resolution); // Load settings from config.json var dataDirectory = Config.Get("data-directory", "../../../Data"); var startDate = fromDate.ConvertToUtc(TimeZones.NewYork); var endDate = toDate.ConvertToUtc(TimeZones.NewYork); // Create an instance of the downloader const string market = Market.USA; SecurityType securityType = SecurityType.Equity; using (var downloader = new IEXDataDownloader(apiKey = apiKey)) { foreach (var ticker in tickers) { // Download the data var symbolObject = Symbol.Create(ticker, securityType, market); var data = downloader.Get(symbolObject, castResolution, startDate, endDate); // Save the data var writer = new LeanDataWriter(castResolution, symbolObject, dataDirectory); writer.Write(data); } } } catch (Exception err) { Log.Error(err); } Console.ReadLine(); }
public void LeanDataWriter_CanWriteCrypto() { var filePath = LeanData.GenerateZipFilePath(_dataDirectory, _crypto, _date, Resolution.Second, TickType.Quote); var leanDataWriter = new LeanDataWriter(Resolution.Second, _crypto, _dataDirectory, TickType.Quote); leanDataWriter.Write(GetQuoteBars(_crypto)); Assert.IsTrue(File.Exists(filePath)); Assert.IsFalse(File.Exists(filePath + ".tmp")); var data = QuantConnect.Compression.Unzip(filePath); Assert.AreEqual(data.First().Value.Count(), 3); }
public void LeanDataWriter_CanWriteEquity() { var filePath = LeanData.GenerateZipFilePath(_dataDirectory, _equity, _date, Resolution.Tick, TickType.Trade); var leanDataWriter = new LeanDataWriter(Resolution.Tick, _equity, _dataDirectory); leanDataWriter.Write(GetTicks(_equity)); Assert.IsTrue(File.Exists(filePath)); Assert.IsFalse(File.Exists(filePath + ".tmp")); var data = QuantConnect.Compression.Unzip(filePath); Assert.AreEqual(data.First().Value.Count(), 3); }
/// <summary> /// Primary entry point to the program /// </summary> static void Main(string[] args) { if (args.Length != 4) { Console.WriteLine("Usage: OandaDownloader SYMBOL RESOLUTION FROMDATE TODATE"); Console.WriteLine("SYMBOL = eg EURUSD"); Console.WriteLine("RESOLUTION = Second/Minute/Hour/Daily"); Console.WriteLine("FROMDATE = yyyymmdd"); Console.WriteLine("TODATE = yyyymmdd"); Environment.Exit(1); } try { // Load settings from command line var symbol = args[0]; var resolution = (Resolution)Enum.Parse(typeof(Resolution), args[1]); var startDate = DateTime.ParseExact(args[2], "yyyyMMdd", CultureInfo.InvariantCulture); var endDate = DateTime.ParseExact(args[3], "yyyyMMdd", CultureInfo.InvariantCulture); // Load settings from config.json var dataDirectory = Config.Get("data-directory", "../../../Data"); var accessToken = Config.Get("access-token", "73eba38ad5b44778f9a0c0fec1a66ed1-44f47f052c897b3e1e7f24196bbc071f"); var accountId = Convert.ToInt32(Config.Get("account-id", "621396")); // Download the data var downloader = new OandaDataDownloader(accessToken, accountId); if (!downloader.HasSymbol(symbol)) { throw new ArgumentException("The symbol " + symbol + " is not available."); } var securityType = downloader.GetSecurityType(symbol); var data = downloader.Get(new Symbol(symbol), securityType, resolution, startDate, endDate); // Save the data var writer = new LeanDataWriter(securityType, resolution, symbol, dataDirectory, "oanda"); writer.Write(data); } catch (Exception err) { Log.Error("OandaDownloader(): Error: " + err.Message); } }
/// <summary> /// Yahoo Downloader Toolbox Project For LEAN Algorithmic Trading Engine. /// Original by @chrisdk2015, tidied by @jaredbroad /// </summary> static void Main(string[] args) { if (args.Length != 4) { Console.WriteLine("Usage: YahooDownloader SYMBOLS RESOLUTION FROMDATE TODATE"); Console.WriteLine("SYMBOLS = eg SPY,AAPL"); Console.WriteLine("RESOLUTION = Daily"); Console.WriteLine("FROMDATE = yyyymmdd"); Console.WriteLine("TODATE = yyyymmdd"); Environment.Exit(1); } try { // Load settings from command line var symbols = args[0].Split(','); var resolution = (Resolution)Enum.Parse(typeof(Resolution), args[1]); var startDate = DateTime.ParseExact(args[2], "yyyyMMdd", CultureInfo.InvariantCulture); var endDate = DateTime.ParseExact(args[3], "yyyyMMdd", CultureInfo.InvariantCulture); // Load settings from config.json var dataDirectory = Config.Get("data-directory", "../../../Data"); // Create an instance of the downloader const string market = Market.USA; var downloader = new YahooDataDownloader(); foreach (var symbol in symbols) { // Download the data var sid = SecurityIdentifier.GenerateEquity(symbol, market); var symbolObject = new Symbol(sid, symbol); var data = downloader.Get(symbolObject, SecurityType.Equity, resolution, startDate, endDate); // Save the data var writer = new LeanDataWriter(SecurityType.Equity, resolution, symbolObject, dataDirectory, market); writer.Write(data); } } catch (Exception err) { Log.Error("YahooDownloader(): Error: " + err.Message); } }
public void LeanDataWriter_CanWriteZipWithMultipleContracts(SecurityType securityType) { Symbol contract1; Symbol contract2; if (securityType == SecurityType.Future) { contract1 = Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, new DateTime(2020, 02, 01)); contract2 = Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, new DateTime(2020, 03, 01)); } else if (securityType == SecurityType.Option) { contract1 = Symbol.CreateOption("AAPL", Market.USA, OptionStyle.American, OptionRight.Call, 1, new DateTime(2020, 02, 01)); contract2 = Symbol.CreateOption("AAPL", Market.USA, OptionStyle.American, OptionRight.Call, 1, new DateTime(2020, 03, 01)); } else if (securityType == SecurityType.FutureOption) { contract1 = Symbol.CreateOption(Futures.Indices.SP500EMini, Market.CME, OptionStyle.American, OptionRight.Call, 1, new DateTime(2020, 02, 01)); contract2 = Symbol.CreateOption(Futures.Indices.SP500EMini, Market.CME, OptionStyle.American, OptionRight.Call, 1, new DateTime(2020, 03, 01)); } else { throw new NotImplementedException($"{securityType} not implemented!"); } var filePath1 = LeanData.GenerateZipFilePath(_dataDirectory, contract1, _date, Resolution.Second, TickType.Quote); var leanDataWriter1 = new LeanDataWriter(Resolution.Second, contract1, _dataDirectory, TickType.Quote); leanDataWriter1.Write(GetQuoteBars(contract1)); var filePath2 = LeanData.GenerateZipFilePath(_dataDirectory, contract2, _date, Resolution.Second, TickType.Quote); var leanDataWriter2 = new LeanDataWriter(Resolution.Second, contract2, _dataDirectory, TickType.Quote); leanDataWriter2.Write(GetQuoteBars(contract2)); Assert.AreEqual(filePath1, filePath2); Assert.IsTrue(File.Exists(filePath1)); Assert.IsFalse(File.Exists(filePath1 + ".tmp")); var data = QuantConnect.Compression.Unzip(filePath1).ToDictionary(x => x.Key, x => x.Value.ToList()); Assert.AreEqual(2, data.Count); Assert.That(data.Values, Has.All.Count.EqualTo(3)); }
/// <summary> /// GDAX Downloader Toolbox Project For LEAN Algorithmic Trading Engine. /// </summary> public static void GDAXDownloader(IList <string> tickers, string resolution, DateTime fromDate, DateTime toDate) { Thread.CurrentThread.CurrentCulture = new CultureInfo("en-US"); if (resolution.IsNullOrEmpty() || tickers.IsNullOrEmpty()) { Console.WriteLine("GDAXDownloader ERROR: '--tickers=' or '--resolution=' parameter is missing"); Console.WriteLine("--tickers=ETHUSD,ETHBTC,BTCUSD,etc."); Console.WriteLine("--resolution=Second/Minute/Hour/Daily"); Environment.Exit(1); } var castResolution = (Resolution)Enum.Parse(typeof(Resolution), resolution); try { // Load settings from config.json var dataDirectory = Config.Get("data-directory", "../../../Data"); //todo: will download any exchange but always save as gdax // Create an instance of the downloader const string market = Market.GDAX; var downloader = new GDAXDownloader(); foreach (var ticker in tickers) { // Download the data var symbolObject = Symbol.Create(ticker, SecurityType.Crypto, market); var data = downloader.Get(symbolObject, castResolution, fromDate, toDate); // Save the data var writer = new LeanDataWriter(castResolution, symbolObject, dataDirectory, TickType.Trade); var distinctData = data.GroupBy(i => i.Time, (key, group) => group.First()).ToArray(); writer.Write(distinctData); } Log.Trace("Finish data download. Press any key to continue.."); } catch (Exception err) { Log.Error(err); Log.Trace(err.Message); Log.Trace(err.StackTrace); } Console.ReadLine(); }
/// <summary> /// QuantConnect Google Downloader For LEAN Algorithmic Trading Engine. /// Original by @chrisdk2015, tidied by @jaredbroad /// </summary> public static void Main(string[] args) { if (args.Length != 4) { Console.WriteLine("Usage: GoogleDownloader SYMBOLS RESOLUTION FROMDATE TODATE"); Console.WriteLine("SYMBOLS = eg SPY,AAPL"); Console.WriteLine("RESOLUTION = Minute/Hour"); Console.WriteLine("FROMDATE = yyyymmdd"); Console.WriteLine("TODATE = yyyymmdd"); Environment.Exit(1); } try { // Load settings from command line var symbols = args[0].Split(','); var resolution = (Resolution)Enum.Parse(typeof(Resolution), args[1]); var startDate = DateTime.ParseExact(args[2], "yyyyMMdd", CultureInfo.InvariantCulture); var endDate = DateTime.ParseExact(args[3], "yyyyMMdd", CultureInfo.InvariantCulture); // Load settings from config.json var dataDirectory = Config.Get("data-directory", "../../../Data"); // Create an instance of the downloader const string market = Market.USA; var downloader = new GoogleDataDownloader(); foreach (var symbol in symbols) { // Download the data var sid = SecurityIdentifier.GenerateEquity(symbol, market); var symbolObject = new Symbol(sid, symbol); var data = downloader.Get(symbolObject, SecurityType.Equity, resolution, startDate, endDate); // Save the data var writer = new LeanDataWriter(SecurityType.Equity, resolution, symbolObject, dataDirectory, market); writer.Write(data); } } catch (Exception err) { Log.Error("GoogleDownloader(): Error: " + err.Message); } }
/// <summary> /// Primary entry point to the program /// </summary> static void Main(string[] args) { if (args.Length != 4) { Console.WriteLine("Usage: DukascopyDownloader SYMBOL RESOLUTION FROMDATE TODATE"); Console.WriteLine("SYMBOL = eg EURUSD"); Console.WriteLine("RESOLUTION = Tick/Second/Minute/Hour/Daily"); Console.WriteLine("FROMDATE = yyyymmdd"); Console.WriteLine("TODATE = yyyymmdd"); Environment.Exit(1); } try { // Load settings from command line var symbol = args[0]; var resolution = (Resolution)Enum.Parse(typeof(Resolution), args[1]); var startDate = DateTime.ParseExact(args[2], "yyyyMMdd", CultureInfo.InvariantCulture); var endDate = DateTime.ParseExact(args[3], "yyyyMMdd", CultureInfo.InvariantCulture); // Load settings from config.json var dataDirectory = Config.Get("data-directory", "../../../Data"); // Download the data var downloader = new DukascopyDataDownloader(); if (!downloader.HasSymbol(symbol)) { throw new ArgumentException("The symbol " + symbol + " is not available."); } var securityType = downloader.GetSecurityType(symbol); var data = downloader.Get(new Symbol(symbol), securityType, resolution, startDate, endDate); // Save the data var writer = new LeanDataWriter(securityType, resolution, symbol, dataDirectory, "dukascopy"); writer.Write(data); } catch (Exception err) { Log.Error("DukascopyDownloader(): {0}", err.Message); } }
/// <summary> /// QuantConnect Google Downloader For LEAN Algorithmic Trading Engine. /// Original by @chrisdk2015, tidied by @jaredbroad /// </summary> public static void Main(string[] args) { //if (args.Length != 4) //{ // Console.WriteLine("Usage: GoogleDownloader SYMBOLS RESOLUTION FROMDATE TODATE"); // Console.WriteLine("SYMBOLS = eg SPY,AAPL"); // Console.WriteLine("RESOLUTION = Minute/Hour"); // Console.WriteLine("FROMDATE = yyyymmdd"); // Console.WriteLine("TODATE = yyyymmdd"); // Environment.Exit(1); //} try { // Load settings from command line var symbols = "GARAN,AKBNK".Split(','); var resolution = (Resolution)Enum.Parse(typeof(Resolution), "Minute"); var startDate = DateTime.ParseExact("20150101", "yyyyMMdd", CultureInfo.InvariantCulture); var endDate = DateTime.ParseExact("20160101", "yyyyMMdd", CultureInfo.InvariantCulture); // Load settings from config.json var dataDirectory = Config.Get("data-directory", "../../../Data"); // Create an instance of the downloader const string market = Market.USA; var downloader = new GoogleDataDownloader(); foreach (var symbol in symbols) { // Download the data var symbolObject = Symbol.Create(symbol, SecurityType.Equity, market); var data = downloader.Get(symbolObject, resolution, startDate, endDate); // Save the data var writer = new LeanDataWriter(resolution, symbolObject, dataDirectory); writer.Write(data); } } catch (Exception err) { Log.Error(err); } }
/// <summary> /// Primary entry point to the program /// </summary> static void Main(string[] args) { if (args.Length != 4) { Console.WriteLine("Usage: OandaDownloader SYMBOL RESOLUTION FROMDATE TODATE"); Console.WriteLine("SYMBOL = eg EURUSD"); Console.WriteLine("RESOLUTION = Second/Minute/Hour/Daily"); Console.WriteLine("FROMDATE = yyyymmdd"); Console.WriteLine("TODATE = yyyymmdd"); Environment.Exit(1); } try { // Load settings from command line var symbol = args[0]; var resolution = (Resolution)Enum.Parse(typeof(Resolution), args[1]); var startDate = DateTime.ParseExact(args[2], "yyyyMMdd", CultureInfo.InvariantCulture); var endDate = DateTime.ParseExact(args[3], "yyyyMMdd", CultureInfo.InvariantCulture); // Load settings from config.json var dataDirectory = Config.Get("data-directory", "../../../Data"); var accessToken = Config.Get("access-token", "73eba38ad5b44778f9a0c0fec1a66ed1-44f47f052c897b3e1e7f24196bbc071f"); var accountId = Convert.ToInt32(Config.Get("account-id", "621396")); // Download the data var downloader = new OandaDataDownloader(accessToken, accountId); if (!downloader.HasSymbol(symbol)) throw new ArgumentException("The symbol " + symbol + " is not available."); var securityType = downloader.GetSecurityType(symbol); var data = downloader.Get(new Symbol(symbol), securityType, resolution, startDate, endDate); // Save the data var writer = new LeanDataWriter(securityType, resolution, symbol, dataDirectory, "oanda"); writer.Write(data); } catch (Exception err) { Log.Error("OandaDownloader(): Error: " + err.Message); } }
/// <summary> /// Cryptoiq Downloader Toolbox Project For LEAN Algorithmic Trading Engine. /// </summary> static void Main(string[] args) { if (args.Length == 3) { args = new [] { args[0], DateTime.UtcNow.ToString("yyyyMMdd"), args[1], args[2] }; } else if (args.Length < 4) { Console.WriteLine("Usage: CryptoiqDownloader FROMDATE TODATE EXCHANGE SYMBOL"); Console.WriteLine("FROMDATE = yyyymmdd"); Console.WriteLine("TODATE = yyyymmdd"); Environment.Exit(1); } try { // Load settings from command line var startDate = DateTime.ParseExact(args[0], "yyyyMMdd", CultureInfo.InvariantCulture); var endDate = DateTime.ParseExact(args[1], "yyyyMMdd", CultureInfo.InvariantCulture); // Load settings from config.json var dataDirectory = Config.Get("data-directory", "../../../Data"); var scaleFactor = Config.GetValue("bitfinex-scale-factor", 1m); // Create an instance of the downloader const string market = Market.Bitfinex; var downloader = new CryptoiqDownloader(args[2], scaleFactor); // Download the data var symbolObject = Symbol.Create(args[3], SecurityType.Forex, market); var data = downloader.Get(symbolObject, Resolution.Tick, startDate, endDate); // Save the data var writer = new LeanDataWriter(Resolution.Tick, symbolObject, dataDirectory, TickType.Quote); writer.Write(data); } catch (Exception err) { Log.Error(err); } }
/// <summary> /// Primary entry point to the program. /// </summary> public static void BitfinexDownloader(IList <string> tickers, string resolution, DateTime fromDate, DateTime toDate) { if (resolution.IsNullOrEmpty() || tickers.IsNullOrEmpty()) { Console.WriteLine("BitfinexDownloader ERROR: '--tickers=' or '--resolution=' parameter is missing"); Console.WriteLine("--tickers=eg BTCUSD"); Console.WriteLine("--resolution=Second/Minute/Hour/Daily/All"); Environment.Exit(1); } try { var allResolutions = resolution.ToLowerInvariant() == "all"; var resolutions = allResolutions ? new[] { Resolution.Minute, Resolution.Hour, Resolution.Daily } : new[] { (Resolution)Enum.Parse(typeof(Resolution), resolution) }; // Load settings from config.json var dataDirectory = Config.Get("data-folder", "../../../Data"); using (var downloader = new BitfinexDataDownloader()) { foreach (var ticker in tickers) { // Download the data var symbol = downloader.GetSymbol(ticker); foreach (var castResolution in resolutions) { var data = downloader.Get(new DataDownloaderGetParameters(symbol, castResolution, fromDate, toDate)); // Save the data (single resolution) var writer = new LeanDataWriter(castResolution, symbol, dataDirectory); writer.Write(data); } } } } catch (Exception err) { Log.Error(err); } }
/// <summary> /// Primary entry point to the program /// </summary> static void Main(string[] args) { if (args.Length != 4) { Console.WriteLine("Usage: DukascopyDownloader SYMBOL RESOLUTION FROMDATE TODATE"); Console.WriteLine("SYMBOL = eg EURUSD"); Console.WriteLine("RESOLUTION = Tick/Second/Minute/Hour/Daily"); Console.WriteLine("FROMDATE = yyyymmdd"); Console.WriteLine("TODATE = yyyymmdd"); Environment.Exit(1); } try { // Load settings from command line var symbol = args[0]; var resolution = (Resolution)Enum.Parse(typeof(Resolution), args[1]); var startDate = DateTime.ParseExact(args[2], "yyyyMMdd", CultureInfo.InvariantCulture); var endDate = DateTime.ParseExact(args[3], "yyyyMMdd", CultureInfo.InvariantCulture); // Load settings from config.json var dataDirectory = Config.Get("data-directory", "../../../Data"); // Download the data var downloader = new DukascopyDataDownloader(); if (!downloader.HasSymbol(symbol)) throw new ArgumentException("The symbol " + symbol + " is not available."); var securityType = downloader.GetSecurityType(symbol); var data = downloader.Get(new Symbol(symbol), securityType, resolution, startDate, endDate); // Save the data var writer = new LeanDataWriter(securityType, resolution, symbol, dataDirectory, "dukascopy"); writer.Write(data); } catch (Exception err) { Log.Error("DukascopyDownloader(): {0}", err.Message); } }
/// <summary> /// Cryptoiq Downloader Toolbox Project For LEAN Algorithmic Trading Engine. /// </summary> static void Main(string[] args) { if (args.Length == 3) { args = new [] { args[0], DateTime.UtcNow.ToString("yyyyMMdd"), args[1], args[2] }; } else if (args.Length < 4) { Console.WriteLine("Usage: CryptoiqDownloader FROMDATE TODATE EXCHANGE SYMBOL"); Console.WriteLine("FROMDATE = yyyymmdd"); Console.WriteLine("TODATE = yyyymmdd"); Environment.Exit(1); } try { // Load settings from command line var startDate = DateTime.ParseExact(args[0], "yyyyMMdd", CultureInfo.InvariantCulture); var endDate = DateTime.ParseExact(args[1], "yyyyMMdd", CultureInfo.InvariantCulture); // Load settings from config.json var dataDirectory = Config.Get("data-directory", "../../../Data"); var scaleFactor = Config.GetValue("bitfinex-scale-factor", 1m); // Create an instance of the downloader const string market = Market.Bitfinex; var downloader = new CryptoiqDownloader(args[2], scaleFactor); // Download the data var symbolObject = Symbol.Create(args[3], SecurityType.Crypto, market); var data = downloader.Get(symbolObject, Resolution.Tick, startDate, endDate); // Save the data var writer = new LeanDataWriter(Resolution.Tick, symbolObject, dataDirectory, TickType.Quote); writer.Write(data); } catch (Exception err) { Log.Error(err); } }
/// <summary> /// https://msdn.microsoft.com/en-us/library/ff650674.aspx /// </summary> /// private void btnTest_Click(object sender, EventArgs e) { try { OpenFileDialog ofd = new OpenFileDialog(); if (ofd.ShowDialog() == DialogResult.OK) { string filePath = ofd.FileName; // Load settings from command line var symbols = "GARAN".Split(','); var resolution = (Resolution)Enum.Parse(typeof(Resolution), "Minute"); var startDate = DateTime.ParseExact("20150101", "yyyyMMdd", CultureInfo.InvariantCulture); var endDate = DateTime.ParseExact("20160328", "yyyyMMdd", CultureInfo.InvariantCulture); // Load settings from config.json var dataDirectory = Config.Get("data-directory", "../../../Data"); // Create an instance of the downloader const string market = Market.USA; GoogleDataDownloader downloader = new GoogleDataDownloader(); foreach (var symbol in symbols) { // Download the data var symbolObject = Symbol.Create(symbol, SecurityType.Equity, market); var data = downloader.Parse(filePath, symbolObject, resolution, startDate, endDate).OrderBy(o => o.Time); // Save the data var writer = new LeanDataWriter(resolution, symbolObject, dataDirectory); writer.Write(data); } } } catch (Exception err) { Log.Error(err); } }
/// <summary> /// Primary entry point to the program /// </summary> static void Main(string[] args) { if (args.Length != 4) { Console.WriteLine("Usage: DukascopyDownloader SYMBOLS RESOLUTION FROMDATE TODATE"); Console.WriteLine("SYMBOLS = eg EURUSD,USDJPY"); Console.WriteLine("RESOLUTION = Tick/Second/Minute/Hour/Daily/All"); Console.WriteLine("FROMDATE = yyyymmdd"); Console.WriteLine("TODATE = yyyymmdd"); Environment.Exit(1); } try { // Load settings from command line var symbols = args[0].Split(','); var allResolutions = args[1].ToLower() == "all"; var resolution = allResolutions ? Resolution.Tick : (Resolution)Enum.Parse(typeof(Resolution), args[1]); var startDate = DateTime.ParseExact(args[2], "yyyyMMdd", CultureInfo.InvariantCulture); var endDate = DateTime.ParseExact(args[3], "yyyyMMdd", CultureInfo.InvariantCulture); // Load settings from config.json var dataDirectory = Config.Get("data-directory", "../../../Data"); // Download the data const string market = Market.Dukascopy; var downloader = new DukascopyDataDownloader(); foreach (var symbol in symbols) { if (!downloader.HasSymbol(symbol)) throw new ArgumentException("The symbol " + symbol + " is not available."); } foreach (var symbol in symbols) { var securityType = downloader.GetSecurityType(symbol); var symbolObject = new Symbol(GetSid(symbol, securityType), symbol); var data = downloader.Get(symbolObject, securityType, resolution, startDate, endDate); if (allResolutions) { var ticks = data.Cast<Tick>().ToList(); // Save the data (tick resolution) var writer = new LeanDataWriter(securityType, resolution, symbolObject, dataDirectory, market); writer.Write(ticks); // Save the data (other resolutions) foreach (var res in new[] { Resolution.Second, Resolution.Minute, Resolution.Hour, Resolution.Daily }) { var resData = DukascopyDataDownloader.AggregateTicks(symbolObject, ticks, res.ToTimeSpan()); writer = new LeanDataWriter(securityType, res, symbolObject, dataDirectory, market); writer.Write(resData); } } else { // Save the data (single resolution) var writer = new LeanDataWriter(securityType, resolution, symbolObject, dataDirectory, market); writer.Write(data); } } } catch (Exception err) { Log.Error(err); } }
/// <summary> /// Primary entry point to the program /// </summary> private static void Main(string[] args) { if (args.Length != 4) { Console.WriteLine("Usage: FxcmDownloader SYMBOLS RESOLUTION FROMDATE TODATE"); Console.WriteLine("SYMBOLS = eg EURUSD,USDJPY"); Console.WriteLine("RESOLUTION = Second/Minute/Hour/Daily/All"); Console.WriteLine("FROMDATE = yyyymmdd"); Console.WriteLine("TODATE = yyyymmdd"); Environment.Exit(1); } try { Logger.getRootLogger().setLevel(Level.ERROR); BasicConfigurator.configure(new FileAppender(new SimpleLayout(), "FxcmDownloader.log", append: false)); // Load settings from command line var tickers = args[0].Split(','); var allResolutions = args[1].ToLower() == "all"; var resolution = allResolutions ? Resolution.Second : (Resolution)Enum.Parse(typeof(Resolution), args[1]); var startDate = DateTime.ParseExact(args[2], "yyyyMMdd", CultureInfo.InvariantCulture); var endDate = DateTime.ParseExact(args[3], "yyyyMMdd", CultureInfo.InvariantCulture); // Load settings from config.json var dataDirectory = Config.Get("data-directory", "../../../Data"); var server = Config.Get("fxcm-server", "http://www.fxcorporate.com/Hosts.jsp"); var terminal = Config.Get("fxcm-terminal", "Demo"); var userName = Config.Get("fxcm-user-name", "username"); var password = Config.Get("fxcm-password", "password"); // Download the data const string market = "fxcm"; var downloader = new FxcmDataDownloader(server, terminal, userName, password); foreach (var ticker in tickers) { var securityType = downloader.GetSecurityType(ticker); var symbol = Symbol.Create(ticker, securityType, Market.FXCM); if (!downloader.HasSymbol(ticker)) throw new ArgumentException("The symbol " + ticker + " is not available."); var data = downloader.Get(symbol, securityType, resolution, startDate, endDate); if (allResolutions) { var bars = data.Cast<TradeBar>().ToList(); // Save the data (second resolution) var writer = new LeanDataWriter(securityType, resolution, symbol, dataDirectory, market); writer.Write(bars); // Save the data (other resolutions) foreach (var res in new[] {Resolution.Minute, Resolution.Hour, Resolution.Daily}) { var resData = FxcmDataDownloader.AggregateBars(symbol, bars, res.ToTimeSpan()); writer = new LeanDataWriter(securityType, res, symbol, dataDirectory, market); writer.Write(resData); } } else { // Save the data (single resolution) var writer = new LeanDataWriter(securityType, resolution, symbol, dataDirectory, market); writer.Write(data); } } } catch (Exception err) { Log.Error("FxcmDownloader(): Error: " + err.Message); } }
/// <summary> /// Primary entry point to the program /// </summary> static void Main(string[] args) { if (args.Length != 4) { Console.WriteLine("Usage: OandaDownloader SYMBOL RESOLUTION FROMDATE TODATE"); Console.WriteLine("SYMBOLS = eg EURUSD,USDJPY"); Console.WriteLine("RESOLUTION = Second/Minute/Hour/Daily/All"); Console.WriteLine("FROMDATE = yyyymmdd"); Console.WriteLine("TODATE = yyyymmdd"); Environment.Exit(1); } try { // Load settings from command line var symbols = args[0].Split(','); var allResolutions = args[1].ToLower() == "all"; var resolution = allResolutions ? Resolution.Second : (Resolution)Enum.Parse(typeof(Resolution), args[1]); var startDate = DateTime.ParseExact(args[2], "yyyyMMdd", CultureInfo.InvariantCulture); var endDate = DateTime.ParseExact(args[3], "yyyyMMdd", CultureInfo.InvariantCulture); // Load settings from config.json var dataDirectory = Config.Get("data-directory", "../../../Data"); var accessToken = Config.Get("access-token", "73eba38ad5b44778f9a0c0fec1a66ed1-44f47f052c897b3e1e7f24196bbc071f"); var accountId = Convert.ToInt32(Config.Get("account-id", "621396")); // Download the data const string market = "oanda"; var downloader = new OandaDataDownloader(accessToken, accountId); foreach (var symbol in symbols) { if (!downloader.HasSymbol(symbol)) throw new ArgumentException("The symbol " + symbol + " is not available."); } foreach (var symbol in symbols) { var securityType = downloader.GetSecurityType(symbol); var data = downloader.Get(new Symbol(symbol), securityType, resolution, startDate, endDate); if (allResolutions) { var bars = data.Cast<TradeBar>().ToList(); // Save the data (second resolution) var writer = new LeanDataWriter(securityType, resolution, symbol, dataDirectory, market); writer.Write(bars); // Save the data (other resolutions) foreach (var res in new[] { Resolution.Minute, Resolution.Hour, Resolution.Daily }) { var resData = AggregateBars(new Symbol(symbol), bars, res.ToTimeSpan()); writer = new LeanDataWriter(securityType, res, symbol, dataDirectory, market); writer.Write(resData); } } else { // Save the data (single resolution) var writer = new LeanDataWriter(securityType, resolution, symbol, dataDirectory, market); writer.Write(data); } } } catch (Exception err) { Log.Error("OandaDownloader(): Error: " + err.Message); } }