/// <summary>
/// Quandl Bitfinex Toolbox Project For LEAN Algorithmic Trading Engine.
/// </summary>
static void Main(string[] args)
{
if (args.Length != 2)
{
Console.WriteLine("Usage: Downloader FROMDATE APIKEY");
Console.WriteLine("FROMDATE = yyyymmdd");
Environment.Exit(1);
}
try
{
// Load settings from config.json
var dataDirectory = Config.Get("data-directory", "../../../Data");
var scaleFactor = Config.GetInt("bitfinex-scale-factor", 100);
// Create an instance of the downloader
const string market = Market.Bitfinex;
var downloader = new QuandlBitfinexDownloader(args[1], scaleFactor);
// Download the data
var symbol = Symbol.Create("BTCUSD", SecurityType.Forex, market);
var data = downloader.Get(symbol, Resolution.Daily, DateTime.ParseExact(args[0], "yyyyMMdd", CultureInfo.CurrentCulture), DateTime.UtcNow);
// Save the data
var writer = new LeanDataWriter(Resolution.Daily, symbol, dataDirectory, TickType.Quote);
writer.Write(data);
}
catch (Exception err)
{
Log.Error(err);
}
}
/// <summary>
/// QuantConnect Google Downloader For LEAN Algorithmic Trading Engine.
/// Original by @chrisdk2015, tidied by @jaredbroad
/// </summary>
public static void Main(string[] args)
{
if (args.Length != 3)
{
Console.WriteLine("Usage: GoogleDownloader SYMBOL RESOLUTION PERIOD");
Console.WriteLine("SYMBOL = eg SPY");
Console.WriteLine("RESOLUTION = Minute/Hour");
Console.WriteLine("PERIOD = 10 for 10 days intraday data");
Environment.Exit(1);
}
try
{
// Load settings from command line
var symbol = args[0];
var resolution = (Resolution)Enum.Parse(typeof(Resolution), args[1]);
var period = args[2].ToInt32();
// Load settings from config.json
var dataDirectory = Config.Get("data-directory", "../../../Data");
// Download the data
var downloader = new GoogleDataDownloader();
var data = downloader.Get(new Symbol(symbol), SecurityType.Equity, resolution, DateTime.UtcNow.AddDays(-period), DateTime.UtcNow);
// Save the data
var writer = new LeanDataWriter(SecurityType.Equity, resolution, symbol, dataDirectory, "usa");
writer.Write(data);
}
catch (Exception err)
{
Log.Error("GoogleDownloader(): Error: " + err.Message);
}
}
/// <summary>
/// Kraken Downloader Toolbox Project For LEAN Algorithmic Trading Engine.
/// By @matthewsedam
/// </summary>
public static void KrakenDownloader(IList <string> tickers, string resolution, DateTime startDate, DateTime endDate)
{
if (resolution.IsNullOrEmpty() || tickers.IsNullOrEmpty())
{
Console.WriteLine("KrakenDownloader ERROR: '--tickers=' or '--resolution=' parameter is missing");
Console.WriteLine("--tickers=eg XXBTZUSD,XETHZUSD");
Console.WriteLine("--resolution=Minute/Hour/Daily/Tick");
Environment.Exit(1);
}
try
{
var castResolution = (Resolution)Enum.Parse(typeof(Resolution), resolution);
// Load settings from config.json and create downloader
var dataDirectory = Config.Get("data-directory", "../../../Data");
var downloader = new KrakenDataDownloader();
foreach (var pair in tickers)
{
// Download data
var pairObject = Symbol.Create(pair, SecurityType.Crypto, Market.Kraken);
var data = downloader.Get(pairObject, castResolution, startDate, endDate);
// Write data
var writer = new LeanDataWriter(castResolution, pairObject, dataDirectory);
writer.Write(data);
}
}
catch (Exception err)
{
Log.Error(err);
}
}
/// <summary>
/// Create ticks from Raw Kaiko data
/// </summary>
/// <param name="fileLocation">Path to the raw Kaiko data</param>
/// <param name="market">The exchange the data represents</param>
/// <param name="tickType">The tick type being processed</param>
/// <param name="aggregateFunction">Function that parses the Kaiko data file and returns the enumerable of ticks</param>
private static void CreateCryptoTicks(string fileLocation, string market, TickType tickType, Func <Symbol, string, IEnumerable <Tick> > aggregateFunction)
{
foreach (var symbolFolder in Directory.EnumerateDirectories(fileLocation))
{
var symbolDirectoryInfo = new DirectoryInfo(symbolFolder);
// Create symbol from folder name
var symbol = Symbol.Create(symbolDirectoryInfo.Name, SecurityType.Crypto, market);
foreach (var symbolMonthDirectory in Directory.EnumerateDirectories(symbolDirectoryInfo.FullName))
{
foreach (var tradeFile in Directory.EnumerateFiles(symbolMonthDirectory, "*.gz"))
{
string unzippedFile;
// Unzip file
try
{
unzippedFile = Compression.UnGZip(tradeFile, symbolMonthDirectory);
}
catch (Exception e)
{
Log.Error($"KaikoDataConverter.CreateCryptoTicks(): File {tradeFile} cannot be unzipped. Exception {e}");
continue;
}
// Write the ticks
var writer = new LeanDataWriter(Resolution.Tick, symbol, Globals.DataFolder, tickType);
writer.Write(aggregateFunction(symbol, unzippedFile));
// Clean up unzipped file
File.Delete(unzippedFile);
}
}
}
}
/// <summary>
/// Create ticks from Raw Kaiko data
/// </summary>
/// <param name="fileLocation">Path to the raw Kaiko data</param>
/// <param name="market">The exchange the data represents</param>
/// <param name="tickType">The tick type being processed</param>
/// <param name="aggregateFunction">Function that parses the Kaiko data file and returns the enumerable of ticks</param>
private static void CreateCryptoTicks(string fileLocation, string market, TickType tickType, Func <Symbol, string, IEnumerable <Tick> > aggregateFunction)
{
foreach (var symbolFolder in Directory.EnumerateDirectories(fileLocation))
{
var symbolDirectoryInfo = new DirectoryInfo(symbolFolder);
// Create symbol from folder name
var symbol = Symbol.Create(symbolDirectoryInfo.Name, SecurityType.Crypto, market);
foreach (var symbolMonthDirectory in Directory.EnumerateDirectories(symbolDirectoryInfo.FullName))
{
foreach (var tradeFile in Directory.EnumerateFiles(symbolMonthDirectory))
{
// Unzip file
var unzippedFile = Compression.UnGZip(tradeFile, symbolMonthDirectory);
// Write the ticks
var writer = new LeanDataWriter(Resolution.Tick, symbol, Globals.DataFolder, tickType);
writer.Write(aggregateFunction(symbol, unzippedFile));
// Clean up unzipped file
File.Delete(unzippedFile);
}
}
}
}
/// <summary>
/// Quandl Bitfinex Toolbox Project For LEAN Algorithmic Trading Engine.
/// </summary>
public static void QuandlBitfinexDownloader(DateTime fromDate, string apiKey)
{
if (apiKey.IsNullOrEmpty())
{
Console.WriteLine("QuandlBitfinexDownloader ERROR: '--api-key=' parameter is missing");
Environment.Exit(1);
}
try
{
// Load settings from config.json
var dataDirectory = Config.Get("data-directory", "../../../Data");
// Create an instance of the downloader
const string market = Market.Bitfinex;
var downloader = new QuandlBitfinexDownloader(apiKey);
// Download the data
var symbol = Symbol.Create("BTCUSD", SecurityType.Forex, market);
var data = downloader.Get(symbol, Resolution.Daily, fromDate, DateTime.UtcNow);
// Save the data
var writer = new LeanDataWriter(Resolution.Daily, symbol, dataDirectory, TickType.Quote);
writer.Write(data);
}
catch (Exception err)
{
Log.Error(err);
}
}
/// <summary>
/// QuantConnect Google Downloader For LEAN Algorithmic Trading Engine.
/// Original by @chrisdk2015, tidied by @jaredbroad
/// </summary>
public static void Main(string[] args)
{
if (args.Length != 3)
{
Console.WriteLine("Usage: GoogleDownloader SYMBOL RESOLUTION PERIOD");
Console.WriteLine("SYMBOL = eg SPY");
Console.WriteLine("RESOLUTION = Minute/Hour");
Console.WriteLine("PERIOD = 10 for 10 days intraday data");
Environment.Exit(1);
}
try
{
// Load settings from command line
var symbol = args[0];
var resolution = (Resolution)Enum.Parse(typeof(Resolution), args[1]);
var period = args[2].ToInt32();
// Load settings from config.json
var dataDirectory = Config.Get("data-directory", "../../../Data");
// Download the data
var downloader = new GoogleDataDownloader();
var data = downloader.Get(new Symbol(symbol), SecurityType.Equity, resolution, DateTime.UtcNow.AddDays(-period), DateTime.UtcNow);
// Save the data
var writer = new LeanDataWriter(SecurityType.Equity, resolution, symbol, dataDirectory, "usa");
writer.Write(data);
}
catch (Exception err)
{
Log.Error("GoogleDownloader(): Error: " + err.Message);
}
}
/// <summary>
/// Yahoo Downloader Toolbox Project For LEAN Algorithmic Trading Engine.
/// Original by @chrisdk2015, tidied by @jaredbroad
/// </summary>
static void Main(string[] args)
{
if (args.Length != 1)
{
Console.WriteLine("Usage: YahooDownloader SYMBOL");
Console.WriteLine("Usage: Place the data into your LEAN Data directory: /data/equity/usa/daily/SYMBOL.zip");
Console.WriteLine("SYMBOL = eg SPY");
Environment.Exit(1);
}
//Command line inputs: symbol
var symbol = args[0];
var dataDirectory = Config.Get("data-directory", "../../../Data");
try
{
//Get Yahoo Downloader:
var yahooDownloader = new YahooDataDownloader();
var enumerableYahoo = yahooDownloader.Get(new Symbol(symbol), SecurityType.Equity, Resolution.Daily, DateTime.MinValue, DateTime.UtcNow);
//Get LEAN Data Writer:
var writer = new LeanDataWriter(SecurityType.Equity, Resolution.Daily, symbol, dataDirectory, "usa");
writer.Write(enumerableYahoo);
}
catch (Exception err)
{
Log.Error("YahooDownloader(): Error: " + err.Message);
}
}
/// <summary>
/// Quandl Bitfinex Toolbox Project For LEAN Algorithmic Trading Engine.
/// </summary>
static void Main(string[] args)
{
if (args.Length != 2)
{
Console.WriteLine("Usage: Downloader FROMDATE APIKEY");
Console.WriteLine("FROMDATE = yyyymmdd");
Environment.Exit(1);
}
try
{
// Load settings from config.json
var dataDirectory = Config.Get("data-directory", "../../../Data");
var scaleFactor = Config.GetInt("bitfinex-scale-factor", 100);
// Create an instance of the downloader
const string market = Market.Bitfinex;
var downloader = new QuandlBitfinexDownloader(args[1], scaleFactor);
// Download the data
var symbol = Symbol.Create("BTCUSD", SecurityType.Forex, market);
var data = downloader.Get(symbol, Resolution.Daily, DateTime.ParseExact(args[0], "yyyyMMdd", CultureInfo.CurrentCulture), DateTime.UtcNow);
// Save the data
var writer = new LeanDataWriter(Resolution.Daily, symbol, dataDirectory, TickType.Quote);
writer.Write(data);
}
catch (Exception err)
{
Log.Error(err);
}
}
/// <summary>
/// Cryptoiq Downloader Toolbox Project For LEAN Algorithmic Trading Engine.
/// </summary>
public static void CryptoiqDownloader(IList <string> tickers, string exchange, DateTime startDate, DateTime endDate)
{
if (exchange.IsNullOrEmpty() || tickers.IsNullOrEmpty())
{
Console.WriteLine("CryptoiqDownloader ERROR: '--exchange=' or '--tickers=' parameter is missing");
Environment.Exit(1);
}
try
{
// Load settings from config.json
var dataDirectory = Config.Get("data-directory", "../../../Data");
//todo: will download any exchange but always save as gdax
// Create an instance of the downloader
const string market = Market.GDAX;
var downloader = new CryptoiqDownloader(exchange);
foreach (var ticker in tickers)
{
// Download the data
var symbolObject = Symbol.Create(ticker, SecurityType.Crypto, market);
var data = downloader.Get(symbolObject, Resolution.Tick, startDate, endDate);
// Save the data
var writer = new LeanDataWriter(Resolution.Tick, symbolObject, dataDirectory, TickType.Quote);
writer.Write(data);
}
}
catch (Exception err)
{
Log.Error(err);
}
}
/// <summary>
/// Yahoo Downloader Toolbox Project For LEAN Algorithmic Trading Engine.
/// Original by @chrisdk2015, tidied by @jaredbroad
/// </summary>
static void Main(string[] args)
{
if (args.Length != 1)
{
Console.WriteLine("Usage: YahooDownloader SYMBOL");
Console.WriteLine("Usage: Place the data into your LEAN Data directory: /data/equity/usa/daily/SYMBOL.zip");
Console.WriteLine("SYMBOL = eg SPY");
Environment.Exit(1);
}
//Command line inputs: symbol
var symbol = args[0];
var dataDirectory = Config.Get("data-directory", "../../../Data");
try
{
//Get Yahoo Downloader:
var yahooDownloader = new YahooDataDownloader();
var enumerableYahoo = yahooDownloader.Get(new Symbol(symbol), SecurityType.Equity, Resolution.Daily, DateTime.MinValue, DateTime.UtcNow);
//Get LEAN Data Writer:
var writer = new LeanDataWriter(SecurityType.Equity, Resolution.Daily, symbol, dataDirectory, "usa");
writer.Write(enumerableYahoo);
}
catch (Exception err)
{
Log.Error("YahooDownloader(): Error: " + err.Message);
}
}
public void LeanDataWriter_MultipleDays()
{
var leanDataWriter = new LeanDataWriter(Resolution.Second, _forex, _dataDirectory, TickType.Quote);
var sourceData = new List <QuoteBar>
{
new (Parse.DateTime("3/16/2021 12:00:00 PM"), _forex, new Bar(1m, 2m, 3m, 4m), 1, new Bar(5m, 6m, 7m, 8m), 2)
};
for (var i = 1; i < 100; i++)
{
sourceData.Add(new QuoteBar(sourceData.Last().Time.AddDays(1),
_forex,
new Bar(1m, 2m, 3m, 4m),
1, new Bar(5m, 6m, 7m, 8m),
2));
}
leanDataWriter.Write(sourceData);
foreach (var bar in sourceData)
{
var filePath = LeanData.GenerateZipFilePath(_dataDirectory, _forex, bar.Time, Resolution.Second, TickType.Quote);
Assert.IsTrue(File.Exists(filePath));
Assert.IsFalse(File.Exists(filePath + ".tmp"));
var data = QuantConnect.Compression.Unzip(filePath).Single();
Assert.AreEqual(1, data.Value.Count());
Assert.IsTrue(data.Key.Contains(bar.Time.ToStringInvariant(DateFormat.EightCharacter)), $"Key {data.Key} BarTime: {bar.Time}");
}
}
public void RespectsWritePolicy(WritePolicy?writePolicy, Resolution resolution)
{
var filePath = LeanData.GenerateZipFilePath(_dataDirectory, _crypto, _date, resolution, TickType.Quote);
if (File.Exists(filePath))
{
File.Delete(filePath);
}
var loopCount = 3;
var dataPointsPerLoop = 2;
for (var i = 0; i < loopCount; i++)
{
var leanDataWriter = new LeanDataWriter(resolution, _crypto, _dataDirectory, TickType.Quote, writePolicy: writePolicy);
var quoteBar = new QuoteBar(Parse.DateTime("3/16/2017 12:00:00 PM").AddHours(i), _crypto, new Bar(1m, 2m, 3m, 4m), 1,
new Bar(5m, 6m, 7m, 8m), 2);
// same quote twice! it has the same time, so it will be dropped when merging
leanDataWriter.Write(Enumerable.Repeat(quoteBar, dataPointsPerLoop));
Assert.IsTrue(File.Exists(filePath));
Assert.IsFalse(File.Exists(filePath + ".tmp"));
}
var data = QuantConnect.Compression.Unzip(filePath).First().Value;
switch (writePolicy)
{
case WritePolicy.Overwrite:
Assert.AreEqual(dataPointsPerLoop, data.Count);
break;
case WritePolicy.Merge:
Assert.AreEqual(loopCount, data.Count);
break;
case WritePolicy.Append:
Assert.AreEqual(dataPointsPerLoop * loopCount, data.Count);
break;
case null:
if (resolution >= Resolution.Hour)
{
// will merge by default
Assert.AreEqual(loopCount, data.Count);
}
else
{
// overwrite
Assert.AreEqual(dataPointsPerLoop, data.Count);
}
break;
default:
throw new ArgumentOutOfRangeException(nameof(writePolicy), writePolicy, null);
}
}
/// <summary>
/// Processes the entry.
/// </summary>
/// <param name="coinapiDataReader">The coinapi data reader.</param>
/// <param name="file">The file.</param>
private void ProcessEntry(CoinApiDataReader coinapiDataReader, FileInfo file)
{
var entryData = coinapiDataReader.GetCoinApiEntryData(file, _processingDate);
if (!SupportedMarkets.Contains(entryData.Symbol.ID.Market))
{
// only convert data for supported exchanges
return;
}
// materialize the enumerable into a list, since we need to enumerate over it twice
var ticks = coinapiDataReader.ProcessCoinApiEntry(entryData, file).OrderBy(t => t.Time).ToList();
var writer = new LeanDataWriter(Resolution.Tick, entryData.Symbol, _destinationFolder.FullName, entryData.TickType);
writer.Write(ticks);
Log.Trace($"CoinApiDataConverter(): Starting consolidation for {entryData.Symbol.Value} {entryData.TickType}");
var consolidators = new List <TickAggregator>();
if (entryData.TickType == TickType.Trade)
{
consolidators.AddRange(new[]
{
new TradeTickAggregator(Resolution.Second),
new TradeTickAggregator(Resolution.Minute)
});
}
else
{
consolidators.AddRange(new[]
{
new QuoteTickAggregator(Resolution.Second),
new QuoteTickAggregator(Resolution.Minute)
});
}
foreach (var tick in ticks)
{
if (tick.Suspicious)
{
// When CoinAPI loses connectivity to the exchange, they indicate
// it in the data by providing a value of `-1` for bid/ask price.
// We will keep it in tick data, but will remove it from consolidated data.
continue;
}
foreach (var consolidator in consolidators)
{
consolidator.Update(tick);
}
}
foreach (var consolidator in consolidators)
{
writer = new LeanDataWriter(consolidator.Resolution, entryData.Symbol, _destinationFolder.FullName, entryData.TickType);
writer.Write(consolidator.Flush());
}
}
public static void GenerateRandomData(RandomDataGeneratorSettings settings, ConsoleLeveledOutput output)
{
// can specify a seed value in this ctor if determinism is desired
var randomValueGenerator = new RandomValueGenerator();
var symbolGenerator = new SymbolGenerator(settings, randomValueGenerator);
var tickGenerator = new TickGenerator(settings, randomValueGenerator);
output.Warn.WriteLine($"Begin data generation of {settings.SymbolCount} randomly generated {settings.SecurityType} assets...");
// iterate over our randomly generated symbols
var count = 0;
var progress = 0d;
var previousMonth = -1;
foreach (var symbol in symbolGenerator.GenerateRandomSymbols())
{
output.Warn.WriteLine($"\tSymbol[{++count}]: {symbol} Progress: {progress:0.0}% - Generating data...");
// define aggregators via settings
var aggregators = settings.CreateAggregators().ToList();
// generate and consolidate data
foreach (var tick in tickGenerator.GenerateTicks(symbol))
{
if (tick.Time.Month != previousMonth)
{
output.Info.WriteLine($"\tMonth: {tick.Time:MMMM}");
previousMonth = tick.Time.Month;
}
foreach (var item in aggregators)
{
item.Consolidator.Update(tick);
}
}
// count each stage as a point, so total points is 2*symbol-count
// and the current progress is twice the current, but less one because we haven't finished writing data yet
progress = 100 * (2 * count - 1) / (2.0 * settings.SymbolCount);
output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} Progress: {progress:0.0}% - Saving data in LEAN format");
// persist consolidated data to disk
foreach (var item in aggregators)
{
var writer = new LeanDataWriter(item.Resolution, symbol, Globals.DataFolder, item.TickType);
// send the flushed data into the writer. pulling the flushed list is very important,
// lest we likely wouldn't get the last piece of data stuck in the consolidator
writer.Write(item.Flush());
}
// update progress
progress = 100 * (2 * count) / (2.0 * settings.SymbolCount);
output.Warn.WriteLine($"\tSymbol[{count}]: {symbol} Progress: {progress:0.0}% - Symbol data generation and output completed");
}
output.Info.WriteLine("Random data generation has completed.");
}
/// <summary>
/// Create ticks from Raw Kaiko data
/// </summary>
/// <param name="fileLocation">Path to the raw Kaiko data</param>
/// <param name="market">The exchange the data represents</param>
/// <param name="tickType">The tick type being processed</param>
/// <param name="aggregateFunction">Function that parses the Kaiko data file and returns the enumerable of ticks</param>
private static void CreateCryptoTicks(string fileLocation, string market, TickType tickType, Func <Symbol, string, IEnumerable <Tick> > aggregateFunction, string ticker)
{
// Create symbol from folder name
var symbol = Symbol.Create(ticker, SecurityType.Crypto, market);
foreach (var tradeFile in Directory.EnumerateFiles(fileLocation))
{
// Write the ticks
var writer = new LeanDataWriter(Resolution.Tick, symbol, Globals.DataFolder, tickType);
writer.Write(aggregateFunction(symbol, tradeFile));
}
}
/// <summary>
/// GDAX Downloader Toolbox Project For LEAN Algorithmic Trading Engine.
/// </summary>
static void Main(string[] args)
{
Thread.CurrentThread.CurrentCulture = new CultureInfo("en-US");
if (args.Length < 3)
{
Console.WriteLine("Usage: GDAX Downloader SYMBOL RESOLUTION FROMDATE TODATE");
Console.WriteLine("SYMBOL = ETHUSD, ETHBTC, BTCUSD etc.");
Console.WriteLine("RESOLUTION = Second/Minute/Hour/Daily");
Console.WriteLine("FROMDATE = yyyyMMdd HH:mm:ss");
Console.WriteLine("TODATE = yyyyMMdd HH:mm:ss");
Environment.Exit(1);
}
try
{
var resolution = (Resolution)Enum.Parse(typeof(Resolution), args[1]);
// Load settings from command line
var startDate = DateTime.ParseExact(args[2], "yyyyMMdd HH:mm:ss", CultureInfo.InvariantCulture);
var endDate = DateTime.UtcNow;
if (args[3] != null)
{
endDate = DateTime.ParseExact(args[3], "yyyyMMdd HH:mm:ss", CultureInfo.InvariantCulture);
}
// Load settings from config.json
var dataDirectory = Config.Get("data-directory", "../../../Data");
//todo: will download any exchange but always save as gdax
// Create an instance of the downloader
const string market = Market.GDAX;
var downloader = new GDAXDownloader();
// Download the data
var symbolObject = Symbol.Create(args[0], SecurityType.Crypto, market);
var data = downloader.Get(symbolObject, resolution, startDate, endDate);
// Save the data
var writer = new LeanDataWriter(resolution, symbolObject, dataDirectory, TickType.Trade);
var distinctData = data.GroupBy(i => i.Time, (key, group) => group.First()).ToArray();
writer.Write(distinctData);
Log.Trace("Finish data download. Press any key to continue..");
}
catch (Exception err)
{
Log.Error(err);
Log.Trace(err.Message);
Log.Trace(err.StackTrace);
}
// Console.ReadLine();
}
/// <summary>
/// Processes the entry.
/// </summary>
/// <param name="coinapiDataReader">The coinapi data reader.</param>
/// <param name="file">The file.</param>
private void ProcessEntry(CoinApiDataReader coinapiDataReader, FileInfo file)
{
var entryData = coinapiDataReader.GetCoinApiEntryData(file, _processingDate, _market);
if (!SupportedMarkets.Contains(entryData.Symbol.ID.Market))
{
// only convert data for supported exchanges
return;
}
// materialize the enumerable into a list, since we need to enumerate over it twice
var ticks = coinapiDataReader.ProcessCoinApiEntry(entryData, file).ToList();
var writer = new LeanDataWriter(Resolution.Tick, entryData.Symbol, _destinationFolder.FullName, entryData.TickType);
writer.Write(ticks);
Log.Trace($"CoinApiDataConverter(): Starting consolidation for {entryData.Symbol.Value} {entryData.TickType}");
var consolidators = new List <TickAggregator>();
if (entryData.TickType == TickType.Trade)
{
consolidators.AddRange(new[]
{
new TradeTickAggregator(Resolution.Second),
new TradeTickAggregator(Resolution.Minute)
});
}
else
{
consolidators.AddRange(new[]
{
new QuoteTickAggregator(Resolution.Second),
new QuoteTickAggregator(Resolution.Minute)
});
}
foreach (var tick in ticks)
{
foreach (var consolidator in consolidators)
{
consolidator.Update(tick);
}
}
foreach (var consolidator in consolidators)
{
writer = new LeanDataWriter(consolidator.Resolution, entryData.Symbol, _destinationFolder.FullName, entryData.TickType);
writer.Write(consolidator.Flush());
}
}
/// <summary>
/// Determines if it should downloads new data and retrieves data from disc
/// </summary>
/// <param name="key">A string representing where the data is stored</param>
/// <returns>A <see cref="Stream"/> of the data requested</returns>
public override Stream Fetch(string key)
{
return(DownloadOnce(key, s =>
{
if (LeanData.TryParsePath(key, out var symbol, out var date, out var resolution, out var tickType, out var dataType))
{
var dataTimeZone = MarketHoursDatabase.FromDataFolder().GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType);
DateTime startTimeUtc;
DateTime endTimeUtc;
if (resolution < Resolution.Hour)
{
// we can get the date from the path
startTimeUtc = date.ConvertToUtc(dataTimeZone);
endTimeUtc = date.Add(resolution.ToTimeSpan()).ConvertToUtc(dataTimeZone);
}
else
{
try
{
startTimeUtc = symbol.ID.Date;
}
catch (InvalidOperationException)
{
startTimeUtc = Time.BeginningOfTime;
}
endTimeUtc = DateTime.UtcNow;
}
// Save the data
var writer = new LeanDataWriter(resolution, symbol, Globals.DataFolder, tickType);
try
{
var getParams = new DataDownloaderGetParameters(symbol, resolution, startTimeUtc, endTimeUtc, tickType);
var data = _dataDownloader.Get(getParams)
.Where(baseData => symbol.SecurityType == SecurityType.Base || baseData.GetType() == dataType)
// for canonical symbols, downloader will return data for all of the chain
.GroupBy(baseData => baseData.Symbol);
foreach (var dataPerSymbol in data)
{
writer.Write(dataPerSymbol);
}
}
catch (Exception e)
{
Log.Error(e);
}
}
}));
/// <summary>
/// Primary entry point to the program.
/// </summary>
public static void DataDownloader(IList <string> tickers, string resolution, DateTime fromDate, DateTime toDate)
{
if (resolution.IsNullOrEmpty() || tickers.IsNullOrEmpty())
{
Console.WriteLine("BinanceDownloader ERROR: '--tickers=' or '--resolution=' parameter is missing");
Console.WriteLine("--tickers=eg BTCUSD");
Console.WriteLine("--resolution=Minute/Hour/Daily/All");
Environment.Exit(1);
}
try
{
var allResolutions = resolution.Equals("all", StringComparison.OrdinalIgnoreCase);
var castResolution = allResolutions ? Resolution.Minute : (Resolution)Enum.Parse(typeof(Resolution), resolution);
// Load settings from config.json
var dataDirectory = Config.Get("data-folder", "../../../Data");
using (var downloader = new BinanceDataDownloader())
{
foreach (var ticker in tickers)
{
// Download the data
var startDate = fromDate;
var symbol = downloader.GetSymbol(ticker);
var data = downloader.Get(symbol, castResolution, fromDate, toDate);
var bars = data.Cast <TradeBar>().ToList();
// Save the data (single resolution)
var writer = new LeanDataWriter(castResolution, symbol, dataDirectory);
writer.Write(bars);
if (allResolutions)
{
// Save the data (other resolutions)
foreach (var res in new[] { Resolution.Hour, Resolution.Daily })
{
var resData = LeanData.AggregateTradeBars(bars, symbol, res.ToTimeSpan());
writer = new LeanDataWriter(res, symbol, dataDirectory);
writer.Write(resData);
}
}
}
}
}
catch (Exception err)
{
Log.Error(err);
}
}
/// <summary>
/// GDAX Downloader Toolbox Project For LEAN Algorithmic Trading Engine.
/// </summary>
static void Main(string[] args)
{
Thread.CurrentThread.CurrentCulture = new CultureInfo("en-US");
if (args.Length == 2)
{
args = new [] { args[0], DateTime.UtcNow.ToString("yyyyMMdd"), args[1] };
}
else if (args.Length < 3)
{
Console.WriteLine("Usage: GDAX Downloader SYMBOL FROMDATE TODATE");
Console.WriteLine("FROMDATE = yyyymmdd");
Console.WriteLine("TODATE = yyyymmdd");
Environment.Exit(1);
}
try
{
// Load settings from command line
var startDate = DateTime.ParseExact(args[1], "yyyyMMdd", CultureInfo.InvariantCulture);
var endDate = DateTime.ParseExact(args[2], "yyyyMMdd", CultureInfo.InvariantCulture);
// Load settings from config.json
var dataDirectory = Config.Get("data-directory", "../../../Data");
//todo: will download any exchange but always save as gdax
// Create an instance of the downloader
const string market = Market.GDAX;
var downloader = new GDAXDownloader();
// Download the data
var symbolObject = Symbol.Create(args[0], SecurityType.Crypto, market);
var data = downloader.Get(symbolObject, Resolution.Hour, startDate, endDate);
// Save the data
var writer = new LeanDataWriter(Resolution.Hour, symbolObject, dataDirectory, TickType.Trade);
var distinctData = data.GroupBy(i => i.Time, (key, group) => group.First()).ToArray();
writer.Write(distinctData);
Log.Trace("Finish data download");
}
catch (Exception err)
{
Log.Error(err);
Log.Trace(err.Message);
Log.Trace(err.StackTrace);
}
Console.ReadLine();
}
/// <summary>
/// Primary entry point to the program. This program only supports FOREX for now.
/// </summary>
public static void IEXDownloader(IList <string> tickers, string resolution, DateTime fromDate, DateTime toDate, string apiKey)
{
if (resolution.IsNullOrEmpty() || tickers.IsNullOrEmpty())
{
Console.WriteLine("IEXDownloader ERROR: '--tickers=' or '--resolution=' parameter is missing");
Console.WriteLine("--tickers=eg SPY,AAPL");
Console.WriteLine("--resolution=Minute/Daily");
Environment.Exit(1);
}
if (apiKey.IsNullOrEmpty())
{
Console.WriteLine("QuandlBitfinexDownloader ERROR: '--api-key=' parameter is missing");
Environment.Exit(1);
}
try
{
// Load settings from command line
var castResolution = (Resolution)Enum.Parse(typeof(Resolution), resolution);
// Load settings from config.json
var dataDirectory = Config.Get("data-directory", "../../../Data");
var startDate = fromDate.ConvertToUtc(TimeZones.NewYork);
var endDate = toDate.ConvertToUtc(TimeZones.NewYork);
// Create an instance of the downloader
const string market = Market.USA;
SecurityType securityType = SecurityType.Equity;
using (var downloader = new IEXDataDownloader(apiKey = apiKey))
{
foreach (var ticker in tickers)
{
// Download the data
var symbolObject = Symbol.Create(ticker, securityType, market);
var data = downloader.Get(symbolObject, castResolution, startDate, endDate);
// Save the data
var writer = new LeanDataWriter(castResolution, symbolObject, dataDirectory);
writer.Write(data);
}
}
}
catch (Exception err)
{
Log.Error(err);
}
Console.ReadLine();
}
public void LeanDataWriter_CanWriteCrypto()
{
var filePath = LeanData.GenerateZipFilePath(_dataDirectory, _crypto, _date, Resolution.Second, TickType.Quote);
var leanDataWriter = new LeanDataWriter(Resolution.Second, _crypto, _dataDirectory, TickType.Quote);
leanDataWriter.Write(GetQuoteBars(_crypto));
Assert.IsTrue(File.Exists(filePath));
Assert.IsFalse(File.Exists(filePath + ".tmp"));
var data = QuantConnect.Compression.Unzip(filePath);
Assert.AreEqual(data.First().Value.Count(), 3);
}
public void LeanDataWriter_CanWriteEquity()
{
var filePath = LeanData.GenerateZipFilePath(_dataDirectory, _equity, _date, Resolution.Tick, TickType.Trade);
var leanDataWriter = new LeanDataWriter(Resolution.Tick, _equity, _dataDirectory);
leanDataWriter.Write(GetTicks(_equity));
Assert.IsTrue(File.Exists(filePath));
Assert.IsFalse(File.Exists(filePath + ".tmp"));
var data = QuantConnect.Compression.Unzip(filePath);
Assert.AreEqual(data.First().Value.Count(), 3);
}
public void CanDownloadAndSave(SecurityType securityType, TickType tickType, Resolution resolution)
{
var symbol = Symbols.GetBySecurityType(securityType);
var startTimeUtc = GetRepoDataDates(securityType, resolution);
// Override for this case because symbol from Symbols does not have data included
if (securityType == SecurityType.Option)
{
symbol = Symbols.CreateOptionSymbol("GOOG", OptionRight.Call, 770, new DateTime(2015, 12, 24));
startTimeUtc = new DateTime(2015, 12, 23);
}
// EndTime based on start, only do 1 day for anything less than hour because we compare datafiles below
// and minute and finer resolutions store by day
var endTimeUtc = startTimeUtc + TimeSpan.FromDays(resolution >= Resolution.Hour ? 15 : 1);
// Create our writer and LocalHistory brokerage to "download" from
var writer = new LeanDataWriter(_dataDirectory, resolution, securityType, tickType);
var brokerage = new LocalHistoryBrokerage();
var symbols = new List <Symbol>()
{
symbol
};
// "Download" and write to file
writer.DownloadAndSave(brokerage, symbols, startTimeUtc, endTimeUtc);
// Verify the file exists where we expect
var filePath = LeanData.GenerateZipFilePath(_dataDirectory, symbol, startTimeUtc, resolution, tickType);
Assert.IsTrue(File.Exists(filePath));
// Read the file and data
var reader = new LeanDataReader(filePath);
var dataFromFile = reader.Parse().ToList();
// Ensure its not empty and it is actually for this symbol
Assert.IsNotEmpty(dataFromFile);
Assert.IsTrue(dataFromFile.All(x => x.Symbol == symbol));
// Get history directly ourselves and compare with the data in the file
var history = GetHistory(brokerage, resolution, securityType, symbol, tickType, startTimeUtc, endTimeUtc);
CollectionAssert.AreEqual(history.Select(x => x.Time), dataFromFile.Select(x => x.Time));
brokerage.Dispose();
}
/// <summary>
/// Primary entry point to the program
/// </summary>
static void Main(string[] args)
{
if (args.Length != 4)
{
Console.WriteLine("Usage: OandaDownloader SYMBOL RESOLUTION FROMDATE TODATE");
Console.WriteLine("SYMBOL = eg EURUSD");
Console.WriteLine("RESOLUTION = Second/Minute/Hour/Daily");
Console.WriteLine("FROMDATE = yyyymmdd");
Console.WriteLine("TODATE = yyyymmdd");
Environment.Exit(1);
}
try
{
// Load settings from command line
var symbol = args[0];
var resolution = (Resolution)Enum.Parse(typeof(Resolution), args[1]);
var startDate = DateTime.ParseExact(args[2], "yyyyMMdd", CultureInfo.InvariantCulture);
var endDate = DateTime.ParseExact(args[3], "yyyyMMdd", CultureInfo.InvariantCulture);
// Load settings from config.json
var dataDirectory = Config.Get("data-directory", "../../../Data");
var accessToken = Config.Get("access-token", "73eba38ad5b44778f9a0c0fec1a66ed1-44f47f052c897b3e1e7f24196bbc071f");
var accountId = Convert.ToInt32(Config.Get("account-id", "621396"));
// Download the data
var downloader = new OandaDataDownloader(accessToken, accountId);
if (!downloader.HasSymbol(symbol))
{
throw new ArgumentException("The symbol " + symbol + " is not available.");
}
var securityType = downloader.GetSecurityType(symbol);
var data = downloader.Get(new Symbol(symbol), securityType, resolution, startDate, endDate);
// Save the data
var writer = new LeanDataWriter(securityType, resolution, symbol, dataDirectory, "oanda");
writer.Write(data);
}
catch (Exception err)
{
Log.Error("OandaDownloader(): Error: " + err.Message);
}
}
/// <summary>
/// GDAX Downloader Toolbox Project For LEAN Algorithmic Trading Engine.
/// </summary>
public static void GDAXDownloader(IList <string> tickers, string resolution, DateTime fromDate, DateTime toDate)
{
Thread.CurrentThread.CurrentCulture = new CultureInfo("en-US");
if (resolution.IsNullOrEmpty() || tickers.IsNullOrEmpty())
{
Console.WriteLine("GDAXDownloader ERROR: '--tickers=' or '--resolution=' parameter is missing");
Console.WriteLine("--tickers=ETHUSD,ETHBTC,BTCUSD,etc.");
Console.WriteLine("--resolution=Second/Minute/Hour/Daily");
Environment.Exit(1);
}
var castResolution = (Resolution)Enum.Parse(typeof(Resolution), resolution);
try
{
// Load settings from config.json
var dataDirectory = Config.Get("data-directory", "../../../Data");
//todo: will download any exchange but always save as gdax
// Create an instance of the downloader
const string market = Market.GDAX;
var downloader = new GDAXDownloader();
foreach (var ticker in tickers)
{
// Download the data
var symbolObject = Symbol.Create(ticker, SecurityType.Crypto, market);
var data = downloader.Get(symbolObject, castResolution, fromDate, toDate);
// Save the data
var writer = new LeanDataWriter(castResolution, symbolObject, dataDirectory, TickType.Trade);
var distinctData = data.GroupBy(i => i.Time, (key, group) => group.First()).ToArray();
writer.Write(distinctData);
}
Log.Trace("Finish data download. Press any key to continue..");
}
catch (Exception err)
{
Log.Error(err);
Log.Trace(err.Message);
Log.Trace(err.StackTrace);
}
Console.ReadLine();
}
public void LeanDataWriter_CanWriteZipWithMultipleContracts(SecurityType securityType)
{
Symbol contract1;
Symbol contract2;
if (securityType == SecurityType.Future)
{
contract1 = Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, new DateTime(2020, 02, 01));
contract2 = Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, new DateTime(2020, 03, 01));
}
else if (securityType == SecurityType.Option)
{
contract1 = Symbol.CreateOption("AAPL", Market.USA, OptionStyle.American, OptionRight.Call, 1, new DateTime(2020, 02, 01));
contract2 = Symbol.CreateOption("AAPL", Market.USA, OptionStyle.American, OptionRight.Call, 1, new DateTime(2020, 03, 01));
}
else if (securityType == SecurityType.FutureOption)
{
contract1 = Symbol.CreateOption(Futures.Indices.SP500EMini, Market.CME, OptionStyle.American, OptionRight.Call, 1, new DateTime(2020, 02, 01));
contract2 = Symbol.CreateOption(Futures.Indices.SP500EMini, Market.CME, OptionStyle.American, OptionRight.Call, 1, new DateTime(2020, 03, 01));
}
else
{
throw new NotImplementedException($"{securityType} not implemented!");
}
var filePath1 = LeanData.GenerateZipFilePath(_dataDirectory, contract1, _date, Resolution.Second, TickType.Quote);
var leanDataWriter1 = new LeanDataWriter(Resolution.Second, contract1, _dataDirectory, TickType.Quote);
leanDataWriter1.Write(GetQuoteBars(contract1));
var filePath2 = LeanData.GenerateZipFilePath(_dataDirectory, contract2, _date, Resolution.Second, TickType.Quote);
var leanDataWriter2 = new LeanDataWriter(Resolution.Second, contract2, _dataDirectory, TickType.Quote);
leanDataWriter2.Write(GetQuoteBars(contract2));
Assert.AreEqual(filePath1, filePath2);
Assert.IsTrue(File.Exists(filePath1));
Assert.IsFalse(File.Exists(filePath1 + ".tmp"));
var data = QuantConnect.Compression.Unzip(filePath1).ToDictionary(x => x.Key, x => x.Value.ToList());
Assert.AreEqual(2, data.Count);
Assert.That(data.Values, Has.All.Count.EqualTo(3));
}
/// <summary>
/// Yahoo Downloader Toolbox Project For LEAN Algorithmic Trading Engine.
/// Original by @chrisdk2015, tidied by @jaredbroad
/// </summary>
static void Main(string[] args)
{
if (args.Length != 4)
{
Console.WriteLine("Usage: YahooDownloader SYMBOLS RESOLUTION FROMDATE TODATE");
Console.WriteLine("SYMBOLS = eg SPY,AAPL");
Console.WriteLine("RESOLUTION = Daily");
Console.WriteLine("FROMDATE = yyyymmdd");
Console.WriteLine("TODATE = yyyymmdd");
Environment.Exit(1);
}
try
{
// Load settings from command line
var symbols = args[0].Split(',');
var resolution = (Resolution)Enum.Parse(typeof(Resolution), args[1]);
var startDate = DateTime.ParseExact(args[2], "yyyyMMdd", CultureInfo.InvariantCulture);
var endDate = DateTime.ParseExact(args[3], "yyyyMMdd", CultureInfo.InvariantCulture);
// Load settings from config.json
var dataDirectory = Config.Get("data-directory", "../../../Data");
// Create an instance of the downloader
const string market = Market.USA;
var downloader = new YahooDataDownloader();
foreach (var symbol in symbols)
{
// Download the data
var sid = SecurityIdentifier.GenerateEquity(symbol, market);
var symbolObject = new Symbol(sid, symbol);
var data = downloader.Get(symbolObject, SecurityType.Equity, resolution, startDate, endDate);
// Save the data
var writer = new LeanDataWriter(SecurityType.Equity, resolution, symbolObject, dataDirectory, market);
writer.Write(data);
}
}
catch (Exception err)
{
Log.Error("YahooDownloader(): Error: " + err.Message);
}
}
/// <summary>
/// QuantConnect Google Downloader For LEAN Algorithmic Trading Engine.
/// Original by @chrisdk2015, tidied by @jaredbroad
/// </summary>
public static void Main(string[] args)
{
if (args.Length != 4)
{
Console.WriteLine("Usage: GoogleDownloader SYMBOLS RESOLUTION FROMDATE TODATE");
Console.WriteLine("SYMBOLS = eg SPY,AAPL");
Console.WriteLine("RESOLUTION = Minute/Hour");
Console.WriteLine("FROMDATE = yyyymmdd");
Console.WriteLine("TODATE = yyyymmdd");
Environment.Exit(1);
}
try
{
// Load settings from command line
var symbols = args[0].Split(',');
var resolution = (Resolution)Enum.Parse(typeof(Resolution), args[1]);
var startDate = DateTime.ParseExact(args[2], "yyyyMMdd", CultureInfo.InvariantCulture);
var endDate = DateTime.ParseExact(args[3], "yyyyMMdd", CultureInfo.InvariantCulture);
// Load settings from config.json
var dataDirectory = Config.Get("data-directory", "../../../Data");
// Create an instance of the downloader
const string market = Market.USA;
var downloader = new GoogleDataDownloader();
foreach (var symbol in symbols)
{
// Download the data
var sid = SecurityIdentifier.GenerateEquity(symbol, market);
var symbolObject = new Symbol(sid, symbol);
var data = downloader.Get(symbolObject, SecurityType.Equity, resolution, startDate, endDate);
// Save the data
var writer = new LeanDataWriter(SecurityType.Equity, resolution, symbolObject, dataDirectory, market);
writer.Write(data);
}
}
catch (Exception err)
{
Log.Error("GoogleDownloader(): Error: " + err.Message);
}
}
/// <summary>
/// QuantConnect Google Downloader For LEAN Algorithmic Trading Engine.
/// Original by @chrisdk2015, tidied by @jaredbroad
/// </summary>
public static void Main(string[] args)
{
//if (args.Length != 4)
//{
// Console.WriteLine("Usage: GoogleDownloader SYMBOLS RESOLUTION FROMDATE TODATE");
// Console.WriteLine("SYMBOLS = eg SPY,AAPL");
// Console.WriteLine("RESOLUTION = Minute/Hour");
// Console.WriteLine("FROMDATE = yyyymmdd");
// Console.WriteLine("TODATE = yyyymmdd");
// Environment.Exit(1);
//}
try
{
// Load settings from command line
var symbols = "GARAN,AKBNK".Split(',');
var resolution = (Resolution)Enum.Parse(typeof(Resolution), "Minute");
var startDate = DateTime.ParseExact("20150101", "yyyyMMdd", CultureInfo.InvariantCulture);
var endDate = DateTime.ParseExact("20160101", "yyyyMMdd", CultureInfo.InvariantCulture);
// Load settings from config.json
var dataDirectory = Config.Get("data-directory", "../../../Data");
// Create an instance of the downloader
const string market = Market.USA;
var downloader = new GoogleDataDownloader();
foreach (var symbol in symbols)
{
// Download the data
var symbolObject = Symbol.Create(symbol, SecurityType.Equity, market);
var data = downloader.Get(symbolObject, resolution, startDate, endDate);
// Save the data
var writer = new LeanDataWriter(resolution, symbolObject, dataDirectory);
writer.Write(data);
}
}
catch (Exception err)
{
Log.Error(err);
}
}
/// <summary>
/// Primary entry point to the program
/// </summary>
static void Main(string[] args)
{
if (args.Length != 4)
{
Console.WriteLine("Usage: DukascopyDownloader SYMBOL RESOLUTION FROMDATE TODATE");
Console.WriteLine("SYMBOL = eg EURUSD");
Console.WriteLine("RESOLUTION = Tick/Second/Minute/Hour/Daily");
Console.WriteLine("FROMDATE = yyyymmdd");
Console.WriteLine("TODATE = yyyymmdd");
Environment.Exit(1);
}
try
{
// Load settings from command line
var symbol = args[0];
var resolution = (Resolution)Enum.Parse(typeof(Resolution), args[1]);
var startDate = DateTime.ParseExact(args[2], "yyyyMMdd", CultureInfo.InvariantCulture);
var endDate = DateTime.ParseExact(args[3], "yyyyMMdd", CultureInfo.InvariantCulture);
// Load settings from config.json
var dataDirectory = Config.Get("data-directory", "../../../Data");
// Download the data
var downloader = new DukascopyDataDownloader();
if (!downloader.HasSymbol(symbol))
{
throw new ArgumentException("The symbol " + symbol + " is not available.");
}
var securityType = downloader.GetSecurityType(symbol);
var data = downloader.Get(new Symbol(symbol), securityType, resolution, startDate, endDate);
// Save the data
var writer = new LeanDataWriter(securityType, resolution, symbol, dataDirectory, "dukascopy");
writer.Write(data);
}
catch (Exception err)
{
Log.Error("DukascopyDownloader(): {0}", err.Message);
}
}
/// <summary>
/// Primary entry point to the program
/// </summary>
static void Main(string[] args)
{
if (args.Length != 4)
{
Console.WriteLine("Usage: OandaDownloader SYMBOL RESOLUTION FROMDATE TODATE");
Console.WriteLine("SYMBOL = eg EURUSD");
Console.WriteLine("RESOLUTION = Second/Minute/Hour/Daily");
Console.WriteLine("FROMDATE = yyyymmdd");
Console.WriteLine("TODATE = yyyymmdd");
Environment.Exit(1);
}
try
{
// Load settings from command line
var symbol = args[0];
var resolution = (Resolution)Enum.Parse(typeof(Resolution), args[1]);
var startDate = DateTime.ParseExact(args[2], "yyyyMMdd", CultureInfo.InvariantCulture);
var endDate = DateTime.ParseExact(args[3], "yyyyMMdd", CultureInfo.InvariantCulture);
// Load settings from config.json
var dataDirectory = Config.Get("data-directory", "../../../Data");
var accessToken = Config.Get("access-token", "73eba38ad5b44778f9a0c0fec1a66ed1-44f47f052c897b3e1e7f24196bbc071f");
var accountId = Convert.ToInt32(Config.Get("account-id", "621396"));
// Download the data
var downloader = new OandaDataDownloader(accessToken, accountId);
if (!downloader.HasSymbol(symbol))
throw new ArgumentException("The symbol " + symbol + " is not available.");
var securityType = downloader.GetSecurityType(symbol);
var data = downloader.Get(new Symbol(symbol), securityType, resolution, startDate, endDate);
// Save the data
var writer = new LeanDataWriter(securityType, resolution, symbol, dataDirectory, "oanda");
writer.Write(data);
}
catch (Exception err)
{
Log.Error("OandaDownloader(): Error: " + err.Message);
}
}
/// <summary>
/// Cryptoiq Downloader Toolbox Project For LEAN Algorithmic Trading Engine.
/// </summary>
static void Main(string[] args)
{
if (args.Length == 3)
{
args = new [] { args[0], DateTime.UtcNow.ToString("yyyyMMdd"), args[1], args[2] };
}
else if (args.Length < 4)
{
Console.WriteLine("Usage: CryptoiqDownloader FROMDATE TODATE EXCHANGE SYMBOL");
Console.WriteLine("FROMDATE = yyyymmdd");
Console.WriteLine("TODATE = yyyymmdd");
Environment.Exit(1);
}
try
{
// Load settings from command line
var startDate = DateTime.ParseExact(args[0], "yyyyMMdd", CultureInfo.InvariantCulture);
var endDate = DateTime.ParseExact(args[1], "yyyyMMdd", CultureInfo.InvariantCulture);
// Load settings from config.json
var dataDirectory = Config.Get("data-directory", "../../../Data");
var scaleFactor = Config.GetValue("bitfinex-scale-factor", 1m);
// Create an instance of the downloader
const string market = Market.Bitfinex;
var downloader = new CryptoiqDownloader(args[2], scaleFactor);
// Download the data
var symbolObject = Symbol.Create(args[3], SecurityType.Crypto, market);
var data = downloader.Get(symbolObject, Resolution.Tick, startDate, endDate);
// Save the data
var writer = new LeanDataWriter(Resolution.Tick, symbolObject, dataDirectory, TickType.Quote);
writer.Write(data);
}
catch (Exception err)
{
Log.Error(err);
}
}
/// <summary>
/// Primary entry point to the program
/// </summary>
static void Main(string[] args)
{
if (args.Length != 4)
{
Console.WriteLine("Usage: DukascopyDownloader SYMBOL RESOLUTION FROMDATE TODATE");
Console.WriteLine("SYMBOL = eg EURUSD");
Console.WriteLine("RESOLUTION = Tick/Second/Minute/Hour/Daily");
Console.WriteLine("FROMDATE = yyyymmdd");
Console.WriteLine("TODATE = yyyymmdd");
Environment.Exit(1);
}
try
{
// Load settings from command line
var symbol = args[0];
var resolution = (Resolution)Enum.Parse(typeof(Resolution), args[1]);
var startDate = DateTime.ParseExact(args[2], "yyyyMMdd", CultureInfo.InvariantCulture);
var endDate = DateTime.ParseExact(args[3], "yyyyMMdd", CultureInfo.InvariantCulture);
// Load settings from config.json
var dataDirectory = Config.Get("data-directory", "../../../Data");
// Download the data
var downloader = new DukascopyDataDownloader();
if (!downloader.HasSymbol(symbol))
throw new ArgumentException("The symbol " + symbol + " is not available.");
var securityType = downloader.GetSecurityType(symbol);
var data = downloader.Get(new Symbol(symbol), securityType, resolution, startDate, endDate);
// Save the data
var writer = new LeanDataWriter(securityType, resolution, symbol, dataDirectory, "dukascopy");
writer.Write(data);
}
catch (Exception err)
{
Log.Error("DukascopyDownloader(): {0}", err.Message);
}
}
/// <summary>
/// Primary entry point to the program.
/// </summary>
public static void BitfinexDownloader(IList <string> tickers, string resolution, DateTime fromDate, DateTime toDate)
{
if (resolution.IsNullOrEmpty() || tickers.IsNullOrEmpty())
{
Console.WriteLine("BitfinexDownloader ERROR: '--tickers=' or '--resolution=' parameter is missing");
Console.WriteLine("--tickers=eg BTCUSD");
Console.WriteLine("--resolution=Second/Minute/Hour/Daily/All");
Environment.Exit(1);
}
try
{
var allResolutions = resolution.ToLowerInvariant() == "all";
var resolutions = allResolutions ?
new[] { Resolution.Minute, Resolution.Hour, Resolution.Daily } :
new[] { (Resolution)Enum.Parse(typeof(Resolution), resolution) };
// Load settings from config.json
var dataDirectory = Config.Get("data-folder", "../../../Data");
using (var downloader = new BitfinexDataDownloader())
{
foreach (var ticker in tickers)
{
// Download the data
var symbol = downloader.GetSymbol(ticker);
foreach (var castResolution in resolutions)
{
var data = downloader.Get(new DataDownloaderGetParameters(symbol, castResolution, fromDate, toDate));
// Save the data (single resolution)
var writer = new LeanDataWriter(castResolution, symbol, dataDirectory);
writer.Write(data);
}
}
}
}
catch (Exception err)
{
Log.Error(err);
}
}
/// <summary>
/// Cryptoiq Downloader Toolbox Project For LEAN Algorithmic Trading Engine.
/// </summary>
static void Main(string[] args)
{
if (args.Length == 3)
{
args = new [] { args[0], DateTime.UtcNow.ToString("yyyyMMdd"), args[1], args[2] };
}
else if (args.Length < 4)
{
Console.WriteLine("Usage: CryptoiqDownloader FROMDATE TODATE EXCHANGE SYMBOL");
Console.WriteLine("FROMDATE = yyyymmdd");
Console.WriteLine("TODATE = yyyymmdd");
Environment.Exit(1);
}
try
{
// Load settings from command line
var startDate = DateTime.ParseExact(args[0], "yyyyMMdd", CultureInfo.InvariantCulture);
var endDate = DateTime.ParseExact(args[1], "yyyyMMdd", CultureInfo.InvariantCulture);
// Load settings from config.json
var dataDirectory = Config.Get("data-directory", "../../../Data");
var scaleFactor = Config.GetValue("bitfinex-scale-factor", 1m);
// Create an instance of the downloader
const string market = Market.Bitfinex;
var downloader = new CryptoiqDownloader(args[2], scaleFactor);
// Download the data
var symbolObject = Symbol.Create(args[3], SecurityType.Forex, market);
var data = downloader.Get(symbolObject, Resolution.Tick, startDate, endDate);
// Save the data
var writer = new LeanDataWriter(Resolution.Tick, symbolObject, dataDirectory, TickType.Quote);
writer.Write(data);
}
catch (Exception err)
{
Log.Error(err);
}
}
/// <summary>
/// Primary entry point to the program
/// </summary>
private static void Main(string[] args)
{
if (args.Length != 4)
{
Console.WriteLine("Usage: FxcmDownloader SYMBOLS RESOLUTION FROMDATE TODATE");
Console.WriteLine("SYMBOLS = eg EURUSD,USDJPY");
Console.WriteLine("RESOLUTION = Second/Minute/Hour/Daily/All");
Console.WriteLine("FROMDATE = yyyymmdd");
Console.WriteLine("TODATE = yyyymmdd");
Environment.Exit(1);
}
try
{
Logger.getRootLogger().setLevel(Level.ERROR);
BasicConfigurator.configure(new FileAppender(new SimpleLayout(), "FxcmDownloader.log", append: false));
// Load settings from command line
var tickers = args[0].Split(',');
var allResolutions = args[1].ToLower() == "all";
var resolution = allResolutions ? Resolution.Second : (Resolution)Enum.Parse(typeof(Resolution), args[1]);
var startDate = DateTime.ParseExact(args[2], "yyyyMMdd", CultureInfo.InvariantCulture);
var endDate = DateTime.ParseExact(args[3], "yyyyMMdd", CultureInfo.InvariantCulture);
// Load settings from config.json
var dataDirectory = Config.Get("data-directory", "../../../Data");
var server = Config.Get("fxcm-server", "http://www.fxcorporate.com/Hosts.jsp");
var terminal = Config.Get("fxcm-terminal", "Demo");
var userName = Config.Get("fxcm-user-name", "username");
var password = Config.Get("fxcm-password", "password");
// Download the data
const string market = "fxcm";
var downloader = new FxcmDataDownloader(server, terminal, userName, password);
foreach (var ticker in tickers)
{
var securityType = downloader.GetSecurityType(ticker);
var symbol = Symbol.Create(ticker, securityType, Market.FXCM);
if (!downloader.HasSymbol(ticker))
throw new ArgumentException("The symbol " + ticker + " is not available.");
var data = downloader.Get(symbol, securityType, resolution, startDate, endDate);
if (allResolutions)
{
var bars = data.Cast<TradeBar>().ToList();
// Save the data (second resolution)
var writer = new LeanDataWriter(securityType, resolution, symbol, dataDirectory, market);
writer.Write(bars);
// Save the data (other resolutions)
foreach (var res in new[] {Resolution.Minute, Resolution.Hour, Resolution.Daily})
{
var resData = FxcmDataDownloader.AggregateBars(symbol, bars, res.ToTimeSpan());
writer = new LeanDataWriter(securityType, res, symbol, dataDirectory, market);
writer.Write(resData);
}
}
else
{
// Save the data (single resolution)
var writer = new LeanDataWriter(securityType, resolution, symbol, dataDirectory, market);
writer.Write(data);
}
}
}
catch (Exception err)
{
Log.Error("FxcmDownloader(): Error: " + err.Message);
}
}
/// <summary>
/// Primary entry point to the program
/// </summary>
static void Main(string[] args)
{
if (args.Length != 4)
{
Console.WriteLine("Usage: OandaDownloader SYMBOL RESOLUTION FROMDATE TODATE");
Console.WriteLine("SYMBOLS = eg EURUSD,USDJPY");
Console.WriteLine("RESOLUTION = Second/Minute/Hour/Daily/All");
Console.WriteLine("FROMDATE = yyyymmdd");
Console.WriteLine("TODATE = yyyymmdd");
Environment.Exit(1);
}
try
{
// Load settings from command line
var symbols = args[0].Split(',');
var allResolutions = args[1].ToLower() == "all";
var resolution = allResolutions ? Resolution.Second : (Resolution)Enum.Parse(typeof(Resolution), args[1]);
var startDate = DateTime.ParseExact(args[2], "yyyyMMdd", CultureInfo.InvariantCulture);
var endDate = DateTime.ParseExact(args[3], "yyyyMMdd", CultureInfo.InvariantCulture);
// Load settings from config.json
var dataDirectory = Config.Get("data-directory", "../../../Data");
var accessToken = Config.Get("access-token", "73eba38ad5b44778f9a0c0fec1a66ed1-44f47f052c897b3e1e7f24196bbc071f");
var accountId = Convert.ToInt32(Config.Get("account-id", "621396"));
// Download the data
const string market = "oanda";
var downloader = new OandaDataDownloader(accessToken, accountId);
foreach (var symbol in symbols)
{
if (!downloader.HasSymbol(symbol))
throw new ArgumentException("The symbol " + symbol + " is not available.");
}
foreach (var symbol in symbols)
{
var securityType = downloader.GetSecurityType(symbol);
var data = downloader.Get(new Symbol(symbol), securityType, resolution, startDate, endDate);
if (allResolutions)
{
var bars = data.Cast<TradeBar>().ToList();
// Save the data (second resolution)
var writer = new LeanDataWriter(securityType, resolution, symbol, dataDirectory, market);
writer.Write(bars);
// Save the data (other resolutions)
foreach (var res in new[] { Resolution.Minute, Resolution.Hour, Resolution.Daily })
{
var resData = AggregateBars(new Symbol(symbol), bars, res.ToTimeSpan());
writer = new LeanDataWriter(securityType, res, symbol, dataDirectory, market);
writer.Write(resData);
}
}
else
{
// Save the data (single resolution)
var writer = new LeanDataWriter(securityType, resolution, symbol, dataDirectory, market);
writer.Write(data);
}
}
}
catch (Exception err)
{
Log.Error("OandaDownloader(): Error: " + err.Message);
}
}
/// <summary>
/// Primary entry point to the program
/// </summary>
static void Main(string[] args)
{
if (args.Length != 4)
{
Console.WriteLine("Usage: DukascopyDownloader SYMBOLS RESOLUTION FROMDATE TODATE");
Console.WriteLine("SYMBOLS = eg EURUSD,USDJPY");
Console.WriteLine("RESOLUTION = Tick/Second/Minute/Hour/Daily/All");
Console.WriteLine("FROMDATE = yyyymmdd");
Console.WriteLine("TODATE = yyyymmdd");
Environment.Exit(1);
}
try
{
// Load settings from command line
var symbols = args[0].Split(',');
var allResolutions = args[1].ToLower() == "all";
var resolution = allResolutions ? Resolution.Tick : (Resolution)Enum.Parse(typeof(Resolution), args[1]);
var startDate = DateTime.ParseExact(args[2], "yyyyMMdd", CultureInfo.InvariantCulture);
var endDate = DateTime.ParseExact(args[3], "yyyyMMdd", CultureInfo.InvariantCulture);
// Load settings from config.json
var dataDirectory = Config.Get("data-directory", "../../../Data");
// Download the data
const string market = Market.Dukascopy;
var downloader = new DukascopyDataDownloader();
foreach (var symbol in symbols)
{
if (!downloader.HasSymbol(symbol))
throw new ArgumentException("The symbol " + symbol + " is not available.");
}
foreach (var symbol in symbols)
{
var securityType = downloader.GetSecurityType(symbol);
var symbolObject = new Symbol(GetSid(symbol, securityType), symbol);
var data = downloader.Get(symbolObject, securityType, resolution, startDate, endDate);
if (allResolutions)
{
var ticks = data.Cast<Tick>().ToList();
// Save the data (tick resolution)
var writer = new LeanDataWriter(securityType, resolution, symbolObject, dataDirectory, market);
writer.Write(ticks);
// Save the data (other resolutions)
foreach (var res in new[] { Resolution.Second, Resolution.Minute, Resolution.Hour, Resolution.Daily })
{
var resData = DukascopyDataDownloader.AggregateTicks(symbolObject, ticks, res.ToTimeSpan());
writer = new LeanDataWriter(securityType, res, symbolObject, dataDirectory, market);
writer.Write(resData);
}
}
else
{
// Save the data (single resolution)
var writer = new LeanDataWriter(securityType, resolution, symbolObject, dataDirectory, market);
writer.Write(data);
}
}
}
catch (Exception err)
{
Log.Error(err);
}
}
