/// <summary>
/// Primary entry point to the program
/// </summary>
public static void OandaDownloader(IList <string> tickers, string resolution, DateTime startDate, DateTime endDate)
{
if (resolution.IsNullOrEmpty() || tickers.IsNullOrEmpty())
{
Console.WriteLine("OandaDownloader ERROR: '--tickers=' or '--resolution=' parameter is missing");
Console.WriteLine("--tickers=eg EURUSD,USDJPY");
Console.WriteLine("--resolution=Second/Minute/Hour/Daily/All");
Environment.Exit(1);
}
try
{
// Load settings from command line
var allResolutions = resolution.ToLowerInvariant() == "all";
var castResolution = allResolutions ? Resolution.Second : (Resolution)Enum.Parse(typeof(Resolution), resolution);
// Load settings from config.json
var dataDirectory = Config.Get("data-directory", "../../../Data");
var accessToken = Config.Get("oanda-access-token", "73eba38ad5b44778f9a0c0fec1a66ed1-44f47f052c897b3e1e7f24196bbc071f");
var accountId = Config.Get("oanda-account-id", "621396");
// Create an instance of the downloader
const string market = Market.Oanda;
var downloader = new OandaDataDownloader(accessToken, accountId);
foreach (var ticker in tickers)
{
if (!downloader.HasSymbol(ticker))
{
throw new ArgumentException("The ticker " + ticker + " is not available.");
}
}
foreach (var ticker in tickers)
{
// Download the data
var securityType = downloader.GetSecurityType(ticker);
var symbol = Symbol.Create(ticker, securityType, market);
var data = downloader.Get(symbol, castResolution, startDate, endDate);
if (allResolutions)
{
var bars = data.Cast <QuoteBar>().ToList();
// Save the data (second resolution)
var writer = new LeanDataWriter(castResolution, symbol, dataDirectory);
writer.Write(bars);
// Save the data (other resolutions)
foreach (var res in new[] { Resolution.Minute, Resolution.Hour, Resolution.Daily })
{
var resData = LeanData.AggregateQuoteBars(bars, symbol, res.ToTimeSpan());
writer = new LeanDataWriter(res, symbol, dataDirectory);
writer.Write(resData);
}
}
else
{
// Save the data (single resolution)
var writer = new LeanDataWriter(castResolution, symbol, dataDirectory);
writer.Write(data);
}
}
}
catch (Exception err)
{
Log.Error(err);
}
}
/// <summary>
/// Primary entry point to the program. This program only supports FOREX for now.
/// </summary>
public static void IBDownloader(IList <string> tickers, string resolution, DateTime fromDate, DateTime toDate)
{
if (resolution.IsNullOrEmpty() || tickers.IsNullOrEmpty())
{
Console.WriteLine("IBDownloader ERROR: '--tickers=' or '--resolution=' parameter is missing");
Console.WriteLine("--tickers=eg EURUSD,USDJPY");
Console.WriteLine("--resolution=Second/Minute/Hour/Daily/All");
Environment.Exit(1);
}
try
{
var allResolutions = resolution.ToLowerInvariant() == "all";
var castResolution = allResolutions ? Resolution.Second : (Resolution)Enum.Parse(typeof(Resolution), resolution);
var startDate = fromDate.ConvertToUtc(TimeZones.NewYork);
var endDate = toDate.ConvertToUtc(TimeZones.NewYork);
// fix end date
endDate = new DateTime(Math.Min(endDate.Ticks, DateTime.Now.AddDays(-1).Ticks));
// Max number of histoy days
int maxDays = 1;
if (!allResolutions)
{
switch (castResolution)
{
case Resolution.Daily:
maxDays = 365;
break;
case Resolution.Hour:
maxDays = 30;
break;
case Resolution.Minute:
maxDays = 10;
break;
}
}
// Load settings from config.json
var dataDirectory = Config.Get("data-folder", "../../../Data");
// Only FOREX for now
SecurityType securityType = SecurityType.Forex;
string market = Market.FXCM;
using (var downloader = new IBDataDownloader())
{
foreach (var ticker in tickers)
{
// Download the data
var symbol = Symbol.Create(ticker, securityType, market);
var auxEndDate = startDate.AddDays(maxDays);
auxEndDate = new DateTime(Math.Min(auxEndDate.Ticks, endDate.Ticks));
while (startDate < auxEndDate)
{
var data = downloader.Get(new DataDownloaderGetParameters(symbol, castResolution, startDate, auxEndDate, TickType.Quote));
var bars = data.Cast <QuoteBar>().ToList();
if (allResolutions)
{
// Save the data (second resolution)
var writer = new LeanDataWriter(castResolution, symbol, dataDirectory);
writer.Write(bars);
// Save the data (other resolutions)
foreach (var res in new[] { Resolution.Minute, Resolution.Hour, Resolution.Daily })
{
var resData = LeanData.AggregateQuoteBars(bars, symbol, res.ToTimeSpan());
writer = new LeanDataWriter(res, symbol, dataDirectory);
writer.Write(resData);
}
}
else
{
// Save the data (single resolution)
var writer = new LeanDataWriter(castResolution, symbol, dataDirectory);
writer.Write(data);
}
startDate = auxEndDate;
auxEndDate = auxEndDate.AddDays(maxDays);
auxEndDate = new DateTime(Math.Min(auxEndDate.Ticks, endDate.Ticks));
}
}
}
}
catch (Exception err)
{
Log.Error(err);
}
}
/// <summary>
/// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC).
/// </summary>
/// <param name="symbol">Symbol for the data we're looking for.</param>
/// <param name="resolution">Resolution of the data request</param>
/// <param name="startUtc">Start time of the data in UTC</param>
/// <param name="endUtc">End time of the data in UTC</param>
/// <returns>Enumerable of base data for this symbol</returns>
public IEnumerable <BaseData> Get(Symbol symbol, Resolution resolution, DateTime startUtc, DateTime endUtc)
{
if (!_symbolMapper.IsKnownLeanSymbol(symbol))
{
throw new ArgumentException("Invalid symbol requested: " + symbol.Value);
}
if (resolution == Resolution.Tick)
{
throw new NotSupportedException("Resolution not available: " + resolution);
}
if (symbol.ID.SecurityType != SecurityType.Forex && symbol.ID.SecurityType != SecurityType.Cfd)
{
throw new NotSupportedException("SecurityType not available: " + symbol.ID.SecurityType);
}
if (endUtc < startUtc)
{
throw new ArgumentException("The end date must be greater or equal than the start date.");
}
var barsTotalInPeriod = new List <QuoteBar>();
var barsToSave = new List <QuoteBar>();
// set the starting date/time
var date = startUtc;
var startDateTime = date;
// loop until last date
while (startDateTime <= endUtc.AddDays(1))
{
// request blocks of 5-second bars with a starting date/time
var bars = _brokerage.DownloadQuoteBars(symbol, startDateTime, endUtc.AddDays(1), Resolution.Second, DateTimeZone.Utc).ToList();
if (bars.Count == 0)
{
break;
}
var groupedBars = GroupBarsByDate(bars);
if (groupedBars.Count > 1)
{
// we received more than one day, so we save the completed days and continue
while (groupedBars.Count > 1)
{
var currentDate = groupedBars.Keys.First();
if (currentDate > endUtc)
{
break;
}
barsToSave.AddRange(groupedBars[currentDate]);
barsTotalInPeriod.AddRange(barsToSave);
barsToSave.Clear();
// remove the completed date
groupedBars.Remove(currentDate);
}
// update the current date
date = groupedBars.Keys.First();
if (date <= endUtc)
{
barsToSave.AddRange(groupedBars[date]);
}
}
else
{
var currentDate = groupedBars.Keys.First();
if (currentDate > endUtc)
{
break;
}
// update the current date
date = currentDate;
barsToSave.AddRange(groupedBars[date]);
}
// calculate the next request datetime (next 5-sec bar time)
startDateTime = bars[bars.Count - 1].Time.AddSeconds(5);
}
if (barsToSave.Count > 0)
{
barsTotalInPeriod.AddRange(barsToSave);
}
switch (resolution)
{
case Resolution.Second:
case Resolution.Minute:
case Resolution.Hour:
case Resolution.Daily:
foreach (var bar in LeanData.AggregateQuoteBars(barsTotalInPeriod, symbol, resolution.ToTimeSpan()))
{
yield return(bar);
}
break;
}
}