/// <summary>
/// Load data into memory.
/// </summary>
/// <param name="startTime">start of load range</param>
/// <param name="endTime">end of load range</param>
public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime)
{
List <Bar> data = new List <Bar>();
try
{
if (startTime < (DateTime)_firstTime)
{
startTime = (DateTime)_firstTime;
}
//if (endTime > (DateTime)LastTime)
// endTime = (DateTime)LastTime;
var cacheKey = new CacheId(null, "", 0,
Info[DataSourceParam.nickName].GetHashCode(),
startTime.GetHashCode(),
endTime.GetHashCode());
List <Bar> retrievalFunction()
{
DateTime t1 = DateTime.Now;
Output.Write(string.Format("DataSourceFred: loading data for {0}...", Info[DataSourceParam.nickName]));
List <Bar> rawBars = new List <Bar>();
JObject jsonData = getData(startTime, endTime);
var e = ((JArray)jsonData["observations"]).GetEnumerator();
while (e.MoveNext())
{
var bar = e.Current;
DateTime date = DateTime.Parse((string)bar["date"], CultureInfo.InvariantCulture).Date
+ DateTime.Parse(Info[DataSourceParam.time], CultureInfo.InvariantCulture).TimeOfDay;
string valueString = (string)bar["value"];
if (valueString == ".")
{
continue; // missing value, avoid throwing exception here
}
double value;
try
{
value = double.Parse(valueString, CultureInfo.InvariantCulture);
}
catch
{
// when we get here, this was probably a missing value,
// which FRED substitutes with "."
// we ignore and move on, AlignWithMarket will take
// care of the issue gracefully
continue;
}
rawBars.Add(Bar.NewOHLC(
Info[DataSourceParam.ticker],
date,
value, value, value, value,
0));
}
List <Bar> alignedBars = DataSourceHelper.AlignWithMarket(rawBars, startTime, endTime);
DateTime t2 = DateTime.Now;
Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds));
return(alignedBars);
};
data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true);
// FIXME: this is far from ideal. We want to make sure that retired
// series are not extended indefinitely
_lastTime = data.FindLast(b => true).Time;
}
catch (Exception e)
{
throw new Exception(
string.Format("DataSourceFred: failed to load quotes for {0}, {1}",
Info[DataSourceParam.nickName], e.Message));
}
if (data.Count == 0)
{
throw new Exception(string.Format("DataSourceFred: no data for {0}", Info[DataSourceParam.nickName]));
}
CachedData = data;
return(data);
}
/// <summary>
/// Load data into memory.
/// </summary>
/// <param name="startTime">start of load range</param>
/// <param name="endTime">end of load range</param>
public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime)
{
List <Bar> data = new List <Bar>();
try
{
//if (startTime < (DateTime)FirstTime)
// startTime = (DateTime)FirstTime;
//if (endTime > (DateTime)LastTime)
// endTime = (DateTime)LastTime;
var cacheKey = new CacheId().AddParameters(
Info[DataSourceParam.nickName].GetHashCode(),
startTime.GetHashCode(),
endTime.GetHashCode());
List <Bar> retrievalFunction()
{
DateTime t1 = DateTime.Now;
Output.Write(string.Format("DataSourceYahoo: loading data for {0}...", Info[DataSourceParam.nickName]));
JObject jsonData = getPrices(startTime, endTime);
/*
* Yahoo JSON format, as of 07/02/2019
*
* [JSON]
* chart
* result
* [0]
* meta
* currency
* symbol
* ...
* timestamp
* [0]: 511108200
* [1]: 511194600
* ...
* indicators
* quote
* [0]
* low
* [0]: 0.08854
* [1]: 0.09722
* ...
* close
* volume
* open
* high
* adjclose
* [0]
* adjclose
* [0]: 0.06999
* [1]: 0.07249
*/
List <Bar> bars = new List <Bar>();
var timestamps = (JArray)jsonData["chart"]["result"][0]["timestamp"];
var opens = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["open"];
var highs = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["high"];
var lows = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["low"];
var closes = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["close"];
var volumes = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["volume"];
var adjcloses = (JArray)jsonData["chart"]["result"][0]["indicators"]["adjclose"][0]["adjclose"];
var eT = timestamps.GetEnumerator();
var eO = opens.GetEnumerator();
var eH = highs.GetEnumerator();
var eL = lows.GetEnumerator();
var eC = closes.GetEnumerator();
var eV = volumes.GetEnumerator();
var eAC = adjcloses.GetEnumerator();
while (eT.MoveNext() && eO.MoveNext() && eH.MoveNext() &&
eL.MoveNext() && eC.MoveNext() && eV.MoveNext() && eAC.MoveNext())
{
DateTime t = fromUnixTime((long)eT.Current).Date
+ DateTime.Parse("16:00").TimeOfDay;
Bar bar = null;
try
{
// Yahoo taints the results by filling in null values
// we try to handle this gracefully in the catch block
double o = (double)eO.Current;
double h = (double)eH.Current;
double l = (double)eL.Current;
double c = (double)eC.Current;
long v = (long)eV.Current;
double ac = (double)eAC.Current;
// adjust prices according to the adjusted close.
// note the volume is adjusted the opposite way.
double ao = o * ac / c;
double ah = h * ac / c;
double al = l * ac / c;
long av = (long)(v * c / ac);
bar = Bar.NewOHLC(
Info[DataSourceParam.ticker],
t,
ao, ah, al, ac,
av);
}
catch
{
if (bars.Count < 1)
{
continue;
}
Bar prevBar = bars.Last();
bar = Bar.NewOHLC(
Info[DataSourceParam.ticker],
t,
prevBar.Open, prevBar.High, prevBar.Low, prevBar.Close,
prevBar.Volume);
}
if (t >= startTime && t <= endTime)
{
bars.Add(bar);
}
}
DateTime t2 = DateTime.Now;
Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds));
return(bars);
};
data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true);
}
catch (Exception e)
{
throw new Exception(
string.Format("DataSourceYahoo: failed to load quotes for {0}, {1}",
Info[DataSourceParam.nickName], e.Message));
}
if (data.Count == 0)
{
throw new Exception(string.Format("DataSourceYahoo: no data for {0}", Info[DataSourceParam.nickName]));
}
CachedData = data;
return(data);
}
