/// <summary>
/// Load data into memory.
/// </summary>
/// <param name="startTime">start of load range</param>
/// <param name="endTime">end of load range</param>
override public void LoadData(DateTime startTime, DateTime endTime)
{
try
{
//if (startTime < (DateTime)FirstTime)
// startTime = (DateTime)FirstTime;
//if (endTime > (DateTime)LastTime)
// endTime = (DateTime)LastTime;
var cacheKey = new CacheId(null, "", 0,
Info[DataSourceParam.nickName].GetHashCode(),
startTime.GetHashCode(),
endTime.GetHashCode());
List <Bar> retrievalFunction()
{
DateTime t1 = DateTime.Now;
Output.Write(string.Format("DataSourceYahoo: loading data for {0}...", Info[DataSourceParam.nickName]));
JObject jsonData = getPrices(startTime, endTime);
/*
* Yahoo JSON format, as of 07/02/2019
*
* [JSON]
* chart
* result
* [0]
* meta
* currency
* symbol
* ...
* timestamp
* [0]: 511108200
* [1]: 511194600
* ...
* indicators
* quote
* [0]
* low
* [0]: 0.08854
* [1]: 0.09722
* ...
* close
* volume
* open
* high
* adjclose
* [0]
* adjclose
* [0]: 0.06999
* [1]: 0.07249
*/
List <Bar> bars = new List <Bar>();
var timestamps = (JArray)jsonData["chart"]["result"][0]["timestamp"];
var opens = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["open"];
var highs = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["high"];
var lows = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["low"];
var closes = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["close"];
var volumes = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["volume"];
var adjcloses = (JArray)jsonData["chart"]["result"][0]["indicators"]["adjclose"][0]["adjclose"];
var eT = timestamps.GetEnumerator();
var eO = opens.GetEnumerator();
var eH = highs.GetEnumerator();
var eL = lows.GetEnumerator();
var eC = closes.GetEnumerator();
var eV = volumes.GetEnumerator();
var eAC = adjcloses.GetEnumerator();
while (eT.MoveNext() && eO.MoveNext() && eH.MoveNext() &&
eL.MoveNext() && eC.MoveNext() && eV.MoveNext() && eAC.MoveNext())
{
DateTime t = fromUnixTime((long)eT.Current).Date
+ DateTime.Parse("16:00").TimeOfDay;
Bar bar = null;
try
{
// Yahoo taints the results by filling in null values
// we try to handle this gracefully in the catch block
double o = (double)eO.Current;
double h = (double)eH.Current;
double l = (double)eL.Current;
double c = (double)eC.Current;
long v = (long)eV.Current;
double ac = (double)eAC.Current;
// adjust prices according to the adjusted close.
// note the volume is adjusted the opposite way.
double ao = o * ac / c;
double ah = h * ac / c;
double al = l * ac / c;
long av = (long)(v * c / ac);
bar = Bar.NewOHLC(
Info[DataSourceParam.ticker],
t,
ao, ah, al, ac,
av);
}
catch
{
if (bars.Count < 1)
{
continue;
}
Bar prevBar = bars.Last();
bar = Bar.NewOHLC(
Info[DataSourceParam.ticker],
t,
prevBar.Open, prevBar.High, prevBar.Low, prevBar.Close,
prevBar.Volume);
}
if (t >= startTime && t <= endTime)
{
bars.Add(bar);
}
}
DateTime t2 = DateTime.Now;
Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds));
return(bars);
};
Data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction);
}
catch (Exception e)
{
throw new Exception(
string.Format("DataSourceYahoo: failed to load quotes for {0}, {1}",
Info[DataSourceParam.nickName], e.Message));
}
if ((Data as List <Bar>).Count == 0)
{
throw new Exception(string.Format("DataSourceYahoo: no data for {0}", Info[DataSourceParam.nickName]));
}
}
/// <summary>
/// Load data into memory.
/// </summary>
/// <param name="startTime">start of load range</param>
/// <param name="endTime">end of load range</param>
public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime)
{
List <Bar> data = new List <Bar>();
try
{
if (startTime < (DateTime)_firstTime)
{
startTime = (DateTime)_firstTime;
}
//if (endTime > (DateTime)LastTime)
// endTime = (DateTime)LastTime;
var cacheKey = new CacheId().AddParameters(
Info[DataSourceParam.nickName].GetHashCode(),
startTime.GetHashCode(),
endTime.GetHashCode());
List <Bar> retrievalFunction()
{
DateTime t1 = DateTime.Now;
Output.Write(string.Format("DataSourceTiingo: loading data for {0}...", Info[DataSourceParam.nickName]));
List <Bar> bars = new List <Bar>();
JArray jsonData = getPrices(startTime, endTime);
var e = jsonData.GetEnumerator();
while (e.MoveNext())
{
var bar = e.Current;
DateTime date = DateTime.Parse((string)bar["date"], CultureInfo.InvariantCulture).Date
+ DateTime.Parse(Info[DataSourceParam.time]).TimeOfDay;
double open = (double)bar["adjOpen"];
double high = (double)bar["adjHigh"];
double low = (double)bar["adjLow"];
double close = (double)bar["adjClose"];
long volume = (long)bar["adjVolume"];
if (date >= startTime && date <= endTime)
{
bars.Add(Bar.NewOHLC(
Info[DataSourceParam.ticker],
date,
open, high, low, close,
volume));
}
}
DateTime t2 = DateTime.Now;
Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds));
return(bars);
};
data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true);;
}
catch (Exception e)
{
throw new Exception(
string.Format("DataSourceTiingo: failed to load quotes for {0}, {1}",
Info[DataSourceParam.nickName], e.Message));
}
if (data.Count == 0)
{
throw new Exception(string.Format("DataSourceTiingo: no data for {0}", Info[DataSourceParam.nickName]));
}
CachedData = data;
return(data);
}
private string getMeta()
{
string cachePath = Path.Combine(GlobalSettings.HomePath, "Cache", Info[DataSourceParam.nickName2]);
string metaCache = Path.Combine(cachePath, "yahoo_meta");
bool writeToDisk = false;
string rawMeta = null;
bool validMeta()
{
if (rawMeta == null)
{
return(false);
}
if (rawMeta.Length < 10)
{
return(false);
}
return(true);
}
//--- 1) try to read meta from disk
if (File.Exists(metaCache))
{
using (StreamReader mc = new StreamReader(File.Open(metaCache, FileMode.Open)))
rawMeta = mc.ReadToEnd();
}
//--- 2) if failed, try to retrieve from web
if (!validMeta())
{
Output.WriteLine("DataSourceYahoo: retrieving meta for {0}", Info[DataSourceParam.nickName]);
string url = string.Format(
@"http://finance.yahoo.com/quote/"
+ "{0}",
convertSymbol(Info[DataSourceParam.symbolYahoo]));
using (var client = new WebClient())
rawMeta = client.DownloadString(url);
writeToDisk = true;
}
//--- 3) if failed, return
if (!validMeta())
{
return(null);
}
//--- 4) write to disk
if (writeToDisk)
{
Directory.CreateDirectory(cachePath);
using (StreamWriter mc = new StreamWriter(File.Open(metaCache, FileMode.Create)))
mc.Write(rawMeta);
}
return(rawMeta);
}