/// <summary>
/// Load data into memory.
/// </summary>
/// <param name="startTime">start of load range</param>
/// <param name="endTime">end of load range</param>
override public void LoadData(DateTime startTime, DateTime endTime)
{
var cacheKey = new CacheId(null, "", 0,
Info[DataSourceParam.nickName].GetHashCode(),
startTime.GetHashCode(),
endTime.GetHashCode());
List <Bar> retrievalFunction()
{
DateTime t1 = DateTime.Now;
Output.Write(string.Format("DataSourceNorgate: loading data for {0}...", Info[DataSourceParam.nickName]));
List <Bar> bars = new List <Bar>();
LoadData(bars, startTime, endTime);
DateTime t2 = DateTime.Now;
Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds));
return(bars);
};
List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction);
if (data.Count == 0)
{
throw new Exception(string.Format("DataSourceNorgate: no data for {0}", Info[DataSourceParam.nickName]));
}
Data = data;
}
/// <summary>
/// Load data into memory.
/// </summary>
/// <param name="startTime">start of load range</param>
/// <param name="endTime">end of load range</param>
public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime)
{
var cacheKey = new CacheId().AddParameters(
Info[DataSourceParam.nickName].GetHashCode(),
startTime.GetHashCode(),
endTime.GetHashCode());
List <Bar> retrievalFunction()
{
DateTime t1 = DateTime.Now;
Output.Write(string.Format("{0}: loading data for {1}...", GetType().Name, Info[DataSourceParam.nickName2]));
List <Bar> bars = new List <Bar>();
LoadData(bars, startTime, endTime);
DateTime t2 = DateTime.Now;
Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds));
return(bars);
};
List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true);
if (data.Count == 0 && !Info[DataSourceParam.dataFeed].ToLower().Contains("accept_no_data"))
{
throw new Exception(string.Format("{0}: no data for {1}", GetType().Name, Info[DataSourceParam.nickName2]));
}
CachedData = data;
return(data);
}
/// <summary>
/// Create new cache id.
/// </summary>
/// <param name="parentId">parent cache id, or null</param>
/// <param name="memberName">member function name, or ""</param>
/// <param name="lineNumber">line number or 0</param>
/// <param name="parameterIds">list of parameter ids</param>
public CacheId(CacheId parentId = null, [CallerMemberName] string memberName = "", [CallerLineNumber] int lineNumber = 0, params int[] parameterIds)
{
//--- call stack key
#if true
keyCallStack = parentId != null ? parentId.keyCallStack : SEED;
keyCallStack = CombineId(keyCallStack, memberName.GetHashCode());
keyCallStack = CombineId(keyCallStack, lineNumber);
#else
// this is safer, as we don't need to keep track of the stack trace
// however, this is also really slow, especially when running
// indicator-rich algorithms in the optimizer
keyCallStack = StackTraceId();
#endif
#if DEBUG_STACK
callStack = parentId?.callStack ?? "";
callStack += string.Format("/{0} ({1})", memberName, lineNumber);
#endif
//--- parameter key
keyParameters = parentId != null ? parentId.keyParameters : SEED;
foreach (var id in parameterIds)
{
keyParameters = CombineId(keyParameters, id);
}
}
/// <summary>
/// Create new cache id, adding a number of parameters to the previous
/// cache id.
/// </summary>
/// <param name="parameterIds"></param>
/// <returns>new cache id</returns>
public CacheId AddParameters(params int[] parameterIds)
{
var id = new CacheId(this);
id.ApplyParameters(id, parameterIds);
return(id);
}
/// <summary>
/// Load data into memory.
/// </summary>
/// <param name="startTime">start of load range</param>
/// <param name="endTime">end of load range</param>
public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime)
{
var cacheKey = new CacheId(null, "", 0,
Info[DataSourceParam.nickName].GetHashCode(),
startTime.GetHashCode(),
endTime.GetHashCode());
List <Bar> retrievalFunction()
{
DateTime t1 = DateTime.Now;
Output.WriteLine(string.Format("DataSourceFakeOptions: generating data for {0}...", Info[DataSourceParam.nickName]));
List <Bar> data = new List <Bar>();
LoadData(data, startTime, endTime);
DateTime t2 = DateTime.Now;
Output.WriteLine(string.Format("DataSourceFakeOptions: finished after {0:F1} seconds", (t2 - t1).TotalSeconds));
return(data);
}
var data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true);
CachedData = data;
return(data);
}
private void ApplyParameters(CacheId parentId, params int[] parameterIds)
{
keyParameters = parentId != null ? parentId.keyParameters : SEED;
foreach (var id in parameterIds)
{
keyParameters = CombineId(keyParameters, id);
}
}
private void ApplyCallStack(CacheId parentId, string memberName, int lineNumber)
{
keyCallStack = parentId != null ? parentId.keyCallStack : SEED;
keyCallStack = CombineId(keyCallStack, memberName.GetHashCode());
keyCallStack = CombineId(keyCallStack, lineNumber);
// this is safer, as we don't need to keep track of the stack trace
// however, this is also really slow, especially when running
// indicator-rich algorithms in the optimizer
// keyCallStack = StackTraceId();
}
/// <summary>
/// Load data into memory.
/// </summary>
/// <param name="startTime">start of load range</param>
/// <param name="endTime">end of load range</param>
public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime)
{
var cacheKey = new CacheId(null, "", 0,
Info[DataSourceParam.nickName].GetHashCode(),
startTime.GetHashCode(),
endTime.GetHashCode());
List <Bar> retrievalFunction()
{
DateTime t1 = DateTime.Now;
Output.Write(string.Format("DataSourceCsv: loading data for {0}...", Info[DataSourceParam.nickName]));
List <Bar> bars = new List <Bar>();
if (File.Exists(Info[DataSourceParam.dataPath]))
{
LoadFile(bars, Info[DataSourceParam.dataPath], startTime, endTime);
}
else if (Directory.Exists(Info[DataSourceParam.dataPath]))
{
LoadDir(bars, Info[DataSourceParam.dataPath], startTime, endTime);
}
// this should never happen - we create an empty directory in DataSource.New
else
{
throw new Exception("DataSourceCsv: data path not found");
}
DateTime t2 = DateTime.Now;
Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds));
if (_lastTime == null ||
_lastTime < endTime)
{
UpdateData(bars, startTime, endTime);
}
return(bars);
};
List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true);
if (data.Count == 0)
{
throw new Exception(string.Format("DataSourceCsv: no data for {0}", Info[DataSourceParam.nickName]));
}
CachedData = data;
return(data);
}
/// <summary>
/// Retrieve data from cache.
/// </summary>
/// <param name="id">unique ID of data</param>
/// <param name="initialRetrieval">lambda to retrieve data not found in cache</param>
/// <param name="global">set to true, to cache globally across all threads</param>
/// <returns>cached data</returns>
static public T GetData(CacheId id, Func <T> initialRetrieval, bool global = false)
{
#if DISABLE_CACHE
return(initialRetrieval());
#else
int key = id.Key;
#if THREAD_LOCAL
// separate thread-local and global caches
if (global)
{
lock (_globalCache)
{
if (!_globalCache.ContainsKey(key))
{
_globalCache[key] = initialRetrieval();
}
return(_globalCache[key]);
}
}
else
{
if (_threadCache == null)
{
_threadCache = new Dictionary <int, T>();
}
if (!_threadCache.ContainsKey(key))
{
_threadCache[key] = initialRetrieval();
}
return(_threadCache[key]);
}
#else
// unified cache across all threads
lock (_globalCache)
{
if (!_globalCache.ContainsKey(key))
{
_globalCache[key] = initialRetrieval();
}
return(_globalCache[key]);
}
#endif
#endif
}
/// <summary>
/// Load data into memory.
/// </summary>
/// <param name="startTime">start of load range</param>
/// <param name="endTime">end of load range</param>
override public void LoadData(DateTime startTime, DateTime endTime)
{
var cacheKey = new CacheId(null, "", 0,
_algoName.GetHashCode(),
startTime.GetHashCode(),
endTime.GetHashCode());
List <Bar> retrievalFunction()
{
try
{
DateTime t1 = DateTime.Now;
Output.WriteLine(string.Format("DataSourceAlgorithm: generating data for {0}...", Info[DataSourceParam.nickName]));
var algo = (SubclassableAlgorithm)AlgorithmLoader.InstantiateAlgorithm(_algoName);
algo.SubclassedStartTime = startTime;
algo.SubclassedEndTime = endTime;
algo.ParentDataSource = this;
algo.SubclassedData = new List <Bar>();;
algo.Run();
DateTime t2 = DateTime.Now;
Output.WriteLine(string.Format("DataSourceAlgorithm: finished after {0:F1} seconds", (t2 - t1).TotalSeconds));
return(algo.SubclassedData);
}
catch
{
throw new Exception("DataSourceAlgorithm: failed to run sub-classed algorithm " + _algoName);
}
}
List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction);
if (data.Count == 0)
{
throw new Exception(string.Format("DataSourceNorgate: no data for {0}", Info[DataSourceParam.nickName]));
}
Data = data;
}
/// <summary>
/// Retrieve data from cache.
/// </summary>
/// <param name="id">unique ID of data</param>
/// <param name="initialRetrieval">lambda to retrieve data not found in cache</param>
/// <returns>cached data</returns>
static public T GetData(CacheId id, Func <T> initialRetrieval)
{
#if DISABLE_CACHE
return(initialRetrieval());
#else
int key = id.Key;
//lock (_lockCache)
lock (_cache)
{
if (!_cache.ContainsKey(key))
{
_cache[key] = initialRetrieval();
}
return(_cache[key]);
}
#endif
}
/// <summary>
/// Clone parent cache id. This constructor is only used for internal purposes.
/// </summary>
/// <param name="parentId">parent to clone</param>
private CacheId(CacheId parentId)
{
keyCallStack = parentId.keyCallStack;
keyParameters = parentId.keyParameters;
}
/// <summary>
/// Load data into memory.
/// </summary>
/// <param name="startTime">start of load range</param>
/// <param name="endTime">end of load range</param>
override public void LoadData(DateTime startTime, DateTime endTime)
{
try
{
if (startTime < (DateTime)FirstTime)
{
startTime = (DateTime)FirstTime;
}
//if (endTime > (DateTime)LastTime)
// endTime = (DateTime)LastTime;
var cacheKey = new CacheId(null, "", 0,
Info[DataSourceParam.nickName].GetHashCode(),
startTime.GetHashCode(),
endTime.GetHashCode());
List <Bar> retrievalFunction()
{
DateTime t1 = DateTime.Now;
Output.Write(string.Format("DataSourceTiingo: loading data for {0}...", Info[DataSourceParam.nickName]));
List <Bar> bars = new List <Bar>();
JArray jsonData = getPrices(startTime, endTime);
var e = jsonData.GetEnumerator();
while (e.MoveNext())
{
var bar = e.Current;
DateTime date = DateTime.Parse((string)bar["date"], CultureInfo.InvariantCulture).Date
+ DateTime.Parse(Info[DataSourceParam.time]).TimeOfDay;
double open = (double)bar["adjOpen"];
double high = (double)bar["adjHigh"];
double low = (double)bar["adjLow"];
double close = (double)bar["adjClose"];
long volume = (long)bar["adjVolume"];
if (date >= startTime && date <= endTime)
{
bars.Add(Bar.NewOHLC(
Info[DataSourceParam.ticker],
date,
open, high, low, close,
volume));
}
}
DateTime t2 = DateTime.Now;
Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds));
return(bars);
};
Data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction);;
}
catch (Exception e)
{
throw new Exception(
string.Format("DataSourceTiingo: failed to load quotes for {0}, {1}",
Info[DataSourceParam.nickName], e.Message));
}
if ((Data as List <Bar>).Count == 0)
{
throw new Exception(string.Format("DataSourceTiingo: no data for {0}", Info[DataSourceParam.nickName]));
}
}
/// <summary>
/// Load data into memory.
/// </summary>
/// <param name="startTime">start of load range</param>
/// <param name="endTime">end of load range</param>
public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime)
{
var cacheKey = new CacheId(null, "", 0,
Info[DataSourceParam.nickName].GetHashCode(),
startTime.GetHashCode(),
endTime.GetHashCode());
List <Bar> retrievalFunction()
{
DateTime t1 = DateTime.Now;
//Output.Write(string.Format("{0}: loading data for {1}...", GetType().Name, Info[DataSourceParam.nickName]));
// load data from specified data sources
// and save as list of bars, in reverse order
Dictionary <string, List <Bar> > dsBars = new Dictionary <string, List <Bar> >();
foreach (var nick in _symbols)
{
var d = DataSource.New(nick);
try
{
var data = d.LoadData(startTime, endTime);
dsBars[nick] = data.Reverse().ToList();
}
catch (Exception /*e*/)
{
Output.WriteLine("{0}: {1} failed to load {2}", this.GetType().Name, Info[DataSourceParam.nickName], nick);
// add an empty list, if need be
// this will be ignored further down during splicing
if (!dsBars.ContainsKey(nick))
{
dsBars[nick] = new List <Bar>();
}
}
//Output.WriteLine("{0}: {1} data range {2:MM/dd/yyyy}, {3:MM/dd/yyyy}", GetType().Name, nick, d.FirstTime, d.LastTime);
}
// create enumerators for all data sources
Dictionary <string, IEnumerator <Bar> > dsEnums = new Dictionary <string, IEnumerator <Bar> >();
Dictionary <string, bool> dsHasData = new Dictionary <string, bool>();
foreach (var nick in _symbols)
{
dsEnums[nick] = dsBars[nick].GetEnumerator();
dsHasData[nick] = dsEnums[nick].MoveNext();
}
// skip bars from all proxy datasources, so that no proxy
// has bars after the primary datasource
// example: extending GLD w/ XAUUSD. because XAUUSD has
// a different trading calendar, XAUUSD might have bars
// after GLD, leading to faulty results
var lastPrimary = dsEnums[_symbols.First()].Current.Time;
foreach (var nick in _symbols)
{
while (dsHasData[nick] && dsEnums[nick].Current.Time > lastPrimary)
{
dsHasData[nick] = dsEnums[nick].MoveNext();
}
}
// collect bars
List <Bar> bars = new List <Bar>();
Dictionary <string, double?> dsScale = new Dictionary <string, double?>();
_symbols.ForEach(n => dsScale[n] = null);
dsScale[_symbols.First()] = 1.0;
while (dsHasData.Values.Aggregate((a, b) => a || b))
{
// find most-recent timestamp
DateTime ts = _symbols
.Where(n => dsHasData[n])
.Select(n => dsEnums[n].Current.Time)
.Max(t => t);
Bar bar = null;
foreach (var nick in _symbols)
{
// no data: continue
if (!dsHasData[nick])
{
continue;
}
// older bar: continue
if (dsEnums[nick].Current.Time < ts)
{
continue;
}
if (bar == null)
{
// highest priority bar
Bar rawBar = dsEnums[nick].Current;
// we might get here, with dsScale not set yet.
// this is the best we can do to fix things
if (dsScale[nick] == null)
{
dsScale[nick] = bars.Last().Open / rawBar.Close;
}
double open = rawBar.Open * (double)dsScale[nick];
double high = rawBar.High * (double)dsScale[nick];
double low = rawBar.Low * (double)dsScale[nick];
double close = rawBar.Close * (double)dsScale[nick];
long volume = 0;
bar = Bar.NewOHLC(Info[DataSourceParam.ticker], ts, open, high, low, close, volume);
bars.Add(bar);
}
else
{
// lower priority bars
Bar rawBar = dsEnums[nick].Current;
List <double> scales = new List <double>
{
bar.Open / rawBar.Open,
bar.High / rawBar.High,
bar.Low / rawBar.Low,
bar.Close / rawBar.Close,
};
dsScale[nick] = scales.Average();
}
dsHasData[nick] = dsEnums[nick].MoveNext();
}
}
// reverse order of bars
bars.Reverse();
DateTime t2 = DateTime.Now;
//Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds));
return(bars);
};
List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true);
if (data.Count == 0)
{
throw new Exception(string.Format("{0}: no data for {1}", GetType().Name, Info[DataSourceParam.nickName]));
}
CachedData = data;
return(data);
}
/// <summary>
/// Load data into memory.
/// </summary>
/// <param name="startTime">start of load range</param>
/// <param name="endTime">end of load range</param>
override public void LoadData(DateTime startTime, DateTime endTime)
{
try
{
if (startTime < (DateTime)FirstTime)
{
startTime = (DateTime)FirstTime;
}
//if (endTime > (DateTime)LastTime)
// endTime = (DateTime)LastTime;
var cacheKey = new CacheId(null, "", 0,
Info[DataSourceParam.nickName].GetHashCode(),
startTime.GetHashCode(),
endTime.GetHashCode());
List <Bar> retrievalFunction()
{
DateTime t1 = DateTime.Now;
Output.Write(string.Format("DataSourceFred: loading data for {0}...", Info[DataSourceParam.nickName]));
List <Bar> rawBars = new List <Bar>();
JObject jsonData = getData(startTime, endTime);
var e = ((JArray)jsonData["observations"]).GetEnumerator();
while (e.MoveNext())
{
var bar = e.Current;
DateTime date = DateTime.Parse((string)bar["date"]).Date
+ DateTime.Parse(Info[DataSourceParam.time]).TimeOfDay;
string valueString = (string)bar["value"];
double value;
try
{
value = double.Parse(valueString);
}
catch
{
// when we get here, this was probably a missing value,
// which FRED substitutes with "."
// we ignore and move on, AlignWithMarket will take
// care of the issue gracefully
continue;
}
rawBars.Add(Bar.NewOHLC(
Info[DataSourceParam.ticker],
date,
value, value, value, value,
0));
}
List <Bar> alignedBars = DataSourceHelper.AlignWithMarket(rawBars, startTime, endTime);
DateTime t2 = DateTime.Now;
Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds));
return(alignedBars);
};
Data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction);
// FIXME: this is far from ideal. We want to make sure that retired
// series are not extended indefinitely
LastTime = (Data as List <Bar>).FindLast(b => true).Time;
}
catch (Exception e)
{
throw new Exception(
string.Format("DataSourceFred: failed to load quotes for {0}, {1}",
Info[DataSourceParam.nickName], e.Message));
}
if ((Data as List <Bar>).Count == 0)
{
throw new Exception(string.Format("DataSourceFred: no data for {0}", Info[DataSourceParam.nickName]));
}
}
/// <summary>
/// Load data into memory.
/// </summary>
/// <param name="startTime">start of load range</param>
/// <param name="endTime">end of load range</param>
override public void LoadData(DateTime startTime, DateTime endTime)
{
var cacheKey = new CacheId(null, "", 0,
Info[DataSourceParam.nickName].GetHashCode(),
startTime.GetHashCode(),
endTime.GetHashCode());
List <Bar> retrievalFunction()
{
DateTime t1 = DateTime.Now;
//Output.Write(string.Format("{0}: loading data for {1}...", GetType().Name, Info[DataSourceParam.nickName]));
// load data from specified data sources
// and save as list of bars, in reverse order
Dictionary <string, List <Bar> > dsBars = new Dictionary <string, List <Bar> >();
foreach (var nick in _symbols)
{
var d = DataSource.New(nick);
try
{
d.LoadData(startTime, endTime);
dsBars[nick] = d.Data.Reverse().ToList();
}
catch (Exception e)
{
Output.WriteLine("{0}: {1} failed to load {2}", this.GetType().Name, Info[DataSourceParam.nickName], nick);
// add an empty list, if need be
// this will be ignored further down during splicing
if (!dsBars.ContainsKey(nick))
{
dsBars[nick] = new List <Bar>();
}
}
//Output.WriteLine("{0}: {1} data range {2:MM/dd/yyyy}, {3:MM/dd/yyyy}", GetType().Name, nick, d.FirstTime, d.LastTime);
}
// create enumerators for all data sources
Dictionary <string, IEnumerator <Bar> > dsEnums = new Dictionary <string, IEnumerator <Bar> >();
Dictionary <string, bool> dsHasData = new Dictionary <string, bool>();
foreach (var nick in _symbols)
{
dsEnums[nick] = dsBars[nick].GetEnumerator();
dsHasData[nick] = dsEnums[nick].MoveNext();
}
// collect bars
List <Bar> bars = new List <Bar>();
Dictionary <string, double> dsScale = new Dictionary <string, double>();
_symbols.ForEach(n => dsScale[n] = 1.0);
while (dsHasData.Values.Aggregate((a, b) => a || b))
{
// find most-recent timestamp
DateTime ts = _symbols
.Where(n => dsHasData[n])
.Select(n => dsEnums[n].Current.Time)
.Max(t => t);
Bar bar = null;
foreach (var nick in _symbols)
{
// no data: continue
if (!dsHasData[nick])
{
continue;
}
// older bar: continue
if (dsEnums[nick].Current.Time < ts)
{
continue;
}
if (bar == null)
{
// highest priority bar
Bar rawBar = dsEnums[nick].Current;
double open = rawBar.Open * dsScale[nick];
double high = rawBar.High * dsScale[nick];
double low = rawBar.Low * dsScale[nick];
double close = rawBar.Close * dsScale[nick];
long volume = 0;
bar = Bar.NewOHLC(Info[DataSourceParam.ticker], ts, open, high, low, close, volume);
bars.Add(bar);
}
else
{
// lower priority bars
Bar rawBar = dsEnums[nick].Current;
List <double> scales = new List <double>
{
bar.Open / rawBar.Open,
bar.High / rawBar.High,
bar.Low / rawBar.Low,
bar.Close / rawBar.Close,
};
dsScale[nick] = scales.Average();
}
dsHasData[nick] = dsEnums[nick].MoveNext();
}
}
// reverse order of bars
bars.Reverse();
DateTime t2 = DateTime.Now;
//Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds));
return(bars);
};
List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction);
if (data.Count == 0)
{
throw new Exception(string.Format("{0}: no data for {1}", GetType().Name, Info[DataSourceParam.nickName]));
}
Data = data;
}
/// <summary>
/// Run sub-classed algorithm and return bars as enumerable.
/// </summary>
/// <param name="startTime">start of load range</param>
/// <param name="endTime">end of load range</param>
/// <returns>bars created from sub-classed algo</returns>
public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime)
{
#if true
_algo.IsDataSource = true;
if (_algo.CanRunAsChild && Info.ContainsKey(DataSourceParam.allowSync))
{
// for child algorithms, we bypass the cache and run the
// child bar-for-bar and in sync with its parent
// we don't want to rely on the algorithm to provide a unique name.
// therefore, we replace the bar symbol with the algorithm's hash
var algoHashHex = string.Format("{0:X}", _algo.Name, _algo.GetHashCode());
foreach (var bar in _algo.Run(startTime, endTime))
{
var bar2 = new Bar(
algoHashHex,
bar.Time,
bar.Open, bar.High, bar.Low, bar.Close, bar.Volume, bar.HasOHLC,
bar.Bid, bar.Ask, bar.BidVolume, bar.AskVolume, bar.HasBidAsk,
bar.OptionExpiry, bar.OptionStrike, bar.OptionIsPut);
yield return(bar2);
if (!_algo.IsLastBar)
{
// the simulator core needs to know the next bar's
// timestamp. at the same time, we want to avoid
// child algorithms from running one bar ahead of
// the main algo. we fix this issue by returning
// a dummy bar, announcing the next timestamp.
var dummy = Bar.NewValue(
null, _algo.NextSimTime, 0.0);
yield return(dummy);
}
}
yield break;
}
else
{
// for all other algorithms, we run the algo
// all at once and save the result in the cache
var algoNick = Info[DataSourceParam.nickName];
var cacheKey = new CacheId().AddParameters(
algoNick.GetHashCode(), // _algoName.GetHashCode(),
startTime.GetHashCode(),
endTime.GetHashCode());
List <Bar> retrievalFunction()
{
try
{
DateTime t1 = DateTime.Now;
Output.WriteLine(string.Format("DataSourceAlgorithm: generating data for {0}...", Info[DataSourceParam.nickName]));
var bars = _algo.Run(startTime, endTime)
.ToList();
DateTime t2 = DateTime.Now;
Output.WriteLine(string.Format("DataSourceAlgorithm: finished after {0:F1} seconds", (t2 - t1).TotalSeconds));
return(bars);
}
catch
{
throw new Exception("DataSourceAlgorithm: failed to run sub-classed algorithm " + algoNick);
}
}
List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true);
if (data.Count == 0)
{
throw new Exception(string.Format("DataSourceAlgorithm: no data for {0}", Info[DataSourceParam.nickName]));
}
CachedData = data;
foreach (var bar in data)
{
yield return(bar);
}
}
#else
var algoNick = Info[DataSourceParam.nickName];
var cacheKey = new CacheId(null, "", 0,
algoNick.GetHashCode(), // _algoName.GetHashCode(),
startTime.GetHashCode(),
endTime.GetHashCode());
List <Bar> retrievalFunction()
{
try
{
DateTime t1 = DateTime.Now;
Output.WriteLine(string.Format("DataSourceAlgorithm: generating data for {0}...", Info[DataSourceParam.nickName]));
_algo.StartTime = startTime;
_algo.EndTime = endTime;
_algo.ParentDataSource = this;
_algo.SubclassedData = new List <Bar>();;
_algo.Run();
DateTime t2 = DateTime.Now;
Output.WriteLine(string.Format("DataSourceAlgorithm: finished after {0:F1} seconds", (t2 - t1).TotalSeconds));
return(_algo.SubclassedData);
}
catch
{
throw new Exception("DataSourceAlgorithm: failed to run sub-classed algorithm " + algoNick);
}
}
List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true);
if (data.Count == 0)
{
throw new Exception(string.Format("DataSourceAlgorithm: no data for {0}", Info[DataSourceParam.nickName]));
}
Data = data;
#endif
}
/// <summary>
/// Load data into memory.
/// </summary>
/// <param name="startTime">start of load range</param>
/// <param name="endTime">end of load range</param>
public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime)
{
List <Bar> data = new List <Bar>();
try
{
if (startTime < (DateTime)_firstTime)
{
startTime = (DateTime)_firstTime;
}
var cacheKey = new CacheId().AddParameters(
Info[DataSourceParam.nickName].GetHashCode(),
startTime.GetHashCode(),
endTime.GetHashCode());
List <Bar> retrievalFunction()
{
DateTime t1 = DateTime.Now;
Output.Write(string.Format("DataSourceStooq: loading data for {0}...", Info[DataSourceParam.nickName]));
List <Bar> bars = new List <Bar>();
List <StooqDataRow> stooqData = getPrices(startTime, endTime);
var e = stooqData.GetEnumerator();
while (e.MoveNext())
{
var bar = e.Current;
DateTime date = bar.Date;
if (date >= startTime && date <= endTime)
{
bars.Add(Bar.NewOHLC(
Info[DataSourceParam.ticker],
date,
bar.Open, bar.High, bar.Low, bar.Close,
bar.Volume));
}
}
DateTime t2 = DateTime.Now;
Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds));
return(bars);
};
data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true);;
}
catch (Exception e)
{
throw new Exception(
string.Format("DataSourceStooq: failed to load quotes for {0}, {1}",
Info[DataSourceParam.nickName], e.Message));
}
if (data.Count == 0)
{
throw new Exception(string.Format("DataSourceStooq: no data for {0}", Info[DataSourceParam.nickName]));
}
CachedData = data;
return(data);
}
/// <summary>
/// Load data into memory.
/// </summary>
/// <param name="startTime">start of load range</param>
/// <param name="endTime">end of load range</param>
public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime)
{
List <Bar> data = new List <Bar>();
try
{
//if (startTime < (DateTime)FirstTime)
// startTime = (DateTime)FirstTime;
//if (endTime > (DateTime)LastTime)
// endTime = (DateTime)LastTime;
var cacheKey = new CacheId(null, "", 0,
Info[DataSourceParam.nickName].GetHashCode(),
startTime.GetHashCode(),
endTime.GetHashCode());
List <Bar> retrievalFunction()
{
DateTime t1 = DateTime.Now;
Output.Write(string.Format("DataSourceYahoo: loading data for {0}...", Info[DataSourceParam.nickName]));
JObject jsonData = getPrices(startTime, endTime);
/*
* Yahoo JSON format, as of 07/02/2019
*
* [JSON]
* chart
* result
* [0]
* meta
* currency
* symbol
* ...
* timestamp
* [0]: 511108200
* [1]: 511194600
* ...
* indicators
* quote
* [0]
* low
* [0]: 0.08854
* [1]: 0.09722
* ...
* close
* volume
* open
* high
* adjclose
* [0]
* adjclose
* [0]: 0.06999
* [1]: 0.07249
*/
List <Bar> bars = new List <Bar>();
var timestamps = (JArray)jsonData["chart"]["result"][0]["timestamp"];
var opens = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["open"];
var highs = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["high"];
var lows = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["low"];
var closes = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["close"];
var volumes = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["volume"];
var adjcloses = (JArray)jsonData["chart"]["result"][0]["indicators"]["adjclose"][0]["adjclose"];
var eT = timestamps.GetEnumerator();
var eO = opens.GetEnumerator();
var eH = highs.GetEnumerator();
var eL = lows.GetEnumerator();
var eC = closes.GetEnumerator();
var eV = volumes.GetEnumerator();
var eAC = adjcloses.GetEnumerator();
while (eT.MoveNext() && eO.MoveNext() && eH.MoveNext() &&
eL.MoveNext() && eC.MoveNext() && eV.MoveNext() && eAC.MoveNext())
{
DateTime t = fromUnixTime((long)eT.Current).Date
+ DateTime.Parse("16:00").TimeOfDay;
Bar bar = null;
try
{
// Yahoo taints the results by filling in null values
// we try to handle this gracefully in the catch block
double o = (double)eO.Current;
double h = (double)eH.Current;
double l = (double)eL.Current;
double c = (double)eC.Current;
long v = (long)eV.Current;
double ac = (double)eAC.Current;
// adjust prices according to the adjusted close.
// note the volume is adjusted the opposite way.
double ao = o * ac / c;
double ah = h * ac / c;
double al = l * ac / c;
long av = (long)(v * c / ac);
bar = Bar.NewOHLC(
Info[DataSourceParam.ticker],
t,
ao, ah, al, ac,
av);
}
catch
{
if (bars.Count < 1)
{
continue;
}
Bar prevBar = bars.Last();
bar = Bar.NewOHLC(
Info[DataSourceParam.ticker],
t,
prevBar.Open, prevBar.High, prevBar.Low, prevBar.Close,
prevBar.Volume);
}
if (t >= startTime && t <= endTime)
{
bars.Add(bar);
}
}
DateTime t2 = DateTime.Now;
Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds));
return(bars);
};
data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true);
}
catch (Exception e)
{
throw new Exception(
string.Format("DataSourceYahoo: failed to load quotes for {0}, {1}",
Info[DataSourceParam.nickName], e.Message));
}
if (data.Count == 0)
{
throw new Exception(string.Format("DataSourceYahoo: no data for {0}", Info[DataSourceParam.nickName]));
}
CachedData = data;
return(data);
}
/// <summary>
/// Create new cache id. Typically, this constructor is used without parameters,
/// resulting in a new top-level cache id, specific to the current member function
/// and line number.
/// </summary>
/// <param name="parentId">parent cache id. Typically not required. Pass in value when nesting indicators.</param>
/// <param name="memberName">member function name. Typically filled in by compiler services. Pass in value when creating indicators.</param>
/// <param name="lineNumber">line number. Typically filled in by compiler services. Pass in value when creating indicators.</param>
public CacheId(CacheId parentId = null, [CallerMemberName] string memberName = "", [CallerLineNumber] int lineNumber = 0)
{
ApplyCallStack(parentId, memberName, lineNumber);
ApplyParameters(parentId);
}
/// <summary>
/// Run sub-classed algorithm and return bars as enumerable.
/// </summary>
/// <param name="startTime">start of load range</param>
/// <param name="endTime">end of load range</param>
/// <returns>bars created from sub-classed algo</returns>
public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime)
{
#if true
_algo.IsDataSource = true;
if (_algo.IsChildAlgorithm)
{
// for child algorithms, we bypass the cache and run the
// child bar-for-bar and in sync with its parent
foreach (var bar in _algo.Run(startTime, endTime))
{
yield return(bar);
}
yield break;
}
else
{
// for all other algorithms, we run the algo
// all at once and save the result in the cache
var algoNick = Info[DataSourceParam.nickName];
var cacheKey = new CacheId(null, "", 0,
algoNick.GetHashCode(), // _algoName.GetHashCode(),
startTime.GetHashCode(),
endTime.GetHashCode());
List <Bar> retrievalFunction()
{
try
{
DateTime t1 = DateTime.Now;
Output.WriteLine(string.Format("DataSourceAlgorithm: generating data for {0}...", Info[DataSourceParam.nickName]));
var bars = _algo.Run(startTime, endTime)
.ToList();
DateTime t2 = DateTime.Now;
Output.WriteLine(string.Format("DataSourceAlgorithm: finished after {0:F1} seconds", (t2 - t1).TotalSeconds));
return(bars);
}
catch
{
throw new Exception("DataSourceAlgorithm: failed to run sub-classed algorithm " + algoNick);
}
}
List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true);
if (data.Count == 0)
{
throw new Exception(string.Format("DataSourceAlgorithm: no data for {0}", Info[DataSourceParam.nickName]));
}
CachedData = data;
foreach (var bar in data)
{
yield return(bar);
}
}
#else
var algoNick = Info[DataSourceParam.nickName];
var cacheKey = new CacheId(null, "", 0,
algoNick.GetHashCode(), // _algoName.GetHashCode(),
startTime.GetHashCode(),
endTime.GetHashCode());
List <Bar> retrievalFunction()
{
try
{
DateTime t1 = DateTime.Now;
Output.WriteLine(string.Format("DataSourceAlgorithm: generating data for {0}...", Info[DataSourceParam.nickName]));
_algo.StartTime = startTime;
_algo.EndTime = endTime;
_algo.ParentDataSource = this;
_algo.SubclassedData = new List <Bar>();;
_algo.Run();
DateTime t2 = DateTime.Now;
Output.WriteLine(string.Format("DataSourceAlgorithm: finished after {0:F1} seconds", (t2 - t1).TotalSeconds));
return(_algo.SubclassedData);
}
catch
{
throw new Exception("DataSourceAlgorithm: failed to run sub-classed algorithm " + algoNick);
}
}
List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true);
if (data.Count == 0)
{
throw new Exception(string.Format("DataSourceAlgorithm: no data for {0}", Info[DataSourceParam.nickName]));
}
Data = data;
#endif
}
/// <summary>
/// Create new cache id. This constructor is specifically aimed at building indicators. Therefore,
/// all parameters are mandatory. The cache id created by this constructor is idential to one
/// created using new CacheId(...).AddParameters(...)
/// </summary>
/// <param name="parentId">parent cache id</param>
/// <param name="memberName">member function name, or ""</param>
/// <param name="lineNumber">line number or 0</param>
/// <param name="parameterIds">list of parameter ids</param>
public CacheId(CacheId parentId, string memberName, int lineNumber, params int[] parameterIds)
{
ApplyCallStack(parentId, memberName, lineNumber);
ApplyParameters(parentId, parameterIds);
}
