/// <summary> /// Load data into memory. /// </summary> /// <param name="startTime">start of load range</param> /// <param name="endTime">end of load range</param> override public void LoadData(DateTime startTime, DateTime endTime) { var cacheKey = new CacheId(null, "", 0, Info[DataSourceParam.nickName].GetHashCode(), startTime.GetHashCode(), endTime.GetHashCode()); List <Bar> retrievalFunction() { DateTime t1 = DateTime.Now; Output.Write(string.Format("DataSourceNorgate: loading data for {0}...", Info[DataSourceParam.nickName])); List <Bar> bars = new List <Bar>(); LoadData(bars, startTime, endTime); DateTime t2 = DateTime.Now; Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds)); return(bars); }; List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction); if (data.Count == 0) { throw new Exception(string.Format("DataSourceNorgate: no data for {0}", Info[DataSourceParam.nickName])); } Data = data; }
/// <summary> /// Load data into memory. /// </summary> /// <param name="startTime">start of load range</param> /// <param name="endTime">end of load range</param> public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime) { var cacheKey = new CacheId().AddParameters( Info[DataSourceParam.nickName].GetHashCode(), startTime.GetHashCode(), endTime.GetHashCode()); List <Bar> retrievalFunction() { DateTime t1 = DateTime.Now; Output.Write(string.Format("{0}: loading data for {1}...", GetType().Name, Info[DataSourceParam.nickName2])); List <Bar> bars = new List <Bar>(); LoadData(bars, startTime, endTime); DateTime t2 = DateTime.Now; Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds)); return(bars); }; List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true); if (data.Count == 0 && !Info[DataSourceParam.dataFeed].ToLower().Contains("accept_no_data")) { throw new Exception(string.Format("{0}: no data for {1}", GetType().Name, Info[DataSourceParam.nickName2])); } CachedData = data; return(data); }
/// <summary> /// Create new cache id. /// </summary> /// <param name="parentId">parent cache id, or null</param> /// <param name="memberName">member function name, or ""</param> /// <param name="lineNumber">line number or 0</param> /// <param name="parameterIds">list of parameter ids</param> public CacheId(CacheId parentId = null, [CallerMemberName] string memberName = "", [CallerLineNumber] int lineNumber = 0, params int[] parameterIds) { //--- call stack key #if true keyCallStack = parentId != null ? parentId.keyCallStack : SEED; keyCallStack = CombineId(keyCallStack, memberName.GetHashCode()); keyCallStack = CombineId(keyCallStack, lineNumber); #else // this is safer, as we don't need to keep track of the stack trace // however, this is also really slow, especially when running // indicator-rich algorithms in the optimizer keyCallStack = StackTraceId(); #endif #if DEBUG_STACK callStack = parentId?.callStack ?? ""; callStack += string.Format("/{0} ({1})", memberName, lineNumber); #endif //--- parameter key keyParameters = parentId != null ? parentId.keyParameters : SEED; foreach (var id in parameterIds) { keyParameters = CombineId(keyParameters, id); } }
/// <summary> /// Create new cache id, adding a number of parameters to the previous /// cache id. /// </summary> /// <param name="parameterIds"></param> /// <returns>new cache id</returns> public CacheId AddParameters(params int[] parameterIds) { var id = new CacheId(this); id.ApplyParameters(id, parameterIds); return(id); }
/// <summary> /// Load data into memory. /// </summary> /// <param name="startTime">start of load range</param> /// <param name="endTime">end of load range</param> public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime) { var cacheKey = new CacheId(null, "", 0, Info[DataSourceParam.nickName].GetHashCode(), startTime.GetHashCode(), endTime.GetHashCode()); List <Bar> retrievalFunction() { DateTime t1 = DateTime.Now; Output.WriteLine(string.Format("DataSourceFakeOptions: generating data for {0}...", Info[DataSourceParam.nickName])); List <Bar> data = new List <Bar>(); LoadData(data, startTime, endTime); DateTime t2 = DateTime.Now; Output.WriteLine(string.Format("DataSourceFakeOptions: finished after {0:F1} seconds", (t2 - t1).TotalSeconds)); return(data); } var data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true); CachedData = data; return(data); }
private void ApplyParameters(CacheId parentId, params int[] parameterIds) { keyParameters = parentId != null ? parentId.keyParameters : SEED; foreach (var id in parameterIds) { keyParameters = CombineId(keyParameters, id); } }
private void ApplyCallStack(CacheId parentId, string memberName, int lineNumber) { keyCallStack = parentId != null ? parentId.keyCallStack : SEED; keyCallStack = CombineId(keyCallStack, memberName.GetHashCode()); keyCallStack = CombineId(keyCallStack, lineNumber); // this is safer, as we don't need to keep track of the stack trace // however, this is also really slow, especially when running // indicator-rich algorithms in the optimizer // keyCallStack = StackTraceId(); }
/// <summary> /// Load data into memory. /// </summary> /// <param name="startTime">start of load range</param> /// <param name="endTime">end of load range</param> public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime) { var cacheKey = new CacheId(null, "", 0, Info[DataSourceParam.nickName].GetHashCode(), startTime.GetHashCode(), endTime.GetHashCode()); List <Bar> retrievalFunction() { DateTime t1 = DateTime.Now; Output.Write(string.Format("DataSourceCsv: loading data for {0}...", Info[DataSourceParam.nickName])); List <Bar> bars = new List <Bar>(); if (File.Exists(Info[DataSourceParam.dataPath])) { LoadFile(bars, Info[DataSourceParam.dataPath], startTime, endTime); } else if (Directory.Exists(Info[DataSourceParam.dataPath])) { LoadDir(bars, Info[DataSourceParam.dataPath], startTime, endTime); } // this should never happen - we create an empty directory in DataSource.New else { throw new Exception("DataSourceCsv: data path not found"); } DateTime t2 = DateTime.Now; Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds)); if (_lastTime == null || _lastTime < endTime) { UpdateData(bars, startTime, endTime); } return(bars); }; List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true); if (data.Count == 0) { throw new Exception(string.Format("DataSourceCsv: no data for {0}", Info[DataSourceParam.nickName])); } CachedData = data; return(data); }
/// <summary> /// Retrieve data from cache. /// </summary> /// <param name="id">unique ID of data</param> /// <param name="initialRetrieval">lambda to retrieve data not found in cache</param> /// <param name="global">set to true, to cache globally across all threads</param> /// <returns>cached data</returns> static public T GetData(CacheId id, Func <T> initialRetrieval, bool global = false) { #if DISABLE_CACHE return(initialRetrieval()); #else int key = id.Key; #if THREAD_LOCAL // separate thread-local and global caches if (global) { lock (_globalCache) { if (!_globalCache.ContainsKey(key)) { _globalCache[key] = initialRetrieval(); } return(_globalCache[key]); } } else { if (_threadCache == null) { _threadCache = new Dictionary <int, T>(); } if (!_threadCache.ContainsKey(key)) { _threadCache[key] = initialRetrieval(); } return(_threadCache[key]); } #else // unified cache across all threads lock (_globalCache) { if (!_globalCache.ContainsKey(key)) { _globalCache[key] = initialRetrieval(); } return(_globalCache[key]); } #endif #endif }
/// <summary> /// Load data into memory. /// </summary> /// <param name="startTime">start of load range</param> /// <param name="endTime">end of load range</param> override public void LoadData(DateTime startTime, DateTime endTime) { var cacheKey = new CacheId(null, "", 0, _algoName.GetHashCode(), startTime.GetHashCode(), endTime.GetHashCode()); List <Bar> retrievalFunction() { try { DateTime t1 = DateTime.Now; Output.WriteLine(string.Format("DataSourceAlgorithm: generating data for {0}...", Info[DataSourceParam.nickName])); var algo = (SubclassableAlgorithm)AlgorithmLoader.InstantiateAlgorithm(_algoName); algo.SubclassedStartTime = startTime; algo.SubclassedEndTime = endTime; algo.ParentDataSource = this; algo.SubclassedData = new List <Bar>();; algo.Run(); DateTime t2 = DateTime.Now; Output.WriteLine(string.Format("DataSourceAlgorithm: finished after {0:F1} seconds", (t2 - t1).TotalSeconds)); return(algo.SubclassedData); } catch { throw new Exception("DataSourceAlgorithm: failed to run sub-classed algorithm " + _algoName); } } List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction); if (data.Count == 0) { throw new Exception(string.Format("DataSourceNorgate: no data for {0}", Info[DataSourceParam.nickName])); } Data = data; }
/// <summary> /// Retrieve data from cache. /// </summary> /// <param name="id">unique ID of data</param> /// <param name="initialRetrieval">lambda to retrieve data not found in cache</param> /// <returns>cached data</returns> static public T GetData(CacheId id, Func <T> initialRetrieval) { #if DISABLE_CACHE return(initialRetrieval()); #else int key = id.Key; //lock (_lockCache) lock (_cache) { if (!_cache.ContainsKey(key)) { _cache[key] = initialRetrieval(); } return(_cache[key]); } #endif }
/// <summary> /// Clone parent cache id. This constructor is only used for internal purposes. /// </summary> /// <param name="parentId">parent to clone</param> private CacheId(CacheId parentId) { keyCallStack = parentId.keyCallStack; keyParameters = parentId.keyParameters; }
/// <summary> /// Load data into memory. /// </summary> /// <param name="startTime">start of load range</param> /// <param name="endTime">end of load range</param> override public void LoadData(DateTime startTime, DateTime endTime) { try { if (startTime < (DateTime)FirstTime) { startTime = (DateTime)FirstTime; } //if (endTime > (DateTime)LastTime) // endTime = (DateTime)LastTime; var cacheKey = new CacheId(null, "", 0, Info[DataSourceParam.nickName].GetHashCode(), startTime.GetHashCode(), endTime.GetHashCode()); List <Bar> retrievalFunction() { DateTime t1 = DateTime.Now; Output.Write(string.Format("DataSourceTiingo: loading data for {0}...", Info[DataSourceParam.nickName])); List <Bar> bars = new List <Bar>(); JArray jsonData = getPrices(startTime, endTime); var e = jsonData.GetEnumerator(); while (e.MoveNext()) { var bar = e.Current; DateTime date = DateTime.Parse((string)bar["date"], CultureInfo.InvariantCulture).Date + DateTime.Parse(Info[DataSourceParam.time]).TimeOfDay; double open = (double)bar["adjOpen"]; double high = (double)bar["adjHigh"]; double low = (double)bar["adjLow"]; double close = (double)bar["adjClose"]; long volume = (long)bar["adjVolume"]; if (date >= startTime && date <= endTime) { bars.Add(Bar.NewOHLC( Info[DataSourceParam.ticker], date, open, high, low, close, volume)); } } DateTime t2 = DateTime.Now; Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds)); return(bars); }; Data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction);; } catch (Exception e) { throw new Exception( string.Format("DataSourceTiingo: failed to load quotes for {0}, {1}", Info[DataSourceParam.nickName], e.Message)); } if ((Data as List <Bar>).Count == 0) { throw new Exception(string.Format("DataSourceTiingo: no data for {0}", Info[DataSourceParam.nickName])); } }
/// <summary> /// Load data into memory. /// </summary> /// <param name="startTime">start of load range</param> /// <param name="endTime">end of load range</param> public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime) { var cacheKey = new CacheId(null, "", 0, Info[DataSourceParam.nickName].GetHashCode(), startTime.GetHashCode(), endTime.GetHashCode()); List <Bar> retrievalFunction() { DateTime t1 = DateTime.Now; //Output.Write(string.Format("{0}: loading data for {1}...", GetType().Name, Info[DataSourceParam.nickName])); // load data from specified data sources // and save as list of bars, in reverse order Dictionary <string, List <Bar> > dsBars = new Dictionary <string, List <Bar> >(); foreach (var nick in _symbols) { var d = DataSource.New(nick); try { var data = d.LoadData(startTime, endTime); dsBars[nick] = data.Reverse().ToList(); } catch (Exception /*e*/) { Output.WriteLine("{0}: {1} failed to load {2}", this.GetType().Name, Info[DataSourceParam.nickName], nick); // add an empty list, if need be // this will be ignored further down during splicing if (!dsBars.ContainsKey(nick)) { dsBars[nick] = new List <Bar>(); } } //Output.WriteLine("{0}: {1} data range {2:MM/dd/yyyy}, {3:MM/dd/yyyy}", GetType().Name, nick, d.FirstTime, d.LastTime); } // create enumerators for all data sources Dictionary <string, IEnumerator <Bar> > dsEnums = new Dictionary <string, IEnumerator <Bar> >(); Dictionary <string, bool> dsHasData = new Dictionary <string, bool>(); foreach (var nick in _symbols) { dsEnums[nick] = dsBars[nick].GetEnumerator(); dsHasData[nick] = dsEnums[nick].MoveNext(); } // skip bars from all proxy datasources, so that no proxy // has bars after the primary datasource // example: extending GLD w/ XAUUSD. because XAUUSD has // a different trading calendar, XAUUSD might have bars // after GLD, leading to faulty results var lastPrimary = dsEnums[_symbols.First()].Current.Time; foreach (var nick in _symbols) { while (dsHasData[nick] && dsEnums[nick].Current.Time > lastPrimary) { dsHasData[nick] = dsEnums[nick].MoveNext(); } } // collect bars List <Bar> bars = new List <Bar>(); Dictionary <string, double?> dsScale = new Dictionary <string, double?>(); _symbols.ForEach(n => dsScale[n] = null); dsScale[_symbols.First()] = 1.0; while (dsHasData.Values.Aggregate((a, b) => a || b)) { // find most-recent timestamp DateTime ts = _symbols .Where(n => dsHasData[n]) .Select(n => dsEnums[n].Current.Time) .Max(t => t); Bar bar = null; foreach (var nick in _symbols) { // no data: continue if (!dsHasData[nick]) { continue; } // older bar: continue if (dsEnums[nick].Current.Time < ts) { continue; } if (bar == null) { // highest priority bar Bar rawBar = dsEnums[nick].Current; // we might get here, with dsScale not set yet. // this is the best we can do to fix things if (dsScale[nick] == null) { dsScale[nick] = bars.Last().Open / rawBar.Close; } double open = rawBar.Open * (double)dsScale[nick]; double high = rawBar.High * (double)dsScale[nick]; double low = rawBar.Low * (double)dsScale[nick]; double close = rawBar.Close * (double)dsScale[nick]; long volume = 0; bar = Bar.NewOHLC(Info[DataSourceParam.ticker], ts, open, high, low, close, volume); bars.Add(bar); } else { // lower priority bars Bar rawBar = dsEnums[nick].Current; List <double> scales = new List <double> { bar.Open / rawBar.Open, bar.High / rawBar.High, bar.Low / rawBar.Low, bar.Close / rawBar.Close, }; dsScale[nick] = scales.Average(); } dsHasData[nick] = dsEnums[nick].MoveNext(); } } // reverse order of bars bars.Reverse(); DateTime t2 = DateTime.Now; //Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds)); return(bars); }; List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true); if (data.Count == 0) { throw new Exception(string.Format("{0}: no data for {1}", GetType().Name, Info[DataSourceParam.nickName])); } CachedData = data; return(data); }
/// <summary> /// Load data into memory. /// </summary> /// <param name="startTime">start of load range</param> /// <param name="endTime">end of load range</param> override public void LoadData(DateTime startTime, DateTime endTime) { try { if (startTime < (DateTime)FirstTime) { startTime = (DateTime)FirstTime; } //if (endTime > (DateTime)LastTime) // endTime = (DateTime)LastTime; var cacheKey = new CacheId(null, "", 0, Info[DataSourceParam.nickName].GetHashCode(), startTime.GetHashCode(), endTime.GetHashCode()); List <Bar> retrievalFunction() { DateTime t1 = DateTime.Now; Output.Write(string.Format("DataSourceFred: loading data for {0}...", Info[DataSourceParam.nickName])); List <Bar> rawBars = new List <Bar>(); JObject jsonData = getData(startTime, endTime); var e = ((JArray)jsonData["observations"]).GetEnumerator(); while (e.MoveNext()) { var bar = e.Current; DateTime date = DateTime.Parse((string)bar["date"]).Date + DateTime.Parse(Info[DataSourceParam.time]).TimeOfDay; string valueString = (string)bar["value"]; double value; try { value = double.Parse(valueString); } catch { // when we get here, this was probably a missing value, // which FRED substitutes with "." // we ignore and move on, AlignWithMarket will take // care of the issue gracefully continue; } rawBars.Add(Bar.NewOHLC( Info[DataSourceParam.ticker], date, value, value, value, value, 0)); } List <Bar> alignedBars = DataSourceHelper.AlignWithMarket(rawBars, startTime, endTime); DateTime t2 = DateTime.Now; Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds)); return(alignedBars); }; Data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction); // FIXME: this is far from ideal. We want to make sure that retired // series are not extended indefinitely LastTime = (Data as List <Bar>).FindLast(b => true).Time; } catch (Exception e) { throw new Exception( string.Format("DataSourceFred: failed to load quotes for {0}, {1}", Info[DataSourceParam.nickName], e.Message)); } if ((Data as List <Bar>).Count == 0) { throw new Exception(string.Format("DataSourceFred: no data for {0}", Info[DataSourceParam.nickName])); } }
/// <summary> /// Load data into memory. /// </summary> /// <param name="startTime">start of load range</param> /// <param name="endTime">end of load range</param> override public void LoadData(DateTime startTime, DateTime endTime) { var cacheKey = new CacheId(null, "", 0, Info[DataSourceParam.nickName].GetHashCode(), startTime.GetHashCode(), endTime.GetHashCode()); List <Bar> retrievalFunction() { DateTime t1 = DateTime.Now; //Output.Write(string.Format("{0}: loading data for {1}...", GetType().Name, Info[DataSourceParam.nickName])); // load data from specified data sources // and save as list of bars, in reverse order Dictionary <string, List <Bar> > dsBars = new Dictionary <string, List <Bar> >(); foreach (var nick in _symbols) { var d = DataSource.New(nick); try { d.LoadData(startTime, endTime); dsBars[nick] = d.Data.Reverse().ToList(); } catch (Exception e) { Output.WriteLine("{0}: {1} failed to load {2}", this.GetType().Name, Info[DataSourceParam.nickName], nick); // add an empty list, if need be // this will be ignored further down during splicing if (!dsBars.ContainsKey(nick)) { dsBars[nick] = new List <Bar>(); } } //Output.WriteLine("{0}: {1} data range {2:MM/dd/yyyy}, {3:MM/dd/yyyy}", GetType().Name, nick, d.FirstTime, d.LastTime); } // create enumerators for all data sources Dictionary <string, IEnumerator <Bar> > dsEnums = new Dictionary <string, IEnumerator <Bar> >(); Dictionary <string, bool> dsHasData = new Dictionary <string, bool>(); foreach (var nick in _symbols) { dsEnums[nick] = dsBars[nick].GetEnumerator(); dsHasData[nick] = dsEnums[nick].MoveNext(); } // collect bars List <Bar> bars = new List <Bar>(); Dictionary <string, double> dsScale = new Dictionary <string, double>(); _symbols.ForEach(n => dsScale[n] = 1.0); while (dsHasData.Values.Aggregate((a, b) => a || b)) { // find most-recent timestamp DateTime ts = _symbols .Where(n => dsHasData[n]) .Select(n => dsEnums[n].Current.Time) .Max(t => t); Bar bar = null; foreach (var nick in _symbols) { // no data: continue if (!dsHasData[nick]) { continue; } // older bar: continue if (dsEnums[nick].Current.Time < ts) { continue; } if (bar == null) { // highest priority bar Bar rawBar = dsEnums[nick].Current; double open = rawBar.Open * dsScale[nick]; double high = rawBar.High * dsScale[nick]; double low = rawBar.Low * dsScale[nick]; double close = rawBar.Close * dsScale[nick]; long volume = 0; bar = Bar.NewOHLC(Info[DataSourceParam.ticker], ts, open, high, low, close, volume); bars.Add(bar); } else { // lower priority bars Bar rawBar = dsEnums[nick].Current; List <double> scales = new List <double> { bar.Open / rawBar.Open, bar.High / rawBar.High, bar.Low / rawBar.Low, bar.Close / rawBar.Close, }; dsScale[nick] = scales.Average(); } dsHasData[nick] = dsEnums[nick].MoveNext(); } } // reverse order of bars bars.Reverse(); DateTime t2 = DateTime.Now; //Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds)); return(bars); }; List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction); if (data.Count == 0) { throw new Exception(string.Format("{0}: no data for {1}", GetType().Name, Info[DataSourceParam.nickName])); } Data = data; }
/// <summary> /// Run sub-classed algorithm and return bars as enumerable. /// </summary> /// <param name="startTime">start of load range</param> /// <param name="endTime">end of load range</param> /// <returns>bars created from sub-classed algo</returns> public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime) { #if true _algo.IsDataSource = true; if (_algo.CanRunAsChild && Info.ContainsKey(DataSourceParam.allowSync)) { // for child algorithms, we bypass the cache and run the // child bar-for-bar and in sync with its parent // we don't want to rely on the algorithm to provide a unique name. // therefore, we replace the bar symbol with the algorithm's hash var algoHashHex = string.Format("{0:X}", _algo.Name, _algo.GetHashCode()); foreach (var bar in _algo.Run(startTime, endTime)) { var bar2 = new Bar( algoHashHex, bar.Time, bar.Open, bar.High, bar.Low, bar.Close, bar.Volume, bar.HasOHLC, bar.Bid, bar.Ask, bar.BidVolume, bar.AskVolume, bar.HasBidAsk, bar.OptionExpiry, bar.OptionStrike, bar.OptionIsPut); yield return(bar2); if (!_algo.IsLastBar) { // the simulator core needs to know the next bar's // timestamp. at the same time, we want to avoid // child algorithms from running one bar ahead of // the main algo. we fix this issue by returning // a dummy bar, announcing the next timestamp. var dummy = Bar.NewValue( null, _algo.NextSimTime, 0.0); yield return(dummy); } } yield break; } else { // for all other algorithms, we run the algo // all at once and save the result in the cache var algoNick = Info[DataSourceParam.nickName]; var cacheKey = new CacheId().AddParameters( algoNick.GetHashCode(), // _algoName.GetHashCode(), startTime.GetHashCode(), endTime.GetHashCode()); List <Bar> retrievalFunction() { try { DateTime t1 = DateTime.Now; Output.WriteLine(string.Format("DataSourceAlgorithm: generating data for {0}...", Info[DataSourceParam.nickName])); var bars = _algo.Run(startTime, endTime) .ToList(); DateTime t2 = DateTime.Now; Output.WriteLine(string.Format("DataSourceAlgorithm: finished after {0:F1} seconds", (t2 - t1).TotalSeconds)); return(bars); } catch { throw new Exception("DataSourceAlgorithm: failed to run sub-classed algorithm " + algoNick); } } List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true); if (data.Count == 0) { throw new Exception(string.Format("DataSourceAlgorithm: no data for {0}", Info[DataSourceParam.nickName])); } CachedData = data; foreach (var bar in data) { yield return(bar); } } #else var algoNick = Info[DataSourceParam.nickName]; var cacheKey = new CacheId(null, "", 0, algoNick.GetHashCode(), // _algoName.GetHashCode(), startTime.GetHashCode(), endTime.GetHashCode()); List <Bar> retrievalFunction() { try { DateTime t1 = DateTime.Now; Output.WriteLine(string.Format("DataSourceAlgorithm: generating data for {0}...", Info[DataSourceParam.nickName])); _algo.StartTime = startTime; _algo.EndTime = endTime; _algo.ParentDataSource = this; _algo.SubclassedData = new List <Bar>();; _algo.Run(); DateTime t2 = DateTime.Now; Output.WriteLine(string.Format("DataSourceAlgorithm: finished after {0:F1} seconds", (t2 - t1).TotalSeconds)); return(_algo.SubclassedData); } catch { throw new Exception("DataSourceAlgorithm: failed to run sub-classed algorithm " + algoNick); } } List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true); if (data.Count == 0) { throw new Exception(string.Format("DataSourceAlgorithm: no data for {0}", Info[DataSourceParam.nickName])); } Data = data; #endif }
/// <summary> /// Load data into memory. /// </summary> /// <param name="startTime">start of load range</param> /// <param name="endTime">end of load range</param> public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime) { List <Bar> data = new List <Bar>(); try { if (startTime < (DateTime)_firstTime) { startTime = (DateTime)_firstTime; } var cacheKey = new CacheId().AddParameters( Info[DataSourceParam.nickName].GetHashCode(), startTime.GetHashCode(), endTime.GetHashCode()); List <Bar> retrievalFunction() { DateTime t1 = DateTime.Now; Output.Write(string.Format("DataSourceStooq: loading data for {0}...", Info[DataSourceParam.nickName])); List <Bar> bars = new List <Bar>(); List <StooqDataRow> stooqData = getPrices(startTime, endTime); var e = stooqData.GetEnumerator(); while (e.MoveNext()) { var bar = e.Current; DateTime date = bar.Date; if (date >= startTime && date <= endTime) { bars.Add(Bar.NewOHLC( Info[DataSourceParam.ticker], date, bar.Open, bar.High, bar.Low, bar.Close, bar.Volume)); } } DateTime t2 = DateTime.Now; Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds)); return(bars); }; data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true);; } catch (Exception e) { throw new Exception( string.Format("DataSourceStooq: failed to load quotes for {0}, {1}", Info[DataSourceParam.nickName], e.Message)); } if (data.Count == 0) { throw new Exception(string.Format("DataSourceStooq: no data for {0}", Info[DataSourceParam.nickName])); } CachedData = data; return(data); }
/// <summary> /// Load data into memory. /// </summary> /// <param name="startTime">start of load range</param> /// <param name="endTime">end of load range</param> public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime) { List <Bar> data = new List <Bar>(); try { //if (startTime < (DateTime)FirstTime) // startTime = (DateTime)FirstTime; //if (endTime > (DateTime)LastTime) // endTime = (DateTime)LastTime; var cacheKey = new CacheId(null, "", 0, Info[DataSourceParam.nickName].GetHashCode(), startTime.GetHashCode(), endTime.GetHashCode()); List <Bar> retrievalFunction() { DateTime t1 = DateTime.Now; Output.Write(string.Format("DataSourceYahoo: loading data for {0}...", Info[DataSourceParam.nickName])); JObject jsonData = getPrices(startTime, endTime); /* * Yahoo JSON format, as of 07/02/2019 * * [JSON] * chart * result * [0] * meta * currency * symbol * ... * timestamp * [0]: 511108200 * [1]: 511194600 * ... * indicators * quote * [0] * low * [0]: 0.08854 * [1]: 0.09722 * ... * close * volume * open * high * adjclose * [0] * adjclose * [0]: 0.06999 * [1]: 0.07249 */ List <Bar> bars = new List <Bar>(); var timestamps = (JArray)jsonData["chart"]["result"][0]["timestamp"]; var opens = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["open"]; var highs = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["high"]; var lows = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["low"]; var closes = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["close"]; var volumes = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["volume"]; var adjcloses = (JArray)jsonData["chart"]["result"][0]["indicators"]["adjclose"][0]["adjclose"]; var eT = timestamps.GetEnumerator(); var eO = opens.GetEnumerator(); var eH = highs.GetEnumerator(); var eL = lows.GetEnumerator(); var eC = closes.GetEnumerator(); var eV = volumes.GetEnumerator(); var eAC = adjcloses.GetEnumerator(); while (eT.MoveNext() && eO.MoveNext() && eH.MoveNext() && eL.MoveNext() && eC.MoveNext() && eV.MoveNext() && eAC.MoveNext()) { DateTime t = fromUnixTime((long)eT.Current).Date + DateTime.Parse("16:00").TimeOfDay; Bar bar = null; try { // Yahoo taints the results by filling in null values // we try to handle this gracefully in the catch block double o = (double)eO.Current; double h = (double)eH.Current; double l = (double)eL.Current; double c = (double)eC.Current; long v = (long)eV.Current; double ac = (double)eAC.Current; // adjust prices according to the adjusted close. // note the volume is adjusted the opposite way. double ao = o * ac / c; double ah = h * ac / c; double al = l * ac / c; long av = (long)(v * c / ac); bar = Bar.NewOHLC( Info[DataSourceParam.ticker], t, ao, ah, al, ac, av); } catch { if (bars.Count < 1) { continue; } Bar prevBar = bars.Last(); bar = Bar.NewOHLC( Info[DataSourceParam.ticker], t, prevBar.Open, prevBar.High, prevBar.Low, prevBar.Close, prevBar.Volume); } if (t >= startTime && t <= endTime) { bars.Add(bar); } } DateTime t2 = DateTime.Now; Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds)); return(bars); }; data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true); } catch (Exception e) { throw new Exception( string.Format("DataSourceYahoo: failed to load quotes for {0}, {1}", Info[DataSourceParam.nickName], e.Message)); } if (data.Count == 0) { throw new Exception(string.Format("DataSourceYahoo: no data for {0}", Info[DataSourceParam.nickName])); } CachedData = data; return(data); }
/// <summary> /// Create new cache id. Typically, this constructor is used without parameters, /// resulting in a new top-level cache id, specific to the current member function /// and line number. /// </summary> /// <param name="parentId">parent cache id. Typically not required. Pass in value when nesting indicators.</param> /// <param name="memberName">member function name. Typically filled in by compiler services. Pass in value when creating indicators.</param> /// <param name="lineNumber">line number. Typically filled in by compiler services. Pass in value when creating indicators.</param> public CacheId(CacheId parentId = null, [CallerMemberName] string memberName = "", [CallerLineNumber] int lineNumber = 0) { ApplyCallStack(parentId, memberName, lineNumber); ApplyParameters(parentId); }
/// <summary> /// Run sub-classed algorithm and return bars as enumerable. /// </summary> /// <param name="startTime">start of load range</param> /// <param name="endTime">end of load range</param> /// <returns>bars created from sub-classed algo</returns> public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime) { #if true _algo.IsDataSource = true; if (_algo.IsChildAlgorithm) { // for child algorithms, we bypass the cache and run the // child bar-for-bar and in sync with its parent foreach (var bar in _algo.Run(startTime, endTime)) { yield return(bar); } yield break; } else { // for all other algorithms, we run the algo // all at once and save the result in the cache var algoNick = Info[DataSourceParam.nickName]; var cacheKey = new CacheId(null, "", 0, algoNick.GetHashCode(), // _algoName.GetHashCode(), startTime.GetHashCode(), endTime.GetHashCode()); List <Bar> retrievalFunction() { try { DateTime t1 = DateTime.Now; Output.WriteLine(string.Format("DataSourceAlgorithm: generating data for {0}...", Info[DataSourceParam.nickName])); var bars = _algo.Run(startTime, endTime) .ToList(); DateTime t2 = DateTime.Now; Output.WriteLine(string.Format("DataSourceAlgorithm: finished after {0:F1} seconds", (t2 - t1).TotalSeconds)); return(bars); } catch { throw new Exception("DataSourceAlgorithm: failed to run sub-classed algorithm " + algoNick); } } List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true); if (data.Count == 0) { throw new Exception(string.Format("DataSourceAlgorithm: no data for {0}", Info[DataSourceParam.nickName])); } CachedData = data; foreach (var bar in data) { yield return(bar); } } #else var algoNick = Info[DataSourceParam.nickName]; var cacheKey = new CacheId(null, "", 0, algoNick.GetHashCode(), // _algoName.GetHashCode(), startTime.GetHashCode(), endTime.GetHashCode()); List <Bar> retrievalFunction() { try { DateTime t1 = DateTime.Now; Output.WriteLine(string.Format("DataSourceAlgorithm: generating data for {0}...", Info[DataSourceParam.nickName])); _algo.StartTime = startTime; _algo.EndTime = endTime; _algo.ParentDataSource = this; _algo.SubclassedData = new List <Bar>();; _algo.Run(); DateTime t2 = DateTime.Now; Output.WriteLine(string.Format("DataSourceAlgorithm: finished after {0:F1} seconds", (t2 - t1).TotalSeconds)); return(_algo.SubclassedData); } catch { throw new Exception("DataSourceAlgorithm: failed to run sub-classed algorithm " + algoNick); } } List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true); if (data.Count == 0) { throw new Exception(string.Format("DataSourceAlgorithm: no data for {0}", Info[DataSourceParam.nickName])); } Data = data; #endif }
/// <summary> /// Create new cache id. This constructor is specifically aimed at building indicators. Therefore, /// all parameters are mandatory. The cache id created by this constructor is idential to one /// created using new CacheId(...).AddParameters(...) /// </summary> /// <param name="parentId">parent cache id</param> /// <param name="memberName">member function name, or ""</param> /// <param name="lineNumber">line number or 0</param> /// <param name="parameterIds">list of parameter ids</param> public CacheId(CacheId parentId, string memberName, int lineNumber, params int[] parameterIds) { ApplyCallStack(parentId, memberName, lineNumber); ApplyParameters(parentId, parameterIds); }