public override void Run() { AddDataSource(SPX); int i = 0; foreach (var s in SimTimes) { // increment, until raw bar has a timestamp // LARGER (not equal) than the simulator. while (i < _rawBars.Count && SimTime[0] >= _rawBars[i].Time) { i++; } // now the raw bar we want is the previous one if (i > 0) { var rawBar = _rawBars[i - 1]; var alignedBar = Bar.NewOHLC( rawBar.Symbol, SimTime[0], rawBar.Open, rawBar.High, rawBar.Low, rawBar.Close, rawBar.Volume); _alignedBars.Add(alignedBar); } } }
/// <summary> /// Add sub-classed bar: arbitrary value /// </summary> /// <param name="value">value to copy bar's OHLC</param> protected void AddSubclassedBar(double value) { if (ParentDataSource != null) { Bar bar = Bar.NewOHLC( ParentDataSource.Info[DataSourceParam.ticker], SimTime[0], value, value, value, value, 0); AddSubclassedBar(bar); } }
/// <summary> /// Run data update. /// </summary> /// <param name="startTime">start of update range</param> /// <param name="endTime">end of update range</param> /// <returns>enumerable of updated bars</returns> override public IEnumerable <Bar> UpdateData(DateTime startTime, DateTime endTime) { string url = string.Format(URL_TEMPLATE, Info[DataSourceParam.symbolFred], startTime, endTime); var rawBars = new List <Bar>(); using (var client = new WebClient()) { string rawData = client.DownloadString(url); using (MemoryStream ms = new MemoryStream(Encoding.UTF8.GetBytes(rawData))) using (StreamReader reader = new StreamReader(ms)) { string header = reader.ReadLine(); // skip header line for (string line; (line = reader.ReadLine()) != null;) { if (line.Length == 0) { continue; // to handle end of file } string[] items = (Info[DataSourceParam.ticker] + "," + line).Split(','); var timestamp = DateTime.Parse(string.Format(PARSE_INFO[DataSourceParam.date], items)).Date + DateTime.Parse(string.Format(PARSE_INFO[DataSourceParam.time], items)).TimeOfDay; var close = double.Parse(string.Format(PARSE_INFO[DataSourceParam.close], items)); var bar = Bar.NewOHLC( Info[DataSourceParam.ticker], timestamp, close, close, close, close, default(long)); rawBars.Add(bar); } } } // using (var client = new WebClient()) // use a simulator instance to align bars w/ S&P 500 var alignedBars = new List <Bar>(); var align = new AlignWithMarket(startTime, endTime, rawBars, alignedBars); align.Run(); return(alignedBars); }
/// <summary> /// Load data into memory. /// </summary> /// <param name="startTime">start of load range</param> /// <param name="endTime">end of load range</param> override public void LoadData(DateTime startTime, DateTime endTime) { try { if (startTime < (DateTime)FirstTime) { startTime = (DateTime)FirstTime; } //if (endTime > (DateTime)LastTime) // endTime = (DateTime)LastTime; var cacheKey = new CacheId(null, "", 0, Info[DataSourceParam.nickName].GetHashCode(), startTime.GetHashCode(), endTime.GetHashCode()); List <Bar> retrievalFunction() { DateTime t1 = DateTime.Now; Output.Write(string.Format("DataSourceTiingo: loading data for {0}...", Info[DataSourceParam.nickName])); List <Bar> bars = new List <Bar>(); JArray jsonData = getPrices(startTime, endTime); var e = jsonData.GetEnumerator(); while (e.MoveNext()) { var bar = e.Current; DateTime date = DateTime.Parse((string)bar["date"], CultureInfo.InvariantCulture).Date + DateTime.Parse(Info[DataSourceParam.time]).TimeOfDay; double open = (double)bar["adjOpen"]; double high = (double)bar["adjHigh"]; double low = (double)bar["adjLow"]; double close = (double)bar["adjClose"]; long volume = (long)bar["adjVolume"]; if (date >= startTime && date <= endTime) { bars.Add(Bar.NewOHLC( Info[DataSourceParam.ticker], date, open, high, low, close, volume)); } } DateTime t2 = DateTime.Now; Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds)); return(bars); }; Data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction);; } catch (Exception e) { throw new Exception( string.Format("DataSourceTiingo: failed to load quotes for {0}, {1}", Info[DataSourceParam.nickName], e.Message)); } if ((Data as List <Bar>).Count == 0) { throw new Exception(string.Format("DataSourceTiingo: no data for {0}", Info[DataSourceParam.nickName])); } }
/// <summary> /// Load data into memory. /// </summary> /// <param name="startTime">start of load range</param> /// <param name="endTime">end of load range</param> override public void LoadData(DateTime startTime, DateTime endTime) { try { if (startTime < (DateTime)FirstTime) { startTime = (DateTime)FirstTime; } //if (endTime > (DateTime)LastTime) // endTime = (DateTime)LastTime; var cacheKey = new CacheId(null, "", 0, Info[DataSourceParam.nickName].GetHashCode(), startTime.GetHashCode(), endTime.GetHashCode()); List <Bar> retrievalFunction() { DateTime t1 = DateTime.Now; Output.Write(string.Format("DataSourceFred: loading data for {0}...", Info[DataSourceParam.nickName])); List <Bar> rawBars = new List <Bar>(); JObject jsonData = getData(startTime, endTime); var e = ((JArray)jsonData["observations"]).GetEnumerator(); while (e.MoveNext()) { var bar = e.Current; DateTime date = DateTime.Parse((string)bar["date"]).Date + DateTime.Parse(Info[DataSourceParam.time]).TimeOfDay; string valueString = (string)bar["value"]; double value; try { value = double.Parse(valueString); } catch { // when we get here, this was probably a missing value, // which FRED substitutes with "." // we ignore and move on, AlignWithMarket will take // care of the issue gracefully continue; } rawBars.Add(Bar.NewOHLC( Info[DataSourceParam.ticker], date, value, value, value, value, 0)); } List <Bar> alignedBars = DataSourceHelper.AlignWithMarket(rawBars, startTime, endTime); DateTime t2 = DateTime.Now; Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds)); return(alignedBars); }; Data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction); // FIXME: this is far from ideal. We want to make sure that retired // series are not extended indefinitely LastTime = (Data as List <Bar>).FindLast(b => true).Time; } catch (Exception e) { throw new Exception( string.Format("DataSourceFred: failed to load quotes for {0}, {1}", Info[DataSourceParam.nickName], e.Message)); } if ((Data as List <Bar>).Count == 0) { throw new Exception(string.Format("DataSourceFred: no data for {0}", Info[DataSourceParam.nickName])); } }
/// <summary> /// Run sub-classed algorithm and return bars as enumerable. /// </summary> /// <param name="startTime">start of load range</param> /// <param name="endTime">end of load range</param> /// <returns>bars created from sub-classed algo</returns> public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime) { #if true _algo.IsDataSource = true; if (_algo.CanRunAsChild && Info.ContainsKey(DataSourceParam.allowSync)) { // for child algorithms, we bypass the cache and run the // child bar-for-bar and in sync with its parent foreach (var bar in _algo.Run(startTime, endTime)) { yield return(bar); if (!_algo.IsLastBar) { // the simulator core needs to know the next bar's // timestamp. at the same time, we want to avoid // child algorithms from running one bar ahead of // the main algo. we fix this issue by returning // a dummy bar, announcing the next timestamp. var dummy = Bar.NewOHLC( null, _algo.NextSimTime, 0.0, 0.0, 0.0, 0.0, 0); yield return(dummy); } } yield break; } else { // for all other algorithms, we run the algo // all at once and save the result in the cache var algoNick = Info[DataSourceParam.nickName]; var cacheKey = new CacheId().AddParameters( algoNick.GetHashCode(), // _algoName.GetHashCode(), startTime.GetHashCode(), endTime.GetHashCode()); List <Bar> retrievalFunction() { try { DateTime t1 = DateTime.Now; Output.WriteLine(string.Format("DataSourceAlgorithm: generating data for {0}...", Info[DataSourceParam.nickName])); var bars = _algo.Run(startTime, endTime) .ToList(); DateTime t2 = DateTime.Now; Output.WriteLine(string.Format("DataSourceAlgorithm: finished after {0:F1} seconds", (t2 - t1).TotalSeconds)); return(bars); } catch { throw new Exception("DataSourceAlgorithm: failed to run sub-classed algorithm " + algoNick); } } List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true); if (data.Count == 0) { throw new Exception(string.Format("DataSourceAlgorithm: no data for {0}", Info[DataSourceParam.nickName])); } CachedData = data; foreach (var bar in data) { yield return(bar); } } #else var algoNick = Info[DataSourceParam.nickName]; var cacheKey = new CacheId(null, "", 0, algoNick.GetHashCode(), // _algoName.GetHashCode(), startTime.GetHashCode(), endTime.GetHashCode()); List <Bar> retrievalFunction() { try { DateTime t1 = DateTime.Now; Output.WriteLine(string.Format("DataSourceAlgorithm: generating data for {0}...", Info[DataSourceParam.nickName])); _algo.StartTime = startTime; _algo.EndTime = endTime; _algo.ParentDataSource = this; _algo.SubclassedData = new List <Bar>();; _algo.Run(); DateTime t2 = DateTime.Now; Output.WriteLine(string.Format("DataSourceAlgorithm: finished after {0:F1} seconds", (t2 - t1).TotalSeconds)); return(_algo.SubclassedData); } catch { throw new Exception("DataSourceAlgorithm: failed to run sub-classed algorithm " + algoNick); } } List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true); if (data.Count == 0) { throw new Exception(string.Format("DataSourceAlgorithm: no data for {0}", Info[DataSourceParam.nickName])); } Data = data; #endif }
/// <summary> /// Load data into memory. /// </summary> /// <param name="startTime">start of load range</param> /// <param name="endTime">end of load range</param> public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime) { List <Bar> data = new List <Bar>(); try { //if (startTime < (DateTime)FirstTime) // startTime = (DateTime)FirstTime; //if (endTime > (DateTime)LastTime) // endTime = (DateTime)LastTime; var cacheKey = new CacheId(null, "", 0, Info[DataSourceParam.nickName].GetHashCode(), startTime.GetHashCode(), endTime.GetHashCode()); List <Bar> retrievalFunction() { DateTime t1 = DateTime.Now; Output.Write(string.Format("DataSourceYahoo: loading data for {0}...", Info[DataSourceParam.nickName])); JObject jsonData = getPrices(startTime, endTime); /* * Yahoo JSON format, as of 07/02/2019 * * [JSON] * chart * result * [0] * meta * currency * symbol * ... * timestamp * [0]: 511108200 * [1]: 511194600 * ... * indicators * quote * [0] * low * [0]: 0.08854 * [1]: 0.09722 * ... * close * volume * open * high * adjclose * [0] * adjclose * [0]: 0.06999 * [1]: 0.07249 */ List <Bar> bars = new List <Bar>(); var timestamps = (JArray)jsonData["chart"]["result"][0]["timestamp"]; var opens = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["open"]; var highs = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["high"]; var lows = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["low"]; var closes = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["close"]; var volumes = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["volume"]; var adjcloses = (JArray)jsonData["chart"]["result"][0]["indicators"]["adjclose"][0]["adjclose"]; var eT = timestamps.GetEnumerator(); var eO = opens.GetEnumerator(); var eH = highs.GetEnumerator(); var eL = lows.GetEnumerator(); var eC = closes.GetEnumerator(); var eV = volumes.GetEnumerator(); var eAC = adjcloses.GetEnumerator(); while (eT.MoveNext() && eO.MoveNext() && eH.MoveNext() && eL.MoveNext() && eC.MoveNext() && eV.MoveNext() && eAC.MoveNext()) { DateTime t = fromUnixTime((long)eT.Current).Date + DateTime.Parse("16:00").TimeOfDay; Bar bar = null; try { // Yahoo taints the results by filling in null values // we try to handle this gracefully in the catch block double o = (double)eO.Current; double h = (double)eH.Current; double l = (double)eL.Current; double c = (double)eC.Current; long v = (long)eV.Current; double ac = (double)eAC.Current; // adjust prices according to the adjusted close. // note the volume is adjusted the opposite way. double ao = o * ac / c; double ah = h * ac / c; double al = l * ac / c; long av = (long)(v * c / ac); bar = Bar.NewOHLC( Info[DataSourceParam.ticker], t, ao, ah, al, ac, av); } catch { if (bars.Count < 1) { continue; } Bar prevBar = bars.Last(); bar = Bar.NewOHLC( Info[DataSourceParam.ticker], t, prevBar.Open, prevBar.High, prevBar.Low, prevBar.Close, prevBar.Volume); } if (t >= startTime && t <= endTime) { bars.Add(bar); } } DateTime t2 = DateTime.Now; Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds)); return(bars); }; data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true); } catch (Exception e) { throw new Exception( string.Format("DataSourceYahoo: failed to load quotes for {0}, {1}", Info[DataSourceParam.nickName], e.Message)); } if (data.Count == 0) { throw new Exception(string.Format("DataSourceYahoo: no data for {0}", Info[DataSourceParam.nickName])); } CachedData = data; return(data); }
/// <summary> /// Load data into memory. /// </summary> /// <param name="startTime">start of load range</param> /// <param name="endTime">end of load range</param> public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime) { List <Bar> data = new List <Bar>(); try { if (startTime < (DateTime)_firstTime) { startTime = (DateTime)_firstTime; } var cacheKey = new CacheId().AddParameters( Info[DataSourceParam.nickName].GetHashCode(), startTime.GetHashCode(), endTime.GetHashCode()); List <Bar> retrievalFunction() { DateTime t1 = DateTime.Now; Output.Write(string.Format("DataSourceStooq: loading data for {0}...", Info[DataSourceParam.nickName])); List <Bar> bars = new List <Bar>(); List <StooqDataRow> stooqData = getPrices(startTime, endTime); var e = stooqData.GetEnumerator(); while (e.MoveNext()) { var bar = e.Current; DateTime date = bar.Date; if (date >= startTime && date <= endTime) { bars.Add(Bar.NewOHLC( Info[DataSourceParam.ticker], date, bar.Open, bar.High, bar.Low, bar.Close, bar.Volume)); } } DateTime t2 = DateTime.Now; Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds)); return(bars); }; data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true);; } catch (Exception e) { throw new Exception( string.Format("DataSourceStooq: failed to load quotes for {0}, {1}", Info[DataSourceParam.nickName], e.Message)); } if (data.Count == 0) { throw new Exception(string.Format("DataSourceStooq: no data for {0}", Info[DataSourceParam.nickName])); } CachedData = data; return(data); }
/// <summary> /// Load data into memory. /// </summary> /// <param name="startTime">start of load range</param> /// <param name="endTime">end of load range</param> override public void LoadData(DateTime startTime, DateTime endTime) { var cacheKey = new CacheId(null, "", 0, Info[DataSourceParam.nickName].GetHashCode(), startTime.GetHashCode(), endTime.GetHashCode()); List <Bar> retrievalFunction() { DateTime t1 = DateTime.Now; //Output.Write(string.Format("{0}: loading data for {1}...", GetType().Name, Info[DataSourceParam.nickName])); // load data from specified data sources // and save as list of bars, in reverse order Dictionary <string, List <Bar> > dsBars = new Dictionary <string, List <Bar> >(); foreach (var nick in _symbols) { var d = DataSource.New(nick); try { d.LoadData(startTime, endTime); dsBars[nick] = d.Data.Reverse().ToList(); } catch (Exception e) { Output.WriteLine("{0}: {1} failed to load {2}", this.GetType().Name, Info[DataSourceParam.nickName], nick); // add an empty list, if need be // this will be ignored further down during splicing if (!dsBars.ContainsKey(nick)) { dsBars[nick] = new List <Bar>(); } } //Output.WriteLine("{0}: {1} data range {2:MM/dd/yyyy}, {3:MM/dd/yyyy}", GetType().Name, nick, d.FirstTime, d.LastTime); } // create enumerators for all data sources Dictionary <string, IEnumerator <Bar> > dsEnums = new Dictionary <string, IEnumerator <Bar> >(); Dictionary <string, bool> dsHasData = new Dictionary <string, bool>(); foreach (var nick in _symbols) { dsEnums[nick] = dsBars[nick].GetEnumerator(); dsHasData[nick] = dsEnums[nick].MoveNext(); } // collect bars List <Bar> bars = new List <Bar>(); Dictionary <string, double> dsScale = new Dictionary <string, double>(); _symbols.ForEach(n => dsScale[n] = 1.0); while (dsHasData.Values.Aggregate((a, b) => a || b)) { // find most-recent timestamp DateTime ts = _symbols .Where(n => dsHasData[n]) .Select(n => dsEnums[n].Current.Time) .Max(t => t); Bar bar = null; foreach (var nick in _symbols) { // no data: continue if (!dsHasData[nick]) { continue; } // older bar: continue if (dsEnums[nick].Current.Time < ts) { continue; } if (bar == null) { // highest priority bar Bar rawBar = dsEnums[nick].Current; double open = rawBar.Open * dsScale[nick]; double high = rawBar.High * dsScale[nick]; double low = rawBar.Low * dsScale[nick]; double close = rawBar.Close * dsScale[nick]; long volume = 0; bar = Bar.NewOHLC(Info[DataSourceParam.ticker], ts, open, high, low, close, volume); bars.Add(bar); } else { // lower priority bars Bar rawBar = dsEnums[nick].Current; List <double> scales = new List <double> { bar.Open / rawBar.Open, bar.High / rawBar.High, bar.Low / rawBar.Low, bar.Close / rawBar.Close, }; dsScale[nick] = scales.Average(); } dsHasData[nick] = dsEnums[nick].MoveNext(); } } // reverse order of bars bars.Reverse(); DateTime t2 = DateTime.Now; //Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds)); return(bars); }; List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction); if (data.Count == 0) { throw new Exception(string.Format("{0}: no data for {1}", GetType().Name, Info[DataSourceParam.nickName])); } Data = data; }
/// <summary> /// Load data into memory. /// </summary> /// <param name="startTime">start of load range</param> /// <param name="endTime">end of load range</param> public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime) { var cacheKey = new CacheId(null, "", 0, Info[DataSourceParam.nickName].GetHashCode(), startTime.GetHashCode(), endTime.GetHashCode()); List <Bar> retrievalFunction() { DateTime t1 = DateTime.Now; //Output.Write(string.Format("{0}: loading data for {1}...", GetType().Name, Info[DataSourceParam.nickName])); // load data from specified data sources // and save as list of bars, in reverse order Dictionary <string, List <Bar> > dsBars = new Dictionary <string, List <Bar> >(); foreach (var nick in _symbols) { var d = DataSource.New(nick); try { var data = d.LoadData(startTime, endTime); dsBars[nick] = data.Reverse().ToList(); } catch (Exception /*e*/) { Output.WriteLine("{0}: {1} failed to load {2}", this.GetType().Name, Info[DataSourceParam.nickName], nick); // add an empty list, if need be // this will be ignored further down during splicing if (!dsBars.ContainsKey(nick)) { dsBars[nick] = new List <Bar>(); } } //Output.WriteLine("{0}: {1} data range {2:MM/dd/yyyy}, {3:MM/dd/yyyy}", GetType().Name, nick, d.FirstTime, d.LastTime); } // create enumerators for all data sources Dictionary <string, IEnumerator <Bar> > dsEnums = new Dictionary <string, IEnumerator <Bar> >(); Dictionary <string, bool> dsHasData = new Dictionary <string, bool>(); foreach (var nick in _symbols) { dsEnums[nick] = dsBars[nick].GetEnumerator(); dsHasData[nick] = dsEnums[nick].MoveNext(); } // skip bars from all proxy datasources, so that no proxy // has bars after the primary datasource // example: extending GLD w/ XAUUSD. because XAUUSD has // a different trading calendar, XAUUSD might have bars // after GLD, leading to faulty results var lastPrimary = dsEnums[_symbols.First()].Current.Time; foreach (var nick in _symbols) { while (dsHasData[nick] && dsEnums[nick].Current.Time > lastPrimary) { dsHasData[nick] = dsEnums[nick].MoveNext(); } } // collect bars List <Bar> bars = new List <Bar>(); Dictionary <string, double?> dsScale = new Dictionary <string, double?>(); _symbols.ForEach(n => dsScale[n] = null); dsScale[_symbols.First()] = 1.0; while (dsHasData.Values.Aggregate((a, b) => a || b)) { // find most-recent timestamp DateTime ts = _symbols .Where(n => dsHasData[n]) .Select(n => dsEnums[n].Current.Time) .Max(t => t); Bar bar = null; foreach (var nick in _symbols) { // no data: continue if (!dsHasData[nick]) { continue; } // older bar: continue if (dsEnums[nick].Current.Time < ts) { continue; } if (bar == null) { // highest priority bar Bar rawBar = dsEnums[nick].Current; // we might get here, with dsScale not set yet. // this is the best we can do to fix things if (dsScale[nick] == null) { dsScale[nick] = bars.Last().Open / rawBar.Close; } double open = rawBar.Open * (double)dsScale[nick]; double high = rawBar.High * (double)dsScale[nick]; double low = rawBar.Low * (double)dsScale[nick]; double close = rawBar.Close * (double)dsScale[nick]; long volume = 0; bar = Bar.NewOHLC(Info[DataSourceParam.ticker], ts, open, high, low, close, volume); bars.Add(bar); } else { // lower priority bars Bar rawBar = dsEnums[nick].Current; List <double> scales = new List <double> { bar.Open / rawBar.Open, bar.High / rawBar.High, bar.Low / rawBar.Low, bar.Close / rawBar.Close, }; dsScale[nick] = scales.Average(); } dsHasData[nick] = dsEnums[nick].MoveNext(); } } // reverse order of bars bars.Reverse(); DateTime t2 = DateTime.Now; //Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds)); return(bars); }; List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true); if (data.Count == 0) { throw new Exception(string.Format("{0}: no data for {1}", GetType().Name, Info[DataSourceParam.nickName])); } CachedData = data; return(data); }