/// <summary> /// Load data into memory. /// </summary> /// <param name="startTime">start of load range</param> /// <param name="endTime">end of load range</param> public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime) { var cacheKey = new CacheId().AddParameters( Info[DataSourceParam.nickName].GetHashCode(), startTime.GetHashCode(), endTime.GetHashCode()); List <Bar> retrievalFunction() { DateTime t1 = DateTime.Now; //Output.Write(string.Format("{0}: loading data for {1}...", GetType().Name, Info[DataSourceParam.nickName])); // load data from specified data sources // and save as list of bars, in reverse order Dictionary <string, List <Bar> > dsBars = new Dictionary <string, List <Bar> >(); foreach (var nick in _symbols) { var d = DataSource.New(nick); try { var data = d.LoadData(startTime, endTime); dsBars[nick] = data.Reverse().ToList(); } catch (Exception /*e*/) { Output.WriteLine("{0}: {1} failed to load {2}", this.GetType().Name, Info[DataSourceParam.nickName], nick); // add an empty list, if need be // this will be ignored further down during splicing if (!dsBars.ContainsKey(nick)) { dsBars[nick] = new List <Bar>(); } } //Output.WriteLine("{0}: {1} data range {2:MM/dd/yyyy}, {3:MM/dd/yyyy}", GetType().Name, nick, d.FirstTime, d.LastTime); } // create enumerators for all data sources Dictionary <string, IEnumerator <Bar> > dsEnums = new Dictionary <string, IEnumerator <Bar> >(); Dictionary <string, bool> dsHasData = new Dictionary <string, bool>(); foreach (var nick in _symbols) { dsEnums[nick] = dsBars[nick].GetEnumerator(); dsHasData[nick] = dsEnums[nick].MoveNext(); } // skip bars from all proxy datasources, so that no proxy // has bars after the primary datasource // example: extending GLD w/ XAUUSD. because XAUUSD has // a different trading calendar, XAUUSD might have bars // after GLD, leading to faulty results var lastPrimary = dsEnums[_symbols.First()].Current.Time; foreach (var nick in _symbols) { while (dsHasData[nick] && dsEnums[nick].Current.Time > lastPrimary) { dsHasData[nick] = dsEnums[nick].MoveNext(); } } // collect bars List <Bar> bars = new List <Bar>(); Dictionary <string, double?> dsScale = new Dictionary <string, double?>(); _symbols.ForEach(n => dsScale[n] = null); dsScale[_symbols.First()] = 1.0; while (dsHasData.Values.Aggregate((a, b) => a || b)) { // find most-recent timestamp DateTime ts = _symbols .Where(n => dsHasData[n]) .Select(n => dsEnums[n].Current.Time) .Max(t => t); Bar bar = null; foreach (var nick in _symbols) { // no data: continue if (!dsHasData[nick]) { continue; } // older bar: continue if (dsEnums[nick].Current.Time < ts) { continue; } if (bar == null) { // highest priority bar Bar rawBar = dsEnums[nick].Current; // we might get here, with dsScale not set yet. // this is the best we can do to fix things if (dsScale[nick] == null) { dsScale[nick] = bars.Last().Open / rawBar.Close; } double open = rawBar.Open * (double)dsScale[nick]; double high = rawBar.High * (double)dsScale[nick]; double low = rawBar.Low * (double)dsScale[nick]; double close = rawBar.Close * (double)dsScale[nick]; long volume = 0; bar = Bar.NewOHLC(Info[DataSourceParam.ticker], ts, open, high, low, close, volume); bars.Add(bar); } else { // lower priority bars Bar rawBar = dsEnums[nick].Current; List <double> scales = new List <double> { bar.Open / rawBar.Open, bar.High / rawBar.High, bar.Low / rawBar.Low, bar.Close / rawBar.Close, }; dsScale[nick] = scales.Average(); } dsHasData[nick] = dsEnums[nick].MoveNext(); } } // reverse order of bars bars.Reverse(); DateTime t2 = DateTime.Now; //Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds)); return(bars); }; List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true); if (data.Count == 0) { throw new Exception(string.Format("{0}: no data for {1}", GetType().Name, Info[DataSourceParam.nickName])); } CachedData = data; return(data); }