public override void Run()
            {
                AddDataSource(SPX);

                int i = 0;

                foreach (var s in SimTimes)
                {
                    // increment, until raw bar has a timestamp
                    // LARGER (not equal) than the simulator.
                    while (i < _rawBars.Count &&
                           SimTime[0] >= _rawBars[i].Time)
                    {
                        i++;
                    }

                    // now the raw bar we want is the previous one
                    if (i > 0)
                    {
                        var rawBar     = _rawBars[i - 1];
                        var alignedBar = Bar.NewOHLC(
                            rawBar.Symbol, SimTime[0],
                            rawBar.Open, rawBar.High, rawBar.Low, rawBar.Close, rawBar.Volume);

                        _alignedBars.Add(alignedBar);
                    }
                }
            }
Пример #2
0
        /// <summary>
        /// Add sub-classed bar: arbitrary value
        /// </summary>
        /// <param name="value">value to copy bar's OHLC</param>
        protected void AddSubclassedBar(double value)
        {
            if (ParentDataSource != null)
            {
                Bar bar = Bar.NewOHLC(
                    ParentDataSource.Info[DataSourceParam.ticker],
                    SimTime[0],
                    value, value, value, value, 0);

                AddSubclassedBar(bar);
            }
        }
            /// <summary>
            /// Run data update.
            /// </summary>
            /// <param name="startTime">start of update range</param>
            /// <param name="endTime">end of update range</param>
            /// <returns>enumerable of updated bars</returns>
            override public IEnumerable <Bar> UpdateData(DateTime startTime, DateTime endTime)
            {
                string url = string.Format(URL_TEMPLATE,
                                           Info[DataSourceParam.symbolFred], startTime, endTime);

                var rawBars = new List <Bar>();

                using (var client = new WebClient())
                {
                    string rawData = client.DownloadString(url);

                    using (MemoryStream ms = new MemoryStream(Encoding.UTF8.GetBytes(rawData)))
                        using (StreamReader reader = new StreamReader(ms))
                        {
                            string header = reader.ReadLine(); // skip header line

                            for (string line; (line = reader.ReadLine()) != null;)
                            {
                                if (line.Length == 0)
                                {
                                    continue; // to handle end of file
                                }
                                string[] items = (Info[DataSourceParam.ticker] + "," + line).Split(',');

                                var timestamp =
                                    DateTime.Parse(string.Format(PARSE_INFO[DataSourceParam.date], items)).Date
                                    + DateTime.Parse(string.Format(PARSE_INFO[DataSourceParam.time], items)).TimeOfDay;

                                var close = double.Parse(string.Format(PARSE_INFO[DataSourceParam.close], items));

                                var bar = Bar.NewOHLC(
                                    Info[DataSourceParam.ticker], timestamp,
                                    close, close, close, close, default(long));

                                rawBars.Add(bar);
                            }
                        }
                } // using (var client = new WebClient())

                // use a simulator instance to align bars w/ S&P 500
                var alignedBars = new List <Bar>();
                var align       = new AlignWithMarket(startTime, endTime, rawBars, alignedBars);

                align.Run();

                return(alignedBars);
            }
Пример #4
0
            /// <summary>
            /// Load data into memory.
            /// </summary>
            /// <param name="startTime">start of load range</param>
            /// <param name="endTime">end of load range</param>
            override public void LoadData(DateTime startTime, DateTime endTime)
            {
                try
                {
                    if (startTime < (DateTime)FirstTime)
                    {
                        startTime = (DateTime)FirstTime;
                    }

                    //if (endTime > (DateTime)LastTime)
                    //    endTime = (DateTime)LastTime;

                    var cacheKey = new CacheId(null, "", 0,
                                               Info[DataSourceParam.nickName].GetHashCode(),
                                               startTime.GetHashCode(),
                                               endTime.GetHashCode());

                    List <Bar> retrievalFunction()
                    {
                        DateTime t1 = DateTime.Now;

                        Output.Write(string.Format("DataSourceTiingo: loading data for {0}...", Info[DataSourceParam.nickName]));

                        List <Bar> bars = new List <Bar>();

                        JArray jsonData = getPrices(startTime, endTime);
                        var    e        = jsonData.GetEnumerator();

                        while (e.MoveNext())
                        {
                            var bar = e.Current;

                            DateTime date = DateTime.Parse((string)bar["date"], CultureInfo.InvariantCulture).Date
                                            + DateTime.Parse(Info[DataSourceParam.time]).TimeOfDay;

                            double open   = (double)bar["adjOpen"];
                            double high   = (double)bar["adjHigh"];
                            double low    = (double)bar["adjLow"];
                            double close  = (double)bar["adjClose"];
                            long   volume = (long)bar["adjVolume"];

                            if (date >= startTime && date <= endTime)
                            {
                                bars.Add(Bar.NewOHLC(
                                             Info[DataSourceParam.ticker],
                                             date,
                                             open, high, low, close,
                                             volume));
                            }
                        }

                        DateTime t2 = DateTime.Now;

                        Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds));

                        return(bars);
                    };

                    Data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction);;
                }

                catch (Exception e)
                {
                    throw new Exception(
                              string.Format("DataSourceTiingo: failed to load quotes for {0}, {1}",
                                            Info[DataSourceParam.nickName], e.Message));
                }

                if ((Data as List <Bar>).Count == 0)
                {
                    throw new Exception(string.Format("DataSourceTiingo: no data for {0}", Info[DataSourceParam.nickName]));
                }
            }
Пример #5
0
            /// <summary>
            /// Load data into memory.
            /// </summary>
            /// <param name="startTime">start of load range</param>
            /// <param name="endTime">end of load range</param>
            override public void LoadData(DateTime startTime, DateTime endTime)
            {
                try
                {
                    if (startTime < (DateTime)FirstTime)
                    {
                        startTime = (DateTime)FirstTime;
                    }

                    //if (endTime > (DateTime)LastTime)
                    //    endTime = (DateTime)LastTime;

                    var cacheKey = new CacheId(null, "", 0,
                                               Info[DataSourceParam.nickName].GetHashCode(),
                                               startTime.GetHashCode(),
                                               endTime.GetHashCode());

                    List <Bar> retrievalFunction()
                    {
                        DateTime t1 = DateTime.Now;

                        Output.Write(string.Format("DataSourceFred: loading data for {0}...", Info[DataSourceParam.nickName]));

                        List <Bar> rawBars = new List <Bar>();

                        JObject jsonData = getData(startTime, endTime);
                        var     e        = ((JArray)jsonData["observations"]).GetEnumerator();

                        while (e.MoveNext())
                        {
                            var bar = e.Current;

                            DateTime date = DateTime.Parse((string)bar["date"]).Date
                                            + DateTime.Parse(Info[DataSourceParam.time]).TimeOfDay;

                            string valueString = (string)bar["value"];
                            double value;
                            try
                            {
                                value = double.Parse(valueString);
                            }
                            catch
                            {
                                // when we get here, this was probably a missing value,
                                // which FRED substitutes with "."
                                // we ignore and move on, AlignWithMarket will take
                                // care of the issue gracefully
                                continue;
                            }

                            rawBars.Add(Bar.NewOHLC(
                                            Info[DataSourceParam.ticker],
                                            date,
                                            value, value, value, value,
                                            0));
                        }

                        List <Bar> alignedBars = DataSourceHelper.AlignWithMarket(rawBars, startTime, endTime);

                        DateTime t2 = DateTime.Now;

                        Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds));

                        return(alignedBars);
                    };

                    Data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction);

                    // FIXME: this is far from ideal. We want to make sure that retired
                    //        series are not extended indefinitely
                    LastTime = (Data as List <Bar>).FindLast(b => true).Time;
                }

                catch (Exception e)
                {
                    throw new Exception(
                              string.Format("DataSourceFred: failed to load quotes for {0}, {1}",
                                            Info[DataSourceParam.nickName], e.Message));
                }

                if ((Data as List <Bar>).Count == 0)
                {
                    throw new Exception(string.Format("DataSourceFred: no data for {0}", Info[DataSourceParam.nickName]));
                }
            }
Пример #6
0
            /// <summary>
            /// Run sub-classed algorithm and return bars as enumerable.
            /// </summary>
            /// <param name="startTime">start of load range</param>
            /// <param name="endTime">end of load range</param>
            /// <returns>bars created from sub-classed algo</returns>
            public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime)
            {
#if true
                _algo.IsDataSource = true;

                if (_algo.CanRunAsChild && Info.ContainsKey(DataSourceParam.allowSync))
                {
                    // for child algorithms, we bypass the cache and run the
                    // child bar-for-bar and in sync with its parent

                    foreach (var bar in _algo.Run(startTime, endTime))
                    {
                        yield return(bar);

                        if (!_algo.IsLastBar)
                        {
                            // the simulator core needs to know the next bar's
                            // timestamp. at the same time, we want to avoid
                            // child algorithms from running one bar ahead of
                            // the main algo. we fix this issue by returning
                            // a dummy bar, announcing the next timestamp.

                            var dummy = Bar.NewOHLC(
                                null, _algo.NextSimTime,
                                0.0, 0.0, 0.0, 0.0, 0);

                            yield return(dummy);
                        }
                    }

                    yield break;
                }
                else
                {
                    // for all other algorithms, we run the algo
                    // all at once and save the result in the cache

                    var algoNick = Info[DataSourceParam.nickName];

                    var cacheKey = new CacheId().AddParameters(
                        algoNick.GetHashCode(), // _algoName.GetHashCode(),
                        startTime.GetHashCode(),
                        endTime.GetHashCode());

                    List <Bar> retrievalFunction()
                    {
                        try
                        {
                            DateTime t1 = DateTime.Now;
                            Output.WriteLine(string.Format("DataSourceAlgorithm: generating data for {0}...", Info[DataSourceParam.nickName]));

                            var bars = _algo.Run(startTime, endTime)
                                       .ToList();

                            DateTime t2 = DateTime.Now;
                            Output.WriteLine(string.Format("DataSourceAlgorithm: finished after {0:F1} seconds", (t2 - t1).TotalSeconds));

                            return(bars);
                        }

                        catch
                        {
                            throw new Exception("DataSourceAlgorithm: failed to run sub-classed algorithm " + algoNick);
                        }
                    }

                    List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true);

                    if (data.Count == 0)
                    {
                        throw new Exception(string.Format("DataSourceAlgorithm: no data for {0}", Info[DataSourceParam.nickName]));
                    }

                    CachedData = data;
                    foreach (var bar in data)
                    {
                        yield return(bar);
                    }
                }
#else
                var algoNick = Info[DataSourceParam.nickName];

                var cacheKey = new CacheId(null, "", 0,
                                           algoNick.GetHashCode(), // _algoName.GetHashCode(),
                                           startTime.GetHashCode(),
                                           endTime.GetHashCode());

                List <Bar> retrievalFunction()
                {
                    try
                    {
                        DateTime t1 = DateTime.Now;
                        Output.WriteLine(string.Format("DataSourceAlgorithm: generating data for {0}...", Info[DataSourceParam.nickName]));

                        _algo.StartTime        = startTime;
                        _algo.EndTime          = endTime;
                        _algo.ParentDataSource = this;

                        _algo.SubclassedData = new List <Bar>();;

                        _algo.Run();

                        DateTime t2 = DateTime.Now;
                        Output.WriteLine(string.Format("DataSourceAlgorithm: finished after {0:F1} seconds", (t2 - t1).TotalSeconds));

                        return(_algo.SubclassedData);
                    }

                    catch
                    {
                        throw new Exception("DataSourceAlgorithm: failed to run sub-classed algorithm " + algoNick);
                    }
                }

                List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true);

                if (data.Count == 0)
                {
                    throw new Exception(string.Format("DataSourceAlgorithm: no data for {0}", Info[DataSourceParam.nickName]));
                }

                Data = data;
#endif
            }
Пример #7
0
            /// <summary>
            /// Load data into memory.
            /// </summary>
            /// <param name="startTime">start of load range</param>
            /// <param name="endTime">end of load range</param>
            public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime)
            {
                List <Bar> data = new List <Bar>();

                try
                {
                    //if (startTime < (DateTime)FirstTime)
                    //    startTime = (DateTime)FirstTime;

                    //if (endTime > (DateTime)LastTime)
                    //    endTime = (DateTime)LastTime;

                    var cacheKey = new CacheId(null, "", 0,
                                               Info[DataSourceParam.nickName].GetHashCode(),
                                               startTime.GetHashCode(),
                                               endTime.GetHashCode());

                    List <Bar> retrievalFunction()
                    {
                        DateTime t1 = DateTime.Now;

                        Output.Write(string.Format("DataSourceYahoo: loading data for {0}...", Info[DataSourceParam.nickName]));

                        JObject jsonData = getPrices(startTime, endTime);

                        /*
                         * Yahoo JSON format, as of 07/02/2019
                         *
                         * [JSON]
                         *  chart
                         *      result
                         *          [0]
                         *              meta
                         *                  currency
                         *                  symbol
                         *                  ...
                         *              timestamp
                         *                  [0]: 511108200
                         *                  [1]: 511194600
                         *                  ...
                         *              indicators
                         *                  quote
                         *                      [0]
                         *                          low
                         *                              [0]: 0.08854
                         *                              [1]: 0.09722
                         *                              ...
                         *                          close
                         *                          volume
                         *                          open
                         *                          high
                         *                  adjclose
                         *                      [0]
                         *                          adjclose
                         *                              [0]: 0.06999
                         *                              [1]: 0.07249
                         */

                        List <Bar> bars = new List <Bar>();

                        var timestamps = (JArray)jsonData["chart"]["result"][0]["timestamp"];
                        var opens      = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["open"];
                        var highs      = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["high"];
                        var lows       = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["low"];
                        var closes     = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["close"];
                        var volumes    = (JArray)jsonData["chart"]["result"][0]["indicators"]["quote"][0]["volume"];
                        var adjcloses  = (JArray)jsonData["chart"]["result"][0]["indicators"]["adjclose"][0]["adjclose"];

                        var eT  = timestamps.GetEnumerator();
                        var eO  = opens.GetEnumerator();
                        var eH  = highs.GetEnumerator();
                        var eL  = lows.GetEnumerator();
                        var eC  = closes.GetEnumerator();
                        var eV  = volumes.GetEnumerator();
                        var eAC = adjcloses.GetEnumerator();

                        while (eT.MoveNext() && eO.MoveNext() && eH.MoveNext() &&
                               eL.MoveNext() && eC.MoveNext() && eV.MoveNext() && eAC.MoveNext())
                        {
                            DateTime t = fromUnixTime((long)eT.Current).Date
                                         + DateTime.Parse("16:00").TimeOfDay;

                            Bar bar = null;
                            try
                            {
                                // Yahoo taints the results by filling in null values
                                // we try to handle this gracefully in the catch block

                                double o  = (double)eO.Current;
                                double h  = (double)eH.Current;
                                double l  = (double)eL.Current;
                                double c  = (double)eC.Current;
                                long   v  = (long)eV.Current;
                                double ac = (double)eAC.Current;

                                // adjust prices according to the adjusted close.
                                // note the volume is adjusted the opposite way.
                                double ao = o * ac / c;
                                double ah = h * ac / c;
                                double al = l * ac / c;
                                long   av = (long)(v * c / ac);

                                bar = Bar.NewOHLC(
                                    Info[DataSourceParam.ticker],
                                    t,
                                    ao, ah, al, ac,
                                    av);
                            }
                            catch
                            {
                                if (bars.Count < 1)
                                {
                                    continue;
                                }

                                Bar prevBar = bars.Last();

                                bar = Bar.NewOHLC(
                                    Info[DataSourceParam.ticker],
                                    t,
                                    prevBar.Open, prevBar.High, prevBar.Low, prevBar.Close,
                                    prevBar.Volume);
                            }

                            if (t >= startTime && t <= endTime)
                            {
                                bars.Add(bar);
                            }
                        }

                        DateTime t2 = DateTime.Now;

                        Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds));

                        return(bars);
                    };

                    data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true);
                }

                catch (Exception e)
                {
                    throw new Exception(
                              string.Format("DataSourceYahoo: failed to load quotes for {0}, {1}",
                                            Info[DataSourceParam.nickName], e.Message));
                }

                if (data.Count == 0)
                {
                    throw new Exception(string.Format("DataSourceYahoo: no data for {0}", Info[DataSourceParam.nickName]));
                }

                CachedData = data;
                return(data);
            }
Пример #8
0
            /// <summary>
            /// Load data into memory.
            /// </summary>
            /// <param name="startTime">start of load range</param>
            /// <param name="endTime">end of load range</param>
            public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime)
            {
                List <Bar> data = new List <Bar>();

                try
                {
                    if (startTime < (DateTime)_firstTime)
                    {
                        startTime = (DateTime)_firstTime;
                    }

                    var cacheKey = new CacheId().AddParameters(
                        Info[DataSourceParam.nickName].GetHashCode(),
                        startTime.GetHashCode(),
                        endTime.GetHashCode());

                    List <Bar> retrievalFunction()
                    {
                        DateTime t1 = DateTime.Now;

                        Output.Write(string.Format("DataSourceStooq: loading data for {0}...", Info[DataSourceParam.nickName]));

                        List <Bar> bars = new List <Bar>();

                        List <StooqDataRow> stooqData = getPrices(startTime, endTime);
                        var e = stooqData.GetEnumerator();

                        while (e.MoveNext())
                        {
                            var bar = e.Current;

                            DateTime date = bar.Date;

                            if (date >= startTime && date <= endTime)
                            {
                                bars.Add(Bar.NewOHLC(
                                             Info[DataSourceParam.ticker],
                                             date,
                                             bar.Open, bar.High, bar.Low, bar.Close,
                                             bar.Volume));
                            }
                        }

                        DateTime t2 = DateTime.Now;

                        Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds));

                        return(bars);
                    };

                    data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true);;
                }

                catch (Exception e)
                {
                    throw new Exception(
                              string.Format("DataSourceStooq: failed to load quotes for {0}, {1}",
                                            Info[DataSourceParam.nickName], e.Message));
                }

                if (data.Count == 0)
                {
                    throw new Exception(string.Format("DataSourceStooq: no data for {0}", Info[DataSourceParam.nickName]));
                }

                CachedData = data;
                return(data);
            }
Пример #9
0
            /// <summary>
            /// Load data into memory.
            /// </summary>
            /// <param name="startTime">start of load range</param>
            /// <param name="endTime">end of load range</param>
            override public void LoadData(DateTime startTime, DateTime endTime)
            {
                var cacheKey = new CacheId(null, "", 0,
                                           Info[DataSourceParam.nickName].GetHashCode(),
                                           startTime.GetHashCode(),
                                           endTime.GetHashCode());

                List <Bar> retrievalFunction()
                {
                    DateTime t1 = DateTime.Now;
                    //Output.Write(string.Format("{0}: loading data for {1}...", GetType().Name, Info[DataSourceParam.nickName]));

                    // load data from specified data sources
                    // and save as list of bars, in reverse order
                    Dictionary <string, List <Bar> > dsBars = new Dictionary <string, List <Bar> >();

                    foreach (var nick in _symbols)
                    {
                        var d = DataSource.New(nick);

                        try
                        {
                            d.LoadData(startTime, endTime);
                            dsBars[nick] = d.Data.Reverse().ToList();
                        }
                        catch (Exception e)
                        {
                            Output.WriteLine("{0}: {1} failed to load {2}", this.GetType().Name, Info[DataSourceParam.nickName], nick);

                            // add an empty list, if need be
                            // this will be ignored further down during splicing
                            if (!dsBars.ContainsKey(nick))
                            {
                                dsBars[nick] = new List <Bar>();
                            }
                        }

                        //Output.WriteLine("{0}: {1} data range {2:MM/dd/yyyy}, {3:MM/dd/yyyy}", GetType().Name, nick, d.FirstTime, d.LastTime);
                    }

                    // create enumerators for all data sources
                    Dictionary <string, IEnumerator <Bar> > dsEnums = new Dictionary <string, IEnumerator <Bar> >();
                    Dictionary <string, bool> dsHasData             = new Dictionary <string, bool>();

                    foreach (var nick in _symbols)
                    {
                        dsEnums[nick]   = dsBars[nick].GetEnumerator();
                        dsHasData[nick] = dsEnums[nick].MoveNext();
                    }

                    // collect bars
                    List <Bar> bars = new List <Bar>();
                    Dictionary <string, double> dsScale = new Dictionary <string, double>();

                    _symbols.ForEach(n => dsScale[n] = 1.0);
                    while (dsHasData.Values.Aggregate((a, b) => a || b))
                    {
                        // find most-recent timestamp
                        DateTime ts = _symbols
                                      .Where(n => dsHasData[n])
                                      .Select(n => dsEnums[n].Current.Time)
                                      .Max(t => t);

                        Bar bar = null;

                        foreach (var nick in _symbols)
                        {
                            // no data: continue
                            if (!dsHasData[nick])
                            {
                                continue;
                            }

                            // older bar: continue
                            if (dsEnums[nick].Current.Time < ts)
                            {
                                continue;
                            }

                            if (bar == null)
                            {
                                // highest priority bar
                                Bar    rawBar = dsEnums[nick].Current;
                                double open   = rawBar.Open * dsScale[nick];
                                double high   = rawBar.High * dsScale[nick];
                                double low    = rawBar.Low * dsScale[nick];
                                double close  = rawBar.Close * dsScale[nick];
                                long   volume = 0;

                                bar = Bar.NewOHLC(Info[DataSourceParam.ticker], ts, open, high, low, close, volume);

                                bars.Add(bar);
                            }
                            else
                            {
                                // lower priority bars
                                Bar rawBar = dsEnums[nick].Current;

                                List <double> scales = new List <double>
                                {
                                    bar.Open / rawBar.Open,
                                    bar.High / rawBar.High,
                                    bar.Low / rawBar.Low,
                                    bar.Close / rawBar.Close,
                                };

                                dsScale[nick] = scales.Average();
                            }

                            dsHasData[nick] = dsEnums[nick].MoveNext();
                        }
                    }

                    // reverse order of bars
                    bars.Reverse();

                    DateTime t2 = DateTime.Now;

                    //Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds));

                    return(bars);
                };

                List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction);

                if (data.Count == 0)
                {
                    throw new Exception(string.Format("{0}: no data for {1}", GetType().Name, Info[DataSourceParam.nickName]));
                }

                Data = data;
            }
Пример #10
0
            /// <summary>
            /// Load data into memory.
            /// </summary>
            /// <param name="startTime">start of load range</param>
            /// <param name="endTime">end of load range</param>
            public override IEnumerable <Bar> LoadData(DateTime startTime, DateTime endTime)
            {
                var cacheKey = new CacheId(null, "", 0,
                                           Info[DataSourceParam.nickName].GetHashCode(),
                                           startTime.GetHashCode(),
                                           endTime.GetHashCode());

                List <Bar> retrievalFunction()
                {
                    DateTime t1 = DateTime.Now;
                    //Output.Write(string.Format("{0}: loading data for {1}...", GetType().Name, Info[DataSourceParam.nickName]));

                    // load data from specified data sources
                    // and save as list of bars, in reverse order
                    Dictionary <string, List <Bar> > dsBars = new Dictionary <string, List <Bar> >();

                    foreach (var nick in _symbols)
                    {
                        var d = DataSource.New(nick);

                        try
                        {
                            var data = d.LoadData(startTime, endTime);
                            dsBars[nick] = data.Reverse().ToList();
                        }
                        catch (Exception /*e*/)
                        {
                            Output.WriteLine("{0}: {1} failed to load {2}", this.GetType().Name, Info[DataSourceParam.nickName], nick);

                            // add an empty list, if need be
                            // this will be ignored further down during splicing
                            if (!dsBars.ContainsKey(nick))
                            {
                                dsBars[nick] = new List <Bar>();
                            }
                        }

                        //Output.WriteLine("{0}: {1} data range {2:MM/dd/yyyy}, {3:MM/dd/yyyy}", GetType().Name, nick, d.FirstTime, d.LastTime);
                    }

                    // create enumerators for all data sources
                    Dictionary <string, IEnumerator <Bar> > dsEnums = new Dictionary <string, IEnumerator <Bar> >();
                    Dictionary <string, bool> dsHasData             = new Dictionary <string, bool>();

                    foreach (var nick in _symbols)
                    {
                        dsEnums[nick]   = dsBars[nick].GetEnumerator();
                        dsHasData[nick] = dsEnums[nick].MoveNext();
                    }

                    // skip bars from all proxy datasources, so that no proxy
                    // has bars after the primary datasource
                    // example: extending GLD w/ XAUUSD. because XAUUSD has
                    // a different trading calendar, XAUUSD might have bars
                    // after GLD, leading to faulty results
                    var lastPrimary = dsEnums[_symbols.First()].Current.Time;

                    foreach (var nick in _symbols)
                    {
                        while (dsHasData[nick] && dsEnums[nick].Current.Time > lastPrimary)
                        {
                            dsHasData[nick] = dsEnums[nick].MoveNext();
                        }
                    }

                    // collect bars
                    List <Bar> bars = new List <Bar>();

                    Dictionary <string, double?> dsScale = new Dictionary <string, double?>();

                    _symbols.ForEach(n => dsScale[n] = null);
                    dsScale[_symbols.First()]        = 1.0;

                    while (dsHasData.Values.Aggregate((a, b) => a || b))
                    {
                        // find most-recent timestamp
                        DateTime ts = _symbols
                                      .Where(n => dsHasData[n])
                                      .Select(n => dsEnums[n].Current.Time)
                                      .Max(t => t);

                        Bar bar = null;

                        foreach (var nick in _symbols)
                        {
                            // no data: continue
                            if (!dsHasData[nick])
                            {
                                continue;
                            }

                            // older bar: continue
                            if (dsEnums[nick].Current.Time < ts)
                            {
                                continue;
                            }

                            if (bar == null)
                            {
                                // highest priority bar
                                Bar rawBar = dsEnums[nick].Current;

                                // we might get here, with dsScale not set yet.
                                // this is the best we can do to fix things
                                if (dsScale[nick] == null)
                                {
                                    dsScale[nick] = bars.Last().Open / rawBar.Close;
                                }

                                double open   = rawBar.Open * (double)dsScale[nick];
                                double high   = rawBar.High * (double)dsScale[nick];
                                double low    = rawBar.Low * (double)dsScale[nick];
                                double close  = rawBar.Close * (double)dsScale[nick];
                                long   volume = 0;

                                bar = Bar.NewOHLC(Info[DataSourceParam.ticker], ts, open, high, low, close, volume);

                                bars.Add(bar);
                            }
                            else
                            {
                                // lower priority bars
                                Bar rawBar = dsEnums[nick].Current;

                                List <double> scales = new List <double>
                                {
                                    bar.Open / rawBar.Open,
                                    bar.High / rawBar.High,
                                    bar.Low / rawBar.Low,
                                    bar.Close / rawBar.Close,
                                };

                                dsScale[nick] = scales.Average();
                            }

                            dsHasData[nick] = dsEnums[nick].MoveNext();
                        }
                    }

                    // reverse order of bars
                    bars.Reverse();

                    DateTime t2 = DateTime.Now;

                    //Output.WriteLine(string.Format(" finished after {0:F1} seconds", (t2 - t1).TotalSeconds));

                    return(bars);
                };

                List <Bar> data = Cache <List <Bar> > .GetData(cacheKey, retrievalFunction, true);

                if (data.Count == 0)
                {
                    throw new Exception(string.Format("{0}: no data for {1}", GetType().Name, Info[DataSourceParam.nickName]));
                }

                CachedData = data;
                return(data);
            }