protected bool ShouldInclude(FIBond bond, DateTime? valueDate, string criteriaType, string criteriaValue)
        {
            bool include = true;

            var maturity = bond.FI.Maturity;

            if (include && valueDate.HasValue)
            {
                include =
                    (bond.IssueDate.HasValue && bond.IssueDate.Value <= valueDate.Value) &&
                    (maturity.HasValue && maturity.Value > valueDate.Value);
            }

            // do we have criteria?
            if (include && criteriaType == "term")
            {
                int termNum = 0;
                if (Int32.TryParse(criteriaValue, out termNum))
                {
                    if (valueDate.HasValue)
                    {
                        include = SmoothBondCurveCalculator.ShouldBeIncluded(bond, valueDate.Value, valueDate.Value, termNum);
                    }
                    else if (maturity.HasValue)
                    {
                        var cutoffDate = maturity.Value.AddTenor(Tenor.BusinessDays(-5));
                        include = SmoothBondCurveCalculator.ShouldBeIncluded(bond, cutoffDate, cutoffDate, termNum);
                    }
                }
            }

            return include;
        }
        public static FI GetOrCreateMLPBond(MLPBondInstrument instrument, SymmetryEntities dc, ThrowBehavior behavior = ThrowBehavior.Throw)
        {
            if(string.IsNullOrWhiteSpace(instrument.ISIN))
            {
                return null;
            }
            FIChangeTracking.DisableChangeTracking(ref dc);
            FI fi = null;
            try
            {
                // create instrument
                fi = FIHelpers.GetFIBySymmetryCode(instrument.ISIN, dc, behavior);
                if (fi == null)
                {
                    fi = new FI
                    {
                        InstrumentType = InstrumentType.Bond,
                        Currency = null,
                        DisplayName = instrument.MlpTicker,
                        Maturity = DateTime.FromOADate(instrument.Maturity),
                        SymmetryCode = instrument.ISIN
                    };
                    dc.FIs.Add(fi);
                }
                else
                {
                    var newCcy = GetBondCurrencyForBbTicker(instrument.BbTicker);
                    if (!string.IsNullOrEmpty(newCcy) && fi.Currency != newCcy)
                    {
                        fi.Currency = newCcy;
                        FIChangeTracking.SetModified(fi, dc);
                    }
                }
                dc.SaveChanges();

                // set identifiers
                SetOrUpdateIdentifier(fi, IdentifierType.ISIN, instrument.ISIN, dc);
                SetOrUpdateIdentifier(fi, IdentifierType.CUSIP, instrument.CUSIP, dc);
                SetOrUpdateIdentifier(fi, IdentifierType.MLPCode, instrument.MlpTicker, dc);
                SetOrUpdateIdentifier(fi, IdentifierType.RIC, instrument.RIC, dc);
                SetOrUpdateIdentifier(fi, IdentifierType.Bloomberg, instrument.ISIN + " Govt", dc);
                dc.SaveChanges();

                // create bond
                var fiBond = dc.FIBonds.SingleOrDefault(b => b.FIID == fi.FIID);
                if (fiBond == null)
                {
                    fiBond = new FIBond
                    {
                        FIID = fi.FIID,
                        Market = instrument.GetMarket(),
                        Coupon = instrument.Coupon.HasValue ? instrument.Coupon.Value : 0,
                        EffectiveDate = DateTime.FromOADate(instrument.EffectiveDate),
                        FirstCouponDate = DateTime.FromOADate(instrument.FirstCouponDate),
                        IssueDate = DateTime.FromOADate(instrument.IssueDate),
                        Series = instrument.Series,
                        Term = instrument.Term
                    };
                    dc.FIBonds.Add(fiBond);
                }
                else
                {
                    var coupon = instrument.Coupon.HasValue ? instrument.Coupon.Value : 0;
                    var effectiveDate = DateTime.FromOADate(instrument.EffectiveDate);
                    var firstCouponDate = DateTime.FromOADate(instrument.FirstCouponDate);
                    var issueDate = DateTime.FromOADate(instrument.IssueDate);

                    if (fiBond.Market != instrument.GetMarket() ||
                        fiBond.Coupon != coupon ||
                        fiBond.EffectiveDate != effectiveDate ||
                        fiBond.FirstCouponDate != firstCouponDate ||
                        fiBond.IssueDate != issueDate ||
                        fiBond.Series != instrument.Series ||
                        fiBond.Term != instrument.Term)
                    {
                        fiBond.Market = instrument.GetMarket();
                        fiBond.Coupon = coupon;
                        fiBond.EffectiveDate = effectiveDate;
                        fiBond.FirstCouponDate = firstCouponDate;
                        fiBond.IssueDate = issueDate;
                        fiBond.Series = instrument.Series;
                        fiBond.Term = instrument.Term;
                        FIChangeTracking.SetModified(fiBond, dc);
                    }                   
                }
                dc.SaveChanges();
            }
            catch (Exception ex)
            {
                Exceptions.Rethrow("GetOrCreateMLPBond", behavior, ex);
            }
            finally
            {
                FIChangeTracking.EnableChangeTracking(ref dc);
            }
            return fi;
        }
    protected override void processResponsesFromDataScope(Data.SymmetryEntities dc_, Data.MarketSnap saveToThisSnap_, string marketSnapCode_)
    {
      OutputToCSV(m_resultsFromDataScope);

      if (m_resultsFromDataScope == null) return;

      var config = BondDiscoveryConfig.GetInstance();

      foreach (var response in m_resultsFromDataScope)
      {
        foreach (var row in response.Rows)
        {
          try
          {
            var isin = GetValueFromObjectArray<string>((int)FieldsToReturn.InstrumentID, row, ThrowBehavior.Throw);
            var ric = GetValueFromObjectArray<string>((int)FieldsToReturn.TradewebRIC, row, ThrowBehavior.DontThrow);
            var maturity = GetValueFromObjectArray<DateTime>((int)FieldsToReturn.Maturity, row, ThrowBehavior.Throw);
            var description = GetValueFromObjectArray<string>((int)FieldsToReturn.Description, row, ThrowBehavior.Throw);
            var currency = GetValueFromObjectArray<string>((int)FieldsToReturn.Currency, row, ThrowBehavior.DontThrow);
            var coupon = GetValueFromObjectArray<double>((int)FieldsToReturn.Coupon, row, ThrowBehavior.Throw);
            var auctionDate = GetValueFromObjectArray<DateTime?>((int)FieldsToReturn.AuctionDate, row, ThrowBehavior.DontThrow);
            var effectiveDate = GetValueFromObjectArray<DateTime?>((int)FieldsToReturn.EffectiveDate, row, ThrowBehavior.DontThrow);
            var firstCouponDate = GetValueFromObjectArray<DateTime?>((int)FieldsToReturn.FirstCouponDate, row, ThrowBehavior.DontThrow);
            var issueDate = GetValueFromObjectArray<DateTime?>((int)FieldsToReturn.IssueDate, row, ThrowBehavior.DontThrow);
            var series = GetValueFromObjectArray<string>((int)FieldsToReturn.Series, row, ThrowBehavior.DontThrow);
            var term = GetValueFromObjectArray<int?>((int)FieldsToReturn.Term, row, ThrowBehavior.DontThrow);
            var assetSubType = GetValueFromObjectArray<string>((int)FieldsToReturn.AssetSubType,row,ThrowBehavior.DontThrow);
            var couponClass = GetValueFromObjectArray<string>((int) FieldsToReturn.CouponClassCode, row, ThrowBehavior.DontThrow);
            var countryCode = GetValueFromObjectArray<string>((int) FieldsToReturn.CountryCode, row, ThrowBehavior.DontThrow);
            var inflationFlag = GetValueFromObjectArray<string>((int)FieldsToReturn.InflationProtectedFlag,row,ThrowBehavior.DontThrow);
            var floaterFlag = GetValueFromObjectArray<string>((int)FieldsToReturn.FloaterFlag,row,ThrowBehavior.DontThrow);

            FIChangeTracking.DisableChangeTracking(ref dc_);

            try
            {
              SLog.log.DebugFormat("Adding new bond to database with ISIN={0}", isin);

              // create instrument
              var fi = new FI
              {
                InstrumentType = InstrumentType.Bond,
                Currency = currency,
                DisplayName = description,
                Maturity = maturity,
                SymmetryCode = isin
              };
              dc_.FIs.Add(fi);

              dc_.SaveChanges();

              FIBondHelpers.SetOrUpdateIdentifier(fi, IdentifierType.ISIN, isin, dc_);
              FIBondHelpers.SetOrUpdateIdentifier(fi, IdentifierType.RIC, ric, dc_);
              FIBondHelpers.SetOrUpdateIdentifier(fi, IdentifierType.Bloomberg, string.Format("{0} Govt", isin), dc_);
              dc_.SaveChanges();

              // create bond
              var fiBond = new FIBond
              {
                FIID = fi.FIID,
                Coupon = Convert.ToDecimal(coupon),
                EffectiveDate = effectiveDate,
                FirstCouponDate = firstCouponDate,
                AuctionDate = auctionDate,
                IssueDate = issueDate,
                Series = series,
                Term = term,
                Market = config == null ? assetSubType : config.GetMarket(countryCode, assetSubType),
                IsFloater=floaterFlag!=null && String.Compare("Y", floaterFlag, StringComparison.OrdinalIgnoreCase)==0 ? 1 : 0,
                IsInflationLinked = inflationFlag!=null && String.Compare("Y", inflationFlag, StringComparison.OrdinalIgnoreCase)==0 ? 1 : 0,
              };
              dc_.FIBonds.Add(fiBond);
              dc_.SaveChanges();
            }
            finally
            {
              FIChangeTracking.EnableChangeTracking(ref dc_);
            }

          }
          catch
          {
            continue; // a required value wasn't found
          }



        } // end inner for
      } // end first for

    }