public async Task UpdateList(IList<SeriesConfig> list_, ICarbonClient client_)
{
// remove current items
var currentItems = list_.Where(x => x.Source != null && x.Source.ToLower().Equals(SourceName)).ToArray();
foreach (var item in currentItems)
list_.Remove(item);
// search for frames that match the regex
Logger.Info(string.Format("Searching for dataframes with regex=[{0}]", MonikerSearchRegex),
typeof (DataframeSearcherBase));
var monikers = await client_.GetSavedIdentifiersAsync("AdHocData", MonikerSearchRegex, true);
foreach (var moniker in monikers)
{
Logger.Info(string.Format("Considering [{0}]...", moniker), typeof (DataframeSearcherBase));
try
{
var frame = await client_.GetFrameAsync<DateTime>(moniker);
foreach (var columnName_ in frame.ColumnKeys)
{
try
{
// attempt to try and get a timeseries
var series = frame.ExtractTimeSeries<double>(columnName_);
if (series == null) continue;
var config = getConfigLine(moniker, columnName_);
if (config == null) continue;
list_.Add(config);
}
catch
{
//Logger.Info(string.Format("Not going to add column [{0}] as can't get doubles from it", columnName_),
// typeof (DataframeSearcherBase));
}
}
}
catch (Exception ex_)
{
Logger.Error(string.Format("Error getting and reflecting on frame {0}", moniker),
typeof (DataframeSearcherBase), ex_);
}
}
}
//-----------------------------------------------------------------------------------
// function to return carry adjusted smooth series, together with Bond ISIN, series price, convfac
public static Frame<DateTime, String> GenerateCarryAdjSeries(
string ticker,
DateTime start,
DateTime end,
string countryCode,
ICarbonClient client = null
)
//-----------------------------------------------------------------------------------
{
//variables used for futureSeriesBuilder
var eRollMethod = TimeSeriesUtilities.GetRollMethods("FirstNoticeDate");
var eInterpolationTypes = TimeSeriesUtilities.GetInterpolationType("None");
var eSmoothingTypes = TimeSeriesUtilities.GetSmoothingTypes("None");
#region Configs
SpreadTimeSeriesConfigs config = new SpreadTimeSeriesConfigs(countryCode, false);
string ccy = config.ccy;
string calendarCode = config.calendarcode;
string futuresStaticMoniker = config.futuresStaticMoniker;
string carbonEnv = config.carbonEnv;
BondAnalytics.Country country = config.country;
BondAnalytics.DayCountType dct = config.dct;
List<string> contractSortOrder = config.contractSortOrder;
List<DateTime> holidays = TimeSeriesUtilities.GetHolidays(calendarCode, client);
#endregion
// in rolling series US is following calendar of USGS rather than USNY
var dateRange = TimeSeriesUtilities.GetAllDatesInDateime(start, end, countryCode.ToUpper() == "US" ? "USGS" : calendarCode, client, "1b");
//Get Rolling Series
HashSet<string> futuresList = new HashSet<string> { "TU", "FV", "TY", "UXY", "US", "WN", "DU", "OE", "RX", "UB", "IK", "BTS", "OAT" };
bool tickerIsFuture = futuresList.Contains(ticker);
bool isCT = ticker.ToUpper().Contains("CT");
var rollType = isCT ? "ct" : "ctd";
string moniker = string.Format("symapp.roll.{0}.{1}.{2}", rollType, ticker.Market(), ticker).ToLower();
string fmoniker;
// Carbon frame //
var frame = client.GetFrameAsync<DateTime>(moniker).Result;
if (!frame.ColumnKeys.Contains("price")) return null;// need better null value checker here
Series<DateTime, string> bondISIN = null;
Series<DateTime, double> bondPrice = null;
Dictionary<DateTime, string> futureTicker = new Dictionary<DateTime, string>();
Dictionary<DateTime, double> convFac = new Dictionary<DateTime, double>();
Dictionary<DateTime, double> futurePrice = new Dictionary<DateTime, double>();
Dictionary<DateTime, double> smoothSeries = new Dictionary<DateTime, double>();
Series<DateTime, double> futureConvertedPrice = null;
//Repo series and carry series
Dictionary<DateTime, double> repoSeries = new Dictionary<DateTime, double>();
Dictionary<DateTime, double> carrySeries = new Dictionary<DateTime, double>();
//DV01 series (Future DV01 = CTD DV01/ConvFac; CT DV01 = Bond DV01)
Dictionary<DateTime, double> DV01Series = new Dictionary<DateTime, double>();
//Variables needed for futures
Frame<string, string> combinedStatic = null;
int RowNum = 0;
try
{
bondISIN = frame.GetColumn<string>("ticker").Between(start, end);//Need check on sorted by date
bondPrice = frame.GetColumn<double>("price").Between(start, end);
}
catch (Exception)
{
//Error handling
}
//Future tickers and Future Price as well as conversion factors
if (tickerIsFuture)
{
fmoniker = string.Format("symapp.roll.{0}.{1}.{2}", "f" + rollType, ticker.Market(), ticker).ToLower();
frame = client.GetFrameAsync<DateTime>(fmoniker).Result;
if (!frame.ColumnIndex.KeySequence.Contains("price")) return null;//need better null value checker here
try
{
futureConvertedPrice = frame.GetColumn<double>("price").Between(start, end).Sort();
}
catch (Exception)
{
//Error handling
}
//Get the future static table
//Futures
List<string> tickers = new List<string>();
tickers.Add(ticker.ToUpper() + ".Front");
FuturesSeriesBuilder
seriesBuilder = new FuturesSeriesBuilder(tickers, dateRange, holidays, futuresStaticMoniker, client, carbonEnv, rollMthd: eRollMethod, interpType: eInterpolationTypes, contractSortOrder: config.contractSortOrder, smoothingType: eSmoothingTypes);
var futStatic = seriesBuilder.FuturesStatic;
//Bond Static
Dictionary<String, TimeSeriesBondStatic> bndStatics = TimeSeriesUtilities.GetBondStaticsFromFuturesBuilder(seriesBuilder, config, client);
combinedStatic = SpreadSeriesBuilder.combineFuturesAndBondStatic(bndStatics, futStatic);
RowNum = combinedStatic.RowCount;
}
double futureConvertedPx = 0.0;
double lastRepo = 0.0;
double fwdPrice = 0.0;
object dailyRepo = null;
bool bHaveEverthing = true;
DateTime prevD8 = DateTime.MinValue;
//loop through each date to find the corresponding future ticker and conversion factors
// and add repo and carry for bond series
// can we get the sorted key list
#region buildSeries
foreach (DateTime d8 in bondISIN.Keys.OfType<DateTime>())
{
// DV01Series
string currBondISIN = bondISIN.TryGet(d8).Value;
Double bondDV01 = 0.0;
DateTime settleDate = d8;
var currBondStatic = AnalyticsUtilities.GetBondStaticData(client, currBondISIN);
if (currBondStatic == null)
{
bHaveEverthing = false;
DV01Series.Add(d8, 0.0);//Need better handling on empty bond static
}
else
{
if (config.bondSettleLag > 0)
{
settleDate = BondAnalytics.NextBusDay(settleDate.AddDays(1), holidays);
}
if (config.bondSettleLag > 1)
{
settleDate = BondAnalytics.NextBusDay(settleDate.AddDays(1), holidays);
}
bondDV01 = BondAnalytics.SolveYield(country, settleDate, bondPrice.TryGet(d8).Value, currBondStatic.EffectiveDate, currBondStatic.FirstCouponDate,
currBondStatic.MaturityDate, currBondStatic.Coupon, currBondStatic.CpnFreq)[1];
DV01Series.Add(d8, bondDV01);
}
if (tickerIsFuture)
{
//Need gaurantee that the table is sorted
int i = 0;
while (combinedStatic.GetColumn<DateTime>("fut_notice_first").GetAt(i) <= d8 && i < RowNum)
{
i++;
}
if (i == RowNum)
{
futureTicker.Add(d8, string.Empty);
convFac.Add(d8, double.NaN);
futurePrice.Add(d8, double.NaN);
}
else
{
futureTicker.Add(d8, combinedStatic.GetColumn<string>("ticker").GetAt(i));
convFac.Add(d8, combinedStatic.GetColumn<double>("fut_cnvs_factor").GetAt(i));
futureConvertedPx = futureConvertedPrice.TryGet(d8).Value;
futurePrice.Add(d8, futureConvertedPx / combinedStatic.GetColumn<double>("fut_cnvs_factor").GetAt(i));
//update CTD DV01 to be future DV01
DV01Series[d8] = DV01Series[d8] / combinedStatic.GetColumn<double>("fut_cnvs_factor").GetAt(i);
}
//Add SmoothSeries
if (prevD8 == DateTime.MinValue)
{
smoothSeries.Add(d8, futurePrice[d8]);
}
else
{
//check if there is ticker change
if (futureTicker[d8] == futureTicker[prevD8])
{
smoothSeries.Add(d8, smoothSeries[prevD8] + futurePrice[d8] - futurePrice[prevD8]);
}
else
{
//find the old future price on current date
string oldFutureTicker = futureTicker[prevD8];
string full_ticker = oldFutureTicker + " COMDTY";
var oldFuturePrice = client.GetCbnTimeSeriesDataCell(full_ticker, "eod", d8, "settle");
if (oldFuturePrice == null)
{
full_ticker = AnalyticsUtilities.GetAlternativeFuturesName(full_ticker);
oldFuturePrice = client.GetCbnTimeSeriesDataCell(full_ticker, "eod", d8, "settle");
}
//Otherwise report error: assume no price change a good way?
if (oldFuturePrice == null) oldFuturePrice = futurePrice[prevD8];
smoothSeries.Add(d8, smoothSeries[prevD8] + (oldFuturePrice as double? ?? double.NaN) - futurePrice[prevD8]);// potential problem if current res is NaN, all following value would be NaN
}
}
}
else
{
//repo and carry series for bond
try
{
dailyRepo = client.GetCbnTimeSeriesDataCell(currBondISIN, "eod-mlp", d8, "wtdRate");
// repoSeries.Add(d8, dailyRepo as double?);
}
catch
{
//Missing Data
//need to convert it
}
lastRepo = (dailyRepo as double?) ?? lastRepo;
repoSeries.Add(d8, lastRepo);
//need bond static info
if (currBondStatic == null)
{
bHaveEverthing = false;
carrySeries.Add(d8, 0.0);//Need better handling for missing bond static
smoothSeries.Add(d8, 0.0);//Need better handling for missing bond static
}
else
{
//calculate one-day fwdPrice
//Current next business day function doesn't handle number of days so get settleDate the hard way.
//one day carry
DateTime fwdDate = settleDate;
fwdDate = BondAnalytics.NextBusDay(fwdDate.AddDays(1), holidays);
if (fwdDate <= settleDate)
{
//error handling here : fwdDate must be bigger than settleDate
}
fwdPrice = BondAnalytics.CalcBondFwd(country, settleDate, bondPrice.TryGet(d8).Value, currBondStatic.EffectiveDate, currBondStatic.FirstCouponDate, currBondStatic.MaturityDate,
currBondStatic.Coupon, currBondStatic.CpnFreq, fwdDate, lastRepo)[0];
carrySeries.Add(d8, bondPrice.TryGet(d8).Value - fwdPrice);
//Smooth series for bond
if (prevD8 == DateTime.MinValue)
{
smoothSeries.Add(d8, bondPrice.TryGet(d8).Value);//assume no carry added on day 1
}
else
{
if (bondISIN.TryGet(prevD8).Value == bondISIN.TryGet(d8).Value)
{
//no change in series
smoothSeries.Add(d8, smoothSeries[prevD8] + bondPrice.TryGet(d8).Value - bondPrice.TryGet(prevD8).Value);
}
else
{
string oldBondISIN = bondISIN.TryGet(prevD8).Value;
var oldBondPrice = client.GetCbnTimeSeriesDataCell(oldBondISIN, "eod-mlp", d8, "close");
if (oldBondPrice == null) oldBondPrice = bondPrice.TryGet(prevD8).Value;// May need better way to handle this error
smoothSeries.Add(d8, smoothSeries[prevD8] + (oldBondPrice as double? ?? double.NaN) - bondPrice.TryGet(prevD8).Value);
}
}
}
}
//update prevD8
prevD8 = d8;
}
#endregion
//Sort the dictionary and package into frame for output
if (tickerIsFuture)
{
//building using dictionary
Dictionary<string, Series<DateTime, string>> resHolder = new Dictionary<string, Series<DateTime, string>>();
resHolder["futureTicker"] = futureTicker.ToSeries();
var res = Frame.FromColumns(resHolder);
res.AddColumn("futruePrice", futurePrice.ToSeries());
res.AddColumn("CTD_ISIN", bondISIN);
res.AddColumn("ConvFac", convFac.ToSeries());
res.AddColumn("DV01", DV01Series.ToSeries());
res.AddColumn("smoothSeries", smoothSeries.ToSeries());
res.SortRowsByKey();
return res;
}
else
{
//if bond
Dictionary<string, Series<DateTime, string>> resHolder = new Dictionary<string, Series<DateTime, string>>();
resHolder["BondISIN"] = bondISIN;
var res = Frame.FromColumns(resHolder);
res.AddColumn("BondPrice", bondPrice);
res.AddColumn("repo", repoSeries.ToSeries());
res.AddColumn("carry", carrySeries.ToSeries());
res.AddColumn("DV01", DV01Series.ToSeries());
res.AddColumn("smoothSeries", smoothSeries.ToSeries());
res.SortRowsByKey();
return res;
}
}
//----------------------------------------------------------------------------------------
public static Frame<DateTime, String> GenerateRollingBondForwardSeries(
List<string> tickers,
DateTime start,
DateTime end,
string tenor,
string countryCode,
string measure,
string holidayCode,
bool bOIS = true,
int optExDivDays = 0,
bool bSpreadOrRepo = true,
List<double> reposOrSpreads = null,
ICarbonClient client = null)
//----------------------------------------------------------------------------------------
{
// Checks
if (client == null)
{
throw new ArgumentException("No Carbon Client!");
}
#region cache
string all_tickers = ";";
foreach (var s in tickers) all_tickers += s;
string allRepoOis = "";
if (reposOrSpreads != null)
foreach (var s in reposOrSpreads) allRepoOis+= s;
string cacheKey = "GenerateRollingBondForwardSeries" + tickers.ToString() + start.ToString() + end.ToString() +
tenor + countryCode + measure + holidayCode + bOIS.ToString()
+ optExDivDays.ToString() + bSpreadOrRepo.ToString() + allRepoOis;
if (TimeSeriesCache.ContainsKey(cacheKey))
{
return TimeSeriesCache[cacheKey];
}
#endregion
#region Checks
if (start >end) throw new ArgumentException("#Error: start date cannot be after end date");
#endregion
var settings = new List<Tuple<string/*moniker*/, string/*tickercol*/, string/*pricecol*/>>();
// Get Configs
foreach (var ticker in tickers)
{
if (ticker.Contains('.')) // it's a CTD series!
{
settings.Add(MonikerConfigs.getCTDMoniker(countryCode,ticker,client));
}
else if (ticker.ToLower().Contains("ct"))
{
settings.Add(MonikerConfigs.getCTMoniker(countryCode, ticker, client));
}
else
{
throw new ArgumentException("#Error: Invalid ticker found - " + ticker);
}
}
#region DATA
bool bHaveEverything = true;
var configs = new SpreadTimeSeriesConfigs(countryCode, !bOIS);
var spreadConventions = new SpreadConventions(configs);
var dateRange = TimeSeriesUtilities.GetAllDatesInDateime(start, end, configs.calendarcode, client, "1b"); // Force daily sampling
var fwdDates = dateRange.Select(dte => dte.AddTenor(Tenor.FromString(tenor))).ToList();
var holidays = TimeSeriesUtilities.GetHolidays(holidayCode,client);
// Parcel input repo rates or spreads
Series<DateTime, double> inputSrs = null;
if (reposOrSpreads != null )
if (reposOrSpreads.Count != dateRange.Count && reposOrSpreads.Count != 1)
{
throw new ArgumentException("#Error: number of spreads or rates do not much the days in range");
}
else
{
var sb = new SeriesBuilder<DateTime, double>();
double scale_factor = bSpreadOrRepo ? 10000 : 1;
for (int i = 0; i < dateRange.Count; ++i)
{
if (reposOrSpreads.Count == 1)
{
sb.Add(dateRange[i], reposOrSpreads[0]/scale_factor);
}
else
{
sb.Add(dateRange[i], reposOrSpreads[i]/scale_factor);
}
}
inputSrs = sb.Series;
}
// Swap Curves
var curves = TimeSeriesUtilities.GetCurvesFromCarbon(dateRange, configs.ccy, client);
if (curves == null) bHaveEverything = false;
// Prices and tickers from ad-hoc persistence
var frames = new Dictionary<string, Frame<DateTime, string>>();
for (int i = 0; i < tickers.Count; ++i)
{
var setting = settings[i];
var ticker = tickers[i];
var moniker = setting.Item1;
var tickercol = setting.Item2;
var pricecol = setting.Item3;
var tmp_df = client.GetFrameAsync<DateTime>(moniker, "AdHocData").Result;
if (tmp_df == null)
{
bHaveEverything = false;
}
else
{
// Extract price and ticker cols only
var colsOfInterest = new Dictionary<string, Series<DateTime,object>>();
// Filter data to contain date range
tmp_df = tmp_df.Where(kvp => (kvp.Key >= start) && (kvp.Key <= end));
colsOfInterest["price"] = tmp_df.GetColumn<object>(pricecol);
colsOfInterest["isin"] = tmp_df.GetColumn<object>(tickercol);
frames[ticker] = Frame.FromColumns(colsOfInterest);
}
}
//Bond static
var bondIsinSet = new HashSet<string>();
foreach(var kvp in frames)
{
var ticker = kvp.Key;
var data = kvp.Value;
var isins = data.GetColumn<string>("isin").Values;
//isins.Select(isin => bondIsinSet.Add(isin));
foreach (string isin in isins)
{
bondIsinSet.Add(isin);
}
}
var bondStatics = client.BulkGetBondStaticData( bondIsinSet.ToList());
if (curves == null || bondStatics == null ) return null;
#endregion
#region RepoRegion
Series<DateTime, double> repoSrs = null;
if (reposOrSpreads == null)
{
// Load default repo-OIS Spread
double[,] repoRates =
client.GetRepoRateFromSavedRepoOisSpread(BondAnalytics.CountryMappings[countryCode], dateRange,
fwdDates, holidays);
// Break out if no data
if (repoRates == null) return null;
var repoSrsBlder = new SeriesBuilder<DateTime, double>();
for (int i = 0; i < dateRange.Count; ++i)
{
repoSrsBlder.Add(dateRange[i], repoRates[i, 1]);
}
repoSrs = repoSrsBlder.Series;
}
else
{
// User-defined repo-OIS Spread or repo rates
if (bSpreadOrRepo)
{
//Inputs are spreads
double[,] repoRates =
client.CalcRepoRatesFromOisSpread(inputSrs, BondAnalytics.CountryMappings[countryCode], dateRange, fwdDates, holidays);
// Break out if no data
if (repoRates == null) return null;
var repoSrsBlder = new SeriesBuilder<DateTime, double>();
for (int i = 0; i < dateRange.Count; ++i)
{
repoSrsBlder.Add(dateRange[i], repoRates[i, 1]);
}
repoSrs = repoSrsBlder.Series;
}
else
{
//Inputs are repo rates
repoSrs = inputSrs;
}
}
#endregion
// Calculation loop
var forecastCurves = curves.Item1;
var discountCurves = curves.Item2;
// Overwrite forecast curve if we want OIS
if (bOIS)
{
forecastCurves = discountCurves;
}
var outputs = new Dictionary<string, Series<DateTime, double>> ();
foreach (string ticker in tickers)
{
var sb = new SeriesBuilder<DateTime, double>();
var df = frames[ticker];
var idx = df.RowKeys;
var isins = df.GetColumn<string>("isin");
var prices = df.GetColumn<double>("price");
foreach (var dte in idx)
{
var tryisin = isins.TryGet(dte);
var tryprice = prices.TryGet(dte);
var tryrepo = repoSrs.TryGet(dte);
if (!tryprice.HasValue || !tryisin.HasValue || !tryrepo.HasValue)
continue;
var isin = tryisin.Value;
var price = tryprice.Value;
var repo = tryrepo.Value;
var fwddate = dte.AddTenor(Tenor.FromString(tenor));
var bondstatic = bondStatics[isin];
double value = double.NaN;
if (measure == "Repo")
{
value = repo;
}
else
{
var fwdprice = BondAnalytics.CalcBondFwd(configs.country, dte, price, bondstatic.EffectiveDate,
bondstatic.FirstCouponDate, bondstatic.MaturityDate, bondstatic.Coupon, bondstatic.CpnFreq,
fwddate, repo, optExDivDays)[0];
if (measure == "Price")
{
value = fwdprice;
}
else
{
value = MeasureCalculator.calcFwdBondMeasure(measure, bondstatic, fwdprice, dte, fwddate,
spreadConventions, forecastCurves, discountCurves, holidays);
}
}
sb.Add(dte,value);
}
outputs[ticker] = sb.Series;
}
var consolidatedMeasures = Frame.FromColumns(outputs);
// Cache results
TimeSeriesCache[cacheKey] = consolidatedMeasures;
return consolidatedMeasures;
}