public Frame <int, string> CreateTermDataFrame(Frame <int, string> frame, string termsColumn, string classifierColumnName)
{
var stopWords = _fileManager.ReadFromFile(_appSettings.StopWordsFilePath)
.Distinct()
.Select(x => x.ToLower())
.ToList();
var termsByRows = frame.GetColumn <string>(termsColumn).GetAllValues().Select((row, index) =>
{
var seriesBuilder = new SeriesBuilder <string, int>();
var rowWords = Regex.Matches(row.Value, "[a-zA-Z]+('(s|d|t|ve|m))?")
.Cast <Match>()
.Where(word => !stopWords.Contains(word.Value.ToLower()))
.Select(word => word.Value.ToLower())
.Distinct();
foreach (var word in rowWords)
{
seriesBuilder.Add(word, 1);
}
return(KeyValue.Create(index, seriesBuilder.Series));
});
var termDataFrame = Frame.FromRows(termsByRows).FillMissing(0);
termDataFrame.AddColumn(classifierColumnName, frame.GetColumn <int>(classifierColumnName));
termDataFrame.SaveCsv(_appSettings.TermsVectorFilePath);
return(termDataFrame);
}
private static Series <DateTime, MarketDataElement> addMarketDataElementWithTimeOnlyToSeries(DateTime dateTime, Series <DateTime, MarketDataElement> series)
{
MarketDataElement currentElement = MarketDataUtil.createHLOC(dateTime, "HLOC", "HSI", "FUT");
currentElement.volume = 900;
MarketDataUtil.setHLOC(currentElement, 20200, 20000, 20050, 20150);
if (series == null)
{
series = new SeriesBuilder <DateTime, MarketDataElement>()
{
{ dateTime, currentElement }
}.Series;
}
else
{
if (!series.ContainsKey(dateTime))
{
series = series.Merge(new SeriesBuilder <DateTime, MarketDataElement>()
{
{ dateTime, currentElement }
}.Series);
}
}
return(series);
}
public async Task Setup()
{
var author = AuthorBuilder.WithLibrary(LibraryId).Build();
var series = SeriesBuilder.WithLibrary(LibraryId).Build();
var categories = CategoryBuilder.WithLibrary(LibraryId).Build(3);
var book = new BookView
{
Title = RandomData.Name,
Authors = new List <AuthorView> {
new AuthorView {
Id = author.Id
}
},
SeriesId = series.Id,
SeriesIndex = 1,
SeriesName = series.Name,
Language = RandomData.Locale,
Categories = RandomData.PickRandom(categories, 2).Select(c => new CategoryView {
Id = c.Id
})
};
_response = await Client.PostObject($"/libraries/{LibraryId}/books", book);
_bookAssert = BookAssert.WithResponse(_response).InLibrary(LibraryId);
}
public void Build_WithoutPublisher_ThrowsException()
{
var builder = new SeriesBuilder();
var bc = builder.Details.Name("Series").Description("Desc");
var ex = Assert.Throws <ValidationException>(() => bc.Build());
}
private Frame <int, string> CreateFrame(DataResponse response)
{
if (response != null)
{
var rows = Enumerable.Range(0, response.data.Count).Select(i =>
{
// Build each row using series builder & return
// KeyValue representing row key with row data
var sb = new SeriesBuilder <string>();
for (int j = 0; j < response.headers.First().Count(); j++)
{
string displayName;
if (string.IsNullOrEmpty(response.headers.First()[j].displayName))
{
displayName = "None";
}
else
{
displayName = response.headers.First()[j].displayName;
}
sb.Add(displayName, response.data[i][j].Value);
}
return(KeyValue.Create(i, sb.Series));
});
return(Frame.FromRows(rows));
}
else
{
return(Frame.CreateEmpty <int, string>());
}
}
private Frame <string, string> CreateBestMatchFrame(SymbologySearchResponse response)
{
List <KeyValuePair <string, Series <string, string> > > list = new List <KeyValuePair <string, Series <string, string> > >();
foreach (var symbol in response.mappedSymbols)
{
string ric = symbol.symbol;
SeriesBuilder <string, string> sb;
if (symbol.bestMatch != null)
{
sb = new SeriesBuilder <string, string>();
FieldInfo[] fields = typeof(BestMatch).GetFields();
foreach (var field in fields)
{
var tmp = field.GetValue(symbol.bestMatch)?.ToString();
if (!string.IsNullOrEmpty(tmp))
{
sb.Add(field.Name, field.GetValue(symbol.bestMatch).ToString());
}
}
// sb.Series.Print();
list.Add(KeyValue.Create(ric, sb.Series));
}
}
return(Frame.FromRows(list));
}
public void updatePreMergeRTBarSeries(RTDataBar bar)
{
if (isDataMerged)
{
return;
}
DateTime time = bar.time;
MarketDataElement currentRTBData = MarketDataUtil.convertBarToMarketDataElement(bar);
if (preMergeRTBarSeries == null)
{
preMergeRTBarSeries = new SeriesBuilder <DateTime, MarketDataElement>()
{
{ time, currentRTBData }
}.Series;
}
else
{
if (!preMergeRTBarSeries.ContainsKey(time))
{
preMergeRTBarSeries = preMergeRTBarSeries.Merge(new SeriesBuilder <DateTime, MarketDataElement>()
{
{ time, currentRTBData }
}.Series);
}
}
return;
}
public async Task Setup()
{
var series = SeriesBuilder.WithLibrary(LibraryId).Build(4);
var expected = series.PickRandom();
_response = await Client.DeleteAsync($"/libraries/{LibraryId}/series/{expected.Id}");
}
private Frame <int, string> CreateWordFrame(Series <int, string> rows)
{
var wordsByRows = rows.GetAllValues()
.Select((x, i) =>
{
var sb = new SeriesBuilder <string, int>();
ISet <string> words = new HashSet <string>(
Regex.Matches(x.Value, "[a-zA-Z]+('(s|d|t|ve|m))?")
.Cast <Match>()
.Select(y => y.Value.ToLower())
.ToArray()
);
// Encode words appeared in each row with 1
foreach (string w in words)
{
sb.Add(w, 1);
}
return(KeyValue.Create(i, sb.Series));
});
// Create a data frame from the rows we just created
// And encode missing values with 0
var wordFrame = Frame.FromRows(wordsByRows)
.FillMissing(0);
return(wordFrame);
}
public async Task Setup()
{
var series = SeriesBuilder.WithLibrary(LibraryId).Build();
_seriesId = series.Id;
_newImage = RandomData.Bytes;
_response = await Client.PutFile($"/libraries/{LibraryId}/series/{series.Id}/image", _newImage);
}
public async Task Setup()
{
SeriesBuilder.WithLibrary(LibraryId).WithBooks(3).Build(4);
_response = await Client.GetAsync($"/libraries/{LibraryId}/series?pageNumber=1&pageSize=10");
_assert = new PagingAssert <SeriesView>(_response);
}
public async Task Setup()
{
SeriesBuilder.WithLibrary(LibraryId).WithBooks(3).WithNamePattern("SearchSeries").Build(50);
_response = await Client.GetAsync($"/libraries/{LibraryId}/series?query=SearchSeries&pageNumber=5&pageSize=10");
_assert = new PagingAssert <SeriesView>(_response);
}
public async Task Setup()
{
var Series = SeriesBuilder.WithLibrary(LibraryId).WithBooks(3).WithoutImage().Build(20);
_response = await Client.GetAsync($"/libraries/{LibraryId}/series?query={_searchedSeries}");
_assert = new PagingAssert <SeriesView>(_response);
}
public async Task Setup()
{
var series = SeriesBuilder.WithLibrary(LibraryId).Build();
series.Name = RandomData.Name;
_response = await Client.PutObject($"/libraries/{LibraryId}/series/{series.Id}", series);
}
private Frame <string, string> CreateFrame(SymbologySearchResponse response)
{
List <KeyValuePair <string, Series <string, string> > > list = new List <KeyValuePair <string, Series <string, string> > >();
SeriesBuilder <string, string> sb;
foreach (var symbol in response.mappedSymbols)
{
string ric = symbol.symbol;
sb = new SeriesBuilder <string, string>();
FieldInfo[] fields = typeof(MappedSymbol).GetFields();
string temp = "";
foreach (var field in fields)
{
temp = "";
if (field.FieldType == typeof(List <string>))
{
List <string> tmpList = (List <string>)field.GetValue(symbol);
if (tmpList != null)
{
temp = string.Join(",", tmpList);
}
}
else if (field.FieldType == typeof(BestMatch))
{
if (symbol.bestMatch != null)
{
// sb = new SeriesBuilder<string, string>();
FieldInfo[] bestMatchFields = typeof(BestMatch).GetFields();
foreach (var bestMatchField in bestMatchFields)
{
var tmp = bestMatchField.GetValue(symbol.bestMatch)?.ToString();
if (!string.IsNullOrEmpty(tmp))
{
sb.Add("BestMatch." + bestMatchField.Name, tmp);
}
}
continue;
}
}
else
{
temp = field.GetValue(symbol)?.ToString();
}
if (!string.IsNullOrEmpty(temp) && field.Name != "symbol")
{
sb.Add(field.Name, temp);
}
}
list.Add(KeyValue.Create(ric, sb.Series));
}
return(Frame.FromRows(list));
}
public async Task Setup()
{
_series = SeriesBuilder.WithLibrary(LibraryId).Build();
_SeriesBooks = BookBuilder.WithLibrary(LibraryId).WithSeries(_series).IsPublic().Build(25);
SeriesBuilder.WithLibrary(LibraryId).WithBooks(3).Build();
_response = await Client.GetAsync($"/libraries/{LibraryId}/books?pageNumber=4&pageSize=10&seriesid={_series.Id}");
_assert = new PagingAssert <BookView>(_response);
}
public async Task Setup()
{
var series = SeriesBuilder.WithLibrary(LibraryId).Build(4);
_expected = series.PickRandom();
_response = await Client.GetAsync($"/libraries/{LibraryId}/series/{_expected.Id}");
_assert = SeriesAssert.WithResponse(_response).InLibrary(LibraryId);
}
/// <summary>
/// For a frame of type Frame<TRow,TCol> mutate its rows of type TVal and create a new column with the results
/// </summary>
/// <typeparam name="TRow">Row Type</typeparam>
/// <typeparam name="TVal">Value Type</typeparam>
/// <typeparam name="TCol">Column Type</typeparam>
/// <param name="myFrame"></param>
/// <param name="mutatorMethod">delegate for transformation</param>
/// <returns>Series<K, V></returns>
public static Series <TRow, TVal> Mutate <TRow, TVal, TCol>(this Frame <TRow, TCol> myFrame, Func <TVal[], TVal> mutatorMethod)
{
SeriesBuilder <TRow, TVal> result = new SeriesBuilder <TRow, TVal>();
foreach (TRow key in myFrame.Rows.Keys)
{
TVal colResult = mutatorMethod(myFrame.Rows[key].GetValues <TVal>().ToArray());
result.Add(key, colResult);
}
return(result.ToSeries());
}
public void GenDeedle()
{
var rows = Enumerable.Range(0, 100).Select(i =>
{
var sb = new SeriesBuilder <string>();
sb.Add("Index", i);
sb.Add("Sin", Math.Sin(i / 100.0));
sb.Add("Cos", Math.Cos(i / 100.0));
return(KeyValue.Create(i, sb.Series));
});
}
public void Add(RegularTimeSeriesRawData item)
{
var dimensions = new string[_dimensions.Length];
var attributes = new object[_attributes.Length];
foreach (var dim in item.Dimensions)
{
var dimIndex = _dimensionIdsMap[dim.DimensionId];
dimensions[dimIndex] = dim.Key;
attributes[dimIndex] = dim.Name;
_dimensionKeyMaps[dimIndex][dim.Name] = dim.Key;
}
if (item.TimeSeriesAttributes != null)
{
foreach (var attr in item.TimeSeriesAttributes)
{
var attrIndex = _attributeIds[attr.Key];
attributes[attrIndex] = attr.Value;
}
}
var frequency = item.Frequency.ToFrequency();
var rangeTuple = Tuple.Create(item.StartDate, item.EndDate, frequency);
IReadOnlyList <DateTime> timeRange;
if (!_timeRangeCache.TryGetValue(rangeTuple, out timeRange))
{
timeRange = TimeFormat.InvariantTimeFormat.ExpandRangeSelection(item.StartDate, item.EndDate, frequency);
_timeRangeCache[rangeTuple] = timeRange;
}
var valuesBuilder = new SeriesBuilder <DateTime, double>();
int valueIndex = 0;
foreach (var valueItem in item.Values)
{
if (valueItem != null)
{
valuesBuilder.Add(timeRange[valueIndex], Convert.ToDouble(valueItem));
}
valueIndex++;
}
var values = new TimeSeriesValues(
item,
frequency,
new AttributesMap <string>(_dimensions, dimensions),
new AttributesMap <object>(_attributes, attributes),
valuesBuilder.Series);
_values[values] = values;
}
public Boolean updateTimeBarSeries(IBMessage message)
{
Boolean isAdded = false;
if ((!isDataMerged && getNeedMergeFlag()) || !getNeedMergeFlag())
{
return(isAdded);
}
Series <DateTime, MarketDataElement> TimeBarSeries = appMDManager.getTimeBarSeries();
if (TimeBarSeries == null)
{
return(isAdded);
}
HistoricalDataMessage histMessage = (HistoricalDataMessage)message;
double h = Convert.ToDouble(histMessage.High);
double l = Convert.ToDouble(histMessage.Low);
double o = Convert.ToDouble(histMessage.Open);
double c = Convert.ToDouble(histMessage.Close);
double vol = Convert.ToDouble(histMessage.Volume);
DateTime time = DateTime.ParseExact(histMessage.Date.Trim(), "yyyyMMdd HH:mm:ss", CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal);
MarketDataElement currentHsiData = MarketDataUtil.createHLOC(time, "HLOC", "HSI", "FUT");
currentHsiData.volume = vol;
currentHsiData.time = time;
if (TimeBarSeries == null)
{
TimeBarSeries = new SeriesBuilder <DateTime, MarketDataElement>()
{
{ time, currentHsiData }
}.Series;
isAdded = true;
}
else
{
MarketDataElement element = TimeBarSeries.GetAt(TimeBarSeries.KeyCount - 1);
if (!TimeBarSeries.ContainsKey(time))
{
TimeBarSeries = TimeBarSeries.Merge(new SeriesBuilder <DateTime, MarketDataElement>()
{
{ time, currentHsiData }
}.Series);
isAdded = true;
}
}
return(isAdded);
}
public static MovingAverageResult CalculateMovingAverage(
List <DataPoint> series,
int daysMovingAverage,
DateTime beginDate,
DateTime endDate,
MovingAverageType movingAverageType = MovingAverageType.Simple)
{
if (daysMovingAverage <= 1)
{
throw new ArgumentException("daysMovingAverage has to be greater than 1");
}
series = series.OrderBy(o => o.DateTime)
.ToList();
List <DataPoint> collapsedSeries = series.GroupBy(g => g.DateTime)
.Select(s => new DataPoint {
DateTime = s.Key, Value = s.Sum(z => z.Value)
})
.ToList();
SeriesBuilder <DateTime, decimal> dateSeries = new SeriesBuilder <DateTime, decimal>();
foreach (DataPoint dataPoint in collapsedSeries)
{
dateSeries.Add(dataPoint.DateTime, dataPoint.Value ?? 0m);
}
Series <DateTime, decimal> movingAverage = dateSeries
.Series
.Window(daysMovingAverage)
.Select(s => s.Value.Mean());
List <DataPoint> movingAverageSeries = movingAverage
.GetAllObservations()
.Select(s => new DataPoint
{
DateTime = s.Key,
Value = s.Value.Value,
SeriesName = string.Format("{0}-day Moving Average", daysMovingAverage)
})
.ToList();
series = series
.Where(w => w.DateTime >= beginDate)
.Where(w => w.DateTime <= endDate)
.OrderBy(o => o.SeriesName)
.ThenBy(o => o.DateTime)
.ToList();
return(new MovingAverageResult(series, movingAverageSeries, movingAverageType));
}
public async Task Setup()
{
var series = SeriesBuilder.WithLibrary(LibraryId).WithBooks(3).Build(4);
var selectedSeries = series.PickRandom();
_expected = new SeriesView {
Name = RandomData.Name
};
_response = await Client.PutObject($"/libraries/{LibraryId}/series/{selectedSeries.Id}", _expected);
_assert = SeriesAssert.WithResponse(_response).InLibrary(LibraryId);
}
public async Task Setup()
{
_series = SeriesBuilder.WithLibrary(LibraryId).Build();
var books = BookBuilder.WithLibrary(LibraryId).WithSeries(_series).Build(1);
_bookToUpdate = books.PickRandom();
var book = new BookView {
Title = RandomData.Text, SeriesId = -RandomData.Number
};
_response = await Client.PutObject($"/libraries/{LibraryId}/books/{_bookToUpdate.Id}", book);
}
public async Task Setup()
{
_series = SeriesBuilder.WithLibrary(LibraryId).Build();
_seriesBooks = BookBuilder.WithLibrary(LibraryId)
.WithSeries(_series)
.IsPublic()
.AddToFavorites(AccountId)
.Build(5);
SeriesBuilder.WithLibrary(LibraryId).WithBooks(3).Build();
_response = await Client.GetAsync($"/libraries/{LibraryId}/books?pageNumber=1&pageSize=10&favorite=true");
_assert = new PagingAssert <BookView>(_response);
}
private static Series <string, double> ColumnWiseSum(Frame <int, string> frame, string exclude)
{
var sb = new SeriesBuilder <string, double>();
foreach (string colname in frame.ColumnKeys)
{
double frequency = frame[colname].Sum();
if (!colname.Equals(exclude))
{
sb.Add(colname, frequency);
}
}
return(sb.ToSeries());
}
public static void SeriesBuilderSupportsDynamic()
{
for (int i = 1; i <= 2; i++) // Run this twice, to check that instance references are right
{
dynamic sb = new SeriesBuilder <string>();
sb.ID = 1;
sb.Value = 3.4;
sb.Date = DateTime.Today;
var actual = sb.Series;
var expected =
new SeriesBuilder <string, object> {
{ "ID", 1 }, { "Value", 3.4 }, { "Date", DateTime.Today }
}.Series;
Assert.AreEqual(expected, actual);
}
}
private static Frame <int, string> CreateCategories(Series <int, string> rows, string originalColName)
{
var categoriesByRows = rows.GetAllValues().Select((x, i) =>
{
// Encode the categories appeared in each row with 1
var sb = new SeriesBuilder <string, int>();
sb.Add(String.Format("{0}_{1}", originalColName, x.Value), 1);
return(KeyValue.Create(i, sb.Series));
});
// Create a data frame from the rows we just created
// And encode missing values with 0
var categoriesDF = Frame.FromRows(categoriesByRows).FillMissing(0);
return(categoriesDF);
}
public async Task Setup()
{
_library2Builder = Services.GetService <LibraryDataBuilder>();
var library2 = _library2Builder.Build();
var series = SeriesBuilder.WithLibrary(library2.Id).Build();
var author = AuthorBuilder.WithLibrary(LibraryId).Build();
var book = new BookView {
Title = new Faker().Random.String(), Authors = new List <AuthorView> {
new AuthorView {
Id = author.Id
}
}, SeriesId = series.Id
};
_response = await Client.PostObject($"/libraries/{LibraryId}/books", book);
}
public void Build_ValidCall()
{
var builder = new SeriesBuilder();
var publisher = new PublisherBuilder()
.Details
.Name("DC Comics")
.Created(new DateTime(1997, 01, 01))
.Build();
var bc = builder
.Details
.Name("Series")
.Description("Desc")
.Publisher(publisher)
.Build();
Assert.NotNull(bc);
}
public static void CanAggregateLettersUsingChunking()
{
var nums =
(from n in Enumerable.Range(0, 10)
select KeyValue.Create(n, (char)('a' + n))).ToSeries();
var actual =
nums.Aggregate(Aggregation.ChunkSize<int>(5, Boundary.Skip),
segment => segment.Data.Keys.First(),
segment => new OptionalValue<string>(new string(segment.Data.Values.ToArray())));
var expected =
new SeriesBuilder<int, string> {
{ 0, "abcde" },
{ 5, "fghij" }}.Series;
Assert.AreEqual(expected, actual);
}
public static void CanAggregateLettersUsingChunkWhile()
{
var nums =
new SeriesBuilder<int, char>
{ {0, 'a'}, {10, 'b'}, {11, 'c'} }.Series;
var actual =
nums.Aggregate(Aggregation.ChunkWhile<int>((k1, k2) => k2 - k1 < 10),
segment => segment.Data.Keys.First(),
segment => new OptionalValue<string>(new string(segment.Data.Values.ToArray())));
var expected =
new SeriesBuilder<int, string> {
{ 0, "a" },
{ 10, "bc" }}.Series;
Assert.AreEqual(expected, actual);
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="name">Name of the gesture</param>
/// <param name="mode">Mode of the gesture</param>
public Gesture(string name, GestureMode mode)
{
//_currentLapse = 0;
Name = name;
Mode = mode;
Active = false;
Confidence = 0;
Count = 0;
Progress = 0;
ConfidenceDict = new Dictionary<GestureConfidenceLevels, GestureConfidenceData>
{
{GestureConfidenceLevels.Low, new GestureConfidenceData(GestureConfidenceLevels.Low)},
{GestureConfidenceLevels.Average, new GestureConfidenceData(GestureConfidenceLevels.Average)},
{GestureConfidenceLevels.Good, new GestureConfidenceData(GestureConfidenceLevels.Good)},
{GestureConfidenceLevels.Better, new GestureConfidenceData(GestureConfidenceLevels.Better)},
{GestureConfidenceLevels.Best, new GestureConfidenceData(GestureConfidenceLevels.Best)},
};
_gestureSeries = new SeriesBuilder<long, int>();
}
public static void SeriesBuilderSupportsDynamic()
{
for (int i = 1; i <= 2; i++) // Run this twice, to check that instance references are right
{
dynamic sb = new SeriesBuilder<string>();
sb.ID = 1;
sb.Value = 3.4;
sb.Date = DateTime.Today;
Assert.AreEqual(sb.ID, 1);
Assert.AreEqual(sb.Value, 3.4);
Assert.AreEqual(sb.Date, DateTime.Today);
SeriesBuilder<string> staticTypedSb = sb;
var actual = staticTypedSb.Series;
var expected =
new SeriesBuilder<string, object> { { "ID", 1 }, { "Value", 3.4 }, { "Date", DateTime.Today } }.Series;
Assert.AreEqual(expected, actual);
}
}
//------------------------------------------------------------------------------------------------
public Frame<DateTime, string> computeFuturesSpreads()
//------------------------------------------------------------------------------------------------
{
Log_.InfoFormat("Computing spreads...");
CombinedStatic = SpreadSeriesBuilder.combineFuturesAndBondStatic(BondStatic,FuturesStatic);
var columns = FuturesPrices.Columns;
FrameBuilder.Columns<DateTime, string> frameblder = new FrameBuilder.Columns<DateTime, string>();
// Iterate over columns for prices
foreach (var col in FuturesPrices.ColumnKeys)
{
var series = columns[col].As<double>();
//series = series.Select(kvp => calcFuturesMeasure(SpreadMetric_, col, kvp.Value, kvp.Key));
var dte_idx = series.Keys.ToList();
SeriesBuilder<DateTime,double> sb = new SeriesBuilder<DateTime, double>();
Tuple<DateTime, double> prev = null;
/* THIS ABSOLUTELY HAS TO BE IN ASCENDING ORDER */
dte_idx.Sort();
foreach(var dte in dte_idx)
{
var res = series.TryGet(dte);
if (res.HasValue)
{
double price = res.Value;
var staticRow = CombinedStatic.Rows.TryGet(col).ValueOrDefault;
/* This section is to account for deficiencies in our database. */
// Quit if we don't have static
if (staticRow == null)
{
sb.Add(dte, double.NaN);
continue;
}
// Quit if we have a bad price
if (price == 0.0)
{
sb.Add(dte, double.NaN);
continue;
}
// Quit if contract isn't even trading
var firstTradeDate = staticRow.GetAs<DateTime>("fut_first_trade_dt");
var lastTradeDate = staticRow.GetAs<DateTime>("last_tradeable_dt");
if (dte > lastTradeDate || dte < firstTradeDate)
{
sb.Add(dte, double.NaN);
continue;
}
// Get price for AUD
if (Conv_.Country == BondAnalytics.Country.AU)
{
DateTime maturity = TimeSeriesUtilities.LazySafeTryGetAs(staticRow, "Maturity", DateTime.MinValue);
int tenor = (int)Math.Round((maturity - dte).Days / 365.25);
price = BondAnalytics.CalcAUDFuturesContractValue(price, tenor);
}
// ASSUMPTION: Our DB seems to front fill by default. Skip days where no data is present
/*if (prev != null && price == prev.Item2)
{
sb.Add(dte, double.NaN);
continue;
} */
double metric = calcFuturesMeasure(SpreadMetric_, col, price, dte, previous: prev);
prev = new Tuple<DateTime, double>(dte, price);
sb.Add(dte, metric);
}
else
{
sb.Add(dte, double.NaN);
}
}
frameblder.Add(col, sb.Series);
}
var output = frameblder.Frame;
Log_.InfoFormat("Complete:");
//output.Print();
return output.SortRowsByKey();
}
public static void Samples([CallerFilePath] string file = "")
{
var root = Path.GetDirectoryName(file);
// ------------------------------------------------------------
// Creating time series
// ------------------------------------------------------------
// [create-builder]
var numNames = new SeriesBuilder<int, string>() {
{ 1, "one" }, { 2, "two" }, { 3, "three" } }.Series;
numNames.Print();
// [/create-builder]
// [create-heterogen]
// Create series builder and use it via 'dynamic'
var nameNumsBuild = new SeriesBuilder<string, int>();
dynamic nameNumsDyn = nameNumsBuild;
nameNumsDyn.One = 1;
nameNumsDyn.Two = 2;
nameNumsDyn.Three = 3;
// Build series and print it
var nameNums = nameNumsBuild.Series;
nameNums.Print();
// [/create-heterogen]
// [create-ordinal]
var rnd = new Random();
var randNums = Enumerable.Range(0, 100)
.Select(_ => rnd.NextDouble()).ToOrdinalSeries();
randNums.Print();
// [/create-ordinal]
// [create-kvp]
var sin =
( from i in Enumerable.Range(0, 1000)
let x = i / 100.0
select KeyValue.Create(x, Math.Sin(x)) ).ToSeries();
sin.Print();
// [/create-kvp]
// [create-sparse]
var opts =
(from i in Enumerable.Range(0, 10)
let v = OptionalValue.OfNullable(LookupEven(i))
select KeyValue.Create(i, v)).ToSparseSeries();
opts.Print();
// [/create-sparse]
// [create-csv]
var frame = Frame.ReadCsv(Path.Combine(root, "../data/stocks/msft.csv"));
var frameDate = frame.IndexRows<DateTime>("Date").OrderRows();
var msftOpen = frameDate.GetSeries<double>("Open");
msftOpen.Print();
// [/create-csv]
// ------------------------------------------------------------
// Lookup and slicing
// ------------------------------------------------------------
// [lookup-key]
// Get value for a specified int and string key
var tenth = randNums[10];
var one = nameNums["One"];
// Get first and last value using index
var fst = nameNums.GetAt(0);
var lst = nameNums.GetAt(nameNums.KeyCount - 1);
// [/lookup-key]
// [lookup-opt]
// Get value as OptionalValue<T> and use it
var opt = opts.TryGet(5);
if (opt.HasValue) Console.Write(opt.Value);
// For value types, we can convert to nullable type
int? value1 = opts.TryGet(5).AsNullable();
int? value2 = opts.TryGetAt(0).AsNullable();
// [/lookup-opt]
// [lookup-ord]
// Get value exactly at the specified key
var jan3 = msftOpen
.Get(new DateTime(2012, 1, 3));
// Get value at a key or for the nearest previous date
var beforeJan1 = msftOpen
.Get(new DateTime(2012, 1, 1), Lookup.NearestSmaller);
// Get value at a key or for the nearest later date
var afterJan1 = msftOpen
.Get(new DateTime(2012, 1, 1), Lookup.NearestGreater);
// [/lookup-ord]
// [lookup-slice]
// Get a series starting/ending at
// the specified date (inclusive)
var msftStartIncl = msftOpen.StartAt(new DateTime(2012, 1, 1));
var msftEndIncl = msftOpen.EndAt(new DateTime(2012, 12, 31));
// Get a series starting/ending after/before
// the specified date (exclusive)
var msftStartExcl = msftOpen.After(new DateTime(2012, 1, 1));
var msftEndExcl = msftOpen.Before(new DateTime(2012, 12, 31));
// Get prices for 2012 (both keys are inclusive)
var msft2012 = msftOpen.Between
(new DateTime(2012, 1, 1), new DateTime(2012, 12, 31));
// [/lookup-slice]
// ------------------------------------------------------------
// Statistics and calculations
// ------------------------------------------------------------
// [calc-stat]
// Calculate median & mean price
var msftMed = msft2012.Median();
var msftAvg = msft2012.Mean();
// Calculate sum of square differences
var msftDiff = msft2012 - msftAvg;
var msftSq = (msftDiff * msftDiff).Sum();
// [/calc-stat]
// [calc-diff]
// Subtract previous day value from current day
var msftChange = msft2012 - msft2012.Shift(1);
// Use built-in Diff method to do the same
var msftChangeAlt = msft2012.Diff(1);
// Get biggest loss and biggest gain
var minMsChange = msftChange.Min();
var maxMsChange = msftChange.Max();
// [/calc-diff]
// [calc-custom]
var wackyStat = msft2012.Observations.Select(kvp =>
kvp.Value / (kvp.Key - msft2012.FirstKey()).TotalDays).Sum();
// [/calc-custom]
// ------------------------------------------------------------
// Missing data
// ------------------------------------------------------------
// [fill-const-drop]
// Fill missing data with constant
var fillConst = opts.FillMissing(-1);
fillConst.Print();
// Drop keys with no value from the series
var drop = opts.DropMissing();
drop.Print();
// [/fill-const-drop]
// [fill-dir]
// Fill with previous available value
var fillFwd = opts.FillMissing(Direction.Forward);
fillFwd.Print();
// Fill with the next available value
var fillBwd = opts.FillMissing(Direction.Backward);
fillBwd.Print();
// [/fill-dir]
// ------------------------------------------------------------
// Windows and chunks, grouping
// ------------------------------------------------------------
// [aggreg-group]
// Group random numbers by the first digit & get distribution
var buckets = randNums
.GroupBy(kvp => (int)(kvp.Value * 10))
.Select(kvp => OptionalValue.Create(kvp.Value.KeyCount));
buckets.Print();
// [/aggreg-group]
// [aggreg-win]
// Average over 25 element floating window
var monthlyWinMean = msft2012.WindowInto(25, win => win.Mean());
// Get floating window over 5 elements as series of series
// and then apply average on each series individually
var weeklyWinMean = msft2012.Window(5).Select(kvp =>
kvp.Value.Mean());
// [/aggreg-win]
// [aggreg-chunk]
// Get chunks of size 25 and mean each (disjoint) chunk
var monthlyChunkMean = msft2012.ChunkInto(25, win => win.Mean());
// Get series containing individual chunks (as series)
var weeklyChunkMean = msft2012.Chunk(5).Select(kvp =>
kvp.Value.Mean());
// [/aggreg-chunk]
// [aggreg-pair]
// For each key, get the previous value and average them
var twoDayAvgs = msft2012.Pairwise().Select(kvp =>
(kvp.Value.Item1 + kvp.Value.Item2) / 2.0);
// [/aggreg-pair]
// [aggreg-any]
msft2012.Aggregate
( // Get chunks while the month & year of the keys are the same
Aggregation.ChunkWhile<DateTime>((k1, k2) =>
k1.Month == k2.Month && k2.Year == k1.Year ),
// For each chunk, return the first key as the key and
// either average value or missing value if it was empty
chunk => KeyValue.Create
( chunk.Data.FirstKey(),
chunk.Data.ValueCount > 0 ?
OptionalValue.Create(chunk.Data.Mean()) :
OptionalValue.Empty<double>() ) );
// [/aggreg-any]
// ------------------------------------------------------------
// Operations (Select, where)
// ------------------------------------------------------------
// [linq-methods]
var overMean = msft2012
.Select(kvp => kvp.Value - msftAvg)
.Where(kvp => kvp.Value > 0.0).KeyCount;
// [/linq-methods]
// [linq-query]
var underMean =
( from kvp in msft2012
where kvp.Value - msftAvg < 0.0
select kvp ).KeyCount;
// [/linq-query]
Console.WriteLine(overMean);
Console.WriteLine(underMean);
// ------------------------------------------------------------
// Indexing and sampling & resampling
// ------------------------------------------------------------
// [index-keys]
// Turn DateTime keys into DateTimeOffset keys
var byOffs = msft2012.SelectKeys(kvp =>
new DateTimeOffset(kvp.Key));
// Replace keys with ordinal numbers 0 .. KeyCount-1
var byInt = msft2012.IndexOrdinally();
// [/index-keys]
// [index-with]
// Replace keys with explictly specified new keys
var byDays = numNames.IndexWith(new[] {
DateTime.Today,
DateTime.Today.AddDays(1.0),
DateTime.Today.AddDays(2.0) });
// [/index-with]
}
//----------------------------------------------------------------------------------------------------------------------------------
public static object[,] AggregateBasisRisk(object[,] DataTable, object[,] Filters, object[,] FuturesTable,
object[,] PriceTable,
string Country, double dv01Scaling, bool bLiveOrEOD, string Currency, bool bInclSnED)
//----------------------------------------------------------------------------------------------------------------------------------
{
// WARNING: This is a very case specific tactical function. Used in the BasisPositionAndRisk sheet to aggregate risk.
try
{
// Init data frames.
Frame<string, string> dataFrame = MainHelpers.getDeedleFrameFromObjectArray(DataTable);
Frame<string, string> filtersFrame = MainHelpers.getDeedleFrameFromObjectArray(Filters, hasHeader: false,
hasIndex: true);
Frame<string, string> futuresFrame = MainHelpers.getDeedleFrameFromObjectArray(FuturesTable);
futuresFrame = futuresFrame.IndexRows<string>("Contract", keepColumn: true);
Frame<string, string> priceFrame = MainHelpers.getDeedleFrameFromObjectArray(PriceTable);
priceFrame =
priceFrame.Where(
kvp =>
(kvp.Value.TryGetAs<string>("Bond").ValueOrDefault != null) &&
(kvp.Value.TryGetAs<string>("Bond").ValueOrDefault != "ExcelErrorNA"));
priceFrame = priceFrame.IndexRows<string>("Bond", keepColumn: true);
Frame<string, string> aggregateFrame;
#region InputValidation
// Pull out all column names into sets.
var dataHeaders = dataFrame.ColumnKeys.ToDictionary(v => v, v => "DataTable");
// we'll use the values for error handling later
var filterRows = filtersFrame.RowKeys.ToDictionary(v => v, v => "Filter Rows");
var futuresHeaders = futuresFrame.ColumnKeys.ToDictionary(v => v, v => "FuturesTable");
var priceHeaders = priceFrame.ColumnKeys.ToDictionary(v => v, v => "PriceTable");
// Hardcoded input name requirements, and OT report generated has to conform to this;
List<string> dataHeaderEssentials = new List<string>()
{
"Entity",
"B:Strategy Type",
"B:Strategy Sub Type",
"B:Portfolio",
"B:Strategy",
"Curve Currency",
"Curve Description",
"Product Type",
"Description",
"Maturity Date",
"Current Notional",
"S:Country Of Risk",
"S:ISIN",
"Delta"
};
List<string> dataHeaderRenames = new List<string>()
{
"Entity",
"StrategyType",
"StrategySubType",
"Portfolio",
"Strategy",
"Currency",
"CurveDescription",
"ProductType",
"Description",
"Bond Maturity",
"Current Notional",
"CountryOfRisk",
"ISIN",
"Delta"
};
List<string> filterRowEssentials = new List<string>()
{
"Entity",
"StrategyType",
"StrategySubType",
"Portfolio",
"Strategy",
};
List<string> futuresHeaderEssentials = new List<string>()
{
"Contract",
"Bond",
"ISIN",
"Future Quotes",
"Bond Quotes",
"Contract Type",
"Maturity"
};
List<string> priceHeaderEssentials = new List<string>()
{
"Bond",
"S:ISIN",
"PositionPrice",
"ClosePrice"
};
List<string> validInstruments = new List<string>()
{
"Bond",
"BondFuture"
};
if (bInclSnED)
{
validInstruments.Add("Swap");
validInstruments.Add("InterestFuture");
}
// CHECK KEY HEADERS.
List<Tuple<List<string>, Dictionary<string, string>>> checkList = new List
<Tuple<List<string>, Dictionary<string, string>>>()
{
new Tuple<List<string>, Dictionary<string, string>>(dataHeaderEssentials, dataHeaders),
new Tuple<List<string>, Dictionary<string, string>>(futuresHeaderEssentials, futuresHeaders),
new Tuple<List<string>, Dictionary<string, string>>(filterRowEssentials, filterRows),
new Tuple<List<string>, Dictionary<string, string>>(priceHeaderEssentials, priceHeaders)
};
foreach (var test in checkList)
{
List<string> essentials = test.Item1;
Dictionary<string, string> header = test.Item2;
foreach (string name in essentials)
{
if (!header.ContainsKey(name))
{
//A bit of a hack, but we only needed the keys in the dictionary, and this puts the values to good use too.
throw new ArgumentException("#Missing " + name + " in " + header.First().Value);
}
}
}
#endregion
#region FrameInitialisaiton
// Extract Essential Columns.
dataFrame = dataFrame.Columns[dataHeaderEssentials];
for (int i = 0; i < dataHeaderEssentials.Count; ++i)
{
dataFrame.RenameColumn(dataHeaderEssentials[i], dataHeaderRenames[i]);
}
filtersFrame = filtersFrame.Rows[filterRowEssentials];
futuresFrame = futuresFrame.Columns[futuresHeaderEssentials];
priceFrame = priceFrame.Columns[priceHeaderEssentials];
// Construct the lookup table for filters
Dictionary<string, HashSet<string>> filterMap = new Dictionary<string, HashSet<string>>();
var srsKeys = filtersFrame.RowKeys.ToList();
foreach (var filterField in srsKeys)
{
string fieldCSVs = filtersFrame.GetRow<string>(filterField).FirstValue();
//var tmpKeys = srs.Keys;
//var tmpVals = srs.Values;
//string filterField = srsKeys[ int.Parse( srs.Keys.First())];
//string fieldCSVs = srs.Values.First().ToString(); // case insensitive comparisons for filter
HashSet<string> values = new HashSet<string>(fieldCSVs.Trim().Split(',').ToList());
filterMap[filterField] = values;
}
filterMap["ProductType"] = new HashSet<string>() {"Bond", "BondFuture"};
if (bInclSnED)
{
filterMap["ProductType"].Add("Swap");
filterMap["ProductType"].Add("InterestFuture");
//Currency is needed if Swap is included
filterMap["Currency"] = new HashSet<string>() { Currency.ToUpper() };
}
// Function is only compatible with Bonds and future instruments
filterMap["CountryOfRisk"] = new HashSet<string>() {Country /*.ToLower()*/};
// Filter data table
dataFrame = dataFrame.Where(kvp =>
{
bool bOK = true;
var row = kvp.Value;
foreach (var keyVal in filterMap)
{
string field = keyVal.Key;
HashSet<string> validFields = keyVal.Value;
//Country for swap is empty, hence must include ccy into filtering list//this is not safe for EURO swaps!!!
if (field == "CountryOfRisk" )
{
var ProdType = row.TryGet("ProductType").ValueOrDefault;
if (ProdType != null && (ProdType.ToString() == "Swap" || ProdType.ToString() == "InterestFuture")) continue;
}
var tmpRowEntry = row.TryGet(field).ValueOrDefault;
string rowEntry = tmpRowEntry != null ? tmpRowEntry.ToString() : ""; // we stripped spaces while parsing the filters
bOK = bOK && rowEntry != null && (validFields.Contains("*") || validFields.Contains(rowEntry));
}
return bOK;
});
#endregion
#region DataAggregation
if (dataFrame.IsEmpty)
{
throw new ArgumentNullException("No data present after applying filters!");
}
//Rename Description region for Swap to be date only
if (bInclSnED)
{
//Loop through each row to update the Description of Swap to expiration only
var rowKeyList = dataFrame.RowKeys.ToList();
var sbuilder = new SeriesBuilder<string, string>();
foreach (var rowIndx in rowKeyList)
{
var currRow = dataFrame.Rows[rowIndx];
string newDescription = null;
if (currRow.GetAs<string>("ProductType") == "Swap")
{
newDescription = currRow.GetAs<string>("Description");
//newDescription = "Swap-"+ newDescription.Substring(newDescription.LastIndexOf(" ")+1);
newDescription = "Swap-" + newDescription;
}
else
{
newDescription = currRow.GetAs<string>("Description");
}
sbuilder.Add(rowIndx, newDescription);
}
//add Description and drop original description column
dataFrame.DropColumn("Description");
dataFrame.AddColumn("Description", sbuilder.Series);
}
// MLP Alloc Frames
var fullNotionalSrs = dataFrame.GetColumn<double>("Current Notional");
var fullDataFrameKeys = dataFrame.Columns["Description"];
var fullDataFrameEntities = dataFrame.Columns["Entity"];
var mlpSrs = fullNotionalSrs.Where(kvp => fullDataFrameEntities.GetAs<string>(kvp.Key) == "MLP");
var mlpnotionals =
mlpSrs.GroupBy(kvp => fullDataFrameKeys.GetAs<string>(kvp.Key)).Select(x => x.Value.Sum());
//Aggregate Notionals and Deltas of position.
var dataFrameKeys = dataFrame.Columns["Description"];
var dataFrameEntities = dataFrame.Columns["Entity"];
var notionalSrs = dataFrame.GetColumn<double>("Current Notional");
var dv01Srs = dataFrame.GetColumn<double>("Delta");
var notionals =
notionalSrs.GroupBy(kvp => dataFrameKeys.GetAs<string>(kvp.Key)).Select(x => x.Value.Sum());
var dv01s = dv01Srs.GroupBy(kvp => dataFrameKeys.GetAs<string>(kvp.Key)).Select(x => x.Value.Sum());
SortedDictionary<string, Series<string, object>>
uniqueRows = new SortedDictionary<string, Series<string, object>>();
var rowKeys = dataFrame.RowKeys.ToList();
foreach (var rowIdx in rowKeys)
{
var row = dataFrame.Rows[rowIdx];
object tmp = row.Get("Description");
//for swap
string prodType = row.GetAs<string>("ProductType");
string instrument = tmp == null ? "" : (string) tmp;
// Get values out
tmp = notionals.TryGet(instrument).ValueOrDefault;
double totalNotional = tmp == null ? 0.0 : (double) tmp;
tmp = dv01s.TryGet(instrument).ValueOrDefault;
double totalDV01 = tmp == null ? 0.0 : (double) tmp*dv01Scaling; // Scale DV01s here
tmp = mlpnotionals.TryGet(instrument).ValueOrDefault;
double mlpNotional = tmp == null ? 0.0 : (double) tmp;
// Create the output row here
if (!uniqueRows.ContainsKey(instrument))
{
#region AssignRowValues
double bond01 = 0.0;
double fut01 = 0.0;
double swap01 = 0.0;
double ED01 = 0.0;
double balancesheet = 0.0;
double futNotional = 0.0;
double swapNotional = 0.0;
double EDNotional = 0.0;
double totNotional = 0.0;
double MLPAlloc = 0.0;
double prevfront = 0.0;
double front = 0.0;
double back = 0.0;
double back2 = 0.0;
DateTime bondMaturity;
string stmp = row.TryGetAs<string>("Bond Maturity").ValueOrDefault;
try
{
//try to convert this to datetime direct
bondMaturity = DateTime.ParseExact(stmp, "MM/dd/yyyy",
System.Globalization.CultureInfo.InvariantCulture);
}
catch
{
//Otherwise, just pull straight
bondMaturity = DateTime.FromOADate(row.TryGetAs<Double>("Bond Maturity").ValueOrDefault);
}
var futRow = futuresFrame.TryGetRow<string>(instrument).ValueOrDefault;
if (futRow != null)
{
futNotional = totalNotional;
fut01 = totalDV01;
double dateNum = 0.0;
bool parsed = double.TryParse(futRow.TryGet("Maturity").ValueOrDefault, out dateNum);
bondMaturity = parsed? DateTime.FromOADate(dateNum) : DateTime.MinValue;
}
else if(prodType == "Swap")
{
//Duplicated records due to exposure to different curve risk
//Not suitable for GBP swaps since GBP swaps can have exposure to 3 curves
if (instrument.Contains("IBOR") && !instrument.Contains("FEDFUNDS"))
{
totalNotional = totalNotional / 2;
mlpNotional = mlpNotional / 2;
}
swapNotional = totalNotional;
swap01 = totalDV01;
}
else if(prodType == "InterestFuture")
{
if (instrument.Contains("Eurodollar") || instrument.Contains("Euribor"))
{
totalNotional = totalNotional / 2;
mlpNotional = mlpNotional / 2;
}
if (instrument.Contains("Sterling"))
{
totalNotional = totalNotional / 3;
mlpNotional = mlpNotional / 3;
}
EDNotional = totalNotional;
ED01 = totalDV01;
}
else
{
var isin = priceFrame.Columns["S:ISIN"].GetAs<string>(instrument);
var ctdRows =
futuresFrame.Where(x => x.Value.GetAs<string>("ISIN") == isin);
string priceCol = bLiveOrEOD ? "PositionPrice" : "ClosePrice";
double price = priceFrame.Columns[priceCol].TryGetAs<double>(instrument).OrDefault(0.0);
if (!ctdRows.IsEmpty)
{
var ctdRow = ctdRows.GetRowAt<string>(0);
string ctdMonth = ctdRow.Get("Contract Type");
switch (ctdMonth)
{
case "FM-1":
prevfront = totalDV01;
break;
case "FM1":
front = totalDV01;
break;
case "BM1":
back = totalDV01;
break;
case "BM2":
back2 = totalDV01;
break;
default:
bond01 = totalDV01;
SLog.log.WarnFormat(
"{0} is CTD, but contract month isn' specified. Defaulting to bond 01",
instrument);
break;
}
}
else
{
bond01 = totalDV01;
}
balancesheet = price*totalNotional;
}
MLPAlloc = mlpNotional / totalNotional;
#endregion
// Processing data
#region BuildRow
// Create entry if doesn't exist
var sb = new SeriesBuilder<string, object>();
string addOn = bLiveOrEOD ? "" : "EOD";
sb.Add("CountryOfRisk" + addOn, row.TryGet("CountryOfRisk").ValueOrDefault);
if (bInclSnED) sb.Add("Product Type" + addOn, row.TryGet("ProductType").ValueOrDefault);
sb.Add("BondMaturity" + addOn, bondMaturity);
sb.Add("Description" + addOn, instrument);
sb.Add("ISIN" + addOn, row.TryGet("ISIN").ValueOrDefault);
sb.Add("Bond DV01" + addOn, bond01);
sb.Add("Fut DV01" + addOn, fut01);
if (bInclSnED)
{
sb.Add("Swap DV01" + addOn, swap01);
sb.Add("Interest Future DV01" + addOn, ED01);
}
sb.Add("Total DV01" + addOn, totalDV01);
sb.Add("Bond BalanceSheet" + addOn, balancesheet);
sb.Add("Fut Notional" + addOn, futNotional);
if (bInclSnED)
{
sb.Add("Swap Notional" + addOn, swapNotional);
sb.Add("Interest Future Notional" + addOn, EDNotional);
}
sb.Add("Total Notional" + addOn, totalNotional);
sb.Add("MLP Allocation" + addOn, MLPAlloc);
sb.Add("FM-1" + addOn, prevfront);
sb.Add("FM1" + addOn, front);
sb.Add("BM1" + addOn, back);
sb.Add("BM2" + addOn, back2);
uniqueRows[bondMaturity.ToShortDateString() + "_" + instrument] = sb.Series;
#endregion
}
}
#endregion
// Package outputs
string appendix = bLiveOrEOD ? "" : "EOD";
aggregateFrame = Frame.FromRows(uniqueRows.AsEnumerable());
aggregateFrame = FrameExtensions.SortRows(aggregateFrame, "BondMaturity" + appendix);
var output = MainHelpers.getFrameAsObjArr(aggregateFrame, sortkeys: false);
output[0, 0] = "ID";
return output;
}
catch (Exception ex)
{
SLog.log.ErrorFormat("Error in SymAggregateBasisRiskReportData(): {0}", ex.Message);
// RETURN HEADERS, to avoid messing pivot charts up
// Create entry if doesn't exist
var sb = new SeriesBuilder<string, object>();
string addOn = bLiveOrEOD ? "" : "EOD";
sb.Add("CountryOfRisk" + addOn, "#Error");
sb.Add("BondMaturity" + addOn, "#Error");
sb.Add("Description" + addOn, "#Error");
sb.Add("ISIN" + addOn, "#Error");
sb.Add("Bond DV01" + addOn, "#Error");
sb.Add("Fut DV01" + addOn, "#Error");
if (bInclSnED)
{
sb.Add("Swap DV01" + addOn, "#Error");
sb.Add("Interest Future DV01" + addOn, "Error");
}
sb.Add("Total DV01" + addOn, "#Error");
sb.Add("Bond BalanceSheet" + addOn, "#Error");
sb.Add("Fut Notional" + addOn, "#Error");
if (bInclSnED)
{
sb.Add("Swap Notional" + addOn, "#Error");
sb.Add("Interest Future Notional" + addOn, "#Error");
}
sb.Add("Total Notional" + addOn, "#Error");
sb.Add("MLP Allocation" + addOn, "#Error");
sb.Add("FM-1" + addOn, "#Error");
sb.Add("FM1" + addOn, "#Error");
sb.Add("BM1" + addOn, "#Error");
sb.Add("BM2" + addOn, "#Error");
SortedDictionary<string, Series<string, object>>
uniqueRows = new SortedDictionary<string, Series<string, object>>();
uniqueRows["Error"] = sb.Series;
string appendix = bLiveOrEOD ? "" : "EOD";
Frame<string, string> aggregateFrame = Frame.FromRows(uniqueRows.AsEnumerable());
aggregateFrame = FrameExtensions.SortRows(aggregateFrame, "BondMaturity" + appendix);
var output = MainHelpers.getFrameAsObjArr(aggregateFrame, sortkeys: false);
output[0, 0] = "ID";
return output;
}
}
public static void Samples([CallerFilePath] string file = "")
{
var root = Path.GetDirectoryName(file);
// ------------------------------------------------------------
// Creating and loading data frames
// ------------------------------------------------------------
// [create-records]
// Create a collection of anonymous types
var objects = Enumerable.Range(0, 100).Select(i =>
new { Group = "g" + (i/10).ToString(), Number = i });
// Create data frame with properties as column names
var dfObjects = Frame.FromRecords(objects);
dfObjects.Print();
// [/create-records]
// [create-rows]
// Generate collection of rows
var rows = Enumerable.Range(0, 100).Select(i => {
// Build each row using series builder & return
// KeyValue representing row key with row data
var sb = new SeriesBuilder<string>();
sb.Add("Index", i);
sb.Add("Sin", Math.Sin(i / 100.0));
sb.Add("Cos", Math.Cos(i / 100.0));
return KeyValue.Create(i, sb.Series);
});
// Turn sequence of row information into data frame
var df = Frame.FromRows(rows);
// [/create-rows]
// [create-csv]
// Read MSFT & FB stock prices from a CSV file
var msftRaw = Frame.ReadCsv(Path.Combine(root, "../data/stocks/msft.csv"));
var fbRaw = Frame.ReadCsv(Path.Combine(root, "../data/stocks/fb.csv"));
// [/create-csv]
// ------------------------------------------------------------
// Working with row and column indices
// ------------------------------------------------------------
// [index-date]
// Get MSFT & FB stock prices indexed by data
var msft = msftRaw.IndexRows<DateTime>("Date").OrderRows();
var fb = fbRaw.IndexRows<DateTime>("Date").OrderRows();
// And rename columns to avoid overlap
msft.RenameSeries(s => "Msft" + s);
fb.RenameSeries(s => "Fb" + s);
// [/index-date]
// [index-cols]
// Read US debt data from a CSV file
var debt = Frame.ReadCsv(Path.Combine(root, "../data/us-debt.csv"));
// Index by Year column and
var debtByYear = debt
.IndexRows<int>("Year")
.IndexColumnsWith(new[] { "Year", "GDP", "Population", "Debt", "?" });
// [/index-cols]
// ------------------------------------------------------------
// Joining and aligning data frames
// ------------------------------------------------------------
// [join-inout]
// Inner join (take intersection of dates)
var joinIn = msft.Join(fb, JoinKind.Inner);
// Outer join (take union & fill with missing)
var joinOut = msft.Join(fb, JoinKind.Outer);
// [/join-inout]
// [join-lookup]
// Shift MSFT observations by +1 hour for testing
var msftShift = msft.SelectRowKeys(k => k.Key.AddHours(1.0));
// MSFT data are missing because keys do not match
var joinLeftWrong = fb.Join(msftShift, JoinKind.Left);
// This works! Find the value for the nearest smaller key
// (that is, for the nearest earlier time with value)
var joinLeft = fb.Join(msftShift, JoinKind.Left, Lookup.NearestSmaller);
joinLeft.Print();
// [/join-lookup]
// ------------------------------------------------------------
// Accessing data and series operations
// ------------------------------------------------------------
// [series-get]
// Get MSFT and FB opening prices and calculate the difference
var msOpen = joinIn.GetSeries<double>("MsftOpen");
var msClose = joinIn.GetSeries<double>("MsftClose");
var msDiff = msClose - msOpen;
// [/series-get]
// [series-dropadd]
// Drop series from a data frame
joinIn.DropSeries("MsftAdj Close");
joinIn.DropSeries("FbAdj Close");
// Add new series to a frame
joinIn.AddSeries("MsftDiff", msDiff);
joinIn.Print();
// [/series-dropadd]
// [series-rows]
// Get row and then look at row properties
var row = joinIn.Rows[new DateTime(2013, 1, 4)];
var msLo = row.GetAs<double>("MsftLow");
var msHi = row.GetAs<double>("MsftHigh");
// Get value for a specified column & row keys
var diff = joinIn["MsftDiff", new DateTime(2013, 1, 4)];
// [/series-rows]
// ------------------------------------------------------------
// LINQ to data frame
// ------------------------------------------------------------
// [linq-select]
// Project rows into a new series using the Select method
var diffs = joinIn.Rows.Select(kvp =>
kvp.Value.GetAs<double>("MsftOpen") -
kvp.Value.GetAs<double>("FbOpen"));
// [/linq-select]
// [linq-where]
// Filter rows using a specified condition
var msftGreaterRows = joinIn.Rows.Where(kvp =>
kvp.Value.GetAs<double>("MsftOpen") >
kvp.Value.GetAs<double>("FbOpen"));
// Transform row collection into a new data frame
var msftGreaterDf = Frame.FromRows(msftGreaterRows);
// [/linq-where]
}
// GBP is only currently setup to get spot ASW terms. Will move on to futures only if it becomes necessary.
//----------------------------------------------------------------------------------------
public static void run(DateTime start, DateTime end, string samplingFreq, string outputFolderPath)
//----------------------------------------------------------------------------------------
{
Console.WriteLine("Commencing GBP Run");
// INIT
BondAnalytics.Country eCountry = BondAnalytics.Country.UK;
List<string> SwapMeasures = new List<string>() { "MMS", "Spread", "TrueSpread" }; //
List<string> BondMeasures = new List<string>() { "Price", "Yield"};
// FIND ALL SECURITIES.
DateTime fDate = DateTime.Today.AddDays(30);
List<string> rawmlptickers = DataHandler.GetAllSecurities("UKT", "Term", fDate);
// Remove everything with AuC in it
List<string> mlptickers = new List<string>();
foreach (string tic in rawmlptickers)
{
if (!tic.ToLower().Contains("auc"))
{
mlptickers.Add(tic);
}
}
Console.WriteLine();
Console.WriteLine("Found UKT tickers");
foreach (string tic in mlptickers)
{
Console.WriteLine(tic);
}
// GET THE RIGHT STATICS
var fcstCfg = new SpreadTimeSeriesConfigs("UK", bForecastCurve:true);
var discCfg = new SpreadTimeSeriesConfigs("UK", bForecastCurve:false);
var bondStaticMap = new Dictionary<string, BondStatic>();
foreach (string tic in mlptickers)
{
bondStaticMap[tic] = DataHandler.GetBondStatic(tic);
}
// GET DATES.
Console.WriteLine();
Console.WriteLine("Setting Dates");
var res = DataHandler.GetCarbonClient().GetCalendarAsync("GBLO");
res.Wait();
var holsInLocalDate = res.Result.Dates;
List<DateTime> hols = holsInLocalDate.Select(d => d.ToDateTime()).ToList();
DateTime startDate = start.AddTenor(Tenor.FromString("-1b"), "").AddTenor(Tenor.FromString("1b"), "");
DateTime nextDate = end.AddTenor(Tenor.FromString("1b"), "").AddTenor(Tenor.FromString("-1b"), "");
List<DateTime> dateRange = new List<DateTime>();
if (startDate > nextDate)
{
throw new ArgumentException("Start date cannot be after end date");
}
var tenor = Tenor.FromString("-" + samplingFreq);
while (nextDate >= startDate)
{
dateRange.Add(nextDate);
nextDate = BondAnalytics.PrevBusDay(nextDate.AddTenor(tenor, ""), hols);
}
// GET ALL PRICES.
Console.WriteLine();
Console.WriteLine("Setting Prices");
var result = DataHandler.GetBondPricesAsync(mlptickers, dateRange, eCountry);
Frame<DateTime, string> bondFrame;
if (result.IsCompleted) bondFrame = result.Result;
else throw new ArgumentNullException("No bond prices came back!");
bondFrame.Print();
// GET ALL SWAP CURVES.
Console.WriteLine();
Console.WriteLine("Getting Swap Cruves");
var fCurves = DataHandler.getDiscountCurvesInDateRangeAsync(fcstCfg.SymForecastCurveName, dateRange).Result;
var dCurves = DataHandler.getDiscountCurvesInDateRangeAsync(fcstCfg.SymDiscountCurveName, dateRange).Result;
// GET ALL MEASURES.
var output = new Dictionary<string/*measure*/,Frame<DateTime,string>>();
var configs = new List<SpreadTimeSeriesConfigs>(){fcstCfg, discCfg};
// Swap measures
foreach (string measure in SwapMeasures)
{
Console.WriteLine("Calculating {0}", measure);
foreach (var config in configs)
{
Console.WriteLine("Forecast curve ? {0}", config);
var valueMap = new Dictionary<string/*id*/, Series<DateTime,double>>();
bool bForecast = config.bForecastCurve;
foreach (var ticker in mlptickers)
{
var srs = bondFrame.Columns[ticker];
var bndStatic = bondStaticMap[ticker];
var sb = new SeriesBuilder<DateTime, double>();
foreach (DateTime dte in srs.Keys)
{
// Skip if we're missing data somewhere
if (!(dCurves.ContainsKey(dte) &&
srs.ContainsKey(dte)))
continue;
if (config.bForecastCurve && !fCurves.ContainsKey(dte)) // only press on if we need forecast curves
continue;
// Proceed.
DiscountCurve discount = dCurves[dte];
DiscountCurve forecast = bForecast ? fCurves[dte] : discount;
DateTime settle = BondAnalytics.NextBusDay(dte.Date.AddDays(config.bondSettleLag), hols);
double price = srs.TryGetAs<double>(dte).OrDefault(double.NaN);
if (double.IsNaN(price)) continue;
double value = BondASWCalc.calcMeasure( measure, price, eCountry,
dte, settle, discount, forecast,
bndStatic, config, hols);
if (double.IsNaN(value)) continue;
sb.Add(dte, value);
}
valueMap[ticker] = sb.Series;
Console.WriteLine("Completed for {0}", ticker);
valueMap[ticker].Print();
}
var df = Frame.FromColumns(valueMap);
string prefix = bForecast ? "L" : "OIS";
output[prefix + measure] = df;
Console.WriteLine();
Console.WriteLine("Swap Measures complete.");
}
}
//Bond Measure
foreach(string measure in BondMeasures)
{
var config = fcstCfg;
Console.WriteLine("Calculating {0}", measure);
var valueMap = new Dictionary<string/*id*/, Series<DateTime, double>>();
foreach (var ticker in mlptickers)
{
var srs = bondFrame.Columns[ticker];
var bndStatic = bondStaticMap[ticker];
var sb = new SeriesBuilder<DateTime, double>();
foreach (DateTime dte in srs.Keys)
{
// Skip if we're missing data somewhere
if (!srs.ContainsKey(dte))
continue;
// Proceed.
DateTime settle = BondAnalytics.NextBusDay(dte.Date.AddDays(config.bondSettleLag), hols);
double price = srs.TryGetAs<double>(dte).OrDefault(double.NaN);
if (double.IsNaN(price)) continue;
double value = BondASWCalc.calcMeasure(measure, price, eCountry,
dte, settle, null, null,
bndStatic, config, hols);
if (double.IsNaN(value)) continue;
sb.Add(dte, value);
}
valueMap[ticker] = sb.Series;
Console.WriteLine("Completed for {0}", ticker);
valueMap[ticker].Print();
}
var df = Frame.FromColumns(valueMap);
output[ measure] = df;
Console.WriteLine();
Console.WriteLine("Bond Measures complete.");
}
// SAVE OUTPUT.
Console.WriteLine("Saving CSV Files");
var renameMap = new Dictionary<string, string>()
{
{"LTrueSpread", "LOAS"},
{"OISTrueSpread", "OISOAS"},
{"LSpread", "LASW"},
{"OISSpread", "OISASW"}
};
foreach (var kvp in output)
{
var measure = kvp.Key;
var df = convertFrameToString(kvp.Value);
if (renameMap.ContainsKey(measure))
{
measure = renameMap[measure];
}
var fileName = outputFolderPath + "\\" + eCountry.ToString()+ "_" + measure + ".csv";
df = df.FillMissing("");
df.SaveCsv(fileName,includeRowKeys:true);
Console.WriteLine("Saved {0}", fileName);
}
}
//----------------------------------------------------------------------------------------
public static Frame<DateTime, string> SymBondFwdTimeSeries( string[] BondId,
DateTime StartDate,
DateTime EndDate,
String TenorString,
DateTime FwdDate,
object[,] SpreadOrPrice,
String Measure,
Boolean bOIS,
Boolean bSpread,
List<DateTime> Holidays)
//----------------------------------------------------------------------------------------
{
bool bHaveEverything = true;
CarbonClient cbnClient = DataHandler.GetCarbonClient();
try
{
int nBonds = BondId.GetLength(0);
// Holidays can be given as holiday code (string) or as array of dates.
// Get Bond static data
#region Bond Static Data
object o = null;
DateTime[] maturity = new DateTime[nBonds],
effectiveDate = new DateTime[nBonds],
firstCpnDate = new DateTime[nBonds];
double[] coupon = new double[nBonds];
bool bEUR = false;
long bondCpnFreq = 6;
BondAnalytics.Country eCountry = BondAnalytics.Country.US, prevCountry = BondAnalytics.Country.US;
for (int i = 0; i < nBonds; i++)
{
var lid = ((String) BondId[i]).Trim().ToLower();
BondStatic bndStatic = DataHandler.GetBondStatic(lid);
maturity[i] = DateTime.FromOADate(bndStatic.Maturity);
effectiveDate[i] = DateTime.FromOADate(bndStatic.EffectiveDate);
firstCpnDate[i] = DateTime.FromOADate(bndStatic.FirstCoupon);
coupon[i] = bndStatic.Coupon;
BondAnalytics.CountryMappings.TryGetValue(((String) BondId[i]).Substring(0, 2), out eCountry);
if (i > 0 && eCountry != prevCountry)
{
throw new ArgumentException("#Error! All bonds must be same country!");
}
prevCountry = eCountry;
}
// Let's use spreadseries config
string sCountry = ((String) BondId[0]).Substring(0, 2); // since all countries are the same
var fcstConfig = new SpreadTimeSeriesConfigs(sCountry, bForecastCurve: true);
var discConfig = new SpreadTimeSeriesConfigs(sCountry, bForecastCurve: false);
bEUR = (eCountry == BondAnalytics.Country.DE || eCountry == BondAnalytics.Country.FR ||
eCountry == BondAnalytics.Country.IT || eCountry == BondAnalytics.Country.ES ||
eCountry == BondAnalytics.Country.BE
? true
: false);
bondCpnFreq = fcstConfig.bndCouponFreq;
//(eCountry == BondAnalytics.Country.US || eCountry == BondAnalytics.Country.UK || eCountry == BondAnalytics.Country.JP || eCountry == BondAnalytics.Country.CA ? 6 : 12);
#endregion
// Build timeseries calendar
#region DateSeries
List<DateTime> dateSeries = new List<DateTime>();
// Ensure start and end date are good business days. Move dates forward then back to minimise chance of future dates in series.
DateTime startDate = StartDate.AddTenor(Tenor.FromString("1b"), "")
.AddTenor(Tenor.FromString("-1b"), "");
DateTime nextDate = EndDate.AddTenor(Tenor.FromString("1b"), "").AddTenor(Tenor.FromString("-1b"), "");
if (startDate > nextDate)
{
throw new ArgumentException("nextDate must be after start date");
}
var tenor = Tenor.FromString("-" + TenorString);
while (nextDate >= startDate)
{
dateSeries.Add(nextDate);
nextDate = BondAnalytics.PrevBusDay(nextDate.AddTenor(tenor, ""), Holidays);
}
int M = dateSeries.Count();
// Keep a track of pricing errors so that a failure for one bond / date does not have to cause a failure of the whole function.
Boolean[,] bPricingCheck = new Boolean[M, nBonds];
#endregion
// Need discount curve before calculating repo series.
#region InterestCurves
String discCurveName = discConfig.SymDiscountCurveName;
String fcstCurveName = fcstConfig.SymForecastCurveName;
String ccy = fcstConfig.ccy;
// Convert DiscountCurve Object to tuples.
var DiscCurves = new SortedDictionary<DateTime, Tuple<DateTime, double>[]>();
var FcstCurves = new SortedDictionary<DateTime, Tuple<DateTime, double>[]>();
// DISC.
SortedDictionary<DateTime, DiscountCurve> DiscCurvesRaw =
DataHandler.getDiscountCurvesInDateRange(discCurveName, dateSeries);
foreach (var dte in DiscCurvesRaw.Keys)
{
DiscCurves[dte] = DiscCurvesRaw[dte].AsTuples();
}
// FCST.
if (!bOIS && (Measure != "Repo" && Measure != "Yield" && Measure != "Price")) // only get fcst curves here
{
SortedDictionary<DateTime, DiscountCurve> FcstCurvesRaw =
DataHandler.getDiscountCurvesInDateRange(fcstCurveName, dateSeries);
foreach (var dte in FcstCurvesRaw.Keys)
{
FcstCurves[dte] = FcstCurvesRaw[dte].AsTuples();
}
}
#endregion InterestCurves
// Get repo spread. Three cases: default is 20 day moving average of GC-OIS, alternatives are input flat rate or interpolation from input table.
#region RepoHistory
double[,] fwdPrices = new double[M, nBonds];
double[,] repoSpread = new double[M, nBonds];
// Two column array - dates / spreads. Linear interpolation in date.
if ((SpreadOrPrice.GetLength(1) == 2 || SpreadOrPrice.GetLength(1) == nBonds + 1) && SpreadOrPrice.GetLength(0) > 1)
{
int nRepos = SpreadOrPrice.GetLength(0);
int nCols = SpreadOrPrice.GetLength(1);
DateTime[] rDates = new DateTime[nRepos];
double[,] rSpreads = new double[nRepos, nBonds];
for (int j = 0; j < nBonds; j++)
{
for (int i = 0; i < nRepos; i++)
{
if (SpreadOrPrice[i, 0] is double)
{
rDates[i] = DateTime.FromOADate((double)SpreadOrPrice[i, 0]).Date;
if (nCols > 1 + j && SpreadOrPrice[i, 1 + j] is double)
rSpreads[i, j] = (double)SpreadOrPrice[i, 1 + j];
else if (1 + j >= nCols)
{
rSpreads[i, j] = rSpreads[i, j - 1];
}
else
{
// Spread override is not a double, pricing fails. Use -1e8 as an error warning.
rSpreads[i, j] = -1e8;
}
}
else
{
rDates[i] = DateTime.MinValue;
}
}
}
// Not very efficient, but need to identify matching dates when not necessarily in order because of input errors.
if (bSpread)
{
int k;
for (int i = 0; i < M; i++)
{
for (k = 0; k < nRepos; k++)
{
if (rDates[k].Date == dateSeries[i].Date)
{
for (int j = 0; j < nBonds; j++)
{
if (rSpreads[i, j] < -1e7)
bPricingCheck[i, j] = true;
else
repoSpread[i, j] = rSpreads[i, j] / 10000.0;
}
break;
}
}
if (k == nRepos)
{
for (int j = 0; j < nBonds; j++)
{
bPricingCheck[i, j] = true;
}
}
}
}
else
{
int k;
for (int i = 0; i < M; i++)
{
for (k = 0; k < nRepos; k++)
{
if (rDates[k].Date == dateSeries[i].Date)
{
for (int j = 0; j < nBonds; j++)
{
if (rSpreads[i, j] < -1e7)
bPricingCheck[i, j] = true;
else
fwdPrices[i, j] = rSpreads[i, j];
}
break;
}
}
if (k == nRepos)
{
for (int j = 0; j < nBonds; j++)
{
bPricingCheck[i, j] = true;
}
}
}
}
}
// Cases: Single entry, double then this is the flat spread.
else if (SpreadOrPrice.GetLength(0) == 1 && SpreadOrPrice.GetLength(1) == 1 && SpreadOrPrice[0, 0] is double)
{
if (bSpread)
{
for (int j = 0; j < nBonds; j++)
{
for (int i = 0; i < M; i++)
{
repoSpread[i, j] = (double)SpreadOrPrice[0, 0] / 10000.0;
}
}
}
else
{
for (int j = 0; j < nBonds; j++)
{
for (int i = 0; i < M; i++)
{
fwdPrices[i, j] = (double)SpreadOrPrice[0, 0];
}
}
}
}
// Vector, length nBonds
else if (((SpreadOrPrice.GetLength(0) == nBonds && SpreadOrPrice.GetLength(1) == 1) || (SpreadOrPrice.GetLength(1) == nBonds && SpreadOrPrice.GetLength(0) == 1)) && SpreadOrPrice[0, 0] is double)
{
bool bRow = SpreadOrPrice.GetLength(1) == 1;
if (bSpread)
{
for (int j = 0; j < nBonds; j++)
{
for (int i = 0; i < M; i++)
repoSpread[i, j] = (double)(bRow ? SpreadOrPrice[j, 0] : SpreadOrPrice[0, j]) / 10000.0;
}
}
else
{
for (int j = 0; j < nBonds; j++)
{
for (int i = 0; i < M; i++)
fwdPrices[i, j] = (double)(bRow ? SpreadOrPrice[j, 0] : SpreadOrPrice[0, j]);
}
}
}
// Otherwise (empty, or not a double or a two column array).
else
{
bSpread = true;
// Repo spread is 20 day moving average of GC - OIS
double[] gcHist = new double[M + 19];
double[] oisHist = new double[M + 19];
var opGC = DataHandler.GetRepoRates(eCountry, dateSeries);
for (int i = 0; i < M; i++)
{
// Copy GC values over (we need to extend this later)
gcHist[i] = opGC[i];
if (DiscCurves.ContainsKey(dateSeries[i]))
{
oisHist[i] = Math.Round((1.0 / DiscCurves[dateSeries[i]].ElementAt(1).Item2 - 1.0) * 360.0 / (DiscCurves[dateSeries[i]].ElementAt(1).Item1 - DiscCurves[dateSeries[i]].ElementAt(0).Item1).TotalDays, 5);
}
else
{
bHaveEverything = false;
}
}
// Dates prior to dateSeries[0]
List<DateTime> prevDates = new List<DateTime>();
for (int i = 0; i < 19; i++)
{
prevDates.Add(BondAnalytics.PrevBusDay((i == 0 ? dateSeries[M - 1] : prevDates.Last()).AddDays(-1), Holidays));
}
// Get old curves
var PrevDiscCurves = new SortedDictionary<DateTime, Tuple<DateTime, double>[]>();
var prevDiscCurvesRaw = DataHandler.getDiscountCurvesInDateRange(discCurveName, prevDates);
foreach (var kvp in prevDiscCurvesRaw)
{
PrevDiscCurves[kvp.Key] = kvp.Value.AsTuples();
}
// Get old rates
double[] oldRepoRates = DataHandler.GetRepoRates(eCountry, prevDates);
for (int i = 0; i < 19; i++)
{
o = oldRepoRates[i];
if (o is double)
{
gcHist[M + i] = (double)o / 100.0;
}
else if (i > 0)
{
gcHist[M + i] = gcHist[M + i];
}
else
gcHist[M + i] = 0.0;
if (PrevDiscCurves.ContainsKey(prevDates[i]))
{
oisHist[M + i] = Math.Round((1.0 / PrevDiscCurves[prevDates[i]].ElementAt(1).Item2 - 1.0) * 360.0 / (PrevDiscCurves[prevDates[i]].ElementAt(1).Item1 - PrevDiscCurves[prevDates[i]].ElementAt(0).Item1).TotalDays, 5);
}
else
{
bHaveEverything = false;
}
}
for (int i = 0; i < M; i++)
{
for (int j = i; j < i + 20; j++)
{
repoSpread[i, 0] += (gcHist[j] - oisHist[j]) / 20.0;
}
}
for (int i = 0; i < M; i++)
for (int j = 1; j < nBonds; j++)
repoSpread[i, j] = repoSpread[i, 0];
}
#endregion
// Get Price Data (Clean Prices, Repo rates - i.e. term OIS rates + repo spreads)
#region GetPricingData
List<double[]> priceSeries = new List<double[]>();
List<double[]> repoSeries = new List<double[]>();
// Let's get price data from symmetry service
if (bSpread)
{
for (int i = 0; i < M; i++)
{
priceSeries.Add(new double[nBonds]);
repoSeries.Add(new double[nBonds]);
DateTime settleDate = BondAnalytics.NextBusDay(dateSeries[i].Date.AddDays(1), Holidays);
if (eCountry != BondAnalytics.Country.US)
{
settleDate = BondAnalytics.NextBusDay(dateSeries[i].Date.AddDays(2), Holidays);
}
for (int j = 0; j < nBonds; j++)
{
if (settleDate.Date > FwdDate.Date)
{
bPricingCheck[i, j] = true;
continue;
}
var priceMonikerName = ((String)BondId[j]).ToLower().Trim();
var priceMonikerDate = dateSeries[i];
var closeTup = DataHandler.GetCloseSnap(priceMonikerDate, eCountry);
var priceMonikerClose = closeTup.Item1;
var priceMonikerSource = closeTup.Item2;
try
{
o = DataHandler.GetHistoryPrice(priceMonikerName, priceMonikerDate, priceMonikerClose, priceMonikerSource);
}
catch
{
}
if (o is double)
priceSeries[i][j] = (double)o;
else if (dateSeries[i].Date < effectiveDate[j].Date || !DiscCurves.ContainsKey(dateSeries[i]))
{
// No price information available
bPricingCheck[i, j] = true;
}
else
bHaveEverything = false;
}
double oisSwap = 0.0;
if (DiscCurves.ContainsKey(dateSeries[i]))
{
DateTime[] discDfDates = DiscCurves[dateSeries[i]].Select(x => x.Item1).ToArray();
double[] discDfs = DiscCurves[dateSeries[i]].Select(x => x.Item2).ToArray();
oisSwap = settleDate.Date < FwdDate.Date ? BondAnalytics.CalcMMS(settleDate, FwdDate, BondAnalytics.DayCountType.Act360, 12, 12, discDfDates, discDfs, discDfDates, discDfs, Holidays, null, null, null, null, null, DateTime.MinValue, 5) : 0.0;
}
for (int j = 0; j < nBonds; j++)
{
repoSeries[i][j] = oisSwap + repoSpread[i, j];
}
}
}
#endregion
// Calculate measures
#region CalcMeasures
List<double[]> outputSeries = new List<double[]>();
for (int i = 0; i < M; i++)
{
outputSeries.Add(new double[nBonds + (Measure == "Repo" ? 1 : 0)]);
DateTime settleDate = BondAnalytics.NextBusDay(dateSeries[i].Date.AddDays(1), Holidays);
if (eCountry != BondAnalytics.Country.US)
{
settleDate = BondAnalytics.NextBusDay(dateSeries[i].Date.AddDays(2), Holidays);
}
for (int j = 0; j < nBonds; j++)
{
// Only attempt to price if there is a chance of success!
if (bPricingCheck[i, j])
continue;
double fwdPrice = bSpread ? BondAnalytics.CalcBondFwd(eCountry, settleDate, priceSeries[i][j], effectiveDate[j], firstCpnDate[j], maturity[j], coupon[j], bondCpnFreq, FwdDate, repoSeries[i][j])[0] : fwdPrices[i, j];
switch (Measure)
{
case "Price":
{
outputSeries[i][j] = fwdPrice;
break;
}
case "Yield":
{
outputSeries[i][j] = 10000.0 * BondAnalytics.SolveYield(eCountry, FwdDate, fwdPrice, effectiveDate[j], firstCpnDate[j], maturity[j], coupon[j], bondCpnFreq)[0];
break;
}
case "TrueSpread":
{
if ((bOIS && DiscCurves.ContainsKey(dateSeries[i])) || FcstCurves.ContainsKey(dateSeries[i]))
{
DateTime[] dfDates = (bOIS ? DiscCurves[dateSeries[i]].Select(x => x.Item1).ToArray() : FcstCurves[dateSeries[i]].Select(x => x.Item1).ToArray());
double[] dfs = (bOIS ? DiscCurves[dateSeries[i]].Select(x => x.Item2).ToArray() : FcstCurves[dateSeries[i]].Select(x => x.Item2).ToArray());
// Check. Sometimes (holidays) the first date on the curve can be repeated, which will cause an exception in the pricer.
if (dfDates[0] == dfDates[1])
{
dfDates = dfDates.GetSubArray(1, dfDates.GetLength(0)).ToArray();
dfs = dfs.GetSubArray(1, dfs.GetLength(0)).ToArray();
}
outputSeries[i][j] = -10000.0 * BondAnalytics.SolveZSpread(eCountry, FwdDate, fwdPrice, effectiveDate[j], firstCpnDate[j], maturity[j], coupon[j], bondCpnFreq, dfDates, dfs, Holidays);
}
else
{
bPricingCheck[i, j] = true;
}
break;
}
case "MMS":
{
if (DiscCurves.ContainsKey(dateSeries[i]) && (bOIS || FcstCurves.ContainsKey(dateSeries[i])))
{
DateTime[] discDfDates = DiscCurves[dateSeries[i]].Select(x => x.Item1).ToArray();
DateTime[] fcstDfDates = (bOIS ? discDfDates : FcstCurves[dateSeries[i]].Select(x => x.Item1).ToArray());
double[] discDfs = DiscCurves[dateSeries[i]].Select(x => x.Item2).ToArray();
double[] fcstDfs = (bOIS ? discDfs : FcstCurves[dateSeries[i]].Select(x => x.Item2).ToArray());
BondAnalytics.DayCountType dct = (bOIS ? BondAnalytics.DayCountType.Act360 : BondAnalytics.DayCountType.E30360);
int fixedFreq = (bEUR || bOIS ? 12 : 6);
int floatFreq = (bOIS ? 12 : (bEUR ? 6 : 3));
double mms = BondAnalytics.CalcMMS(FwdDate, maturity[j], dct, fixedFreq, floatFreq, discDfDates, discDfs, fcstDfDates, fcstDfs, Holidays, null, null, null, null, null, firstCpnDate[j], (bOIS ? 5 : 0));
outputSeries[i][j] = 10000.0 * mms;
}
else
{
bPricingCheck[i, j] = true;
}
break;
}
case "Spread":
{
if (DiscCurves.ContainsKey(dateSeries[i]) && (bOIS || FcstCurves.ContainsKey(dateSeries[i])))
{
DateTime[] discDfDates = DiscCurves[dateSeries[i]].Select(x => x.Item1).ToArray();
DateTime[] fcstDfDates = (bOIS ? discDfDates : FcstCurves[dateSeries[i]].Select(x => x.Item1).ToArray());
double[] discDfs = DiscCurves[dateSeries[i]].Select(x => x.Item2).ToArray();
double[] fcstDfs = (bOIS ? discDfs : FcstCurves[dateSeries[i]].Select(x => x.Item2).ToArray());
BondAnalytics.DayCountType dct = (bOIS ? BondAnalytics.DayCountType.Act360 : BondAnalytics.DayCountType.E30360);
int fixedFreq = (bEUR || bOIS ? 12 : 6);
int floatFreq = (bOIS ? 12 : (bEUR ? 6 : 3));
double mms = BondAnalytics.CalcMMS(FwdDate, maturity[j], dct, fixedFreq, floatFreq, discDfDates, discDfs, fcstDfDates, fcstDfs, Holidays, null, null, null, null, null, firstCpnDate[j], (bOIS ? 5 : 0));
double yield = BondAnalytics.SolveYield(eCountry, FwdDate, fwdPrice, effectiveDate[j], firstCpnDate[j], maturity[j], coupon[j], bondCpnFreq)[0];
outputSeries[i][j] = 10000.0 * (mms - yield);
}
else
{
bPricingCheck[i, j] = true;
}
break;
}
case "Repo":
{
if (bSpread)
{
outputSeries[i][j] = 10000 * repoSpread[i, j];
if (j == 0)
outputSeries[i][nBonds] = 10000 * (repoSeries[i][0] - repoSpread[i, 0]);
}
break;
}
}
}
}
#endregion
Dictionary<DateTime, Series<string,double>> output = new Dictionary<DateTime, Series<string, double>>();
for (int i = 0; i < M; i++)
{
SeriesBuilder<string, double> sb = new SeriesBuilder<string, double>();
for (int j = 0; j < nBonds + (Measure == "Repo" ? 1 : 0); j++)
{
if (j < nBonds && bPricingCheck[i, j])
{
sb.Add(BondId[j], double.NaN);
}
else
{
if (j >= nBonds)
{
sb.Add("Repo", outputSeries[i][j]);
}
else
{
sb.Add(BondId[j], outputSeries[i][j]);
}
}
}
output[dateSeries[i].Date] = sb.Series;
}
var result = Frame.FromRows(output);
return result;
}
catch (Exception ex)
{
Log_.ErrorFormat("Error in BondASWCalc: {0}", ex.Message);
throw ex;
}
}
public void MakeTimeSeriesFrame()
{
var indexByDate = false;
var random = new Random(10);
var rows = Enumerable.Range(0, 5).Select(i =>
{
var builder = new SeriesBuilder<string, object>
{
{"ticker", "a"},
{"date", DateTime.Today.AddDays(-i)},
{"open", Math.Round(random.NextDouble(), 10)},
{"close", Math.Round(random.NextDouble(), 10)},
{"settle", Math.Round(random.NextDouble(), 10)}
};
return KeyValue.Create(i, builder.Series);
}).ToList();
if (!indexByDate)
{
rows.AddRange(Enumerable.Range(5, 5).Select(i =>
{
var builder = new SeriesBuilder<string, object>
{
{"ticker", "b"},
{"date", DateTime.Today.AddDays(-(i - 5))},
{"open", Math.Round(random.NextDouble(), 10)},
{"close", Math.Round(random.NextDouble(), 10)},
{"settle", Math.Round(random.NextDouble(), 10)}
};
return KeyValue.Create(i, builder.Series);
}));
}
var frame = Frame.FromRows(rows);
if (!indexByDate) frame.Print();
if (indexByDate)
{
var indexed = frame.IndexRows<DateTime>("date").SortRowsByKey();
indexed.Print();
//return indexed;
}
//return frame;
}
public void ShouldMergeFrame()
{
var random = new Random(10);
var rows = Enumerable.Range(0, 5).Select(i =>
{
var builder = new SeriesBuilder<string, object>
{
{"ticker", "ABC" + i + " Comdty"},
{"expiry", DateTime.Today.AddDays(-i)},
{"ccy", "USD"},
{"number", Math.Round(random.NextDouble(), 10)}
};
return KeyValue.Create(i, builder.Series);
}).ToList();
var frame = Frame.FromRows(rows);
var indexed = frame.IndexRows<string>("ticker").SortRowsByKey();
indexed.Print();
var newrows =new[]{KeyValue.Create(7, new SeriesBuilder<string, object>
{
{"ticker", "ABC" + 7 + " Comdty"},
{"expiry", DateTime.Today.AddDays(5)},
{"ccy", "USD"},
{"number", Math.Round(random.NextDouble(), 10)}
}.Series)}.ToList();
var newframe = Frame.FromRows(newrows);
newframe.Print();
var mergedFrame = frame.Merge(newframe);
mergedFrame.Print();
}
// Allows intialisation code reuse
//------------------------------------------------------------------------------------------------
private void initFutureSeriesBuilder(List<string> tickers, List<DateTime> daterange, List<DateTime> holidays,
string futuresStaticMoniker, ICarbonClient client, string carbonEnv = "PRD",
RollMethods rolMthd = RollMethods.FirstNoticeDate,
InterpolationTypes interpType = InterpolationTypes.None,
List<string> contractSortOrder = null ,
Dictionary<string,DateTime> rollmap = null,
SmoothingTypes smoothingType = SmoothingTypes.None)
//------------------------------------------------------------------------------------------------
{
// defaults
dateRange_ = daterange;
startDate_ = daterange.Min();
endDate_ = daterange.Max();
carbonEnv_ = carbonEnv;
rollMethod_ = rolMthd; //RollMethods.LastTradeDate; //
holidays_ = holidays;
futuresStaticMoniker_ = futuresStaticMoniker;
interpType_ = interpType;
smoothingType_ = smoothingType;
if (contractSortOrder == null)
{
contractSortOrder_ = new List<string>();
}
else
{
contractSortOrder_ = contractSortOrder;
}
if (rolMthd == RollMethods.Custom && rollmap == null)
{
throw new ArgumentException("Custom roll dates required for custom rolls!");
}
ContractMonthLookupTable_ = new Dictionary<ContractMonths, string>
{
{ContractMonths.PrevFrontMonth, "PrevFront"},
{ContractMonths.FrontMonth, "Front"},
{ContractMonths.BackMonth, "Back"},
{ContractMonths.SecondBackMonth, "SecondBack"}
};
Tuple<List<Tuple<string, ContractMonths>> /*rootTickers*/, List<string> /*contracts*/> tickerListTuple;
// Get Static
FuturesStatic = getFuturesStatic(tickers, futuresStaticMoniker, client, carbonEnv, out tickerListTuple, startDate_, endDate_, contractSortOrder_);
// Add custom roll if present
if (rollmap != null)
{
var sb = new SeriesBuilder<string,DateTime>();
foreach (var kvp in rollmap)
{
sb.Add(kvp.Key, kvp.Value);
}
FuturesStatic.AddColumn("custom_roll", sb.Series);
}
inputTickers_ = tickers;
contractTickers_ = tickerListTuple.Item2;
RollingContractSpecs_ = tickerListTuple.Item1;
var contracts = FuturesStatic.RowKeys.ToList(); // get all the tickers (in case no tickers supplied
// Get prices
Console.WriteLine("Getting Futures prices");
RawFuturesPrices = getAllFuturesPricesInDateRange(contracts, startDate_, endDate_, client);
RawFuturesPrices.Print();
// Align Futures Prices & interpolate gaps
alignRawPriceFrameWithDateRange(dateRange_);
//interpolateRawPriceFrame(interpType_);
}
public void MakeStaticDataFrame(bool index = false)
{
var random = new Random(10);
var rows = Enumerable.Range(0, 5).Select(i =>
{
var builder = new SeriesBuilder<string, object>
{
{"ticker", "ABC" + i + " Comdty"},
{"expiry", DateTime.Today.AddDays(-i)},
{"ccy", "USD"},
{"number", Math.Round(random.NextDouble(), 10)}
};
return KeyValue.Create(i, builder.Series);
}).ToList();
var frame = Frame.FromRows(rows);
if (!index) frame.Print();
if (index)
{
var indexed = frame.IndexRows<string>("ticker").SortRowsByKey();
indexed.Print();
//return indexed;
}
//return frame;
}
getFuturesStatic(List< string> tickers , string futuresStaticMoniker, ICarbonClient client, string carbonEnv,
out Tuple<List<Tuple<string, ContractMonths>> /*rootTickers*/, List<string> /*contracts*/> tickerSplitTuple,
DateTime? vStartDate = null, DateTime? vEndDate = null, List<string> contractSortOrder = null )
//------------------------------------------------------------------------------------------------
{
// Intialisation
if (tickers == null)
{
tickers = new List<string>();
}
DateTime startDate, endDate;
if (vStartDate == null)
{
startDate = DateTime.MinValue;
}
else
{
startDate = vStartDate.Value;
}
if (vStartDate == null)
{
endDate = DateTime.MaxValue;
}
else
{
endDate = vEndDate.Value;
}
if (contractSortOrder == null)
{
contractSortOrder = new List<string>();
}
string cacheKey = getCacheKey(tickers, futuresStaticMoniker, startDate, endDate, contractSortOrder);
// Divide inputs according to whether they should be used for rolling futures or not
tickerSplitTuple = splitRollingFuturesFromRawTickers(tickers);
List<string> contractTickers = tickerSplitTuple.Item2;
List<Tuple<string, ContractMonths>> rootTickers = tickerSplitTuple.Item1;
// Get Static From cache here if we have it, tickersplittuple has been set.
Frame<string, string> cachedValue;
if (StaticCache_.TryGetValue(cacheKey, out cachedValue)) return cachedValue;
// Get all the roots from list
List< string> rollingFuturesIDs = new List<string>();
foreach (var tup in rootTickers)
{
rollingFuturesIDs.Add(tup.Item1);
}
// Get test sets.
// This allows us to map back names used in futures static moniker, to our original inputs
var bbgTickerMap = processBBGTickerList(contractTickers);
HashSet<string> rollingSet = new HashSet<string>(rollingFuturesIDs);
HashSet<string> tickerSet = new HashSet<string>(bbgTickerMap.Keys.AsEnumerable());
// Processing inputs
Frame<string, string> output;
var rowList = new List<KeyValuePair<int/*id*/, Series<string, object>/*row*/>>();
var df = client.GetDataFrame(futuresStaticMoniker);
var tmp = df.ToList();
var col = tmp[0].Value as List<object>;
var ind = tmp[1].Value as List<object>;
var dat = tmp[2].Value as List<object>;
//Find it's first trade date col
int firstTradeDtCol = 0;
int futDeliveryDtCol = 0;
int typeCol = 0;
for (int j = 0; j < col.Count; ++j)
{
string heading = col[j].ToString();
if (heading == "fut_first_trade_dt")
{
firstTradeDtCol = j;
}
if (heading == "fut_dlv_dt_last")
{
futDeliveryDtCol = j;
}
if(heading == "type")
{
typeCol = j;
}
}
//client.getdat
SeriesBuilder<string> sb;
int i;
for (i = 0; i < ind.Count; ++i)
{
sb = new SeriesBuilder<string>();
var data = dat[i] as List<object>;
var contract = data[typeCol].ToString();
string index = (string)ind[i];
if ((tickerSet.IsEmpty() && rollingSet.IsEmpty())
|| (!tickerSet.Contains(index) && !rollingSet.Contains(contract)))
{
continue;
}
DateTime firstTradeDate = DateTime.ParseExact(data[firstTradeDtCol].ToString(), "%dd/%MM/%yyyy", CultureInfo.InvariantCulture);
DateTime futDeliveryDate = DateTime.ParseExact(data[futDeliveryDtCol].ToString(), "%dd/%MM/%yyyy", CultureInfo.InvariantCulture);
if (rollingSet.Contains(contract) && (futDeliveryDate < startDate || firstTradeDate > endDate))
{
continue;
}
if (bbgTickerMap.ContainsKey(index))
sb.Add("ticker", bbgTickerMap[index]);
else
sb.Add("ticker", index); // ignores the mappings, in the rolling contract case
for (int h = 0; h < col.Count; ++h)
{
sb.Add((string)col[h], data[h]);
}
rowList.Add(KeyValue.Create(i, sb.Series));
}
output = Frame.FromRows(rowList).IndexRows<string>("ticker", keepColumn: true);
if (rootTickers.IsEmpty())
{
output = sortFrameByInput(output, tickers);
}
else
{
output = sortFrameByNaturalOrder(output, contractSortOrder);
}
StaticCache_.TryAdd(cacheKey, output.Clone());
return output;
}
//------------------------------------------------------------------------------------------------
public void setRollingFuturesPrices(string outputfilepath = "")
//------------------------------------------------------------------------------------------------
{
Console.WriteLine("Setting Rolling Futures:");
var clonedPrices = TimeSeriesUtilities.DeepDeedleClone<DateTime, string, double>(RawFuturesPrices);
double offset = 0.0; // how much to shift the entire series by
// Compute the differences instead
if (smoothingType_ != SmoothingTypes.None)
{
clonedPrices = interpolateRawPriceFrame(clonedPrices, InterpolationTypes.FlatForward); // required to calculate differences correctly
clonedPrices = clonedPrices.SortRowsByKey();
clonedPrices = clonedPrices.Diff(1); // assume we do have all the required dates, as per calendar specified
}
// Init
int i = 0;
var rowList = new List<KeyValuePair<int/*id*/, Series<string, object>/*row*/>>();
var contracts = getTypesOfFuturesContracts();
foreach (string contract in contracts)
{
// Get only the statics for a particular type
var contractSubset = FuturesStatic.Where(kvp => kvp.Value.GetAs<string>("type") == contract);
// Start adding to builder
foreach (DateTime dte in dateRange_)
{
var contractsByMonth = getXMonthContracts(dte, contractSubset, rollMethod_); // roll on first notice date
Console.WriteLine("{0}: prevFM = {1}, FM = {2}, BM = {3}, BM2 = {4}", dte.ToShortDateString(), contractsByMonth[ContractMonths.PrevFrontMonth], contractsByMonth[ContractMonths.FrontMonth], contractsByMonth[ContractMonths.BackMonth], contractsByMonth[ContractMonths.SecondBackMonth]);
// Add an entry for each futures contract month
foreach (KeyValuePair<ContractMonths, string/*ticker*/> entry in contractsByMonth)
{
ContractMonths contractMonth = entry.Key;
string ticker = entry.Value;
if (ticker == "")
{
continue;
}
var priceRow = clonedPrices.TryGetRow<double>(dte).ValueOrDefault;
double price = (priceRow == null) ? double.NaN : priceRow.TryGet(ticker).ValueOrDefault;
var staticRow = FuturesStatic.GetRow<string>(ticker);
// All formats here:
if (price != 0.0 || smoothingType_ != SmoothingTypes.None) // so the flat forward bits get picked up
{
var sb = new SeriesBuilder<string>();
sb.Add("date", dte);
sb.Add("deliverydate", staticRow.Get(getRollField(RollMethods.DeliveryDate)));
sb.Add("rolldate", staticRow.Get(getRollField(rollMethod_)));
sb.Add("ticker", ticker);
sb.Add("type", staticRow.Get("type"));
sb.Add("contractmonth", contractMonth.ToString());
sb.Add("price", price);
sb.Add("convfact", staticRow.Get("fut_cnvs_factor"));
sb.Add("ctdisin", staticRow.Get("fut_ctd_isin"));
sb.Add("ctdmaturity", staticRow.Get("maturity"));
sb.Add("ctdcoupon", staticRow.Get("cpn"));
rowList.Add(KeyValue.Create(i, sb.Series));
++i;
}
}
}
}
RollingFuturesData = Frame.FromRows(rowList);
// This needs to be reassembled in a different function
Console.WriteLine("Rolled Series");
RollingFuturesData.Print();
if (outputfilepath != "")
{
RollingFuturesData.SaveCsv("RollingFuturesDump.csv");
clonedPrices.SaveCsv("RawFutures.csv");
}
}
//------------------------------------------------------------------------------------------------
public Frame<DateTime, string> interpolateRawPriceFrame(Frame<DateTime, string> df, InterpolationTypes interpType)
//------------------------------------------------------------------------------------------------
{
int frameLength = df.RowCount;
int columnCount = df.ColumnCount;
var columns = RawFuturesPrices.Columns;
if (interpType == InterpolationTypes.None) return df;
Dictionary<string, Series<DateTime, double>> resList = new Dictionary<string, Series<DateTime, double>>();
foreach (string colName in RawFuturesPrices.ColumnKeys)
{
var srs = columns[colName];
var idx = df.RowKeys.ToList();
double prev = double.NaN;
idx.Sort();
// Find first non-NaN number
List<int> startIndices = new List<int>();
List<int> endIndices = new List<int>();
SeriesBuilder<DateTime, double> sb = new SeriesBuilder<DateTime, double>();
// Forward pass for start idx
int validIdx = -1;
for (int i = 0; i < frameLength; ++i)
{
double current = srs.TryGetAs<double>(idx[i]).OrDefault(double.NaN);
// Interpolate if missing
if (!double.IsNaN(current))
{
validIdx = i;
//continue;
}
startIndices.Add(validIdx);
}
// Backward pass for end idx
validIdx = -1;
for (int i = frameLength-1; i >= 0; --i)
{
double current = srs.TryGetAs<double>(idx[i]).OrDefault(double.NaN);
// Interpolate if missing
if (!double.IsNaN(current))
{
validIdx = i;
//continue;
}
endIndices.Add(validIdx);
}
endIndices.Reverse();
// Run over and do interpolation
for (int i = 0; i < frameLength; ++i)
{
int startidx = startIndices[i];
int endidx = endIndices[i];
double current = double.NaN;
DateTime dte = idx[i];
// Cases for interpolation....
if (startidx == -1)
{
// Just take last price if it's there
if (endidx == -1) continue;
else
{
startidx = endidx;
}
}
if (endidx == -1)
{
endidx = startidx;
}
if (startidx == endidx)
{
current = srs.GetAs<double>(idx[startidx]);
}
else
{
switch (interpType)
{
case InterpolationTypes.Linear:
DateTime startDate = idx[startidx];
DateTime endDate = idx[endidx];
double startValue = srs.GetAs<double>(startDate);
double endValue = srs.GetAs<double>(endDate);
current = startValue +
(endValue - startValue)*
(dte - startDate).TotalDays / (endDate - startDate).TotalDays;
break;
case InterpolationTypes.FlatForward:
current = srs.GetAs<double>(idx[startidx]);
break;
default:
throw new ArgumentException("Illegal interpolation type (we shouldn't be here...)");
}
}
sb.Add(dte, current);
}
resList[colName] = sb.Series;
}
df = Frame.FromColumns(resList);
return df;
}
combineFuturesAndBondStatic(Dictionary<string/*isin*/, TimeSeriesBondStatic> bondstatic,
Frame<string,string> futstatic)
//------------------------------------------------------------------------------------------------
{
bool bDeliverable = (bondstatic.Count != 0); // for non deliverable contracts, we don't feed in any bond static info
var rowList = new List<KeyValuePair<int/*id*/, Series<string, object>/*row*/>>();
int i = 0;
foreach (var kvp in bondstatic)
{
TimeSeriesBondStatic staticRow = kvp.Value;
var sb = new SeriesBuilder<string>();
sb.Add("ISIN", staticRow.ISIN);
sb.Add("Maturity", staticRow.Maturity.ToShortDateString());
sb.Add("EffectiveDate", staticRow.EffectiveDate.ToShortDateString());
sb.Add("FirstCouponDate", staticRow.FirstCouponDate.ToShortDateString());
sb.Add("Coupon", staticRow.Coupon);
rowList.Add(KeyValue.Create(i, sb.Series));
++i;
}
var df = Frame.FromRows(rowList);
var bonddf = df.IsEmpty? null :df.IndexRows<string>("ISIN");
// Change indexing structure to deal with futures
Dictionary<string, Series<string, object>>
outputRows = new Dictionary<string, Series<string, object>>();
foreach (string colkey in futstatic.RowKeys)
{
var row = futstatic.Rows[colkey];
string ctdisin = row.GetAs<string>("fut_ctd_isin");
// Let's check to make sure that all EffectiveDate, FirstCouponDate,Maturity etc are on the table
// and ignore the bond static if so.
DateTime maturity = row.TryGetAs<DateTime>("maturity").OrDefault(DateTime.MinValue);
DateTime effDate = row.TryGetAs<DateTime>("eff_dt").OrDefault(DateTime.MinValue);
DateTime firstCpnDate = row.TryGetAs<DateTime>("first_cpn_dt").OrDefault(DateTime.MinValue);
double coupon = row.TryGetAs<double>("cpn").OrDefault(double.NaN);
if ( (maturity == DateTime.MinValue || effDate == DateTime.MinValue || firstCpnDate == DateTime.MinValue || double.IsNaN(coupon) ) && bDeliverable)
{
var bondstaticSrs = bonddf.Rows[ctdisin];
outputRows[colkey] = row.Merge(bondstaticSrs);
}
else
{
var sb = new SeriesBuilder<string>();
sb.Add("ISIN", ctdisin);
sb.Add("Maturity", (maturity != DateTime.MinValue)? maturity.ToShortDateString():"-");
sb.Add("EffectiveDate", (effDate != DateTime.MinValue)? effDate.Date.ToShortDateString() :"-");
sb.Add("FirstCouponDate", (firstCpnDate != DateTime.MinValue) ? firstCpnDate.ToShortDateString():"-");
sb.Add("Coupon", coupon);
outputRows[colkey] = row.Merge(sb.Series);
}
}
var output = Frame.FromRows(outputRows.AsEnumerable());
//Log_.InfoFormat("Static Combined");
//output.Print();
return output.IndexRows<string>("ticker", keepColumn:true);
}
public void ShouldUpdateFromCarbonFrame()
{
//var moniker = "symapp.roll.ctd.us.ty";
var moniker = "symapp.events.config";
var frame = CarbonModel.RetrieveFrameFromCarbon<int>(moniker).Result;
//frame.Print();
var filteredrows = frame.Rows.Where(r => r.Key != 180);
var filteredframe = Frame.FromRows(filteredrows);
//filteredframe.Print();
//var isins = frame.GetColumn<string>("ticker");
//var truespread = frame.GetColumn<double>("truespread");
var builder = new SeriesBuilder<string, object>
{
{"Index", 180},
{"EventID", 181},
{"EventCode", "US Midterm"},
{"Country", "US"},
{"Type", "ECO"},
{"HasTimeElement", "FALSE"},
{"BbgTicker", ""},
{"RetrievalType", 4},
{"Image", null},
};
var keyvalue = KeyValue.Create(180, builder.Series);
var newrows = new[] { keyvalue }.ToList();
var newframe = Frame.FromRows(newrows);
var indexed = newframe.IndexRows<int>("Index").SortRowsByKey();
//indexed.Print();
var joinframe = filteredframe.Merge(indexed);
joinframe.Print();
var result = CarbonModel.SaveFrameToCarbon(moniker, joinframe).Result;
}
private static Frame<string, string> convertFrameToString(Frame<DateTime, string> input )
{
var outputMap = new Dictionary<string, Series<string, double>>();
foreach (DateTime dte in input.RowKeys)
{
var row = input.GetRow<double>(dte);
var sb = new SeriesBuilder<string, double>();
foreach (string id in row.Keys)
{
var value = row.TryGet(id).OrDefault(double.NaN);
sb.Add(id, value);
}
outputMap[dte.ToString("yyyy-MM-dd")] = sb.Series;
}
return Frame.FromRows(outputMap);
}
//------------------------------------------------------------------------------------------------
public static Frame<string, string> sortFrameByInput(Frame<string, string> input, List<string> tickers)
//------------------------------------------------------------------------------------------------
{
Frame<string, string> output = input;
// Make a colums to sort on
var sb = new SeriesBuilder<string>();
int i = 0;
foreach (string ticker in tickers)
{
sb.Add(ticker, i.ToString());
++i;
}
// Format Output
output.AddColumn("SortOrder", sb.Series);
var sortDF = output.IndexRows<int>("SortOrder").SortRowsByKey();
//sortDF.Print();
output = sortDF.IndexRows<string>("ticker", keepColumn: true);
//output.Print();
return output;
}
//----------------------------------------------------------------------------------------
public static Frame<DateTime, String> GenerateRollingBondForwardSeries(
List<string> tickers,
DateTime start,
DateTime end,
string tenor,
string countryCode,
string measure,
string holidayCode,
bool bOIS = true,
int optExDivDays = 0,
bool bSpreadOrRepo = true,
List<double> reposOrSpreads = null,
ICarbonClient client = null)
//----------------------------------------------------------------------------------------
{
// Checks
if (client == null)
{
throw new ArgumentException("No Carbon Client!");
}
#region cache
string all_tickers = ";";
foreach (var s in tickers) all_tickers += s;
string allRepoOis = "";
if (reposOrSpreads != null)
foreach (var s in reposOrSpreads) allRepoOis+= s;
string cacheKey = "GenerateRollingBondForwardSeries" + tickers.ToString() + start.ToString() + end.ToString() +
tenor + countryCode + measure + holidayCode + bOIS.ToString()
+ optExDivDays.ToString() + bSpreadOrRepo.ToString() + allRepoOis;
if (TimeSeriesCache.ContainsKey(cacheKey))
{
return TimeSeriesCache[cacheKey];
}
#endregion
#region Checks
if (start >end) throw new ArgumentException("#Error: start date cannot be after end date");
#endregion
var settings = new List<Tuple<string/*moniker*/, string/*tickercol*/, string/*pricecol*/>>();
// Get Configs
foreach (var ticker in tickers)
{
if (ticker.Contains('.')) // it's a CTD series!
{
settings.Add(MonikerConfigs.getCTDMoniker(countryCode,ticker,client));
}
else if (ticker.ToLower().Contains("ct"))
{
settings.Add(MonikerConfigs.getCTMoniker(countryCode, ticker, client));
}
else
{
throw new ArgumentException("#Error: Invalid ticker found - " + ticker);
}
}
#region DATA
bool bHaveEverything = true;
var configs = new SpreadTimeSeriesConfigs(countryCode, !bOIS);
var spreadConventions = new SpreadConventions(configs);
var dateRange = TimeSeriesUtilities.GetAllDatesInDateime(start, end, configs.calendarcode, client, "1b"); // Force daily sampling
var fwdDates = dateRange.Select(dte => dte.AddTenor(Tenor.FromString(tenor))).ToList();
var holidays = TimeSeriesUtilities.GetHolidays(holidayCode,client);
// Parcel input repo rates or spreads
Series<DateTime, double> inputSrs = null;
if (reposOrSpreads != null )
if (reposOrSpreads.Count != dateRange.Count && reposOrSpreads.Count != 1)
{
throw new ArgumentException("#Error: number of spreads or rates do not much the days in range");
}
else
{
var sb = new SeriesBuilder<DateTime, double>();
double scale_factor = bSpreadOrRepo ? 10000 : 1;
for (int i = 0; i < dateRange.Count; ++i)
{
if (reposOrSpreads.Count == 1)
{
sb.Add(dateRange[i], reposOrSpreads[0]/scale_factor);
}
else
{
sb.Add(dateRange[i], reposOrSpreads[i]/scale_factor);
}
}
inputSrs = sb.Series;
}
// Swap Curves
var curves = TimeSeriesUtilities.GetCurvesFromCarbon(dateRange, configs.ccy, client);
if (curves == null) bHaveEverything = false;
// Prices and tickers from ad-hoc persistence
var frames = new Dictionary<string, Frame<DateTime, string>>();
for (int i = 0; i < tickers.Count; ++i)
{
var setting = settings[i];
var ticker = tickers[i];
var moniker = setting.Item1;
var tickercol = setting.Item2;
var pricecol = setting.Item3;
var tmp_df = client.GetFrameAsync<DateTime>(moniker, "AdHocData").Result;
if (tmp_df == null)
{
bHaveEverything = false;
}
else
{
// Extract price and ticker cols only
var colsOfInterest = new Dictionary<string, Series<DateTime,object>>();
// Filter data to contain date range
tmp_df = tmp_df.Where(kvp => (kvp.Key >= start) && (kvp.Key <= end));
colsOfInterest["price"] = tmp_df.GetColumn<object>(pricecol);
colsOfInterest["isin"] = tmp_df.GetColumn<object>(tickercol);
frames[ticker] = Frame.FromColumns(colsOfInterest);
}
}
//Bond static
var bondIsinSet = new HashSet<string>();
foreach(var kvp in frames)
{
var ticker = kvp.Key;
var data = kvp.Value;
var isins = data.GetColumn<string>("isin").Values;
//isins.Select(isin => bondIsinSet.Add(isin));
foreach (string isin in isins)
{
bondIsinSet.Add(isin);
}
}
var bondStatics = client.BulkGetBondStaticData( bondIsinSet.ToList());
if (curves == null || bondStatics == null ) return null;
#endregion
#region RepoRegion
Series<DateTime, double> repoSrs = null;
if (reposOrSpreads == null)
{
// Load default repo-OIS Spread
double[,] repoRates =
client.GetRepoRateFromSavedRepoOisSpread(BondAnalytics.CountryMappings[countryCode], dateRange,
fwdDates, holidays);
// Break out if no data
if (repoRates == null) return null;
var repoSrsBlder = new SeriesBuilder<DateTime, double>();
for (int i = 0; i < dateRange.Count; ++i)
{
repoSrsBlder.Add(dateRange[i], repoRates[i, 1]);
}
repoSrs = repoSrsBlder.Series;
}
else
{
// User-defined repo-OIS Spread or repo rates
if (bSpreadOrRepo)
{
//Inputs are spreads
double[,] repoRates =
client.CalcRepoRatesFromOisSpread(inputSrs, BondAnalytics.CountryMappings[countryCode], dateRange, fwdDates, holidays);
// Break out if no data
if (repoRates == null) return null;
var repoSrsBlder = new SeriesBuilder<DateTime, double>();
for (int i = 0; i < dateRange.Count; ++i)
{
repoSrsBlder.Add(dateRange[i], repoRates[i, 1]);
}
repoSrs = repoSrsBlder.Series;
}
else
{
//Inputs are repo rates
repoSrs = inputSrs;
}
}
#endregion
// Calculation loop
var forecastCurves = curves.Item1;
var discountCurves = curves.Item2;
// Overwrite forecast curve if we want OIS
if (bOIS)
{
forecastCurves = discountCurves;
}
var outputs = new Dictionary<string, Series<DateTime, double>> ();
foreach (string ticker in tickers)
{
var sb = new SeriesBuilder<DateTime, double>();
var df = frames[ticker];
var idx = df.RowKeys;
var isins = df.GetColumn<string>("isin");
var prices = df.GetColumn<double>("price");
foreach (var dte in idx)
{
var tryisin = isins.TryGet(dte);
var tryprice = prices.TryGet(dte);
var tryrepo = repoSrs.TryGet(dte);
if (!tryprice.HasValue || !tryisin.HasValue || !tryrepo.HasValue)
continue;
var isin = tryisin.Value;
var price = tryprice.Value;
var repo = tryrepo.Value;
var fwddate = dte.AddTenor(Tenor.FromString(tenor));
var bondstatic = bondStatics[isin];
double value = double.NaN;
if (measure == "Repo")
{
value = repo;
}
else
{
var fwdprice = BondAnalytics.CalcBondFwd(configs.country, dte, price, bondstatic.EffectiveDate,
bondstatic.FirstCouponDate, bondstatic.MaturityDate, bondstatic.Coupon, bondstatic.CpnFreq,
fwddate, repo, optExDivDays)[0];
if (measure == "Price")
{
value = fwdprice;
}
else
{
value = MeasureCalculator.calcFwdBondMeasure(measure, bondstatic, fwdprice, dte, fwddate,
spreadConventions, forecastCurves, discountCurves, holidays);
}
}
sb.Add(dte,value);
}
outputs[ticker] = sb.Series;
}
var consolidatedMeasures = Frame.FromColumns(outputs);
// Cache results
TimeSeriesCache[cacheKey] = consolidatedMeasures;
return consolidatedMeasures;
}
//----------------------------------------------------------------------------------------
public async static Task<Frame<DateTime, string>> GetBondPricesAsync(List<string> bonds, List<DateTime> dates,
BondAnalytics.Country eCountry )
//----------------------------------------------------------------------------------------
{
var opMap = new ConcurrentDictionary<DateTime, Series<string, double>>();
// Let's trigger all the data to start coming back
ConcurrentDictionary<string, Task<GenericPrice>> asyncTable = new ConcurrentDictionary<string, Task<GenericPrice>>();
List<Tuple<DateTime, string>> dateBondList = new List<Tuple<DateTime, string>>();
foreach (var dte in dates)
foreach (string bond in bonds)
{
dateBondList.Add(new Tuple<DateTime, string>(dte, bond));
}
Parallel.ForEach(dateBondList, item =>
{
DateTime dte = item.Item1;
string bond = item.Item2;
Console.WriteLine("Triggering async for {0}", bond + "_" + dte.ToShortDateString());
var closeSnapTup = GetCloseSnap(dte, eCountry);
string closeSnap = closeSnapTup.Item1;
string source = closeSnapTup.Item2;
asyncTable[bond + "_" + dte.ToShortDateString()] = GetHistoryPriceAsync(bond, dte, closeSnap, source);
});
// Wait for 10 secs to do its thing
Thread.Sleep(PauseDuration);
foreach(var dte in dates)
{
Console.WriteLine("Getting prices on {0}", dte.ToShortDateString());
var sb = new SeriesBuilder<string, double>();
foreach (var bond in bonds)
{
GenericPrice res = await asyncTable[bond + "_" + dte.ToShortDateString()];
if (res == null)
{
sb.Add(bond, double.NaN);
}
else
sb.Add(bond, res.Price);
}
opMap[dte] = sb.Series;
}
var df = Frame.FromRows(opMap);
df = df.SortRowsByKey();
return df;
}
