public static ConstructGen<PairTrade> CalculateWeights(ComID[] commodities_)
{
// calculate the weighting
var con = new ConstructGen<PairTrade>(commodities_.Select(x => x.Name).ToArray());
for (int i = 0; i < con.ArrayLength; ++i)
con.SetColumnValues(i, CalculateWeights(commodities_[i]));
if (con.NeedsToSortKeys())
con.SortKeys();
return con;
}
private void btnCombinePnl_Click(object sender, EventArgs e)
{
var all = spreadWeightGeneratorCollectionGrid1.ListOfGenerators;
if (all.Count() == 0) return;
ConstructGen<double> con = new ConstructGen<double>(all.Count());
con.ColumnHeadings = new string[con.ArrayLength];
for (int i = 0; i < con.ArrayLength; ++i)
{
var item = all.ElementAt(i);
con.ColumnHeadings[i] = item.ToString();
con.SetColumnValues(i, item.GetSimplePnl());
}
if (con.NeedsToSortKeys())
con.SortKeys();
var eval = new ReturnsEval.DataSeriesEvaluator("Combined", ReturnsEval.DataSeriesType.Returns);
eval.AddInnerSeries(con.Dates.ToArray(), con.ToArray(), con.ColumnHeadings);
eval.Display("Combined");
}
internal static void Go()
{
var listOfTenors = new[]
{
"1Y",
"2Y",
"3Y",
"4Y",
"5Y",
"6Y",
"7Y",
"8Y",
"9Y",
"10Y",
"11Y",
"12Y",
"15Y",
"20Y",
"25Y",
"30Y",
"40Y",
"1Y1Y",
"1Y2Y",
"1Y3Y",
"1Y4Y",
"1Y5Y",
"1Y6Y",
"1Y7Y",
"1Y8Y",
"1Y9Y",
"1Y10Y",
"1Y11Y",
"1Y12Y",
"1Y15Y",
"1Y20Y",
"1Y25Y",
"1Y30Y",
"1Y40Y",
"2Y1Y",
"2Y2Y",
"2Y3Y",
"2Y4Y",
"2Y5Y",
"2Y6Y",
"2Y7Y",
"2Y8Y",
"2Y9Y",
"2Y10Y",
"2Y11Y",
"2Y12Y",
"2Y15Y",
"2Y20Y",
"2Y25Y",
"2Y30Y",
"2Y40Y",
"3Y1Y",
"3Y2Y",
"3Y3Y",
"3Y4Y",
"3Y5Y",
"3Y6Y",
"3Y7Y",
"3Y8Y",
"3Y9Y",
"3Y10Y",
"3Y11Y",
"3Y12Y",
"3Y15Y",
"3Y20Y",
"3Y25Y",
"3Y30Y",
"3Y40Y",
"4Y1Y",
"4Y2Y",
"4Y3Y",
"4Y4Y",
"4Y5Y",
"4Y6Y",
"4Y7Y",
"4Y8Y",
"4Y9Y",
"4Y10Y",
"4Y12Y",
"4Y15Y",
"4Y20Y",
"4Y25Y",
"4Y30Y",
"4Y40Y",
"5Y1Y",
"5Y2Y",
"5Y3Y",
"5Y4Y",
"5Y5Y",
"5Y6Y",
"5Y7Y",
"5Y8Y",
"5Y9Y",
"5Y10Y",
"5Y12Y",
"5Y15Y",
"5Y20Y",
"5Y25Y",
"5Y30Y",
"5Y40Y",
"6Y1Y",
"6Y2Y",
"6Y3Y",
"6Y4Y",
"6Y5Y",
"6Y6Y",
"6Y7Y",
"6Y8Y",
"6Y9Y",
"6Y10Y",
"6Y12Y",
"6Y15Y",
"6Y20Y",
"6Y25Y",
"6Y30Y",
"7Y1Y",
"7Y2Y",
"7Y3Y",
"7Y4Y",
"7Y5Y",
"7Y6Y",
"7Y7Y",
"7Y8Y",
"7Y9Y",
"7Y10Y",
"7Y11Y",
"7Y12Y",
"7Y15Y",
"7Y20Y",
"7Y25Y",
"7Y30Y",
"8Y1Y",
"8Y2Y",
"8Y3Y",
"8Y4Y",
"8Y5Y",
"8Y6Y",
"8Y7Y",
"8Y8Y",
"8Y9Y",
"8Y10Y",
"8Y12Y",
"8Y15Y",
"8Y20Y",
"8Y25Y",
"8Y30Y",
"9Y1Y",
"9Y2Y",
"9Y3Y",
"9Y4Y",
"9Y5Y",
"9Y6Y",
"9Y7Y",
"9Y8Y",
"9Y9Y",
"9Y10Y",
"9Y11Y",
"9Y12Y",
"9Y15Y",
"9Y20Y",
"9Y25Y",
"9Y30Y",
"10Y1Y",
"10Y2Y",
"10Y3Y",
"10Y4Y",
"10Y5Y",
"10Y10Y",
"10Y15Y",
"10Y20Y",
"10Y30Y",
"11Y1Y",
"11Y2Y",
"11Y3Y",
"11Y4Y",
"11Y5Y",
"11Y6Y",
"11Y7Y",
"11Y10Y",
"11Y15Y",
"11Y20Y",
"11Y25Y",
"11Y30Y",
"12Y1Y",
"12Y2Y",
"12Y3Y",
"12Y4Y",
"12Y5Y",
"12Y8Y",
"12Y10Y",
"12Y20Y",
"12Y25Y",
"15Y1Y",
"15Y2Y",
"15Y3Y",
"15Y4Y",
"15Y5Y",
"15Y6Y",
"15Y7Y",
"15Y8Y",
"15Y9Y",
"15Y10Y",
"15Y15Y",
"20Y5Y",
"20Y10Y",
"25Y5Y",
"30Y10Y",
};
ProductList.RegisterMongoClasses();
foreach (var curve in new[] { CurveNames.USD3M, CurveNames.EUR6M} )
{
var con = new ConstructGen<double>(listOfTenors);
for (int i = 0; i < listOfTenors.Length; ++i)
{
var split = listOfTenors[i].Trim('Y').Split('Y');
int start=0, foward=0;
if (split.Length == 1)
{
foward = int.Parse(split[0]);
}
else
{
start = int.Parse(split[0]);
foward = int.Parse(split[1]);
}
var series = xYyY.Get(curve, start, foward);
con.SetColumnValues(i, series.A_Prices);
}
if (con.NeedsToSortKeys())
con.SortKeys();
con.WriteToCSV(string.Format(@"e:\temp\Mark_Swaps_{0}.csv", curve));
}
}
private void buildData()
{
var pxDates = new List<DateTime>();
var pxValues = new List<double>();
for (int y = 2003;y <= DateTime.Now.Year; ++y)
{
// find the contracts
var conLon = Long.Underlying.Futures.Where(x => x.Expiry.Year-Long.YearOffset == y && x.Expiry.Month == (int)Long.Month).FirstOrDefault();
var conShort = Short.Underlying.Futures.Where(x => x.Expiry.Year-Short.YearOffset == y && x.Expiry.Month == (int)Short.Month).FirstOrDefault();
if (conLon != null && conShort != null)
{
m_contractsLongShort.Add(y, new KeyValuePair<ComFutureMeta, ComFutureMeta>(conLon, conShort));
// last trade of this pair is the earliest lastTrade date of the two
var lastTrade = (conLon.LastTrade < conShort.LastTrade) ? conLon.LastLastDate : conShort.LastLastDate;
var dataStart = lastTrade.AddYears(-1);
if (MyCalendar.IsWeekend(dataStart)) dataStart = MyCalendar.NextWeekDay(dataStart);
ConstructGen<double> con = new ConstructGen<double>(new string[] { conLon.Ticker, conShort.Ticker, "Diff", "Normalized" });
con.SetColumnValues((int)dataColumns.Long, conLon.Prices.GetSubValues(dataStart, lastTrade));
con.SetColumnValues((int)dataColumns.Short, conShort.Prices.GetSubValues(dataStart, lastTrade));
if (con.NeedsToSortKeys()) con.SortKeys();
if (con.Dates.Count == 0)
continue;
// calculate differences
foreach (DateTime date in con.Keys)
{
double[] d = con.GetValues(date);
// if we have a value for both contracts on this day
if (d[(int)dataColumns.Long] != 0d && d[(int)dataColumns.Short] != 0d)
{
// save down the difference
d[(int)dataColumns.Diff] = d[(int)dataColumns.Long] - d[(int)dataColumns.Short];
if (date.Year == y)
{
pxDates.Add(date);
pxValues.Add(d[2]);
}
}
}
// normalize differences
{
DatedDataCollectionGen<double> diffs = con.GetColumnValuesAsDDC((int)dataColumns.Diff);
if (diffs==null || diffs.Length == 0)
continue;
var min = diffs.Data.Min();
var max = diffs.Data.Max();
var normArr = new double[diffs.Length];
for (int i = 0; i < normArr.Length; ++i)
normArr[i] = (diffs.Data[i] - min) / (max - min);
con.SetColumnValues((int)dataColumns.NormalizedDiff, new DatedDataCollectionGen<double>(diffs.Dates, normArr));
}
m_yearToPxs.Add(y, con);
}
}
m_hasBuiltData = true;
}
public void Go()
{
var allweights=new ConstructGen<WeightsLine>(Spreads.Length);
// mark each of the individual spread weight entry/exit points in the construct - could well be different dates per spread...
for(int i=0;i<Spreads.Length;++i)
{
var wts = Spreads[i].GenerateWeights();
foreach (var wt in wts)
{
if (wt.EntryDate <= DateTime.Today) allweights.SetValue(wt.EntryDate, i, wt);
if (wt.ExitDate <= DateTime.Today) allweights.SetValue(wt.ExitDate, i, wt);
}
}
allweights.SortKeys();
// on each date, note the positions that are carried over from an earlier trade on the same day, so that we have a
// full picture of what is in play on that day
WeightsLine[] prev = null;
foreach (var date in allweights.Dates)
{
var todays = allweights.GetValues(date);
if (prev != null)
{
for (int i = 0; i < todays.Length; ++i)
{
if (prev[i] != null && todays[i]==null && date <= prev[i].ExitDate)
todays[i] = prev[i];
}
}
prev = todays;
}
if (allweights.NeedsToSortKeys()) allweights.SortKeys();
// go through each of the dates to generate a covariance and scale the positions
foreach (DateTime date in allweights.Keys)
{
var arr = allweights.GetValues(date);
// build up list of indicies that are live on the current date
var liveIndicies = new List<int>();
for (int i = 0; i < arr.Length; ++i)
if (arr[i] != null && arr[i].ExitDate > date)
liveIndicies.Add(i);
if (!liveIndicies.Any()) continue;
var liveItems = liveIndicies.Select(x => arr[x]).ToArray();
// for all live items form an array of recent returns over of length 'NumDaysForCovariance'
var returns = new double[NumDaysForCovariance, liveIndicies.Count()];
var rawSpreadWeights = new double[liveIndicies.Count()];
for (int i = 0; i < liveIndicies.Count; ++i)
{
var indexReturns = liveItems[i].GetAllSpreadReturns();
// have prices been updated?
if (indexReturns.LastDate < date)
continue;
int indexOfDate = indexReturns.IndexOfElsePrevious(date);
--indexOfDate;
var slice = indexReturns.Data.Slice(indexOfDate - NumDaysForCovariance + 1, NumDaysForCovariance);
rawSpreadWeights[i] = liveItems[i].SpreadWeight/Statistics.Stdev(slice);
returns.SetColumn(i, slice);
}
// buil the covariance
var covar = new CovarianceItem(Utils.CalculateCovariance(returns));
// vol bucketing
var targetvol = liveItems.Length*0.02;
// scale the weights
var newwts = covar.ScaleSeries(rawSpreadWeights, targetvol);
for (int i = 0; i < newwts.Length; ++i)
liveItems[i].AddCombineWeight(date, newwts[i]);
}
}
public static void Test()
{
const int lookback = 5;
const WeightGeneratorType genType = WeightGeneratorType.LongestZScoreOverThreshold;
SpreadWeightGenerator[] arr = new[]
{
new SpreadWeightGenerator(
new WeightGeneratorArgs()
{
Lookback = lookback,
WeightGenerationType = genType,
MinWindowLength = 50,
ZScoreThreshold = 1.3d
},
new SpreadDefinition(
new MonthYearOffset(ComIDs.Sugar, 0, MonthCode.V),
new MonthYearOffset(ComIDs.Sugar, 1, MonthCode.K),
false)),
//new SpreadWeightGenerator(
// new WeightGeneratorArgs()
// {
// Lookback = lookback,
// WeightGenerationType = genType,
// MinWindowLength = 40,
// ZScoreThreshold = 1d
// },
// new SpreadDefinition(
// new MonthYearOffset(ComIDs.Sugar, 0, MonthCode.V),
// new MonthYearOffset(ComIDs.Sugar, 1, MonthCode.K),
// false)),
//new SpreadWeightGenerator(
// new WeightGeneratorArgs()
// {
// Lookback = lookback,
// WeightGenerationType = genType,
// MinWindowLength = 40,
// ZScoreThreshold = 1.5d
// },
// new SpreadDefinition(
// new MonthYearOffset(ComIDs.NatGas, 0, MonthCode.H),
// new MonthYearOffset(ComIDs.NatGas, 0, MonthCode.X),
// false)),
new SpreadWeightGenerator(
new WeightGeneratorArgs()
{
Lookback = lookback,
WeightGenerationType = genType,
MinWindowLength = 60,
ZScoreThreshold = 1.2d
},
new SpreadDefinition(
new MonthYearOffset(ComIDs.NymexGas, 0, MonthCode.H),
new MonthYearOffset(ComIDs.NymexGas, 0, MonthCode.X),
false)),
//new SpreadWeightGenerator(
// new WeightGeneratorArgs()
// {
// Lookback = lookback,
// WeightGenerationType = genType,
// MinWindowLength = 90,
// ZScoreThreshold = 1.7d
// },
// new SpreadDefinition(
// new MonthYearOffset(ComIDs.NatGas, 0, MonthCode.H),
// new MonthYearOffset(ComIDs.NatGas, 0, MonthCode.X),
// false)),
//new SpreadWeightGenerator(
// new WeightGeneratorArgs()
// {
// Lookback = lookback,
// WeightGenerationType = genType,
// MinWindowLength = 40,
// ZScoreThreshold = 1.6d
// },
// new SpreadDefinition(
// new MonthYearOffset(ComIDs.Corn, 0, MonthCode.U),
// new MonthYearOffset(ComIDs.Corn, 0, MonthCode.Z),
// false)),
new SpreadWeightGenerator(
new WeightGeneratorArgs()
{
Lookback = lookback,
WeightGenerationType = genType,
MinWindowLength = 60,
ZScoreThreshold = 1.3d
},
new SpreadDefinition(
new MonthYearOffset(ComIDs.Corn, 0, MonthCode.U),
new MonthYearOffset(ComIDs.Corn, 0, MonthCode.Z),
false)),
//new SpreadWeightGenerator(
// new WeightGeneratorArgs()
// {
// Lookback = lookback,
// WeightGenerationType = genType,
// MinWindowLength = 90,
// ZScoreThreshold = 0.8d
// },
// new SpreadDefinition(
// new MonthYearOffset(ComIDs.Corn, 0, MonthCode.U),
// new MonthYearOffset(ComIDs.Corn, 0, MonthCode.Z),
// false)),
new SpreadWeightGenerator(
new WeightGeneratorArgs()
{
Lookback = lookback,
WeightGenerationType = genType,
MinWindowLength = 40,
ZScoreThreshold = 1.5d
},
new SpreadDefinition(
new MonthYearOffset(ComIDs.Wheat, 0, MonthCode.U),
new MonthYearOffset(ComIDs.Wheat, 0, MonthCode.Z),
false)),
//new SpreadWeightGenerator(
// new WeightGeneratorArgs()
// {
// Lookback = lookback,
// WeightGenerationType = genType,
// MinWindowLength = 50,
// ZScoreThreshold = 1.3d
// },
// new SpreadDefinition(
// new MonthYearOffset(ComIDs.Wheat, 0, MonthCode.U),
// new MonthYearOffset(ComIDs.Wheat, 0, MonthCode.Z),
// false)),
//new SpreadWeightGenerator(
// new WeightGeneratorArgs()
// {
// Lookback = lookback,
// WeightGenerationType = genType,
// MinWindowLength = 70,
// ZScoreThreshold = 1.1d
// },
// new SpreadDefinition(
// new MonthYearOffset(ComIDs.Wheat, 0, MonthCode.U),
// new MonthYearOffset(ComIDs.Wheat, 0, MonthCode.Z),
// false)),
new SpreadWeightGenerator(
new WeightGeneratorArgs()
{
Lookback = lookback,
WeightGenerationType = genType,
MinWindowLength = 50,
ZScoreThreshold = 1.6d
},
new SpreadDefinition(
new MonthYearOffset(ComIDs.RBOB, 0, MonthCode.J),
new MonthYearOffset(ComIDs.RBOB, 0, MonthCode.U),
false)),
};
var comb = new SpreadWeightGeneratorCombiner(arr) {NumDaysForCovariance = 42, TargetVol = 0.06};
comb.Go();
{
var combinedPnl = new ConstructGen<double>(arr.Length);
combinedPnl.ColumnHeadings =
arr.Select(x => string.Format("{0} / {1} / {2}", x.Spread, x.Args.MinWindowLength, x.Args.ZScoreThreshold))
.ToArray();
for (int i = 0; i < arr.Length; ++i)
combinedPnl.SetColumnValues(i, arr[i].GetCombinedPnl());
if (combinedPnl.NeedsToSortKeys())
combinedPnl.SortKeys();
var eval = new ReturnsEval.DataSeriesEvaluator("Combined", ReturnsEval.DataSeriesType.Returns);
eval.AddInnerSeries(combinedPnl.Dates.ToArray(), combinedPnl.ToArray(), combinedPnl.ColumnHeadings);
eval.Display("Combined");
combinedPnl.SumRows().ToCumulative().DisplayLineChart("combined pnl of scaled weights");
}
}
public static ConstructGen<double> GetInvoiceSpreadsAsConstruct()
{
var configs = invoiceSpreadConfigs().ToArray();
var con = new ConstructGen<double>(
configs.Select(x => string.Format("{0}_{1}_{2}", x.Future, x.Series + 1, x.Curve.ToString())).ToArray());
for (int i = 0; i < configs.Length; ++i)
{
var spreads = getInvoiceSpreadsCollectionFromMongo(configs[i]);
con.SetColumnValues(i,
new DatedDataCollectionGen<double>(spreads.Lines.Select(x => x.Date).ToArray(),
spreads.Lines.Select(x => x.InvoiceSpread ?? 0d).ToArray()));
}
if (con.NeedsToSortKeys())
con.SortKeys();
// feed forward missing values
{
var values = con.GetValues(con.Dates[0]);
for (int i = 1; i < con.Dates.Count; ++i)
{
var date = con.Dates[i];
var todayValues = con.GetValues(date);
for (int j = 0; j < todayValues.Length; ++j)
{
if (todayValues[j] == 0d)
todayValues[j] = values[j];
}
values = todayValues;
}
}
return con;
}
public static void Go()
{
var yearOfFile = 2015;
var ricList = new[] {"ED", "FEI", "FGBL", "FGBM", "FGBS", "TY", "FLG", "FSS", "FGB", "FBTP", "FOAT", "ES", "FDX"};
var contractMonths = (MonthCode[])Enum.GetValues(typeof(MonthCode));
var sourceDir = @"E:\futuresData\MarkFiles";
var markDir = @"e:\futuresData\MarkEllis2";
var dict = new SortedDictionary<string, DatedDataCollectionGen<double>>();
var keys = new List<string>();
foreach (var ricStart in ricList)
{
foreach (var contractYear in new[] { 2015, 2016, 2017 })
{
foreach (var contractMonth in contractMonths)
{
var dates = new List<DateTime>();
var values = new List<double>();
foreach (var monthOfFile in new[] { 6, 7, 8 })
{
var searchString = string.Format("*{0}-{1}-{2}{3}{4}.csv", yearOfFile, monthOfFile.ToString("00"), ricStart, contractMonth, contractYear - 2010);
var files = Directory.GetFiles(sourceDir, searchString);
if (files.Length == 0)
continue;
foreach (var item in CsvFile.Read<FuturesIntradaySaver.FuturesLineItem>(files[0]))
{
if (item.gmtDate.HasValue && (item.gmtDate.Value.Minute == 32 || item.gmtDate.Value.Minute == 2))
{
item.Date = TimeZoneHelper.ConvertGmtTimeToLondon(item.gmtDate.Value);
item.Time = string.Empty;
if (item.Date.Hour < 8)
continue;
if (item.Date.Hour > 17)
continue;
dates.Add(item.Date);
values.Add((item.CloseAsk + item.CloseBid) / 2d);
}
}
//list.ToCsv(string.Join("\\", markDir,
// string.Format("{0}-{1}-{2}{3}{4}", yearOfFile, monthOfFile.ToString("00"), UpdateStaticDataForFutures.MAPS[ricStart].Item1, contractMonth, contractYear - 2010)));
}
if(dates.Count>0)
{
try
{
var ticker = string.Format("{0}{1}{2}", UpdateStaticDataForFutures.MAPS[ricStart].Item1, contractMonth, contractYear - 2010);
keys.Add(ticker);
dict.Add(ticker, new DatedDataCollectionGen<double>(dates.ToArray(), values.ToArray()));
}
catch (Exception ex_)
{
Console.WriteLine(ex_.ToString());
}
}
}
}
}
var con = new ConstructGen<double>(keys.ToArray());
for(int i=0;i<keys.Count;++i)
{
con.SetColumnValues(i,dict[keys[i]]);
}
if (con.NeedsToSortKeys())
con.SortKeys();
con.WriteToCSV(@"e:\futuresData\markDataTo1732.csv");
}
protected virtual ConstructGen<double> getData(DateTime startDate_, DateTime endDate_, bool refresh_)
{
ConstructGen<double> con = null;
try
{
var ds = Singleton<ConnMngr>.Instance.Execute(queryDBName, getQueryString(startDate_, endDate_));
if (ConnMngr.HasResult(ds))
{
var list = new List<tempRowExtract>();
foreach (DataRow row in ds.Tables[0].Rows)
{
var id = DataRowHelpers.GetInt(row, idColumn);
var value = DataRowHelpers.GetDouble(row, valueColumn);
var date = DataRowHelpers.GetDate(row, dateColumn);
list.Add(new tempRowExtract(id, date, value));
}
var colMapping = getConstructIndicies(list);
con = new ConstructGen<double>(colMapping.Keys.Select(x=>x.ToString()).ToArray());
foreach (var item in list)
{
con.SetValue(item.Date, colMapping[item.ID], item.Value);
}
if (con.NeedsToSortKeys())
con.SortKeys();
postProcessConstruct(con);
}
}
catch (Exception ex_)
{
Logger.Error(string.Format("Error retrieving data (sql={0}). Error:{1}", getQueryString(startDate_, endDate_), ex_.Message), GetType(), ex_);
}
return con;
}
public ConstructGen<double> AllProductPrices(bool fillInGapsWithPrevious_ = true)
{
// start by generating price series
ConstructGen<double> prices = new ConstructGen<double>(Products.Select(x => x.Name).ToArray());
for (int i = 0; i < Products.Count; ++i)
prices.SetColumnValues(i, Products[i].Prices);
if (prices.NeedsToSortKeys())
prices.SortKeys();
// fill in any missing values (holidays)
if(fillInGapsWithPrevious_)
{
double[] yesterday = null;
foreach (var date in prices.Dates)
{
var today = prices.GetValues(date);
if (yesterday != null)
{
for(int i=0;i<today.Length;++i)
if (today[i] == 0d)
today[i] = yesterday[i];
}
yesterday = today;
}
}
return prices;
}
public static void Compare(string indexStart_, string suffix_)
{
var closePrices = GetData(indexStart_, suffix_);
var closePrices2 = GetData2(indexStart_, suffix_);
var con = new ConstructGen<double>(new string[] { "First", "Second" });
con.SetColumnValues(0, closePrices);
con.SetColumnValues(1, closePrices2);
if (con.NeedsToSortKeys())
con.SortKeys();
con.DisplayInGrid("Compare");
}