public RegressionResult(ILevels independent_, DatedDataCollectionGen<double[]> regDataInput_, NMathStats.LinearRegression lin_, NMathStats.LinearRegressionAnova anova_)
{
m_independent = independent_;
m_lin = lin_;
m_regressionData = regDataInput_;
m_anova = anova_;
}
private DatedDataCollectionGen<double> getSignal(DatedDataCollectionGen<double> prices_, double convention_, int windowLength_, double zScoreThreshold_)
{
var periodReturns = getPeriodReturns(prices_, convention_, windowLength_);
var zScores = getZScores(prices_, convention_, windowLength_, ZScoreIsFromMean);
var dates = periodReturns.Length < zScores.Length ? periodReturns.Dates : zScores.Dates;
var scores = new double[dates.Length];
for (int i = 1; i < dates.Length; ++i)
{
var date = dates[i];
var zScore = zScores.ValueOnDate(date);
var ret = periodReturns.ValueOnDate(date);
var targetScore =
Math.Abs(zScore) < zScoreThreshold_
? 0d
: ret > 0d ? 1d : -1d;
var prevScore = scores[i - 1];
var shift = Math.Min(Math.Abs(targetScore - prevScore), ScoreShift);
if (targetScore < prevScore)
scores[i] = prevScore - shift;
else
scores[i] = prevScore + shift;
}
return new DatedDataCollectionGen<double>(dates, scores);
}
private void btnGen_Click(object sender, EventArgs e)
{
KeyValuePair<ConstructGen<double>, ReturnsEval.DataSeriesEvaluator> kvp = SI.Research.FXStrat.BacktestHelper.getWE(m_calc.Strat);
DatedDataCollectionGen<double> cum = new DatedDataCollectionGen<double>(kvp.Value.All.Dates,kvp.Value.All.CumulativeReturnSeries);
DatedDataCollectionGen<double> indic = m_calc.GetIndicator();
if (indic == null) return;
indic = indic.GetValuesBetweenDates(cum.Dates[0], cum.LastDate);
if (cbAbsIndic.Checked)
indic = indic.ToAbs();
zoneChart1.LineThickness = 2;
zoneChart1.Create(cum, indic, new ZoneDefinition[] {
new ZoneDefinition("under",double.MinValue,m_calc.DisplayThreshold,Color.Tomato),
new ZoneDefinition("over",m_calc.DisplayThreshold,double.MaxValue,Color.LightGreen)
});
if (cbPlotIndic.Checked)
zoneChart1.AddSeries(indic, m_calc.IndicType.ToString(), Color.AliceBlue, 80, "##0.0#");
}
protected ATMVolsRankGroup(FXGroup group_)
{
var currencies = Singleton<FXIDs>.Instance.Where(x => x.IsGroup(group_)).ToArray();
var oneWeek = new ConstructGen<double>(currencies.Select(x => x.Code).ToArray());
var oneMonth = new ConstructGen<double>(currencies.Select(x => x.Code).ToArray());
for (int i = 0; i < currencies.Length; ++i)
{
{
var vols = BbgTalk.HistoryRequester.GetHistory(DataConstants.DATA_START, currencies[i].AtTheMoneyVolTicker_1W, "PX_LAST", false);
oneWeek.SetColumnValues(i, vols);
}
{
var vols = BbgTalk.HistoryRequester.GetHistory(DataConstants.DATA_START, currencies[i].AtTheMoneyVolTicker_1M, "PX_LAST", false);
oneMonth.SetColumnValues(i, vols);
}
}
{
oneWeek.SortKeys();
var avg1W = oneWeek.AvgRows();
ATM_1W_Avg = avg1W;
ATM_1W_o6M = avg1W.ToPercentileRanked(126);
ATM_1W_o1Y = avg1W.ToPercentileRanked(252);
}
{
oneMonth.SortKeys();
var avg1M = oneMonth.AvgRows();
ATM_1M_Avg = avg1M;
ATM_1M_o6M = avg1M.ToPercentileRanked(126);
ATM_1M_o1Y = avg1M.ToPercentileRanked(252);
}
}
protected DatedDataCollectionGen<double> ensureAllWeekdays(DatedDataCollectionGen<double> input_)
{
var dates = new List<DateTime>();
var values = new List<double>();
var date = input_.Dates[0];
while (date <= input_.Dates.Last())
{
int index = input_.IndexOf(date);
if (index == -1)
{
dates.Add(date);
values.Add(double.NaN);
}
else
{
dates.Add(input_.Dates[index]);
values.Add(input_.Data[index]);
}
date = MyCalendar.NextWeekDay(date);
}
return new DatedDataCollectionGen<double>(dates.ToArray(), values.ToArray());
}
public DatedDataCollectionGen<double> GenerateWeightSeries(DatedDataCollectionGen<double> prices_, double convention_, DateTime? minDate_)
{
var periodReturns = getPeriodReturns(prices_, convention_);
var zScores = getZScores(prices_, convention_);
double lastScore = 0d;
var dates = periodReturns.Length < zScores.Length ? periodReturns.Dates : zScores.Dates;
var scores = new double[dates.Length];
for (int i = 1; i < dates.Length; ++i)
{
var date = dates[i];
var zScore = zScores.ValueOnDate(date);
var ret = periodReturns.ValueOnDate(date);
var targetScore =
Math.Abs(zScore) < ZScoreThreshold
? 0d
: ret > 0d ? 1d : -1d;
var prevScore = scores[i - 1];
var shift = Math.Min(Math.Abs(targetScore - prevScore), ScoreShift);
if (targetScore < prevScore)
scores[i] = prevScore - shift;
else
scores[i] = prevScore + shift;
}
return new DatedDataCollectionGen<double>(dates, scores);
}
public override DatedDataCollectionGen<double> GenerateWeightSeries(Backtest.ProductBase product_, DateTime? minDate_)
{
var pxs = product_.Prices;
var mas = new DatedDataCollectionGen<double>[m_windows.Length];
for (int i = 0; i < mas.Length; ++i)
mas[i] = HelperMethods.GetMA(pxs, m_windows[i]);
double[] scores = new double[mas[0].Length];
var dates = mas[0].Dates;
for (int i = 0; i < dates.Length; ++i)
{
var date = dates[i];
var px = pxs.ValueOnDate(date);
for (int m = 0; m < mas.Length; ++m)
{
var maValue = mas[m].ValueOnDate(date);
if (px > maValue)
scores[i] += m_weightings[m_windows[m]];
else
scores[i] -= m_weightings[m_windows[m]];
}
}
return new DatedDataCollectionGen<double>(dates, scores);
}
public DataAroundEvent(DateTime eventDate_, IntradayFuture usedFuture_, DatedDataCollectionGen<double> data_, int numPointsAroundEvent_)
{
EventDate = eventDate_;
Future = usedFuture_;
RawFutureDataAroundEvent = data_;
NumPointsAroundEvent = numPointsAroundEvent_;
}
private static DatedDataCollectionGen<double> unwind_1d(DatedDataCollectionGen<double> input_)
{
var dates = input_.Dates.Slice(0, input_.Length - 1);
var values = input_.Data.Slice(1, input_.Length - 1);
return new DatedDataCollectionGen<double>(dates, values);
}
public DatedDataCollectionGen<double> GetSeries(Periodicity type_)
{
if (m_lastType == type_ && m_cachedSeries != null)
return m_cachedSeries;
m_lastType = type_;
var ds = m_eval.ToPeriod(type_, DoubleCollationType.Sum);
if (m_lag == 0)
{
m_cachedSeries = ds;
return ds;
}
double[] values=new double[ds.Data.Length-m_lag];
DateTime[] dates=new DateTime[ds.Data.Length-m_lag];
for (int i = m_lag; i < ds.Data.Length; ++i)
{
values[i-m_lag]=ds.Data[i-m_lag];
dates[i-m_lag]=ds.Dates[i];
}
m_cachedSeries = ds = new DatedDataCollectionGen<double>(dates, values);
return ds;
}
public void ApplyData(string name_, DateTime[] xAxis_, double[] yAxis_)
{
//seasonalityChartSingle1.AddLineChart(name_, xAxis_, yAxis_);
var dateddata = new DatedDataCollectionGen<double>(xAxis_, yAxis_);
Chart.SetValues(dateddata, dateddata, name_);
updateGrid(name_, xAxis_, yAxis_);
}
public override double GetVol(DateTime date_, VolType type_, int window_ = 252)
{
if(m_histVols==null)
m_histVols = HelperMethods.GetRollingStat(Prices.ToReturns(), window_, Statistics.Stdev);
return m_histVols.ValueOnDate(date_);
}
virtual protected DatedDataCollectionGen<double> getData(DateTime startDate_, DateTime endDate_)
{
DatedDataCollectionGen<double> d = null;
string sql = getQueryString(startDate_, endDate_);
try
{
var ds = Singleton<ConnMngr>.Instance.Execute(getDBName(), getQueryString(startDate_, endDate_));
if (ConnMngr.HasResult(ds))
{
var dates = new List<DateTime>();
var values = new List<double>();
foreach (DataRow row in ds.Tables[0].Rows)
{
var value = Convert.ToDouble(row[valueColumn]);
var date = Convert.ToDateTime(row[dateColumn]);
dates.Add(date);
values.Add(value);
}
d = new DatedDataCollectionGen<double>(dates.ToArray(), values.ToArray());
}
}
catch (Exception ex_)
{
this.LogError(string.Format("Error getting data from database. sql=('{0}')", sql),ex_);
}
return d;
}
public void Create(DatedDataCollectionGen<BarDataPoint> points_)
{
dt.Rows.Clear();
// order of columns is important
// http://help.infragistics.com/Help/NetAdvantage/WinForms/2012.2/CLR4.0/html/Chart_Working_with_Candle_Chart_Data.html
if (dt.Columns.Count == 0)
{
dt.Columns.Add("Date", typeof(DateTime));
dt.Columns.Add("Open", typeof(double));
dt.Columns.Add("High", typeof(double));
dt.Columns.Add("Low", typeof(double));
dt.Columns.Add("Close", typeof(double));
dt.Columns.Add("Volume", typeof(double));
setuptooltip();
}
for (int i = 0; i < points_.Length; ++i)
{
dt.LoadDataRow(new object[] {
points_.Dates[i],
points_.Data[i].Open,
points_.Data[i].High,
points_.Data[i].Low,
points_.Data[i].Close,
points_.Data[i].NumDataPoints
}, true);
}
ultraChart.DataSource = dt;
LookFeel.ProcessChart(this);
ultraChart.DataBind();
}
public CorrelationValue(DatedDataCollectionGen<double> rollingCorrelations_, int windowLength_, CorrelationMeasure displayMeasure_, CorrelationMeasure colorMeasure_ )
{
RollingCorrelations = rollingCorrelations_;
WindowLength = windowLength_;
DisplayMeasure = displayMeasure_;
ColorMeasure = colorMeasure_;
}
public Portfolio(string rootName_, DatedDataCollectionGen<double> allocationTo_, double pnlMultiplier_, Portfolio parent_=null)
{
RootName = rootName_;
AllocationToThis = allocationTo_;
PnlMultiplier = pnlMultiplier_;
m_parentPortfolio = parent_;
}
public static async Task<DatedDataCollectionGen<double>> Get(string name_, CarbonClient cc_, bool force_ = false)
{
if (_cache.ContainsKey(name_) && !force_)
return _cache[name_];
try
{
var con = await DataFrameHelper.GetDataFrameByRow(
name_: string.Format("ssys.trs.{0}",name_),
keyExtractor_: x => (DateTime) x,
valueExtractor_: (dict, vals) => Convert.ToDouble(vals[0]),
cc_: cc_);
var ddc = new DatedDataCollectionGen<double>(con.Keys.ToArray(), con.GetColumnValues(0));
_cache[name_] = ddc;
return ddc;
}
catch (Exception ex_)
{
_cache[name_] = null;
Logger.Error(string.Format("Error getting timeseries for [{0}]", name_), typeof (TimeSeries), ex_);
return null;
}
}
public void Refresh(bool forceReRead_)
{
Logger.Debug(string.Format("CommodsMultiScore.Refresh() : Rebuilding scores from scratch. ForceReRead_=={0}", forceReRead_.ToString()), GetType());
var spotReturns =
Singleton<ComIndexPrices>.Instance.GetData(DataConstants.DATA_START, DateTime.Today, forceReRead_).ToReturns();
var prices = new double[spotReturns.Dates.Count];
for (int i = 0; i < spotReturns.Dates.Count; ++i)
{
var today = spotReturns.Dates[i];
if (today == DateTime.Today) // explicitly don't want 'today's' value as that will be handled by 'live'
break;
var todayValues = spotReturns.GetValues(today);
prices[i] = i == 0
? 100d
: todayValues.Any(x => (Math.Abs(x)) > 1e-07)
? prices[i - 1]*(1d - todayValues.Where(x => Math.Abs(x) > 1e-07).Average())
: prices[i - 1];
}
Pxs = new DatedDataCollectionGen<double>(spotReturns.Dates.ToArray(), prices);
Scores = HelperMethods.CalculateStdevFromMean(Pxs, NumDays, NumDays);
}
public CorrelationAnalyzer(DatedDataCollectionGen<double> series1_, DatedDataCollectionGen<double> series2_, Period p_, int numOfPeriods_)
{
m_series1 = series1_;
m_series2 = series2_;
m_period = p_;
m_numPeriod = numOfPeriods_;
// find common dates
List<DateTime> commonDates = new List<DateTime>();
List<DateTime> stratDates = new List<DateTime>(m_series1.Dates);
foreach (DateTime date in m_series2.Dates)
if (stratDates.Contains(date))
commonDates.Add(date);
QuickSort.Sort<DateTime>(commonDates);
List<double> first = new List<double>();
List<double> second = new List<double>();
// get the values for each of the common dates
foreach (DateTime date in commonDates)
{
first.Add(m_series1.ValueOnDate(date));
second.Add(m_series2.ValueOnDate(date));
}
m_series1 = new DatedDataCollectionGen<double>(commonDates.ToArray(), first.ToArray());
m_series2 = new DatedDataCollectionGen<double>(commonDates.ToArray(), second.ToArray());
recalc();
}
public void Bind(ReturnsEval.DataSeriesEvaluator eval_, ReturnsEval.Period period_)
{
m_eval = eval_;
m_period = period_;
ClearSeries();
DatedDataCollectionGen<double> daily = eval_.Daily;
AddSeries(eval_.Daily.Dates, eval_.Daily.CumulativeReturnSeries, "Cumulative Pnl", 40, "##0.0#%");
{
DatedDataCollectionGen<double>[] years = new DatedDataCollectionGen<double>[]
{
daily.ToYearsCumulativeYTD(),
daily.ToYearsCumulativeYTD()
};
int index = 1;
for (int i = 0; i < years[0].Length; ++i)
{
if (i>0 && years[0].Dates[i].Year != years[0].Dates[i - 1].Year)
index = (index == 1) ? 0 : 1;
years[index].Data[i] = 0d;
}
if (ShowYearly)
{
AddSeries(years[0].Dates, years[0].Data, "YTD Cumulative(A)", 40, "##0.0#%");
AddSeries(years[1].Dates, years[1].Data, "YTD Cumulative(B)", 40, "##0.0#%");
}
}
//AddSeries(daily.Dates, daily.ToYearsCumulativeYTD().Data, "YTD Cumulative", 40, "##0.0#%");
//AddSeries(daily.Dates, daily.ToMonthsCumulativeMTD().Data, "MTD Cumulative", 40, "##0.0#%");
}
internal override DatedDataCollectionGen<double> GetTrades(DatedDataCollectionGen<double> levels_)
{
var ma = HelperMethods.GetMA(levels_, MA);
var trades = ma.Dates.Select((date, index) => levels_.ValueOnDate(date) > ma.Data[index] ? 1d : -1d).ToArray();
return new DatedDataCollectionGen<double>(ma.Dates, trades);
}
public DataInterEvent(EventsPick.EventDateSpan span_, bool dateOnly_, object instrument_, DatedDataCollectionGen<double> data_, DataInterEventArgs args_)
{
EventSpan = span_;
Instrument = instrument_;
RawData = data_;
Args = args_;
DateOnly = dateOnly_;
m_processedData = processToAroundAnchor(RawData);
}
public EventsPostFilter(PostFilterType postFilter, FilterOperator filterOperator, double value, DatedDataCollectionGen<double> sourceCollections, string instrumentString, Tuple<string, DatedDataCollectionGen<double>> rawData)
{
_postFilter = postFilter;
_filterOperator = filterOperator;
valueToCompare = value;
//FetchValueToCompare = fetchValueToCompare;
this.sourceCollections = sourceCollections;
_instrumentString = instrumentString;
_rawData = rawData;
}
public DatedDataCollectionGen<double> GenerateWeightSeries(DatedDataCollectionGen<double> prices_, double convention_, DateTime? minDate_)
{
var periodReturns = getPeriodReturns(prices_, convention_);
var zScores = getZScores(prices_, convention_);
var contextMAs = HelperMethods.GetRollingStat(prices_, Context, x => x.Average());
double lastScore = 0d;
var dates = new[]
{
periodReturns.Dates,
zScores.Dates,
contextMAs.Dates
}.OrderBy(x => x.Length)
.First();
var scores = new double[dates.Length];
for (int i = 1; i < dates.Length; ++i)
{
var date = dates[i];
var zScore = zScores.ValueOnDate(date);
var ret = periodReturns.ValueOnDate(date);
var price = prices_.ValueOnDate(date);
var contextMA = contextMAs.ValueOnDate(date);
var targetScore =
Math.Abs(zScore) < ZScoreThreshold
? 0d
: ret > 0d ? 1d : -1d;
// matches context?
if (targetScore == 1)
{
targetScore = (price >= contextMA) ? targetScore : 0d;
}
else if (targetScore == -1)
{
targetScore = (price <= contextMA) ? targetScore : 0d;
}
var prevScore = scores[i - 1];
var shift = Math.Min(Math.Abs(targetScore - prevScore), ScoreShift);
if (targetScore < prevScore)
scores[i] = prevScore - shift;
else
scores[i] = prevScore + shift;
}
return new DatedDataCollectionGen<double>(dates, scores);
}
public DatedDataCollectionGen<double> GenerateWeightSeries(DatedDataCollectionGen<double> prices_, DateTime? minDate_)
{
var maF = HelperMethods.GetMA(prices_, FastMA);
var maS = HelperMethods.GetMA(prices_, SlowMA);
var arr = new double[maS.Length];
for (int i = 0; i < maS.Length; ++i) arr[i] = maF.ValueOnDate(maS.Dates[i]) - maS.Data[i];
return new DatedDataCollectionGen<double>(maS.Dates, arr);
}
private static DatedDataCollectionGen<double> combineAllocation(DatedDataCollectionGen<double> a_, DatedDataCollectionGen<double> b_)
{
// this should only be used when at the top level, when the allocation will be assumed to be '1'
if (a_ == null) return b_;
if (b_ == null) return a_;
var distinctDates = a_.Dates.AppendWith(b_.Dates).Distinct().OrderBy(x => x).ToArray();
var allocs = distinctDates.Select(date => a_.ValueOnDate(date)*b_.ValueOnDate(date)).ToArray();
return new DatedDataCollectionGen<double>(distinctDates, allocs);
}
public static Tuple<DatedDataCollectionGen<string>, DatedDataCollectionGen<double>> GetItemsFromCarbonFrame(string identifier, double multiplier,
Func<RollResults, RollResultContractItem> GetGenerationResult,
Func<RollResultContractItem, DatedDataCollectionGen<RollResultItem>> GetResult)
{
bool isCT = identifier.ToUpper().Contains("CT");
var rolltype = isCT ? "ct" : "ctd";
var moniker =
("symapp.roll." + rolltype + "." + identifier.Market() + "." + identifier).ToLower();
RollResults savedResults=null;
// clear the cache every day
if(DateTime.UtcNow.Date > cacheTimeStamp)
CarbonRollResultsCache.Clear();
// our static cache is moniker name + utc today date. we use utc as we don't want to clear the cache at HK midnight.
var cacheKey = moniker + "_" + DateTime.UtcNow.Date.ToString("yy-MM-dd");
if (!CarbonRollResultsCache.ContainsKey(cacheKey))
{
var savedFrame = CarbonModel.RetrieveFrameFromCarbon(moniker).Result;
savedResults = savedFrame.ToRollResult(moniker);
CarbonRollResultsCache[cacheKey] = savedResults;
cacheTimeStamp = DateTime.UtcNow.Date;
}
else
savedResults = CarbonRollResultsCache[cacheKey];
var generationResult = GetGenerationResult(savedResults);
var genResultCollection = GetResult(generationResult);
var value = genResultCollection.Data.Select(d => d.Value * multiplier).ToArray();
var dates = genResultCollection.Dates.ToArray();
var bonds = genResultCollection.Data.Select(d => d.Name).ToArray();
// fill in zeroes
for (int i = 1; i < value.Length; ++i)
if (value[i].IsZero())
{
// if bonds haven't changed then can take the previous price
//if (bonds[i].SymmetryCode.Equals(bonds[i - 1].SymmetryCode))
value[i] = value[i - 1];
//else
// value[i] = double.NaN;
}
var validatedResult = new DatedDataCollectionGen<double>(dates, value);
return new Tuple<DatedDataCollectionGen<string>, DatedDataCollectionGen<double>>(
new DatedDataCollectionGen<string>(dates, bonds),
validatedResult);
//HelperMethods.RemoveNans(validatedResult));
}
private DatedDataCollectionGen<double> preProcess(DatedDataCollectionGen<double> series_)
{
if (m_args.OmitYears == null)
{
if (m_args.EndYear.HasValue)
return series_.GetSubValues(x => x.Year < m_args.StartYear || x.Year > m_args.EndYear);
return series_;
}
if(m_args.EndYear.HasValue)
return series_.GetSubValues(x => m_args.OmitYears.Contains(x.Year) || x.Year < m_args.StartYear || x.Year > m_args.EndYear);
return series_.GetSubValues(x => m_args.OmitYears.Contains(x.Year) || x.Year < m_args.StartYear);
}
public static EventsPostFilter DeMark(DeMarkEventType markEventType, int window, RetrieveEventDataDelegate FetchValueToCompare)
{
var retrievedData = FetchValueToCompare();
var prices = retrievedData.Item2;
var instrumentString = retrievedData.Item1;
var setups = SI.Strategy.Technicals.DeMark.Analysis.GetSetups(prices.ToContruct(), 0, 0, 0, 0, window)
.Where(x => x.EventType == markEventType && x.Children != null);
var events = setups.Select(x => new {a = x.Children.First(c => c.SeriesIndex == window).Date, b = (double)window}).ToArray();
var dateWindowDeMarkCollection = new DatedDataCollectionGen<double>(events.Select(e => e.a).ToArray(),
events.Select(e => e.b).ToArray());
return new EventsPostFilter(PostFilterType.DeMarkBuy, FilterOperator.Equal, window, dateWindowDeMarkCollection, instrumentString, retrievedData);
}
public void Create(DatedDataCollectionGen<double> x_, DatedDataCollectionGen<double> y_, string xName_, string yName_)
{
var con = new ConstructGenGen<string,double>(2);
con.ColumnHeadings = new string[] { xName_, yName_ };
var ds = new DatedDataCollectionGen<double>[] { x_, y_ };
for (int y = 0; y < ds.Length; ++y)
for (int i = 0; i < ds[y].Length; ++i)
con.SetValue(ds[y].Dates[i].ToString("dd-MMM-yyyy"), y, ds[y].Data[i]);
Create(con);
}
