//this is asych for the calling Task.WhenAll
//but does not necessarily need internal asych awaits
public async Task RunAlgorithmAsync(List <List <double> > data)
{
try
{
//minimal data requirement is first five cols
if (_colNames.Count() < 5 ||
_mathTerms.Count() == 0)
{
ErrorMessage = "Regression requires at least one dependent variable and one independent variable.";
return;
}
if (data.Count() < 5)
{
//185 same as other analysis
ErrorMessage = "Regression requires at least 2 rows of observed data and 3 rows of scoring data.";
return;
}
//convert data to a Math.Net Matrix
//v185 uses same ci technique as algos 2,3 and 4 -last 3 rows are used to generate ci
List <List <double> > dataci = data.Skip(data.Count - _scoreRows).ToList();
data.Reverse();
List <List <double> > dataobs = data.Skip(_scoreRows).ToList();
dataobs.Reverse();
//actual observed values
Vector <double> y = Shared.GetYData(dataobs);
Matrix <double> x = Shared.GetDoubleMatrix(dataobs, _colNames, _depColNames);
Matrix <double> ci = Shared.GetDoubleMatrix(dataci, _colNames, _depColNames);
//model expected values - get the coefficents
//use normal equations regression
Vector <double> p = MultipleRegression.NormalEquations(x, y);
//but note that this runs without errors in more cases but still does not give good results
//Vector<double> p = MultipleRegression.QR(x, y);
if (p.Count() != ci.Row(_scoreRows - 1).Count())
{
//185 same as other analysis
ErrorMessage = "The scoring and training datasets have different numbers of columns.";
return;
}
//get the predicted yhats
Vector <double> yhat = GetYHatandSetQTPred(y.Count, x, p, ci.Row(_scoreRows - 1).ToArray());
//get the durbin-watson d statistic
double d = GetDurbinWatson(y, yhat);
double SSE = 0;
//sum of the square of the error (between the predicted, p, and observed, y);
SSE = Distance.SSD(yhat, y);
double rSquared = GoodnessOfFit.RSquared(yhat, y);
//sum of the square of the regression (between the predicted, p, and observed mean, statsY.Mean);
double SSR = 0;
for (int i = 0; i < yhat.Count(); i++)
{
SSR += Math.Pow((yhat[i] - y.Mean()), 2);
}
//set joint vars properties
//degrees freedom
double dfR = x.ColumnCount - 1;
double dfE = x.RowCount - x.ColumnCount;
int idfR = x.ColumnCount - 1;
int idfE = x.RowCount - x.ColumnCount;
double s2 = SSE / dfE;
double s = Math.Sqrt(s2);
double MSR = SSR / dfR;
double MSE = SSE / dfE;
double FValue = MSR / MSE;
double adjRSquared = 1 - ((x.RowCount - 1) * (MSE / (SSE + SSR)));
double pValue = Shared.GetPValueForFDist(idfR, idfE, FValue);
//correct 2 tailed t test
//double TCritialValue = ExcelFunctions.TInv(0.05, idfE);
//so do this
double dbCI = CalculatorHelpers.GetConfidenceIntervalProb(_confidenceInt);
double tCriticalValue = ExcelFunctions.TInv(dbCI, idfE);
//set each coeff properties
//coeffs st error
//use matrix math to get the standard error of coefficients
Matrix <double> xt = x.Transpose();
//matrix x'x
Matrix <double> xx = xt.Multiply(x);
Matrix <double> xxminus1 = xx.Inverse();
double sxx = 0;
double[] xiSE = new double[x.ColumnCount];
//coeff tstats
double[] xiT = new double[x.ColumnCount];
//lower value for pvalue
double[] xiP = new double[x.ColumnCount];
for (int i = 0; i < x.ColumnCount; i++)
{
//use the matrix techniques shown on p 717 of Mendenhall and Sincich
sxx = s * Math.Sqrt(xxminus1.Column(i)[i]);
xiSE[i] = sxx;
xiT[i] = p[i] / sxx;
xiP[i] = Shared.GetPValueForTDist(idfE, xiT[i], 0, 1);
}
double FCriticalValue = 0;
string FGreaterFCritical = string.Empty;
if (_subalgorithm == Calculator1.MATH_SUBTYPES.subalgorithm8.ToString())
{
//anova regression
//anova critical fvalue test
//FCriticalValue = ExcelFunctions.FInv(1 - _confidenceInt, idfR, idfE);
FCriticalValue = ExcelFunctions.FInv(dbCI, idfR, idfE);
FGreaterFCritical = (FValue > FCriticalValue) ? "true" : "false";
SetAnovaIntervals(0, p, xiSE, tCriticalValue);
SetAnovaIntervals(1, p, xiSE, tCriticalValue);
SetAnovaIntervals(2, p, xiSE, tCriticalValue);
}
else
{
//set QTM ci and pi intervals
SetQTIntervals(0, s, xxminus1, ci.Row(_scoreRows - 1).ToArray(), p, tCriticalValue);
SetQTIntervals(1, s, xxminus1, ci.Row(_scoreRows - 2).ToArray(), p, tCriticalValue);
SetQTIntervals(2, s, xxminus1, ci.Row(_scoreRows - 3).ToArray(), p, tCriticalValue);
}
//add the data to a string builder
StringBuilder sb = new StringBuilder();
sb.AppendLine("regression results");
//dep var has to be in the 4 column always
string sLine = string.Concat("dependent variable: ", _colNames[3]);
sb.AppendLine(sLine);
string[] cols = new string[] { "source", "df", "SS", "MS" };
sb.AppendLine(Shared.GetLine(cols, true));
cols = new string[] { "model", dfR.ToString("F0"), SSR.ToString("F4"), MSR.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { "error ", dfE.ToString("F0"), SSE.ToString("F4"), MSE.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { "total ", (dfR + dfE).ToString("F0"), (SSR + SSE).ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
sb.AppendLine(string.Empty);
cols = new string[] { "R-squared", rSquared.ToString("F4"), "Adj R-squared", adjRSquared.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { "F value", FValue.ToString("F4"), "prob > F", pValue.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
sb.AppendLine(string.Empty);
cols = new string[] { GetName("variable"), "coefficient", "stand error", "T-ratio", "prob > T" };
sb.AppendLine(Shared.GetLine(cols, true));
for (int i = 0; i < p.Count(); i++)
{
if (i == 0)
{
cols = new string[] { GetName(_depColNames[i]), p[i].ToString("F5"), xiSE[i].ToString("F4"), xiT[i].ToString("F4"), xiP[i].ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
}
else
{
cols = new string[] { GetName(_depColNames[i]), p[i].ToString("F5"), xiSE[i].ToString("F4"), xiT[i].ToString("F4"), xiP[i].ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
}
}
cols = new string[] { "durbin-watson: ", d.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
if (_subalgorithm == Calculator1.MATH_SUBTYPES.subalgorithm8.ToString())
{
cols = new string[] { "F Critical Value", FCriticalValue.ToString("F5"), "F > F Critical", FGreaterFCritical };
sb.AppendLine(Shared.GetLine(cols, true));
cols = new string[] { "estimate", "predicted", string.Concat("lower ", _confidenceInt.ToString(), "%"), string.Concat("upper ", _confidenceInt.ToString(), "%") };
sb.AppendLine(Shared.GetLine(cols, true));
cols = new string[] { "Col 0 Mean CI ", QTPredicted.ToString("F4"), QTL.ToString("F4"), QTU.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { "Col 1 - 0 Mean CI ", QTPredicted10.ToString("F4"), QTL10.ToString("F4"), QTU10.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { "Col 2 - 0 Mean CI ", QTPredicted20.ToString("F4"), QTL20.ToString("F4"), QTU20.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
}
else
{
cols = new string[] { "estimate", "predicted", string.Concat("lower ", _confidenceInt.ToString(), "%"), string.Concat("upper ", _confidenceInt.ToString(), "%") };
sb.AppendLine(Shared.GetLine(cols, true));
cols = new string[] { "QTM CI ", QTPredicted.ToString("F4"), QTL.ToString("F4"), QTU.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { "QTM PI ", QTPredicted.ToString("F4"), (QTPredicted - QTPI).ToString("F4"), (QTPredicted + QTPI).ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
string sRow = string.Concat("row ", data.Count - 2);
cols = new string[] { sRow };
sb.AppendLine(Shared.GetLine(cols, true));
cols = new string[] { "CI ", QTPredicted10.ToString("F4"), QTL10.ToString("F4"), QTU10.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { "PI ", QTPredicted10.ToString("F4"), (QTPredicted10 - QTPI10).ToString("F4"), (QTPredicted10 + QTPI10).ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
sRow = string.Concat("row ", data.Count - 1);
cols = new string[] { sRow };
sb.AppendLine(Shared.GetLine(cols, true));
cols = new string[] { "CI ", QTPredicted20.ToString("F4"), QTL20.ToString("F4"), QTU20.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { "PI ", QTPredicted20.ToString("F4"), (QTPredicted20 - QTPI20).ToString("F4"), (QTPredicted20 + QTPI20).ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
}
if (this.MathResult.ToLower().StartsWith("http"))
{
string sError = string.Empty;
bool bHasSaved = CalculatorHelpers.SaveTextInURI(
_params.ExtensionDocToCalcURI, sb.ToString(), this.MathResult, out sError);
if (!string.IsNullOrEmpty(sError))
{
this.MathResult += sError;
}
}
else
{
this.MathResult = sb.ToString();
}
}
catch (Exception ex)
{
this.ErrorMessage = ex.Message;
}
}
public async Task <bool> RunSubAlgorithm1or2Async(string inputFilePath, string rFile)
{
bool bHasCalcs = false;
string sBaseURL =
"https://ussouthcentral.services.azureml.net/workspaces/d454361ecdcb4ec4b03fa1aec5a7c0e2/services/b10e6b4c4e63438999cc45147bbe006c/jobs";
if (_subalgorithm == Calculator1.MATH_SUBTYPES.subalgorithm2.ToString())
{
sBaseURL = "https://ussouthcentral.services.azureml.net/workspaces/d454361ecdcb4ec4b03fa1aec5a7c0e2/services/abd32060dc014d0e8fe1256e0f694daa/jobs";
}
string sPlatForm = _params.ExtensionDocToCalcURI.URIDataManager.PlatformType.ToString();
if (sPlatForm != CalculatorHelpers.PLATFORM_TYPES.azure.ToString())
{
sBaseURL = "https://ussouthcentral.services.azureml.net/workspaces/d454361ecdcb4ec4b03fa1aec5a7c0e2/services/b10e6b4c4e63438999cc45147bbe006c/execute?api-version=2.0&details=true";
if (_subalgorithm == Calculator1.MATH_SUBTYPES.subalgorithm2.ToString())
{
sBaseURL = "https://ussouthcentral.services.azureml.net/workspaces/d454361ecdcb4ec4b03fa1aec5a7c0e2/services/abd32060dc014d0e8fe1256e0f694daa/execute?api-version=2.0&details=true";
}
}
//r web service is default
string sApiKey =
"fxBeL9LJ3ORm0kW0DtKhT99OfUK6YgBlc59crizYhlxKoEjRd3kuDHvPRuehCQ02VJhPPXcdYTp2pDUynb9gMA==";
if (_subalgorithm == Calculator1.MATH_SUBTYPES.subalgorithm2.ToString())
{
//python
sApiKey =
"/bjDNKx4OWdMIQu6CkvWCIhcfUOCTp9jUE9kD7uylwhOYyhVFOqAFA7M75mJjHS6p6jnAhCvFn1jSl678gzPVA==";
}
string sError = string.Empty;
//convert the script file to the script string expected by the algorithm
List <string> rlines = new List <string>();
rlines = CalculatorHelpers.ReadLines(_params.ExtensionDocToCalcURI, rFile, out sError);
this.ErrorMessage += sError;
if (rlines == null)
{
this.ErrorMessage += string.Concat(" ", Errors.MakeStandardErrorMsg("DATAURL_BAD"));
return(bHasCalcs);
}
if (rlines.Count == 0)
{
this.ErrorMessage += string.Concat(" ", Errors.MakeStandardErrorMsg("DATAURL_BAD"));
return(bHasCalcs);
}
StringBuilder sbR = new StringBuilder();
for (int i = 0; i < rlines.Count(); i++)
{
sbR.AppendLine(rlines[i]);
}
string rScript = sbR.ToString();
//web server expects to store in temp/randomid/name.csv
//web service stores in temp blob
string sOutputDataURL = CalculatorHelpers.GetTempContainerPath("Routput.csv");
//web service expects urls that start with container names
//regular rproject file must be stored in JDataURL
string sInputContainerPath = CalculatorHelpers.GetContainerPathFromFullURIPath("resources", inputFilePath);
//async wait so that results can be stored in output file location and parsed into string lines
SetResponse(sBaseURL, sApiKey, sInputContainerPath, sOutputDataURL, rScript).Wait();
StringBuilder sb = new StringBuilder();
//if web service successully saved the results, the response will start with Success
if (_response.StartsWith("Success"))
{
//return the output file contents in a string list of lines
//must convert container path to full path
string sOutputFullDataURL = string.Concat("https://devtreks1.blob.core.windows.net/", sOutputDataURL);
List <string> lines = new List <string>();
//azure emulator can't process real Azure URL so this won't work
//instead, double check that output url is actually saved
lines = CalculatorHelpers.ReadLines(_params.ExtensionDocToCalcURI, sOutputFullDataURL, out sError);
this.ErrorMessage += sError;
//this results in endless wait-try ReadLinesAsync(sOutputDataURL).ConfigureAwait(false)
//lines = await CalculatorHelpers.ReadLinesAsync(sOutputDataURL);
if (lines == null)
{
this.ErrorMessage += string.Concat(" ", Errors.MakeStandardErrorMsg("DATAURL_BAD"));
return(bHasCalcs);
}
if (lines.Count == 0)
{
this.ErrorMessage += string.Concat(" ", Errors.MakeStandardErrorMsg("DATAURL_BAD"));
return(bHasCalcs);
}
sb = new StringBuilder();
if (_subalgorithm == Calculator1.MATH_SUBTYPES.subalgorithm2.ToString())
{
sb.AppendLine("py results");
}
else
{
sb.AppendLine("r results");
}
//dep var has to be in the R project 1st column
string sLine = string.Concat("first variable: ", _colNames[0]);
string[] line = new List <string>().ToArray();
for (int i = 0; i < lines.Count(); i++)
{
line = lines[i].Split(Constants.CSV_DELIMITERS);
//lineout[1] = CalculatorHelpers.ConvertStringToDouble(line[0]).ToString("N4", CultureInfo.InvariantCulture);
sb.AppendLine(Shared.GetLine(line, false));
}
if (this.MathResult.ToLower().StartsWith("http"))
{
sError = string.Empty;
bool bHasSaved = CalculatorHelpers.SaveTextInURI(
_params.ExtensionDocToCalcURI, sb.ToString(), this.MathResult, out sError);
if (!string.IsNullOrEmpty(sError))
{
this.MathResult += sError;
}
}
else
{
this.MathResult = sb.ToString();
}
bHasCalcs = true;
//last line of string should have the QTM vars
if (line != null)
{
int iPos = 0;
if (line[iPos] != null)
{
this.QTPredicted = CalculatorHelpers.ConvertStringToDouble(line[iPos]);
}
iPos = 1;
if (line[iPos] != null)
{
this.QTL = CalculatorHelpers.ConvertStringToDouble(line[iPos]);
}
iPos = 2;
if (line[iPos] != null)
{
this.QTU = CalculatorHelpers.ConvertStringToDouble(line[iPos]);
}
}
}
else
{
this.ErrorMessage += string.Concat(_response, "The calculations could not be run using the web service.");
}
return(bHasCalcs);
}
public async Task <bool> RunSubAlgorithm3Async(string inputFile1Path, string inputFile2Path)
{
bool bHasCalcs = false;
string sBaseURL =
"https://ussouthcentral.services.azureml.net/workspaces/d454361ecdcb4ec4b03fa1aec5a7c0e2/services/8b1074b465ea4258a11ec48ce64ae257/jobs";
string sPlatForm = CalculatorHelpers.GetPlatform(_params.ExtensionDocToCalcURI, inputFile1Path);
if (sPlatForm != CalculatorHelpers.PLATFORM_TYPES.azure.ToString())
{
sBaseURL = "https://ussouthcentral.services.azureml.net/workspaces/d454361ecdcb4ec4b03fa1aec5a7c0e2/services/8b1074b465ea4258a11ec48ce64ae257/execute?api-version=2.0&details=true";
}
string sApiKey =
"RO2Ev5dRSKqNJ4jz+zoT0qDntEsKbyizbgZKlhOR2vGztsjBD3S3C8nmIlZI9TbbmCcsw+unwhky1GgZ5qiHmg==";
string sError = string.Empty;
//web server expects to store in temp/randomid/name.csv
//scoring results
string sOutputData1URL = CalculatorHelpers.GetTempContainerPath("outputdata1.csv");
//model results
string sOutputData2URL = CalculatorHelpers.GetTempContainerPath("outputdata2.csv");
//web service expects urls that start with container names
string sInput1ContainerPath = CalculatorHelpers.GetContainerPathFromFullURIPath("resources", inputFile1Path);
string sInput2ContainerPath = CalculatorHelpers.GetContainerPathFromFullURIPath("resources", inputFile2Path);
//async wait so that results can be stored in output file location and parsed into string lines
SetResponse2(sBaseURL, sApiKey, sInput1ContainerPath, sInput2ContainerPath, sOutputData1URL, sOutputData2URL).Wait();
StringBuilder sb = new StringBuilder();
//if web service successully saved the results, the response will start with Success
if (_response.StartsWith("Success"))
{
//return the output file contents in a string list of lines
//must convert container path to full path
string sOutput1FullDataURL = string.Concat("https://devtreks1.blob.core.windows.net/", sOutputData1URL);
List <string> lines = new List <string>();
//azure emulator can't process real Azure URL so this won't work
//instead, double check that output url is actually saved
lines = CalculatorHelpers.ReadLines(_params.ExtensionDocToCalcURI, sOutput1FullDataURL, out sError);
this.ErrorMessage += sError;
//this results in endless wait
//lines = await CalculatorHelpers.ReadLinesAsync(sOutputDataURL);
if (lines == null)
{
this.ErrorMessage += string.Concat(" ", Errors.MakeStandardErrorMsg("DATAURL_BAD"));
return(bHasCalcs);
}
if (lines.Count == 0)
{
this.ErrorMessage += string.Concat(" ", Errors.MakeStandardErrorMsg("DATAURL_BAD"));
return(bHasCalcs);
}
sb = new StringBuilder();
sb.AppendLine("aml results");
//dep var has to be in the R project 1st column
//string sLine = string.Concat("first variable: ", _colNames[0]);
string[] line = new List <string>().ToArray();
int iPos = 0;
for (int i = 0; i < lines.Count(); i++)
{
line = lines[i].Split(Constants.CSV_DELIMITERS);
//lineout[1] = CalculatorHelpers.ConvertStringToDouble(line[0]).ToString("N4", CultureInfo.InvariantCulture);
sb.AppendLine(Shared.GetLine(line, false));
}
if (this.MathResult.ToLower().StartsWith("http"))
{
sError = string.Empty;
bool bHasSaved = CalculatorHelpers.SaveTextInURI(
_params.ExtensionDocToCalcURI, sb.ToString(), this.MathResult, out sError);
if (!string.IsNullOrEmpty(sError))
{
this.MathResult += sError;
}
}
else
{
this.MathResult = sb.ToString();
}
bHasCalcs = true;
//last line of string should have the QTM vars
if (line != null)
{
//last string is prediction
iPos = line.Count() - 1;
//int iPos = line.Count() - 3;
if (line[iPos] != null)
{
this.QTPredicted = CalculatorHelpers.ConvertStringToDouble(line[iPos]);
}
}
string sOutput2FullDataURL = string.Concat("https://devtreks1.blob.core.windows.net/", sOutputData2URL);
lines = new List <string>();
//azure emulator can't process real Azure URL so this won't work
//instead, double check that output url is actually saved
lines = CalculatorHelpers.ReadLines(_params.ExtensionDocToCalcURI, sOutput2FullDataURL, out sError);
this.ErrorMessage += sError;
if (lines == null)
{
this.ErrorMessage += string.Concat(" ", Errors.MakeStandardErrorMsg("DATAURL_BAD"));
return(bHasCalcs);
}
if (lines.Count == 0)
{
this.ErrorMessage += string.Concat(" ", Errors.MakeStandardErrorMsg("DATAURL_BAD"));
return(bHasCalcs);
}
sb = new StringBuilder();
//dep var has to be in the R project 1st column
//string sLine = string.Concat("first variable: ", _colNames[0]);
line = new List <string>().ToArray();
double dbCI = 0;
for (int i = 0; i < lines.Count(); i++)
{
line = lines[i].Split(Constants.CSV_DELIMITERS);
if (line != null)
{
iPos = 0;
//used to derive conf interval
dbCI = CalculatorHelpers.ConvertStringToDouble(line[iPos]);
sb.AppendLine(string.Format("{0} {1}", "Mean Absolute Error: ", dbCI.ToString()));
double dbScore = 0;
if (line.Count() >= 2)
{
iPos = 1;
dbScore = CalculatorHelpers.ConvertStringToDouble(line[iPos]);
sb.AppendLine(string.Format("{0} {1}", "Root Mean Squared Error: ", dbScore.ToString()));
}
if (line.Count() >= 3)
{
iPos = 2;
dbScore = CalculatorHelpers.ConvertStringToDouble(line[iPos]);
sb.AppendLine(string.Format("{0} {1}", "Relative Absolute Error: ", dbScore.ToString()));
}
if (line.Count() >= 4)
{
iPos = 3;
dbScore = CalculatorHelpers.ConvertStringToDouble(line[iPos]);
sb.AppendLine(string.Format("{0} {1}", "Relative Squared Error: ", dbScore.ToString()));
}
if (line.Count() >= 5)
{
iPos = 4;
dbScore = CalculatorHelpers.ConvertStringToDouble(line[iPos]);
sb.AppendLine(string.Format("{0} {1}", "Coefficient of Determination: ", dbScore.ToString()));
}
//sb.AppendLine(Shared.GetLine(line, false));
}
}
this.MathResult += sb.ToString();
bHasCalcs = true;
//last line of string should have the QTM vars
if (line != null)
{
if (line[iPos] != null)
{
dbCI = CalculatorHelpers.ConvertStringToDouble(line[iPos]);
}
this.QTL = this.QTPredicted - dbCI;
this.QTU = this.QTPredicted + dbCI;
}
}
else
{
this.ErrorMessage += "The calculations could not be run using the web service.";
}
return(bHasCalcs);
}
//this is asych for the calling Task.WhenAll
//but does not necessarily need internal asych awaits
public async Task <bool> RunAlgorithmAsync(List <List <double> > data)
{
bool bHasCalculations = false;
try
{
//minimal data requirement is first five cols
if (_colNames.Count() < 5 ||
_mathTerms.Count() == 0)
{
ErrorMessage = "Complete randomized anova requires at least 1 dependent variable and 1 independent variable. Randomized block and factorial anovas require at least 2 independent variables.";
return(bHasCalculations);
}
if (data.Count() < 5)
{
//185 same as other analysis
ErrorMessage = "Anova requires at least 2 rows of observed data and 3 rows of scoring data.";
return(bHasCalculations);
}
//convert data to a Math.Net Matrix
//last 3 rows are used to generate ci
List <List <double> > dataci = data.Skip(data.Count - _scoreRows).ToList();
data.Reverse();
List <List <double> > dataobs = data.Skip(_scoreRows).ToList();
dataobs.Reverse();
//actual observed values
Vector <double> y = Shared.GetYData(dataobs);
//treatments or factor1 levels
Matrix <double> treatments = Shared.GetDistinctMatrix(dataobs, 1);
//206 condition added due to M and E dataset indexing
_totalsNeeded = treatments.ColumnCount;
double tDF = treatments.ColumnCount - 1;
//step 1. get total of observed data
double yTotal = y.Sum();
//step 2. get total of observed data squared
double yTotal2 = y.PointwisePower(2).Sum();
//step 3. set CM
double CM = Math.Pow(yTotal, 2) / y.Count();
//step 4. treatments and blocks
double SSTotal = yTotal2 - CM;
//add the data to a string builder
StringBuilder sb = new StringBuilder();
sb.AppendLine("anova results");
//5 col array
string[] cols = new string[] { "source", "df", "SS", "MS", "F" };
sb.AppendLine(Shared.GetLine(cols, true));
List <List <double> > totals = new List <List <double> >(_totalsNeeded);
//min treatment block required
bool bIsBlock = (_depColNames.Contains("treatment") && _depColNames.Contains("block")) ? true : false;
//min 2 factors required
bool bIsFactorial = (_depColNames.Contains("factor1") && _depColNames.Contains("factor2")) ? true : false;
bool bIsComplete = (bIsBlock == false && bIsFactorial == false) ? true : false;
if (bIsComplete)
{
double eDF = y.Count() - treatments.ColumnCount;
//step 5.treatments (correct to divide by rows)
double SST = ((treatments.ColumnSums().PointwisePower(2).Sum()) / treatments.RowCount) - CM;
//step 6. error
double SSE = SSTotal - SST;
//step 7. mean treatment
double MST = SST / tDF;
//step 8. mean error
double MSE = SSE / (y.Count() - treatments.ColumnCount);
//step 9. F treatments
double FT = MST / MSE;
//tests
double s = Math.Pow(MSE, 0.5);
//correct 2 tailed t test
int itDF = CalculatorHelpers.ConvertStringToInt(tDF.ToString());
int ieDF = CalculatorHelpers.ConvertStringToInt(eDF.ToString());
double dbCI = CalculatorHelpers.GetConfidenceIntervalProb(_confidenceInt);
//TINV divides dbCI by 2 to get student t
double tCriticalValue = ExcelFunctions.TInv(dbCI, ieDF);
//prevents an error in Finv
if (itDF == 0)
{
itDF = 1;
}
double FCriticalTValue = ExcelFunctions.FInv(dbCI, itDF, ieDF);
string FTGreaterFCritical = (FT > FCriticalTValue) ? "true" : "false";
for (int i = 0; i < _totalsNeeded; i++)
{
//206 condition added due to M and E dataset indexing
if (i < treatments.ColumnCount)
{
SetAnovaIntervals(i, totals, treatments, tCriticalValue, s,
CalculatorHelpers.ConvertStringToDouble(treatments.RowCount.ToString()),
FT, FCriticalTValue, bIsComplete);
}
}
this.DataToAnalyze.Add(Label, totals);
////add the data to a string builder
cols = new string[] { "treats", itDF.ToString("F0"), SST.ToString("F4"), MST.ToString("F4"), FT.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { "error ", ieDF.ToString("F0"), SSE.ToString("F4"), MSE.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { "total ", (y.Count() - 1).ToString("F0"), (SSTotal).ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { string.Concat("F Crit ", "treats"), FCriticalTValue.ToString("F5"), "F > F Critical", FTGreaterFCritical };
sb.AppendLine(Shared.GetLine(cols, true));
}
else
{
//observations per cell for factorials (data[0] if first row of data)
double r = Shared.GetObservationsPerCell(dataobs, 1, data[0].ElementAt(1), data[0].ElementAt(2));
//blocks or factor2 levels
Matrix <double> blocks = Shared.GetDistinctMatrix(dataobs, 2);
double bDF = blocks.ColumnCount - 1;
double eDF = y.Count() - treatments.ColumnCount - blocks.ColumnCount + 1;
if (bIsFactorial)
{
eDF = (treatments.ColumnCount * blocks.ColumnCount) * (r - 1);
}
//factorial interaction df
double tbDF = tDF * bDF;
//step 5.treatments (correct to divide by r)
double SST = ((treatments.ColumnSums().PointwisePower(2).Sum()) / (blocks.ColumnCount * r)) - CM;
//step 6. blocks
double SSB = ((blocks.ColumnSums().PointwisePower(2).Sum()) / (treatments.ColumnCount * r)) - CM;
//factor level interaction
double SSFL = 0;
//step 7. error
double SSE = 0;
if (bIsFactorial)
{
double totalinteraction = Shared.GetTotalInteraction(blocks, r);
//watch block.colcount for 2 x 3 factorials
SSFL = (totalinteraction / r) - SST - SSB - CM;
//step 7. error
SSE = SSTotal - SST - SSB - SSFL;
}
else
{
//step 7. error
SSE = SSTotal - SST - SSB;
}
//step 8. mean treatment
double MST = SST / tDF;
//step 9. mean block
double MSB = SSB / bDF;
//step 10. mean error
double MSE = SSE / eDF;
double MSFL = SSFL / tbDF;
//step 11. F treatments
double FT = MST / MSE;
//step 12. F blocks
double FB = MSB / MSE;
double FTB = MSFL / MSE;
//tests
double s = Math.Pow(MSE, 0.5);
//correct 2 tailed t test
int itDF = CalculatorHelpers.ConvertStringToInt(tDF.ToString());
int ibDF = CalculatorHelpers.ConvertStringToInt(bDF.ToString());
int itbDF = CalculatorHelpers.ConvertStringToInt(tbDF.ToString());
int ieDF = CalculatorHelpers.ConvertStringToInt(eDF.ToString());
double dbCI = CalculatorHelpers.GetConfidenceIntervalProb(_confidenceInt);
//TINV divides dbCI by 2 to get student t
double tCriticalValue = ExcelFunctions.TInv(dbCI, ieDF);
//prevents an error in Finv
if (itDF == 0)
{
itDF = 1;
}
double FCriticalTValue = ExcelFunctions.FInv(dbCI, itDF, ieDF);
string FTGreaterFCritical = (FT > FCriticalTValue) ? "true" : "false";
//prevents an error in Finv
if (ibDF == 0)
{
ibDF = 1;
}
double FCriticalBValue = ExcelFunctions.FInv(dbCI, ibDF, ieDF);
string FBGreaterFCritical = (FB > FCriticalBValue) ? "true" : "false";
//prevents an error in Finv
if (itbDF == 0)
{
itbDF = 1;
}
double FCriticalTBValue = ExcelFunctions.FInv(dbCI, itbDF, ieDF);
string FTBGreaterFCritical = (FTB > FCriticalTBValue) ? "true" : "false";
//List<List<double>> totals = new List<List<double>>(_totalsNeeded);
if (bIsFactorial)
{
//unless custom stylesheets are developed, can only display factor 1 - factor 2 diffs
//build a matrix equivalent to treatments -1 row, variable cols
Matrix <double> torbs = Matrix <double> .Build.Dense(1, _totalsNeeded);
List <double> mrow = new List <double>(_totalsNeeded);
for (int i = 0; i < _totalsNeeded; i++)
{
double cellMean = Shared.GetMeanPerCell(dataobs, 1, 2, i, i, r);
mrow.Add(cellMean);
}
torbs.SetRow(0, mrow.ToArray());
for (int i = 0; i < _totalsNeeded; i++)
{
//206 condition added due to M and E dataset indexing
if (i < treatments.ColumnCount)
{
//treatments
SetAnovaIntervals(i, totals, torbs, tCriticalValue, s,
CalculatorHelpers.ConvertStringToDouble(blocks.ColumnCount.ToString()),
FT, FCriticalTValue, bIsComplete);
}
}
//cell1Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 0, 0, r);
//cell2Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 0, 1, r);
//SetAnovaIntervals2(1, cell1Mean, cell2Mean, tCriticalValue, s, r);
//cell1Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 0, 0, r);
//cell2Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 2, 0, r);
//SetAnovaIntervals2(2, cell1Mean, cell2Mean, tCriticalValue, s, r);
//double cell1Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 0, 0, r);
//double cell2Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 1, 0, r);
//SetAnovaIntervals2(0, cell1Mean, cell2Mean, tCriticalValue, s, r);
//cell1Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 0, 0, r);
//cell2Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 0, 1, r);
//SetAnovaIntervals2(1, cell1Mean, cell2Mean, tCriticalValue, s, r);
//cell1Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 0, 0, r);
//cell2Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 2, 0, r);
//SetAnovaIntervals2(2, cell1Mean, cell2Mean, tCriticalValue, s, r);
}
else
{
//unless custom stylesheets are developed, need to only display treatment diffs
for (int i = 0; i < _totalsNeeded; i++)
{
//206 condition added due to M and E dataset indexing
if (i < treatments.ColumnCount)
{
//treatments
SetAnovaIntervals(i, totals, treatments, tCriticalValue, s,
CalculatorHelpers.ConvertStringToDouble(blocks.ColumnCount.ToString()),
FT, FCriticalTValue, bIsComplete);
////blocks
//SetAnovaIntervals(i, totals, blocks, tCriticalValue, s,
// CalculatorHelpers.ConvertStringToDouble(treatments.ColumnCount.ToString()),
// FB, FCriticalBValue, bIsComplete);
//interactions
}
}
}
this.DataToAnalyze.Add(Label, totals);
string sTreats = "treats ";
string sBlocks = "blocks ";
if (bIsFactorial)
{
sTreats = "factor1 ";
sBlocks = "factor2 ";
}
////add the data to a string builder
//StringBuilder sb = new StringBuilder();
//sb.AppendLine("anova results");
cols = new string[] { sTreats, itDF.ToString("F0"), SST.ToString("F4"), MST.ToString("F4"), FT.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { sBlocks, bDF.ToString("F0"), SSB.ToString("F4"), MSB.ToString("F4"), FB.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
if (bIsFactorial)
{
cols = new string[] { "interacts ", tbDF.ToString("F0"), SSFL.ToString("F4"), MSFL.ToString("F4"), FTB.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
}
cols = new string[] { "error ", ieDF.ToString("F0"), SSE.ToString("F4"), MSE.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { "total ", (y.Count() - 1).ToString("F0"), (SSTotal).ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { string.Concat("F Crit ", sTreats), FCriticalTValue.ToString("F5"), "F > F Critical", FTGreaterFCritical };
sb.AppendLine(Shared.GetLine(cols, true));
cols = new string[] { string.Concat("F Crit ", sBlocks), FCriticalBValue.ToString("F5"), "F > F Critical", FBGreaterFCritical };
sb.AppendLine(Shared.GetLine(cols, true));
if (bIsFactorial)
{
cols = new string[] { "F Crit Interacts", FCriticalTBValue.ToString("F5"), "F > F Critical", FTBGreaterFCritical };
sb.AppendLine(Shared.GetLine(cols, true));
}
}
cols = new string[] { "estimate", "mean diff", string.Concat("lower ", _confidenceInt.ToString(), "%"), string.Concat("upper ", _confidenceInt.ToString(), "%") };
sb.AppendLine(Shared.GetLine(cols, true));
//same report for calculator and analyzer
for (int i = 0; i < _totalsNeeded; i++)
{
if (totals[i].Count >= 5)
{
if (i == 0)
{
QTPredicted = totals[i].ElementAt(0);
QTL = QTPredicted - totals[i].ElementAt(4);
QTU = QTPredicted + totals[i].ElementAt(4);
cols = new string[] { "Treat 1 Mean ", QTPredicted.ToString("F4"), QTL.ToString("F4"), QTU.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
}
else
{
QTPredicted = totals[i].ElementAt(1);
QTL = QTPredicted - totals[i].ElementAt(2);
QTU = QTPredicted + totals[i].ElementAt(2);
cols = new string[] { string.Concat("xminus1 ", i.ToString(), " "), QTPredicted.ToString("F4"), QTL.ToString("F4"), QTU.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
QTPredicted = totals[i].ElementAt(3);
QTL = QTPredicted - totals[i].ElementAt(4);
QTU = QTPredicted + totals[i].ElementAt(4);
cols = new string[] { string.Concat("base ", i.ToString(), " "), QTPredicted.ToString("F4"), QTL.ToString("F4"), QTU.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
}
}
}
if (this.MathResult.ToLower().StartsWith("http"))
{
bool bHasSaved = await CalculatorHelpers.SaveTextInURI(
_params.ExtensionDocToCalcURI, sb.ToString(), this.MathResult);
if (!string.IsNullOrEmpty(_params.ExtensionDocToCalcURI.ErrorMessage))
{
this.MathResult += _params.ExtensionDocToCalcURI.ErrorMessage;
//done with errormsg
_params.ExtensionDocToCalcURI.ErrorMessage = string.Empty;
}
}
else
{
this.MathResult = sb.ToString();
}
bHasCalculations = true;
}
catch (Exception ex)
{
this.ErrorMessage = ex.Message;
}
return(bHasCalculations);
}