private async Task <bool> SetMathResult(List <List <string> > rowNames, StringBuilder sb = null)
{
bool bHasSet = false;
if (sb == null)
{
//add the data to a string builder
sb = new StringBuilder();
if (_subalgorithm == MATHML_SUBTYPES.subalgorithm_03.ToString())
{
sb.AppendLine("ml results");
string[] newColNames = new string[_actualColNames.Length + 5];
for (int i = 0; i < _actualColNames.Length; i++)
{
newColNames[i] = _actualColNames[i];
}
//new cols changed by algo
newColNames[_actualColNames.Length] = "mse";
newColNames[_actualColNames.Length + 1] = "accuracy";
newColNames[_actualColNames.Length + 2] = "qtm";
newColNames[_actualColNames.Length + 3] = "qtl";
newColNames[_actualColNames.Length + 4] = "qtu";
_actualColNames = newColNames;
CalculatorHelpers.SetIndMathResult(sb, _actualColNames, rowNames, DataResults);
}
}
if (this.IndicatorQT.MathResult.ToLower().StartsWith("http"))
{
bool bHasSaved = await CalculatorHelpers.SaveTextInURI(
Params.ExtensionDocToCalcURI, sb.ToString(), this.IndicatorQT.MathResult);
if (!string.IsNullOrEmpty(Params.ExtensionDocToCalcURI.ErrorMessage))
{
this.IndicatorQT.MathResult += Params.ExtensionDocToCalcURI.ErrorMessage;
//done with errormsg
Params.ExtensionDocToCalcURI.ErrorMessage = string.Empty;
}
else
{
bHasSet = true;
}
}
else
{
this.IndicatorQT.MathResult = sb.ToString();
bHasSet = true;
}
return(bHasSet);
}
//this is asych for the calling Task.WhenAll
//but does not necessarily need internal asych awaits
public async Task RunAlgorithmAsync(List <List <double> > data)
{
try
{
//minimal data requirement is first five cols
if (_colNames.Count() < 5 ||
_mathTerms.Count() == 0)
{
ErrorMessage = "Regression requires at least one dependent variable and one independent variable.";
return;
}
if (data.Count() < 5)
{
//185 same as other analysis
ErrorMessage = "Regression requires at least 2 rows of observed data and 3 rows of scoring data.";
return;
}
//convert data to a Math.Net Matrix
//v185 uses same ci technique as algos 2,3 and 4 -last 3 rows are used to generate ci
List <List <double> > dataci = data.Skip(data.Count - _scoreRows).ToList();
data.Reverse();
List <List <double> > dataobs = data.Skip(_scoreRows).ToList();
dataobs.Reverse();
//actual observed values
Vector <double> y = Shared.GetYData(dataobs);
Matrix <double> x = Shared.GetDoubleMatrix(dataobs, _colNames, _depColNames);
Matrix <double> ci = Shared.GetDoubleMatrix(dataci, _colNames, _depColNames);
//model expected values - get the coefficents
//use normal equations regression
Vector <double> p = MultipleRegression.NormalEquations(x, y);
//but note that this runs without errors in more cases but still does not give good results
//Vector<double> p = MultipleRegression.QR(x, y);
if (p.Count() != ci.Row(_scoreRows - 1).Count())
{
//185 same as other analysis
ErrorMessage = "The scoring and training datasets have different numbers of columns.";
return;
}
//get the predicted yhats
Vector <double> yhat = GetYHatandSetQTPred(y.Count, x, p, ci.Row(_scoreRows - 1).ToArray());
//get the durbin-watson d statistic
double d = GetDurbinWatson(y, yhat);
double SSE = 0;
//sum of the square of the error (between the predicted, p, and observed, y);
SSE = Distance.SSD(yhat, y);
double rSquared = GoodnessOfFit.RSquared(yhat, y);
//sum of the square of the regression (between the predicted, p, and observed mean, statsY.Mean);
double SSR = 0;
for (int i = 0; i < yhat.Count(); i++)
{
SSR += Math.Pow((yhat[i] - y.Mean()), 2);
}
//set joint vars properties
//degrees freedom
double dfR = x.ColumnCount - 1;
double dfE = x.RowCount - x.ColumnCount;
int idfR = x.ColumnCount - 1;
int idfE = x.RowCount - x.ColumnCount;
double s2 = SSE / dfE;
double s = Math.Sqrt(s2);
double MSR = SSR / dfR;
double MSE = SSE / dfE;
double FValue = MSR / MSE;
double adjRSquared = 1 - ((x.RowCount - 1) * (MSE / (SSE + SSR)));
double pValue = Shared.GetPValueForFDist(idfR, idfE, FValue);
//correct 2 tailed t test
//double TCritialValue = ExcelFunctions.TInv(0.05, idfE);
//so do this
double dbCI = CalculatorHelpers.GetConfidenceIntervalProb(_confidenceInt);
double tCriticalValue = ExcelFunctions.TInv(dbCI, idfE);
//set each coeff properties
//coeffs st error
//use matrix math to get the standard error of coefficients
Matrix <double> xt = x.Transpose();
//matrix x'x
Matrix <double> xx = xt.Multiply(x);
Matrix <double> xxminus1 = xx.Inverse();
double sxx = 0;
double[] xiSE = new double[x.ColumnCount];
//coeff tstats
double[] xiT = new double[x.ColumnCount];
//lower value for pvalue
double[] xiP = new double[x.ColumnCount];
for (int i = 0; i < x.ColumnCount; i++)
{
//use the matrix techniques shown on p 717 of Mendenhall and Sincich
sxx = s * Math.Sqrt(xxminus1.Column(i)[i]);
xiSE[i] = sxx;
xiT[i] = p[i] / sxx;
xiP[i] = Shared.GetPValueForTDist(idfE, xiT[i], 0, 1);
}
double FCriticalValue = 0;
string FGreaterFCritical = string.Empty;
if (_subalgorithm == Calculator1.MATH_SUBTYPES.subalgorithm8.ToString())
{
//anova regression
//anova critical fvalue test
//FCriticalValue = ExcelFunctions.FInv(1 - _confidenceInt, idfR, idfE);
FCriticalValue = ExcelFunctions.FInv(dbCI, idfR, idfE);
FGreaterFCritical = (FValue > FCriticalValue) ? "true" : "false";
SetAnovaIntervals(0, p, xiSE, tCriticalValue);
SetAnovaIntervals(1, p, xiSE, tCriticalValue);
SetAnovaIntervals(2, p, xiSE, tCriticalValue);
}
else
{
//set QTM ci and pi intervals
SetQTIntervals(0, s, xxminus1, ci.Row(_scoreRows - 1).ToArray(), p, tCriticalValue);
SetQTIntervals(1, s, xxminus1, ci.Row(_scoreRows - 2).ToArray(), p, tCriticalValue);
SetQTIntervals(2, s, xxminus1, ci.Row(_scoreRows - 3).ToArray(), p, tCriticalValue);
}
//add the data to a string builder
StringBuilder sb = new StringBuilder();
sb.AppendLine("regression results");
//dep var has to be in the 4 column always
string sLine = string.Concat("dependent variable: ", _colNames[3]);
sb.AppendLine(sLine);
string[] cols = new string[] { "source", "df", "SS", "MS" };
sb.AppendLine(Shared.GetLine(cols, true));
cols = new string[] { "model", dfR.ToString("F0"), SSR.ToString("F4"), MSR.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { "error ", dfE.ToString("F0"), SSE.ToString("F4"), MSE.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { "total ", (dfR + dfE).ToString("F0"), (SSR + SSE).ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
sb.AppendLine(string.Empty);
cols = new string[] { "R-squared", rSquared.ToString("F4"), "Adj R-squared", adjRSquared.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { "F value", FValue.ToString("F4"), "prob > F", pValue.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
sb.AppendLine(string.Empty);
cols = new string[] { GetName("variable"), "coefficient", "stand error", "T-ratio", "prob > T" };
sb.AppendLine(Shared.GetLine(cols, true));
for (int i = 0; i < p.Count(); i++)
{
if (i == 0)
{
cols = new string[] { GetName(_depColNames[i]), p[i].ToString("F5"), xiSE[i].ToString("F4"), xiT[i].ToString("F4"), xiP[i].ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
}
else
{
cols = new string[] { GetName(_depColNames[i]), p[i].ToString("F5"), xiSE[i].ToString("F4"), xiT[i].ToString("F4"), xiP[i].ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
}
}
cols = new string[] { "durbin-watson: ", d.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
if (_subalgorithm == Calculator1.MATH_SUBTYPES.subalgorithm8.ToString())
{
cols = new string[] { "F Critical Value", FCriticalValue.ToString("F5"), "F > F Critical", FGreaterFCritical };
sb.AppendLine(Shared.GetLine(cols, true));
cols = new string[] { "estimate", "predicted", string.Concat("lower ", _confidenceInt.ToString(), "%"), string.Concat("upper ", _confidenceInt.ToString(), "%") };
sb.AppendLine(Shared.GetLine(cols, true));
cols = new string[] { "Col 0 Mean CI ", QTPredicted.ToString("F4"), QTL.ToString("F4"), QTU.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { "Col 1 - 0 Mean CI ", QTPredicted10.ToString("F4"), QTL10.ToString("F4"), QTU10.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { "Col 2 - 0 Mean CI ", QTPredicted20.ToString("F4"), QTL20.ToString("F4"), QTU20.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
}
else
{
cols = new string[] { "estimate", "predicted", string.Concat("lower ", _confidenceInt.ToString(), "%"), string.Concat("upper ", _confidenceInt.ToString(), "%") };
sb.AppendLine(Shared.GetLine(cols, true));
cols = new string[] { "QTM CI ", QTPredicted.ToString("F4"), QTL.ToString("F4"), QTU.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { "QTM PI ", QTPredicted.ToString("F4"), (QTPredicted - QTPI).ToString("F4"), (QTPredicted + QTPI).ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
string sRow = string.Concat("row ", data.Count - 2);
cols = new string[] { sRow };
sb.AppendLine(Shared.GetLine(cols, true));
cols = new string[] { "CI ", QTPredicted10.ToString("F4"), QTL10.ToString("F4"), QTU10.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { "PI ", QTPredicted10.ToString("F4"), (QTPredicted10 - QTPI10).ToString("F4"), (QTPredicted10 + QTPI10).ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
sRow = string.Concat("row ", data.Count - 1);
cols = new string[] { sRow };
sb.AppendLine(Shared.GetLine(cols, true));
cols = new string[] { "CI ", QTPredicted20.ToString("F4"), QTL20.ToString("F4"), QTU20.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { "PI ", QTPredicted20.ToString("F4"), (QTPredicted20 - QTPI20).ToString("F4"), (QTPredicted20 + QTPI20).ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
}
if (this.MathResult.ToLower().StartsWith("http"))
{
string sError = string.Empty;
bool bHasSaved = CalculatorHelpers.SaveTextInURI(
_params.ExtensionDocToCalcURI, sb.ToString(), this.MathResult, out sError);
if (!string.IsNullOrEmpty(sError))
{
this.MathResult += sError;
}
}
else
{
this.MathResult = sb.ToString();
}
}
catch (Exception ex)
{
this.ErrorMessage = ex.Message;
}
}
public async Task <bool> RunSubAlgorithm3Async(string inputFile1Path, string inputFile2Path)
{
bool bHasCalcs = false;
string sBaseURL =
"https://ussouthcentral.services.azureml.net/workspaces/d454361ecdcb4ec4b03fa1aec5a7c0e2/services/8b1074b465ea4258a11ec48ce64ae257/jobs";
string sPlatForm = CalculatorHelpers.GetPlatform(_params.ExtensionDocToCalcURI, inputFile1Path);
if (sPlatForm != CalculatorHelpers.PLATFORM_TYPES.azure.ToString())
{
sBaseURL = "https://ussouthcentral.services.azureml.net/workspaces/d454361ecdcb4ec4b03fa1aec5a7c0e2/services/8b1074b465ea4258a11ec48ce64ae257/execute?api-version=2.0&details=true";
}
string sApiKey =
"RO2Ev5dRSKqNJ4jz+zoT0qDntEsKbyizbgZKlhOR2vGztsjBD3S3C8nmIlZI9TbbmCcsw+unwhky1GgZ5qiHmg==";
string sError = string.Empty;
//web server expects to store in temp/randomid/name.csv
//scoring results
string sOutputData1URL = CalculatorHelpers.GetTempContainerPath("outputdata1.csv");
//model results
string sOutputData2URL = CalculatorHelpers.GetTempContainerPath("outputdata2.csv");
//web service expects urls that start with container names
string sInput1ContainerPath = CalculatorHelpers.GetContainerPathFromFullURIPath("resources", inputFile1Path);
string sInput2ContainerPath = CalculatorHelpers.GetContainerPathFromFullURIPath("resources", inputFile2Path);
//async wait so that results can be stored in output file location and parsed into string lines
SetResponse2(sBaseURL, sApiKey, sInput1ContainerPath, sInput2ContainerPath, sOutputData1URL, sOutputData2URL).Wait();
StringBuilder sb = new StringBuilder();
//if web service successully saved the results, the response will start with Success
if (_response.StartsWith("Success"))
{
//return the output file contents in a string list of lines
//must convert container path to full path
string sOutput1FullDataURL = string.Concat("https://devtreks1.blob.core.windows.net/", sOutputData1URL);
List <string> lines = new List <string>();
//azure emulator can't process real Azure URL so this won't work
//instead, double check that output url is actually saved
lines = CalculatorHelpers.ReadLines(_params.ExtensionDocToCalcURI, sOutput1FullDataURL, out sError);
this.ErrorMessage += sError;
//this results in endless wait
//lines = await CalculatorHelpers.ReadLinesAsync(sOutputDataURL);
if (lines == null)
{
this.ErrorMessage += string.Concat(" ", Errors.MakeStandardErrorMsg("DATAURL_BAD"));
return(bHasCalcs);
}
if (lines.Count == 0)
{
this.ErrorMessage += string.Concat(" ", Errors.MakeStandardErrorMsg("DATAURL_BAD"));
return(bHasCalcs);
}
sb = new StringBuilder();
sb.AppendLine("aml results");
//dep var has to be in the R project 1st column
//string sLine = string.Concat("first variable: ", _colNames[0]);
string[] line = new List <string>().ToArray();
int iPos = 0;
for (int i = 0; i < lines.Count(); i++)
{
line = lines[i].Split(Constants.CSV_DELIMITERS);
//lineout[1] = CalculatorHelpers.ConvertStringToDouble(line[0]).ToString("N4", CultureInfo.InvariantCulture);
sb.AppendLine(Shared.GetLine(line, false));
}
if (this.MathResult.ToLower().StartsWith("http"))
{
sError = string.Empty;
bool bHasSaved = CalculatorHelpers.SaveTextInURI(
_params.ExtensionDocToCalcURI, sb.ToString(), this.MathResult, out sError);
if (!string.IsNullOrEmpty(sError))
{
this.MathResult += sError;
}
}
else
{
this.MathResult = sb.ToString();
}
bHasCalcs = true;
//last line of string should have the QTM vars
if (line != null)
{
//last string is prediction
iPos = line.Count() - 1;
//int iPos = line.Count() - 3;
if (line[iPos] != null)
{
this.QTPredicted = CalculatorHelpers.ConvertStringToDouble(line[iPos]);
}
}
string sOutput2FullDataURL = string.Concat("https://devtreks1.blob.core.windows.net/", sOutputData2URL);
lines = new List <string>();
//azure emulator can't process real Azure URL so this won't work
//instead, double check that output url is actually saved
lines = CalculatorHelpers.ReadLines(_params.ExtensionDocToCalcURI, sOutput2FullDataURL, out sError);
this.ErrorMessage += sError;
if (lines == null)
{
this.ErrorMessage += string.Concat(" ", Errors.MakeStandardErrorMsg("DATAURL_BAD"));
return(bHasCalcs);
}
if (lines.Count == 0)
{
this.ErrorMessage += string.Concat(" ", Errors.MakeStandardErrorMsg("DATAURL_BAD"));
return(bHasCalcs);
}
sb = new StringBuilder();
//dep var has to be in the R project 1st column
//string sLine = string.Concat("first variable: ", _colNames[0]);
line = new List <string>().ToArray();
double dbCI = 0;
for (int i = 0; i < lines.Count(); i++)
{
line = lines[i].Split(Constants.CSV_DELIMITERS);
if (line != null)
{
iPos = 0;
//used to derive conf interval
dbCI = CalculatorHelpers.ConvertStringToDouble(line[iPos]);
sb.AppendLine(string.Format("{0} {1}", "Mean Absolute Error: ", dbCI.ToString()));
double dbScore = 0;
if (line.Count() >= 2)
{
iPos = 1;
dbScore = CalculatorHelpers.ConvertStringToDouble(line[iPos]);
sb.AppendLine(string.Format("{0} {1}", "Root Mean Squared Error: ", dbScore.ToString()));
}
if (line.Count() >= 3)
{
iPos = 2;
dbScore = CalculatorHelpers.ConvertStringToDouble(line[iPos]);
sb.AppendLine(string.Format("{0} {1}", "Relative Absolute Error: ", dbScore.ToString()));
}
if (line.Count() >= 4)
{
iPos = 3;
dbScore = CalculatorHelpers.ConvertStringToDouble(line[iPos]);
sb.AppendLine(string.Format("{0} {1}", "Relative Squared Error: ", dbScore.ToString()));
}
if (line.Count() >= 5)
{
iPos = 4;
dbScore = CalculatorHelpers.ConvertStringToDouble(line[iPos]);
sb.AppendLine(string.Format("{0} {1}", "Coefficient of Determination: ", dbScore.ToString()));
}
//sb.AppendLine(Shared.GetLine(line, false));
}
}
this.MathResult += sb.ToString();
bHasCalcs = true;
//last line of string should have the QTM vars
if (line != null)
{
if (line[iPos] != null)
{
dbCI = CalculatorHelpers.ConvertStringToDouble(line[iPos]);
}
this.QTL = this.QTPredicted - dbCI;
this.QTU = this.QTPredicted + dbCI;
}
}
else
{
this.ErrorMessage += "The calculations could not be run using the web service.";
}
return(bHasCalcs);
}
public async Task <bool> RunSubAlgorithm1or2Async(string inputFilePath, string rFile)
{
bool bHasCalcs = false;
string sBaseURL =
"https://ussouthcentral.services.azureml.net/workspaces/d454361ecdcb4ec4b03fa1aec5a7c0e2/services/b10e6b4c4e63438999cc45147bbe006c/jobs";
if (_subalgorithm == Calculator1.MATH_SUBTYPES.subalgorithm2.ToString())
{
sBaseURL = "https://ussouthcentral.services.azureml.net/workspaces/d454361ecdcb4ec4b03fa1aec5a7c0e2/services/abd32060dc014d0e8fe1256e0f694daa/jobs";
}
string sPlatForm = _params.ExtensionDocToCalcURI.URIDataManager.PlatformType.ToString();
if (sPlatForm != CalculatorHelpers.PLATFORM_TYPES.azure.ToString())
{
sBaseURL = "https://ussouthcentral.services.azureml.net/workspaces/d454361ecdcb4ec4b03fa1aec5a7c0e2/services/b10e6b4c4e63438999cc45147bbe006c/execute?api-version=2.0&details=true";
if (_subalgorithm == Calculator1.MATH_SUBTYPES.subalgorithm2.ToString())
{
sBaseURL = "https://ussouthcentral.services.azureml.net/workspaces/d454361ecdcb4ec4b03fa1aec5a7c0e2/services/abd32060dc014d0e8fe1256e0f694daa/execute?api-version=2.0&details=true";
}
}
//r web service is default
string sApiKey =
"fxBeL9LJ3ORm0kW0DtKhT99OfUK6YgBlc59crizYhlxKoEjRd3kuDHvPRuehCQ02VJhPPXcdYTp2pDUynb9gMA==";
if (_subalgorithm == Calculator1.MATH_SUBTYPES.subalgorithm2.ToString())
{
//python
sApiKey =
"/bjDNKx4OWdMIQu6CkvWCIhcfUOCTp9jUE9kD7uylwhOYyhVFOqAFA7M75mJjHS6p6jnAhCvFn1jSl678gzPVA==";
}
string sError = string.Empty;
//convert the script file to the script string expected by the algorithm
List <string> rlines = new List <string>();
rlines = CalculatorHelpers.ReadLines(_params.ExtensionDocToCalcURI, rFile, out sError);
this.ErrorMessage += sError;
if (rlines == null)
{
this.ErrorMessage += string.Concat(" ", Errors.MakeStandardErrorMsg("DATAURL_BAD"));
return(bHasCalcs);
}
if (rlines.Count == 0)
{
this.ErrorMessage += string.Concat(" ", Errors.MakeStandardErrorMsg("DATAURL_BAD"));
return(bHasCalcs);
}
StringBuilder sbR = new StringBuilder();
for (int i = 0; i < rlines.Count(); i++)
{
sbR.AppendLine(rlines[i]);
}
string rScript = sbR.ToString();
//web server expects to store in temp/randomid/name.csv
//web service stores in temp blob
string sOutputDataURL = CalculatorHelpers.GetTempContainerPath("Routput.csv");
//web service expects urls that start with container names
//regular rproject file must be stored in JDataURL
string sInputContainerPath = CalculatorHelpers.GetContainerPathFromFullURIPath("resources", inputFilePath);
//async wait so that results can be stored in output file location and parsed into string lines
SetResponse(sBaseURL, sApiKey, sInputContainerPath, sOutputDataURL, rScript).Wait();
StringBuilder sb = new StringBuilder();
//if web service successully saved the results, the response will start with Success
if (_response.StartsWith("Success"))
{
//return the output file contents in a string list of lines
//must convert container path to full path
string sOutputFullDataURL = string.Concat("https://devtreks1.blob.core.windows.net/", sOutputDataURL);
List <string> lines = new List <string>();
//azure emulator can't process real Azure URL so this won't work
//instead, double check that output url is actually saved
lines = CalculatorHelpers.ReadLines(_params.ExtensionDocToCalcURI, sOutputFullDataURL, out sError);
this.ErrorMessage += sError;
//this results in endless wait-try ReadLinesAsync(sOutputDataURL).ConfigureAwait(false)
//lines = await CalculatorHelpers.ReadLinesAsync(sOutputDataURL);
if (lines == null)
{
this.ErrorMessage += string.Concat(" ", Errors.MakeStandardErrorMsg("DATAURL_BAD"));
return(bHasCalcs);
}
if (lines.Count == 0)
{
this.ErrorMessage += string.Concat(" ", Errors.MakeStandardErrorMsg("DATAURL_BAD"));
return(bHasCalcs);
}
sb = new StringBuilder();
if (_subalgorithm == Calculator1.MATH_SUBTYPES.subalgorithm2.ToString())
{
sb.AppendLine("py results");
}
else
{
sb.AppendLine("r results");
}
//dep var has to be in the R project 1st column
string sLine = string.Concat("first variable: ", _colNames[0]);
string[] line = new List <string>().ToArray();
for (int i = 0; i < lines.Count(); i++)
{
line = lines[i].Split(Constants.CSV_DELIMITERS);
//lineout[1] = CalculatorHelpers.ConvertStringToDouble(line[0]).ToString("N4", CultureInfo.InvariantCulture);
sb.AppendLine(Shared.GetLine(line, false));
}
if (this.MathResult.ToLower().StartsWith("http"))
{
sError = string.Empty;
bool bHasSaved = CalculatorHelpers.SaveTextInURI(
_params.ExtensionDocToCalcURI, sb.ToString(), this.MathResult, out sError);
if (!string.IsNullOrEmpty(sError))
{
this.MathResult += sError;
}
}
else
{
this.MathResult = sb.ToString();
}
bHasCalcs = true;
//last line of string should have the QTM vars
if (line != null)
{
int iPos = 0;
if (line[iPos] != null)
{
this.QTPredicted = CalculatorHelpers.ConvertStringToDouble(line[iPos]);
}
iPos = 1;
if (line[iPos] != null)
{
this.QTL = CalculatorHelpers.ConvertStringToDouble(line[iPos]);
}
iPos = 2;
if (line[iPos] != null)
{
this.QTU = CalculatorHelpers.ConvertStringToDouble(line[iPos]);
}
}
}
else
{
this.ErrorMessage += string.Concat(_response, "The calculations could not be run using the web service.");
}
return(bHasCalcs);
}
public async Task <bool> RunAlgorithmAsync(List <List <double> > data)
{
bool bHasCalculation = false;
try
{
//add the qs to end of list for running in algo
data.Add(_qs);
//this algorith uses standard arrays
double[,] problemData = Shared.GetDoubleArray(data);
StringBuilder sb = new StringBuilder();
//set the rank params needed for the needed matrixes
rows = problemData.GetUpperBound(0) + 1;
cols = problemData.GetUpperBound(1) + 1;
outs = GetDistinctOutputValues(problemData, cols);
inputVarCount = cols - 1;
if (inputVarCount > maxInputs)
{
//no: need to catch it before the data is sent here
}
outputValCount = outs.Count();
rnd = new Random(159); // 159 makes 'good' output
double[][] trainMatrix = null;
double[][] testMatrix = null;
//Generating train and test matrices using an 80%-20% split
MakeTrainAndTest(problemData, out trainMatrix, out testMatrix);
sb.AppendLine("\nFirst few rows of training matrix, using an 80%-20% split, are:");
//First few rows of training matrix are
Helpers.ShowMatrix(sb, trainMatrix, inputVarCount, 5);
//the first and last params are legit; not sure of the correct basis for 2nd
NeuralNetwork nn = new NeuralNetwork(inputVarCount, inputVarCount + 1, outputValCount);
//Training to find best neural network weights using PSO with cross entropy error
double[] bestWeights = nn.Train(sb, trainMatrix);
sb.AppendLine("\nBest weights found:");
Helpers.ShowVector(sb, bestWeights, 2, true);
//Loading best weights into neural network
nn.SetWeights(bestWeights);
sb.AppendLine("\nExamples of the neural network accuracy:\n");
//Analyzing the neural network accuracy on the test data
double accuracy = nn.Test(sb, testMatrix);
this.QTL = accuracy;
this.QTLUnit = "percent accuracy";
sb.AppendLine("Prediction accuracy = " + accuracy.ToString("F4"));
this.QTPredicted = nn.QTPredicted;
sb.AppendLine("Predicted QT = " + this.QTPredicted.ToString("F4"));
if (this.MathResult.ToLower().StartsWith("http"))
{
string sError = string.Empty;
bool bHasSaved = CalculatorHelpers.SaveTextInURI(
_params.ExtensionDocToCalcURI, sb.ToString(), this.MathResult, out sError);
if (!string.IsNullOrEmpty(sError))
{
this.MathResult += sError;
}
}
else
{
this.MathResult = sb.ToString();
}
bHasCalculation = true;
}
catch (Exception ex)
{
this.ErrorMessage = ex.Message;
}
return(bHasCalculation);
}
public async Task <bool> SetObservedMathResult(string mathType, string distType,
string mathSubType, double mostLikely, double[] obsTs)
{
bool bHasSet = false;
//this is observed data rather than randomly sampled data
StringBuilder sb = new StringBuilder();
if (obsTs.Count() > 0)
{
//xy array of 10 points on cdf
//var array = qTs.ToArray();
Array.Sort(obsTs);
int i00 = 0;
int i10 = (int)(obsTs.Length * .10);
int i20 = (int)(obsTs.Length * .20);
int i30 = (int)(obsTs.Length * .30);
int i40 = (int)(obsTs.Length * .40);
int i50 = (int)(obsTs.Length * .50);
int i60 = (int)(obsTs.Length * .60);
int i70 = (int)(obsTs.Length * .70);
int i80 = (int)(obsTs.Length * .80);
int i90 = (int)(obsTs.Length * .90);
int i100 = (int)(obsTs.Length - 1);
sb.AppendLine(Errors.GetMessage("STATS_DESC5"));
sb.AppendLine(string.Concat(
MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(obsTs, obsTs[i00]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(obsTs, obsTs[i10]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(obsTs, obsTs[i20]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(obsTs, obsTs[i30]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(obsTs, obsTs[i40]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(obsTs, obsTs[i50]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(obsTs, obsTs[i60]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(obsTs, obsTs[i70]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(obsTs, obsTs[i80]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(obsTs, obsTs[i90]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(obsTs, obsTs[i100]).ToString("N2", CultureInfo.InvariantCulture)));
sb.AppendLine(string.Concat(
obsTs[i00].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
obsTs[i10].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
obsTs[i20].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
obsTs[i30].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
obsTs[i40].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
obsTs[i50].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
obsTs[i60].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
obsTs[i70].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
obsTs[i80].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
obsTs[i90].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
obsTs[i100].ToString("F4", CultureInfo.InvariantCulture)));
sb.AppendLine(Errors.GetMessage("STATS_DESC3"));
sb.AppendLine(Errors.GetMessage("STATS_DESC2"));
//qT is the result of math and stored in sixth col
var stats = new MathNet.Numerics.Statistics.DescriptiveStatistics(obsTs);
sb.AppendLine(string.Concat(stats.Count.ToString(), Constants.CSV_DELIMITER, obsTs.Sum().ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
stats.Mean.ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
MathNet.Numerics.Statistics.Statistics.Median(obsTs).ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
stats.StandardDeviation.ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, stats.Variance.ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
stats.Minimum.ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, stats.Maximum.ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER));
string sLowerCI = string.Concat(Errors.GetMessage("LOWER"), this.CILevel.ToString(), Errors.GetMessage("CI_PCT"));
string sUpperCI = string.Concat(Errors.GetMessage("UPPER"), this.CILevel.ToString(), Errors.GetMessage("CI_PCT"));
this.IndicatorQT.QTM = stats.Mean;
this.IndicatorQT.QTL = stats.Mean - CalculatorHelpers.GetConfidenceInterval(this.CILevel, stats.Count, stats.StandardDeviation);
this.IndicatorQT.QTU = stats.Mean + CalculatorHelpers.GetConfidenceInterval(this.CILevel, stats.Count, stats.StandardDeviation);
this.IndicatorQT.QTLUnit = sLowerCI;
this.IndicatorQT.QTUUnit = sUpperCI;
}
else
{
sb.AppendLine(Errors.MakeStandardErrorMsg("DATAURL_BADDATA"));
}
if (this.IndicatorQT.MathResult.ToLower().StartsWith("http"))
{
bool bHasSaved = await CalculatorHelpers.SaveTextInURI(
Params.ExtensionDocToCalcURI, sb.ToString(), this.IndicatorQT.MathResult);
if (!string.IsNullOrEmpty(Params.ExtensionDocToCalcURI.ErrorMessage))
{
this.IndicatorQT.MathResult += Params.ExtensionDocToCalcURI.ErrorMessage;
//done with errormsg
Params.ExtensionDocToCalcURI.ErrorMessage = string.Empty;
}
}
else
{
this.IndicatorQT.MathResult = sb.ToString();
}
bHasSet = true;
return(bHasSet);
}
public async Task <bool> SetRange(double mostLikelyEstimate, double lowEstimate, double highEstimate,
double[] data, List <double> cdf = null)
{
bool bHasSet = false;
StringBuilder sb = new StringBuilder();
if (data == null)
{
sb.AppendLine("This indicator does not have the properties needed for descriptive statistics.");
}
if (data.Count() == 0)
{
sb.AppendLine("This indicator does not have the properties needed for descriptive statistics.");
}
//default has no range of values
double lowRange = lowEstimate;
double highRange = highEstimate;
double mostLikely = mostLikelyEstimate;
string mostLikelyUnit = string.Empty;
string lowUnit = string.Empty;
string highUnit = string.Empty;
bool bNeedsCDF = false;
var stats = new MathNet.Numerics.Statistics.DescriptiveStatistics(data);
string sLowerCI = string.Concat(Errors.GetMessage("LOWER"), this.CILevel.ToString(), Errors.GetMessage("CI_PCT"));
string sUpperCI = string.Concat(Errors.GetMessage("UPPER"), this.CILevel.ToString(), Errors.GetMessage("CI_PCT"));
//some data has all zeros and does not use sampled data
if (stats.Mean != 0)
{
mostLikely = stats.Mean;
mostLikelyUnit = "mean";
lowRange = mostLikely - CalculatorHelpers.GetConfidenceInterval(this.CILevel, stats.Count, stats.StandardDeviation);
lowUnit = sLowerCI;
highRange = mostLikely + CalculatorHelpers.GetConfidenceInterval(this.CILevel, stats.Count, stats.StandardDeviation);
highUnit = sUpperCI;
bNeedsCDF = true;
}
if (bNeedsCDF)
{
//csv strings use f4 not n4
sb.AppendLine(Errors.GetMessage("STATS_DESC1"));
sb.AppendLine(Errors.GetMessage("STATS_DESC2"));
sb.AppendLine(string.Concat(stats.Count.ToString(), ", ", data.Sum().ToString("F4", CultureInfo.InvariantCulture), ", ",
stats.Mean.ToString("F4", CultureInfo.InvariantCulture), ", ",
MathNet.Numerics.Statistics.Statistics.Median(data).ToString("F4", CultureInfo.InvariantCulture), ", ",
stats.StandardDeviation.ToString("F4", CultureInfo.InvariantCulture), ", ", stats.Variance.ToString("F4", CultureInfo.InvariantCulture), ", ",
stats.Minimum.ToString("F4", CultureInfo.InvariantCulture), ", ", stats.Maximum.ToString("F4", CultureInfo.InvariantCulture), ", "));
//xy array of 10 points on cdf
var array = data.ToArray();
Array.Sort(array);
int i00 = 0;
int i10 = (int)(array.Length * .10);
int i20 = (int)(array.Length * .20);
int i30 = (int)(array.Length * .30);
int i40 = (int)(array.Length * .40);
int i50 = (int)(array.Length * .50);
int i60 = (int)(array.Length * .60);
int i70 = (int)(array.Length * .70);
int i80 = (int)(array.Length * .80);
int i90 = (int)(array.Length * .90);
int i100 = (int)(array.Length - 1);
sb.AppendLine(Errors.GetMessage("STATS_DESC4"));
sb.AppendLine(string.Concat(
MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(data, array[i00]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(data, array[i10]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(data, array[i20]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(data, array[i30]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(data, array[i40]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(data, array[i50]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(data, array[i60]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(data, array[i70]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(data, array[i80]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(data, array[i90]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(data, array[i100]).ToString("N2", CultureInfo.InvariantCulture)));
sb.AppendLine(string.Concat(
array[i00].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
array[i10].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
array[i20].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
array[i30].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
array[i40].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
array[i50].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
array[i60].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
array[i70].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
array[i80].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
array[i90].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER,
array[i100].ToString("F4", CultureInfo.InvariantCulture)));
if (cdf == null)
{
cdf = new List <double>();
}
cdf.Add(array[i00]);
cdf.Add(array[i10]);
cdf.Add(array[i20]);
cdf.Add(array[i30]);
cdf.Add(array[i40]);
cdf.Add(array[i50]);
cdf.Add(array[i60]);
cdf.Add(array[i70]);
cdf.Add(array[i80]);
cdf.Add(array[i90]);
cdf.Add(array[i100]);
}
else
{
sb.AppendLine(Errors.GetMessage("STATS_DESC6"));
}
//doubles need restricted digits
mostLikely = Math.Round(mostLikely, 4);
lowRange = Math.Round(lowRange, 4);
highRange = Math.Round(highRange, 4);
string sD1Unit = "low";
string sD2Unit = "high";
this.IndicatorQT.QTD1 = lowEstimate;
if (this.IndicatorQT.QTD1Unit == string.Empty || this.IndicatorQT.QTD1Unit == Constants.NONE)
{
this.IndicatorQT.QTD1Unit = sD1Unit;
}
this.IndicatorQT.QTD2 = highEstimate;
if (this.IndicatorQT.QTD2Unit == string.Empty || this.IndicatorQT.QTD2Unit == Constants.NONE)
{
this.IndicatorQT.QTD2Unit = sD2Unit;
}
//computed results
this.IndicatorQT.QTM = mostLikely;
if (this.IndicatorQT.QTMUnit == string.Empty || this.IndicatorQT.QTMUnit == Constants.NONE)
{
this.IndicatorQT.QTMUnit = mostLikelyUnit;
}
this.IndicatorQT.QTL = lowRange;
if (this.IndicatorQT.QTLUnit == string.Empty || this.IndicatorQT.QTLUnit == Constants.NONE)
{
this.IndicatorQT.QTLUnit = lowUnit;
}
this.IndicatorQT.QTU = highRange;
if (this.IndicatorQT.QTUUnit == string.Empty || this.IndicatorQT.QTUUnit == Constants.NONE)
{
this.IndicatorQT.QTUUnit = highUnit;
}
if (this.IndicatorQT.MathResult.ToLower().StartsWith("http"))
{
bool bHasSaved = await CalculatorHelpers.SaveTextInURI(
Params.ExtensionDocToCalcURI, sb.ToString(), this.IndicatorQT.MathResult);
if (!string.IsNullOrEmpty(Params.ExtensionDocToCalcURI.ErrorMessage))
{
this.IndicatorQT.MathResult += Params.ExtensionDocToCalcURI.ErrorMessage;
//done with errormsg
Params.ExtensionDocToCalcURI.ErrorMessage = string.Empty;
}
}
else
{
this.IndicatorQT.MathResult = sb.ToString();
}
bHasSet = true;
return(bHasSet);
}
/// <summary>
/// Run the simulated annealing algo
/// </summary>
/// <param name="problemData">matrix of energy requirements (hour to complete task)</param>
/// <param name="currTemp">initial temperature from which cooldown starts</param>
/// <param name="alpha">cooling rate</param>
/// <param name="penalty">a penalty when a worker has more than 1 task</param>
/// <param name="maxIteration">number of iterations</param>
public async Task <bool> RunAlgorithmAsync(List <List <double> > data)
{
bool bHasCalculation = false;
try
{
random = new Random(0);
//this algorith uses standard arrays
double[,] problemData = Shared.GetDoubleArray(data);
int[] state = RandomState(problemData);
double energy = Energy(state, problemData, _penalty);
int[] bestState = state;
double bestEnergy = energy;
int[] adjState;
double adjEnergy;
int iteration = 0;
while (iteration < _maxiteration && _currtemp > 0.0001)
{
adjState = AdjacentState(state, problemData);
adjEnergy = Energy(adjState, problemData, _penalty);
if (adjEnergy < bestEnergy)
{
bestState = adjState;
bestEnergy = adjEnergy;
}
double p = random.NextDouble(); // [0, 1.0)
if (AcceptanceProb(energy, adjEnergy, _currtemp) > p)
{
state = adjState;
energy = adjEnergy;
}
_currtemp = _currtemp * _alpha; // cool down; annealing schedule
++iteration;
} // while
this.BestEnergy = bestEnergy;
if (this.MathResult.ToLower().StartsWith("http"))
{
bool bHasSaved = await CalculatorHelpers.SaveTextInURI(_params.ExtensionDocToCalcURI,
Interpret(bestState, problemData), this.MathResult);
if (!string.IsNullOrEmpty(_params.ExtensionDocToCalcURI.ErrorMessage))
{
this.MathResult += _params.ExtensionDocToCalcURI.ErrorMessage;
//done with errormsg
_params.ExtensionDocToCalcURI.ErrorMessage = string.Empty;
}
}
else
{
this.MathResult = Interpret(bestState, problemData);
}
bHasCalculation = true;
}
catch (Exception ex)
{
ErrorMessage = ex.Message;
}
return(bHasCalculation);
}
//this is asych for the calling Task.WhenAll
//but does not necessarily need internal asych awaits
public async Task <bool> RunAlgorithmAsync(List <List <double> > data)
{
bool bHasCalculations = false;
try
{
//minimal data requirement is first five cols
if (_colNames.Count() < 5 ||
_mathTerms.Count() == 0)
{
ErrorMessage = "Complete randomized anova requires at least 1 dependent variable and 1 independent variable. Randomized block and factorial anovas require at least 2 independent variables.";
return(bHasCalculations);
}
if (data.Count() < 5)
{
//185 same as other analysis
ErrorMessage = "Anova requires at least 2 rows of observed data and 3 rows of scoring data.";
return(bHasCalculations);
}
//convert data to a Math.Net Matrix
//last 3 rows are used to generate ci
List <List <double> > dataci = data.Skip(data.Count - _scoreRows).ToList();
data.Reverse();
List <List <double> > dataobs = data.Skip(_scoreRows).ToList();
dataobs.Reverse();
//actual observed values
Vector <double> y = Shared.GetYData(dataobs);
//treatments or factor1 levels
Matrix <double> treatments = Shared.GetDistinctMatrix(dataobs, 1);
//206 condition added due to M and E dataset indexing
_totalsNeeded = treatments.ColumnCount;
double tDF = treatments.ColumnCount - 1;
//step 1. get total of observed data
double yTotal = y.Sum();
//step 2. get total of observed data squared
double yTotal2 = y.PointwisePower(2).Sum();
//step 3. set CM
double CM = Math.Pow(yTotal, 2) / y.Count();
//step 4. treatments and blocks
double SSTotal = yTotal2 - CM;
//add the data to a string builder
StringBuilder sb = new StringBuilder();
sb.AppendLine("anova results");
//5 col array
string[] cols = new string[] { "source", "df", "SS", "MS", "F" };
sb.AppendLine(Shared.GetLine(cols, true));
List <List <double> > totals = new List <List <double> >(_totalsNeeded);
//min treatment block required
bool bIsBlock = (_depColNames.Contains("treatment") && _depColNames.Contains("block")) ? true : false;
//min 2 factors required
bool bIsFactorial = (_depColNames.Contains("factor1") && _depColNames.Contains("factor2")) ? true : false;
bool bIsComplete = (bIsBlock == false && bIsFactorial == false) ? true : false;
if (bIsComplete)
{
double eDF = y.Count() - treatments.ColumnCount;
//step 5.treatments (correct to divide by rows)
double SST = ((treatments.ColumnSums().PointwisePower(2).Sum()) / treatments.RowCount) - CM;
//step 6. error
double SSE = SSTotal - SST;
//step 7. mean treatment
double MST = SST / tDF;
//step 8. mean error
double MSE = SSE / (y.Count() - treatments.ColumnCount);
//step 9. F treatments
double FT = MST / MSE;
//tests
double s = Math.Pow(MSE, 0.5);
//correct 2 tailed t test
int itDF = CalculatorHelpers.ConvertStringToInt(tDF.ToString());
int ieDF = CalculatorHelpers.ConvertStringToInt(eDF.ToString());
double dbCI = CalculatorHelpers.GetConfidenceIntervalProb(_confidenceInt);
//TINV divides dbCI by 2 to get student t
double tCriticalValue = ExcelFunctions.TInv(dbCI, ieDF);
//prevents an error in Finv
if (itDF == 0)
{
itDF = 1;
}
double FCriticalTValue = ExcelFunctions.FInv(dbCI, itDF, ieDF);
string FTGreaterFCritical = (FT > FCriticalTValue) ? "true" : "false";
for (int i = 0; i < _totalsNeeded; i++)
{
//206 condition added due to M and E dataset indexing
if (i < treatments.ColumnCount)
{
SetAnovaIntervals(i, totals, treatments, tCriticalValue, s,
CalculatorHelpers.ConvertStringToDouble(treatments.RowCount.ToString()),
FT, FCriticalTValue, bIsComplete);
}
}
this.DataToAnalyze.Add(Label, totals);
////add the data to a string builder
cols = new string[] { "treats", itDF.ToString("F0"), SST.ToString("F4"), MST.ToString("F4"), FT.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { "error ", ieDF.ToString("F0"), SSE.ToString("F4"), MSE.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { "total ", (y.Count() - 1).ToString("F0"), (SSTotal).ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { string.Concat("F Crit ", "treats"), FCriticalTValue.ToString("F5"), "F > F Critical", FTGreaterFCritical };
sb.AppendLine(Shared.GetLine(cols, true));
}
else
{
//observations per cell for factorials (data[0] if first row of data)
double r = Shared.GetObservationsPerCell(dataobs, 1, data[0].ElementAt(1), data[0].ElementAt(2));
//blocks or factor2 levels
Matrix <double> blocks = Shared.GetDistinctMatrix(dataobs, 2);
double bDF = blocks.ColumnCount - 1;
double eDF = y.Count() - treatments.ColumnCount - blocks.ColumnCount + 1;
if (bIsFactorial)
{
eDF = (treatments.ColumnCount * blocks.ColumnCount) * (r - 1);
}
//factorial interaction df
double tbDF = tDF * bDF;
//step 5.treatments (correct to divide by r)
double SST = ((treatments.ColumnSums().PointwisePower(2).Sum()) / (blocks.ColumnCount * r)) - CM;
//step 6. blocks
double SSB = ((blocks.ColumnSums().PointwisePower(2).Sum()) / (treatments.ColumnCount * r)) - CM;
//factor level interaction
double SSFL = 0;
//step 7. error
double SSE = 0;
if (bIsFactorial)
{
double totalinteraction = Shared.GetTotalInteraction(blocks, r);
//watch block.colcount for 2 x 3 factorials
SSFL = (totalinteraction / r) - SST - SSB - CM;
//step 7. error
SSE = SSTotal - SST - SSB - SSFL;
}
else
{
//step 7. error
SSE = SSTotal - SST - SSB;
}
//step 8. mean treatment
double MST = SST / tDF;
//step 9. mean block
double MSB = SSB / bDF;
//step 10. mean error
double MSE = SSE / eDF;
double MSFL = SSFL / tbDF;
//step 11. F treatments
double FT = MST / MSE;
//step 12. F blocks
double FB = MSB / MSE;
double FTB = MSFL / MSE;
//tests
double s = Math.Pow(MSE, 0.5);
//correct 2 tailed t test
int itDF = CalculatorHelpers.ConvertStringToInt(tDF.ToString());
int ibDF = CalculatorHelpers.ConvertStringToInt(bDF.ToString());
int itbDF = CalculatorHelpers.ConvertStringToInt(tbDF.ToString());
int ieDF = CalculatorHelpers.ConvertStringToInt(eDF.ToString());
double dbCI = CalculatorHelpers.GetConfidenceIntervalProb(_confidenceInt);
//TINV divides dbCI by 2 to get student t
double tCriticalValue = ExcelFunctions.TInv(dbCI, ieDF);
//prevents an error in Finv
if (itDF == 0)
{
itDF = 1;
}
double FCriticalTValue = ExcelFunctions.FInv(dbCI, itDF, ieDF);
string FTGreaterFCritical = (FT > FCriticalTValue) ? "true" : "false";
//prevents an error in Finv
if (ibDF == 0)
{
ibDF = 1;
}
double FCriticalBValue = ExcelFunctions.FInv(dbCI, ibDF, ieDF);
string FBGreaterFCritical = (FB > FCriticalBValue) ? "true" : "false";
//prevents an error in Finv
if (itbDF == 0)
{
itbDF = 1;
}
double FCriticalTBValue = ExcelFunctions.FInv(dbCI, itbDF, ieDF);
string FTBGreaterFCritical = (FTB > FCriticalTBValue) ? "true" : "false";
//List<List<double>> totals = new List<List<double>>(_totalsNeeded);
if (bIsFactorial)
{
//unless custom stylesheets are developed, can only display factor 1 - factor 2 diffs
//build a matrix equivalent to treatments -1 row, variable cols
Matrix <double> torbs = Matrix <double> .Build.Dense(1, _totalsNeeded);
List <double> mrow = new List <double>(_totalsNeeded);
for (int i = 0; i < _totalsNeeded; i++)
{
double cellMean = Shared.GetMeanPerCell(dataobs, 1, 2, i, i, r);
mrow.Add(cellMean);
}
torbs.SetRow(0, mrow.ToArray());
for (int i = 0; i < _totalsNeeded; i++)
{
//206 condition added due to M and E dataset indexing
if (i < treatments.ColumnCount)
{
//treatments
SetAnovaIntervals(i, totals, torbs, tCriticalValue, s,
CalculatorHelpers.ConvertStringToDouble(blocks.ColumnCount.ToString()),
FT, FCriticalTValue, bIsComplete);
}
}
//cell1Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 0, 0, r);
//cell2Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 0, 1, r);
//SetAnovaIntervals2(1, cell1Mean, cell2Mean, tCriticalValue, s, r);
//cell1Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 0, 0, r);
//cell2Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 2, 0, r);
//SetAnovaIntervals2(2, cell1Mean, cell2Mean, tCriticalValue, s, r);
//double cell1Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 0, 0, r);
//double cell2Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 1, 0, r);
//SetAnovaIntervals2(0, cell1Mean, cell2Mean, tCriticalValue, s, r);
//cell1Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 0, 0, r);
//cell2Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 0, 1, r);
//SetAnovaIntervals2(1, cell1Mean, cell2Mean, tCriticalValue, s, r);
//cell1Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 0, 0, r);
//cell2Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 2, 0, r);
//SetAnovaIntervals2(2, cell1Mean, cell2Mean, tCriticalValue, s, r);
}
else
{
//unless custom stylesheets are developed, need to only display treatment diffs
for (int i = 0; i < _totalsNeeded; i++)
{
//206 condition added due to M and E dataset indexing
if (i < treatments.ColumnCount)
{
//treatments
SetAnovaIntervals(i, totals, treatments, tCriticalValue, s,
CalculatorHelpers.ConvertStringToDouble(blocks.ColumnCount.ToString()),
FT, FCriticalTValue, bIsComplete);
////blocks
//SetAnovaIntervals(i, totals, blocks, tCriticalValue, s,
// CalculatorHelpers.ConvertStringToDouble(treatments.ColumnCount.ToString()),
// FB, FCriticalBValue, bIsComplete);
//interactions
}
}
}
this.DataToAnalyze.Add(Label, totals);
string sTreats = "treats ";
string sBlocks = "blocks ";
if (bIsFactorial)
{
sTreats = "factor1 ";
sBlocks = "factor2 ";
}
////add the data to a string builder
//StringBuilder sb = new StringBuilder();
//sb.AppendLine("anova results");
cols = new string[] { sTreats, itDF.ToString("F0"), SST.ToString("F4"), MST.ToString("F4"), FT.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { sBlocks, bDF.ToString("F0"), SSB.ToString("F4"), MSB.ToString("F4"), FB.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
if (bIsFactorial)
{
cols = new string[] { "interacts ", tbDF.ToString("F0"), SSFL.ToString("F4"), MSFL.ToString("F4"), FTB.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
}
cols = new string[] { "error ", ieDF.ToString("F0"), SSE.ToString("F4"), MSE.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { "total ", (y.Count() - 1).ToString("F0"), (SSTotal).ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
cols = new string[] { string.Concat("F Crit ", sTreats), FCriticalTValue.ToString("F5"), "F > F Critical", FTGreaterFCritical };
sb.AppendLine(Shared.GetLine(cols, true));
cols = new string[] { string.Concat("F Crit ", sBlocks), FCriticalBValue.ToString("F5"), "F > F Critical", FBGreaterFCritical };
sb.AppendLine(Shared.GetLine(cols, true));
if (bIsFactorial)
{
cols = new string[] { "F Crit Interacts", FCriticalTBValue.ToString("F5"), "F > F Critical", FTBGreaterFCritical };
sb.AppendLine(Shared.GetLine(cols, true));
}
}
cols = new string[] { "estimate", "mean diff", string.Concat("lower ", _confidenceInt.ToString(), "%"), string.Concat("upper ", _confidenceInt.ToString(), "%") };
sb.AppendLine(Shared.GetLine(cols, true));
//same report for calculator and analyzer
for (int i = 0; i < _totalsNeeded; i++)
{
if (totals[i].Count >= 5)
{
if (i == 0)
{
QTPredicted = totals[i].ElementAt(0);
QTL = QTPredicted - totals[i].ElementAt(4);
QTU = QTPredicted + totals[i].ElementAt(4);
cols = new string[] { "Treat 1 Mean ", QTPredicted.ToString("F4"), QTL.ToString("F4"), QTU.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
}
else
{
QTPredicted = totals[i].ElementAt(1);
QTL = QTPredicted - totals[i].ElementAt(2);
QTU = QTPredicted + totals[i].ElementAt(2);
cols = new string[] { string.Concat("xminus1 ", i.ToString(), " "), QTPredicted.ToString("F4"), QTL.ToString("F4"), QTU.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
QTPredicted = totals[i].ElementAt(3);
QTL = QTPredicted - totals[i].ElementAt(4);
QTU = QTPredicted + totals[i].ElementAt(4);
cols = new string[] { string.Concat("base ", i.ToString(), " "), QTPredicted.ToString("F4"), QTL.ToString("F4"), QTU.ToString("F4") };
sb.AppendLine(Shared.GetLine(cols, false));
}
}
}
if (this.MathResult.ToLower().StartsWith("http"))
{
bool bHasSaved = await CalculatorHelpers.SaveTextInURI(
_params.ExtensionDocToCalcURI, sb.ToString(), this.MathResult);
if (!string.IsNullOrEmpty(_params.ExtensionDocToCalcURI.ErrorMessage))
{
this.MathResult += _params.ExtensionDocToCalcURI.ErrorMessage;
//done with errormsg
_params.ExtensionDocToCalcURI.ErrorMessage = string.Empty;
}
}
else
{
this.MathResult = sb.ToString();
}
bHasCalculations = true;
}
catch (Exception ex)
{
this.ErrorMessage = ex.Message;
}
return(bHasCalculations);
}
//running this truly async returns to UI w/o saving final calcs or an endless wait
public async Task <bool> RunAlgorithmAsync(string inputFilePath, string scriptFilePath,
System.Threading.CancellationToken ctk)
{
bool bHasCalcs = false;
try
{
this.ErrorMessage = string.Empty;
if (string.IsNullOrEmpty(inputFilePath) || (!inputFilePath.EndsWith(".csv")))
{
this.ErrorMessage = "The dataset file URL has not been added to the Data URL. The file must be stored in a Resource and use a csv file extension.";
}
if (string.IsNullOrEmpty(scriptFilePath) || (!scriptFilePath.EndsWith(".txt")))
{
this.ErrorMessage += "The script file URL has not been added to the Joint Data.The file must be stored in a Resource and use a txt file extension.";
}
//unblock after debug
if (!string.IsNullOrEmpty(this.ErrorMessage))
{
return(bHasCalcs);
}
string sError = string.Empty;
StringBuilder sb = new StringBuilder();
//get the path to the script executable
string sScriptExecutable = CalculatorHelpers.GetAppSettingString(
this._params.ExtensionDocToCalcURI, "RExecutable");
if (_algorithm == Calculator1.MATH_TYPES.algorithm3.ToString())
{
//python must be installed to automatically run
sScriptExecutable = CalculatorHelpers.GetAppSettingString(
this._params.ExtensionDocToCalcURI, "PyExecutable");
if (_subalgorithm == Calculator1.MATH_SUBTYPES.subalgorithm1.ToString())
{
//python scripts must be run by executable as '.pyw' files
//save the 'txt' file as a 'pyw' file in temp path
//has to be done each time because can't be sure when scriptfile changed last
if (!scriptFilePath.EndsWith(".pyw"))
{
string sPyScript = CalculatorHelpers.ReadText(_params.ExtensionDocToCalcURI, scriptFilePath, out sError);
if (!string.IsNullOrEmpty(sPyScript))
{
string sFileName = Path.GetFileName(scriptFilePath);
string sPyScriptFileName = sFileName.Replace(".txt", ".pyw");
bool bIsLocalCache = false;
string sPyPath = CalculatorHelpers.GetTempDocsPath(_params.ExtensionDocToCalcURI, bIsLocalCache, sPyScriptFileName);
bool bHasFile = CalculatorHelpers.SaveTextInURI(_params.ExtensionDocToCalcURI, sPyScript, sPyPath, out sError);
scriptFilePath = sPyPath;
}
}
}
sb.AppendLine("python results");
}
else
{
if (_subalgorithm == Calculator1.MATH_SUBTYPES.subalgorithm1.ToString())
{
//check for 2.0.2 -R Open can run from a url
//rscript.exe can't run from a url
if (scriptFilePath.StartsWith("http"))
{
//convert it to a filesystem path
//make sure that both localhost and localhost:44300 have a copy of the file
string sRFilePath = CalculatorHelpers.ConvertFullURIToFilePath(
this._params.ExtensionDocToCalcURI, scriptFilePath);
scriptFilePath = sRFilePath;
}
}
//r is default
sb.AppendLine("r results");
}
string sLastLine = string.Empty;
//2.0.2: algo 2 subalgo2 is r or algo 3 subalgo2 Python; subalgo 2 is virtual machine
if (_subalgorithm == Calculator1.MATH_SUBTYPES.subalgorithm2.ToString())
{
//run on remote servers that have the DevTreksStatsApi WebApi app deployed
string sStatType = Data.Helpers.StatScript.STAT_TYPE.r.ToString();
if (_algorithm == Calculator1.MATH_TYPES.algorithm3.ToString())
{
//python is significantly slower than R
sStatType = Data.Helpers.StatScript.STAT_TYPE.py.ToString();
}
else if (_algorithm == Calculator1.MATH_TYPES.algorithm6.ToString())
{
//julia has not been tested in 2.0.2
sStatType = Data.Helpers.StatScript.STAT_TYPE.julia.ToString();
}
DevTreks.Data.Helpers.StatScript statScript
= DevTreks.Data.Helpers.StatScript.GetStatScript(
sStatType, scriptFilePath, inputFilePath);
string sPlatformType = CalculatorHelpers.GetAppSettingString(
this._params.ExtensionDocToCalcURI, "PlatformType");
if (sPlatformType.Contains("azure"))
{
//webapi web domain
if (inputFilePath.Contains("localhost"))
{
statScript.DefaultWebDomain = "http://localhost:5000/";
}
else
{
statScript.DefaultWebDomain = "http://devtreksapi1.southcentralus.cloudapp.azure.com/";
}
}
else
{
if (inputFilePath.Contains("localhost"))
{
statScript.DefaultWebDomain = "http://localhost:5000/";
}
else
{
//run tests on cloud webapi site too
statScript.DefaultWebDomain = "http://devtreksapi1.southcentralus.cloudapp.azure.com/";
}
}
//use a console app to post to a webapi CreateClient controller action
bool bIsSuccess = await CalculatorHelpers.ClientCreate(statScript);
if (bIsSuccess)
{
if ((!string.IsNullOrEmpty(statScript.StatisticalResult)))
{
List <string> lines = CalculatorHelpers
.GetLinesFromUTF8Encoding(statScript.StatisticalResult);
if (lines != null)
{
if (lines.Count > 0)
{
//store the result in the MathResult (or in the MathResult.URL below)
sb.Append(statScript.StatisticalResult);
sLastLine = lines.Last();
if (string.IsNullOrEmpty(sLastLine))
{
int iSecondToLast = lines.Count - 2;
sLastLine = lines[iSecondToLast];
}
}
}
}
else
{
if ((!string.IsNullOrEmpty(statScript.StatisticalResult)))
{
this.MathResult += statScript.ErrorMessage;
}
else
{
this.MathResult += "The remote server returned a successful response header but failed to generate the statistical results.";
}
}
}
}
else
{
//default subalgo1 runs statpackages on the same server
sLastLine = RunScript(sb, sScriptExecutable, scriptFilePath, inputFilePath);
}
if (this.MathResult.ToLower().StartsWith("http"))
{
sError = string.Empty;
bool bHasSaved = CalculatorHelpers.SaveTextInURI(
_params.ExtensionDocToCalcURI, sb.ToString(), this.MathResult, out sError);
if (!string.IsNullOrEmpty(sError))
{
this.MathResult += sError;
}
}
else
{
this.MathResult = sb.ToString();
}
bHasCalcs = true;
//last line of string should have the QTM vars
if (!string.IsNullOrEmpty(sLastLine))
{
string[] vars = sLastLine.Split(Constants.CSV_DELIMITERS);
bool bHasVars = false;
if (vars != null)
{
if (vars.Count() > 1)
{
bHasVars = true;
}
if (!bHasVars)
{
//try space delimited
vars = sLastLine.Split(' ');
bHasVars = true;
}
if (vars != null)
{
//row count may be in first pos
int iPos = vars.Count() - 3;
if (vars[iPos] != null)
{
this.QTPredicted = CalculatorHelpers.ConvertStringToDouble(vars[iPos]);
}
iPos = vars.Count() - 2;
if (vars[iPos] != null)
{
this.QTL = CalculatorHelpers.ConvertStringToDouble(vars[iPos]);
}
iPos = vars.Count() - 1;
if (vars[iPos] != null)
{
this.QTU = CalculatorHelpers.ConvertStringToDouble(vars[iPos]);
}
}
}
}
else
{
this.MathResult = "The script did not run successfully. Please check the dataset and script. Verify their urls.";
}
}
catch (Exception ex)
{
this.ErrorMessage += ex.Message;
}
return(bHasCalcs);
}
