private async Task <bool> SetMathResult(List <List <string> > rowNames, StringBuilder sb = null) { bool bHasSet = false; if (sb == null) { //add the data to a string builder sb = new StringBuilder(); if (_subalgorithm == MATHML_SUBTYPES.subalgorithm_03.ToString()) { sb.AppendLine("ml results"); string[] newColNames = new string[_actualColNames.Length + 5]; for (int i = 0; i < _actualColNames.Length; i++) { newColNames[i] = _actualColNames[i]; } //new cols changed by algo newColNames[_actualColNames.Length] = "mse"; newColNames[_actualColNames.Length + 1] = "accuracy"; newColNames[_actualColNames.Length + 2] = "qtm"; newColNames[_actualColNames.Length + 3] = "qtl"; newColNames[_actualColNames.Length + 4] = "qtu"; _actualColNames = newColNames; CalculatorHelpers.SetIndMathResult(sb, _actualColNames, rowNames, DataResults); } } if (this.IndicatorQT.MathResult.ToLower().StartsWith("http")) { bool bHasSaved = await CalculatorHelpers.SaveTextInURI( Params.ExtensionDocToCalcURI, sb.ToString(), this.IndicatorQT.MathResult); if (!string.IsNullOrEmpty(Params.ExtensionDocToCalcURI.ErrorMessage)) { this.IndicatorQT.MathResult += Params.ExtensionDocToCalcURI.ErrorMessage; //done with errormsg Params.ExtensionDocToCalcURI.ErrorMessage = string.Empty; } else { bHasSet = true; } } else { this.IndicatorQT.MathResult = sb.ToString(); bHasSet = true; } return(bHasSet); }
//this is asych for the calling Task.WhenAll //but does not necessarily need internal asych awaits public async Task RunAlgorithmAsync(List <List <double> > data) { try { //minimal data requirement is first five cols if (_colNames.Count() < 5 || _mathTerms.Count() == 0) { ErrorMessage = "Regression requires at least one dependent variable and one independent variable."; return; } if (data.Count() < 5) { //185 same as other analysis ErrorMessage = "Regression requires at least 2 rows of observed data and 3 rows of scoring data."; return; } //convert data to a Math.Net Matrix //v185 uses same ci technique as algos 2,3 and 4 -last 3 rows are used to generate ci List <List <double> > dataci = data.Skip(data.Count - _scoreRows).ToList(); data.Reverse(); List <List <double> > dataobs = data.Skip(_scoreRows).ToList(); dataobs.Reverse(); //actual observed values Vector <double> y = Shared.GetYData(dataobs); Matrix <double> x = Shared.GetDoubleMatrix(dataobs, _colNames, _depColNames); Matrix <double> ci = Shared.GetDoubleMatrix(dataci, _colNames, _depColNames); //model expected values - get the coefficents //use normal equations regression Vector <double> p = MultipleRegression.NormalEquations(x, y); //but note that this runs without errors in more cases but still does not give good results //Vector<double> p = MultipleRegression.QR(x, y); if (p.Count() != ci.Row(_scoreRows - 1).Count()) { //185 same as other analysis ErrorMessage = "The scoring and training datasets have different numbers of columns."; return; } //get the predicted yhats Vector <double> yhat = GetYHatandSetQTPred(y.Count, x, p, ci.Row(_scoreRows - 1).ToArray()); //get the durbin-watson d statistic double d = GetDurbinWatson(y, yhat); double SSE = 0; //sum of the square of the error (between the predicted, p, and observed, y); SSE = Distance.SSD(yhat, y); double rSquared = GoodnessOfFit.RSquared(yhat, y); //sum of the square of the regression (between the predicted, p, and observed mean, statsY.Mean); double SSR = 0; for (int i = 0; i < yhat.Count(); i++) { SSR += Math.Pow((yhat[i] - y.Mean()), 2); } //set joint vars properties //degrees freedom double dfR = x.ColumnCount - 1; double dfE = x.RowCount - x.ColumnCount; int idfR = x.ColumnCount - 1; int idfE = x.RowCount - x.ColumnCount; double s2 = SSE / dfE; double s = Math.Sqrt(s2); double MSR = SSR / dfR; double MSE = SSE / dfE; double FValue = MSR / MSE; double adjRSquared = 1 - ((x.RowCount - 1) * (MSE / (SSE + SSR))); double pValue = Shared.GetPValueForFDist(idfR, idfE, FValue); //correct 2 tailed t test //double TCritialValue = ExcelFunctions.TInv(0.05, idfE); //so do this double dbCI = CalculatorHelpers.GetConfidenceIntervalProb(_confidenceInt); double tCriticalValue = ExcelFunctions.TInv(dbCI, idfE); //set each coeff properties //coeffs st error //use matrix math to get the standard error of coefficients Matrix <double> xt = x.Transpose(); //matrix x'x Matrix <double> xx = xt.Multiply(x); Matrix <double> xxminus1 = xx.Inverse(); double sxx = 0; double[] xiSE = new double[x.ColumnCount]; //coeff tstats double[] xiT = new double[x.ColumnCount]; //lower value for pvalue double[] xiP = new double[x.ColumnCount]; for (int i = 0; i < x.ColumnCount; i++) { //use the matrix techniques shown on p 717 of Mendenhall and Sincich sxx = s * Math.Sqrt(xxminus1.Column(i)[i]); xiSE[i] = sxx; xiT[i] = p[i] / sxx; xiP[i] = Shared.GetPValueForTDist(idfE, xiT[i], 0, 1); } double FCriticalValue = 0; string FGreaterFCritical = string.Empty; if (_subalgorithm == Calculator1.MATH_SUBTYPES.subalgorithm8.ToString()) { //anova regression //anova critical fvalue test //FCriticalValue = ExcelFunctions.FInv(1 - _confidenceInt, idfR, idfE); FCriticalValue = ExcelFunctions.FInv(dbCI, idfR, idfE); FGreaterFCritical = (FValue > FCriticalValue) ? "true" : "false"; SetAnovaIntervals(0, p, xiSE, tCriticalValue); SetAnovaIntervals(1, p, xiSE, tCriticalValue); SetAnovaIntervals(2, p, xiSE, tCriticalValue); } else { //set QTM ci and pi intervals SetQTIntervals(0, s, xxminus1, ci.Row(_scoreRows - 1).ToArray(), p, tCriticalValue); SetQTIntervals(1, s, xxminus1, ci.Row(_scoreRows - 2).ToArray(), p, tCriticalValue); SetQTIntervals(2, s, xxminus1, ci.Row(_scoreRows - 3).ToArray(), p, tCriticalValue); } //add the data to a string builder StringBuilder sb = new StringBuilder(); sb.AppendLine("regression results"); //dep var has to be in the 4 column always string sLine = string.Concat("dependent variable: ", _colNames[3]); sb.AppendLine(sLine); string[] cols = new string[] { "source", "df", "SS", "MS" }; sb.AppendLine(Shared.GetLine(cols, true)); cols = new string[] { "model", dfR.ToString("F0"), SSR.ToString("F4"), MSR.ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); cols = new string[] { "error ", dfE.ToString("F0"), SSE.ToString("F4"), MSE.ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); cols = new string[] { "total ", (dfR + dfE).ToString("F0"), (SSR + SSE).ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); sb.AppendLine(string.Empty); cols = new string[] { "R-squared", rSquared.ToString("F4"), "Adj R-squared", adjRSquared.ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); cols = new string[] { "F value", FValue.ToString("F4"), "prob > F", pValue.ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); sb.AppendLine(string.Empty); cols = new string[] { GetName("variable"), "coefficient", "stand error", "T-ratio", "prob > T" }; sb.AppendLine(Shared.GetLine(cols, true)); for (int i = 0; i < p.Count(); i++) { if (i == 0) { cols = new string[] { GetName(_depColNames[i]), p[i].ToString("F5"), xiSE[i].ToString("F4"), xiT[i].ToString("F4"), xiP[i].ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); } else { cols = new string[] { GetName(_depColNames[i]), p[i].ToString("F5"), xiSE[i].ToString("F4"), xiT[i].ToString("F4"), xiP[i].ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); } } cols = new string[] { "durbin-watson: ", d.ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); if (_subalgorithm == Calculator1.MATH_SUBTYPES.subalgorithm8.ToString()) { cols = new string[] { "F Critical Value", FCriticalValue.ToString("F5"), "F > F Critical", FGreaterFCritical }; sb.AppendLine(Shared.GetLine(cols, true)); cols = new string[] { "estimate", "predicted", string.Concat("lower ", _confidenceInt.ToString(), "%"), string.Concat("upper ", _confidenceInt.ToString(), "%") }; sb.AppendLine(Shared.GetLine(cols, true)); cols = new string[] { "Col 0 Mean CI ", QTPredicted.ToString("F4"), QTL.ToString("F4"), QTU.ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); cols = new string[] { "Col 1 - 0 Mean CI ", QTPredicted10.ToString("F4"), QTL10.ToString("F4"), QTU10.ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); cols = new string[] { "Col 2 - 0 Mean CI ", QTPredicted20.ToString("F4"), QTL20.ToString("F4"), QTU20.ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); } else { cols = new string[] { "estimate", "predicted", string.Concat("lower ", _confidenceInt.ToString(), "%"), string.Concat("upper ", _confidenceInt.ToString(), "%") }; sb.AppendLine(Shared.GetLine(cols, true)); cols = new string[] { "QTM CI ", QTPredicted.ToString("F4"), QTL.ToString("F4"), QTU.ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); cols = new string[] { "QTM PI ", QTPredicted.ToString("F4"), (QTPredicted - QTPI).ToString("F4"), (QTPredicted + QTPI).ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); string sRow = string.Concat("row ", data.Count - 2); cols = new string[] { sRow }; sb.AppendLine(Shared.GetLine(cols, true)); cols = new string[] { "CI ", QTPredicted10.ToString("F4"), QTL10.ToString("F4"), QTU10.ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); cols = new string[] { "PI ", QTPredicted10.ToString("F4"), (QTPredicted10 - QTPI10).ToString("F4"), (QTPredicted10 + QTPI10).ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); sRow = string.Concat("row ", data.Count - 1); cols = new string[] { sRow }; sb.AppendLine(Shared.GetLine(cols, true)); cols = new string[] { "CI ", QTPredicted20.ToString("F4"), QTL20.ToString("F4"), QTU20.ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); cols = new string[] { "PI ", QTPredicted20.ToString("F4"), (QTPredicted20 - QTPI20).ToString("F4"), (QTPredicted20 + QTPI20).ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); } if (this.MathResult.ToLower().StartsWith("http")) { string sError = string.Empty; bool bHasSaved = CalculatorHelpers.SaveTextInURI( _params.ExtensionDocToCalcURI, sb.ToString(), this.MathResult, out sError); if (!string.IsNullOrEmpty(sError)) { this.MathResult += sError; } } else { this.MathResult = sb.ToString(); } } catch (Exception ex) { this.ErrorMessage = ex.Message; } }
public async Task <bool> RunSubAlgorithm3Async(string inputFile1Path, string inputFile2Path) { bool bHasCalcs = false; string sBaseURL = "https://ussouthcentral.services.azureml.net/workspaces/d454361ecdcb4ec4b03fa1aec5a7c0e2/services/8b1074b465ea4258a11ec48ce64ae257/jobs"; string sPlatForm = CalculatorHelpers.GetPlatform(_params.ExtensionDocToCalcURI, inputFile1Path); if (sPlatForm != CalculatorHelpers.PLATFORM_TYPES.azure.ToString()) { sBaseURL = "https://ussouthcentral.services.azureml.net/workspaces/d454361ecdcb4ec4b03fa1aec5a7c0e2/services/8b1074b465ea4258a11ec48ce64ae257/execute?api-version=2.0&details=true"; } string sApiKey = "RO2Ev5dRSKqNJ4jz+zoT0qDntEsKbyizbgZKlhOR2vGztsjBD3S3C8nmIlZI9TbbmCcsw+unwhky1GgZ5qiHmg=="; string sError = string.Empty; //web server expects to store in temp/randomid/name.csv //scoring results string sOutputData1URL = CalculatorHelpers.GetTempContainerPath("outputdata1.csv"); //model results string sOutputData2URL = CalculatorHelpers.GetTempContainerPath("outputdata2.csv"); //web service expects urls that start with container names string sInput1ContainerPath = CalculatorHelpers.GetContainerPathFromFullURIPath("resources", inputFile1Path); string sInput2ContainerPath = CalculatorHelpers.GetContainerPathFromFullURIPath("resources", inputFile2Path); //async wait so that results can be stored in output file location and parsed into string lines SetResponse2(sBaseURL, sApiKey, sInput1ContainerPath, sInput2ContainerPath, sOutputData1URL, sOutputData2URL).Wait(); StringBuilder sb = new StringBuilder(); //if web service successully saved the results, the response will start with Success if (_response.StartsWith("Success")) { //return the output file contents in a string list of lines //must convert container path to full path string sOutput1FullDataURL = string.Concat("https://devtreks1.blob.core.windows.net/", sOutputData1URL); List <string> lines = new List <string>(); //azure emulator can't process real Azure URL so this won't work //instead, double check that output url is actually saved lines = CalculatorHelpers.ReadLines(_params.ExtensionDocToCalcURI, sOutput1FullDataURL, out sError); this.ErrorMessage += sError; //this results in endless wait //lines = await CalculatorHelpers.ReadLinesAsync(sOutputDataURL); if (lines == null) { this.ErrorMessage += string.Concat(" ", Errors.MakeStandardErrorMsg("DATAURL_BAD")); return(bHasCalcs); } if (lines.Count == 0) { this.ErrorMessage += string.Concat(" ", Errors.MakeStandardErrorMsg("DATAURL_BAD")); return(bHasCalcs); } sb = new StringBuilder(); sb.AppendLine("aml results"); //dep var has to be in the R project 1st column //string sLine = string.Concat("first variable: ", _colNames[0]); string[] line = new List <string>().ToArray(); int iPos = 0; for (int i = 0; i < lines.Count(); i++) { line = lines[i].Split(Constants.CSV_DELIMITERS); //lineout[1] = CalculatorHelpers.ConvertStringToDouble(line[0]).ToString("N4", CultureInfo.InvariantCulture); sb.AppendLine(Shared.GetLine(line, false)); } if (this.MathResult.ToLower().StartsWith("http")) { sError = string.Empty; bool bHasSaved = CalculatorHelpers.SaveTextInURI( _params.ExtensionDocToCalcURI, sb.ToString(), this.MathResult, out sError); if (!string.IsNullOrEmpty(sError)) { this.MathResult += sError; } } else { this.MathResult = sb.ToString(); } bHasCalcs = true; //last line of string should have the QTM vars if (line != null) { //last string is prediction iPos = line.Count() - 1; //int iPos = line.Count() - 3; if (line[iPos] != null) { this.QTPredicted = CalculatorHelpers.ConvertStringToDouble(line[iPos]); } } string sOutput2FullDataURL = string.Concat("https://devtreks1.blob.core.windows.net/", sOutputData2URL); lines = new List <string>(); //azure emulator can't process real Azure URL so this won't work //instead, double check that output url is actually saved lines = CalculatorHelpers.ReadLines(_params.ExtensionDocToCalcURI, sOutput2FullDataURL, out sError); this.ErrorMessage += sError; if (lines == null) { this.ErrorMessage += string.Concat(" ", Errors.MakeStandardErrorMsg("DATAURL_BAD")); return(bHasCalcs); } if (lines.Count == 0) { this.ErrorMessage += string.Concat(" ", Errors.MakeStandardErrorMsg("DATAURL_BAD")); return(bHasCalcs); } sb = new StringBuilder(); //dep var has to be in the R project 1st column //string sLine = string.Concat("first variable: ", _colNames[0]); line = new List <string>().ToArray(); double dbCI = 0; for (int i = 0; i < lines.Count(); i++) { line = lines[i].Split(Constants.CSV_DELIMITERS); if (line != null) { iPos = 0; //used to derive conf interval dbCI = CalculatorHelpers.ConvertStringToDouble(line[iPos]); sb.AppendLine(string.Format("{0} {1}", "Mean Absolute Error: ", dbCI.ToString())); double dbScore = 0; if (line.Count() >= 2) { iPos = 1; dbScore = CalculatorHelpers.ConvertStringToDouble(line[iPos]); sb.AppendLine(string.Format("{0} {1}", "Root Mean Squared Error: ", dbScore.ToString())); } if (line.Count() >= 3) { iPos = 2; dbScore = CalculatorHelpers.ConvertStringToDouble(line[iPos]); sb.AppendLine(string.Format("{0} {1}", "Relative Absolute Error: ", dbScore.ToString())); } if (line.Count() >= 4) { iPos = 3; dbScore = CalculatorHelpers.ConvertStringToDouble(line[iPos]); sb.AppendLine(string.Format("{0} {1}", "Relative Squared Error: ", dbScore.ToString())); } if (line.Count() >= 5) { iPos = 4; dbScore = CalculatorHelpers.ConvertStringToDouble(line[iPos]); sb.AppendLine(string.Format("{0} {1}", "Coefficient of Determination: ", dbScore.ToString())); } //sb.AppendLine(Shared.GetLine(line, false)); } } this.MathResult += sb.ToString(); bHasCalcs = true; //last line of string should have the QTM vars if (line != null) { if (line[iPos] != null) { dbCI = CalculatorHelpers.ConvertStringToDouble(line[iPos]); } this.QTL = this.QTPredicted - dbCI; this.QTU = this.QTPredicted + dbCI; } } else { this.ErrorMessage += "The calculations could not be run using the web service."; } return(bHasCalcs); }
public async Task <bool> RunSubAlgorithm1or2Async(string inputFilePath, string rFile) { bool bHasCalcs = false; string sBaseURL = "https://ussouthcentral.services.azureml.net/workspaces/d454361ecdcb4ec4b03fa1aec5a7c0e2/services/b10e6b4c4e63438999cc45147bbe006c/jobs"; if (_subalgorithm == Calculator1.MATH_SUBTYPES.subalgorithm2.ToString()) { sBaseURL = "https://ussouthcentral.services.azureml.net/workspaces/d454361ecdcb4ec4b03fa1aec5a7c0e2/services/abd32060dc014d0e8fe1256e0f694daa/jobs"; } string sPlatForm = _params.ExtensionDocToCalcURI.URIDataManager.PlatformType.ToString(); if (sPlatForm != CalculatorHelpers.PLATFORM_TYPES.azure.ToString()) { sBaseURL = "https://ussouthcentral.services.azureml.net/workspaces/d454361ecdcb4ec4b03fa1aec5a7c0e2/services/b10e6b4c4e63438999cc45147bbe006c/execute?api-version=2.0&details=true"; if (_subalgorithm == Calculator1.MATH_SUBTYPES.subalgorithm2.ToString()) { sBaseURL = "https://ussouthcentral.services.azureml.net/workspaces/d454361ecdcb4ec4b03fa1aec5a7c0e2/services/abd32060dc014d0e8fe1256e0f694daa/execute?api-version=2.0&details=true"; } } //r web service is default string sApiKey = "fxBeL9LJ3ORm0kW0DtKhT99OfUK6YgBlc59crizYhlxKoEjRd3kuDHvPRuehCQ02VJhPPXcdYTp2pDUynb9gMA=="; if (_subalgorithm == Calculator1.MATH_SUBTYPES.subalgorithm2.ToString()) { //python sApiKey = "/bjDNKx4OWdMIQu6CkvWCIhcfUOCTp9jUE9kD7uylwhOYyhVFOqAFA7M75mJjHS6p6jnAhCvFn1jSl678gzPVA=="; } string sError = string.Empty; //convert the script file to the script string expected by the algorithm List <string> rlines = new List <string>(); rlines = CalculatorHelpers.ReadLines(_params.ExtensionDocToCalcURI, rFile, out sError); this.ErrorMessage += sError; if (rlines == null) { this.ErrorMessage += string.Concat(" ", Errors.MakeStandardErrorMsg("DATAURL_BAD")); return(bHasCalcs); } if (rlines.Count == 0) { this.ErrorMessage += string.Concat(" ", Errors.MakeStandardErrorMsg("DATAURL_BAD")); return(bHasCalcs); } StringBuilder sbR = new StringBuilder(); for (int i = 0; i < rlines.Count(); i++) { sbR.AppendLine(rlines[i]); } string rScript = sbR.ToString(); //web server expects to store in temp/randomid/name.csv //web service stores in temp blob string sOutputDataURL = CalculatorHelpers.GetTempContainerPath("Routput.csv"); //web service expects urls that start with container names //regular rproject file must be stored in JDataURL string sInputContainerPath = CalculatorHelpers.GetContainerPathFromFullURIPath("resources", inputFilePath); //async wait so that results can be stored in output file location and parsed into string lines SetResponse(sBaseURL, sApiKey, sInputContainerPath, sOutputDataURL, rScript).Wait(); StringBuilder sb = new StringBuilder(); //if web service successully saved the results, the response will start with Success if (_response.StartsWith("Success")) { //return the output file contents in a string list of lines //must convert container path to full path string sOutputFullDataURL = string.Concat("https://devtreks1.blob.core.windows.net/", sOutputDataURL); List <string> lines = new List <string>(); //azure emulator can't process real Azure URL so this won't work //instead, double check that output url is actually saved lines = CalculatorHelpers.ReadLines(_params.ExtensionDocToCalcURI, sOutputFullDataURL, out sError); this.ErrorMessage += sError; //this results in endless wait-try ReadLinesAsync(sOutputDataURL).ConfigureAwait(false) //lines = await CalculatorHelpers.ReadLinesAsync(sOutputDataURL); if (lines == null) { this.ErrorMessage += string.Concat(" ", Errors.MakeStandardErrorMsg("DATAURL_BAD")); return(bHasCalcs); } if (lines.Count == 0) { this.ErrorMessage += string.Concat(" ", Errors.MakeStandardErrorMsg("DATAURL_BAD")); return(bHasCalcs); } sb = new StringBuilder(); if (_subalgorithm == Calculator1.MATH_SUBTYPES.subalgorithm2.ToString()) { sb.AppendLine("py results"); } else { sb.AppendLine("r results"); } //dep var has to be in the R project 1st column string sLine = string.Concat("first variable: ", _colNames[0]); string[] line = new List <string>().ToArray(); for (int i = 0; i < lines.Count(); i++) { line = lines[i].Split(Constants.CSV_DELIMITERS); //lineout[1] = CalculatorHelpers.ConvertStringToDouble(line[0]).ToString("N4", CultureInfo.InvariantCulture); sb.AppendLine(Shared.GetLine(line, false)); } if (this.MathResult.ToLower().StartsWith("http")) { sError = string.Empty; bool bHasSaved = CalculatorHelpers.SaveTextInURI( _params.ExtensionDocToCalcURI, sb.ToString(), this.MathResult, out sError); if (!string.IsNullOrEmpty(sError)) { this.MathResult += sError; } } else { this.MathResult = sb.ToString(); } bHasCalcs = true; //last line of string should have the QTM vars if (line != null) { int iPos = 0; if (line[iPos] != null) { this.QTPredicted = CalculatorHelpers.ConvertStringToDouble(line[iPos]); } iPos = 1; if (line[iPos] != null) { this.QTL = CalculatorHelpers.ConvertStringToDouble(line[iPos]); } iPos = 2; if (line[iPos] != null) { this.QTU = CalculatorHelpers.ConvertStringToDouble(line[iPos]); } } } else { this.ErrorMessage += string.Concat(_response, "The calculations could not be run using the web service."); } return(bHasCalcs); }
public async Task <bool> RunAlgorithmAsync(List <List <double> > data) { bool bHasCalculation = false; try { //add the qs to end of list for running in algo data.Add(_qs); //this algorith uses standard arrays double[,] problemData = Shared.GetDoubleArray(data); StringBuilder sb = new StringBuilder(); //set the rank params needed for the needed matrixes rows = problemData.GetUpperBound(0) + 1; cols = problemData.GetUpperBound(1) + 1; outs = GetDistinctOutputValues(problemData, cols); inputVarCount = cols - 1; if (inputVarCount > maxInputs) { //no: need to catch it before the data is sent here } outputValCount = outs.Count(); rnd = new Random(159); // 159 makes 'good' output double[][] trainMatrix = null; double[][] testMatrix = null; //Generating train and test matrices using an 80%-20% split MakeTrainAndTest(problemData, out trainMatrix, out testMatrix); sb.AppendLine("\nFirst few rows of training matrix, using an 80%-20% split, are:"); //First few rows of training matrix are Helpers.ShowMatrix(sb, trainMatrix, inputVarCount, 5); //the first and last params are legit; not sure of the correct basis for 2nd NeuralNetwork nn = new NeuralNetwork(inputVarCount, inputVarCount + 1, outputValCount); //Training to find best neural network weights using PSO with cross entropy error double[] bestWeights = nn.Train(sb, trainMatrix); sb.AppendLine("\nBest weights found:"); Helpers.ShowVector(sb, bestWeights, 2, true); //Loading best weights into neural network nn.SetWeights(bestWeights); sb.AppendLine("\nExamples of the neural network accuracy:\n"); //Analyzing the neural network accuracy on the test data double accuracy = nn.Test(sb, testMatrix); this.QTL = accuracy; this.QTLUnit = "percent accuracy"; sb.AppendLine("Prediction accuracy = " + accuracy.ToString("F4")); this.QTPredicted = nn.QTPredicted; sb.AppendLine("Predicted QT = " + this.QTPredicted.ToString("F4")); if (this.MathResult.ToLower().StartsWith("http")) { string sError = string.Empty; bool bHasSaved = CalculatorHelpers.SaveTextInURI( _params.ExtensionDocToCalcURI, sb.ToString(), this.MathResult, out sError); if (!string.IsNullOrEmpty(sError)) { this.MathResult += sError; } } else { this.MathResult = sb.ToString(); } bHasCalculation = true; } catch (Exception ex) { this.ErrorMessage = ex.Message; } return(bHasCalculation); }
public async Task <bool> SetObservedMathResult(string mathType, string distType, string mathSubType, double mostLikely, double[] obsTs) { bool bHasSet = false; //this is observed data rather than randomly sampled data StringBuilder sb = new StringBuilder(); if (obsTs.Count() > 0) { //xy array of 10 points on cdf //var array = qTs.ToArray(); Array.Sort(obsTs); int i00 = 0; int i10 = (int)(obsTs.Length * .10); int i20 = (int)(obsTs.Length * .20); int i30 = (int)(obsTs.Length * .30); int i40 = (int)(obsTs.Length * .40); int i50 = (int)(obsTs.Length * .50); int i60 = (int)(obsTs.Length * .60); int i70 = (int)(obsTs.Length * .70); int i80 = (int)(obsTs.Length * .80); int i90 = (int)(obsTs.Length * .90); int i100 = (int)(obsTs.Length - 1); sb.AppendLine(Errors.GetMessage("STATS_DESC5")); sb.AppendLine(string.Concat( MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(obsTs, obsTs[i00]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(obsTs, obsTs[i10]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(obsTs, obsTs[i20]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(obsTs, obsTs[i30]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(obsTs, obsTs[i40]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(obsTs, obsTs[i50]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(obsTs, obsTs[i60]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(obsTs, obsTs[i70]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(obsTs, obsTs[i80]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(obsTs, obsTs[i90]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(obsTs, obsTs[i100]).ToString("N2", CultureInfo.InvariantCulture))); sb.AppendLine(string.Concat( obsTs[i00].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, obsTs[i10].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, obsTs[i20].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, obsTs[i30].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, obsTs[i40].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, obsTs[i50].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, obsTs[i60].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, obsTs[i70].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, obsTs[i80].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, obsTs[i90].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, obsTs[i100].ToString("F4", CultureInfo.InvariantCulture))); sb.AppendLine(Errors.GetMessage("STATS_DESC3")); sb.AppendLine(Errors.GetMessage("STATS_DESC2")); //qT is the result of math and stored in sixth col var stats = new MathNet.Numerics.Statistics.DescriptiveStatistics(obsTs); sb.AppendLine(string.Concat(stats.Count.ToString(), Constants.CSV_DELIMITER, obsTs.Sum().ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, stats.Mean.ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, MathNet.Numerics.Statistics.Statistics.Median(obsTs).ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, stats.StandardDeviation.ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, stats.Variance.ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, stats.Minimum.ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, stats.Maximum.ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER)); string sLowerCI = string.Concat(Errors.GetMessage("LOWER"), this.CILevel.ToString(), Errors.GetMessage("CI_PCT")); string sUpperCI = string.Concat(Errors.GetMessage("UPPER"), this.CILevel.ToString(), Errors.GetMessage("CI_PCT")); this.IndicatorQT.QTM = stats.Mean; this.IndicatorQT.QTL = stats.Mean - CalculatorHelpers.GetConfidenceInterval(this.CILevel, stats.Count, stats.StandardDeviation); this.IndicatorQT.QTU = stats.Mean + CalculatorHelpers.GetConfidenceInterval(this.CILevel, stats.Count, stats.StandardDeviation); this.IndicatorQT.QTLUnit = sLowerCI; this.IndicatorQT.QTUUnit = sUpperCI; } else { sb.AppendLine(Errors.MakeStandardErrorMsg("DATAURL_BADDATA")); } if (this.IndicatorQT.MathResult.ToLower().StartsWith("http")) { bool bHasSaved = await CalculatorHelpers.SaveTextInURI( Params.ExtensionDocToCalcURI, sb.ToString(), this.IndicatorQT.MathResult); if (!string.IsNullOrEmpty(Params.ExtensionDocToCalcURI.ErrorMessage)) { this.IndicatorQT.MathResult += Params.ExtensionDocToCalcURI.ErrorMessage; //done with errormsg Params.ExtensionDocToCalcURI.ErrorMessage = string.Empty; } } else { this.IndicatorQT.MathResult = sb.ToString(); } bHasSet = true; return(bHasSet); }
public async Task <bool> SetRange(double mostLikelyEstimate, double lowEstimate, double highEstimate, double[] data, List <double> cdf = null) { bool bHasSet = false; StringBuilder sb = new StringBuilder(); if (data == null) { sb.AppendLine("This indicator does not have the properties needed for descriptive statistics."); } if (data.Count() == 0) { sb.AppendLine("This indicator does not have the properties needed for descriptive statistics."); } //default has no range of values double lowRange = lowEstimate; double highRange = highEstimate; double mostLikely = mostLikelyEstimate; string mostLikelyUnit = string.Empty; string lowUnit = string.Empty; string highUnit = string.Empty; bool bNeedsCDF = false; var stats = new MathNet.Numerics.Statistics.DescriptiveStatistics(data); string sLowerCI = string.Concat(Errors.GetMessage("LOWER"), this.CILevel.ToString(), Errors.GetMessage("CI_PCT")); string sUpperCI = string.Concat(Errors.GetMessage("UPPER"), this.CILevel.ToString(), Errors.GetMessage("CI_PCT")); //some data has all zeros and does not use sampled data if (stats.Mean != 0) { mostLikely = stats.Mean; mostLikelyUnit = "mean"; lowRange = mostLikely - CalculatorHelpers.GetConfidenceInterval(this.CILevel, stats.Count, stats.StandardDeviation); lowUnit = sLowerCI; highRange = mostLikely + CalculatorHelpers.GetConfidenceInterval(this.CILevel, stats.Count, stats.StandardDeviation); highUnit = sUpperCI; bNeedsCDF = true; } if (bNeedsCDF) { //csv strings use f4 not n4 sb.AppendLine(Errors.GetMessage("STATS_DESC1")); sb.AppendLine(Errors.GetMessage("STATS_DESC2")); sb.AppendLine(string.Concat(stats.Count.ToString(), ", ", data.Sum().ToString("F4", CultureInfo.InvariantCulture), ", ", stats.Mean.ToString("F4", CultureInfo.InvariantCulture), ", ", MathNet.Numerics.Statistics.Statistics.Median(data).ToString("F4", CultureInfo.InvariantCulture), ", ", stats.StandardDeviation.ToString("F4", CultureInfo.InvariantCulture), ", ", stats.Variance.ToString("F4", CultureInfo.InvariantCulture), ", ", stats.Minimum.ToString("F4", CultureInfo.InvariantCulture), ", ", stats.Maximum.ToString("F4", CultureInfo.InvariantCulture), ", ")); //xy array of 10 points on cdf var array = data.ToArray(); Array.Sort(array); int i00 = 0; int i10 = (int)(array.Length * .10); int i20 = (int)(array.Length * .20); int i30 = (int)(array.Length * .30); int i40 = (int)(array.Length * .40); int i50 = (int)(array.Length * .50); int i60 = (int)(array.Length * .60); int i70 = (int)(array.Length * .70); int i80 = (int)(array.Length * .80); int i90 = (int)(array.Length * .90); int i100 = (int)(array.Length - 1); sb.AppendLine(Errors.GetMessage("STATS_DESC4")); sb.AppendLine(string.Concat( MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(data, array[i00]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(data, array[i10]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(data, array[i20]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(data, array[i30]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(data, array[i40]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(data, array[i50]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(data, array[i60]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(data, array[i70]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(data, array[i80]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(data, array[i90]).ToString("N2", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, MathNet.Numerics.Statistics.Statistics.EmpiricalCDF(data, array[i100]).ToString("N2", CultureInfo.InvariantCulture))); sb.AppendLine(string.Concat( array[i00].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, array[i10].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, array[i20].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, array[i30].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, array[i40].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, array[i50].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, array[i60].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, array[i70].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, array[i80].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, array[i90].ToString("F4", CultureInfo.InvariantCulture), Constants.CSV_DELIMITER, array[i100].ToString("F4", CultureInfo.InvariantCulture))); if (cdf == null) { cdf = new List <double>(); } cdf.Add(array[i00]); cdf.Add(array[i10]); cdf.Add(array[i20]); cdf.Add(array[i30]); cdf.Add(array[i40]); cdf.Add(array[i50]); cdf.Add(array[i60]); cdf.Add(array[i70]); cdf.Add(array[i80]); cdf.Add(array[i90]); cdf.Add(array[i100]); } else { sb.AppendLine(Errors.GetMessage("STATS_DESC6")); } //doubles need restricted digits mostLikely = Math.Round(mostLikely, 4); lowRange = Math.Round(lowRange, 4); highRange = Math.Round(highRange, 4); string sD1Unit = "low"; string sD2Unit = "high"; this.IndicatorQT.QTD1 = lowEstimate; if (this.IndicatorQT.QTD1Unit == string.Empty || this.IndicatorQT.QTD1Unit == Constants.NONE) { this.IndicatorQT.QTD1Unit = sD1Unit; } this.IndicatorQT.QTD2 = highEstimate; if (this.IndicatorQT.QTD2Unit == string.Empty || this.IndicatorQT.QTD2Unit == Constants.NONE) { this.IndicatorQT.QTD2Unit = sD2Unit; } //computed results this.IndicatorQT.QTM = mostLikely; if (this.IndicatorQT.QTMUnit == string.Empty || this.IndicatorQT.QTMUnit == Constants.NONE) { this.IndicatorQT.QTMUnit = mostLikelyUnit; } this.IndicatorQT.QTL = lowRange; if (this.IndicatorQT.QTLUnit == string.Empty || this.IndicatorQT.QTLUnit == Constants.NONE) { this.IndicatorQT.QTLUnit = lowUnit; } this.IndicatorQT.QTU = highRange; if (this.IndicatorQT.QTUUnit == string.Empty || this.IndicatorQT.QTUUnit == Constants.NONE) { this.IndicatorQT.QTUUnit = highUnit; } if (this.IndicatorQT.MathResult.ToLower().StartsWith("http")) { bool bHasSaved = await CalculatorHelpers.SaveTextInURI( Params.ExtensionDocToCalcURI, sb.ToString(), this.IndicatorQT.MathResult); if (!string.IsNullOrEmpty(Params.ExtensionDocToCalcURI.ErrorMessage)) { this.IndicatorQT.MathResult += Params.ExtensionDocToCalcURI.ErrorMessage; //done with errormsg Params.ExtensionDocToCalcURI.ErrorMessage = string.Empty; } } else { this.IndicatorQT.MathResult = sb.ToString(); } bHasSet = true; return(bHasSet); }
/// <summary> /// Run the simulated annealing algo /// </summary> /// <param name="problemData">matrix of energy requirements (hour to complete task)</param> /// <param name="currTemp">initial temperature from which cooldown starts</param> /// <param name="alpha">cooling rate</param> /// <param name="penalty">a penalty when a worker has more than 1 task</param> /// <param name="maxIteration">number of iterations</param> public async Task <bool> RunAlgorithmAsync(List <List <double> > data) { bool bHasCalculation = false; try { random = new Random(0); //this algorith uses standard arrays double[,] problemData = Shared.GetDoubleArray(data); int[] state = RandomState(problemData); double energy = Energy(state, problemData, _penalty); int[] bestState = state; double bestEnergy = energy; int[] adjState; double adjEnergy; int iteration = 0; while (iteration < _maxiteration && _currtemp > 0.0001) { adjState = AdjacentState(state, problemData); adjEnergy = Energy(adjState, problemData, _penalty); if (adjEnergy < bestEnergy) { bestState = adjState; bestEnergy = adjEnergy; } double p = random.NextDouble(); // [0, 1.0) if (AcceptanceProb(energy, adjEnergy, _currtemp) > p) { state = adjState; energy = adjEnergy; } _currtemp = _currtemp * _alpha; // cool down; annealing schedule ++iteration; } // while this.BestEnergy = bestEnergy; if (this.MathResult.ToLower().StartsWith("http")) { bool bHasSaved = await CalculatorHelpers.SaveTextInURI(_params.ExtensionDocToCalcURI, Interpret(bestState, problemData), this.MathResult); if (!string.IsNullOrEmpty(_params.ExtensionDocToCalcURI.ErrorMessage)) { this.MathResult += _params.ExtensionDocToCalcURI.ErrorMessage; //done with errormsg _params.ExtensionDocToCalcURI.ErrorMessage = string.Empty; } } else { this.MathResult = Interpret(bestState, problemData); } bHasCalculation = true; } catch (Exception ex) { ErrorMessage = ex.Message; } return(bHasCalculation); }
//this is asych for the calling Task.WhenAll //but does not necessarily need internal asych awaits public async Task <bool> RunAlgorithmAsync(List <List <double> > data) { bool bHasCalculations = false; try { //minimal data requirement is first five cols if (_colNames.Count() < 5 || _mathTerms.Count() == 0) { ErrorMessage = "Complete randomized anova requires at least 1 dependent variable and 1 independent variable. Randomized block and factorial anovas require at least 2 independent variables."; return(bHasCalculations); } if (data.Count() < 5) { //185 same as other analysis ErrorMessage = "Anova requires at least 2 rows of observed data and 3 rows of scoring data."; return(bHasCalculations); } //convert data to a Math.Net Matrix //last 3 rows are used to generate ci List <List <double> > dataci = data.Skip(data.Count - _scoreRows).ToList(); data.Reverse(); List <List <double> > dataobs = data.Skip(_scoreRows).ToList(); dataobs.Reverse(); //actual observed values Vector <double> y = Shared.GetYData(dataobs); //treatments or factor1 levels Matrix <double> treatments = Shared.GetDistinctMatrix(dataobs, 1); //206 condition added due to M and E dataset indexing _totalsNeeded = treatments.ColumnCount; double tDF = treatments.ColumnCount - 1; //step 1. get total of observed data double yTotal = y.Sum(); //step 2. get total of observed data squared double yTotal2 = y.PointwisePower(2).Sum(); //step 3. set CM double CM = Math.Pow(yTotal, 2) / y.Count(); //step 4. treatments and blocks double SSTotal = yTotal2 - CM; //add the data to a string builder StringBuilder sb = new StringBuilder(); sb.AppendLine("anova results"); //5 col array string[] cols = new string[] { "source", "df", "SS", "MS", "F" }; sb.AppendLine(Shared.GetLine(cols, true)); List <List <double> > totals = new List <List <double> >(_totalsNeeded); //min treatment block required bool bIsBlock = (_depColNames.Contains("treatment") && _depColNames.Contains("block")) ? true : false; //min 2 factors required bool bIsFactorial = (_depColNames.Contains("factor1") && _depColNames.Contains("factor2")) ? true : false; bool bIsComplete = (bIsBlock == false && bIsFactorial == false) ? true : false; if (bIsComplete) { double eDF = y.Count() - treatments.ColumnCount; //step 5.treatments (correct to divide by rows) double SST = ((treatments.ColumnSums().PointwisePower(2).Sum()) / treatments.RowCount) - CM; //step 6. error double SSE = SSTotal - SST; //step 7. mean treatment double MST = SST / tDF; //step 8. mean error double MSE = SSE / (y.Count() - treatments.ColumnCount); //step 9. F treatments double FT = MST / MSE; //tests double s = Math.Pow(MSE, 0.5); //correct 2 tailed t test int itDF = CalculatorHelpers.ConvertStringToInt(tDF.ToString()); int ieDF = CalculatorHelpers.ConvertStringToInt(eDF.ToString()); double dbCI = CalculatorHelpers.GetConfidenceIntervalProb(_confidenceInt); //TINV divides dbCI by 2 to get student t double tCriticalValue = ExcelFunctions.TInv(dbCI, ieDF); //prevents an error in Finv if (itDF == 0) { itDF = 1; } double FCriticalTValue = ExcelFunctions.FInv(dbCI, itDF, ieDF); string FTGreaterFCritical = (FT > FCriticalTValue) ? "true" : "false"; for (int i = 0; i < _totalsNeeded; i++) { //206 condition added due to M and E dataset indexing if (i < treatments.ColumnCount) { SetAnovaIntervals(i, totals, treatments, tCriticalValue, s, CalculatorHelpers.ConvertStringToDouble(treatments.RowCount.ToString()), FT, FCriticalTValue, bIsComplete); } } this.DataToAnalyze.Add(Label, totals); ////add the data to a string builder cols = new string[] { "treats", itDF.ToString("F0"), SST.ToString("F4"), MST.ToString("F4"), FT.ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); cols = new string[] { "error ", ieDF.ToString("F0"), SSE.ToString("F4"), MSE.ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); cols = new string[] { "total ", (y.Count() - 1).ToString("F0"), (SSTotal).ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); cols = new string[] { string.Concat("F Crit ", "treats"), FCriticalTValue.ToString("F5"), "F > F Critical", FTGreaterFCritical }; sb.AppendLine(Shared.GetLine(cols, true)); } else { //observations per cell for factorials (data[0] if first row of data) double r = Shared.GetObservationsPerCell(dataobs, 1, data[0].ElementAt(1), data[0].ElementAt(2)); //blocks or factor2 levels Matrix <double> blocks = Shared.GetDistinctMatrix(dataobs, 2); double bDF = blocks.ColumnCount - 1; double eDF = y.Count() - treatments.ColumnCount - blocks.ColumnCount + 1; if (bIsFactorial) { eDF = (treatments.ColumnCount * blocks.ColumnCount) * (r - 1); } //factorial interaction df double tbDF = tDF * bDF; //step 5.treatments (correct to divide by r) double SST = ((treatments.ColumnSums().PointwisePower(2).Sum()) / (blocks.ColumnCount * r)) - CM; //step 6. blocks double SSB = ((blocks.ColumnSums().PointwisePower(2).Sum()) / (treatments.ColumnCount * r)) - CM; //factor level interaction double SSFL = 0; //step 7. error double SSE = 0; if (bIsFactorial) { double totalinteraction = Shared.GetTotalInteraction(blocks, r); //watch block.colcount for 2 x 3 factorials SSFL = (totalinteraction / r) - SST - SSB - CM; //step 7. error SSE = SSTotal - SST - SSB - SSFL; } else { //step 7. error SSE = SSTotal - SST - SSB; } //step 8. mean treatment double MST = SST / tDF; //step 9. mean block double MSB = SSB / bDF; //step 10. mean error double MSE = SSE / eDF; double MSFL = SSFL / tbDF; //step 11. F treatments double FT = MST / MSE; //step 12. F blocks double FB = MSB / MSE; double FTB = MSFL / MSE; //tests double s = Math.Pow(MSE, 0.5); //correct 2 tailed t test int itDF = CalculatorHelpers.ConvertStringToInt(tDF.ToString()); int ibDF = CalculatorHelpers.ConvertStringToInt(bDF.ToString()); int itbDF = CalculatorHelpers.ConvertStringToInt(tbDF.ToString()); int ieDF = CalculatorHelpers.ConvertStringToInt(eDF.ToString()); double dbCI = CalculatorHelpers.GetConfidenceIntervalProb(_confidenceInt); //TINV divides dbCI by 2 to get student t double tCriticalValue = ExcelFunctions.TInv(dbCI, ieDF); //prevents an error in Finv if (itDF == 0) { itDF = 1; } double FCriticalTValue = ExcelFunctions.FInv(dbCI, itDF, ieDF); string FTGreaterFCritical = (FT > FCriticalTValue) ? "true" : "false"; //prevents an error in Finv if (ibDF == 0) { ibDF = 1; } double FCriticalBValue = ExcelFunctions.FInv(dbCI, ibDF, ieDF); string FBGreaterFCritical = (FB > FCriticalBValue) ? "true" : "false"; //prevents an error in Finv if (itbDF == 0) { itbDF = 1; } double FCriticalTBValue = ExcelFunctions.FInv(dbCI, itbDF, ieDF); string FTBGreaterFCritical = (FTB > FCriticalTBValue) ? "true" : "false"; //List<List<double>> totals = new List<List<double>>(_totalsNeeded); if (bIsFactorial) { //unless custom stylesheets are developed, can only display factor 1 - factor 2 diffs //build a matrix equivalent to treatments -1 row, variable cols Matrix <double> torbs = Matrix <double> .Build.Dense(1, _totalsNeeded); List <double> mrow = new List <double>(_totalsNeeded); for (int i = 0; i < _totalsNeeded; i++) { double cellMean = Shared.GetMeanPerCell(dataobs, 1, 2, i, i, r); mrow.Add(cellMean); } torbs.SetRow(0, mrow.ToArray()); for (int i = 0; i < _totalsNeeded; i++) { //206 condition added due to M and E dataset indexing if (i < treatments.ColumnCount) { //treatments SetAnovaIntervals(i, totals, torbs, tCriticalValue, s, CalculatorHelpers.ConvertStringToDouble(blocks.ColumnCount.ToString()), FT, FCriticalTValue, bIsComplete); } } //cell1Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 0, 0, r); //cell2Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 0, 1, r); //SetAnovaIntervals2(1, cell1Mean, cell2Mean, tCriticalValue, s, r); //cell1Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 0, 0, r); //cell2Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 2, 0, r); //SetAnovaIntervals2(2, cell1Mean, cell2Mean, tCriticalValue, s, r); //double cell1Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 0, 0, r); //double cell2Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 1, 0, r); //SetAnovaIntervals2(0, cell1Mean, cell2Mean, tCriticalValue, s, r); //cell1Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 0, 0, r); //cell2Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 0, 1, r); //SetAnovaIntervals2(1, cell1Mean, cell2Mean, tCriticalValue, s, r); //cell1Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 0, 0, r); //cell2Mean = Shared.GetMeanPerCell(dataobs, 1, 2, 2, 0, r); //SetAnovaIntervals2(2, cell1Mean, cell2Mean, tCriticalValue, s, r); } else { //unless custom stylesheets are developed, need to only display treatment diffs for (int i = 0; i < _totalsNeeded; i++) { //206 condition added due to M and E dataset indexing if (i < treatments.ColumnCount) { //treatments SetAnovaIntervals(i, totals, treatments, tCriticalValue, s, CalculatorHelpers.ConvertStringToDouble(blocks.ColumnCount.ToString()), FT, FCriticalTValue, bIsComplete); ////blocks //SetAnovaIntervals(i, totals, blocks, tCriticalValue, s, // CalculatorHelpers.ConvertStringToDouble(treatments.ColumnCount.ToString()), // FB, FCriticalBValue, bIsComplete); //interactions } } } this.DataToAnalyze.Add(Label, totals); string sTreats = "treats "; string sBlocks = "blocks "; if (bIsFactorial) { sTreats = "factor1 "; sBlocks = "factor2 "; } ////add the data to a string builder //StringBuilder sb = new StringBuilder(); //sb.AppendLine("anova results"); cols = new string[] { sTreats, itDF.ToString("F0"), SST.ToString("F4"), MST.ToString("F4"), FT.ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); cols = new string[] { sBlocks, bDF.ToString("F0"), SSB.ToString("F4"), MSB.ToString("F4"), FB.ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); if (bIsFactorial) { cols = new string[] { "interacts ", tbDF.ToString("F0"), SSFL.ToString("F4"), MSFL.ToString("F4"), FTB.ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); } cols = new string[] { "error ", ieDF.ToString("F0"), SSE.ToString("F4"), MSE.ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); cols = new string[] { "total ", (y.Count() - 1).ToString("F0"), (SSTotal).ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); cols = new string[] { string.Concat("F Crit ", sTreats), FCriticalTValue.ToString("F5"), "F > F Critical", FTGreaterFCritical }; sb.AppendLine(Shared.GetLine(cols, true)); cols = new string[] { string.Concat("F Crit ", sBlocks), FCriticalBValue.ToString("F5"), "F > F Critical", FBGreaterFCritical }; sb.AppendLine(Shared.GetLine(cols, true)); if (bIsFactorial) { cols = new string[] { "F Crit Interacts", FCriticalTBValue.ToString("F5"), "F > F Critical", FTBGreaterFCritical }; sb.AppendLine(Shared.GetLine(cols, true)); } } cols = new string[] { "estimate", "mean diff", string.Concat("lower ", _confidenceInt.ToString(), "%"), string.Concat("upper ", _confidenceInt.ToString(), "%") }; sb.AppendLine(Shared.GetLine(cols, true)); //same report for calculator and analyzer for (int i = 0; i < _totalsNeeded; i++) { if (totals[i].Count >= 5) { if (i == 0) { QTPredicted = totals[i].ElementAt(0); QTL = QTPredicted - totals[i].ElementAt(4); QTU = QTPredicted + totals[i].ElementAt(4); cols = new string[] { "Treat 1 Mean ", QTPredicted.ToString("F4"), QTL.ToString("F4"), QTU.ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); } else { QTPredicted = totals[i].ElementAt(1); QTL = QTPredicted - totals[i].ElementAt(2); QTU = QTPredicted + totals[i].ElementAt(2); cols = new string[] { string.Concat("xminus1 ", i.ToString(), " "), QTPredicted.ToString("F4"), QTL.ToString("F4"), QTU.ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); QTPredicted = totals[i].ElementAt(3); QTL = QTPredicted - totals[i].ElementAt(4); QTU = QTPredicted + totals[i].ElementAt(4); cols = new string[] { string.Concat("base ", i.ToString(), " "), QTPredicted.ToString("F4"), QTL.ToString("F4"), QTU.ToString("F4") }; sb.AppendLine(Shared.GetLine(cols, false)); } } } if (this.MathResult.ToLower().StartsWith("http")) { bool bHasSaved = await CalculatorHelpers.SaveTextInURI( _params.ExtensionDocToCalcURI, sb.ToString(), this.MathResult); if (!string.IsNullOrEmpty(_params.ExtensionDocToCalcURI.ErrorMessage)) { this.MathResult += _params.ExtensionDocToCalcURI.ErrorMessage; //done with errormsg _params.ExtensionDocToCalcURI.ErrorMessage = string.Empty; } } else { this.MathResult = sb.ToString(); } bHasCalculations = true; } catch (Exception ex) { this.ErrorMessage = ex.Message; } return(bHasCalculations); }
//running this truly async returns to UI w/o saving final calcs or an endless wait public async Task <bool> RunAlgorithmAsync(string inputFilePath, string scriptFilePath, System.Threading.CancellationToken ctk) { bool bHasCalcs = false; try { this.ErrorMessage = string.Empty; if (string.IsNullOrEmpty(inputFilePath) || (!inputFilePath.EndsWith(".csv"))) { this.ErrorMessage = "The dataset file URL has not been added to the Data URL. The file must be stored in a Resource and use a csv file extension."; } if (string.IsNullOrEmpty(scriptFilePath) || (!scriptFilePath.EndsWith(".txt"))) { this.ErrorMessage += "The script file URL has not been added to the Joint Data.The file must be stored in a Resource and use a txt file extension."; } //unblock after debug if (!string.IsNullOrEmpty(this.ErrorMessage)) { return(bHasCalcs); } string sError = string.Empty; StringBuilder sb = new StringBuilder(); //get the path to the script executable string sScriptExecutable = CalculatorHelpers.GetAppSettingString( this._params.ExtensionDocToCalcURI, "RExecutable"); if (_algorithm == Calculator1.MATH_TYPES.algorithm3.ToString()) { //python must be installed to automatically run sScriptExecutable = CalculatorHelpers.GetAppSettingString( this._params.ExtensionDocToCalcURI, "PyExecutable"); if (_subalgorithm == Calculator1.MATH_SUBTYPES.subalgorithm1.ToString()) { //python scripts must be run by executable as '.pyw' files //save the 'txt' file as a 'pyw' file in temp path //has to be done each time because can't be sure when scriptfile changed last if (!scriptFilePath.EndsWith(".pyw")) { string sPyScript = CalculatorHelpers.ReadText(_params.ExtensionDocToCalcURI, scriptFilePath, out sError); if (!string.IsNullOrEmpty(sPyScript)) { string sFileName = Path.GetFileName(scriptFilePath); string sPyScriptFileName = sFileName.Replace(".txt", ".pyw"); bool bIsLocalCache = false; string sPyPath = CalculatorHelpers.GetTempDocsPath(_params.ExtensionDocToCalcURI, bIsLocalCache, sPyScriptFileName); bool bHasFile = CalculatorHelpers.SaveTextInURI(_params.ExtensionDocToCalcURI, sPyScript, sPyPath, out sError); scriptFilePath = sPyPath; } } } sb.AppendLine("python results"); } else { if (_subalgorithm == Calculator1.MATH_SUBTYPES.subalgorithm1.ToString()) { //check for 2.0.2 -R Open can run from a url //rscript.exe can't run from a url if (scriptFilePath.StartsWith("http")) { //convert it to a filesystem path //make sure that both localhost and localhost:44300 have a copy of the file string sRFilePath = CalculatorHelpers.ConvertFullURIToFilePath( this._params.ExtensionDocToCalcURI, scriptFilePath); scriptFilePath = sRFilePath; } } //r is default sb.AppendLine("r results"); } string sLastLine = string.Empty; //2.0.2: algo 2 subalgo2 is r or algo 3 subalgo2 Python; subalgo 2 is virtual machine if (_subalgorithm == Calculator1.MATH_SUBTYPES.subalgorithm2.ToString()) { //run on remote servers that have the DevTreksStatsApi WebApi app deployed string sStatType = Data.Helpers.StatScript.STAT_TYPE.r.ToString(); if (_algorithm == Calculator1.MATH_TYPES.algorithm3.ToString()) { //python is significantly slower than R sStatType = Data.Helpers.StatScript.STAT_TYPE.py.ToString(); } else if (_algorithm == Calculator1.MATH_TYPES.algorithm6.ToString()) { //julia has not been tested in 2.0.2 sStatType = Data.Helpers.StatScript.STAT_TYPE.julia.ToString(); } DevTreks.Data.Helpers.StatScript statScript = DevTreks.Data.Helpers.StatScript.GetStatScript( sStatType, scriptFilePath, inputFilePath); string sPlatformType = CalculatorHelpers.GetAppSettingString( this._params.ExtensionDocToCalcURI, "PlatformType"); if (sPlatformType.Contains("azure")) { //webapi web domain if (inputFilePath.Contains("localhost")) { statScript.DefaultWebDomain = "http://localhost:5000/"; } else { statScript.DefaultWebDomain = "http://devtreksapi1.southcentralus.cloudapp.azure.com/"; } } else { if (inputFilePath.Contains("localhost")) { statScript.DefaultWebDomain = "http://localhost:5000/"; } else { //run tests on cloud webapi site too statScript.DefaultWebDomain = "http://devtreksapi1.southcentralus.cloudapp.azure.com/"; } } //use a console app to post to a webapi CreateClient controller action bool bIsSuccess = await CalculatorHelpers.ClientCreate(statScript); if (bIsSuccess) { if ((!string.IsNullOrEmpty(statScript.StatisticalResult))) { List <string> lines = CalculatorHelpers .GetLinesFromUTF8Encoding(statScript.StatisticalResult); if (lines != null) { if (lines.Count > 0) { //store the result in the MathResult (or in the MathResult.URL below) sb.Append(statScript.StatisticalResult); sLastLine = lines.Last(); if (string.IsNullOrEmpty(sLastLine)) { int iSecondToLast = lines.Count - 2; sLastLine = lines[iSecondToLast]; } } } } else { if ((!string.IsNullOrEmpty(statScript.StatisticalResult))) { this.MathResult += statScript.ErrorMessage; } else { this.MathResult += "The remote server returned a successful response header but failed to generate the statistical results."; } } } } else { //default subalgo1 runs statpackages on the same server sLastLine = RunScript(sb, sScriptExecutable, scriptFilePath, inputFilePath); } if (this.MathResult.ToLower().StartsWith("http")) { sError = string.Empty; bool bHasSaved = CalculatorHelpers.SaveTextInURI( _params.ExtensionDocToCalcURI, sb.ToString(), this.MathResult, out sError); if (!string.IsNullOrEmpty(sError)) { this.MathResult += sError; } } else { this.MathResult = sb.ToString(); } bHasCalcs = true; //last line of string should have the QTM vars if (!string.IsNullOrEmpty(sLastLine)) { string[] vars = sLastLine.Split(Constants.CSV_DELIMITERS); bool bHasVars = false; if (vars != null) { if (vars.Count() > 1) { bHasVars = true; } if (!bHasVars) { //try space delimited vars = sLastLine.Split(' '); bHasVars = true; } if (vars != null) { //row count may be in first pos int iPos = vars.Count() - 3; if (vars[iPos] != null) { this.QTPredicted = CalculatorHelpers.ConvertStringToDouble(vars[iPos]); } iPos = vars.Count() - 2; if (vars[iPos] != null) { this.QTL = CalculatorHelpers.ConvertStringToDouble(vars[iPos]); } iPos = vars.Count() - 1; if (vars[iPos] != null) { this.QTU = CalculatorHelpers.ConvertStringToDouble(vars[iPos]); } } } } else { this.MathResult = "The script did not run successfully. Please check the dataset and script. Verify their urls."; } } catch (Exception ex) { this.ErrorMessage += ex.Message; } return(bHasCalcs); }