private void PopulateWthNormalParams() { List <WNRdngData> allWNRdngData = _weatherRepository.GetAllReadingsFromStoredProcedure(); var wNRdngDataGroups = allWNRdngData.GroupBy(s => new { s.AccID, s.UtilID, s.UnitID }); foreach (var wNRdngGroup in wNRdngDataGroups) { List <WNRdngData> wNRdngList = wNRdngGroup.OrderBy(s => s.MoID).ToList(); WNRdngData lastRead = wNRdngList.LastOrDefault(); NormalParamsAccord nParamsAccord = new NormalParamsAccord() { AccID = lastRead.AccID, UtilID = lastRead.UtilID, UnitID = lastRead.UnitID, WstID = lastRead.WstID, ZipW = lastRead.Zip, EndDate = lastRead.DateEnd, EMoID = lastRead.EMoID, MoCt = lastRead.MoCt }; try { List <BPPairAccord> allBalancePointStatsFromYear = CalculateOneYearOfDegreeDaysForAllBalancePoints(wNRdngList); int daysInYear = allBalancePointStatsFromYear.FirstOrDefault().DaysInYear; nParamsAccord.DaysInYear = daysInYear; if (allBalancePointStatsFromYear.Count == 0) { //_weatherRepository.InsertWthNormalParams(normalParamsKey, Accord: true); _weatherRepository.InsertWthNormalParamsFinal(nParamsAccord); continue; } AccordResultNew accord = CalculateLinearRegression(allBalancePointStatsFromYear, nParamsAccord); if (accord.FTestFailed) { Console.WriteLine("F Test failed... " + nParamsAccord.AccID + " " + nParamsAccord.UtilID + " " + nParamsAccord.UnitID); } nParamsAccord.B1 = decimal.Round(Convert.ToDecimal(accord.Intercept), 9, MidpointRounding.AwayFromZero); if (accord.IsSimpleSingleRegression == true && accord.HeatingBP > 0) { nParamsAccord.B2 = decimal.Round(Convert.ToDecimal(accord.B2), 9, MidpointRounding.AwayFromZero); nParamsAccord.B3 = accord.HeatingBP; } else if (accord.IsSimpleSingleRegression == true && accord.CoolingBP > 0) { nParamsAccord.B4 = decimal.Round(Convert.ToDecimal(accord.B4), 9, MidpointRounding.AwayFromZero); nParamsAccord.B5 = accord.CoolingBP; } else if (accord.IsMultipleLinearRegression == true) { nParamsAccord.B2 = decimal.Round(Convert.ToDecimal(accord.B2), 9, MidpointRounding.AwayFromZero); nParamsAccord.B3 = accord.HeatingBP; nParamsAccord.B4 = decimal.Round(Convert.ToDecimal(accord.B4), 9, MidpointRounding.AwayFromZero); nParamsAccord.B5 = accord.CoolingBP; } if (!Double.IsNaN(accord.R2Accord) && !Double.IsInfinity(accord.R2Accord)) { nParamsAccord.R2 = decimal.Round(Convert.ToDecimal(accord.R2Accord), 9, MidpointRounding.AwayFromZero); } //nParams.YearOfReadsDateStart = accord.bpPair.YearOfReadsDateStart; //nParams.YearOfReadsDateEnd = accord.bpPair.YearOfReadsDateEnd; //nParams.Readings = accord.bpPair.ReadingsInNormalYear; //nParams.Days = accord.bpPair.DaysInNormalYear; //nParams.WthZipCode = accord.bpPair.WthZipCode; //_weatherRepository.InsertWthNormalParams(nParams, Accord: true); _weatherRepository.InsertWthNormalParamsFinal(nParamsAccord); } catch (Exception e) { Console.WriteLine(nParamsAccord.AccID + " " + nParamsAccord.UtilID + " " + nParamsAccord.UnitID); Console.WriteLine(e.Message); Console.WriteLine(e.StackTrace); } } Console.WriteLine("PopulateWthNormalParams Finished."); }
private AccordResultNew CalculateLinearRegression(List <BPPairAccord> allBalancePointPairs, NormalParamsAccord nPKey) { var allBalancePointGroups = allBalancePointPairs.GroupBy(s => new { s.CoolingBalancePoint, s.HeatingBalancePoint }); List <AccordResultNew> accordResults = new List <AccordResultNew>(); //List<AccordResult> rejectedAccords = new List<AccordResult>(); foreach (var group in allBalancePointGroups) { try { List <BPPairAccord> IdenticalBalancePointPairsFromAllReadings = group.ToList(); BPPairAccord _pointPair = IdenticalBalancePointPairsFromAllReadings.First(); int readingsCount = IdenticalBalancePointPairsFromAllReadings.Count; double[] fullYData = new double[readingsCount]; double[] fullYDataDailyAvg = new double[readingsCount]; double[][] hcddMatrix = new double[readingsCount][]; double[][] hcddMatrixNonDaily = new double[readingsCount][]; foreach (BPPairAccord balancePointPair in IdenticalBalancePointPairsFromAllReadings) { fullYData[IdenticalBalancePointPairsFromAllReadings.IndexOf(balancePointPair)] = (balancePointPair.ActualUsage); fullYDataDailyAvg[IdenticalBalancePointPairsFromAllReadings.IndexOf(balancePointPair)] = (balancePointPair.ActualUsage / balancePointPair.DaysInReading); hcddMatrix[IdenticalBalancePointPairsFromAllReadings.IndexOf(balancePointPair)] = new double[] { (balancePointPair.HeatingDegreeDays / balancePointPair.DaysInReading), (balancePointPair.CoolingDegreeDays / balancePointPair.DaysInReading) }; } double[] avgHddsForEachReadingInYear = new double[readingsCount]; double[] avgCddsForEachReadingInYear = new double[readingsCount]; for (int i = 0; i < readingsCount; i++) { avgHddsForEachReadingInYear[i] = hcddMatrix[i][0]; avgCddsForEachReadingInYear[i] = hcddMatrix[i][1]; } double[] modelParams = new double[3]; modelParams[0] = 0; modelParams[1] = 0; modelParams[2] = 0; //if (fullYData.Sum() == 0) //{ // AccordResultNew empty = new AccordResultNew(); // accordResults.Add(empty); //} if (_pointPair.HeatingBalancePoint == 0 && _pointPair.CoolingBalancePoint == 0) { double[] onesVector = new double[readingsCount]; for (int i = 0; i < readingsCount; i++) { onesVector[i] = 1; } modelParams[0] = Fit.LineThroughOrigin(onesVector, fullYDataDailyAvg); OrdinaryLeastSquares ols = new OrdinaryLeastSquares() { UseIntercept = false }; //SimpleLinearRegression regressionAccord = ols.Learn(onesVector, fullYDataDailyAvg); //double[] predictedAccord = regressionAccord.Transform(onesVector); double r2 = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination(onesVector.Select(x => x * modelParams[0]), fullYDataDailyAvg); //double mean = fullYDataDailyAvg.Mean(); //if (mean != modelParams[0] || mean != regressionAccord.Slope) //{ // Console.WriteLine("Hey!"); //} AccordResultNew accordResult = new AccordResultNew() { IsSimpleSingleRegression = true, HeatingBP = _pointPair.HeatingBalancePoint, CoolingBP = _pointPair.CoolingBalancePoint, Intercept = modelParams[0], R2Accord = r2, }; accordResults.Add(accordResult); } else if (_pointPair.CoolingBalancePoint != 0 && _pointPair.HeatingBalancePoint != 0) { //modelParams = MultipleRegression.QR(hcddMatrix, fullYDataDailyAvg, intercept: true); //Accord //var ols = new OrdinaryLeastSquares() //{ // UseIntercept = true //}; try { MultipleLinearRegressionAnalysis mlra = new MultipleLinearRegressionAnalysis(intercept: true); mlra.Learn(hcddMatrix, fullYDataDailyAvg); // //MultipleLinearRegression regressionAccord = ols.Learn(hcddMatrix, fullYDataDailyAvg); var regressionAccord = mlra.Regression; double[] predicted = regressionAccord.Transform(hcddMatrix); double r2Accord = new RSquaredLoss(numberOfInputs: 2, expected: fullYDataDailyAvg) { Adjust = false }.Loss(predicted); double r2Coeff = regressionAccord.CoefficientOfDetermination(hcddMatrix, fullYDataDailyAvg, adjust: false); bool FTestFailed = !mlra.FTest.Significant; //double r2Math = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination(hcddMatrix.Select( // x => (x[0] * regressionAccord.Weights[0]) + (x[1] * regressionAccord.Weights[1]) + regressionAccord.Intercept //), fullYDataDailyAvg); //double r2MathPred = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination(predicted, fullYDataDailyAvg); AccordResultNew accordResult = new AccordResultNew() { IsMultipleLinearRegression = true, //MultipleRegression = regressionAccord, HeatingBP = _pointPair.HeatingBalancePoint, CoolingBP = _pointPair.CoolingBalancePoint, Intercept = regressionAccord.Intercept, B2 = regressionAccord.Weights[0], B4 = regressionAccord.Weights[1], R2Accord = r2Accord, FTestFailed = FTestFailed }; if (mlra.Coefficients.All(x => x.TTest.Significant)) { accordResults.Add(accordResult); } //else //{ // rejectedAccords.Add(accordResult); //} } catch (Exception e) { Console.WriteLine(nPKey.AccID + " " + nPKey.UtilID + " " + nPKey.UnitID + " " + e.Message + " " + e.StackTrace); } } else if (_pointPair.HeatingBalancePoint > 0) { // Tuple<double, double> heatingTuple = Fit.Line(avgHddsForEachReadingInYear, fullYDataDailyAvg); // modelParams[0] = heatingTuple.Item1; // modelParams[1] = heatingTuple.Item2; // double r = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination( // avgHddsForEachReadingInYear.Select(x => heatingTuple.Item1 + heatingTuple.Item2 * x), fullYDataDailyAvg); OrdinaryLeastSquares ols = new OrdinaryLeastSquares() { UseIntercept = true }; SimpleLinearRegression regressionAccord = ols.Learn(avgHddsForEachReadingInYear, fullYDataDailyAvg); double[] predictedAccord = regressionAccord.Transform(avgHddsForEachReadingInYear); double r2Accord = new RSquaredLoss(1, fullYDataDailyAvg).Loss(predictedAccord); //double rAccord2 = regressionAccord.CoefficientOfDetermination(avgHddsForEachReadingInYear, fullYDataDailyAvg, adjust: false); //double r2Math = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination(avgHddsForEachReadingInYear.Select( // x => (x * regressionAccord.Slope) + regressionAccord.Intercept // ), fullYDataDailyAvg); //double r2 = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination(predictedAccord, fullYDataDailyAvg); int degreesOfFreedom = nPKey.MoCt - 2; double ssx = Math.Sqrt((avgHddsForEachReadingInYear.Subtract(avgHddsForEachReadingInYear.Mean())).Pow(2).Sum()); double s = Math.Sqrt(((fullYDataDailyAvg.Subtract(predictedAccord).Pow(2)).Sum()) / degreesOfFreedom); double error = regressionAccord.GetStandardError(avgHddsForEachReadingInYear, fullYDataDailyAvg); double seSubB = s / ssx; double hypothesizedValue = 0; TTest tTest = new TTest( estimatedValue: regressionAccord.Slope, standardError: seSubB, degreesOfFreedom: degreesOfFreedom, hypothesizedValue: hypothesizedValue, alternate: OneSampleHypothesis.ValueIsDifferentFromHypothesis ); AccordResultNew accordResult = new AccordResultNew() { IsSimpleSingleRegression = true, HeatingBP = _pointPair.HeatingBalancePoint, Intercept = regressionAccord.Intercept, B2 = regressionAccord.Slope, R2Accord = r2Accord }; if (tTest.Significant) { accordResults.Add(accordResult); } //else //{ // rejectedAccords.Add(accordResult); //} } else if (_pointPair.CoolingBalancePoint > 0) { //Tuple<double, double> coolingTuple = Fit.Line(avgCddsForEachReadingInYear, fullYDataDailyAvg); //modelParams[0] = coolingTuple.Item1; //modelParams[2] = coolingTuple.Item2; OrdinaryLeastSquares ols = new OrdinaryLeastSquares() { UseIntercept = true }; SimpleLinearRegression regressionAccord = ols.Learn(avgCddsForEachReadingInYear, fullYDataDailyAvg); double[] predictedAccord = regressionAccord.Transform(avgCddsForEachReadingInYear); double rAccord = new RSquaredLoss(1, fullYDataDailyAvg).Loss(predictedAccord); //double r2Math = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination(avgCddsForEachReadingInYear.Select( // x => (x * regressionAccord.Slope) + regressionAccord.Intercept // ), fullYDataDailyAvg); //double r2 = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination(predictedAccord, fullYDataDailyAvg); int degreesOfFreedom = nPKey.MoCt - 2; double ssx = Math.Sqrt(avgCddsForEachReadingInYear.Subtract(avgCddsForEachReadingInYear.Mean()).Pow(2).Sum()); double s = Math.Sqrt(((fullYDataDailyAvg.Subtract(predictedAccord).Pow(2)).Sum()) / degreesOfFreedom); double seSubB = s / ssx; double hypothesizedValue = 0; double myT = seSubB / regressionAccord.Slope; TTest tTest = new TTest( estimatedValue: regressionAccord.Slope, standardError: seSubB, degreesOfFreedom: degreesOfFreedom, hypothesizedValue: hypothesizedValue, alternate: OneSampleHypothesis.ValueIsDifferentFromHypothesis ); AccordResultNew accordResult = new AccordResultNew() { IsSimpleSingleRegression = true, CoolingBP = _pointPair.CoolingBalancePoint, Intercept = regressionAccord.Intercept, B4 = regressionAccord.Slope, R2Accord = rAccord }; if (tTest.Significant) { accordResults.Add(accordResult); } //else //{ // rejectedAccords.Add(accordResult); //} } ; } catch (Exception e) { Console.WriteLine(nPKey.AccID + " " + nPKey.UtilID + " " + nPKey.UnitID + " " + e.Message + e.StackTrace); } } //rejectedAccords = rejectedAccords.OrderByDescending(s => s.R2Accord).ToList(); AccordResultNew accordWinner = accordResults .Where(s => s.Intercept >= 0) .OrderByDescending(s => s.R2Accord).ToList().FirstOrDefault(); return(accordWinner); }