public List <ReadingsQueryResult> GetReadingsFromExpUsageOriginalCorrected(AccordResult accord) { List <WthNormalParams> myParams = new List <WthNormalParams>(); List <ReadingsQueryResult> allReadings = new List <ReadingsQueryResult>(); string Sql = @"select wnp.AccID, wnp.UtilID, wnp.UnitID, weu.ExpUsage, weu.RdngID from WthNormalParams wnp join WthExpUsage weu on wnp.AccID = weu.AccID and wnp.UtilID = weu.UtilID and wnp.UnitID = weu.UnitID where wnp.R2_New > 0.8 and wnp.AccID = @AccID and wnp.UtilID = @UtilID and wnp.UnitID = @UnitID"; //where wnp.R2_New > 0.8;"; using (IDbConnection db = new SqlConnection(_myConnectionString)) { myParams = db.Query <WthNormalParams>(Sql, new { accord.AccID, accord.UtilID, accord.UnitID }).AsList(); } var myParamsGroups = myParams.GroupBy(p => new { p.AccID, p.UtilID, p.UnitID }); foreach (var group in myParamsGroups) { List <WthNormalParams> wthNormalParams = group.ToList(); foreach (WthNormalParams wnp in wthNormalParams) { ReadingsQueryResult reading = new ReadingsQueryResult(); string Sql2 = @"select RdngID, AccID, UtilID, UnitID as RUnitID, ExpUsage, Units, DateStart, DateEnd, Days from Readings where RdngID = @RdngID;"; using (IDbConnection db = new SqlConnection(_myConnectionString)) { reading = db.Query <ReadingsQueryResult>(Sql2, new { wnp.RdngID }).First(); } allReadings.Add(reading); //string Sql2 = @"select AccID, UtilID, UnitID as RUnitID, RdngID, Units, DateStart, DateEnd, Days from Readings // where RdngID = @RdngID;"; //try //{ // using (IDbConnection db = new SqlConnection(_jitWebData3ConnectionString)) // { // reading = db.Query<ReadingsQueryResult>(Sql2, new { wnp.RdngID }).First(); // } // reading.ExpUsage = wnp.ExpUsage; // allReadings.Add(reading); // try // { // InsertMyReadings(reading); // } // catch (Exception e) { Console.WriteLine(e.Message + " " + e.StackTrace); } //} catch (Exception e) // { // Console.WriteLine(e.Message + " " + wnp.RdngID); // } } } return(allReadings); }
private void PopulateWthNormalParams() { List <WthNormalParams> normalParamsKeys = _weatherRepository.GetNormalParamsKeysForRegression(); foreach (WthNormalParams normalParamsKey in normalParamsKeys) { try { int yearAsInt = normalParamsKey.EndDate_Original.Year - 2000; string desiredStartDate = yearAsInt.ToString(); List <BalancePointPair> allBalancePointStatsFromYear = CalculateOneYearOfDegreeDaysForAllBalancePoints(desiredStartDate, normalParamsKey).ToList(); if (allBalancePointStatsFromYear.Count == 0) { _weatherRepository.InsertWthNormalParams(normalParamsKey, Accord: true); //_weatherRepository.InsertWthNormalParamsFinal(normalParamsKey); continue; } List <AccordResult> accords = CalculateLinearRegression(allBalancePointStatsFromYear, normalParamsKey); accords = accords .Where(s => s.Intercept >= 0) .OrderByDescending(s => s.R2Accord).ToList(); AccordResult bestAccord = accords.FirstOrDefault(); if (bestAccord.IsMultipleLinearRegression) { if (bestAccord.MLRA.FTest.Significant == false) { normalParamsKey.FTestFailed = 1; Console.WriteLine("F Test failed... " + normalParamsKey.AccID + " " + normalParamsKey.UtilID + " " + normalParamsKey.UnitID); } } normalParamsKey.B1_New = decimal.Round(Convert.ToDecimal(bestAccord.Intercept), 9, MidpointRounding.AwayFromZero); normalParamsKey.B2_New = 0; normalParamsKey.B3_New = 0; normalParamsKey.B4_New = 0; normalParamsKey.B5_New = 0; if (bestAccord.IsSimpleSingleRegression == true && bestAccord.HeatingBP > 0) { normalParamsKey.B2_New = decimal.Round(Convert.ToDecimal(bestAccord.SimpleLinearRegression.Slope), 9, MidpointRounding.AwayFromZero); normalParamsKey.B3_New = bestAccord.HeatingBP; } else if (bestAccord.IsSimpleSingleRegression == true && bestAccord.CoolingBP > 0) { normalParamsKey.B4_New = decimal.Round(Convert.ToDecimal(bestAccord.SimpleLinearRegression.Slope), 9, MidpointRounding.AwayFromZero); normalParamsKey.B5_New = bestAccord.CoolingBP; } else if (bestAccord.IsMultipleLinearRegression == true) { normalParamsKey.B2_New = decimal.Round(Convert.ToDecimal(bestAccord.MLRA.Regression.Weights[0]), 9, MidpointRounding.AwayFromZero); normalParamsKey.B3_New = bestAccord.HeatingBP; normalParamsKey.B4_New = decimal.Round(Convert.ToDecimal(bestAccord.MLRA.Regression.Weights[1]), 9, MidpointRounding.AwayFromZero); normalParamsKey.B5_New = bestAccord.CoolingBP; } if (AreDoublesAllNotInfinityNorNaN(new double[] { bestAccord.R2Accord })) { normalParamsKey.R2_New = decimal.Round(Convert.ToDecimal(bestAccord.R2Accord), 9, MidpointRounding.AwayFromZero); } else { normalParamsKey.R2_New = 0; } normalParamsKey.YearOfReadsDateStart = bestAccord.bpPair.YearOfReadsDateStart; normalParamsKey.YearOfReadsDateEnd = bestAccord.bpPair.YearOfReadsDateEnd; normalParamsKey.Readings = bestAccord.bpPair.ReadingsInNormalYear; normalParamsKey.Days = bestAccord.bpPair.DaysInNormalYear; normalParamsKey.WthZipCode = bestAccord.bpPair.WthZipCode; _weatherRepository.InsertWthNormalParams(normalParamsKey, Accord: true); //_weatherRepository.InsertWthNormalParamsFinal(normalParamsKey); } catch (Exception e) { Console.WriteLine(normalParamsKey.AccID + " " + normalParamsKey.EndDate_Original); Console.WriteLine(e.Message); Console.WriteLine(e.StackTrace); } } Console.WriteLine("PopulateWthNormalParams Finished."); }
private void PopulateMyWthExpUsage(AccordResult accord) { //List<ReadingsQueryResult> allReadings = _weatherRepository.GetReadingsFromExpUsageOriginal(); List <ReadingsQueryResult> allReadings = _weatherRepository.GetReadingsFromExpUsageOriginalCorrected(accord); var readingsByAccount = allReadings.GroupBy(r => new { r.AccID, r.UtilID, r.RUnitID }); foreach (var group in readingsByAccount) { try { List <ReadingsQueryResult> readings = group.ToList(); WthNormalParams _params = _weatherRepository.GetParamsForReading(readings.First().AccID, readings.First().UtilID, readings.First().RUnitID); foreach (ReadingsQueryResult reading in readings) { WthExpUsage wthExpUsage = new WthExpUsage(); wthExpUsage.RdngID = reading.RdngID; wthExpUsage.AccID = reading.AccID; wthExpUsage.UtilID = reading.UtilID; wthExpUsage.UnitID = reading.RUnitID; decimal expUsage_Old = decimal.Round(Convert.ToDecimal(reading.ExpUsage), 4, MidpointRounding.AwayFromZero); wthExpUsage.ExpUsage_Old = expUsage_Old; wthExpUsage.Units = reading.Units; List <WeatherData> weatherDataList = _weatherRepository.GetWeatherDataByZipStartAndEndDate(_params.WthZipCode, reading.DateStart, reading.DateEnd); decimal B1 = 0; decimal B2 = 0; decimal B4 = 0; reading.B3 = accord.HeatingBP; reading.B5 = accord.CoolingBP; if (accord.IsSimpleSingleRegression) { if (reading.B3 > 0) { B1 = decimal.Round(Convert.ToDecimal(accord.SimpleLinearRegression.Intercept), 12, MidpointRounding.AwayFromZero); B2 = decimal.Round(Convert.ToDecimal(accord.SimpleLinearRegression.Slope), 12, MidpointRounding.AwayFromZero); B4 = 0; } else if (reading.B5 > 0) { B1 = decimal.Round(Convert.ToDecimal(accord.SimpleLinearRegression.Intercept), 12, MidpointRounding.AwayFromZero); B2 = 0; B4 = decimal.Round(Convert.ToDecimal(accord.SimpleLinearRegression.Slope), 12, MidpointRounding.AwayFromZero); } } else { if (reading.B3 > 0 && reading.B5 > 0) { B1 = decimal.Round(Convert.ToDecimal(accord.MultipleRegression.Weights[0]), 12, MidpointRounding.AwayFromZero); B2 = decimal.Round(Convert.ToDecimal(accord.MultipleRegression.Weights[1]), 12, MidpointRounding.AwayFromZero); B4 = decimal.Round(Convert.ToDecimal(accord.MultipleRegression.Weights[2]), 12, MidpointRounding.AwayFromZero); } else if (reading.B3 > 0) { B1 = decimal.Round(Convert.ToDecimal(accord.MultipleRegression.Weights[0]), 12, MidpointRounding.AwayFromZero); B2 = decimal.Round(Convert.ToDecimal(accord.MultipleRegression.Weights[1]), 12, MidpointRounding.AwayFromZero); } else if (reading.B5 > 0) { B1 = decimal.Round(Convert.ToDecimal(accord.MultipleRegression.Weights[0]), 12, MidpointRounding.AwayFromZero); B4 = decimal.Round(Convert.ToDecimal(accord.MultipleRegression.Weights[1]), 12, MidpointRounding.AwayFromZero); } } HeatingCoolingDegreeDays hcdd = HeatingCoolingDegreeDaysValueOf(reading, weatherDataList); decimal expUsage_New = (B1 * reading.Days) + (B2 * decimal.Round(Convert.ToDecimal(hcdd.HDD), 12, MidpointRounding.AwayFromZero)) + (B4 * decimal.Round(Convert.ToDecimal(hcdd.CDD), 12, MidpointRounding.AwayFromZero)); wthExpUsage.ExpUsage_New = expUsage_New; if (reading.Units != 0) { wthExpUsage.PercentDelta_Old = Math.Abs((expUsage_Old - reading.Units) / reading.Units); wthExpUsage.PercentDelta_New = Math.Abs((expUsage_New - reading.Units) / reading.Units); } wthExpUsage.DateStart = reading.DateStart; wthExpUsage.DateEnd = reading.DateEnd; _weatherRepository.InsertMyWthExpUsage(wthExpUsage, Accord: true); } } catch (Exception e) { Console.WriteLine(e.Message + " " + group.ToList().First().RdngID + " " + group.ToList().First().AccID + " " + group.ToList().First().RUnitID + "\n" + e.StackTrace); } } }
private List <AccordResult> CalculateLinearRegression(List <BalancePointPair> allBalancePointPairs, WthNormalParams normalParamsKey) { var allBalancePointGroups = allBalancePointPairs.GroupBy(s => new { s.CoolingBalancePoint, s.HeatingBalancePoint }); List <AccordResult> accordResults = new List <AccordResult>(); List <AccordResult> rejectedAccords = new List <AccordResult>(); foreach (var group in allBalancePointGroups) { try { List <BalancePointPair> IdenticalBalancePointPairsForAllReadings = group.ToList(); BalancePointPair _pointPair = IdenticalBalancePointPairsForAllReadings.First(); int readingsCount = IdenticalBalancePointPairsForAllReadings.Count; double[] fullYData = new double[readingsCount]; double[] fullYDataDailyAvg = new double[readingsCount]; double[][] hcddMatrix = new double[readingsCount][]; double[][] hcddMatrixNonDaily = new double[readingsCount][]; foreach (BalancePointPair balancePointPair in IdenticalBalancePointPairsForAllReadings) { fullYData[IdenticalBalancePointPairsForAllReadings.IndexOf(balancePointPair)] = (balancePointPair.ActualUsage); fullYDataDailyAvg[IdenticalBalancePointPairsForAllReadings.IndexOf(balancePointPair)] = (balancePointPair.ActualUsage / balancePointPair.DaysInReading); hcddMatrix[IdenticalBalancePointPairsForAllReadings.IndexOf(balancePointPair)] = new double[] { (balancePointPair.HeatingDegreeDays / balancePointPair.DaysInReading), (balancePointPair.CoolingDegreeDays / balancePointPair.DaysInReading) }; } double[] avgHddsForEachReadingInYear = new double[readingsCount]; double[] avgCddsForEachReadingInYear = new double[readingsCount]; for (int i = 0; i < readingsCount; i++) { avgHddsForEachReadingInYear[i] = hcddMatrix[i][0]; avgCddsForEachReadingInYear[i] = hcddMatrix[i][1]; } double[] modelParams = new double[3]; modelParams[0] = 0; modelParams[1] = 0; modelParams[2] = 0; if (fullYData.Sum() == 0) { AccordResult empty = new AccordResult { bpPair = _pointPair }; accordResults.Add(empty); } else if (_pointPair.HeatingBalancePoint == 0 && _pointPair.CoolingBalancePoint == 0) { double[] onesVector = new double[readingsCount]; for (int i = 0; i < readingsCount; i++) { onesVector[i] = 1; } modelParams[0] = Fit.LineThroughOrigin(onesVector, fullYDataDailyAvg); OrdinaryLeastSquares ols = new OrdinaryLeastSquares() { UseIntercept = false }; SimpleLinearRegression regressionAccord = ols.Learn(onesVector, fullYDataDailyAvg); //double[] predictedAccord = regressionAccord.Transform(onesVector); double r2 = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination(onesVector.Select(x => x * modelParams[0]), fullYDataDailyAvg); //double mean = fullYDataDailyAvg.Mean(); //if (mean != modelParams[0] || mean != regressionAccord.Slope) //{ // Console.WriteLine("Hey!"); //} //double r2Accord = regressionAccord.CoefficientOfDetermination(onesVector, fullYDataDailyAvg); //double sxx = onesVector.Subtract(onesVector.Mean()).Pow(2).Sum(); //double hypothesizedValue = 0; //try //{ // TTest test = new TTest( // estimatedValue: regressionAccord.Slope, standardError: sxx, degreesOfFreedom: _pointPair.ReadingsInNormalYear - 2, // hypothesizedValue: hypothesizedValue, alternate: OneSampleHypothesis.ValueIsDifferentFromHypothesis // ); // if (test.Significant) // { AccordResult accordResult = new AccordResult() { SimpleLinearRegression = regressionAccord, R2Accord = r2, IsSimpleSingleRegression = true, HeatingBP = _pointPair.HeatingBalancePoint, CoolingBP = _pointPair.CoolingBalancePoint, Intercept = regressionAccord.Slope, bpPair = _pointPair }; accordResults.Add(accordResult); // } //} //catch (Exception e) //{ // Console.WriteLine(e.Message + e.StackTrace); //} } else if (_pointPair.CoolingBalancePoint != 0 && _pointPair.HeatingBalancePoint != 0) { //modelParams = MultipleRegression.QR(hcddMatrix, fullYDataDailyAvg, intercept: true); //Accord //var ols = new OrdinaryLeastSquares() //{ // UseIntercept = true //}; try { MultipleLinearRegressionAnalysis mlra = new MultipleLinearRegressionAnalysis(intercept: true); mlra.Learn(hcddMatrix, fullYDataDailyAvg); // //MultipleLinearRegression regressionAccord = ols.Learn(hcddMatrix, fullYDataDailyAvg); var regressionAccord = mlra.Regression; double[] predicted = regressionAccord.Transform(hcddMatrix); double r2Accord = new RSquaredLoss(numberOfInputs: 2, expected: fullYDataDailyAvg) { Adjust = false }.Loss(predicted); double r2Coeff = regressionAccord.CoefficientOfDetermination(hcddMatrix, fullYDataDailyAvg, adjust: false); //double r2Math = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination(hcddMatrix.Select( // x => (x[0] * regressionAccord.Weights[0]) + (x[1] * regressionAccord.Weights[1]) + regressionAccord.Intercept //), fullYDataDailyAvg); //double r2MathPred = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination(predicted, fullYDataDailyAvg); AccordResult accordResult = new AccordResult() { //MultipleRegression = regressionAccord, R2Accord = r2Accord, R2Coeff = r2Coeff, HeatingBP = _pointPair.HeatingBalancePoint, CoolingBP = _pointPair.CoolingBalancePoint, IsSimpleSingleRegression = false, MLRA = mlra, Intercept = regressionAccord.Intercept, bpPair = _pointPair, IsMultipleLinearRegression = true }; if (mlra.Coefficients.All(x => x.TTest.Significant)) { accordResults.Add(accordResult); } else { rejectedAccords.Add(accordResult); } } catch (Exception e) { Console.WriteLine(e.Message + " " + e.StackTrace); } } else if (_pointPair.HeatingBalancePoint > 0) { // Tuple<double, double> heatingTuple = Fit.Line(avgHddsForEachReadingInYear, fullYDataDailyAvg); // modelParams[0] = heatingTuple.Item1; // modelParams[1] = heatingTuple.Item2; // double r = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination( // avgHddsForEachReadingInYear.Select(x => heatingTuple.Item1 + heatingTuple.Item2 * x), fullYDataDailyAvg); OrdinaryLeastSquares ols = new OrdinaryLeastSquares() { UseIntercept = true }; SimpleLinearRegression regressionAccord = ols.Learn(avgHddsForEachReadingInYear, fullYDataDailyAvg); double[] predictedAccord = regressionAccord.Transform(avgHddsForEachReadingInYear); double rAccord = new RSquaredLoss(1, fullYDataDailyAvg).Loss(predictedAccord); //double rAccord2 = regressionAccord.CoefficientOfDetermination(avgHddsForEachReadingInYear, fullYDataDailyAvg, adjust: false); //double r2Math = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination(avgHddsForEachReadingInYear.Select( // x => (x * regressionAccord.Slope) + regressionAccord.Intercept // ), fullYDataDailyAvg); //double r2 = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination(predictedAccord, fullYDataDailyAvg); int degreesOfFreedom = _pointPair.ReadingsInNormalYear - 2; double ssx = Math.Sqrt((avgHddsForEachReadingInYear.Subtract(avgHddsForEachReadingInYear.Mean())).Pow(2).Sum()); double s = Math.Sqrt(((fullYDataDailyAvg.Subtract(predictedAccord).Pow(2)).Sum()) / degreesOfFreedom); double error = regressionAccord.GetStandardError(avgHddsForEachReadingInYear, fullYDataDailyAvg); double seSubB = s / ssx; double hypothesizedValue = 0; TTest tTest = new TTest( estimatedValue: regressionAccord.Slope, standardError: seSubB, degreesOfFreedom: degreesOfFreedom, hypothesizedValue: hypothesizedValue, alternate: OneSampleHypothesis.ValueIsDifferentFromHypothesis ); AccordResult accordResult = new AccordResult() { SimpleLinearRegression = regressionAccord, R2Accord = rAccord, IsSimpleSingleRegression = true, HeatingBP = _pointPair.HeatingBalancePoint, CoolingBP = _pointPair.CoolingBalancePoint, TTest = tTest, Intercept = regressionAccord.Intercept, bpPair = _pointPair }; if (tTest.Significant) { accordResults.Add(accordResult); } else { rejectedAccords.Add(accordResult); } } else if (_pointPair.CoolingBalancePoint > 0) { //Tuple<double, double> coolingTuple = Fit.Line(avgCddsForEachReadingInYear, fullYDataDailyAvg); //modelParams[0] = coolingTuple.Item1; //modelParams[2] = coolingTuple.Item2; OrdinaryLeastSquares ols = new OrdinaryLeastSquares() { UseIntercept = true }; SimpleLinearRegression regressionAccord = ols.Learn(avgCddsForEachReadingInYear, fullYDataDailyAvg); double[] predictedAccord = regressionAccord.Transform(avgCddsForEachReadingInYear); double rAccord = new RSquaredLoss(1, fullYDataDailyAvg).Loss(predictedAccord); //double r2Math = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination(avgCddsForEachReadingInYear.Select( // x => (x * regressionAccord.Slope) + regressionAccord.Intercept // ), fullYDataDailyAvg); //double r2 = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination(predictedAccord, fullYDataDailyAvg); int degreesOfFreedom = _pointPair.ReadingsInNormalYear - 2; double ssx = Math.Sqrt(avgCddsForEachReadingInYear.Subtract(avgCddsForEachReadingInYear.Mean()).Pow(2).Sum()); double s = Math.Sqrt(((fullYDataDailyAvg.Subtract(predictedAccord).Pow(2)).Sum()) / degreesOfFreedom); double seSubB = s / ssx; double hypothesizedValue = 0; double myT = seSubB / regressionAccord.Slope; TTest tTest = new TTest( estimatedValue: regressionAccord.Slope, standardError: seSubB, degreesOfFreedom: degreesOfFreedom, hypothesizedValue: hypothesizedValue, alternate: OneSampleHypothesis.ValueIsDifferentFromHypothesis ); AccordResult accordResult = new AccordResult() { SimpleLinearRegression = regressionAccord, R2Accord = rAccord, IsSimpleSingleRegression = true, HeatingBP = _pointPair.HeatingBalancePoint, CoolingBP = _pointPair.CoolingBalancePoint, TTest = tTest, Intercept = regressionAccord.Intercept, bpPair = _pointPair }; if (tTest.Significant) { accordResults.Add(accordResult); } else { rejectedAccords.Add(accordResult); } } ; } catch (Exception e) { Console.WriteLine(normalParamsKey.AccID + " " + normalParamsKey.UtilID + " " + normalParamsKey.UnitID + " " + e.Message + e.StackTrace); } } //rejectedAccords = rejectedAccords.OrderByDescending(s => s.R2Accord).ToList(); //accordResults = accordResults.OrderByDescending(s => s.R2Accord).ToList(); return(accordResults); }
private void PopulateWthNormalParams() { List <WNRdngData> allWNRdngData = _weatherRepository.GetAllReadingsFromStoredProcedure(); var wNRdngDataGroups = allWNRdngData.GroupBy(s => new { s.AccID, s.UtilID, s.UnitID }); List <WthNormalParams> newNormalParamsList = new List <WthNormalParams>(); int updateCount = 0; int insertCount = 0; foreach (var wNRdngGroup in wNRdngDataGroups) { List <WNRdngData> wNRdngList = wNRdngGroup.OrderBy(s => s.MoID).ToList(); WNRdngData lastRead = wNRdngList.LastOrDefault(); WthNormalParams normalParams = new WthNormalParams() { AccID = lastRead.AccID, UtilID = lastRead.UtilID, UnitID = lastRead.UnitID, WstID = lastRead.WstID, ZipW = lastRead.Zip, EndDate = lastRead.DateEnd, EMoID = lastRead.EMoID, MoCt = lastRead.MoCt }; bool normalParamsExists = _weatherRepository.GetWthNormalParamsExists(normalParams); try { List <BalancePointPair> allBalancePointStatsFromYear = CalculateOneYearOfDegreeDaysForAllBalancePoints(wNRdngList); int daysInYear = allBalancePointStatsFromYear.FirstOrDefault().DaysInYear; normalParams.DaysInYear = daysInYear; if (allBalancePointStatsFromYear.Count == 0) { if (normalParamsExists) { _weatherRepository.UpdateWthNormalParams(normalParams); newNormalParamsList.Add(normalParams); updateCount++; Log.Debug($"Updated WthNormalParams >> AccID: {normalParams.AccID}. UtilID: {normalParams.UtilID} UnitID: {normalParams.UnitID} " + $"B1: {normalParams.B1} B2: {normalParams.B2} B3: {normalParams.B3} B4: {normalParams.B4} B5: {normalParams.B5} R2: {normalParams.R2}."); } else { _weatherRepository.InsertWthNormalParams(normalParams); newNormalParamsList.Add(normalParams); insertCount++; Log.Debug($"Inserted WthNormalParams >> AccID: {normalParams.AccID}. UtilID: {normalParams.UtilID} UnitID: {normalParams.UnitID} " + $"B1: {normalParams.B1} B2: {normalParams.B2} B3: {normalParams.B3} B4: {normalParams.B4} B5: {normalParams.B5} R2: {normalParams.R2}."); } continue; } AccordResult accord = CalculateLinearRegression(allBalancePointStatsFromYear, normalParams); if (accord.FTestFailed) { Log.Information("F Test failed... " + normalParams.AccID + " " + normalParams.UtilID + " " + normalParams.UnitID); } normalParams.B1 = decimal.Round(Convert.ToDecimal(accord.Intercept), 9, MidpointRounding.AwayFromZero); if (accord.IsSimpleSingleRegression == true && accord.HeatingBP > 0) { normalParams.B2 = decimal.Round(Convert.ToDecimal(accord.B2), 9, MidpointRounding.AwayFromZero); normalParams.B3 = accord.HeatingBP; } else if (accord.IsSimpleSingleRegression == true && accord.CoolingBP > 0) { normalParams.B4 = decimal.Round(Convert.ToDecimal(accord.B4), 9, MidpointRounding.AwayFromZero); normalParams.B5 = accord.CoolingBP; } else if (accord.IsMultipleLinearRegression == true) { normalParams.B2 = decimal.Round(Convert.ToDecimal(accord.B2), 9, MidpointRounding.AwayFromZero); normalParams.B3 = accord.HeatingBP; normalParams.B4 = decimal.Round(Convert.ToDecimal(accord.B4), 9, MidpointRounding.AwayFromZero); normalParams.B5 = accord.CoolingBP; } if (!Double.IsNaN(accord.R2Accord) && !Double.IsInfinity(accord.R2Accord)) { normalParams.R2 = decimal.Round(Convert.ToDecimal(accord.R2Accord), 9, MidpointRounding.AwayFromZero); } if (normalParamsExists) { _weatherRepository.UpdateWthNormalParams(normalParams); newNormalParamsList.Add(normalParams); updateCount++; Log.Debug($"Updated WthNormalParams >> AccID: {normalParams.AccID}. UtilID: {normalParams.UtilID} UnitID: {normalParams.UnitID} " + $"B1: {normalParams.B1} B2: {normalParams.B2} B3: {normalParams.B3} B4: {normalParams.B4} B5: {normalParams.B5} R2: {normalParams.R2}."); } else { _weatherRepository.InsertWthNormalParams(normalParams); newNormalParamsList.Add(normalParams); insertCount++; Log.Debug($"Inserted WthNormalParams >> AccID: {normalParams.AccID}. UtilID: {normalParams.UtilID} UnitID: {normalParams.UnitID} " + $"B1: {normalParams.B1} B2: {normalParams.B2} B3: {normalParams.B3} B4: {normalParams.B4} B5: {normalParams.B5} R2: {normalParams.R2}."); } } catch (Exception e) { Log.Error(normalParams.AccID + " " + normalParams.UtilID + " " + normalParams.UnitID + " " + e.Message + " " + e.StackTrace); } } Log.Information($"PopulateWthNormalParams Finished. Rows Updated: {updateCount}. Rows Inserted: {insertCount}."); UpdateWthExpUsage(newNormalParamsList); }
private AccordResult CalculateLinearRegression(List <BalancePointPair> allBalancePointPairs, WthNormalParams normalParamsKey) { var allBalancePointGroups = allBalancePointPairs.GroupBy(s => new { s.CoolingBalancePoint, s.HeatingBalancePoint }); List <AccordResult> accordResults = new List <AccordResult>(); foreach (var group in allBalancePointGroups) { try { List <BalancePointPair> IdenticalBalancePointPairsFromAllReadings = group.ToList(); BalancePointPair _pointPair = IdenticalBalancePointPairsFromAllReadings.First(); int readingsCount = IdenticalBalancePointPairsFromAllReadings.Count; double[] fullYData = new double[readingsCount]; double[] fullYDataDailyAvg = new double[readingsCount]; double[][] hcddMatrix = new double[readingsCount][]; double[][] hcddMatrixNonDaily = new double[readingsCount][]; foreach (BalancePointPair balancePointPair in IdenticalBalancePointPairsFromAllReadings) { fullYData[IdenticalBalancePointPairsFromAllReadings.IndexOf(balancePointPair)] = (balancePointPair.ActualUsage); fullYDataDailyAvg[IdenticalBalancePointPairsFromAllReadings.IndexOf(balancePointPair)] = (balancePointPair.ActualUsage / balancePointPair.DaysInReading); hcddMatrix[IdenticalBalancePointPairsFromAllReadings.IndexOf(balancePointPair)] = new double[] { (balancePointPair.HeatingDegreeDays / balancePointPair.DaysInReading), (balancePointPair.CoolingDegreeDays / balancePointPair.DaysInReading) }; } double[] avgHddsForEachReadingInYear = new double[readingsCount]; double[] avgCddsForEachReadingInYear = new double[readingsCount]; for (int i = 0; i < readingsCount; i++) { avgHddsForEachReadingInYear[i] = hcddMatrix[i][0]; avgCddsForEachReadingInYear[i] = hcddMatrix[i][1]; } double[] modelParams = new double[3]; modelParams[0] = 0; modelParams[1] = 0; modelParams[2] = 0; if (_pointPair.HeatingBalancePoint == 0 && _pointPair.CoolingBalancePoint == 0) { double[] onesVector = new double[readingsCount]; for (int i = 0; i < readingsCount; i++) { onesVector[i] = 1; } modelParams[0] = Fit.LineThroughOrigin(onesVector, fullYDataDailyAvg); OrdinaryLeastSquares ols = new OrdinaryLeastSquares() { UseIntercept = false }; double r2 = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination( onesVector.Select(x => x * modelParams[0]), fullYDataDailyAvg); AccordResult accordResult = new AccordResult() { IsSimpleSingleRegression = true, HeatingBP = _pointPair.HeatingBalancePoint, CoolingBP = _pointPair.CoolingBalancePoint, Intercept = modelParams[0], R2Accord = r2, }; accordResults.Add(accordResult); } else if (_pointPair.CoolingBalancePoint != 0 && _pointPair.HeatingBalancePoint != 0) { try { MultipleLinearRegressionAnalysis mlra = new MultipleLinearRegressionAnalysis(intercept: true); mlra.Learn(hcddMatrix, fullYDataDailyAvg); var regressionAccord = mlra.Regression; double[] predicted = regressionAccord.Transform(hcddMatrix); double r2Accord = new RSquaredLoss(numberOfInputs: 2, expected: fullYDataDailyAvg) { Adjust = false }.Loss(predicted); double r2Coeff = regressionAccord.CoefficientOfDetermination(hcddMatrix, fullYDataDailyAvg, adjust: false); bool FTestFailed = !mlra.FTest.Significant; AccordResult accordResult = new AccordResult() { IsMultipleLinearRegression = true, HeatingBP = _pointPair.HeatingBalancePoint, CoolingBP = _pointPair.CoolingBalancePoint, Intercept = regressionAccord.Intercept, B2 = regressionAccord.Weights[0], B4 = regressionAccord.Weights[1], R2Accord = r2Accord, FTestFailed = FTestFailed }; if (mlra.Coefficients.All(x => x.TTest.Significant)) { accordResults.Add(accordResult); } } catch (Exception e) { Log.Debug(normalParamsKey.AccID + " " + normalParamsKey.UtilID + " " + normalParamsKey.UnitID + " " + e.Message + " " + e.StackTrace); } } else if (_pointPair.HeatingBalancePoint > 0) { OrdinaryLeastSquares ols = new OrdinaryLeastSquares() { UseIntercept = true }; SimpleLinearRegression regressionAccord = ols.Learn(avgHddsForEachReadingInYear, fullYDataDailyAvg); double[] predictedAccord = regressionAccord.Transform(avgHddsForEachReadingInYear); double r2Accord = new RSquaredLoss(1, fullYDataDailyAvg).Loss(predictedAccord); int degreesOfFreedom = normalParamsKey.MoCt - 2; double ssx = Math.Sqrt((avgHddsForEachReadingInYear.Subtract(avgHddsForEachReadingInYear.Mean())).Pow(2).Sum()); double s = Math.Sqrt(((fullYDataDailyAvg.Subtract(predictedAccord).Pow(2)).Sum()) / degreesOfFreedom); double error = regressionAccord.GetStandardError(avgHddsForEachReadingInYear, fullYDataDailyAvg); double seSubB = s / ssx; double hypothesizedValue = 0; TTest tTest = new TTest( estimatedValue: regressionAccord.Slope, standardError: seSubB, degreesOfFreedom: degreesOfFreedom, hypothesizedValue: hypothesizedValue, alternate: OneSampleHypothesis.ValueIsDifferentFromHypothesis ); AccordResult accordResult = new AccordResult() { IsSimpleSingleRegression = true, HeatingBP = _pointPair.HeatingBalancePoint, Intercept = regressionAccord.Intercept, B2 = regressionAccord.Slope, R2Accord = r2Accord }; if (tTest.Significant) { accordResults.Add(accordResult); } } else if (_pointPair.CoolingBalancePoint > 0) { OrdinaryLeastSquares ols = new OrdinaryLeastSquares() { UseIntercept = true }; SimpleLinearRegression regressionAccord = ols.Learn(avgCddsForEachReadingInYear, fullYDataDailyAvg); double[] predictedAccord = regressionAccord.Transform(avgCddsForEachReadingInYear); double rAccord = new RSquaredLoss(1, fullYDataDailyAvg).Loss(predictedAccord); int degreesOfFreedom = normalParamsKey.MoCt - 2; double ssx = Math.Sqrt(avgCddsForEachReadingInYear.Subtract(avgCddsForEachReadingInYear.Mean()).Pow(2).Sum()); double s = Math.Sqrt(((fullYDataDailyAvg.Subtract(predictedAccord).Pow(2)).Sum()) / degreesOfFreedom); double seSubB = s / ssx; double hypothesizedValue = 0; double myT = seSubB / regressionAccord.Slope; TTest tTest = new TTest( estimatedValue: regressionAccord.Slope, standardError: seSubB, degreesOfFreedom: degreesOfFreedom, hypothesizedValue: hypothesizedValue, alternate: OneSampleHypothesis.ValueIsDifferentFromHypothesis ); AccordResult accordResult = new AccordResult() { IsSimpleSingleRegression = true, CoolingBP = _pointPair.CoolingBalancePoint, Intercept = regressionAccord.Intercept, B4 = regressionAccord.Slope, R2Accord = rAccord }; if (tTest.Significant) { accordResults.Add(accordResult); } } ; } catch (Exception e) { Log.Debug(normalParamsKey.AccID + " " + normalParamsKey.UtilID + " " + normalParamsKey.UnitID + " " + e.Message + e.StackTrace); } } AccordResult accordWinner = accordResults .Where(s => s.Intercept >= 0) .OrderByDescending(s => s.R2Accord).ToList().FirstOrDefault(); return(accordWinner); }
private void PopulateWthNormalParams() { List <WthNormalParams> newNormalParamsList = new List <WthNormalParams>(); List <WthNormalParams> modelsWithNotTwelveReadings = new List <WthNormalParams>(); List <WthNormalParams> modelsWithReadingOrWeatherIssues = new List <WthNormalParams>(); List <WthNormalParams> modelsWithOtherIssues = new List <WthNormalParams>(); Log.Information("Starting PopulateWthNormalParams(). Searching WNRdngData01 stored procedure... "); List <WNRdngData> allWNRdngData = _weatherRepository.GetAllReadingsFromStoredProcedure(); var wNRdngDataGroups = allWNRdngData.GroupBy(s => new { s.AccID, s.UtilID, s.UnitID }); //List<WNRdngData> heyy = wNRdngDataGroups.Contains(s => new { }) int updateCount = 0; int insertCount = 0; int failCount = 0; foreach (var wNRdngGroup in wNRdngDataGroups) { List <WNRdngData> wNRdngList = wNRdngGroup.OrderBy(s => s.MoID).ToList(); WNRdngData lastRead = wNRdngList.LastOrDefault(); WthNormalParams normalParams = new WthNormalParams() { AccID = lastRead.AccID, UtilID = lastRead.UtilID, UnitID = lastRead.UnitID, WstID = lastRead.WstID, ZipW = lastRead.Zip, EndDate = lastRead.DateEnd, EMoID = lastRead.EMoID, MoCt = lastRead.MoCt }; if (wNRdngList.Count < 6) { Log.Error("There are only {0} readings for AccID/UtilID/UnitID: {1}/{2}/{3}. No model found.", wNRdngList.Count, normalParams.AccID, normalParams.UtilID, normalParams.UnitID); continue; } if (wNRdngList.Count % 12 != 0) { modelsWithNotTwelveReadings.Add(normalParams); Log.Warning($"wNRdngList is not a multiple of 12 .. {wNRdngList.Count} Readings.\"MoCt\": {normalParams.MoCt}, still proceeding..."); } bool normalParamsExists = _weatherRepository.GetWthNormalParamsExists(normalParams); try { List <BalancePointPair> allBalancePointStatsFromYear = CalculateOneYearOfDegreeDaysForAllBalancePoints(wNRdngList); //int daysInYear = allBalancePointStatsFromYear.FirstOrDefault().DaysInYear; //normalParams.DaysInYear = daysInYear; bool success = false; if (allBalancePointStatsFromYear.Count == 0) { Log.Error($"AccID/UtilID/UnitID: {normalParams.AccID}/{normalParams.UtilID}/{normalParams.UnitID} >> " + $"Why are there no BalancePointStatsFromYear?"); if (normalParamsExists) { success = UpdateOrInsertWthNormalParams(normalParams); if (success) { newNormalParamsList.Add(normalParams); updateCount++; } else { failCount++; } } else { success = UpdateOrInsertWthNormalParams(normalParams); if (success) { newNormalParamsList.Add(normalParams); insertCount++; } else { failCount++; } } continue; } AccordResult accord = CalculateLinearRegression(allBalancePointStatsFromYear, normalParams); //if (accord.R2Accord < 0.7500) //{ // success = UpdateOrInsertWthNormalParams(normalParams); // if (success && normalParamsExists) // { // newNormalParamsList.Add(normalParams); // updateCount++; // } // else if (success && !normalParamsExists) // { // newNormalParamsList.Add(normalParams); // insertCount++; // } // else // { // failCount++; // } // continue; //} normalParams.B1 = decimal.Round(Convert.ToDecimal(accord.Intercept), 9, MidpointRounding.AwayFromZero); if (accord.IsSimpleSingleRegression == true && accord.HeatingBP > 0) { normalParams.B2 = decimal.Round(Convert.ToDecimal(accord.B2), 9, MidpointRounding.AwayFromZero); normalParams.B3 = accord.HeatingBP; } else if (accord.IsSimpleSingleRegression == true && accord.CoolingBP > 0) { normalParams.B4 = decimal.Round(Convert.ToDecimal(accord.B4), 9, MidpointRounding.AwayFromZero); normalParams.B5 = accord.CoolingBP; } else if (accord.IsMultipleLinearRegression == true) { normalParams.B2 = decimal.Round(Convert.ToDecimal(accord.B2), 9, MidpointRounding.AwayFromZero); normalParams.B3 = accord.HeatingBP; normalParams.B4 = decimal.Round(Convert.ToDecimal(accord.B4), 9, MidpointRounding.AwayFromZero); normalParams.B5 = accord.CoolingBP; } if (!double.IsNaN(accord.R2Accord) && !Double.IsInfinity(accord.R2Accord)) { normalParams.R2 = decimal.Round(Convert.ToDecimal(accord.R2Accord), 9, MidpointRounding.AwayFromZero); } if (accord.FTestFailed) { Log.Warning($"AccID/UtilID/UnitID: {normalParams.AccID}/{normalParams.UtilID}/{normalParams.UnitID}: " + $"Best Regression Model's F-Test failed and model was not rejected. R2 = {normalParams.R2}"); } success = UpdateOrInsertWthNormalParams(normalParams); if (success && normalParamsExists) { newNormalParamsList.Add(normalParams); updateCount++; } else if (success && !normalParamsExists) { newNormalParamsList.Add(normalParams); insertCount++; } else if (!success) { modelsWithOtherIssues.Add(normalParams); failCount++; } } catch (BadWNRdngDataException bdEx) { failCount++; modelsWithReadingOrWeatherIssues.Add(normalParams); Log.Error(bdEx.Message + " " + bdEx.StackTrace); } catch (Exception e) { failCount++; modelsWithOtherIssues.Add(normalParams); Log.Error($"AccID/UtilID/UnitID: {normalParams.AccID}/{normalParams.UtilID}/{normalParams.UnitID} >> {e.Message} {e.StackTrace}"); } } foreach (WthNormalParams normalParams in modelsWithNotTwelveReadings) { Log.Warning($"AccID/UtilID/UnitID: {normalParams.AccID}/{normalParams.UtilID}/{normalParams.UnitID} >> " + $"Model was inserted but did not have (a multiple of) 12 readings from SP. "); } //foreach(WthNormalParams normalParams in modelsWithReadingOrWeatherIssues) //{ // Log.Warning($"AccID/UtilID/UnitID: {normalParams.AccID}/{normalParams.UtilID}/{normalParams.UnitID} >> No model found. " + // $"Readings had Bad/Null data from SP."); //} foreach (WthNormalParams normalParams in modelsWithOtherIssues) { Log.Error($"AccID/UtilID/UnitID: {normalParams.AccID}/{normalParams.UtilID}/{normalParams.UnitID} >> No model found. See daily log for more detail."); } if (failCount == 0) { Log.Information($"Finished PopulateWthNormalParams(). Rows Updated: {updateCount}. Rows Inserted: {insertCount}. Failures: {failCount}"); } else { Log.Warning($"Finished PopulateWthNormalParams() with failures. Rows Updated: {updateCount}. Rows Inserted: {insertCount}. Failures: {failCount}"); } //UpdateWthExpUsage(newNormalParamsList); }
private AccordResult CalculateLinearRegression(List <BalancePointPair> allBalancePointPairs, WthNormalParams normalParamsKey) { var allBalancePointGroups = allBalancePointPairs.GroupBy(s => new { s.CoolingBalancePoint, s.HeatingBalancePoint }); List <AccordResult> accordResults = new List <AccordResult>(); foreach (var group in allBalancePointGroups) { try { List <BalancePointPair> IdenticalBalancePointPairsFromAllReadings = group.ToList(); BalancePointPair _pointPair = IdenticalBalancePointPairsFromAllReadings.First(); int readingsCount = IdenticalBalancePointPairsFromAllReadings.Count; double[] fullYData = new double[readingsCount]; double[] fullYDataDailyAvg = new double[readingsCount]; double[][] hcddMatrix = new double[readingsCount][]; double[][] hcddMatrixNonDaily = new double[readingsCount][]; foreach (BalancePointPair balancePointPair in IdenticalBalancePointPairsFromAllReadings) { fullYData[IdenticalBalancePointPairsFromAllReadings.IndexOf(balancePointPair)] = (balancePointPair.ActualUsage); fullYDataDailyAvg[IdenticalBalancePointPairsFromAllReadings.IndexOf(balancePointPair)] = (balancePointPair.ActualUsage / balancePointPair.DaysInReading); hcddMatrix[IdenticalBalancePointPairsFromAllReadings.IndexOf(balancePointPair)] = new double[] { (balancePointPair.HeatingDegreeDays / balancePointPair.DaysInReading), (balancePointPair.CoolingDegreeDays / balancePointPair.DaysInReading) }; } if (!(fullYData.Sum() > 0)) { return(new AccordResult()); } double[] avgHddsForEachReadingInYear = new double[readingsCount]; double[] avgCddsForEachReadingInYear = new double[readingsCount]; for (int i = 0; i < readingsCount; i++) { avgHddsForEachReadingInYear[i] = hcddMatrix[i][0]; avgCddsForEachReadingInYear[i] = hcddMatrix[i][1]; } double[] modelParams = new double[3]; modelParams[0] = 0; modelParams[1] = 0; modelParams[2] = 0; if (_pointPair.HeatingBalancePoint == 0 && _pointPair.CoolingBalancePoint == 0) { double[] onesVector = new double[readingsCount]; for (int i = 0; i < readingsCount; i++) { onesVector[i] = 1; } modelParams[0] = Fit.LineThroughOrigin(onesVector, fullYDataDailyAvg); OrdinaryLeastSquares ols = new OrdinaryLeastSquares() { UseIntercept = false }; double r2 = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination( onesVector.Select(x => x * modelParams[0]), fullYDataDailyAvg); AccordResult accordResult = new AccordResult() { IsSimpleSingleRegression = true, HeatingBP = _pointPair.HeatingBalancePoint, CoolingBP = _pointPair.CoolingBalancePoint, Intercept = modelParams[0], R2Accord = r2, //R2Accord = 0 }; accordResults.Add(accordResult); } else if (_pointPair.CoolingBalancePoint != 0 && _pointPair.HeatingBalancePoint != 0) { try { MultipleLinearRegressionAnalysis mlra = new MultipleLinearRegressionAnalysis(intercept: true); mlra.Learn(hcddMatrix, fullYDataDailyAvg); var regressionAccord = mlra.Regression; double[] predictedAccord = regressionAccord.Transform(hcddMatrix); double r2Accord = new RSquaredLoss(numberOfInputs: 2, expected: fullYDataDailyAvg) { Adjust = false }.Loss(predictedAccord); double r2Coeff = regressionAccord.CoefficientOfDetermination(hcddMatrix, fullYDataDailyAvg, adjust: false); bool FTestFailed = !mlra.FTest.Significant; AccordResult accordResult = new AccordResult() { IsMultipleLinearRegression = true, HeatingBP = _pointPair.HeatingBalancePoint, CoolingBP = _pointPair.CoolingBalancePoint, Intercept = regressionAccord.Intercept, B2 = regressionAccord.Weights[0], B4 = regressionAccord.Weights[1], R2Accord = r2Accord, FTestFailed = FTestFailed }; //int degreesOfFreedom = normalParamsKey.MoCt - 3; double degreesOfFreedomAsDouble = mlra.Regression.GetDegreesOfFreedom(readingsCount); int degreesOfFreedom = Convert.ToInt32(degreesOfFreedomAsDouble); //if (degreesOfFreedom != 9) //{ // Log.Warning($"Multivariable regression. DOF expected to be 9. is: {degreesOfFreedom}"); //} //if (degreesOfFreedom != dof) //{ // Console.WriteLine($"dof different. mlra.dof = {dof} expected = {degreesOfFreedom}"); //} double s = Math.Sqrt(fullYDataDailyAvg.Subtract(predictedAccord).Pow(2).Sum() / degreesOfFreedom); double ssxHdd = Math.Sqrt((avgHddsForEachReadingInYear.Subtract(avgHddsForEachReadingInYear.Mean())).Pow(2).Sum()); double ssxCdd = Math.Sqrt((avgCddsForEachReadingInYear.Subtract(avgCddsForEachReadingInYear.Mean())).Pow(2).Sum()); double seSubHdd = s / ssxHdd; double seSubCdd = s / ssxCdd; double tStatisticHdd = regressionAccord.Weights[0] / seSubHdd; double tStatisticCdd = regressionAccord.Weights[1] / seSubCdd; double tCriticalFivePercent = 2.262156; double tCriticalTenPercent = 1.833113; bool myTestHdd = Math.Abs(tStatisticHdd) >= tCriticalTenPercent; bool myTestCdd = Math.Abs(tStatisticCdd) >= tCriticalTenPercent; //if (myTestHdd != mlra.Coefficients[0].TTest.Significant && degreesOfFreedom != 9) //{ // Console.WriteLine($"nope. mystat - {tStatisticHdd} accordstat - {mlra.Coefficients[0].TTest.Statistic} " + // $"accordCritical - {mlra.Coefficients[0].TTest.CriticalValue}"); //} //if (myTestCdd != mlra.Coefficients[1].TTest.Significant && degreesOfFreedom != 9) //{ // Console.WriteLine($"nope. mystat - {tStatisticCdd} accordstat - {mlra.Coefficients[1].TTest.Statistic} " + // $"accordCritical - {mlra.Coefficients[1].TTest.CriticalValue}"); //} //if (mlra.Coefficients.All(x => x.TTest.Significant) && // mlra.Coefficients.All(x => x.Value > 0) && // mlra.Regression.Intercept > 0 && // r2Accord >= 0.7500) //{ // accordResults.Add(accordResult); //} if ( myTestHdd && myTestCdd && mlra.Coefficients.All(x => x.Value > 0) && mlra.Regression.Intercept > 0 //&& accordResult.R2Accord >= 0.75 ) { accordResults.Add(accordResult); } } catch (Exception e) { Log.Debug($"AccID/UtilID/UnitID: {normalParamsKey.AccID}/{normalParamsKey.UtilID}/{normalParamsKey.UnitID} >> " + $"MultipleLinearRegressionAnalysis Exception: {e.Message}"); } } else if (_pointPair.HeatingBalancePoint > 0) { OrdinaryLeastSquares ols = new OrdinaryLeastSquares() { UseIntercept = true }; SimpleLinearRegression regressionAccord = ols.Learn(avgHddsForEachReadingInYear, fullYDataDailyAvg); double[] predictedAccord = regressionAccord.Transform(avgHddsForEachReadingInYear); double r2Accord = new RSquaredLoss(1, fullYDataDailyAvg).Loss(predictedAccord); //int degreesOfFreedom = normalParamsKey.MoCt - 2; double degreesOfFreedomAsDouble = regressionAccord.GetDegreesOfFreedom(readingsCount); int degreesOfFreedom = Convert.ToInt32(degreesOfFreedomAsDouble); //if (degreesOfFreedom != 10) //{ // Log.Warning($"Single variable regression. DOF expected to be 10. is: {degreesOfFreedom}"); //} double ssx = Math.Sqrt((avgHddsForEachReadingInYear.Subtract(avgHddsForEachReadingInYear.Mean())).Pow(2).Sum()); double s = Math.Sqrt(fullYDataDailyAvg.Subtract(predictedAccord).Pow(2).Sum() / degreesOfFreedom); double error = regressionAccord.GetStandardError(avgHddsForEachReadingInYear, fullYDataDailyAvg); double seSubB = s / ssx; double hypothesizedValue = 0; double tStatistic = regressionAccord.Slope / seSubB; double tCriticalFivePercent = 2.228138; double tCriticalTenPercent = 1.812461; bool myTest = Math.Abs(tStatistic) >= tCriticalTenPercent; //TTest tTest = new TTest( // estimatedValue: regressionAccord.Slope, standardError: seSubB, degreesOfFreedom: degreesOfFreedom, // hypothesizedValue: hypothesizedValue, alternate: OneSampleHypothesis.ValueIsDifferentFromHypothesis // ); //if (myTest != tTest.Significant) //{ // Console.WriteLine($"nope. mystat - {tStatistic} accordstat - {tTest.Statistic} accordCritical - {tTest.CriticalValue}"); //} AccordResult accordResult = new AccordResult() { IsSimpleSingleRegression = true, HeatingBP = _pointPair.HeatingBalancePoint, Intercept = regressionAccord.Intercept, B2 = regressionAccord.Slope, R2Accord = r2Accord }; //if (tTest.Significant && accordResult.B2 > 0 && r2Accord >= 0.7500) //{ // accordResults.Add(accordResult); //} if (myTest && accordResult.B2 > 0 && accordResult.Intercept > 0 //&& r2Accord >= 0.7500 ) { accordResults.Add(accordResult); } } else if (_pointPair.CoolingBalancePoint > 0) { OrdinaryLeastSquares ols = new OrdinaryLeastSquares() { UseIntercept = true }; SimpleLinearRegression regressionAccord = ols.Learn(avgCddsForEachReadingInYear, fullYDataDailyAvg); double[] predictedAccord = regressionAccord.Transform(avgCddsForEachReadingInYear); double r2Accord = new RSquaredLoss(1, fullYDataDailyAvg).Loss(predictedAccord); //int degreesOfFreedom = normalParamsKey.MoCt - 2; double degreesOfFreedomAsDouble = regressionAccord.GetDegreesOfFreedom(readingsCount); int degreesOfFreedom = Convert.ToInt32(degreesOfFreedomAsDouble); //if (degreesOfFreedom != 10) //{ // Log.Warning($"Single variable regression. DOF expected to be 10. is: {degreesOfFreedom}"); //} double ssx = Math.Sqrt(avgCddsForEachReadingInYear.Subtract(avgCddsForEachReadingInYear.Mean()).Pow(2).Sum()); double s = Math.Sqrt(fullYDataDailyAvg.Subtract(predictedAccord).Pow(2).Sum() / degreesOfFreedom); double seSubB = s / ssx; double hypothesizedValue = 0; double tStatistic = regressionAccord.Slope / seSubB; double tCriticalFivePercent = 2.22813885198627; double tCriticalTenPercent = 1.812461; bool myTest = Math.Abs(tStatistic) >= tCriticalTenPercent; //TTest tTest = new TTest( // estimatedValue: regressionAccord.Slope, standardError: seSubB, degreesOfFreedom: degreesOfFreedom, // hypothesizedValue: hypothesizedValue, alternate: OneSampleHypothesis.ValueIsDifferentFromHypothesis // ); //if (myTest != tTest.Significant) //{ // Console.WriteLine($"nope. mystat - {tStatistic} accordstat - {tTest.Statistic} accordCritical - {tTest.CriticalValue}"); //} AccordResult accordResult = new AccordResult() { IsSimpleSingleRegression = true, CoolingBP = _pointPair.CoolingBalancePoint, Intercept = regressionAccord.Intercept, B4 = regressionAccord.Slope, R2Accord = r2Accord }; //if (tTest.Significant && accordResult.B4 > 0 && r2Accord >= 0.7500) //{ // accordResults.Add(accordResult); //} if ( myTest && accordResult.B4 > 0 //&& r2Accord >= 0.7500 ) { accordResults.Add(accordResult); } } } catch (Exception e) { Log.Debug($"AccID/UtilID/UnitID: {normalParamsKey.AccID}/{normalParamsKey.UtilID}/{normalParamsKey.UnitID} >> {e.Message} {e.StackTrace}"); } } AccordResult accordWinner = accordResults .Where(s => s.Intercept >= 0) .OrderByDescending(s => s.R2Accord).ToList().FirstOrDefault(); return(accordWinner); }