public List <ReadingsQueryResult> GetReadingsFromExpUsageOriginalCorrected(AccordResult accord)
{
List <WthNormalParams> myParams = new List <WthNormalParams>();
List <ReadingsQueryResult> allReadings = new List <ReadingsQueryResult>();
string Sql = @"select wnp.AccID, wnp.UtilID, wnp.UnitID, weu.ExpUsage, weu.RdngID from WthNormalParams wnp join WthExpUsage weu on wnp.AccID = weu.AccID and
wnp.UtilID = weu.UtilID and wnp.UnitID = weu.UnitID where wnp.R2_New > 0.8 and wnp.AccID = @AccID and wnp.UtilID = @UtilID and wnp.UnitID = @UnitID";
//where wnp.R2_New > 0.8;";
using (IDbConnection db = new SqlConnection(_myConnectionString))
{
myParams = db.Query <WthNormalParams>(Sql, new { accord.AccID, accord.UtilID, accord.UnitID }).AsList();
}
var myParamsGroups = myParams.GroupBy(p => new { p.AccID, p.UtilID, p.UnitID });
foreach (var group in myParamsGroups)
{
List <WthNormalParams> wthNormalParams = group.ToList();
foreach (WthNormalParams wnp in wthNormalParams)
{
ReadingsQueryResult reading = new ReadingsQueryResult();
string Sql2 = @"select RdngID, AccID, UtilID, UnitID as RUnitID, ExpUsage, Units, DateStart, DateEnd, Days from Readings
where RdngID = @RdngID;";
using (IDbConnection db = new SqlConnection(_myConnectionString))
{
reading = db.Query <ReadingsQueryResult>(Sql2, new { wnp.RdngID }).First();
}
allReadings.Add(reading);
//string Sql2 = @"select AccID, UtilID, UnitID as RUnitID, RdngID, Units, DateStart, DateEnd, Days from Readings
// where RdngID = @RdngID;";
//try
//{
// using (IDbConnection db = new SqlConnection(_jitWebData3ConnectionString))
// {
// reading = db.Query<ReadingsQueryResult>(Sql2, new { wnp.RdngID }).First();
// }
// reading.ExpUsage = wnp.ExpUsage;
// allReadings.Add(reading);
// try
// {
// InsertMyReadings(reading);
// }
// catch (Exception e) { Console.WriteLine(e.Message + " " + e.StackTrace); }
//} catch (Exception e)
// {
// Console.WriteLine(e.Message + " " + wnp.RdngID);
// }
}
}
return(allReadings);
}
private void PopulateWthNormalParams()
{
List <WthNormalParams> normalParamsKeys = _weatherRepository.GetNormalParamsKeysForRegression();
foreach (WthNormalParams normalParamsKey in normalParamsKeys)
{
try
{
int yearAsInt = normalParamsKey.EndDate_Original.Year - 2000;
string desiredStartDate = yearAsInt.ToString();
List <BalancePointPair> allBalancePointStatsFromYear = CalculateOneYearOfDegreeDaysForAllBalancePoints(desiredStartDate, normalParamsKey).ToList();
if (allBalancePointStatsFromYear.Count == 0)
{
_weatherRepository.InsertWthNormalParams(normalParamsKey, Accord: true);
//_weatherRepository.InsertWthNormalParamsFinal(normalParamsKey);
continue;
}
List <AccordResult> accords = CalculateLinearRegression(allBalancePointStatsFromYear, normalParamsKey);
accords = accords
.Where(s => s.Intercept >= 0)
.OrderByDescending(s => s.R2Accord).ToList();
AccordResult bestAccord = accords.FirstOrDefault();
if (bestAccord.IsMultipleLinearRegression)
{
if (bestAccord.MLRA.FTest.Significant == false)
{
normalParamsKey.FTestFailed = 1;
Console.WriteLine("F Test failed... " + normalParamsKey.AccID + " " + normalParamsKey.UtilID + " " + normalParamsKey.UnitID);
}
}
normalParamsKey.B1_New = decimal.Round(Convert.ToDecimal(bestAccord.Intercept), 9, MidpointRounding.AwayFromZero);
normalParamsKey.B2_New = 0;
normalParamsKey.B3_New = 0;
normalParamsKey.B4_New = 0;
normalParamsKey.B5_New = 0;
if (bestAccord.IsSimpleSingleRegression == true && bestAccord.HeatingBP > 0)
{
normalParamsKey.B2_New = decimal.Round(Convert.ToDecimal(bestAccord.SimpleLinearRegression.Slope), 9, MidpointRounding.AwayFromZero);
normalParamsKey.B3_New = bestAccord.HeatingBP;
}
else if (bestAccord.IsSimpleSingleRegression == true && bestAccord.CoolingBP > 0)
{
normalParamsKey.B4_New = decimal.Round(Convert.ToDecimal(bestAccord.SimpleLinearRegression.Slope), 9, MidpointRounding.AwayFromZero);
normalParamsKey.B5_New = bestAccord.CoolingBP;
}
else if (bestAccord.IsMultipleLinearRegression == true)
{
normalParamsKey.B2_New = decimal.Round(Convert.ToDecimal(bestAccord.MLRA.Regression.Weights[0]), 9, MidpointRounding.AwayFromZero);
normalParamsKey.B3_New = bestAccord.HeatingBP;
normalParamsKey.B4_New = decimal.Round(Convert.ToDecimal(bestAccord.MLRA.Regression.Weights[1]), 9, MidpointRounding.AwayFromZero);
normalParamsKey.B5_New = bestAccord.CoolingBP;
}
if (AreDoublesAllNotInfinityNorNaN(new double[] { bestAccord.R2Accord }))
{
normalParamsKey.R2_New = decimal.Round(Convert.ToDecimal(bestAccord.R2Accord), 9, MidpointRounding.AwayFromZero);
}
else
{
normalParamsKey.R2_New = 0;
}
normalParamsKey.YearOfReadsDateStart = bestAccord.bpPair.YearOfReadsDateStart;
normalParamsKey.YearOfReadsDateEnd = bestAccord.bpPair.YearOfReadsDateEnd;
normalParamsKey.Readings = bestAccord.bpPair.ReadingsInNormalYear;
normalParamsKey.Days = bestAccord.bpPair.DaysInNormalYear;
normalParamsKey.WthZipCode = bestAccord.bpPair.WthZipCode;
_weatherRepository.InsertWthNormalParams(normalParamsKey, Accord: true);
//_weatherRepository.InsertWthNormalParamsFinal(normalParamsKey);
}
catch (Exception e)
{
Console.WriteLine(normalParamsKey.AccID + " " + normalParamsKey.EndDate_Original);
Console.WriteLine(e.Message);
Console.WriteLine(e.StackTrace);
}
}
Console.WriteLine("PopulateWthNormalParams Finished.");
}
private void PopulateMyWthExpUsage(AccordResult accord)
{
//List<ReadingsQueryResult> allReadings = _weatherRepository.GetReadingsFromExpUsageOriginal();
List <ReadingsQueryResult> allReadings = _weatherRepository.GetReadingsFromExpUsageOriginalCorrected(accord);
var readingsByAccount = allReadings.GroupBy(r => new { r.AccID, r.UtilID, r.RUnitID });
foreach (var group in readingsByAccount)
{
try
{
List <ReadingsQueryResult> readings = group.ToList();
WthNormalParams _params =
_weatherRepository.GetParamsForReading(readings.First().AccID, readings.First().UtilID, readings.First().RUnitID);
foreach (ReadingsQueryResult reading in readings)
{
WthExpUsage wthExpUsage = new WthExpUsage();
wthExpUsage.RdngID = reading.RdngID;
wthExpUsage.AccID = reading.AccID;
wthExpUsage.UtilID = reading.UtilID;
wthExpUsage.UnitID = reading.RUnitID;
decimal expUsage_Old = decimal.Round(Convert.ToDecimal(reading.ExpUsage), 4, MidpointRounding.AwayFromZero);
wthExpUsage.ExpUsage_Old = expUsage_Old;
wthExpUsage.Units = reading.Units;
List <WeatherData> weatherDataList =
_weatherRepository.GetWeatherDataByZipStartAndEndDate(_params.WthZipCode, reading.DateStart, reading.DateEnd);
decimal B1 = 0;
decimal B2 = 0;
decimal B4 = 0;
reading.B3 = accord.HeatingBP;
reading.B5 = accord.CoolingBP;
if (accord.IsSimpleSingleRegression)
{
if (reading.B3 > 0)
{
B1 = decimal.Round(Convert.ToDecimal(accord.SimpleLinearRegression.Intercept), 12, MidpointRounding.AwayFromZero);
B2 = decimal.Round(Convert.ToDecimal(accord.SimpleLinearRegression.Slope), 12, MidpointRounding.AwayFromZero);
B4 = 0;
}
else if (reading.B5 > 0)
{
B1 = decimal.Round(Convert.ToDecimal(accord.SimpleLinearRegression.Intercept), 12, MidpointRounding.AwayFromZero);
B2 = 0;
B4 = decimal.Round(Convert.ToDecimal(accord.SimpleLinearRegression.Slope), 12, MidpointRounding.AwayFromZero);
}
}
else
{
if (reading.B3 > 0 && reading.B5 > 0)
{
B1 = decimal.Round(Convert.ToDecimal(accord.MultipleRegression.Weights[0]), 12, MidpointRounding.AwayFromZero);
B2 = decimal.Round(Convert.ToDecimal(accord.MultipleRegression.Weights[1]), 12, MidpointRounding.AwayFromZero);
B4 = decimal.Round(Convert.ToDecimal(accord.MultipleRegression.Weights[2]), 12, MidpointRounding.AwayFromZero);
}
else if (reading.B3 > 0)
{
B1 = decimal.Round(Convert.ToDecimal(accord.MultipleRegression.Weights[0]), 12, MidpointRounding.AwayFromZero);
B2 = decimal.Round(Convert.ToDecimal(accord.MultipleRegression.Weights[1]), 12, MidpointRounding.AwayFromZero);
}
else if (reading.B5 > 0)
{
B1 = decimal.Round(Convert.ToDecimal(accord.MultipleRegression.Weights[0]), 12, MidpointRounding.AwayFromZero);
B4 = decimal.Round(Convert.ToDecimal(accord.MultipleRegression.Weights[1]), 12, MidpointRounding.AwayFromZero);
}
}
HeatingCoolingDegreeDays hcdd = HeatingCoolingDegreeDaysValueOf(reading, weatherDataList);
decimal expUsage_New = (B1 * reading.Days)
+ (B2 * decimal.Round(Convert.ToDecimal(hcdd.HDD), 12, MidpointRounding.AwayFromZero))
+ (B4 * decimal.Round(Convert.ToDecimal(hcdd.CDD), 12, MidpointRounding.AwayFromZero));
wthExpUsage.ExpUsage_New = expUsage_New;
if (reading.Units != 0)
{
wthExpUsage.PercentDelta_Old = Math.Abs((expUsage_Old - reading.Units) / reading.Units);
wthExpUsage.PercentDelta_New = Math.Abs((expUsage_New - reading.Units) / reading.Units);
}
wthExpUsage.DateStart = reading.DateStart;
wthExpUsage.DateEnd = reading.DateEnd;
_weatherRepository.InsertMyWthExpUsage(wthExpUsage, Accord: true);
}
}
catch (Exception e)
{
Console.WriteLine(e.Message + " " + group.ToList().First().RdngID + " " + group.ToList().First().AccID + " " + group.ToList().First().RUnitID
+ "\n" + e.StackTrace);
}
}
}
private List <AccordResult> CalculateLinearRegression(List <BalancePointPair> allBalancePointPairs, WthNormalParams normalParamsKey)
{
var allBalancePointGroups = allBalancePointPairs.GroupBy(s => new { s.CoolingBalancePoint, s.HeatingBalancePoint });
List <AccordResult> accordResults = new List <AccordResult>();
List <AccordResult> rejectedAccords = new List <AccordResult>();
foreach (var group in allBalancePointGroups)
{
try
{
List <BalancePointPair> IdenticalBalancePointPairsForAllReadings = group.ToList();
BalancePointPair _pointPair = IdenticalBalancePointPairsForAllReadings.First();
int readingsCount = IdenticalBalancePointPairsForAllReadings.Count;
double[] fullYData = new double[readingsCount];
double[] fullYDataDailyAvg = new double[readingsCount];
double[][] hcddMatrix = new double[readingsCount][];
double[][] hcddMatrixNonDaily = new double[readingsCount][];
foreach (BalancePointPair balancePointPair in IdenticalBalancePointPairsForAllReadings)
{
fullYData[IdenticalBalancePointPairsForAllReadings.IndexOf(balancePointPair)] = (balancePointPair.ActualUsage);
fullYDataDailyAvg[IdenticalBalancePointPairsForAllReadings.IndexOf(balancePointPair)]
= (balancePointPair.ActualUsage / balancePointPair.DaysInReading);
hcddMatrix[IdenticalBalancePointPairsForAllReadings.IndexOf(balancePointPair)] = new double[]
{
(balancePointPair.HeatingDegreeDays / balancePointPair.DaysInReading),
(balancePointPair.CoolingDegreeDays / balancePointPair.DaysInReading)
};
}
double[] avgHddsForEachReadingInYear = new double[readingsCount];
double[] avgCddsForEachReadingInYear = new double[readingsCount];
for (int i = 0; i < readingsCount; i++)
{
avgHddsForEachReadingInYear[i] = hcddMatrix[i][0];
avgCddsForEachReadingInYear[i] = hcddMatrix[i][1];
}
double[] modelParams = new double[3];
modelParams[0] = 0;
modelParams[1] = 0;
modelParams[2] = 0;
if (fullYData.Sum() == 0)
{
AccordResult empty = new AccordResult
{
bpPair = _pointPair
};
accordResults.Add(empty);
}
else if (_pointPair.HeatingBalancePoint == 0 && _pointPair.CoolingBalancePoint == 0)
{
double[] onesVector = new double[readingsCount];
for (int i = 0; i < readingsCount; i++)
{
onesVector[i] = 1;
}
modelParams[0] = Fit.LineThroughOrigin(onesVector, fullYDataDailyAvg);
OrdinaryLeastSquares ols = new OrdinaryLeastSquares()
{
UseIntercept = false
};
SimpleLinearRegression regressionAccord = ols.Learn(onesVector, fullYDataDailyAvg);
//double[] predictedAccord = regressionAccord.Transform(onesVector);
double r2 = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination(onesVector.Select(x => x * modelParams[0]), fullYDataDailyAvg);
//double mean = fullYDataDailyAvg.Mean();
//if (mean != modelParams[0] || mean != regressionAccord.Slope)
//{
// Console.WriteLine("Hey!");
//}
//double r2Accord = regressionAccord.CoefficientOfDetermination(onesVector, fullYDataDailyAvg);
//double sxx = onesVector.Subtract(onesVector.Mean()).Pow(2).Sum();
//double hypothesizedValue = 0;
//try
//{
// TTest test = new TTest(
// estimatedValue: regressionAccord.Slope, standardError: sxx, degreesOfFreedom: _pointPair.ReadingsInNormalYear - 2,
// hypothesizedValue: hypothesizedValue, alternate: OneSampleHypothesis.ValueIsDifferentFromHypothesis
// );
// if (test.Significant)
// {
AccordResult accordResult = new AccordResult()
{
SimpleLinearRegression = regressionAccord,
R2Accord = r2,
IsSimpleSingleRegression = true,
HeatingBP = _pointPair.HeatingBalancePoint,
CoolingBP = _pointPair.CoolingBalancePoint,
Intercept = regressionAccord.Slope,
bpPair = _pointPair
};
accordResults.Add(accordResult);
// }
//}
//catch (Exception e)
//{
// Console.WriteLine(e.Message + e.StackTrace);
//}
}
else if (_pointPair.CoolingBalancePoint != 0 && _pointPair.HeatingBalancePoint != 0)
{
//modelParams = MultipleRegression.QR(hcddMatrix, fullYDataDailyAvg, intercept: true);
//Accord
//var ols = new OrdinaryLeastSquares()
//{
// UseIntercept = true
//};
try
{
MultipleLinearRegressionAnalysis mlra = new MultipleLinearRegressionAnalysis(intercept: true);
mlra.Learn(hcddMatrix, fullYDataDailyAvg);
//
//MultipleLinearRegression regressionAccord = ols.Learn(hcddMatrix, fullYDataDailyAvg);
var regressionAccord = mlra.Regression;
double[] predicted = regressionAccord.Transform(hcddMatrix);
double r2Accord = new RSquaredLoss(numberOfInputs: 2, expected: fullYDataDailyAvg)
{
Adjust = false
}.Loss(predicted);
double r2Coeff = regressionAccord.CoefficientOfDetermination(hcddMatrix, fullYDataDailyAvg, adjust: false);
//double r2Math = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination(hcddMatrix.Select(
// x => (x[0] * regressionAccord.Weights[0]) + (x[1] * regressionAccord.Weights[1]) + regressionAccord.Intercept
//), fullYDataDailyAvg);
//double r2MathPred = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination(predicted, fullYDataDailyAvg);
AccordResult accordResult = new AccordResult()
{
//MultipleRegression = regressionAccord,
R2Accord = r2Accord,
R2Coeff = r2Coeff,
HeatingBP = _pointPair.HeatingBalancePoint,
CoolingBP = _pointPair.CoolingBalancePoint,
IsSimpleSingleRegression = false,
MLRA = mlra,
Intercept = regressionAccord.Intercept,
bpPair = _pointPair,
IsMultipleLinearRegression = true
};
if (mlra.Coefficients.All(x => x.TTest.Significant))
{
accordResults.Add(accordResult);
}
else
{
rejectedAccords.Add(accordResult);
}
}
catch (Exception e)
{
Console.WriteLine(e.Message + " " + e.StackTrace);
}
}
else if (_pointPair.HeatingBalancePoint > 0)
{
// Tuple<double, double> heatingTuple = Fit.Line(avgHddsForEachReadingInYear, fullYDataDailyAvg);
// modelParams[0] = heatingTuple.Item1;
// modelParams[1] = heatingTuple.Item2;
// double r = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination(
// avgHddsForEachReadingInYear.Select(x => heatingTuple.Item1 + heatingTuple.Item2 * x), fullYDataDailyAvg);
OrdinaryLeastSquares ols = new OrdinaryLeastSquares()
{
UseIntercept = true
};
SimpleLinearRegression regressionAccord = ols.Learn(avgHddsForEachReadingInYear, fullYDataDailyAvg);
double[] predictedAccord = regressionAccord.Transform(avgHddsForEachReadingInYear);
double rAccord = new RSquaredLoss(1, fullYDataDailyAvg).Loss(predictedAccord);
//double rAccord2 = regressionAccord.CoefficientOfDetermination(avgHddsForEachReadingInYear, fullYDataDailyAvg, adjust: false);
//double r2Math = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination(avgHddsForEachReadingInYear.Select(
// x => (x * regressionAccord.Slope) + regressionAccord.Intercept
// ), fullYDataDailyAvg);
//double r2 = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination(predictedAccord, fullYDataDailyAvg);
int degreesOfFreedom = _pointPair.ReadingsInNormalYear - 2;
double ssx = Math.Sqrt((avgHddsForEachReadingInYear.Subtract(avgHddsForEachReadingInYear.Mean())).Pow(2).Sum());
double s = Math.Sqrt(((fullYDataDailyAvg.Subtract(predictedAccord).Pow(2)).Sum()) / degreesOfFreedom);
double error = regressionAccord.GetStandardError(avgHddsForEachReadingInYear, fullYDataDailyAvg);
double seSubB = s / ssx;
double hypothesizedValue = 0;
TTest tTest = new TTest(
estimatedValue: regressionAccord.Slope, standardError: seSubB, degreesOfFreedom: degreesOfFreedom,
hypothesizedValue: hypothesizedValue, alternate: OneSampleHypothesis.ValueIsDifferentFromHypothesis
);
AccordResult accordResult = new AccordResult()
{
SimpleLinearRegression = regressionAccord,
R2Accord = rAccord,
IsSimpleSingleRegression = true,
HeatingBP = _pointPair.HeatingBalancePoint,
CoolingBP = _pointPair.CoolingBalancePoint,
TTest = tTest,
Intercept = regressionAccord.Intercept,
bpPair = _pointPair
};
if (tTest.Significant)
{
accordResults.Add(accordResult);
}
else
{
rejectedAccords.Add(accordResult);
}
}
else if (_pointPair.CoolingBalancePoint > 0)
{
//Tuple<double, double> coolingTuple = Fit.Line(avgCddsForEachReadingInYear, fullYDataDailyAvg);
//modelParams[0] = coolingTuple.Item1;
//modelParams[2] = coolingTuple.Item2;
OrdinaryLeastSquares ols = new OrdinaryLeastSquares()
{
UseIntercept = true
};
SimpleLinearRegression regressionAccord = ols.Learn(avgCddsForEachReadingInYear, fullYDataDailyAvg);
double[] predictedAccord = regressionAccord.Transform(avgCddsForEachReadingInYear);
double rAccord = new RSquaredLoss(1, fullYDataDailyAvg).Loss(predictedAccord);
//double r2Math = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination(avgCddsForEachReadingInYear.Select(
// x => (x * regressionAccord.Slope) + regressionAccord.Intercept
// ), fullYDataDailyAvg);
//double r2 = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination(predictedAccord, fullYDataDailyAvg);
int degreesOfFreedom = _pointPair.ReadingsInNormalYear - 2;
double ssx = Math.Sqrt(avgCddsForEachReadingInYear.Subtract(avgCddsForEachReadingInYear.Mean()).Pow(2).Sum());
double s = Math.Sqrt(((fullYDataDailyAvg.Subtract(predictedAccord).Pow(2)).Sum()) / degreesOfFreedom);
double seSubB = s / ssx;
double hypothesizedValue = 0;
double myT = seSubB / regressionAccord.Slope;
TTest tTest = new TTest(
estimatedValue: regressionAccord.Slope, standardError: seSubB, degreesOfFreedom: degreesOfFreedom,
hypothesizedValue: hypothesizedValue, alternate: OneSampleHypothesis.ValueIsDifferentFromHypothesis
);
AccordResult accordResult = new AccordResult()
{
SimpleLinearRegression = regressionAccord,
R2Accord = rAccord,
IsSimpleSingleRegression = true,
HeatingBP = _pointPair.HeatingBalancePoint,
CoolingBP = _pointPair.CoolingBalancePoint,
TTest = tTest,
Intercept = regressionAccord.Intercept,
bpPair = _pointPair
};
if (tTest.Significant)
{
accordResults.Add(accordResult);
}
else
{
rejectedAccords.Add(accordResult);
}
}
;
}
catch (Exception e)
{
Console.WriteLine(normalParamsKey.AccID + " " + normalParamsKey.UtilID + " " + normalParamsKey.UnitID + " " + e.Message + e.StackTrace);
}
}
//rejectedAccords = rejectedAccords.OrderByDescending(s => s.R2Accord).ToList();
//accordResults = accordResults.OrderByDescending(s => s.R2Accord).ToList();
return(accordResults);
}
private void PopulateWthNormalParams()
{
List <WNRdngData> allWNRdngData = _weatherRepository.GetAllReadingsFromStoredProcedure();
var wNRdngDataGroups = allWNRdngData.GroupBy(s => new { s.AccID, s.UtilID, s.UnitID });
List <WthNormalParams> newNormalParamsList = new List <WthNormalParams>();
int updateCount = 0;
int insertCount = 0;
foreach (var wNRdngGroup in wNRdngDataGroups)
{
List <WNRdngData> wNRdngList = wNRdngGroup.OrderBy(s => s.MoID).ToList();
WNRdngData lastRead = wNRdngList.LastOrDefault();
WthNormalParams normalParams = new WthNormalParams()
{
AccID = lastRead.AccID,
UtilID = lastRead.UtilID,
UnitID = lastRead.UnitID,
WstID = lastRead.WstID,
ZipW = lastRead.Zip,
EndDate = lastRead.DateEnd,
EMoID = lastRead.EMoID,
MoCt = lastRead.MoCt
};
bool normalParamsExists = _weatherRepository.GetWthNormalParamsExists(normalParams);
try
{
List <BalancePointPair> allBalancePointStatsFromYear = CalculateOneYearOfDegreeDaysForAllBalancePoints(wNRdngList);
int daysInYear = allBalancePointStatsFromYear.FirstOrDefault().DaysInYear;
normalParams.DaysInYear = daysInYear;
if (allBalancePointStatsFromYear.Count == 0)
{
if (normalParamsExists)
{
_weatherRepository.UpdateWthNormalParams(normalParams);
newNormalParamsList.Add(normalParams);
updateCount++;
Log.Debug($"Updated WthNormalParams >> AccID: {normalParams.AccID}. UtilID: {normalParams.UtilID} UnitID: {normalParams.UnitID} " +
$"B1: {normalParams.B1} B2: {normalParams.B2} B3: {normalParams.B3} B4: {normalParams.B4} B5: {normalParams.B5} R2: {normalParams.R2}.");
}
else
{
_weatherRepository.InsertWthNormalParams(normalParams);
newNormalParamsList.Add(normalParams);
insertCount++;
Log.Debug($"Inserted WthNormalParams >> AccID: {normalParams.AccID}. UtilID: {normalParams.UtilID} UnitID: {normalParams.UnitID} " +
$"B1: {normalParams.B1} B2: {normalParams.B2} B3: {normalParams.B3} B4: {normalParams.B4} B5: {normalParams.B5} R2: {normalParams.R2}.");
}
continue;
}
AccordResult accord = CalculateLinearRegression(allBalancePointStatsFromYear, normalParams);
if (accord.FTestFailed)
{
Log.Information("F Test failed... " + normalParams.AccID + " " + normalParams.UtilID + " " + normalParams.UnitID);
}
normalParams.B1 = decimal.Round(Convert.ToDecimal(accord.Intercept), 9, MidpointRounding.AwayFromZero);
if (accord.IsSimpleSingleRegression == true && accord.HeatingBP > 0)
{
normalParams.B2 = decimal.Round(Convert.ToDecimal(accord.B2), 9, MidpointRounding.AwayFromZero);
normalParams.B3 = accord.HeatingBP;
}
else if (accord.IsSimpleSingleRegression == true && accord.CoolingBP > 0)
{
normalParams.B4 = decimal.Round(Convert.ToDecimal(accord.B4), 9, MidpointRounding.AwayFromZero);
normalParams.B5 = accord.CoolingBP;
}
else if (accord.IsMultipleLinearRegression == true)
{
normalParams.B2 = decimal.Round(Convert.ToDecimal(accord.B2), 9, MidpointRounding.AwayFromZero);
normalParams.B3 = accord.HeatingBP;
normalParams.B4 = decimal.Round(Convert.ToDecimal(accord.B4), 9, MidpointRounding.AwayFromZero);
normalParams.B5 = accord.CoolingBP;
}
if (!Double.IsNaN(accord.R2Accord) && !Double.IsInfinity(accord.R2Accord))
{
normalParams.R2 = decimal.Round(Convert.ToDecimal(accord.R2Accord), 9, MidpointRounding.AwayFromZero);
}
if (normalParamsExists)
{
_weatherRepository.UpdateWthNormalParams(normalParams);
newNormalParamsList.Add(normalParams);
updateCount++;
Log.Debug($"Updated WthNormalParams >> AccID: {normalParams.AccID}. UtilID: {normalParams.UtilID} UnitID: {normalParams.UnitID} " +
$"B1: {normalParams.B1} B2: {normalParams.B2} B3: {normalParams.B3} B4: {normalParams.B4} B5: {normalParams.B5} R2: {normalParams.R2}.");
}
else
{
_weatherRepository.InsertWthNormalParams(normalParams);
newNormalParamsList.Add(normalParams);
insertCount++;
Log.Debug($"Inserted WthNormalParams >> AccID: {normalParams.AccID}. UtilID: {normalParams.UtilID} UnitID: {normalParams.UnitID} " +
$"B1: {normalParams.B1} B2: {normalParams.B2} B3: {normalParams.B3} B4: {normalParams.B4} B5: {normalParams.B5} R2: {normalParams.R2}.");
}
}
catch (Exception e)
{
Log.Error(normalParams.AccID + " " + normalParams.UtilID + " " + normalParams.UnitID + " " + e.Message + " " + e.StackTrace);
}
}
Log.Information($"PopulateWthNormalParams Finished. Rows Updated: {updateCount}. Rows Inserted: {insertCount}.");
UpdateWthExpUsage(newNormalParamsList);
}
private AccordResult CalculateLinearRegression(List <BalancePointPair> allBalancePointPairs, WthNormalParams normalParamsKey)
{
var allBalancePointGroups = allBalancePointPairs.GroupBy(s => new { s.CoolingBalancePoint, s.HeatingBalancePoint });
List <AccordResult> accordResults = new List <AccordResult>();
foreach (var group in allBalancePointGroups)
{
try
{
List <BalancePointPair> IdenticalBalancePointPairsFromAllReadings = group.ToList();
BalancePointPair _pointPair = IdenticalBalancePointPairsFromAllReadings.First();
int readingsCount = IdenticalBalancePointPairsFromAllReadings.Count;
double[] fullYData = new double[readingsCount];
double[] fullYDataDailyAvg = new double[readingsCount];
double[][] hcddMatrix = new double[readingsCount][];
double[][] hcddMatrixNonDaily = new double[readingsCount][];
foreach (BalancePointPair balancePointPair in IdenticalBalancePointPairsFromAllReadings)
{
fullYData[IdenticalBalancePointPairsFromAllReadings.IndexOf(balancePointPair)] = (balancePointPair.ActualUsage);
fullYDataDailyAvg[IdenticalBalancePointPairsFromAllReadings.IndexOf(balancePointPair)]
= (balancePointPair.ActualUsage / balancePointPair.DaysInReading);
hcddMatrix[IdenticalBalancePointPairsFromAllReadings.IndexOf(balancePointPair)] = new double[]
{
(balancePointPair.HeatingDegreeDays / balancePointPair.DaysInReading),
(balancePointPair.CoolingDegreeDays / balancePointPair.DaysInReading)
};
}
double[] avgHddsForEachReadingInYear = new double[readingsCount];
double[] avgCddsForEachReadingInYear = new double[readingsCount];
for (int i = 0; i < readingsCount; i++)
{
avgHddsForEachReadingInYear[i] = hcddMatrix[i][0];
avgCddsForEachReadingInYear[i] = hcddMatrix[i][1];
}
double[] modelParams = new double[3];
modelParams[0] = 0;
modelParams[1] = 0;
modelParams[2] = 0;
if (_pointPair.HeatingBalancePoint == 0 && _pointPair.CoolingBalancePoint == 0)
{
double[] onesVector = new double[readingsCount];
for (int i = 0; i < readingsCount; i++)
{
onesVector[i] = 1;
}
modelParams[0] = Fit.LineThroughOrigin(onesVector, fullYDataDailyAvg);
OrdinaryLeastSquares ols = new OrdinaryLeastSquares()
{
UseIntercept = false
};
double r2 = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination(
onesVector.Select(x => x * modelParams[0]), fullYDataDailyAvg);
AccordResult accordResult = new AccordResult()
{
IsSimpleSingleRegression = true,
HeatingBP = _pointPair.HeatingBalancePoint,
CoolingBP = _pointPair.CoolingBalancePoint,
Intercept = modelParams[0],
R2Accord = r2,
};
accordResults.Add(accordResult);
}
else if (_pointPair.CoolingBalancePoint != 0 && _pointPair.HeatingBalancePoint != 0)
{
try
{
MultipleLinearRegressionAnalysis mlra = new MultipleLinearRegressionAnalysis(intercept: true);
mlra.Learn(hcddMatrix, fullYDataDailyAvg);
var regressionAccord = mlra.Regression;
double[] predicted = regressionAccord.Transform(hcddMatrix);
double r2Accord = new RSquaredLoss(numberOfInputs: 2, expected: fullYDataDailyAvg)
{
Adjust = false
}.Loss(predicted);
double r2Coeff = regressionAccord.CoefficientOfDetermination(hcddMatrix, fullYDataDailyAvg, adjust: false);
bool FTestFailed = !mlra.FTest.Significant;
AccordResult accordResult = new AccordResult()
{
IsMultipleLinearRegression = true,
HeatingBP = _pointPair.HeatingBalancePoint,
CoolingBP = _pointPair.CoolingBalancePoint,
Intercept = regressionAccord.Intercept,
B2 = regressionAccord.Weights[0],
B4 = regressionAccord.Weights[1],
R2Accord = r2Accord,
FTestFailed = FTestFailed
};
if (mlra.Coefficients.All(x => x.TTest.Significant))
{
accordResults.Add(accordResult);
}
}
catch (Exception e)
{
Log.Debug(normalParamsKey.AccID + " " + normalParamsKey.UtilID + " " + normalParamsKey.UnitID + " " + e.Message + " " + e.StackTrace);
}
}
else if (_pointPair.HeatingBalancePoint > 0)
{
OrdinaryLeastSquares ols = new OrdinaryLeastSquares()
{
UseIntercept = true
};
SimpleLinearRegression regressionAccord = ols.Learn(avgHddsForEachReadingInYear, fullYDataDailyAvg);
double[] predictedAccord = regressionAccord.Transform(avgHddsForEachReadingInYear);
double r2Accord = new RSquaredLoss(1, fullYDataDailyAvg).Loss(predictedAccord);
int degreesOfFreedom = normalParamsKey.MoCt - 2;
double ssx = Math.Sqrt((avgHddsForEachReadingInYear.Subtract(avgHddsForEachReadingInYear.Mean())).Pow(2).Sum());
double s = Math.Sqrt(((fullYDataDailyAvg.Subtract(predictedAccord).Pow(2)).Sum()) / degreesOfFreedom);
double error = regressionAccord.GetStandardError(avgHddsForEachReadingInYear, fullYDataDailyAvg);
double seSubB = s / ssx;
double hypothesizedValue = 0;
TTest tTest = new TTest(
estimatedValue: regressionAccord.Slope, standardError: seSubB, degreesOfFreedom: degreesOfFreedom,
hypothesizedValue: hypothesizedValue, alternate: OneSampleHypothesis.ValueIsDifferentFromHypothesis
);
AccordResult accordResult = new AccordResult()
{
IsSimpleSingleRegression = true,
HeatingBP = _pointPair.HeatingBalancePoint,
Intercept = regressionAccord.Intercept,
B2 = regressionAccord.Slope,
R2Accord = r2Accord
};
if (tTest.Significant)
{
accordResults.Add(accordResult);
}
}
else if (_pointPair.CoolingBalancePoint > 0)
{
OrdinaryLeastSquares ols = new OrdinaryLeastSquares()
{
UseIntercept = true
};
SimpleLinearRegression regressionAccord = ols.Learn(avgCddsForEachReadingInYear, fullYDataDailyAvg);
double[] predictedAccord = regressionAccord.Transform(avgCddsForEachReadingInYear);
double rAccord = new RSquaredLoss(1, fullYDataDailyAvg).Loss(predictedAccord);
int degreesOfFreedom = normalParamsKey.MoCt - 2;
double ssx = Math.Sqrt(avgCddsForEachReadingInYear.Subtract(avgCddsForEachReadingInYear.Mean()).Pow(2).Sum());
double s = Math.Sqrt(((fullYDataDailyAvg.Subtract(predictedAccord).Pow(2)).Sum()) / degreesOfFreedom);
double seSubB = s / ssx;
double hypothesizedValue = 0;
double myT = seSubB / regressionAccord.Slope;
TTest tTest = new TTest(
estimatedValue: regressionAccord.Slope, standardError: seSubB, degreesOfFreedom: degreesOfFreedom,
hypothesizedValue: hypothesizedValue, alternate: OneSampleHypothesis.ValueIsDifferentFromHypothesis
);
AccordResult accordResult = new AccordResult()
{
IsSimpleSingleRegression = true,
CoolingBP = _pointPair.CoolingBalancePoint,
Intercept = regressionAccord.Intercept,
B4 = regressionAccord.Slope,
R2Accord = rAccord
};
if (tTest.Significant)
{
accordResults.Add(accordResult);
}
}
;
}
catch (Exception e)
{
Log.Debug(normalParamsKey.AccID + " " + normalParamsKey.UtilID + " " + normalParamsKey.UnitID + " " + e.Message + e.StackTrace);
}
}
AccordResult accordWinner = accordResults
.Where(s => s.Intercept >= 0)
.OrderByDescending(s => s.R2Accord).ToList().FirstOrDefault();
return(accordWinner);
}
private void PopulateWthNormalParams()
{
List <WthNormalParams> newNormalParamsList = new List <WthNormalParams>();
List <WthNormalParams> modelsWithNotTwelveReadings = new List <WthNormalParams>();
List <WthNormalParams> modelsWithReadingOrWeatherIssues = new List <WthNormalParams>();
List <WthNormalParams> modelsWithOtherIssues = new List <WthNormalParams>();
Log.Information("Starting PopulateWthNormalParams(). Searching WNRdngData01 stored procedure... ");
List <WNRdngData> allWNRdngData = _weatherRepository.GetAllReadingsFromStoredProcedure();
var wNRdngDataGroups = allWNRdngData.GroupBy(s => new { s.AccID, s.UtilID, s.UnitID });
//List<WNRdngData> heyy = wNRdngDataGroups.Contains(s => new { })
int updateCount = 0;
int insertCount = 0;
int failCount = 0;
foreach (var wNRdngGroup in wNRdngDataGroups)
{
List <WNRdngData> wNRdngList = wNRdngGroup.OrderBy(s => s.MoID).ToList();
WNRdngData lastRead = wNRdngList.LastOrDefault();
WthNormalParams normalParams = new WthNormalParams()
{
AccID = lastRead.AccID,
UtilID = lastRead.UtilID,
UnitID = lastRead.UnitID,
WstID = lastRead.WstID,
ZipW = lastRead.Zip,
EndDate = lastRead.DateEnd,
EMoID = lastRead.EMoID,
MoCt = lastRead.MoCt
};
if (wNRdngList.Count < 6)
{
Log.Error("There are only {0} readings for AccID/UtilID/UnitID: {1}/{2}/{3}. No model found.", wNRdngList.Count,
normalParams.AccID, normalParams.UtilID, normalParams.UnitID);
continue;
}
if (wNRdngList.Count % 12 != 0)
{
modelsWithNotTwelveReadings.Add(normalParams);
Log.Warning($"wNRdngList is not a multiple of 12 .. {wNRdngList.Count} Readings.\"MoCt\": {normalParams.MoCt}, still proceeding...");
}
bool normalParamsExists = _weatherRepository.GetWthNormalParamsExists(normalParams);
try
{
List <BalancePointPair> allBalancePointStatsFromYear = CalculateOneYearOfDegreeDaysForAllBalancePoints(wNRdngList);
//int daysInYear = allBalancePointStatsFromYear.FirstOrDefault().DaysInYear;
//normalParams.DaysInYear = daysInYear;
bool success = false;
if (allBalancePointStatsFromYear.Count == 0)
{
Log.Error($"AccID/UtilID/UnitID: {normalParams.AccID}/{normalParams.UtilID}/{normalParams.UnitID} >> " +
$"Why are there no BalancePointStatsFromYear?");
if (normalParamsExists)
{
success = UpdateOrInsertWthNormalParams(normalParams);
if (success)
{
newNormalParamsList.Add(normalParams);
updateCount++;
}
else
{
failCount++;
}
}
else
{
success = UpdateOrInsertWthNormalParams(normalParams);
if (success)
{
newNormalParamsList.Add(normalParams);
insertCount++;
}
else
{
failCount++;
}
}
continue;
}
AccordResult accord = CalculateLinearRegression(allBalancePointStatsFromYear, normalParams);
//if (accord.R2Accord < 0.7500)
//{
// success = UpdateOrInsertWthNormalParams(normalParams);
// if (success && normalParamsExists)
// {
// newNormalParamsList.Add(normalParams);
// updateCount++;
// }
// else if (success && !normalParamsExists)
// {
// newNormalParamsList.Add(normalParams);
// insertCount++;
// }
// else
// {
// failCount++;
// }
// continue;
//}
normalParams.B1 = decimal.Round(Convert.ToDecimal(accord.Intercept), 9, MidpointRounding.AwayFromZero);
if (accord.IsSimpleSingleRegression == true && accord.HeatingBP > 0)
{
normalParams.B2 = decimal.Round(Convert.ToDecimal(accord.B2), 9, MidpointRounding.AwayFromZero);
normalParams.B3 = accord.HeatingBP;
}
else if (accord.IsSimpleSingleRegression == true && accord.CoolingBP > 0)
{
normalParams.B4 = decimal.Round(Convert.ToDecimal(accord.B4), 9, MidpointRounding.AwayFromZero);
normalParams.B5 = accord.CoolingBP;
}
else if (accord.IsMultipleLinearRegression == true)
{
normalParams.B2 = decimal.Round(Convert.ToDecimal(accord.B2), 9, MidpointRounding.AwayFromZero);
normalParams.B3 = accord.HeatingBP;
normalParams.B4 = decimal.Round(Convert.ToDecimal(accord.B4), 9, MidpointRounding.AwayFromZero);
normalParams.B5 = accord.CoolingBP;
}
if (!double.IsNaN(accord.R2Accord) && !Double.IsInfinity(accord.R2Accord))
{
normalParams.R2 = decimal.Round(Convert.ToDecimal(accord.R2Accord), 9, MidpointRounding.AwayFromZero);
}
if (accord.FTestFailed)
{
Log.Warning($"AccID/UtilID/UnitID: {normalParams.AccID}/{normalParams.UtilID}/{normalParams.UnitID}: " +
$"Best Regression Model's F-Test failed and model was not rejected. R2 = {normalParams.R2}");
}
success = UpdateOrInsertWthNormalParams(normalParams);
if (success && normalParamsExists)
{
newNormalParamsList.Add(normalParams);
updateCount++;
}
else if (success && !normalParamsExists)
{
newNormalParamsList.Add(normalParams);
insertCount++;
}
else if (!success)
{
modelsWithOtherIssues.Add(normalParams);
failCount++;
}
}
catch (BadWNRdngDataException bdEx)
{
failCount++;
modelsWithReadingOrWeatherIssues.Add(normalParams);
Log.Error(bdEx.Message + " " + bdEx.StackTrace);
}
catch (Exception e)
{
failCount++;
modelsWithOtherIssues.Add(normalParams);
Log.Error($"AccID/UtilID/UnitID: {normalParams.AccID}/{normalParams.UtilID}/{normalParams.UnitID} >> {e.Message} {e.StackTrace}");
}
}
foreach (WthNormalParams normalParams in modelsWithNotTwelveReadings)
{
Log.Warning($"AccID/UtilID/UnitID: {normalParams.AccID}/{normalParams.UtilID}/{normalParams.UnitID} >> " +
$"Model was inserted but did not have (a multiple of) 12 readings from SP. ");
}
//foreach(WthNormalParams normalParams in modelsWithReadingOrWeatherIssues)
//{
// Log.Warning($"AccID/UtilID/UnitID: {normalParams.AccID}/{normalParams.UtilID}/{normalParams.UnitID} >> No model found. " +
// $"Readings had Bad/Null data from SP.");
//}
foreach (WthNormalParams normalParams in modelsWithOtherIssues)
{
Log.Error($"AccID/UtilID/UnitID: {normalParams.AccID}/{normalParams.UtilID}/{normalParams.UnitID} >> No model found. See daily log for more detail.");
}
if (failCount == 0)
{
Log.Information($"Finished PopulateWthNormalParams(). Rows Updated: {updateCount}. Rows Inserted: {insertCount}. Failures: {failCount}");
}
else
{
Log.Warning($"Finished PopulateWthNormalParams() with failures. Rows Updated: {updateCount}. Rows Inserted: {insertCount}. Failures: {failCount}");
}
//UpdateWthExpUsage(newNormalParamsList);
}
private AccordResult CalculateLinearRegression(List <BalancePointPair> allBalancePointPairs, WthNormalParams normalParamsKey)
{
var allBalancePointGroups = allBalancePointPairs.GroupBy(s => new { s.CoolingBalancePoint, s.HeatingBalancePoint });
List <AccordResult> accordResults = new List <AccordResult>();
foreach (var group in allBalancePointGroups)
{
try
{
List <BalancePointPair> IdenticalBalancePointPairsFromAllReadings = group.ToList();
BalancePointPair _pointPair = IdenticalBalancePointPairsFromAllReadings.First();
int readingsCount = IdenticalBalancePointPairsFromAllReadings.Count;
double[] fullYData = new double[readingsCount];
double[] fullYDataDailyAvg = new double[readingsCount];
double[][] hcddMatrix = new double[readingsCount][];
double[][] hcddMatrixNonDaily = new double[readingsCount][];
foreach (BalancePointPair balancePointPair in IdenticalBalancePointPairsFromAllReadings)
{
fullYData[IdenticalBalancePointPairsFromAllReadings.IndexOf(balancePointPair)] = (balancePointPair.ActualUsage);
fullYDataDailyAvg[IdenticalBalancePointPairsFromAllReadings.IndexOf(balancePointPair)]
= (balancePointPair.ActualUsage / balancePointPair.DaysInReading);
hcddMatrix[IdenticalBalancePointPairsFromAllReadings.IndexOf(balancePointPair)] = new double[]
{
(balancePointPair.HeatingDegreeDays / balancePointPair.DaysInReading),
(balancePointPair.CoolingDegreeDays / balancePointPair.DaysInReading)
};
}
if (!(fullYData.Sum() > 0))
{
return(new AccordResult());
}
double[] avgHddsForEachReadingInYear = new double[readingsCount];
double[] avgCddsForEachReadingInYear = new double[readingsCount];
for (int i = 0; i < readingsCount; i++)
{
avgHddsForEachReadingInYear[i] = hcddMatrix[i][0];
avgCddsForEachReadingInYear[i] = hcddMatrix[i][1];
}
double[] modelParams = new double[3];
modelParams[0] = 0;
modelParams[1] = 0;
modelParams[2] = 0;
if (_pointPair.HeatingBalancePoint == 0 && _pointPair.CoolingBalancePoint == 0)
{
double[] onesVector = new double[readingsCount];
for (int i = 0; i < readingsCount; i++)
{
onesVector[i] = 1;
}
modelParams[0] = Fit.LineThroughOrigin(onesVector, fullYDataDailyAvg);
OrdinaryLeastSquares ols = new OrdinaryLeastSquares()
{
UseIntercept = false
};
double r2 = MathNet.Numerics.GoodnessOfFit.CoefficientOfDetermination(
onesVector.Select(x => x * modelParams[0]), fullYDataDailyAvg);
AccordResult accordResult = new AccordResult()
{
IsSimpleSingleRegression = true,
HeatingBP = _pointPair.HeatingBalancePoint,
CoolingBP = _pointPair.CoolingBalancePoint,
Intercept = modelParams[0],
R2Accord = r2,
//R2Accord = 0
};
accordResults.Add(accordResult);
}
else if (_pointPair.CoolingBalancePoint != 0 && _pointPair.HeatingBalancePoint != 0)
{
try
{
MultipleLinearRegressionAnalysis mlra = new MultipleLinearRegressionAnalysis(intercept: true);
mlra.Learn(hcddMatrix, fullYDataDailyAvg);
var regressionAccord = mlra.Regression;
double[] predictedAccord = regressionAccord.Transform(hcddMatrix);
double r2Accord = new RSquaredLoss(numberOfInputs: 2, expected: fullYDataDailyAvg)
{
Adjust = false
}.Loss(predictedAccord);
double r2Coeff = regressionAccord.CoefficientOfDetermination(hcddMatrix, fullYDataDailyAvg, adjust: false);
bool FTestFailed = !mlra.FTest.Significant;
AccordResult accordResult = new AccordResult()
{
IsMultipleLinearRegression = true,
HeatingBP = _pointPair.HeatingBalancePoint,
CoolingBP = _pointPair.CoolingBalancePoint,
Intercept = regressionAccord.Intercept,
B2 = regressionAccord.Weights[0],
B4 = regressionAccord.Weights[1],
R2Accord = r2Accord,
FTestFailed = FTestFailed
};
//int degreesOfFreedom = normalParamsKey.MoCt - 3;
double degreesOfFreedomAsDouble = mlra.Regression.GetDegreesOfFreedom(readingsCount);
int degreesOfFreedom = Convert.ToInt32(degreesOfFreedomAsDouble);
//if (degreesOfFreedom != 9)
//{
// Log.Warning($"Multivariable regression. DOF expected to be 9. is: {degreesOfFreedom}");
//}
//if (degreesOfFreedom != dof)
//{
// Console.WriteLine($"dof different. mlra.dof = {dof} expected = {degreesOfFreedom}");
//}
double s = Math.Sqrt(fullYDataDailyAvg.Subtract(predictedAccord).Pow(2).Sum() / degreesOfFreedom);
double ssxHdd = Math.Sqrt((avgHddsForEachReadingInYear.Subtract(avgHddsForEachReadingInYear.Mean())).Pow(2).Sum());
double ssxCdd = Math.Sqrt((avgCddsForEachReadingInYear.Subtract(avgCddsForEachReadingInYear.Mean())).Pow(2).Sum());
double seSubHdd = s / ssxHdd;
double seSubCdd = s / ssxCdd;
double tStatisticHdd = regressionAccord.Weights[0] / seSubHdd;
double tStatisticCdd = regressionAccord.Weights[1] / seSubCdd;
double tCriticalFivePercent = 2.262156;
double tCriticalTenPercent = 1.833113;
bool myTestHdd = Math.Abs(tStatisticHdd) >= tCriticalTenPercent;
bool myTestCdd = Math.Abs(tStatisticCdd) >= tCriticalTenPercent;
//if (myTestHdd != mlra.Coefficients[0].TTest.Significant && degreesOfFreedom != 9)
//{
// Console.WriteLine($"nope. mystat - {tStatisticHdd} accordstat - {mlra.Coefficients[0].TTest.Statistic} " +
// $"accordCritical - {mlra.Coefficients[0].TTest.CriticalValue}");
//}
//if (myTestCdd != mlra.Coefficients[1].TTest.Significant && degreesOfFreedom != 9)
//{
// Console.WriteLine($"nope. mystat - {tStatisticCdd} accordstat - {mlra.Coefficients[1].TTest.Statistic} " +
// $"accordCritical - {mlra.Coefficients[1].TTest.CriticalValue}");
//}
//if (mlra.Coefficients.All(x => x.TTest.Significant) &&
// mlra.Coefficients.All(x => x.Value > 0) &&
// mlra.Regression.Intercept > 0 &&
// r2Accord >= 0.7500)
//{
// accordResults.Add(accordResult);
//}
if (
myTestHdd &&
myTestCdd &&
mlra.Coefficients.All(x => x.Value > 0) &&
mlra.Regression.Intercept > 0
//&& accordResult.R2Accord >= 0.75
)
{
accordResults.Add(accordResult);
}
}
catch (Exception e)
{
Log.Debug($"AccID/UtilID/UnitID: {normalParamsKey.AccID}/{normalParamsKey.UtilID}/{normalParamsKey.UnitID} >> " +
$"MultipleLinearRegressionAnalysis Exception: {e.Message}");
}
}
else if (_pointPair.HeatingBalancePoint > 0)
{
OrdinaryLeastSquares ols = new OrdinaryLeastSquares()
{
UseIntercept = true
};
SimpleLinearRegression regressionAccord = ols.Learn(avgHddsForEachReadingInYear, fullYDataDailyAvg);
double[] predictedAccord = regressionAccord.Transform(avgHddsForEachReadingInYear);
double r2Accord = new RSquaredLoss(1, fullYDataDailyAvg).Loss(predictedAccord);
//int degreesOfFreedom = normalParamsKey.MoCt - 2;
double degreesOfFreedomAsDouble = regressionAccord.GetDegreesOfFreedom(readingsCount);
int degreesOfFreedom = Convert.ToInt32(degreesOfFreedomAsDouble);
//if (degreesOfFreedom != 10)
//{
// Log.Warning($"Single variable regression. DOF expected to be 10. is: {degreesOfFreedom}");
//}
double ssx = Math.Sqrt((avgHddsForEachReadingInYear.Subtract(avgHddsForEachReadingInYear.Mean())).Pow(2).Sum());
double s = Math.Sqrt(fullYDataDailyAvg.Subtract(predictedAccord).Pow(2).Sum() / degreesOfFreedom);
double error = regressionAccord.GetStandardError(avgHddsForEachReadingInYear, fullYDataDailyAvg);
double seSubB = s / ssx;
double hypothesizedValue = 0;
double tStatistic = regressionAccord.Slope / seSubB;
double tCriticalFivePercent = 2.228138;
double tCriticalTenPercent = 1.812461;
bool myTest = Math.Abs(tStatistic) >= tCriticalTenPercent;
//TTest tTest = new TTest(
// estimatedValue: regressionAccord.Slope, standardError: seSubB, degreesOfFreedom: degreesOfFreedom,
// hypothesizedValue: hypothesizedValue, alternate: OneSampleHypothesis.ValueIsDifferentFromHypothesis
// );
//if (myTest != tTest.Significant)
//{
// Console.WriteLine($"nope. mystat - {tStatistic} accordstat - {tTest.Statistic} accordCritical - {tTest.CriticalValue}");
//}
AccordResult accordResult = new AccordResult()
{
IsSimpleSingleRegression = true,
HeatingBP = _pointPair.HeatingBalancePoint,
Intercept = regressionAccord.Intercept,
B2 = regressionAccord.Slope,
R2Accord = r2Accord
};
//if (tTest.Significant && accordResult.B2 > 0 && r2Accord >= 0.7500)
//{
// accordResults.Add(accordResult);
//}
if (myTest &&
accordResult.B2 > 0 &&
accordResult.Intercept > 0
//&& r2Accord >= 0.7500
)
{
accordResults.Add(accordResult);
}
}
else if (_pointPair.CoolingBalancePoint > 0)
{
OrdinaryLeastSquares ols = new OrdinaryLeastSquares()
{
UseIntercept = true
};
SimpleLinearRegression regressionAccord = ols.Learn(avgCddsForEachReadingInYear, fullYDataDailyAvg);
double[] predictedAccord = regressionAccord.Transform(avgCddsForEachReadingInYear);
double r2Accord = new RSquaredLoss(1, fullYDataDailyAvg).Loss(predictedAccord);
//int degreesOfFreedom = normalParamsKey.MoCt - 2;
double degreesOfFreedomAsDouble = regressionAccord.GetDegreesOfFreedom(readingsCount);
int degreesOfFreedom = Convert.ToInt32(degreesOfFreedomAsDouble);
//if (degreesOfFreedom != 10)
//{
// Log.Warning($"Single variable regression. DOF expected to be 10. is: {degreesOfFreedom}");
//}
double ssx = Math.Sqrt(avgCddsForEachReadingInYear.Subtract(avgCddsForEachReadingInYear.Mean()).Pow(2).Sum());
double s = Math.Sqrt(fullYDataDailyAvg.Subtract(predictedAccord).Pow(2).Sum() / degreesOfFreedom);
double seSubB = s / ssx;
double hypothesizedValue = 0;
double tStatistic = regressionAccord.Slope / seSubB;
double tCriticalFivePercent = 2.22813885198627;
double tCriticalTenPercent = 1.812461;
bool myTest = Math.Abs(tStatistic) >= tCriticalTenPercent;
//TTest tTest = new TTest(
// estimatedValue: regressionAccord.Slope, standardError: seSubB, degreesOfFreedom: degreesOfFreedom,
// hypothesizedValue: hypothesizedValue, alternate: OneSampleHypothesis.ValueIsDifferentFromHypothesis
// );
//if (myTest != tTest.Significant)
//{
// Console.WriteLine($"nope. mystat - {tStatistic} accordstat - {tTest.Statistic} accordCritical - {tTest.CriticalValue}");
//}
AccordResult accordResult = new AccordResult()
{
IsSimpleSingleRegression = true,
CoolingBP = _pointPair.CoolingBalancePoint,
Intercept = regressionAccord.Intercept,
B4 = regressionAccord.Slope,
R2Accord = r2Accord
};
//if (tTest.Significant && accordResult.B4 > 0 && r2Accord >= 0.7500)
//{
// accordResults.Add(accordResult);
//}
if (
myTest &&
accordResult.B4 > 0
//&& r2Accord >= 0.7500
)
{
accordResults.Add(accordResult);
}
}
}
catch (Exception e)
{
Log.Debug($"AccID/UtilID/UnitID: {normalParamsKey.AccID}/{normalParamsKey.UtilID}/{normalParamsKey.UnitID} >> {e.Message} {e.StackTrace}");
}
}
AccordResult accordWinner = accordResults
.Where(s => s.Intercept >= 0)
.OrderByDescending(s => s.R2Accord).ToList().FirstOrDefault();
return(accordWinner);
}
