public override Significance Compare(CoreDataSet data, CoreDataSet comparator, IndicatorMetadata metadata)
{
if (CanComparisonGoAhead(data, comparator) == false || !data.IsDenominatorValid)
{
return Significance.None;
}
double unitValue = metadata.Unit.Value;
double P1 = data.Value / unitValue;
double numerator = P1;
double confidenceVariableSquared = Math.Pow(ConfidenceVariable, 2.0);
double P0 = comparator.Value / unitValue;
double N1 = data.Denominator;
double E1 = N1 * P0;
// B
double Bs1 = 1.0 - (E1 / N1);
double Bs2 = 4.0 * E1 * Bs1;
double Bs3 = confidenceVariableSquared + Bs2;
double B = ConfidenceVariable * Math.Sqrt(Bs3);
// Control limits
double A = (2.0 * E1) + confidenceVariableSquared;
double C = 2.0 * (N1 + confidenceVariableSquared);
double L3SD = (A - B) / C;
double U3SD = (A + B) / C;
return GetSignificanceFromSDs(numerator, L3SD, U3SD);
}
private byte[] GetMetadataFileAsBytes(IndicatorMetadata indicatorMetadata,
IList<IndicatorMetadataTextProperty> properties, IDictionary<string, string> descriptive)
{
// Create CSV writer
var csvWriter = new CsvWriter();
csvWriter.AddHeader("Field", "Text");
// Add descriptive metadata properties
foreach (var property in properties)
{
if (descriptive.ContainsKey(property.ColumnName))
{
var text = descriptive[property.ColumnName];
if (string.IsNullOrWhiteSpace(text) == false)
{
text = htmlCleaner.RemoveHtml(text);
csvWriter.AddLine(property.DisplayName, text);
}
}
}
// Add other properties
csvWriter.AddLine("Unit", indicatorMetadata.Unit.Label);
csvWriter.AddLine("Value type", indicatorMetadata.ValueType.Name);
csvWriter.AddLine("Year type", indicatorMetadata.YearType.Name);
var bytes = csvWriter.WriteAsBytes();
return bytes;
}
public void TestDenominatorInvalid()
{
CoreDataSet comparator = new CoreDataSet { Value = 10 };
IndicatorMetadata metadata = new IndicatorMetadata();
Significance sig = New95_0().Compare(new CoreDataSet
{
Value = 10,
LowerCI = 11,
UpperCI = 12,
Denominator = 0
},
comparator, metadata
);
Assert.AreEqual(Significance.None, sig);
sig = New95_0().Compare(new CoreDataSet
{
Value = 10,
LowerCI = 11,
UpperCI = 12,
Denominator = -1
},
comparator, metadata
);
Assert.AreEqual(Significance.None, sig);
}
private void GivenAnIndicatorWhereMaximumValueis100()
{
metadata = new IndicatorMetadata
{
IndicatorId = IndicatorIds.QofPoints
};
}
public void Test_Trend_Result_Provided_For_Every_Area()
{
// Arrange
var areaCode1 = AreaCodes.CountyUa_Leicestershire;
var areaCode2 = AreaCodes.CountyUa_Bexley;
var mockTrendReader = new Mock<ITrendDataReader>();
mockTrendReader.Setup(x => x.GetTrendData(It.IsAny<Grouping>(), It.IsAny<string>()))
.Returns(new List<CoreDataSet> { });
var areas = new List<IArea>
{
new Area { Code = areaCode1},
new Area { Code = areaCode2}
};
var indicatorMetadata = new IndicatorMetadata
{
Unit = new Unit { Value = 1 },
ValueTypeId = ValueTypeId.CrudeRate,
};
// Act
var trendMarkers = new TrendMarkersProvider(mockTrendReader.Object, new TrendMarkerCalculator())
.GetTrendResults(areas, indicatorMetadata, new Grouping());
// Assert
Assert.AreEqual(2, trendMarkers.Count);
Assert.AreEqual(trendMarkers[areaCode1].Marker, TrendMarker.CannotBeCalculated);
Assert.AreEqual(trendMarkers[areaCode2].Marker, TrendMarker.CannotBeCalculated);
}
private static void IndexIndicator(IndicatorMetadata indicatorMetadata,
IEnumerable<IndicatorMetadataTextProperty> properties, IndexWriter writer)
{
Document doc = new Document();
doc.Add(new Field("id", indicatorMetadata.IndicatorId.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED));
var text = indicatorMetadata.Descriptive;
StringBuilder sb = new StringBuilder();
foreach (var indicatorMetadataTextProperty in properties)
{
var key = indicatorMetadataTextProperty.ColumnName;
if (text.ContainsKey(key))
{
sb.Append(text[key]);
sb.Append(" ");
}
}
doc.Add(new Field("IndicatorText",
sb.ToString().ToLower(), Field.Store.NO,
Field.Index.ANALYZED));
writer.AddDocument(doc);
}
public void SetProperties(CkanPackage unsavedPackage, CkanGroup ckanGroup, ProfileParameters parameters,
IndicatorMetadata indicatorMetadata, TimeRange timeRange)
{
SetMetadataProperties(unsavedPackage, indicatorMetadata);
SetPackageProperties(unsavedPackage, ckanGroup, parameters);
SetDateProperties(unsavedPackage, timeRange);
}
/// <summary>
/// Factory method
/// </summary>
public static GroupRootTrendBuilderBase New(Grouping grouping, IndicatorMetadata indicatorMetadata)
{
GroupRootTrendBuilderBase builder = GetAppropriateBuilderType(grouping);
builder.Grouping = grouping;
builder.IndicatorMetadata = indicatorMetadata;
return builder;
}
public override CoreDataSet GetData(Grouping grouping, TimePeriod timePeriod, IndicatorMetadata indicatorMetadata)
{
double val = practiceDataAccess.GetPracticeAggregateDataValue(grouping, timePeriod, Area.Code);
return val.Equals(ValueData.NullValue) ?
null :
new CoreDataSet { Value = val };
}
public override CoreDataSet GetData(Grouping grouping, TimePeriod timePeriod, IndicatorMetadata indicatorMetadata)
{
double val = practiceDataAccess.GetGpDeprivationDecileDataValue(grouping, timePeriod, categoryArea);
return val.Equals(ValueData.NullValue) ?
null :
new CoreDataSet { Value = val };
}
private void FormatDataAndStats(GroupRoot groupRoot, IndicatorMetadata metadata,
TimePeriodFormatter timePeriodFormatter)
{
var formatter = NumericFormatterFactory.New(metadata, groupDataReader);
new GroupRootFormatter().
Format(groupRoot, metadata, timePeriodFormatter, formatter);
}
public void TestNoDataDoesNotCauseException()
{
IndicatorMetadata metadata = new IndicatorMetadata();
GroupRoot root = new GroupRoot();
GetComparisonManager().CompareToCalculateSignficance(root, metadata);
Assert.AreEqual(0, root.Data.Count);
Assert.AreEqual(0, root.Grouping.Count);
}
private static void SetMetadataProperties(CkanPackage package, IndicatorMetadata indicatorMetadata)
{
var htmlCleaner = new HtmlCleaner();
IDictionary<string, string> descriptiveMetadata = indicatorMetadata.Descriptive;
package.Title = descriptiveMetadata[IndicatorMetadataTextColumnNames.Name];
package.Notes = htmlCleaner.RemoveHtml(descriptiveMetadata[IndicatorMetadataTextColumnNames.Definition]);
package.Origin = htmlCleaner.RemoveHtml(descriptiveMetadata[IndicatorMetadataTextColumnNames.Source]);
}
public static AverageCalculator Calculator(int id)
{
var indicatorMetadata = new IndicatorMetadata
{
ValueTypeId = id
};
return AverageCalculatorFactory.New(new List<CoreDataSet>(), indicatorMetadata);
}
public static ITimePeriodTextListBuilder New(bool doNotUseNullBuilder, IndicatorMetadata metadata)
{
if (doNotUseNullBuilder)
{
return new TimePeriodTextListBuilder(metadata);
}
return new NullTimePeriodTextListBuilder();
}
public override IList<CoreDataSet> AddMultipleAreaData(RowLabels rowLabels, Grouping grouping,
TimePeriod timePeriod, IndicatorMetadata metadata, Dictionary<string, Area> areaCodeToParentMap)
{
var dataList = GroupDataReader.GetCoreDataListForAllCategoryAreasOfCategoryAreaType(
grouping, timePeriod, _categoryAreaTypeId, AreaCodes.England);
ProfileDataWriter.AddCategorisedData(Worksheet, rowLabels, dataList, subnationalCategoryIdToCategoryAreaMap);
return dataList;
}
public IndicatorComparisonHelper(IndicatorMetadata indicatorMetadata, Grouping grouping,
IGroupDataReader groupDataReader, PholioReader pholioReader, IArea nationalArea)
{
// Assign constructor parameter to instance variables
this.indicatorMetadata = indicatorMetadata;
this.grouping = grouping;
this.groupDataReader = groupDataReader;
InitComparer(pholioReader, nationalArea);
}
protected void InitBuild(Grouping grouping)
{
if (grouping != null)
{
Grouping = grouping;
IndicatorMetadata = IndicatorMetadataRepository.Instance.GetIndicatorMetadata(grouping.IndicatorId);
Period = new DataPointOffsetCalculator(grouping, DataPointOffset, IndicatorMetadata.YearType).TimePeriod;
Formatter = GetFormatter();
}
}
public void TestTimePeriodsAreFormatted()
{
GroupRoot groupRoot = new GroupRoot();
groupRoot.Grouping.Add(new Grouping { DataPointYear = 2005, YearRange = 1, DataPointQuarter = -1 });
IndicatorMetadata metadata = new IndicatorMetadata { YearTypeId = 1 };
Limits limits = new Limits { Min = 0, Max = 50 };
var formatter = NumericFormatterFactory.NewWithLimits(metadata, limits);
new GroupRootFormatter().Format(groupRoot, metadata, new DataPointTimePeriodFormatter(), formatter);
Assert.AreEqual("2005", groupRoot.Grouping[0].TimePeriodText);
}
public TrendMarkerResult GetTrendMarkerResult(IndicatorMetadata indicatorMetadata, Grouping grouping, IList<CoreDataSet> dataList)
{
var trendRequest = new TrendRequest
{
UnitValue = indicatorMetadata.Unit.Value,
ValueTypeId = indicatorMetadata.ValueTypeId,
Data = dataList,
YearRange = grouping.YearRange,
};
var result = _trendCalculator.GetResults(trendRequest);
return result;
}
private CoreDataSet CalculateData(Grouping grouping, TimePeriod timePeriod, IndicatorMetadata indicatorMetadata)
{
if (RuleShouldCcgAverageBeCalculatedForGroup.Validates(grouping))
{
var dataList = coreDataSetListProvider.GetChildAreaData(grouping, Area, timePeriod);
if (dataList.Any())
{
var population = ccgPopulationProvider.GetPopulation(Area.Code);
return new CcgAverageCalculator(dataList, population, indicatorMetadata).Average;
}
}
return null;
}
public void Format(GroupRoot groupRoot, IndicatorMetadata metadata, TimePeriodFormatter timePeriodFormatter, NumericFormatter formatter)
{
foreach (CoreDataSet coreData in groupRoot.Data)
{
formatter.Format(coreData);
formatter.FormatConfidenceIntervals(coreData);
}
foreach (var grouping in groupRoot.Grouping)
{
formatter.Format(grouping.ComparatorData);
formatter.FormatConfidenceIntervals(grouping.ComparatorData);
timePeriodFormatter.Format(grouping, metadata);
}
}
public void GetPercentileData(TargetComparer targetComparer, Grouping grouping, IndicatorMetadata indicatorMetadata)
{
var bespokeComparer = targetComparer as BespokeTargetPercentileRangeComparer;
if (bespokeComparer != null)
{
var nationalValues = groupDataReader.GetCoreDataForAllAreasOfType(grouping, TimePeriod.GetDataPoint(grouping));
var percentileCalculator = new BespokeTargetPercentileRangeCalculator(nationalValues.Where(x => x.IsValueValid).Select(x => x.Value).ToList());
bespokeComparer.LowerTargetPercentileBenchmarkData =
new CoreDataSet() { Value = percentileCalculator.GetPercentileValue(bespokeComparer.GetLowerTargetPercentile()) };
bespokeComparer.UpperTargetPercentileBenchmarkData =
new CoreDataSet() { Value = percentileCalculator.GetPercentileValue(bespokeComparer.GetUpperTargetPercentile()) };
}
}
public void TestNullAverageWhenRatioValueType()
{
var data = new[] {
new CoreDataSet { Value = 2, Count = 4, Denominator = 2, AreaCode = "a" },
new CoreDataSet { Value = 2, Count = 4, Denominator = 2, AreaCode = "b" }
};
var metadata = new IndicatorMetadata
{
ValueTypeId = ValueTypeId.Ratio,
Unit = new Unit { Value = 1 }
};
Assert.IsNull(new CcgAverageCalculator(data, GetCcgPopulation(), metadata).Average);
}
/// <summary>
/// New numeric formatter instance.
/// </summary>
/// <param name="metadata">Indicator metadata of data to be formatted.</param>
/// <param name="limits">May be null if fixed decimal place is specified in metadata.</param>
public static NumericFormatter NewWithLimits(IndicatorMetadata metadata, Limits limits)
{
int? decimalPlacesDisplayed = metadata.DecimalPlacesDisplayed;
if (decimalPlacesDisplayed.HasValue && decimalPlacesDisplayed.Value >= 0)
{
return new FixedDecimalPlaceFormatter(decimalPlacesDisplayed.Value);
}
switch (metadata.ValueTypeId)
{
case 5:
return new ProportionFormatter(metadata, limits);
default:
return new DefaultFormatter(metadata, limits);
}
}
private TargetComparer GetTargetComparer(IndicatorMetadata metadata, GroupRoot groupRoot, IArea england)
{
var targetComparer = TargetComparerFactory.New(metadata.TargetConfig);
// Initialise the target comparer
if (targetComparer as BespokeTargetPercentileRangeComparer != null)
{
new TargetComparerHelper(groupDataReader, england)
.GetPercentileData(targetComparer, groupRoot.FirstGrouping, metadata);
}
else
{
new TargetComparerHelper(groupDataReader, england)
.AssignExtraDataIfRequired(england, targetComparer, groupRoot.FirstGrouping, metadata);
}
return targetComparer;
}
private void CompareRegionalAgainstNational(GroupRoot groupRoot, IndicatorMetadata metadata)
{
if (comparer != null &&
groupRoot.Grouping.Count > 1 &&
CanCompareSubnationalAgainstNational(groupRoot.FirstGrouping.ComparatorMethodId))
{
Grouping regionalGrouping = groupRoot.GetSubnationalGrouping();
Grouping nationalGrouping = groupRoot.GetNationalGrouping();
if (nationalGrouping != null && regionalGrouping != null && regionalGrouping.ComparatorData != null)
{
Significance significance = comparer.Compare(regionalGrouping.ComparatorData,
nationalGrouping.ComparatorData, metadata);
regionalGrouping.ComparatorData.AddSignificance(nationalGrouping.ComparatorId,
significance);
}
}
}
public static AverageCalculator New(IEnumerable<CoreDataSet> dataList, IndicatorMetadata indicatorMetadata)
{
switch (indicatorMetadata.ValueTypeId)
{
case ValueTypeId.IndirectlyStandardisedRatio:
case ValueTypeId.CrudeRate:
case ValueTypeId.Proportion:
return new WeightedAverageCalculator(new CoreDataSetFilter(dataList),
indicatorMetadata.Unit);
case ValueTypeId.Count:
return new CountAverageCalculator(new CoreDataSetFilter(dataList));
default:
return new NullAverageCalculator();
}
}
public override Significance Compare(CoreDataSet data, CoreDataSet comparator, IndicatorMetadata metadata)
{
if (CanComparisonGoAhead(data, comparator) == false || !data.IsDenominator2Valid)
{
return Significance.None;
}
double unit = metadata.Unit.Value;
double denominator = data.Denominator2;
double E1 = comparator.Value * (denominator / unit);
double l2SD = (StatisticalMethods.PoisLow(E1, ConfidenceVariable) / denominator) * unit;
double u2SD = (StatisticalMethods.PoisHigh(E1, ConfidenceVariable) / denominator) * unit;
return GetSignificanceFromSDs(data.Value, l2SD, u2SD);
}
public void TestComparisionWithDifferentConfidencesWhereSignifancesSame()
{
// Values taken from IndicatorId=1, Year=2010 (search by value if need to retrieve)
CoreDataSet comparator = new CoreDataSet { Value = 8.9611086876279327 };
IndicatorMetadata metadata =
new IndicatorMetadata { Unit = new Unit { Value = 100 } };
//5PP
CoreDataSet data = new CoreDataSet
{
Value = 7.7065767284991562,
LowerCI = 7.0548320523366375,
UpperCI = 8.4130813590727076,
Denominator = 5930,
YearRange = 1
};
Assert.AreEqual(Significance.Better, New95_0().Compare(data, comparator, metadata));
Assert.AreEqual(Significance.Better, New99_8().Compare(data, comparator, metadata));
//5PN
data = new CoreDataSet
{
Value = 9.8988121425428961,
LowerCI = 8.7379055891773572,
UpperCI = 11.195035154784872,
Denominator = 2273,
YearRange = 1
};
Assert.AreEqual(Significance.Same, New95_0().Compare(data, comparator, metadata));
Assert.AreEqual(Significance.Same, New99_8().Compare(data, comparator, metadata));
//5PR
data = new CoreDataSet
{
Value = 12.474645030425965,
LowerCI = 11.088798553603118,
UpperCI = 14.006406155616993,
Denominator = 1972,
YearRange = 1
};
Assert.AreEqual(Significance.Worse, New95_0().Compare(data, comparator, metadata));
Assert.AreEqual(Significance.Worse, New99_8().Compare(data, comparator, metadata));
}
