/// <summary> /// Day Aggregation /// </summary> /// <param name="definition">Aggregation Definition</param> /// <param name="patientId">Patient Id</param> /// <param name="observations">Observations</param> /// <returns></returns> private IEnumerable <IObservation> PerformDayAggregation(AggrConfig definition, string patientId, IEnumerable <PDObservation> observations) { // Get Min and Max Date var minT = observations.Select(e => e.Timestamp).DefaultIfEmpty().Min(); var maxT = observations.Select(e => e.Timestamp).DefaultIfEmpty().Max(); List <IObservation> metaObservations = new List <IObservation>(); var startDate = DateTimeExtensions.FromUnixTimestampMilli(minT); var endDate = DateTimeExtensions.FromUnixTimestampMilli(maxT); var startT = new DateTime(startDate.Year, startDate.Month, startDate.Day, 0, 0, 0, DateTimeKind.Utc).ToUnixTimestampMilli(); var endT = new DateTime(endDate.Year, endDate.Month, endDate.Day, 23, 59, 59, DateTimeKind.Utc).ToUnixTimestampMilli(); //Estimate mean and descriptive statistics for (long i = startT; i < endT; i += dayInterval) { double v = definition.Beta; foreach (var c in definition.Variables) { var v1 = c.Weight * observations.Where(e => e.CodeId == c.Code && e.Timestamp >= i && e.Timestamp < i + dayInterval).Select(e => e.Value).DefaultIfEmpty().Average(); v += v1; } metaObservations.Add(new PDObservation() { CodeId = definition.Code, PatientId = patientId, Timestamp = i, Value = v }); } return(metaObservations); }
private void Thresholding(AggrConfig definition, IEnumerable <IObservation> metaObservations) { var mean = metaObservations.Select(e => e.Value).DefaultIfEmpty(0).Average(); var q2 = metaObservations.Select(e => e.Value * e.Value).DefaultIfEmpty(0).Average(); int n = metaObservations.Count(); var std = q2 / n - mean * mean; //----------------- //THRESHOLDING //----------------- //If not thresholding required return mean if (definition.Threshold) { //Set threshold based on threshold type and value var threshold = definition.ThresholdValue; if (definition.ThresholdType == "std") { threshold = mean + definition.ThresholdValue * std; } //New observation timestamp foreach (var obs in metaObservations) { obs.Value = (obs.Value > threshold) ? 1.0 : 0.0; } } }
/// <summary> /// Load Aggregation Definition from file /// </summary> /// <param name="file"></param> /// <returns></returns> public static AggrConfig LoadFromFile(string file) { AggrConfig ret = null; StreamReader fstr = null; JsonTextReader reader = null; try { fstr = new StreamReader(file); reader = new JsonTextReader(fstr); JsonSerializer serializer = new JsonSerializer(); ret = serializer.Deserialize <AggrConfig>(reader); } catch (Exception ex) { throw ex; } finally { if (reader != null) { reader.Close(); } if (fstr != null) { fstr.Dispose(); } } return(ret); }
/// <summary> /// Save Aggregation Definition to file /// </summary> /// <param name="definition"></param> /// <param name="file"></param> public static void SaveToFile(AggrConfig definition, string file) { StreamWriter fstr = null; JsonTextWriter writer = null; try { fstr = new StreamWriter(file); writer = new JsonTextWriter(fstr); JsonSerializer serializer = new JsonSerializer(); serializer.Serialize(writer, definition); } catch (Exception ex) { throw ex; } finally { if (writer != null) { writer.Close(); } if (fstr != null) { fstr.Dispose(); } } }
/// <summary> /// Time Aggregation /// </summary> /// <param name="definition">Aggregation Definition</param> /// <param name="patientId">Patient Id</param> /// <param name="observations">Observations</param> /// <returns></returns> private IEnumerable <IObservation> PerformTimeAggregation(AggrConfig definition, string patientId, IEnumerable <PDObservation> observations) { List <IObservation> metaObservations = new List <IObservation>(); var minT = observations.Select(e => e.Timestamp).DefaultIfEmpty().Min(); var maxT = observations.Select(e => e.Timestamp).DefaultIfEmpty().Max(); var startDate = DateTimeExtensions.FromUnixTimestampMilli(minT); var endDate = DateTimeExtensions.FromUnixTimestampMilli(maxT); var t0 = new DateTime(startDate.Year, startDate.Month, startDate.Day, 0, 0, 0, DateTimeKind.Utc).ToUnixTimestampMilli(); var tmax = new DateTime(endDate.Year, endDate.Month, endDate.Day, 23, 59, 59, DateTimeKind.Utc).ToUnixTimestampMilli(); for (long j = 0; j <= dayInterval; j += timeInterval) { double m = 0; int n0 = 0; double v = definition.Beta; foreach (var c in definition.Variables) { n0 = 0; m = 0; for (long i = t0; i <= tmax; i += dayInterval) { long te = i + j + timeInterval; long ts = i + j; m += observations.Where(e => e.Timestamp >= ts && e.Timestamp < te && e.CodeId == c.Code).Select(e => e.Value).DefaultIfEmpty(0).Sum(); n0 += observations.Where(e => e.Timestamp >= ts && e.Timestamp < te && e.CodeId == c.Code).Count(); } if (n0 > 0) { v += (m / n0) * c.Weight; } } if (n0 > 0) { metaObservations.Add(new PDObservation() { CodeId = definition.Code, PatientId = patientId, Timestamp = j, Value = v, Weight = n0 }); } } return(metaObservations); }
/// <summary> /// Total Aggregation /// </summary> /// <param name="definition">Aggregation definition</param> /// <param name="patientId">Patient Id</param> /// <param name="timestamp">Timestamp</param> /// <param name="observations">Observations</param> /// <returns></returns> private IEnumerable <IObservation> PerformTotalAggregation(AggrConfig definition, string patientId, long timestamp, IEnumerable <PDObservation> observations) { double v = definition.Beta; foreach (var c in definition.Variables) { var v1 = c.Weight * observations.Where(e => e.CodeId == c.Code).Select(e => e.Value).DefaultIfEmpty().Average(); v += v1; } return(new List <IObservation>() { new PDObservation() { Value = v, PatientId = patientId, CodeId = definition.Code, Timestamp = timestamp } }); }
/// <summary> /// PerformAggregation Aggregation /// </summary> /// <param name="definition">Aggregation Definition</param> /// <param name="patientId">Patient Id</param> /// <param name="timestamp">Timestamp</param> /// <param name="observations">Observations</param> /// <returns></returns> private IEnumerable <IObservation> PerformAggregation(AggrConfig definition, string patientId, long timestamp, IEnumerable <PDObservation> observations) { //----------------- // 1st Level Aggregation //------------ IEnumerable <IObservation> metaObservations = null; //return PerformTotalAggregation(definition, patientId, timestamp, observations); if (definition.AggregationType == TimeAggregationType) { metaObservations = PerformTimeAggregation(definition, patientId, observations); } else if (definition.AggregationType == DayAggregationType) { metaObservations = PerformDayAggregation(definition, patientId, observations); } else { metaObservations = PerformTotalAggregation(definition, patientId, timestamp, observations); } if (metaObservations == null) { //Something wrong happened!! throw new Exception(); } //--------------------- //Threshold meta-observations //--------------------- Thresholding(definition, metaObservations); //--------------------- //2nd Level aggregation //--------------------- return(MetaAggregation(definition, patientId, metaObservations)); }
/// <summary> /// Run Aggregation /// This method /// 1) loads the aggregation definition /// 2) Fetch all required observations using the DataProxy /// 3) Calls the PerformAggregation method to perform the aggregation and returns a new observation /// /// </summary> /// <param name="patientId">Patient Id</param> /// <param name="code">Meta observation Code</param> /// <param name="lastExecutionTime"></param> /// <param name="aggregationType">Overrides the default aggregation type</param> /// <param name="filterType">Overrides the default filter type</param> /// <returns></returns> public async Task <IEnumerable <IObservation> > Run(string patientId, string code, DateTime?lastExecutionTime, string aggregationType = null, string filterType = null) { if (patientId == null) { throw new ArgumentNullException(nameof(patientId)); } if (code == null) { throw new ArgumentNullException(nameof(code)); } // Get Definition File Name from AggrDefinitionDictionary if (_definition == null) { _definition = AggrConfig.FromString(_aggrDefinitionDictionary.GetJsonConfigFromCode(code)); } if (_definition == null) { throw new AggrDefinitionNotFoundException(nameof(_definition)); } if (!string.IsNullOrEmpty(aggregationType)) { _definition.AggregationType = aggregationType; } if (!string.IsNullOrEmpty(filterType)) { _definition.MetaAggregationType = filterType; } var lastExecutionTimeStamp = lastExecutionTime.HasValue ? lastExecutionTime.Value.ToUnixTimestamp() * 1000 : 0; //Get Data List <PDObservation> observations = new List <PDObservation>(); foreach (var c in _definition.Variables) { try { //Get Observations For patient var ret = await _proxy.Get <PDObservation>(MAXTAKE, 0, GetFilter(patientId, c.Code, lastExecutionTimeStamp, _definition.AggregationType), null); observations.AddRange(ret); } catch (Exception ex) { _logger?.LogError(ex, "Error in aggregation"); } } var metaObservation = PerformAggregation(_definition, patientId, lastExecutionTimeStamp, observations); return(metaObservation); }
private AggrConfig LoadAggrDefinition(string name) { return(AggrConfig.LoadFromFile(name)); }
/// <summary> /// 2nd Level Aggregation /// </summary> /// <param name="definition">Aggregation Definition</param> /// <param name="patientId">Patient Id</param> /// <param name="metaObservations">Observations</param> /// <returns></returns> private IEnumerable <IObservation> MetaAggregation(AggrConfig definition, string patientId, IEnumerable <IObservation> metaObservations) { var endT = metaObservations.Select(e => e.Timestamp).DefaultIfEmpty().Max(); if (definition.MetaAggregationType == "sum") { return(new List <IObservation>() { new PDObservation() { CodeId = definition.Code, PatientId = patientId, Timestamp = endT, Value = definition.MetaScale * metaObservations.Select(e => e.Value).DefaultIfEmpty(0).Sum() } }); } else if (definition.MetaAggregationType == "average") { return(new List <IObservation>() { new PDObservation() { CodeId = definition.Code, PatientId = patientId, Timestamp = endT, Value = definition.MetaScale * metaObservations.Select(e => e.Value).DefaultIfEmpty(0).Average() } }); } else if (definition.MetaAggregationType == "mfi") { return(new List <IObservation>() { new PDObservation() { CodeId = definition.Code, PatientId = patientId, Timestamp = endT, Value = definition.MetaScale * (metaObservations.Select(e => e.Value).DefaultIfEmpty(0).Max() - metaObservations.Select(e => e.Value).DefaultIfEmpty(0).Average()) } }); } else if (definition.MetaAggregationType == "count") { return(new List <IObservation>() { new PDObservation() { CodeId = definition.Code, PatientId = patientId, Timestamp = endT, Value = definition.MetaScale * metaObservations.Count() } }); } else if (definition.MetaAggregationType == "max") { return(new List <IObservation>() { new PDObservation() { CodeId = definition.Code, PatientId = patientId, Timestamp = endT, Value = definition.MetaScale * metaObservations.Select(e => e.Value).DefaultIfEmpty(0).Max() } }); } else if (definition.MetaAggregationType == "min") { return(new List <IObservation>() { new PDObservation() { CodeId = definition.Code, PatientId = patientId, Timestamp = endT, Value = definition.MetaScale * metaObservations.Select(e => e.Value).DefaultIfEmpty(0).Min() } }); } else if (definition.MetaAggregationType == "std") { var fmean = metaObservations.Select(e => e.Value).DefaultIfEmpty(0).Average(); var fq2 = metaObservations.Select(e => e.Value * e.Value).DefaultIfEmpty(0).Average(); int fn = metaObservations.Count(); var fstd = fq2 / fn - fmean * fmean; return(new List <IObservation>() { new PDObservation() { CodeId = definition.Code, PatientId = patientId, Timestamp = endT, Value = definition.MetaScale * fstd } }); } else if (definition.MetaAggregationType == "cv") { var fmean = metaObservations.Select(e => e.Value).DefaultIfEmpty(0).Average(); var fq2 = metaObservations.Select(e => e.Value * e.Value).DefaultIfEmpty(0).Average(); int fn = metaObservations.Count(); var fstd = fq2 / fn - fmean * fmean; return(new List <IObservation>() { new PDObservation() { CodeId = definition.Code, PatientId = patientId, Timestamp = endT, Value = definition.MetaScale * 100 * fstd / fmean } }); } else { return(metaObservations); } }
/// <summary> /// Create Aggregation Definition from string /// </summary> /// <param name="configJson">Config in json</param> /// <returns></returns> public static AggrConfig FromString(string configJson) { AggrConfig ret = null; return(ret = JsonConvert.DeserializeObject <AggrConfig>(configJson)); }