/// <summary>
/// Executes the specified context.
/// </summary>
/// <param name="context">The context.</param>
public void Execute(IJobExecutionContext context)
{
var metricSourceValueTypeDataviewGuid = Rock.SystemGuid.DefinedValue.METRIC_SOURCE_VALUE_TYPE_DATAVIEW.AsGuid();
var metricSourceValueTypeSqlGuid = Rock.SystemGuid.DefinedValue.METRIC_SOURCE_VALUE_TYPE_SQL.AsGuid();
var metricsQry = new MetricService(new RockContext()).Queryable().AsNoTracking().Where(
a => a.ScheduleId.HasValue &&
a.SourceValueTypeId.HasValue &&
(a.SourceValueType.Guid == metricSourceValueTypeDataviewGuid || a.SourceValueType.Guid == metricSourceValueTypeSqlGuid));
var metricIdList = metricsQry.OrderBy(a => a.Title).ThenBy(a => a.Subtitle).Select(a => a.Id).ToList();
var metricExceptions = new List <Exception>();
int metricsCalculated = 0;
int metricValuesCalculated = 0;
Metric metric = null;
foreach (var metricId in metricIdList)
{
try
{
using (var rockContextForMetricEntity = new RockContext())
{
var metricService = new MetricService(rockContextForMetricEntity);
metric = metricService.Get(metricId);
var lastRunDateTime = metric.LastRunDateTime ?? metric.CreatedDateTime ?? metric.ModifiedDateTime;
if (lastRunDateTime.HasValue)
{
var currentDateTime = RockDateTime.Now;
// get all the schedule times that were supposed to run since that last time it was scheduled to run
var scheduledDateTimesToProcess = metric.Schedule.GetScheduledStartTimes(lastRunDateTime.Value, currentDateTime).Where(a => a > lastRunDateTime.Value).ToList();
foreach (var scheduleDateTime in scheduledDateTimesToProcess)
{
using (var rockContextForMetricValues = new RockContext())
{
var metricPartitions = new MetricPartitionService(rockContextForMetricValues).Queryable().Where(a => a.MetricId == metric.Id).ToList();
var metricValueService = new MetricValueService(rockContextForMetricValues);
List <ResultValue> resultValues = new List <ResultValue>();
bool getMetricValueDateTimeFromResultSet = false;
if (metric.SourceValueType.Guid == metricSourceValueTypeDataviewGuid)
{
// get the metric value from the DataView
if (metric.DataView != null)
{
var errorMessages = new List <string>();
var qry = metric.DataView.GetQuery(null, null, out errorMessages);
if (metricPartitions.Count > 1 || metricPartitions.First().EntityTypeId.HasValue)
{
throw new NotImplementedException("Partitioned Metrics using DataViews is not supported.");
}
else
{
var resultValue = new ResultValue();
resultValue.Value = Convert.ToDecimal(qry.Count());
resultValue.Partitions = new List <ResultValuePartition>();
resultValue.MetricValueDateTime = scheduleDateTime;
resultValues.Add(resultValue);
}
}
}
else if (metric.SourceValueType.Guid == metricSourceValueTypeSqlGuid)
{
//// calculate the metricValue using the results from the SQL
//// assume SQL is in one of the following forms:
//// -- "SELECT Count(*), Partion0EntityId, Partion1EntityId, Partion2EntityId,.. FROM ..."
//// -- "SELECT Count(*), [MetricValueDateTime], Partion0EntityId, Partion1EntityId, Partion2EntityId,.. FROM ..."
if (!string.IsNullOrWhiteSpace(metric.SourceSql))
{
string formattedSql = metric.SourceSql.ResolveMergeFields(metric.GetMergeObjects(scheduleDateTime));
var tableResult = DbService.GetDataTable(formattedSql, System.Data.CommandType.Text, null);
if (tableResult.Columns.Count >= 2 && tableResult.Columns[1].ColumnName == "MetricValueDateTime")
{
getMetricValueDateTimeFromResultSet = true;
}
foreach (var row in tableResult.Rows.OfType <System.Data.DataRow>())
{
var resultValue = new ResultValue();
resultValue.Value = Convert.ToDecimal(row[0]);
if (getMetricValueDateTimeFromResultSet)
{
resultValue.MetricValueDateTime = Convert.ToDateTime(row[1]);
}
else
{
resultValue.MetricValueDateTime = scheduleDateTime;
}
resultValue.Partitions = new List <ResultValuePartition>();
int partitionPosition = 0;
int partitionFieldIndex = getMetricValueDateTimeFromResultSet ? 2 : 1;
int partitionColumnCount = tableResult.Columns.Count - 1;
while (partitionFieldIndex <= partitionColumnCount)
{
resultValue.Partitions.Add(new ResultValuePartition
{
PartitionPosition = partitionPosition,
EntityId = row[partitionFieldIndex] as int?
});
partitionPosition++;
partitionFieldIndex++;
}
resultValues.Add(resultValue);
}
}
}
metric.LastRunDateTime = scheduleDateTime;
metricsCalculated++;
metricValuesCalculated += resultValues.Count();
if (resultValues.Any())
{
List <MetricValue> metricValuesToAdd = new List <MetricValue>();
foreach (var resultValue in resultValues)
{
var metricValue = new MetricValue();
metricValue.MetricId = metric.Id;
metricValue.MetricValueDateTime = resultValue.MetricValueDateTime;
metricValue.MetricValueType = MetricValueType.Measure;
metricValue.YValue = resultValue.Value;
metricValue.CreatedDateTime = RockDateTime.Now;
metricValue.ModifiedDateTime = RockDateTime.Now;
metricValue.MetricValuePartitions = new List <MetricValuePartition>();
var metricPartitionsByPosition = metricPartitions.OrderBy(a => a.Order).ToList();
if (!resultValue.Partitions.Any() && metricPartitionsByPosition.Count() == 1 && !metricPartitionsByPosition[0].EntityTypeId.HasValue)
{
// a metric with just the default partition (not partitioned by Entity)
var metricPartition = metricPartitionsByPosition[0];
var metricValuePartition = new MetricValuePartition();
metricValuePartition.MetricPartition = metricPartition;
metricValuePartition.MetricPartitionId = metricPartition.Id;
metricValuePartition.MetricValue = metricValue;
metricValuePartition.EntityId = null;
metricValue.MetricValuePartitions.Add(metricValuePartition);
}
else
{
foreach (var partitionResult in resultValue.Partitions)
{
if (metricPartitionsByPosition.Count > partitionResult.PartitionPosition)
{
var metricPartition = metricPartitionsByPosition[partitionResult.PartitionPosition];
var metricValuePartition = new MetricValuePartition();
metricValuePartition.MetricPartition = metricPartition;
metricValuePartition.MetricPartitionId = metricPartition.Id;
metricValuePartition.MetricValue = metricValue;
metricValuePartition.EntityId = partitionResult.EntityId;
metricValue.MetricValuePartitions.Add(metricValuePartition);
}
}
}
if (metricValue.MetricValuePartitions == null || !metricValue.MetricValuePartitions.Any())
{
// shouldn't happen, but just in case
throw new Exception("MetricValue requires at least one Partition Value");
}
else
{
metricValuesToAdd.Add(metricValue);
}
}
// if a single metricValueDateTime was specified, delete any existing metric values for this date and refresh with the current results
var dbTransaction = rockContextForMetricValues.Database.BeginTransaction();
if (getMetricValueDateTimeFromResultSet)
{
var metricValueDateTimes = metricValuesToAdd.Select(a => a.MetricValueDateTime).Distinct().ToList();
foreach (var metricValueDateTime in metricValueDateTimes)
{
bool alreadyHasMetricValues = metricValueService.Queryable().Where(a => a.MetricId == metric.Id && a.MetricValueDateTime == metricValueDateTime).Any();
if (alreadyHasMetricValues)
{
// use direct SQL to clean up any existing metric values
rockContextForMetricValues.Database.ExecuteSqlCommand(@"
DELETE
FROM MetricValuePartition
WHERE MetricValueId IN (
SELECT Id
FROM MetricValue
WHERE MetricId = @metricId
AND MetricValueDateTime = @metricValueDateTime
)
", new SqlParameter("@metricId", metric.Id), new SqlParameter("@metricValueDateTime", metricValueDateTime));
rockContextForMetricValues.Database.ExecuteSqlCommand(@"
DELETE
FROM MetricValue
WHERE MetricId = @metricId
AND MetricValueDateTime = @metricValueDateTime
", new SqlParameter("@metricId", metric.Id), new SqlParameter("@metricValueDateTime", metricValueDateTime));
}
}
}
metricValueService.AddRange(metricValuesToAdd);
// disable savechanges PrePostProcessing since there could be hundreds or thousands of metric values getting inserted/updated
rockContextForMetricValues.SaveChanges(true);
dbTransaction.Commit();
}
rockContextForMetricEntity.SaveChanges();
}
}
}
}
}
catch (Exception ex)
{
metricExceptions.Add(new Exception(string.Format("Exception when calculating metric for {0} ", metric), ex));
}
}
context.Result = string.Format("Calculated a total of {0} metric values for {1} metrics", metricValuesCalculated, metricsCalculated);
if (metricExceptions.Any())
{
throw new AggregateException("One or more metric calculations failed ", metricExceptions);
}
}
/// <summary>
/// Executes the specified context.
/// </summary>
/// <param name="context">The context.</param>
public void Execute(IJobExecutionContext context)
{
var dataMap = context.JobDetail.JobDataMap;
var commandTimeout = dataMap.GetString(AttributeKey.CommandTimeout).AsIntegerOrNull() ?? 300;
var metricSourceValueTypeDataviewGuid = Rock.SystemGuid.DefinedValue.METRIC_SOURCE_VALUE_TYPE_DATAVIEW.AsGuid();
var metricSourceValueTypeSqlGuid = Rock.SystemGuid.DefinedValue.METRIC_SOURCE_VALUE_TYPE_SQL.AsGuid();
var metricSourceValueTypeLavaGuid = Rock.SystemGuid.DefinedValue.METRIC_SOURCE_VALUE_TYPE_LAVA.AsGuid();
Guid[] calculatedSourceTypes = new Guid[] {
metricSourceValueTypeDataviewGuid,
metricSourceValueTypeSqlGuid,
metricSourceValueTypeLavaGuid
};
var metricsQry = new MetricService(new RockContext()).Queryable().AsNoTracking().Where(
a => a.ScheduleId.HasValue &&
a.SourceValueTypeId.HasValue &&
calculatedSourceTypes.Contains(a.SourceValueType.Guid));
var metricIdList = metricsQry.OrderBy(a => a.Title).ThenBy(a => a.Subtitle).Select(a => a.Id).ToList();
var metricExceptions = new List <Exception>();
int metricsCalculated = 0;
int metricValuesCalculated = 0;
Metric metric = null;
foreach (var metricId in metricIdList)
{
try
{
using (var rockContextForMetricEntity = new RockContext())
{
rockContextForMetricEntity.Database.CommandTimeout = commandTimeout;
var metricService = new MetricService(rockContextForMetricEntity);
metric = metricService.Get(metricId);
/* Get the last time the metric has run, if it has never run then set it as the first day of the current week to give it a chance to run now.
* NOTE: This is being used instead of Min Date to prevent a schedule with a start date of months or years back from running for each period since then.
* This would be the case for the "out-of-the-box" Rock daily metrics "Active Records", "Active Families", and "Active Connection Requests" if they
* have never run before. Or if a new user created metric uses a schedule that has an older start date.
*/
DateTime currentDateTime = RockDateTime.Now;
DateTime?startOfWeek = currentDateTime.StartOfWeek(RockDateTime.FirstDayOfWeek);
DateTime?lastRunDateTime = metric.LastRunDateTime ?? startOfWeek;
// get all the schedule times that were supposed to run since that last time it was scheduled to run
var scheduledDateTimesToProcess = metric.Schedule.GetScheduledStartTimes(lastRunDateTime.Value, currentDateTime).Where(a => a > lastRunDateTime.Value).ToList();
foreach (var scheduleDateTime in scheduledDateTimesToProcess)
{
using (var rockContextForMetricValues = new RockContext())
{
rockContextForMetricValues.Database.CommandTimeout = commandTimeout;
var metricPartitions = new MetricPartitionService(rockContextForMetricValues).Queryable().Where(a => a.MetricId == metric.Id).ToList();
var metricValueService = new MetricValueService(rockContextForMetricValues);
List <ResultValue> resultValues = new List <ResultValue>();
bool getMetricValueDateTimeFromResultSet = false;
if (metric.SourceValueType.Guid == metricSourceValueTypeDataviewGuid)
{
// get the metric value from the DataView
if (metric.DataView != null)
{
bool parseCampusPartition = metricPartitions.Count == 1 &&
metric.AutoPartitionOnPrimaryCampus &&
metric.DataView.EntityTypeId == Web.Cache.EntityTypeCache.GetId(SystemGuid.EntityType.PERSON) &&
metricPartitions[0].EntityTypeId == Web.Cache.EntityTypeCache.GetId(SystemGuid.EntityType.CAMPUS);
// Dataview metrics can be partitioned by campus only and AutoPartitionOnPrimaryCampus must be selected.
if (metricPartitions.Count > 1 ||
(metricPartitions[0].EntityTypeId.HasValue && parseCampusPartition == false))
{
throw new NotImplementedException("Partitioned Metrics using DataViews is only supported for Person data views using a single partition of type 'Campus' with 'Auto Partition on Primary Campus' checked. Any other dataview partition configuration is not supported.");
}
Stopwatch stopwatch = Stopwatch.StartNew();
var qry = metric.DataView.GetQuery(new DataViewGetQueryArgs());
if (parseCampusPartition)
{
// Create a dictionary of campus IDs with a person count
var campusPartitionValues = new Dictionary <int, int>();
foreach (var person in qry.ToList())
{
var iPerson = ( Person )person;
var campusId = iPerson.GetCampus()?.Id ?? -1;
campusPartitionValues.TryGetValue(campusId, out var currentCount);
campusPartitionValues[campusId] = currentCount + 1;
}
// Use the dictionary to create the ResultValues for each campus (partition)
foreach (var campusPartitionValue in campusPartitionValues)
{
var resultValue = new ResultValue
{
MetricValueDateTime = scheduleDateTime,
Partitions = new List <ResultValuePartition>(),
Value = campusPartitionValue.Value
};
int?entityId = campusPartitionValue.Key;
if (entityId == -1)
{
entityId = null;
}
resultValue.Partitions.Add(new ResultValuePartition
{
PartitionPosition = 0,
EntityId = entityId
});
resultValues.Add(resultValue);
}
}
else
{
// Put the entire set in one result since there is no partition
resultValues.Add(new ResultValue
{
Value = Convert.ToDecimal(qry.Count()),
MetricValueDateTime = scheduleDateTime,
Partitions = new List <ResultValuePartition>()
});
}
stopwatch.Stop();
DataViewService.AddRunDataViewTransaction(metric.DataView.Id, Convert.ToInt32(stopwatch.Elapsed.TotalMilliseconds));
}
}
else if (metric.SourceValueType.Guid == metricSourceValueTypeSqlGuid)
{
//// calculate the metricValue using the results from the SQL
//// assume SQL is in one of the following forms:
//// -- "SELECT Count(*) FROM ..."
//// -- "SELECT Count(*), [MetricValueDateTime] FROM ..."
//// -- "SELECT Count(*), Partition0EntityId, Partition1EntityId, Partition2EntityId,.. FROM ..."
//// -- "SELECT Count(*), [MetricValueDateTime], Partition0EntityId, Partition1EntityId, Partition2EntityId,.. FROM ..."
if (!string.IsNullOrWhiteSpace(metric.SourceSql))
{
string formattedSql = metric.SourceSql.ResolveMergeFields(metric.GetMergeObjects(scheduleDateTime));
var tableResult = DbService.GetDataTable(formattedSql, System.Data.CommandType.Text, null);
if (tableResult.Columns.Count >= 2 && tableResult.Columns[1].ColumnName == "MetricValueDateTime")
{
getMetricValueDateTimeFromResultSet = true;
}
foreach (var row in tableResult.Rows.OfType <System.Data.DataRow>())
{
var resultValue = new ResultValue();
resultValue.Value = Convert.ToDecimal(row[0] == DBNull.Value ? 0 : row[0]);
if (getMetricValueDateTimeFromResultSet)
{
resultValue.MetricValueDateTime = Convert.ToDateTime(row[1]);
}
else
{
resultValue.MetricValueDateTime = scheduleDateTime;
}
resultValue.Partitions = new List <ResultValuePartition>();
int partitionPosition = 0;
int partitionFieldIndex = getMetricValueDateTimeFromResultSet ? 2 : 1;
int partitionColumnCount = tableResult.Columns.Count - 1;
while (partitionFieldIndex <= partitionColumnCount)
{
resultValue.Partitions.Add(new ResultValuePartition
{
PartitionPosition = partitionPosition,
EntityId = row[partitionFieldIndex] as int?
});
partitionPosition++;
partitionFieldIndex++;
}
resultValues.Add(resultValue);
}
}
}
else if (metric.SourceValueType.Guid == metricSourceValueTypeLavaGuid)
{
//// calculate the metricValue using the results from Lava
//// assume Lava Output is in one of the following forms:
//// A single Count
//// 42
//// A List of Count, MetricValueDateTime
//// 42, 1/1/2017
//// 40, 1/2/2017
//// 49, 1/3/2017
//// A List of Count, Partition0EntityId, Partition1EntityId, Partition2EntityId
//// 42, 201, 450, 654
//// 42, 202, 450, 654
//// 42, 203, 450, 654
//// A List of Count, MetricValueDateTime, Partition0EntityId, Partition1EntityId, Partition2EntityId
//// 42, 1/1/2017, 201, 450, 654
//// 42, 1/2/2017, 202, 450, 654
//// 42, 1/3/2017, 203, 450, 654
if (!string.IsNullOrWhiteSpace(metric.SourceLava))
{
var mergeObjects = metric.GetMergeObjects(scheduleDateTime);
string lavaResult = metric.SourceLava.ResolveMergeFields(mergeObjects, enabledLavaCommands: "All", throwExceptionOnErrors: true);
List <string> resultLines = lavaResult.Split(new string[] { "\n", "\r\n" }, StringSplitOptions.RemoveEmptyEntries)
.Select(a => a.Trim()).Where(a => !string.IsNullOrEmpty(a)).ToList();
List <string[]> resultList = resultLines.Select(a => a.SplitDelimitedValues()).ToList();
if (resultList.Any())
{
if (resultList[0].Length >= 2)
{
// if the value of the data in the 2nd column is a Date, assume that is is the MetricValueDateTime
if (resultList[0][1].AsDateTime().HasValue)
{
getMetricValueDateTimeFromResultSet = true;
}
}
}
foreach (var row in resultList)
{
var resultValue = new ResultValue();
resultValue.Value = row[0].AsDecimal();
if (getMetricValueDateTimeFromResultSet)
{
resultValue.MetricValueDateTime = row[1].AsDateTime() ?? scheduleDateTime;
}
else
{
resultValue.MetricValueDateTime = scheduleDateTime;
}
resultValue.Partitions = new List <ResultValuePartition>();
int partitionPosition = 0;
int partitionFieldIndex = getMetricValueDateTimeFromResultSet ? 2 : 1;
int partitionColumnCount = row.Length - 1;
while (partitionFieldIndex <= partitionColumnCount)
{
resultValue.Partitions.Add(new ResultValuePartition
{
PartitionPosition = partitionPosition,
EntityId = row[partitionFieldIndex].AsIntegerOrNull()
});
partitionPosition++;
partitionFieldIndex++;
}
resultValues.Add(resultValue);
}
}
}
metric.LastRunDateTime = scheduleDateTime;
metricsCalculated++;
metricValuesCalculated += resultValues.Count();
if (resultValues.Any())
{
List <MetricValue> metricValuesToAdd = new List <MetricValue>();
foreach (var resultValue in resultValues)
{
var metricValue = new MetricValue();
metricValue.MetricId = metric.Id;
metricValue.MetricValueDateTime = resultValue.MetricValueDateTime;
metricValue.MetricValueType = MetricValueType.Measure;
metricValue.YValue = resultValue.Value;
metricValue.CreatedDateTime = RockDateTime.Now;
metricValue.ModifiedDateTime = RockDateTime.Now;
metricValue.MetricValuePartitions = new List <MetricValuePartition>();
var metricPartitionsByPosition = metricPartitions.OrderBy(a => a.Order).ToList();
if (!resultValue.Partitions.Any() && metricPartitionsByPosition.Count() == 1 && !metricPartitionsByPosition[0].EntityTypeId.HasValue)
{
// a metric with just the default partition (not partitioned by Entity)
var metricPartition = metricPartitionsByPosition[0];
var metricValuePartition = new MetricValuePartition();
metricValuePartition.MetricPartition = metricPartition;
metricValuePartition.MetricPartitionId = metricPartition.Id;
metricValuePartition.MetricValue = metricValue;
metricValuePartition.EntityId = null;
metricValue.MetricValuePartitions.Add(metricValuePartition);
}
else
{
foreach (var partitionResult in resultValue.Partitions)
{
if (metricPartitionsByPosition.Count > partitionResult.PartitionPosition)
{
var metricPartition = metricPartitionsByPosition[partitionResult.PartitionPosition];
var metricValuePartition = new MetricValuePartition();
metricValuePartition.MetricPartition = metricPartition;
metricValuePartition.MetricPartitionId = metricPartition.Id;
metricValuePartition.MetricValue = metricValue;
metricValuePartition.EntityId = partitionResult.EntityId;
metricValue.MetricValuePartitions.Add(metricValuePartition);
}
}
}
if (metricValue.MetricValuePartitions == null || !metricValue.MetricValuePartitions.Any())
{
// shouldn't happen, but just in case
throw new Exception("MetricValue requires at least one Partition Value");
}
else
{
metricValuesToAdd.Add(metricValue);
}
}
// if a single metricValueDateTime was specified, delete any existing metric values for this date and refresh with the current results
var dbTransaction = rockContextForMetricValues.Database.BeginTransaction();
if (getMetricValueDateTimeFromResultSet)
{
var metricValueDateTimes = metricValuesToAdd.Select(a => a.MetricValueDateTime).Distinct().ToList();
foreach (var metricValueDateTime in metricValueDateTimes)
{
bool alreadyHasMetricValues = metricValueService.Queryable().Where(a => a.MetricId == metric.Id && a.MetricValueDateTime == metricValueDateTime).Any();
if (alreadyHasMetricValues)
{
// use direct SQL to clean up any existing metric values
rockContextForMetricValues.Database.ExecuteSqlCommand(@"
DELETE
FROM MetricValuePartition
WHERE MetricValueId IN (
SELECT Id
FROM MetricValue
WHERE MetricId = @metricId
AND MetricValueDateTime = @metricValueDateTime
)
", new SqlParameter("@metricId", metric.Id), new SqlParameter("@metricValueDateTime", metricValueDateTime));
rockContextForMetricValues.Database.ExecuteSqlCommand(@"
DELETE
FROM MetricValue
WHERE MetricId = @metricId
AND MetricValueDateTime = @metricValueDateTime
", new SqlParameter("@metricId", metric.Id), new SqlParameter("@metricValueDateTime", metricValueDateTime));
}
}
}
metricValueService.AddRange(metricValuesToAdd);
// disable savechanges PrePostProcessing since there could be hundreds or thousands of metric values getting inserted/updated
rockContextForMetricValues.SaveChanges(true);
dbTransaction.Commit();
}
rockContextForMetricEntity.SaveChanges();
}
}
}
}
catch (Exception ex)
{
metricExceptions.Add(new Exception($"Exception when calculating metric for {metric}: {ex.Message}", ex));
}
}
context.Result = string.Format("Calculated a total of {0} metric values for {1} metrics", metricValuesCalculated, metricsCalculated);
if (metricExceptions.Any())
{
throw new AggregateException("One or more metric calculations failed ", metricExceptions);
}
}
