/// <summary>
/// Persists the DataView to the database by updating the DataViewPersistedValues for this DataView. Returns true if successful
/// </summary>
/// <param name="databaseTimeoutSeconds">The database timeout seconds.</param>
public void PersistResult(int?databaseTimeoutSeconds = null)
{
List <string> errorMessages;
var dbContext = this.GetDbContext();
DataViewFilterOverrides dataViewFilterOverrides = new DataViewFilterOverrides();
// set an override so that the Persisted Values aren't used when rebuilding the values from the DataView Query
dataViewFilterOverrides.IgnoreDataViewPersistedValues.Add(this.Id);
var qry = this.GetQuery(null, dbContext, dataViewFilterOverrides, databaseTimeoutSeconds, out errorMessages);
if (!errorMessages.Any())
{
RockContext rockContext = dbContext as RockContext;
if (rockContext == null)
{
rockContext = new RockContext();
}
rockContext.Database.CommandTimeout = databaseTimeoutSeconds;
var savedDataViewPersistedValues = rockContext.DataViewPersistedValues.Where(a => a.DataViewId == this.Id);
var updatedEntityIdsQry = qry.Select(a => a.Id);
if (!(rockContext is RockContext))
{
// if this DataView doesn't use a RockContext get the EntityIds into memory as as a List<int> then back into IQueryable<int> so that we aren't use multiple dbContexts
updatedEntityIdsQry = updatedEntityIdsQry.ToList().AsQueryable();
}
var persistedValuesToRemove = savedDataViewPersistedValues.Where(a => !updatedEntityIdsQry.Any(x => x == a.EntityId));
var persistedEntityIdsToInsert = updatedEntityIdsQry.Where(x => !savedDataViewPersistedValues.Any(a => a.EntityId == x)).ToList();
int removeCount = persistedValuesToRemove.Count();
if (removeCount > 0)
{
// increase the batch size if there are a bunch of rows (and this is a narrow table with no references to it)
int?deleteBatchSize = removeCount > 50000 ? 25000 : ( int? )null;
int rowRemoved = rockContext.BulkDelete(persistedValuesToRemove, deleteBatchSize);
}
if (persistedEntityIdsToInsert.Any())
{
List <DataViewPersistedValue> persistedValuesToInsert = persistedEntityIdsToInsert.OrderBy(a => a)
.Select(a =>
new DataViewPersistedValue
{
DataViewId = this.Id,
EntityId = a
}).ToList();
rockContext.BulkInsert(persistedValuesToInsert);
}
}
}
public IQueryable GetQueryable(Type entityType, Dictionary <int, EntityField> entityFields, Dictionary <int, AttributeCache> attributes, Dictionary <int, ReportField> selectComponents, Rock.Web.UI.Controls.SortProperty sortProperty,
DataViewFilterOverrides dataViewFilterOverrides, int?databaseTimeoutSeconds, bool isCommunication, out List <string> errorMessages, out System.Data.Entity.DbContext reportDbContext)
{
reportDbContext = Reflection.GetDbContextForEntityType(entityType);
ReportGetQueryableArgs reportGetQueryableArgs = new ReportGetQueryableArgs
{
ReportDbContext = reportDbContext as Rock.Data.DbContext,
EntityFields = entityFields,
Attributes = attributes,
SelectComponents = selectComponents,
SortProperty = sortProperty,
DataViewFilterOverrides = dataViewFilterOverrides,
DatabaseTimeoutSeconds = databaseTimeoutSeconds,
IsCommunication = isCommunication
};
errorMessages = null;
return(GetQueryable(reportGetQueryableArgs));
}
/// <summary>
/// Gets the expression.
/// </summary>
/// <param name="dataViewEntityTypeType">Type of the data view entity type.</param>
/// <param name="serviceInstance">The service instance.</param>
/// <param name="parameter">The parameter.</param>
/// <param name="dataViewFilterOverrides">The data view filter overrides.</param>
/// <returns></returns>
/// <exception cref="Rock.Reporting.RockReportingException">
/// EntityTypeId not defined for {this}
/// or
/// Unable to determine EntityType not defined for EntityTypeId {EntityTypeId}
/// or
/// Unable to determine Component for EntityType {entityType.Name}
/// or
/// unable to determine expression for {filter}
/// or
/// Unexpected FilterExpressionType {ExpressionType}
/// </exception>
public Expression GetExpression(Type dataViewEntityTypeType, IService serviceInstance, ParameterExpression parameter, DataViewFilterOverrides dataViewFilterOverrides)
{
switch (ExpressionType)
{
case FilterExpressionType.Filter:
if (!this.EntityTypeId.HasValue)
{
// if this happens, we want to throw an exception to prevent incorrect results
throw new RockDataViewFilterExpressionException(this, $"EntityTypeId not defined for {this}");
}
var entityType = EntityTypeCache.Get(this.EntityTypeId.Value);
if (entityType == null)
{
// if this happens, we want to throw an exception to prevent incorrect results
throw new RockDataViewFilterExpressionException(this, $"Unable to determine EntityType not defined for EntityTypeId {EntityTypeId}");
}
var component = Rock.Reporting.DataFilterContainer.GetComponent(entityType.Name);
if (component == null)
{
// if this happens, we want to throw an exception to prevent incorrect results
throw new RockDataViewFilterExpressionException(this, $"Unable to determine Component for EntityType {entityType.Name}");
}
string selection; // A formatted string representing the filter settings: FieldName, <see cref="ComparisonType">Comparison Type</see>, (optional) Comparison Value(s)
var dataViewFilterOverride = dataViewFilterOverrides?.GetOverride(this.Guid);
if (dataViewFilterOverride != null)
{
if (dataViewFilterOverride.IncludeFilter == false)
{
/*
* 1/15/2021 - Shaun
* This should not assume that returning Expression.Constant( true ) is equivalent to not filtering as this predicate
* may be joined to other predicates and the AND/OR logic may result in an inappropriate filter. Instead, we simply
* return null and allow the caller to handle this in a manner appropriate to the given filter.
*/
// If the dataview filter should not be included, don't have this filter filter anything.
return(null);
}
else
{
selection = dataViewFilterOverride.Selection;
}
}
else
{
selection = this.Selection;
}
Expression expression;
try
{
if (component is IDataFilterWithOverrides)
{
expression = (component as IDataFilterWithOverrides).GetExpressionWithOverrides(dataViewEntityTypeType, serviceInstance, parameter, dataViewFilterOverrides, selection);
}
else
{
expression = component.GetExpression(dataViewEntityTypeType, serviceInstance, parameter, selection);
}
}
catch (RockDataViewFilterExpressionException dex)
{
// components don't know which DataView/DataFilter they are working with, so if there was a RockDataViewFilterExpressionException, let's tell it what DataViewFilter/DataView it was using
dex.SetDataFilterIfNotSet(this);
throw;
}
if (expression == null)
{
// If a DataFilter component returned a null expression, that probably means that it decided not to filter anything. So, we'll interpret that as "Don't Filter"
expression = Expression.Constant(true);
}
return(expression);
case FilterExpressionType.GroupAll:
case FilterExpressionType.GroupAnyFalse:
Expression andExp = null;
foreach (var filter in this.ChildFilters)
{
Expression exp = filter.GetExpression(dataViewEntityTypeType, serviceInstance, parameter, dataViewFilterOverrides);
if (exp == null)
{
// If a DataFilter component returned a null expression, that probably means that it decided not to filter anything. So, we'll interpret that as "Don't Filter"
exp = Expression.Constant(true);
}
if (andExp == null)
{
andExp = exp;
}
else
{
andExp = Expression.AndAlso(andExp, exp);
}
}
if (ExpressionType == FilterExpressionType.GroupAnyFalse &&
andExp != null)
{
// If only one of the conditions must be false, invert the expression so that it becomes the logical equivalent of "NOT ALL".
andExp = Expression.Not(andExp);
}
if (andExp == null)
{
// If there aren't any child filters for a GroupAll/GroupAnyFalse. That is OK, so just don't filter anything.
return(Expression.Constant(true));
}
return(andExp);
case FilterExpressionType.GroupAny:
case FilterExpressionType.GroupAllFalse:
Expression orExp = null;
foreach (DataViewFilter filter in this.ChildFilters)
{
Expression exp = filter.GetExpression(dataViewEntityTypeType, serviceInstance, parameter, dataViewFilterOverrides);
if (exp == null)
{
/*
* 1/15/2021 - Shaun
* Filter expressions of these types (GroupAny/GroupAllFalse) are joined with an OR clause,
* so they must either be defaulted to false or excluded from the where expression altogether
* (otherwise they will return every Person record in the database, because a "True OrElse
* <anything>" predicate will always be true).
*
* Therefore, if this child filter is null, we can simply ignore it and move on to the next one.
*
* Reason: Correcting behavior of dynamic reports where a group is deselected at run time.
*/
continue;
}
if (orExp == null)
{
orExp = exp;
}
else
{
orExp = Expression.OrElse(orExp, exp);
}
}
if (ExpressionType == FilterExpressionType.GroupAllFalse &&
orExp != null)
{
// If all of the conditions must be false, invert the expression so that it becomes the logical equivalent of "NOT ANY".
orExp = Expression.Not(orExp);
}
if (orExp == null)
{
// If there aren't any child filters for a GroupAny/GroupAllFalse. That is OK, so just don't filter anything.
return(Expression.Constant(true));
}
return(orExp);
default:
throw new RockDataViewFilterExpressionException(this, $"Unexpected FilterExpressionType {ExpressionType} ");
}
}
public virtual Expression GetExpression(Type filteredEntityType, IService serviceInstance, ParameterExpression parameter, DataViewFilterOverrides dataViewFilterOverrides, List <string> errorMessages)
{
return(GetExpression(filteredEntityType, serviceInstance, parameter, dataViewFilterOverrides));
}
/// <summary>
/// Gets the Linq expression for the DataViewFilter.
/// </summary>
/// <param name="filteredEntityType">Type of the filtered entity.</param>
/// <param name="serviceInstance">The service instance.</param>
/// <param name="parameter">The parameter.</param>
/// <param name="dataViewFilterOverrides">The data view filter overrides.</param>
/// <param name="errorMessages">The error messages.</param>
/// <returns></returns>
public virtual Expression GetExpression(Type filteredEntityType, IService serviceInstance, ParameterExpression parameter, DataViewFilterOverrides dataViewFilterOverrides, List <string> errorMessages)
{
switch (ExpressionType)
{
case FilterExpressionType.Filter:
if (this.EntityTypeId.HasValue)
{
var entityType = EntityTypeCache.Get(this.EntityTypeId.Value);
if (entityType != null)
{
var component = Rock.Reporting.DataFilterContainer.GetComponent(entityType.Name);
if (component != null)
{
try
{
string selection; // A formatted string representing the filter settings: FieldName, <see cref="ComparisonType">Comparison Type</see>, (optional) Comparison Value(s)
var dataViewFilterOverride = dataViewFilterOverrides?.GetOverride(this.Guid);
if (dataViewFilterOverride != null)
{
if (dataViewFilterOverride.IncludeFilter == false)
{
return(null);
}
else
{
selection = dataViewFilterOverride.Selection;
}
}
else
{
selection = this.Selection;
}
if (component is IDataFilterWithOverrides)
{
return((component as IDataFilterWithOverrides).GetExpressionWithOverrides(filteredEntityType, serviceInstance, parameter, dataViewFilterOverrides, selection));
}
else
{
return(component.GetExpression(filteredEntityType, serviceInstance, parameter, selection));
}
}
catch (SystemException ex)
{
ExceptionLogService.LogException(ex, System.Web.HttpContext.Current);
errorMessages.Add(string.Format("{0}: {1}", component.FormatSelection(filteredEntityType, this.Selection), ex.Message));
}
}
}
}
return(null);
case FilterExpressionType.GroupAll:
case FilterExpressionType.GroupAnyFalse:
Expression andExp = null;
foreach (var filter in this.ChildFilters)
{
Expression exp = filter.GetExpression(filteredEntityType, serviceInstance, parameter, dataViewFilterOverrides, errorMessages);
if (exp != null)
{
if (andExp == null)
{
andExp = exp;
}
else
{
andExp = Expression.AndAlso(andExp, exp);
}
}
}
if (ExpressionType == FilterExpressionType.GroupAnyFalse &&
andExp != null)
{
// If only one of the conditions must be false, invert the expression so that it becomes the logical equivalent of "NOT ALL".
andExp = Expression.Not(andExp);
}
return(andExp);
case FilterExpressionType.GroupAny:
case FilterExpressionType.GroupAllFalse:
Expression orExp = null;
foreach (var filter in this.ChildFilters)
{
Expression exp = filter.GetExpression(filteredEntityType, serviceInstance, parameter, dataViewFilterOverrides, errorMessages);
if (exp != null)
{
if (orExp == null)
{
orExp = exp;
}
else
{
orExp = Expression.OrElse(orExp, exp);
}
}
}
if (ExpressionType == FilterExpressionType.GroupAllFalse &&
orExp != null)
{
// If all of the conditions must be false, invert the expression so that it becomes the logical equivalent of "NOT ANY".
orExp = Expression.Not(orExp);
}
return(orExp);
}
return(null);
}
/// <summary>
/// Gets the queryable.
/// </summary>
/// <param name="entityType">Type of the entity.</param>
/// <param name="entityFields">The entity fields.</param>
/// <param name="attributes">The attributes.</param>
/// <param name="selectComponents">The select components.</param>
/// <param name="sortProperty">The sort property.</param>
/// <param name="dataViewFilterOverrides">The data view filter overrides.</param>
/// <param name="databaseTimeoutSeconds">The database timeout seconds.</param>
/// <param name="isCommunication">if set to <c>true</c> [is communication].</param>
/// <param name="errorMessages">The error messages.</param>
/// <param name="reportDbContext">The report database context.</param>
/// <returns></returns>
/// <exception cref="Exception"></exception>
public IQueryable GetQueryable(Type entityType, Dictionary <int, EntityField> entityFields, Dictionary <int, AttributeCache> attributes, Dictionary <int, ReportField> selectComponents, Rock.Web.UI.Controls.SortProperty sortProperty, DataViewFilterOverrides dataViewFilterOverrides, int?databaseTimeoutSeconds, bool isCommunication, out List <string> errorMessages, out System.Data.Entity.DbContext reportDbContext)
{
errorMessages = new List <string>();
reportDbContext = null;
if (entityType != null)
{
reportDbContext = Reflection.GetDbContextForEntityType(entityType);
IService serviceInstance = Reflection.GetServiceForEntityType(entityType, reportDbContext);
if (databaseTimeoutSeconds.HasValue)
{
reportDbContext.Database.CommandTimeout = databaseTimeoutSeconds.Value;
}
if (serviceInstance != null)
{
ParameterExpression paramExpression = serviceInstance.ParameterExpression;
MemberExpression idExpression = Expression.Property(paramExpression, "Id");
// Get AttributeValue queryable and parameter
var attributeValues = reportDbContext.Set <AttributeValue>();
ParameterExpression attributeValueParameter = Expression.Parameter(typeof(AttributeValue), "v");
// Create the dynamic type
var dynamicFields = new Dictionary <string, Type>();
dynamicFields.Add("Id", typeof(int));
foreach (var f in entityFields)
{
dynamicFields.Add(string.Format("Entity_{0}_{1}", f.Value.Name, f.Key), f.Value.PropertyType);
}
foreach (var a in attributes)
{
dynamicFields.Add(string.Format("Attribute_{0}_{1}", a.Value.Id, a.Key), a.Value.FieldType.Field.AttributeValueFieldType);
}
foreach (var reportField in selectComponents)
{
DataSelectComponent selectComponent = DataSelectContainer.GetComponent(reportField.Value.DataSelectComponentEntityType.Name);
if (selectComponent != null)
{
dynamicFields.Add(string.Format("Data_{0}_{1}", selectComponent.ColumnPropertyName, reportField.Key), selectComponent.ColumnFieldType);
var customSortProperties = selectComponent.SortProperties(reportField.Value.Selection);
if (customSortProperties != null)
{
foreach (var customSortProperty in customSortProperties.Split(','))
{
if (!string.IsNullOrWhiteSpace(customSortProperty))
{
var customSortPropertyType = entityType.GetPropertyType(customSortProperty);
dynamicFields.Add(string.Format("Sort_{0}_{1}", customSortProperty, reportField.Key), customSortPropertyType ?? typeof(string));
}
}
}
if (isCommunication && selectComponent is IRecipientDataSelect)
{
dynamicFields.Add($"Recipient_{selectComponent.ColumnPropertyName}_{reportField.Key}", ((IRecipientDataSelect)selectComponent).RecipientColumnFieldType);
}
}
}
if (dynamicFields.Count == 0)
{
errorMessages.Add("At least one field must be defined");
return(null);
}
Type dynamicType = LinqRuntimeTypeBuilder.GetDynamicType(dynamicFields);
ConstructorInfo methodFromHandle = dynamicType.GetConstructor(Type.EmptyTypes);
// Bind the dynamic fields to their expressions
var bindings = new List <MemberAssignment>();
bindings.Add(Expression.Bind(dynamicType.GetField("id"), idExpression));
foreach (var f in entityFields)
{
bindings.Add(Expression.Bind(dynamicType.GetField(string.Format("entity_{0}_{1}", f.Value.Name, f.Key)), Expression.Property(paramExpression, f.Value.Name)));
}
foreach (var a in attributes)
{
bindings.Add(Expression.Bind(dynamicType.GetField(string.Format("attribute_{0}_{1}", a.Value.Id, a.Key)), GetAttributeValueExpression(attributeValues, attributeValueParameter, idExpression, a.Value.Id)));
}
foreach (var reportField in selectComponents)
{
DataSelectComponent selectComponent = DataSelectContainer.GetComponent(reportField.Value.DataSelectComponentEntityType.Name);
if (selectComponent != null)
{
try
{
var componentExpression = selectComponent.GetExpression(reportDbContext, idExpression, reportField.Value.Selection ?? string.Empty);
if (componentExpression == null)
{
componentExpression = Expression.Constant(null, typeof(string));
}
bindings.Add(Expression.Bind(dynamicType.GetField(string.Format("data_{0}_{1}", selectComponent.ColumnPropertyName, reportField.Key)), componentExpression));
if (isCommunication && selectComponent is IRecipientDataSelect)
{
var recipientPersonIdExpression = ((IRecipientDataSelect)selectComponent).GetRecipientPersonIdExpression(reportDbContext, idExpression, reportField.Value.Selection ?? string.Empty);
if (recipientPersonIdExpression != null)
{
bindings.Add(Expression.Bind(dynamicType.GetField(string.Format("recipient_{0}_{1}", selectComponent.ColumnPropertyName, reportField.Key)), recipientPersonIdExpression));
}
}
var customSortProperties = selectComponent.SortProperties(reportField.Value.Selection);
if (!string.IsNullOrEmpty(customSortProperties))
{
foreach (var customSortProperty in customSortProperties.Split(','))
{
var customSortPropertyParts = customSortProperty.Split('.');
MemberInfo memberInfo = dynamicType.GetField(string.Format("sort_{0}_{1}", customSortProperty, reportField.Key));
Expression memberExpression = null;
foreach (var customSortPropertyPart in customSortPropertyParts)
{
memberExpression = Expression.Property(memberExpression ?? paramExpression, customSortPropertyPart);
}
bindings.Add(Expression.Bind(memberInfo, memberExpression));
}
}
}
catch (Exception ex)
{
throw new Exception(string.Format("Exception in {0}", selectComponent), ex);
}
}
}
ConstructorInfo constructorInfo = dynamicType.GetConstructor(Type.EmptyTypes);
NewExpression newExpression = Expression.New(constructorInfo);
MemberInitExpression memberInitExpression = Expression.MemberInit(newExpression, bindings);
LambdaExpression selector = Expression.Lambda(memberInitExpression, paramExpression);
// NOTE: having a NULL Dataview is OK.
Expression whereExpression = null;
if (this.DataView != null)
{
whereExpression = this.DataView.GetExpression(serviceInstance, paramExpression, dataViewFilterOverrides, out errorMessages);
}
MethodInfo getMethod = serviceInstance.GetType().GetMethod("Get", new Type[] { typeof(ParameterExpression), typeof(Expression), typeof(Rock.Web.UI.Controls.SortProperty), typeof(int?) });
if (getMethod != null)
{
var getResult = getMethod.Invoke(serviceInstance, new object[] { paramExpression, whereExpression, null, null });
var qry = getResult as IQueryable <IEntity>;
var qryExpression = qry.Expression;
// apply the OrderBy clauses to the Expression from whatever columns are specified in sortProperty.Property
string orderByMethod = "OrderBy";
if (sortProperty == null)
{
// if no sorting was specified, sort by Id
sortProperty = new Web.UI.Controls.SortProperty {
Direction = SortDirection.Ascending, Property = "Id"
};
}
/*
* NOTE: The sort property sorting rules can be a little confusing. Here is how it works:
* - SortProperty.Direction of Ascending means sort exactly as what the Columns specification says
* - SortProperty.Direction of Descending means sort the _opposite_ of what the Columns specification says
* Examples:
* 1) SortProperty.Property "LastName desc, FirstName, BirthDate desc" and SortProperty.Direction = Ascending
* OrderBy should be: "order by LastName desc, FirstName, BirthDate desc"
* 2) SortProperty.Property "LastName desc, FirstName, BirthDate desc" and SortProperty.Direction = Descending
* OrderBy should be: "order by LastName, FirstName desc, BirthDate"
*/
foreach (var column in sortProperty.Property.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries))
{
string propertyName;
var direction = sortProperty.Direction;
if (column.EndsWith(" desc", StringComparison.OrdinalIgnoreCase))
{
propertyName = column.Left(column.Length - 5);
// if the column ends with " desc", toggle the direction if sortProperty is Descending
direction = sortProperty.Direction == SortDirection.Ascending ? SortDirection.Descending : SortDirection.Ascending;
}
else
{
propertyName = column;
}
string methodName = direction == SortDirection.Descending ? orderByMethod + "Descending" : orderByMethod;
// Call OrderBy on whatever the Expression is for that Column
var sortMember = bindings.FirstOrDefault(a => a.Member.Name.Equals(propertyName, StringComparison.OrdinalIgnoreCase));
LambdaExpression sortSelector = Expression.Lambda(sortMember.Expression, paramExpression);
qryExpression = Expression.Call(typeof(Queryable), methodName, new Type[] { qry.ElementType, sortSelector.ReturnType }, qryExpression, sortSelector);
orderByMethod = "ThenBy";
}
var selectExpression = Expression.Call(typeof(Queryable), "Select", new Type[] { qry.ElementType, dynamicType }, qryExpression, selector);
var query = qry.Provider.CreateQuery(selectExpression).AsNoTracking();
// cast to a dynamic so that we can do a Queryable.Take (the compiler figures out the T in IQueryable at runtime)
dynamic dquery = query;
if (FetchTop.HasValue)
{
dquery = Queryable.Take(dquery, FetchTop.Value);
}
return(dquery as IQueryable);
}
}
}
return(null);
}
/// <summary>
/// Persists the DataView to the database by updating the DataViewPersistedValues for this DataView. Returns true if successful
/// </summary>
/// <param name="databaseTimeoutSeconds">The database timeout seconds.</param>
public void PersistResult(int?databaseTimeoutSeconds = null)
{
using (var dbContext = this.GetDbContext())
{
var persistStopwatch = Stopwatch.StartNew();
var dataViewFilterOverrides = new DataViewFilterOverrides();
dataViewFilterOverrides.ShouldUpdateStatics = false;
// set an override so that the Persisted Values aren't used when rebuilding the values from the DataView Query
dataViewFilterOverrides.IgnoreDataViewPersistedValues.Add(this.Id);
var dataViewGetQueryArgs = new DataViewGetQueryArgs
{
DbContext = dbContext,
DataViewFilterOverrides = dataViewFilterOverrides,
DatabaseTimeoutSeconds = databaseTimeoutSeconds,
};
var qry = this.GetQuery(dataViewGetQueryArgs);
RockContext rockContext = dbContext as RockContext;
if (rockContext == null)
{
rockContext = new RockContext();
}
rockContext.Database.CommandTimeout = databaseTimeoutSeconds;
var savedDataViewPersistedValues = rockContext.DataViewPersistedValues.Where(a => a.DataViewId == this.Id);
var updatedEntityIdsQry = qry.Select(a => a.Id);
if (!(rockContext is RockContext))
{
// if this DataView doesn't use a RockContext get the EntityIds into memory as a List<int> then back into IQueryable<int> so that we aren't use multiple dbContexts
updatedEntityIdsQry = updatedEntityIdsQry.ToList().AsQueryable();
}
var persistedValuesToRemove = savedDataViewPersistedValues.Where(a => !updatedEntityIdsQry.Any(x => x == a.EntityId));
var persistedEntityIdsToInsert = updatedEntityIdsQry.Where(x => !savedDataViewPersistedValues.Any(a => a.EntityId == x)).ToList();
var removeCount = persistedValuesToRemove.Count();
if (removeCount > 0)
{
// increase the batch size if there are a bunch of rows (and this is a narrow table with no references to it)
int?deleteBatchSize = removeCount > 50000 ? 25000 : ( int? )null;
int rowRemoved = rockContext.BulkDelete(persistedValuesToRemove, deleteBatchSize);
}
if (persistedEntityIdsToInsert.Any())
{
List <DataViewPersistedValue> persistedValuesToInsert = persistedEntityIdsToInsert.OrderBy(a => a)
.Select(a =>
new DataViewPersistedValue
{
DataViewId = this.Id,
EntityId = a
}).ToList();
rockContext.BulkInsert(persistedValuesToInsert);
}
persistStopwatch.Stop();
// Update the Persisted Refresh information.
PersistedLastRefreshDateTime = RockDateTime.Now;
PersistedLastRunDurationMilliseconds = Convert.ToInt32(persistStopwatch.Elapsed.TotalMilliseconds);
}
}
/// <summary>
/// Gets the expression.
/// </summary>
/// <param name="serviceInstance">The service instance.</param>
/// <param name="paramExpression">The parameter expression.</param>
/// <param name="dataViewFilterOverrides">The data view filter overrides.</param>
/// <returns></returns>
/// <exception cref="Rock.Reporting.RockReportingException">
/// Unable to determine Assembly for EntityTypeId { EntityTypeId }
/// or
/// Unable to determine DataView EntityType for { dataViewEntityTypeCache }.
/// or
/// Unable to determine transform expression for TransformEntityTypeId: {TransformEntityTypeId}
/// </exception>
public Expression GetExpression(IService serviceInstance, ParameterExpression paramExpression, DataViewFilterOverrides dataViewFilterOverrides)
{
var dataViewEntityTypeCache = EntityTypeCache.Get(EntityTypeId.Value);
if (dataViewEntityTypeCache?.AssemblyName == null)
{
throw new RockDataViewFilterExpressionException(this.DataViewFilter, $"Unable to determine DataView Assembly for EntityTypeId { EntityTypeId }");
}
Type dataViewEntityTypeType = dataViewEntityTypeCache.GetEntityType();
if (dataViewEntityTypeType == null)
{
throw new RockDataViewFilterExpressionException(this.DataViewFilter, $"Unable to determine DataView EntityType for { dataViewEntityTypeCache }.");
}
// DataViews must have a DataViewFilter (something has gone wrong it doesn't have one)
// Note that DataViewFilterId might be null even though DataViewFilter is not null
// This is because the DataViewFilter might be just in memory and not saved to the database (for example, a Preview or a DynamicReport)
if (this.DataViewFilter == null)
{
throw new RockDataViewFilterExpressionException(this.DataViewFilter, $"DataViewFilter is null for DataView { this.Name } ({this.Id}).");
}
bool usePersistedValues = this.PersistedScheduleIntervalMinutes.HasValue && this.PersistedLastRefreshDateTime.HasValue;
if (dataViewFilterOverrides != null)
{
// don't use persisted values if this DataView in the list of DataViews that should not be persisted due to override
usePersistedValues = usePersistedValues && !dataViewFilterOverrides.IgnoreDataViewPersistedValues.Contains(this.Id);
}
// If dataViewFilterOverrides is null assume true in order to preserve current functionality.
RockLogger.Log.Debug(RockLogDomains.Reporting, "{methodName} dataViewFilterOverrides: {@dataViewFilterOverrides} DataviewId: {DataviewId}", nameof(GetExpression), dataViewFilterOverrides, DataViewFilter.DataViewId);
if (dataViewFilterOverrides == null || dataViewFilterOverrides.ShouldUpdateStatics)
{
DataViewService.AddRunDataViewTransaction(Id);
}
// We need to call GetExpression regardless of whether or not usePresistedValues is true so the child queries get their stats updated.
var filterExpression = DataViewFilter != null?DataViewFilter.GetExpression(dataViewEntityTypeType, serviceInstance, paramExpression, dataViewFilterOverrides) : null;
if (usePersistedValues)
{
// If this is a persisted DataView, get the ids for the expression by querying DataViewPersistedValue instead of evaluating all the filters
var rockContext = serviceInstance.Context as RockContext;
if (rockContext == null)
{
rockContext = new RockContext();
}
var persistedValuesQuery = rockContext.DataViewPersistedValues.Where(a => a.DataViewId == this.Id);
var ids = persistedValuesQuery.Select(v => v.EntityId);
MemberExpression propertyExpression = Expression.Property(paramExpression, "Id");
if (!(serviceInstance.Context is RockContext))
{
// if this DataView doesn't use a RockContext get the EntityIds into memory as a List<int> then back into IQueryable<int> so that we aren't use multiple dbContexts
ids = ids.ToList().AsQueryable();
}
var idsExpression = Expression.Constant(ids.AsQueryable(), typeof(IQueryable <int>));
Expression expression = Expression.Call(typeof(Queryable), "Contains", new Type[] { typeof(int) }, idsExpression, propertyExpression);
return(expression);
}
else
{
if (dataViewEntityTypeCache.Id == EntityTypeCache.Get(typeof(Rock.Model.Person)).Id)
{
var qry = new PersonService(( RockContext )serviceInstance.Context).Queryable(this.IncludeDeceased);
Expression extractedFilterExpression = FilterExpressionExtractor.Extract <Rock.Model.Person>(qry, paramExpression, "p");
if (filterExpression == null)
{
filterExpression = extractedFilterExpression;
}
else
{
filterExpression = Expression.AndAlso(filterExpression, extractedFilterExpression);
}
}
if (this.TransformEntityTypeId.HasValue)
{
Expression transformedExpression = GetTransformExpression(this.TransformEntityTypeId.Value, serviceInstance, paramExpression, filterExpression);
if (transformedExpression == null)
{
// if TransformEntityTypeId is defined, but we got null back, we'll get unexpected results, so throw an exception
throw new RockDataViewFilterExpressionException(this.DataViewFilter, $"Unable to determine transform expression for TransformEntityTypeId: {TransformEntityTypeId}");
}
return(transformedExpression);
}
return(filterExpression);
}
}
public Expression GetExpression(IService serviceInstance, ParameterExpression paramExpression, DataViewFilterOverrides dataViewFilterOverrides, out List <string> errorMessages)
{
errorMessages = new List <string>();
return(GetExpression(serviceInstance, paramExpression, dataViewFilterOverrides));
}
/// <summary>
/// Gets the expression.
/// </summary>
/// <param name="serviceInstance">The service instance.</param>
/// <param name="paramExpression">The parameter expression.</param>
/// <param name="dataViewFilterOverrides">The data view filter overrides.</param>
/// <param name="errorMessages">The error messages.</param>
/// <returns></returns>
public Expression GetExpression(IService serviceInstance, ParameterExpression paramExpression, DataViewFilterOverrides dataViewFilterOverrides, out List <string> errorMessages)
{
errorMessages = new List <string>();
var cachedEntityType = EntityTypeCache.Read(EntityTypeId.Value);
if (cachedEntityType != null && cachedEntityType.AssemblyName != null)
{
Type filteredEntityType = cachedEntityType.GetEntityType();
if (filteredEntityType != null)
{
Expression filterExpression = DataViewFilter != null?DataViewFilter.GetExpression(filteredEntityType, serviceInstance, paramExpression, dataViewFilterOverrides, errorMessages) : null;
Expression transformedExpression = GetTransformExpression(serviceInstance, paramExpression, filterExpression, errorMessages);
if (transformedExpression != null)
{
return(transformedExpression);
}
return(filterExpression);
}
}
return(null);
}
/// <summary>
/// Gets the query.
/// </summary>
/// <param name="sortProperty">The sort property.</param>
/// <param name="dbContext">The database context.</param>
/// <param name="dataViewFilterOverrides">The data view filter overrides.</param>
/// <param name="databaseTimeoutSeconds">The database timeout seconds.</param>
/// <param name="errorMessages">The error messages.</param>
/// <returns></returns>
public IQueryable <IEntity> GetQuery(SortProperty sortProperty, System.Data.Entity.DbContext dbContext, DataViewFilterOverrides dataViewFilterOverrides, int?databaseTimeoutSeconds, out List <string> errorMessages)
{
errorMessages = new List <string>();
if (EntityTypeId.HasValue)
{
var cachedEntityType = EntityTypeCache.Read(EntityTypeId.Value);
if (cachedEntityType != null && cachedEntityType.AssemblyName != null)
{
Type entityType = cachedEntityType.GetEntityType();
if (entityType != null)
{
if (dbContext == null)
{
dbContext = Reflection.GetDbContextForEntityType(entityType);
}
if (databaseTimeoutSeconds.HasValue)
{
dbContext.Database.CommandTimeout = databaseTimeoutSeconds.Value;
}
IService serviceInstance = Reflection.GetServiceForEntityType(entityType, dbContext);
if (serviceInstance != null)
{
ParameterExpression paramExpression = serviceInstance.ParameterExpression;
Expression whereExpression = GetExpression(serviceInstance, paramExpression, dataViewFilterOverrides, out errorMessages);
MethodInfo getMethod = serviceInstance.GetType().GetMethod("Get", new Type[] { typeof(ParameterExpression), typeof(Expression), typeof(SortProperty) });
if (getMethod != null)
{
if (sortProperty == null)
{
// if no sorting is specified, just sort by Id
sortProperty = new SortProperty {
Direction = SortDirection.Ascending, Property = "Id"
};
}
var getResult = getMethod.Invoke(serviceInstance, new object[] { paramExpression, whereExpression, sortProperty });
var qry = getResult as IQueryable <IEntity>;
return(qry);
}
}
}
}
}
return(null);
}
/// <summary>
/// Gets the expression.
/// </summary>
/// <param name="serviceInstance">The service instance.</param>
/// <param name="paramExpression">The parameter expression.</param>
/// <param name="dataViewFilterOverrides">The data view filter overrides.</param>
/// <param name="errorMessages">The error messages.</param>
/// <returns></returns>
public Expression GetExpression(IService serviceInstance, ParameterExpression paramExpression, DataViewFilterOverrides dataViewFilterOverrides, out List <string> errorMessages)
{
errorMessages = new List <string>();
var cachedEntityType = EntityTypeCache.Get(EntityTypeId.Value);
if (cachedEntityType?.AssemblyName == null)
{
return(null);
}
Type filteredEntityType = cachedEntityType.GetEntityType();
if (filteredEntityType == null)
{
return(null);
}
bool usePersistedValues = this.PersistedScheduleIntervalMinutes.HasValue && this.PersistedLastRefreshDateTime.HasValue;
if (dataViewFilterOverrides != null)
{
// don't use persisted values if this dataview is in the list of dataviews that should not be persisted due to override
usePersistedValues = usePersistedValues && !dataViewFilterOverrides.IgnoreDataViewPersistedValues.Contains(this.Id);
}
DataViewService.AddRunDataViewTransaction(Id);
if (usePersistedValues)
{
// If this is a persisted dataview, get the ids for the expression by querying DataViewPersistedValue instead of evaluating all the filters
var rockContext = serviceInstance.Context as RockContext;
if (rockContext == null)
{
rockContext = new RockContext();
}
var persistedValuesQuery = rockContext.DataViewPersistedValues.Where(a => a.DataViewId == this.Id);
var ids = persistedValuesQuery.Select(v => v.EntityId);
MemberExpression propertyExpression = Expression.Property(paramExpression, "Id");
if (!(serviceInstance.Context is RockContext))
{
// if this DataView doesn't use a RockContext get the EntityIds into memory as as a List<int> then back into IQueryable<int> so that we aren't use multiple dbContexts
ids = ids.ToList().AsQueryable();
}
var idsExpression = Expression.Constant(ids.AsQueryable(), typeof(IQueryable <int>));
Expression expression = Expression.Call(typeof(Queryable), "Contains", new Type[] { typeof(int) }, idsExpression, propertyExpression);
return(expression);
}
else
{
Expression filterExpression = DataViewFilter != null?DataViewFilter.GetExpression(filteredEntityType, serviceInstance, paramExpression, dataViewFilterOverrides, errorMessages) : null;
if (cachedEntityType.Id == EntityTypeCache.Get(typeof(Rock.Model.Person)).Id)
{
var qry = new PersonService(( RockContext )serviceInstance.Context).Queryable(this.IncludeDeceased);
Expression extractedFilterExpression = FilterExpressionExtractor.Extract <Rock.Model.Person>(qry, paramExpression, "p");
if (filterExpression == null)
{
filterExpression = extractedFilterExpression;
}
else
{
filterExpression = Expression.AndAlso(filterExpression, extractedFilterExpression);
}
}
Expression transformedExpression = GetTransformExpression(serviceInstance, paramExpression, filterExpression, errorMessages);
if (transformedExpression != null)
{
return(transformedExpression);
}
return(filterExpression);
}
}
/// <summary>
/// Gets the expression.
/// </summary>
/// <param name="serviceInstance">The service instance.</param>
/// <param name="paramExpression">The parameter expression.</param>
/// <param name="dataViewFilterOverrides">The data view filter overrides.</param>
/// <param name="errorMessages">The error messages.</param>
/// <returns></returns>
public Expression GetExpression(IService serviceInstance, ParameterExpression paramExpression, DataViewFilterOverrides dataViewFilterOverrides, out List <string> errorMessages)
{
errorMessages = new List <string>();
var cachedEntityType = EntityTypeCache.Get(EntityTypeId.Value);
if (cachedEntityType != null && cachedEntityType.AssemblyName != null)
{
Type filteredEntityType = cachedEntityType.GetEntityType();
if (filteredEntityType != null)
{
bool usePersistedValues = this.PersistedScheduleIntervalMinutes.HasValue && this.PersistedLastRefreshDateTime.HasValue;
if (dataViewFilterOverrides != null)
{
usePersistedValues = usePersistedValues && !dataViewFilterOverrides.IgnoreDataViewPersistedValues.Contains(this.Id);
}
// don't use persisted values if there are dataViewFilterOverrides
if (usePersistedValues && dataViewFilterOverrides?.Any() == true)
{
usePersistedValues = false;
}
if (usePersistedValues)
{
// If this is a persisted dataview, get the ids for the expression by querying DataViewPersistedValue instead of evaluating all the filters
var rockContext = serviceInstance.Context as RockContext;
if (rockContext == null)
{
rockContext = new RockContext();
}
var persistedValuesQuery = rockContext.DataViewPersistedValues.Where(a => a.DataViewId == this.Id);
var ids = persistedValuesQuery.Select(v => v.EntityId);
MemberExpression propertyExpression = Expression.Property(paramExpression, "Id");
if (!(serviceInstance.Context is RockContext))
{
// if this DataView doesn't use a RockContext get the EntityIds into memory as as a List<int> then back into IQueryable<int> so that we aren't use multiple dbContexts
ids = ids.ToList().AsQueryable();
}
var idsExpression = Expression.Constant(ids.AsQueryable(), typeof(IQueryable <int>));
Expression expression = Expression.Call(typeof(Queryable), "Contains", new Type[] { typeof(int) }, idsExpression, propertyExpression);
return(expression);
}
else
{
Expression filterExpression = DataViewFilter != null?DataViewFilter.GetExpression(filteredEntityType, serviceInstance, paramExpression, dataViewFilterOverrides, errorMessages) : null;
Expression transformedExpression = GetTransformExpression(serviceInstance, paramExpression, filterExpression, errorMessages);
if (transformedExpression != null)
{
return(transformedExpression);
}
return(filterExpression);
}
}
}
return(null);
}
public IQueryable <IEntity> GetQuery(SortProperty sortProperty, System.Data.Entity.DbContext dbContext, DataViewFilterOverrides dataViewFilterOverrides, int?databaseTimeoutSeconds, out List <string> errorMessages)
{
errorMessages = new List <string>();
return(GetQuery(new DataViewGetQueryArgs
{
DbContext = dbContext,
DataViewFilterOverrides = dataViewFilterOverrides,
DatabaseTimeoutSeconds = databaseTimeoutSeconds,
SortProperty = sortProperty
}));
}