private static EntityMetadata SchemaToMetadata(INodeSchema schema, string alias)
{
var metadata = new EntityMetadata
{
LogicalName = alias,
DataProviderId = ProviderId
};
var attributes = schema.Schema
.Where(kvp => kvp.Key.StartsWith(alias + "."))
.Select(kvp =>
{
var attr = CreateAttribute(kvp.Value);
attr.LogicalName = kvp.Key.Substring(alias.Length + 1);
return(attr);
})
.ToArray();
SetSealedProperty(metadata, nameof(metadata.Attributes), attributes);
SetSealedProperty(metadata, nameof(metadata.OneToManyRelationships), CreateOneToManyRelationships("metadata." + alias));
SetSealedProperty(metadata, nameof(metadata.ManyToOneRelationships), CreateManyToOneRelationships("metadata." + alias));
SetSealedProperty(metadata, nameof(metadata.ManyToManyRelationships), Array.Empty <ManyToManyRelationshipMetadata>());
return(metadata);
}
protected void InitializeAggregates(INodeSchema schema, IDictionary <string, Type> parameterTypes)
{
foreach (var aggregate in Aggregates.Where(agg => agg.Value.SqlExpression != null))
{
var sourceExpression = aggregate.Value.SqlExpression;
// Sum and Average need to have Decimal values as input for their calculations to work correctly
if (aggregate.Value.AggregateType == AggregateType.Average || aggregate.Value.AggregateType == AggregateType.Sum)
{
sourceExpression = new ConvertCall {
Parameter = sourceExpression, DataType = new SqlDataTypeReference {
SqlDataTypeOption = SqlDataTypeOption.Decimal
}
}
}
;
aggregate.Value.Expression = sourceExpression.Compile(schema, parameterTypes);
aggregate.Value.ReturnType = aggregate.Value.SqlExpression.GetType(schema, null, parameterTypes);
if (aggregate.Value.AggregateType == AggregateType.Average)
{
if (aggregate.Value.ReturnType == typeof(SqlByte) || aggregate.Value.ReturnType == typeof(SqlInt16))
{
aggregate.Value.ReturnType = typeof(SqlInt32);
}
}
}
}
protected void InitializePartitionedAggregates(INodeSchema schema, IDictionary <string, Type> parameterTypes)
{
foreach (var aggregate in Aggregates)
{
var sourceExpression = aggregate.Key.ToColumnReference();
aggregate.Value.Expression = sourceExpression.Compile(schema, parameterTypes);
aggregate.Value.ReturnType = sourceExpression.GetType(schema, null, parameterTypes);
}
}
protected List <string> GetGroupingColumns(INodeSchema schema)
{
var groupByCols = GroupBy
.Select(col =>
{
var colName = col.GetColumnName();
schema.ContainsColumn(colName, out colName);
return(colName);
})
.ToList();
return(groupByCols);
}
/// <summary>
/// Creates the output record by merging the records that have been matched from the left and right sources
/// </summary>
/// <param name="leftEntity">The data from the left source</param>
/// <param name="leftSchema">The schema of the left source</param>
/// <param name="rightEntity">The data from the right source</param>
/// <param name="rightSchema">The schema of the right source</param>
/// <returns>The merged data</returns>
protected Entity Merge(Entity leftEntity, INodeSchema leftSchema, Entity rightEntity, INodeSchema rightSchema)
{
var merged = new Entity();
if (leftEntity != null)
{
foreach (var attr in leftEntity.Attributes)
{
merged[attr.Key] = attr.Value;
}
}
else
{
foreach (var attr in leftSchema.Schema)
{
merged[attr.Key] = SqlTypeConverter.GetNullValue(attr.Value);
}
}
if (rightEntity != null)
{
foreach (var attr in rightEntity.Attributes)
{
merged[attr.Key] = attr.Value;
}
}
else
{
foreach (var attr in rightSchema.Schema)
{
merged[attr.Key] = SqlTypeConverter.GetNullValue(attr.Value);
}
}
foreach (var definedValue in DefinedValues)
{
if (rightEntity == null)
{
merged[definedValue.Key] = SqlTypeConverter.GetNullValue(rightSchema.Schema[definedValue.Value]);
}
else
{
merged[definedValue.Key] = rightEntity[definedValue.Value];
}
}
return(merged);
}
/// <summary>
/// Creates a new <see cref="NodeSchema"/> as a copy of the supplied schema
/// </summary>
/// <param name="copy">The schema to copy</param>
public NodeSchema(INodeSchema copy)
{
PrimaryKey = copy.PrimaryKey;
foreach (var kvp in copy.Schema)
{
Schema[kvp.Key] = kvp.Value;
}
foreach (var kvp in copy.Aliases)
{
Aliases[kvp.Key] = new List <string>(kvp.Value);
}
SortOrder.AddRange(copy.SortOrder);
}
public string CreateScript(INodeSchema node)
{
StringBuilder sb = new StringBuilder();
sb.AppendLine($"CREATE TABLE {node.Name} (");
sb.AppendLine($"ID INTEGER PRIMARY KEY,");
sb.AppendLine(new AttributeSchemaScript().CreateScript(node.Attributes));
sb.AppendLine(") AS NODE;");
var s = new EdgeSchemaScript(node);
foreach (var edge in node.Edges)
{
sb.AppendLine(s.CreateScript(edge));
}
return(sb.ToString());
}
private void SortSubset(List <Entity> subset, INodeSchema schema, IDictionary <string, Type> parameterTypes, IDictionary <string, object> parameterValues, List <Func <Entity, IDictionary <string, object>, IQueryExecutionOptions, object> > expressions, IQueryExecutionOptions options)
{
// Simple case if there's no need to do any further sorting
if (subset.Count <= 1)
{
return;
}
// Precalculate the sort keys for the remaining sorts
var sortKeys = subset
.ToDictionary(entity => entity, entity => expressions.Skip(PresortedCount).Select(expr => expr(entity, parameterValues, options)).ToList());
// Sort the list according to these sort keys
subset.Sort((x, y) =>
{
var xValues = sortKeys[x];
var yValues = sortKeys[y];
for (var i = 0; i < Sorts.Count - PresortedCount; i++)
{
var comparison = ((IComparable)xValues[i]).CompareTo(yValues[i]);
if (comparison == 0)
{
continue;
}
if (Sorts[PresortedCount + i].SortOrder == SortOrder.Descending)
{
return(-comparison);
}
return(comparison);
}
return(0);
});
}
private IDictionary <string, Type> GetInnerParameterTypes(INodeSchema leftSchema, IDictionary <string, Type> parameterTypes)
{
var innerParameterTypes = parameterTypes;
if (OuterReferences != null)
{
if (parameterTypes == null)
{
innerParameterTypes = new Dictionary <string, Type>();
}
else
{
innerParameterTypes = new Dictionary <string, Type>(parameterTypes);
}
foreach (var kvp in OuterReferences)
{
innerParameterTypes[kvp.Value] = leftSchema.Schema[kvp.Key];
}
}
return(innerParameterTypes);
}
public EdgeSchemaScript(INodeSchema parent)
{
_parent = parent;
}
private void OnRetrievedEntity(Entity entity, INodeSchema schema, IQueryExecutionOptions options, IAttributeMetadataCache metadata)
{
// Expose any formatted values for OptionSetValue and EntityReference values
foreach (var formatted in entity.FormattedValues)
{
if (!entity.Contains(formatted.Key + "name"))
{
entity[formatted.Key + "name"] = formatted.Value;
}
}
if (options.UseLocalTimeZone)
{
// For any datetime values, check the metadata to see if they are affected by timezones and convert them
foreach (var attribute in entity.Attributes.ToList())
{
var entityName = entity.LogicalName;
var attributeName = attribute.Key;
var value = attribute.Value;
if (value is AliasedValue alias)
{
entityName = alias.EntityLogicalName;
attributeName = alias.AttributeLogicalName;
value = alias.Value;
}
if (value is DateTime dt)
{
var meta = metadata[entityName];
var attrMeta = (DateTimeAttributeMetadata)meta.Attributes.Single(a => a.LogicalName == attributeName);
if (attrMeta.DateTimeBehavior == DateTimeBehavior.UserLocal)
{
dt = dt.ToLocalTime();
entity[attribute.Key] = dt;
}
}
}
}
// Prefix all attributes of the main entity with the expected alias
foreach (var attribute in entity.Attributes.Where(attr => !attr.Key.Contains('.') && !(attr.Value is AliasedValue)).ToList())
{
entity[$"{Alias}.{attribute.Key}"] = attribute.Value;
}
// Only prefix aliased values if they're not aggregates
PrefixAliasedScalarAttributes(entity, Entity.Items, Alias);
// Convert aliased values to the underlying value
foreach (var attribute in entity.Attributes.Where(attr => attr.Value is AliasedValue).ToList())
{
var aliasedValue = (AliasedValue)attribute.Value;
// When grouping by EntityName attributes the value is converted from the normal string value to an OptionSetValue
// Convert it back now for consistency
if (_entityNameGroupings.Contains(attribute.Key))
{
int otc;
if (aliasedValue.Value is OptionSetValue osv)
{
otc = osv.Value;
}
else if (aliasedValue.Value is int i)
{
otc = i;
}
else
{
throw new QueryExecutionException($"Expected ObjectTypeCode value, got {aliasedValue.Value} ({aliasedValue.Value?.GetType()})");
}
var meta = metadata[otc];
entity[attribute.Key] = meta.LogicalName;
}
else
{
entity[attribute.Key] = aliasedValue.Value;
}
}
// Copy any grouped values to their full names
if (FetchXml.aggregateSpecified && FetchXml.aggregate)
{
if (Entity.Items != null)
{
foreach (var attr in Entity.Items.OfType <FetchAttributeType>().Where(a => a.groupbySpecified && a.groupby == FetchBoolType.@true))
{
if (entity.Attributes.TryGetValue(attr.alias, out var value))
{
entity[$"{Alias}.{attr.alias}"] = value;
}
}
}
foreach (var linkEntity in Entity.GetLinkEntities().Where(le => le.Items != null))
{
foreach (var attr in linkEntity.Items.OfType <FetchAttributeType>().Where(a => a.groupbySpecified && a.groupby == FetchBoolType.@true))
{
if (entity.Attributes.TryGetValue(attr.alias, out var value))
{
entity[$"{linkEntity.alias}.{attr.alias}"] = value;
}
}
}
}
// Expose the type of lookup values
foreach (var attribute in entity.Attributes.Where(attr => attr.Value is EntityReference).ToList())
{
if (!entity.Contains(attribute.Key + "type"))
{
entity[attribute.Key + "type"] = ((EntityReference)attribute.Value).LogicalName;
}
//entity[attribute.Key] = ((EntityReference)attribute.Value).Id;
}
// Convert values to SQL types
foreach (var col in schema.Schema)
{
object sqlValue;
if (entity.Attributes.TryGetValue(col.Key, out var value) && value != null)
{
sqlValue = SqlTypeConverter.NetToSqlType(DataSource, value);
}
else
{
sqlValue = SqlTypeConverter.GetNullValue(col.Value);
}
if (_primaryKeyColumns.TryGetValue(col.Key, out var logicalName) && sqlValue is SqlGuid guid)
{
sqlValue = new SqlEntityReference(DataSource, logicalName, guid);
}
entity[col.Key] = sqlValue;
}
}
private void AddSchemaColumn(string outputColumn, string sourceColumn, NodeSchema schema, INodeSchema sourceSchema)
{
if (!sourceSchema.ContainsColumn(sourceColumn, out var normalized))
{
return;
}
var mapped = $"{Alias}.{outputColumn}";
schema.Schema[mapped] = sourceSchema.Schema[normalized];
if (normalized == sourceSchema.PrimaryKey)
{
schema.PrimaryKey = mapped;
}
if (!schema.Aliases.TryGetValue(outputColumn, out var aliases))
{
aliases = new List <string>();
schema.Aliases[outputColumn] = aliases;
}
aliases.Add(mapped);
var sorted = sourceSchema.SortOrder.Contains(sourceColumn, StringComparer.OrdinalIgnoreCase);
if (sorted)
{
schema.SortOrder.Add(outputColumn);
}
}
public JoinConditionVisitor(INodeSchema lhs, INodeSchema rhs)
{
_lhs = lhs;
_rhs = rhs;
}
/// <summary>
/// Compiles methods to access the data required for the DML operation
/// </summary>
/// <param name="mappings">The mappings of attribute name to source column</param>
/// <param name="schema">The schema of data source</param>
/// <param name="attributes">The attributes in the target metadata</param>
/// <param name="dateTimeKind">The time zone that datetime values are supplied in</param>
/// <returns></returns>
protected Dictionary <string, Func <Entity, object> > CompileColumnMappings(EntityMetadata metadata, IDictionary <string, string> mappings, INodeSchema schema, IDictionary <string, AttributeMetadata> attributes, DateTimeKind dateTimeKind)
{
var attributeAccessors = new Dictionary <string, Func <Entity, object> >();
var entityParam = Expression.Parameter(typeof(Entity));
foreach (var mapping in mappings)
{
var sourceColumnName = mapping.Value;
var destAttributeName = mapping.Key;
if (!schema.ContainsColumn(sourceColumnName, out sourceColumnName))
{
throw new QueryExecutionException($"Missing source column {mapping.Value}")
{
Node = this
}
}
;
// We might be using a virtual ___type attribute that has a different name in the metadata. We can safely
// ignore these attributes - the attribute names have already been validated in the ExecutionPlanBuilder
if (!attributes.TryGetValue(destAttributeName, out var attr) || attr.AttributeOf != null)
{
continue;
}
var sourceType = schema.Schema[sourceColumnName];
var destType = attr.GetAttributeType();
var destSqlType = SqlTypeConverter.NetToSqlType(destType);
var expr = (Expression)Expression.Property(entityParam, typeof(Entity).GetCustomAttribute <DefaultMemberAttribute>().MemberName, Expression.Constant(sourceColumnName));
var originalExpr = expr;
if (sourceType == typeof(object))
{
// null literal
expr = Expression.Constant(null, destType);
expr = Expr.Box(expr);
}
else
{
expr = SqlTypeConverter.Convert(expr, sourceType);
expr = SqlTypeConverter.Convert(expr, destSqlType);
var convertedExpr = SqlTypeConverter.Convert(expr, destType);
if (attr is LookupAttributeMetadata lookupAttr && lookupAttr.AttributeType != AttributeTypeCode.PartyList)
{
// Special case: intersect attributes can be simple guids
if (metadata.IsIntersect != true)
{
if (sourceType == typeof(SqlEntityReference))
{
expr = SqlTypeConverter.Convert(originalExpr, sourceType);
convertedExpr = SqlTypeConverter.Convert(expr, typeof(EntityReference));
}
else
{
Expression targetExpr;
if (lookupAttr.Targets.Length == 1)
{
targetExpr = Expression.Constant(lookupAttr.Targets[0]);
}
else
{
var sourceTargetColumnName = mappings[destAttributeName + "type"];
var sourceTargetType = schema.Schema[sourceTargetColumnName];
targetExpr = Expression.Property(entityParam, typeof(Entity).GetCustomAttribute <DefaultMemberAttribute>().MemberName, Expression.Constant(sourceTargetColumnName));
targetExpr = SqlTypeConverter.Convert(targetExpr, sourceTargetType);
targetExpr = SqlTypeConverter.Convert(targetExpr, typeof(SqlString));
targetExpr = SqlTypeConverter.Convert(targetExpr, typeof(string));
}
convertedExpr = Expression.New(
typeof(EntityReference).GetConstructor(new[] { typeof(string), typeof(Guid) }),
targetExpr,
Expression.Convert(convertedExpr, typeof(Guid))
);
}
destType = typeof(EntityReference);
}
}
else if (attr is EnumAttributeMetadata && !(attr is MultiSelectPicklistAttributeMetadata))
{
convertedExpr = Expression.New(
typeof(OptionSetValue).GetConstructor(new[] { typeof(int) }),
Expression.Convert(convertedExpr, typeof(int))
);
destType = typeof(OptionSetValue);
}
else if (attr is MoneyAttributeMetadata)
{
convertedExpr = Expression.New(
typeof(Money).GetConstructor(new[] { typeof(decimal) }),
Expression.Convert(expr, typeof(decimal))
);
destType = typeof(Money);
}
else if (attr is DateTimeAttributeMetadata)
{
convertedExpr = Expression.Convert(
Expr.Call(() => DateTime.SpecifyKind(Expr.Arg <DateTime>(), Expr.Arg <DateTimeKind>()),
expr,
Expression.Constant(dateTimeKind)
),
typeof(DateTime?)
);
}
// Check for null on the value BEFORE converting from the SQL to BCL type to avoid e.g. SqlDateTime.Null being converted to 1900-01-01
expr = Expression.Condition(
SqlTypeConverter.NullCheck(expr),
Expression.Constant(null, destType),
convertedExpr);
if (expr.Type.IsValueType)
{
expr = SqlTypeConverter.Convert(expr, typeof(object));
}
}
attributeAccessors[destAttributeName] = Expression.Lambda <Func <Entity, object> >(expr, entityParam).Compile();
}
/// <summary>
/// Gets the records to perform the DML operation on
/// </summary>
/// <param name="org">The <see cref="IOrganizationService"/> to use to get the data</param>
/// <param name="metadata">The <see cref="IAttributeMetadataCache"/> to use to get metadata</param>
/// <param name="options"><see cref="IQueryExecutionOptions"/> to indicate how the query can be executed</param>
/// <param name="parameterTypes">A mapping of parameter names to their related types</param>
/// <param name="parameterValues">A mapping of parameter names to their current values</param>
/// <param name="schema">The schema of the data source</param>
/// <returns>The entities to perform the DML operation on</returns>
protected List <Entity> GetDmlSourceEntities(IDictionary <string, DataSource> dataSources, IQueryExecutionOptions options, IDictionary <string, Type> parameterTypes, IDictionary <string, object> parameterValues, out INodeSchema schema)
{
List <Entity> entities;
if (Source is IDataExecutionPlanNode dataSource)
{
schema = dataSource.GetSchema(dataSources, parameterTypes);
entities = dataSource.Execute(dataSources, options, parameterTypes, parameterValues).ToList();
}
else if (Source is IDataSetExecutionPlanNode dataSetSource)
{
var dataTable = dataSetSource.Execute(dataSources, options, parameterTypes, parameterValues);
// Store the values under the column index as well as name for compatibility with INSERT ... SELECT ...
schema = new NodeSchema();
for (var i = 0; i < dataTable.Columns.Count; i++)
{
var col = dataTable.Columns[i];
((NodeSchema)schema).Schema[col.ColumnName] = col.DataType;
((NodeSchema)schema).Schema[i.ToString()] = col.DataType;
}
entities = dataTable.Rows
.Cast <DataRow>()
.Select(row =>
{
var entity = new Entity();
for (var i = 0; i < dataTable.Columns.Count; i++)
{
entity[dataTable.Columns[i].ColumnName] = row[i];
entity[i.ToString()] = row[i];
}
return(entity);
})
.ToList();
}
else
{
throw new QueryExecutionException("Unexpected data source")
{
Node = this
};
}
return(entities);
}
