private void AddReferenceConfigJoins(LinkEntity linkToReferenced, Entity thisEntity, string field)
{
var referencedType = XrmEntity.GetLookupType(thisEntity.GetFieldValue(field));
var referencedTypeConfig = XrmRecordService.GetTypeConfigs().GetFor(referencedType);
if (referencedTypeConfig != null && referencedTypeConfig.UniqueChildFields != null)
{
foreach (var uniqueField in referencedTypeConfig.UniqueChildFields)
{
var theValue = thisEntity.GetFieldValue($"{field}.{uniqueField}");
if (theValue == null)
{
linkToReferenced.LinkCriteria.AddCondition(new ConditionExpression(uniqueField, ConditionOperator.Null));
}
else if (theValue is EntityReference)
{
var name = XrmEntity.GetLookupName(theValue);
var type = XrmEntity.GetLookupType(theValue);
var nextLinkToReferenced = linkToReferenced.AddLink(type, uniqueField, XrmService.GetPrimaryKeyField(type));
if (name == null)
{
nextLinkToReferenced.LinkCriteria.AddCondition(XrmService.GetPrimaryNameField(type), ConditionOperator.Null);
}
else
{
nextLinkToReferenced.LinkCriteria.AddCondition(XrmService.GetPrimaryNameField(type), ConditionOperator.Equal, name);
AddReferenceConfigJoins(nextLinkToReferenced, thisEntity, $"{field}.{uniqueField}");
}
}
else
{
linkToReferenced.LinkCriteria.AddCondition(new ConditionExpression(uniqueField, ConditionOperator.Equal, XrmService.ConvertToQueryValue(uniqueField, referencedType, theValue)));
}
}
}
}
public void ProcessExport(IEnumerable <ExportRecordType> exports, bool includeNotes, bool includeNNBetweenEntities, LogController controller
, Action <Entity> processEntity, Action <Entity> processAssociation)
{
if (exports == null || !exports.Any())
{
throw new Exception("Error No Record Types To Export");
}
var countToExport = exports.Count();
var countsExported = 0;
var exported = new Dictionary <string, List <Entity> >();
foreach (var exportType in exports)
{
var type = exportType.RecordType == null ? null : exportType.RecordType.Key;
var thisTypeConfig = XrmRecordService.GetTypeConfigs().GetFor(type);
controller.UpdateProgress(countsExported++, countToExport, string.Format("Querying {0} Records", type));
var conditions = new List <ConditionExpression>();
if (type == "list")
{
conditions.Add(new ConditionExpression("type", ConditionOperator.Equal, XrmPicklists.ListType.Dynamic));
}
if (type == "knowledgearticle")
{
conditions.Add(new ConditionExpression("islatestversion", ConditionOperator.Equal, true));
}
//doesn't work for too many notes
//should have option on each or all entities for notes maybe
IEnumerable <Entity> entities;
switch (exportType.Type)
{
case ExportType.AllRecords:
{
entities = XrmService.RetrieveAllAndClauses(type, conditions)
.Where(e => !CheckIgnoreForExport(exportType, e))
.ToArray();
break;
}
case ExportType.FetchXml:
{
var queryExpression = XrmService.ConvertFetchToQueryExpression(exportType.FetchXml);
queryExpression.ColumnSet = new ColumnSet(true);
entities = queryExpression.PageInfo != null && queryExpression.PageInfo.Count > 0
? XrmService.RetrieveFirstX(queryExpression, queryExpression.PageInfo.Count)
: XrmService.RetrieveAll(queryExpression);
entities = entities
.Where(e => !CheckIgnoreForExport(exportType, e))
.ToArray();
break;
}
case ExportType.SpecificRecords:
{
var primaryKey = XrmService.GetPrimaryKeyField(type);
var ids = exportType.SpecificRecordsToExport == null
? new string[0]
: exportType.SpecificRecordsToExport
.Select(r => r.Record == null ? null : r.Record.Id)
.Where(s => !s.IsNullOrWhiteSpace()).Distinct().ToArray();
entities = ids
.Select(id => XrmService.Retrieve(type, new Guid(id)))
.ToArray();
break;
}
default:
{
throw new NotImplementedException(string.Format("No Export Implemented For {0} {1} For {2} Records", typeof(ExportType).Name, exportType.Type, type));
}
}
var excludeFields = exportType.IncludeAllFields
? new string[0]
: XrmService.GetFields(exportType.RecordType.Key).Except(exportType.IncludeOnlyTheseFields.Select(f => f.RecordField == null ? null : f.RecordField.Key).Distinct().ToArray());
if (thisTypeConfig != null)
{
//which need to include the fields if they are needed for parentchild configs
excludeFields = excludeFields.Except(new[] { thisTypeConfig.ParentLookupField }).ToArray();
if (thisTypeConfig.UniqueChildFields != null)
{
excludeFields = excludeFields.Except(thisTypeConfig.UniqueChildFields).ToArray();
}
var fieldsToIncludeInParent = XrmRecordService.GetTypeConfigs().GetParentFieldsRequiredForComparison(type);
var thisTypesParentsConfig = XrmRecordService.GetTypeConfigs().GetFor(thisTypeConfig.ParentLookupType);
if (fieldsToIncludeInParent != null)
{
//if the parent also has a config then we need to use it when matching the parent
//e.g. portal web page access rules -> web page where the web page may be a master or child web page
//so lets include the parents config fields as aliased fields in the exported entity
foreach (var item in entities)
{
var parentId = item.GetLookupGuid(thisTypeConfig.ParentLookupField);
if (parentId.HasValue)
{
var parent = XrmService.Retrieve(thisTypeConfig.ParentLookupType, parentId.Value, fieldsToIncludeInParent);
item.Attributes.AddRange(parent.Attributes.Select(f => new KeyValuePair <string, object>($"{thisTypeConfig.ParentLookupField}.{f.Key}", new AliasedValue(thisTypeConfig.ParentLookupType, f.Key, f.Value))));
}
}
}
var lookupFieldsEnsureNamePopulated = new List <string>();
if (thisTypeConfig.ParentLookupField != null)
{
lookupFieldsEnsureNamePopulated.Add(thisTypeConfig.ParentLookupField);
}
if (thisTypeConfig.UniqueChildFields != null)
{
lookupFieldsEnsureNamePopulated.AddRange(thisTypeConfig.UniqueChildFields.Where(f => XrmService.IsLookup(f, thisTypeConfig.Type)));
}
foreach (var field in lookupFieldsEnsureNamePopulated)
{
controller.UpdateProgress(countsExported++, countToExport, string.Format("Populating Empty Lookups For {0} Records", type));
foreach (var item in entities)
{
var entityReference = item.GetField(field) as EntityReference;
if (entityReference != null && entityReference.Name == null)
{
var referencedTypeNameField = XrmService.GetPrimaryNameField(entityReference.LogicalName);
var referencedRecord = XrmService.Retrieve(entityReference.LogicalName, entityReference.Id, new[] { referencedTypeNameField });
entityReference.Name = referencedRecord.GetStringField(referencedTypeNameField);
}
}
}
}
var primaryField = XrmService.GetPrimaryNameField(exportType.RecordType.Key);
if (excludeFields.Contains(primaryField))
{
excludeFields = excludeFields.Except(new[] { primaryField }).ToArray();
}
if (exportType.ExplicitValuesToSet != null)
{
foreach (var field in exportType.ExplicitValuesToSet)
{
var parseFieldValue = XrmRecordService.ToEntityValue(field.ClearValue ? null : field.ValueToSet);
foreach (var entity in entities)
{
entity.SetField(field.FieldToSet.Key, parseFieldValue, XrmService);
}
if (excludeFields.Contains(field.FieldToSet.Key))
{
excludeFields = excludeFields.Except(new[] { field.FieldToSet.Key }).ToArray();
}
}
}
var fieldsAlwaysExclude = new[] { "calendarrules" };
excludeFields = excludeFields.Union(fieldsAlwaysExclude).ToArray();
var toDo = entities.Count();
var done = 0;
foreach (var entity in entities)
{
controller.UpdateLevel2Progress(done++, toDo, string.Format("Processing {0} Records", type));
entity.RemoveFields(excludeFields);
processEntity(entity);
}
controller.TurnOffLevel2();
if (!exported.ContainsKey(type))
{
exported.Add(type, new List <Entity>());
}
exported[type].AddRange(entities);
if (includeNotes)
{
controller.LogLiteral(string.Format("Querying Notes For {0} Records", type));
var notes = XrmService
.RetrieveAllOrClauses("annotation",
new[] { new ConditionExpression("objecttypecode", ConditionOperator.Equal, type) });
toDo = notes.Count();
done = 0;
foreach (var note in notes)
{
var objectId = note.GetLookupGuid("objectid");
if (objectId.HasValue && entities.Select(e => e.Id).Contains(objectId.Value))
{
controller.UpdateLevel2Progress(done++, toDo, string.Format("Processing Notes For {0} Records", type));
processEntity(note);
}
}
controller.TurnOffLevel2();
}
controller.TurnOffLevel2();
}
var relationshipsDone = new List <string>();
if (includeNNBetweenEntities)
{
countToExport = exports.Count();
countsExported = 0;
foreach (var type in exported.Keys)
{
controller.UpdateProgress(countsExported++, countToExport, string.Format("Exporting {0} Associations", type));
var nnRelationships = XrmService.GetEntityManyToManyRelationships(type)
.Where(
r =>
exported.Keys.Contains(r.Entity1LogicalName) && exported.Keys.Contains(r.Entity2LogicalName));
foreach (var item in nnRelationships)
{
var type1 = item.Entity1LogicalName;
var type2 = item.Entity2LogicalName;
if (!relationshipsDone.Contains(item.SchemaName))
{
controller.LogLiteral(string.Format("Querying {0} Associations", item.SchemaName));
var associations = XrmService.RetrieveAllEntityType(item.IntersectEntityName, null);
var toDo = associations.Count();
var done = 0;
foreach (var association in associations)
{
controller.UpdateLevel2Progress(done++, toDo, string.Format("Processing Notes For {0} Records", type));
if (exported[type1].Any(e => e.Id == association.GetGuidField(item.Entity1IntersectAttribute)) &&
exported[type2].Any(e => e.Id == association.GetGuidField(item.Entity2IntersectAttribute)))
{
processAssociation(association);
}
}
relationshipsDone.Add(item.SchemaName);
controller.TurnOffLevel2();
}
}
}
}
controller.TurnOffLevel2();
}
private void CheckLoadCache(IEnumerable <string> recordsReferenced)
{
var loadTheseOnes = recordsReferenced.ToList();
var typeConfigs = XrmRecordService.GetTypeConfigs();
foreach (var one in loadTheseOnes.ToArray())
{
if (_dontCacheTheseTypes.Contains(one))
{
loadTheseOnes.Remove(one);
}
if (_cachedRecords.ContainsKey(one))
{
loadTheseOnes.Remove(one);
}
else
{
var typeConfig = typeConfigs.GetFor(one);
if (typeConfig != null)
{
loadTheseOnes.Remove(one);
}
}
}
var getMultipleResponses = XrmService.ExecuteMultiple(loadTheseOnes.Select(rt =>
{
var matchFilters = _matchFilters.ContainsKey(rt)
? _matchFilters[rt]
: new ConditionExpression[0];
var query = new QueryExpression(rt)
{
ColumnSet = new ColumnSet(true),
TopCount = _maxCacheCount
};
query.Criteria.Conditions.AddRange(matchFilters);
return(new RetrieveMultipleRequest
{
Query = query
});
}).ToArray());
if (getMultipleResponses.Any())
{
try
{
var i = 0;
foreach (var response in getMultipleResponses)
{
var type = loadTheseOnes.ElementAt(i);
if (response.Fault != null)
{
Response.AddImportError(new DataImportResponseItem(
$"Error Loading Target Records Cache for {type})", new FaultException <OrganizationServiceFault>(response.Fault, response.Fault.Message)));
}
else
{
var records = ((RetrieveMultipleResponse)response.Response).EntityCollection.Entities;
if (!_cachedRecords.ContainsKey(type))
{
_cachedRecords.Add(type, new Dictionary <string, Dictionary <string, List <Entity> > >());
}
var primaryKey = XrmService.GetPrimaryKeyField(type);
if (!_cachedRecords[type].ContainsKey(primaryKey))
{
_cachedRecords[type].Add(primaryKey, new Dictionary <string, List <Entity> >());
}
foreach (var record in records)
{
var cacheMatchString = record.Id.ToString();
if (!_cachedRecords[type][primaryKey].ContainsKey(cacheMatchString))
{
_cachedRecords[type][primaryKey].Add(cacheMatchString, new List <Entity>());
}
_cachedRecords[type][primaryKey][cacheMatchString].Add(record);
}
var primaryNameField = XrmService.GetPrimaryNameField(type);
if (primaryNameField != null)
{
if (!_cachedRecords[type].ContainsKey(primaryNameField))
{
_cachedRecords[type].Add(primaryNameField, new Dictionary <string, List <Entity> >());
}
foreach (var record in records)
{
var cacheMatchString = XrmService.GetFieldAsMatchString(type, primaryNameField, record.GetStringField(primaryNameField));
if (!_cachedRecords[type][primaryNameField].ContainsKey(cacheMatchString))
{
_cachedRecords[type][primaryNameField].Add(cacheMatchString, new List <Entity>());
}
_cachedRecords[type][primaryNameField][cacheMatchString].Add(record);
}
}
}
i++;
}
}
catch (Exception ex)
{
Response.AddImportError(new DataImportResponseItem(
$"Error Loading Target Records Cache ({string.Join(",", loadTheseOnes)})", ex));
}
}
}
