/// <summary>
/// WriteDatatoTargetCRM
/// </summary>
/// <param name="fetchXmlQueriesResultXml">This must contain data for one entity only!</param>
/// <param name="columnsToExcludeToCompareData"></param>
/// <param name="verifyDataImport"></param>
/// <returns></returns>
private void WriteDatatoTargetCrm(Dictionary <string, string> fetchXmlQueriesResultXml,
string[] columnsToExcludeToCompareData, bool verifyDataImport)
{
// load data
var fetchXmlQueryData = LoadFetchXmlData(fetchXmlQueriesResultXml);
// check if there is any data to import
if (fetchXmlQueryData.Entities.Count == 0)
{
return;
}
// Add mapping for OOTB entities (Business Unit, Currency, Team and Security Role)
var targetDataLoader = new DataLoader(OrganizationService);
if (!AddTransformsForEntity(fetchXmlQueryData))
{
return; //don't continue further - eg Security roles can't be imported using this utility
}
// get medata of the entity
var entityMetaData = TransformData.GetEntityMetaData(TargetEntitiesMetaData, fetchXmlQueryData.Entities[0].LogicalName);
if (entityMetaData == null)
{
throw new Exception($"ERROR: MetaData is missing for {fetchXmlQueryData.Entities[0].LogicalName}");
}
EntityMetadata relationshipMetaData = null;
if (entityMetaData.IsIntersect == true) // is this a many to many entity?
{
// load relationship
relationshipMetaData = targetDataLoader.GetEntityMetaData(fetchXmlQueryData.Entities[0].LogicalName, EntityFilters.Relationships);
}
var dataImportVerification = new DataImportVerification(columnsToExcludeToCompareData);
var allRecordsImported = false;
var failedEntities = new List <Guid>();
// this loop is required for self-relationship entity such as Account-ParentAccount,
var importCount = 0;
while (allRecordsImported == false)
{
foreach (var curEntity in fetchXmlQueryData.Entities.ToArray())
{
// first time (importCount == 0) - process all the entities. on second try process only the failed ones!
if (importCount == 0 || (importCount > 0 && failedEntities.Contains(curEntity.Id)))
{
var currentEntityImported = false;
// replace values using transforms - for the first time only
if (importCount == 0)
{
foreach (var a in curEntity.Attributes.ToArray())
{
// replace any Target Data (ie value to be replaced) with the value from the transform file
//if (curEntity.LogicalName == "contact")
_transformData.TransformValue(curEntity, a.Key, a.Value);
}
}
if (entityMetaData.IsIntersect == true) //is this a many to many entity?
{
if (relationshipMetaData?.ManyToManyRelationships == null)
{
throw new Exception($"Relationship is missing for {entityMetaData.LogicalName}");
}
currentEntityImported = UpsertManyManyRecord(curEntity, relationshipMetaData.ManyToManyRelationships[0].SchemaName);
}
else
{
currentEntityImported = UpsertEntityRecord(curEntity, TargetEntitiesMetaData);
}
if (!currentEntityImported)
{
failedEntities.Add(curEntity.Id);
allRecordsImported = currentEntityImported;
}
}
}
importCount++;
// try 3 times max - depth of the self-relation within an entity!!!
// an account is parent of another account that has more than 1 child accounts.
if (importCount > 2)
{
allRecordsImported = true;
}
}
// Verify data import if requested :)
if (verifyDataImport && !entityMetaData.IsIntersect.Value)
{
// give extra time to publish the records - this can be extended to use asyncoperation to find the jobs and let the jobs finish before the Verification starts
if (entityMetaData.LogicalName.Equals(Constant.DuplicateRule.EntityLogicalName, StringComparison.OrdinalIgnoreCase) ||
entityMetaData.LogicalName.Equals(Constant.Workflow.EntityLogicalName, StringComparison.OrdinalIgnoreCase) ||
entityMetaData.LogicalName.Equals(Constant.Sla.EntityLogicalName, StringComparison.OrdinalIgnoreCase))
{
// sleep to let the publishing finishes before continuing ;)
Thread.Sleep(16000);
}
// Verify that the data in FetchXML match with saved data in Target
var verificationMsg = dataImportVerification.VerifyDataImport(OrganizationService, fetchXmlQueryData);
if (!string.IsNullOrWhiteSpace(verificationMsg))
{
Logger?.Invoke(this, verificationMsg);
}
// Build Entity collection to use with Target data verification
DataImportVerification.AddInSourceEntityCollection(SourceData, fetchXmlQueryData);
}
}
public StringBuilder ExecuteFetchXmlQuery(string entityName, string fetchXml, string[] systemFieldsToExclude, bool forComparison = false)
{
var queryResultXml = new StringBuilder();
if (string.IsNullOrWhiteSpace(entityName) || string.IsNullOrWhiteSpace(fetchXml))
{
return(queryResultXml);
}
// convert fetchxml to query expression
var query = ConvertFetchXmlToQueryExpression(fetchXml);
var systemFields = new List <string>();
if (systemFieldsToExclude == null)
{
// match this with "ColumnsToExcludeToCompareData" in app.config - this is included here in case there is no exclusion in the fetch!!!
const string columnsToExclude = "languagecode;createdon;createdby;modifiedon;modifiedby;owningbusinessunit;owninguser;owneridtype;" +
"importsequencenumber;overriddencreatedon;timezoneruleversionnumber;operatorparam;utcconversiontimezonecode;versionnumber;" +
"customertypecode;matchingentitymatchcodetable;baseentitymatchcodetable;slaidunique;slaitemidunique";
systemFields.AddRange(columnsToExclude.Split(';'));
}
else
{
systemFields.AddRange(systemFieldsToExclude);
}
systemFields.Sort();
var sourceDataLoader = new DataLoader(OrganizationService);
var entityMetadata = sourceDataLoader.GetEntityMetaData(query.EntityName, EntityFilters.Attributes);
// exclude all fields tha are in the system fields or IsValidForRead=false or isValidForCreate = false
if (query.ColumnSet.AllColumns)
{
query.ColumnSet.AllColumns = false;
foreach (var attributeMetadata in entityMetadata.Attributes)
{
if (systemFields.Contains(attributeMetadata.LogicalName) || attributeMetadata.IsValidForRead == false ||
attributeMetadata.IsValidForCreate == false)
{
// ignore system fields or those can't be read or used for created
}
else
{
query.ColumnSet.AddColumn(attributeMetadata.LogicalName);
}
}
}
var queryResults = OrganizationService.RetrieveMultiple(query);
if (queryResults.Entities.Count == 0)
{
return(queryResultXml);
}
queryResultXml.AppendLine($"<EntityData Name='{entityName}'>");
foreach (var queryResult in queryResults.Entities)
{
if (!forComparison)
{
foreach (var attributeMetadata in entityMetadata.Attributes)
{
if (systemFields.Contains(attributeMetadata.LogicalName) ||
attributeMetadata.IsValidForCreate == false)
{
// these columns should not be included (even if they are included in the query)
}
else if (query.ColumnSet != null)
{
// if the fetchXml contains the column but the column is not in the queryResult because it has null value
// then we need to add the column in the queryResult so that the column-value is replaced in the target
if (query.ColumnSet.Columns.Contains(attributeMetadata.LogicalName) &&
!queryResult.Attributes.Contains(attributeMetadata.LogicalName))
{
queryResult.Attributes.Add(attributeMetadata.LogicalName, null);
}
}
}
}
if ((queryResult.RowVersion != null) && forComparison)
{
queryResult.RowVersion = null;
}
var typeList = new List <Type> {
queryResult.GetType()
};
queryResultXml.AppendLine(DataContractSerializeObject(queryResult, typeList));
}
queryResultXml.AppendLine("</EntityData>");
return(queryResultXml);
}