public void ReadFetchXmlQueryResultsFromDiskAndImport(string dataFolder, string[] columnsToExcludeToCompareData, bool verifyDataImport = false)
{
SourceData = new Dictionary <string, EntityCollection>();
if (columnsToExcludeToCompareData == null)
{
// match this with "ColumnsToExcludeToCompareData" in app.config - this is included here in case there is no exclusion in the fetch!!!
var columnsToExclude = "languagecode;createdon;createdby;modifiedon;modifiedby;owningbusinessunit;owninguser;owneridtype;" +
"importsequencenumber;overriddencreatedon;timezoneruleversionnumber;operatorparam;utcconversiontimezonecode;versionnumber;" +
"customertypecode;matchingentitymatchcodetable;baseentitymatchcodetable;slaidunique;slaitemidunique;ignoreblankvalues";
columnsToExcludeToCompareData = columnsToExclude.Split(';');
}
/* Load all the reference data related to the security framework (bu, team, security role, mailbox, queue) of the target */
LoadTargetSecurityReferenceData();
foreach (var file in Directory.EnumerateFiles(dataFolder, "*.xml"))
{
// must reset for every fetchxml data
var fetchXmlQueriesResultXml = new Dictionary <string, string>();
Logger?.Invoke(this, "\r\n---\r\nReading : " + file);
var nodes = XElement.Load(File.OpenRead(file));
var entityName = XElement.Load(File.OpenRead(file)).FirstAttribute.Value;
fetchXmlQueriesResultXml.Add(entityName, nodes.ToString());
// save records into Target CRM
WriteDatatoTargetCrm(fetchXmlQueriesResultXml, columnsToExcludeToCompareData, verifyDataImport);
}
// verify data import if requested :)
if (verifyDataImport)
{
// Check if the target has any extra data (potential duplicate) comparing to source
var verificationMsg = new DataImportVerification(columnsToExcludeToCompareData).VerifyTargetDataInSourceFetchXml(
OrganizationService, SourceData);
if (!string.IsNullOrWhiteSpace(verificationMsg))
{
Logger?.Invoke(this, verificationMsg);
}
}
}
/// <summary>
/// WriteDatatoTargetCRM
/// </summary>
/// <param name="fetchXmlQueriesResultXml">This must contain data for one entity only!</param>
/// <param name="columnsToExcludeToCompareData"></param>
/// <param name="verifyDataImport"></param>
/// <returns></returns>
private void WriteDatatoTargetCrm(Dictionary <string, string> fetchXmlQueriesResultXml,
string[] columnsToExcludeToCompareData, bool verifyDataImport)
{
// load data
var fetchXmlQueryData = LoadFetchXmlData(fetchXmlQueriesResultXml);
// check if there is any data to import
if (fetchXmlQueryData.Entities.Count == 0)
{
return;
}
// Add mapping for OOTB entities (Business Unit, Currency, Team and Security Role)
var targetDataLoader = new DataLoader(OrganizationService);
if (!AddTransformsForEntity(fetchXmlQueryData))
{
return; //don't continue further - eg Security roles can't be imported using this utility
}
// get medata of the entity
var entityMetaData = TransformData.GetEntityMetaData(TargetEntitiesMetaData, fetchXmlQueryData.Entities[0].LogicalName);
if (entityMetaData == null)
{
throw new Exception($"ERROR: MetaData is missing for {fetchXmlQueryData.Entities[0].LogicalName}");
}
EntityMetadata relationshipMetaData = null;
if (entityMetaData.IsIntersect == true) // is this a many to many entity?
{
// load relationship
relationshipMetaData = targetDataLoader.GetEntityMetaData(fetchXmlQueryData.Entities[0].LogicalName, EntityFilters.Relationships);
}
var dataImportVerification = new DataImportVerification(columnsToExcludeToCompareData);
var allRecordsImported = false;
var failedEntities = new List <Guid>();
// this loop is required for self-relationship entity such as Account-ParentAccount,
var importCount = 0;
while (allRecordsImported == false)
{
foreach (var curEntity in fetchXmlQueryData.Entities.ToArray())
{
// first time (importCount == 0) - process all the entities. on second try process only the failed ones!
if (importCount == 0 || (importCount > 0 && failedEntities.Contains(curEntity.Id)))
{
var currentEntityImported = false;
// replace values using transforms - for the first time only
if (importCount == 0)
{
foreach (var a in curEntity.Attributes.ToArray())
{
// replace any Target Data (ie value to be replaced) with the value from the transform file
//if (curEntity.LogicalName == "contact")
_transformData.TransformValue(curEntity, a.Key, a.Value);
}
}
if (entityMetaData.IsIntersect == true) //is this a many to many entity?
{
if (relationshipMetaData?.ManyToManyRelationships == null)
{
throw new Exception($"Relationship is missing for {entityMetaData.LogicalName}");
}
currentEntityImported = UpsertManyManyRecord(curEntity, relationshipMetaData.ManyToManyRelationships[0].SchemaName);
}
else
{
currentEntityImported = UpsertEntityRecord(curEntity, TargetEntitiesMetaData);
}
if (!currentEntityImported)
{
failedEntities.Add(curEntity.Id);
allRecordsImported = currentEntityImported;
}
}
}
importCount++;
// try 3 times max - depth of the self-relation within an entity!!!
// an account is parent of another account that has more than 1 child accounts.
if (importCount > 2)
{
allRecordsImported = true;
}
}
// Verify data import if requested :)
if (verifyDataImport && !entityMetaData.IsIntersect.Value)
{
// give extra time to publish the records - this can be extended to use asyncoperation to find the jobs and let the jobs finish before the Verification starts
if (entityMetaData.LogicalName.Equals(Constant.DuplicateRule.EntityLogicalName, StringComparison.OrdinalIgnoreCase) ||
entityMetaData.LogicalName.Equals(Constant.Workflow.EntityLogicalName, StringComparison.OrdinalIgnoreCase) ||
entityMetaData.LogicalName.Equals(Constant.Sla.EntityLogicalName, StringComparison.OrdinalIgnoreCase))
{
// sleep to let the publishing finishes before continuing ;)
Thread.Sleep(16000);
}
// Verify that the data in FetchXML match with saved data in Target
var verificationMsg = dataImportVerification.VerifyDataImport(OrganizationService, fetchXmlQueryData);
if (!string.IsNullOrWhiteSpace(verificationMsg))
{
Logger?.Invoke(this, verificationMsg);
}
// Build Entity collection to use with Target data verification
DataImportVerification.AddInSourceEntityCollection(SourceData, fetchXmlQueryData);
}
}
