public void ReadFetchXmlQueryResultsFromDiskAndImport(string dataFolder, string[] columnsToExcludeToCompareData, bool verifyDataImport = false) { SourceData = new Dictionary <string, EntityCollection>(); if (columnsToExcludeToCompareData == null) { // match this with "ColumnsToExcludeToCompareData" in app.config - this is included here in case there is no exclusion in the fetch!!! var columnsToExclude = "languagecode;createdon;createdby;modifiedon;modifiedby;owningbusinessunit;owninguser;owneridtype;" + "importsequencenumber;overriddencreatedon;timezoneruleversionnumber;operatorparam;utcconversiontimezonecode;versionnumber;" + "customertypecode;matchingentitymatchcodetable;baseentitymatchcodetable;slaidunique;slaitemidunique;ignoreblankvalues"; columnsToExcludeToCompareData = columnsToExclude.Split(';'); } /* Load all the reference data related to the security framework (bu, team, security role, mailbox, queue) of the target */ LoadTargetSecurityReferenceData(); foreach (var file in Directory.EnumerateFiles(dataFolder, "*.xml")) { // must reset for every fetchxml data var fetchXmlQueriesResultXml = new Dictionary <string, string>(); Logger?.Invoke(this, "\r\n---\r\nReading : " + file); var nodes = XElement.Load(File.OpenRead(file)); var entityName = XElement.Load(File.OpenRead(file)).FirstAttribute.Value; fetchXmlQueriesResultXml.Add(entityName, nodes.ToString()); // save records into Target CRM WriteDatatoTargetCrm(fetchXmlQueriesResultXml, columnsToExcludeToCompareData, verifyDataImport); } // verify data import if requested :) if (verifyDataImport) { // Check if the target has any extra data (potential duplicate) comparing to source var verificationMsg = new DataImportVerification(columnsToExcludeToCompareData).VerifyTargetDataInSourceFetchXml( OrganizationService, SourceData); if (!string.IsNullOrWhiteSpace(verificationMsg)) { Logger?.Invoke(this, verificationMsg); } } }
/// <summary> /// WriteDatatoTargetCRM /// </summary> /// <param name="fetchXmlQueriesResultXml">This must contain data for one entity only!</param> /// <param name="columnsToExcludeToCompareData"></param> /// <param name="verifyDataImport"></param> /// <returns></returns> private void WriteDatatoTargetCrm(Dictionary <string, string> fetchXmlQueriesResultXml, string[] columnsToExcludeToCompareData, bool verifyDataImport) { // load data var fetchXmlQueryData = LoadFetchXmlData(fetchXmlQueriesResultXml); // check if there is any data to import if (fetchXmlQueryData.Entities.Count == 0) { return; } // Add mapping for OOTB entities (Business Unit, Currency, Team and Security Role) var targetDataLoader = new DataLoader(OrganizationService); if (!AddTransformsForEntity(fetchXmlQueryData)) { return; //don't continue further - eg Security roles can't be imported using this utility } // get medata of the entity var entityMetaData = TransformData.GetEntityMetaData(TargetEntitiesMetaData, fetchXmlQueryData.Entities[0].LogicalName); if (entityMetaData == null) { throw new Exception($"ERROR: MetaData is missing for {fetchXmlQueryData.Entities[0].LogicalName}"); } EntityMetadata relationshipMetaData = null; if (entityMetaData.IsIntersect == true) // is this a many to many entity? { // load relationship relationshipMetaData = targetDataLoader.GetEntityMetaData(fetchXmlQueryData.Entities[0].LogicalName, EntityFilters.Relationships); } var dataImportVerification = new DataImportVerification(columnsToExcludeToCompareData); var allRecordsImported = false; var failedEntities = new List <Guid>(); // this loop is required for self-relationship entity such as Account-ParentAccount, var importCount = 0; while (allRecordsImported == false) { foreach (var curEntity in fetchXmlQueryData.Entities.ToArray()) { // first time (importCount == 0) - process all the entities. on second try process only the failed ones! if (importCount == 0 || (importCount > 0 && failedEntities.Contains(curEntity.Id))) { var currentEntityImported = false; // replace values using transforms - for the first time only if (importCount == 0) { foreach (var a in curEntity.Attributes.ToArray()) { // replace any Target Data (ie value to be replaced) with the value from the transform file //if (curEntity.LogicalName == "contact") _transformData.TransformValue(curEntity, a.Key, a.Value); } } if (entityMetaData.IsIntersect == true) //is this a many to many entity? { if (relationshipMetaData?.ManyToManyRelationships == null) { throw new Exception($"Relationship is missing for {entityMetaData.LogicalName}"); } currentEntityImported = UpsertManyManyRecord(curEntity, relationshipMetaData.ManyToManyRelationships[0].SchemaName); } else { currentEntityImported = UpsertEntityRecord(curEntity, TargetEntitiesMetaData); } if (!currentEntityImported) { failedEntities.Add(curEntity.Id); allRecordsImported = currentEntityImported; } } } importCount++; // try 3 times max - depth of the self-relation within an entity!!! // an account is parent of another account that has more than 1 child accounts. if (importCount > 2) { allRecordsImported = true; } } // Verify data import if requested :) if (verifyDataImport && !entityMetaData.IsIntersect.Value) { // give extra time to publish the records - this can be extended to use asyncoperation to find the jobs and let the jobs finish before the Verification starts if (entityMetaData.LogicalName.Equals(Constant.DuplicateRule.EntityLogicalName, StringComparison.OrdinalIgnoreCase) || entityMetaData.LogicalName.Equals(Constant.Workflow.EntityLogicalName, StringComparison.OrdinalIgnoreCase) || entityMetaData.LogicalName.Equals(Constant.Sla.EntityLogicalName, StringComparison.OrdinalIgnoreCase)) { // sleep to let the publishing finishes before continuing ;) Thread.Sleep(16000); } // Verify that the data in FetchXML match with saved data in Target var verificationMsg = dataImportVerification.VerifyDataImport(OrganizationService, fetchXmlQueryData); if (!string.IsNullOrWhiteSpace(verificationMsg)) { Logger?.Invoke(this, verificationMsg); } // Build Entity collection to use with Target data verification DataImportVerification.AddInSourceEntityCollection(SourceData, fetchXmlQueryData); } }