/// <summary>
/// runs the import process
/// </summary>
public void Process()
{
OrganizationService sourceService = null;
OrganizationService targetService = null;
//set up logging
logger = LogManager.GetLogger(typeof(Importer));
LogMessage("INFO", "starting job");
//establish connections and/or read source data from file
ParseConnections();
//connect to source and target if necessary
if (!_isFileSource)
{
sourceService = new OrganizationService(_sourceConn);
}
if (!_isFileTarget)
{
targetService = new OrganizationService(_targetConn);
}
//create the guid mappings table
SetupGuidMappings();
LogMessage("INFO", "processing records");
//loop through each job step
for (int i = 0; i < JobSteps.Count; i++)
{
var item = JobSteps[i];
JobStep step = (JobStep)item;
LogMessage("INFO", string.Format("starting step {0}", step.StepName));
//create a list of entities to hold retrieved entities so we can page through results
List <Entity> ec = new List <Entity>();
//if data is coming from a file
if (_isFileSource)
{
LogMessage("INFO", " preparing data from source file for update/import");
//get the recordset in the file that corresponds to the current job step and loop through it
foreach (var e in _savedSourceData.RecordSets[i])
{
//instantiate a new crm entity object
Entity entity = new Entity(e.LogicalName);
entity.Id = e.Id;
entity.LogicalName = e.LogicalName;
//loop through the attributes stored in the file
foreach (ExportAttribute exportAttribute in e.Attributes)
{
//JObject object if we need to parse a complex type
Newtonsoft.Json.Linq.JObject jObject;
//instantiate a new object to hold the attribute value
object attributeValue = null;
//give the attribute the correct name
string attributeName = exportAttribute.AttributeName;
//check the stored attribute type in the file and set the crm entity's attribute values accordingly
switch (exportAttribute.AttributeType)
{
//if it's an entityreference
case "Microsoft.Xrm.Sdk.EntityReference":
jObject = (Newtonsoft.Json.Linq.JObject)exportAttribute.AttributeValue;
EntityReference lookup = new EntityReference((string)jObject["LogicalName"], (Guid)jObject["Id"]);
attributeValue = lookup;
break;
//if it's an optionsetvalue
case "Microsoft.Xrm.Sdk.OptionSetValue":
jObject = (Newtonsoft.Json.Linq.JObject)exportAttribute.AttributeValue;
attributeValue = new OptionSetValue {
Value = (int)jObject["Value"]
};
break;
//if it's money
case "Microsoft.Xrm.Sdk.Money":
jObject = (Newtonsoft.Json.Linq.JObject)exportAttribute.AttributeValue;
attributeValue = new Microsoft.Xrm.Sdk.Money {
Value = (decimal)jObject["Value"]
};
break;
//if it's anything else - i think this covers everything we would typically need
default:
attributeValue = exportAttribute.AttributeValue;
break;
}
//add the attribute name and value to the entity's attributes collection
entity.Attributes.Add(attributeName, attributeValue);
}
//add the entity to the entity collection
ec.Add(entity);
}
}
else //source is live crm org
{
string fetchQuery = step.StepFetch;
LogMessage("INFO", " retrieving records");
// Set the number of records per page to retrieve.
int fetchCount = 5000;
// Initialize the page number.
int pageNumber = 1;
// Specify the current paging cookie. For retrieving the first page,
// pagingCookie should be null.
string pagingCookie = null;
while (true)
{
// Build fetchXml string with the placeholders.
string fetchXml = CreateXml(fetchQuery, pagingCookie, pageNumber, fetchCount);
EntityCollection retrieved = sourceService.RetrieveMultiple(new FetchExpression(fetchXml));
ec.AddRange(retrieved.Entities);
if (retrieved.MoreRecords)
{
// Increment the page number to retrieve the next page.
pageNumber++;
// Set the paging cookie to the paging cookie returned from current results.
pagingCookie = retrieved.PagingCookie;
}
else
{
// If no more records in the result nodes, exit the loop.
break;
}
}
LogMessage("INFO", string.Format(" {0} records retrieved", ec.Count));
}
if (ec.Count > 0)
{
//if the target is a live crm org
if (!_isFileTarget)
{
//loop through each entity in the collection
foreach (Entity entity in ec)
{
//create a list to hold the replacement guids. a second pass is required because c# disallows modifying a collection while enumerating
List <KeyValuePair <string, object> > guidsToUpdate = new List <KeyValuePair <string, object> >();
LogMessage("INFO", string.Format(" processing record {0}, {1}", entity.Id, entity.LogicalName));
try
{
LogMessage("INFO", " processing GUID replacements");
foreach (KeyValuePair <string, object> attribute in entity.Attributes)
{
//LogMessage("INFO",string.Format("Attribute - {0} {1}", attribute.Key, attribute.Value.GetType().ToString()));
if (attribute.Value is Microsoft.Xrm.Sdk.EntityReference)
{
//LogMessage("INFO","getting source");
EntityReference source = ((EntityReference)attribute.Value);
try
{
//LogMessage("INFO","looking for GUID replacement");
Guid sourceId = source.Id;
Guid targetId = _mappings.Find(t => t.sourceId == source.Id).targetId;
source.Id = targetId;
guidsToUpdate.Add(new KeyValuePair <string, object>(attribute.Key, source));
//LogMessage("INFO",string.Format("replacement found - {0} -> {1}", sourceId, targetId));
}
catch (System.NullReferenceException ex)
{
//LogMessage("INFO", "NullReferenceException happened");
//do nothing because nullreferenceexception means there's no guid mapping to use
}
}
}
//now actually update the GUIDs with the mapped values
foreach (KeyValuePair <string, object> attribute in guidsToUpdate)
{
//LogMessage("INFO",string.Format(" replacing attribute GUID {0} {1}", attribute.Key, attribute.Value));
entity[attribute.Key] = attribute.Value;
}
//try to update first
try
{
LogMessage("INFO", " trying target update");
targetService.Update(entity);
LogMessage("INFO", " update ok");
}
catch (FaultException <Microsoft.Xrm.Sdk.OrganizationServiceFault> ex)
{
if (!step.UpdateOnly)
{
LogMessage("INFO", " trying target create");
//if update fails and step is not update-only then try to create
targetService.Create(entity);
LogMessage("INFO", " create ok");
}
else
{
throw new FaultException <Microsoft.Xrm.Sdk.OrganizationServiceFault>(ex.Detail);
}
}
}
catch (FaultException <Microsoft.Xrm.Sdk.OrganizationServiceFault> ex)
{
//if everything fails, log error
//to main log
LogMessage("ERROR", string.Format(" record transfer failed"));
//to record error log
LogMessage("ERROR", string.Format("RECORD ERROR: {0}, {1}, MESSAGE:{2}", entity.Id, entity.LogicalName, ex.Detail?.Message));
//increment the error count
_errorCount++;
}
}
}
}
//if the target is a file - prepare the records in this step for serialization later
if (_isFileTarget)
{
LogMessage("INFO", " preparing records for serialization");
//instantiate a new list of exportentity objects
List <ExportEntity> entitiesToExport = new List <ExportEntity>();
//loop through each entity in the collection
foreach (Entity e in ec)
{
//instantiate a new exportentity object and set its fields appropriately
ExportEntity exportEntity = new ExportEntity();
exportEntity.Id = e.Id;
exportEntity.LogicalName = e.LogicalName;
foreach (var attribute in e.Attributes)
{
//leave out the entity id and logical name from the attribute collection - they cause problems on import
if ((attribute.Key.ToUpper() != e.LogicalName.ToUpper() + "ID") &&
(attribute.Key.ToUpper() != "LOGICALNAME"))
{
ExportAttribute exportAttribute = new ExportAttribute();
exportAttribute.AttributeName = attribute.Key;
exportAttribute.AttributeValue = attribute.Value;
exportAttribute.AttributeType = attribute.Value.GetType().ToString();
exportEntity.Attributes.Add(exportAttribute);
}
}
//add the exportentity object to the recordset
entitiesToExport.Add(exportEntity);
}
//add the recordset to the exporteddata object to be serialized
_savedSourceData.RecordSets.Add(entitiesToExport);
}
}
//if the target is a file - serialize the data and write it to a file
if (_isFileTarget)
{
LogMessage("INFO", " serializing data to target file");
//instantiate a new jsonserializer
JsonSerializer serializer = new JsonSerializer();
//write to the target file path
using (StreamWriter sw = new StreamWriter(_targetFile))
{
using (JsonWriter writer = new JsonTextWriter(sw))
{
//some jsonwriter options
//leave out null values - this might cause problems if trying to unset a value, but not sure what the alternative approach would be
serializer.NullValueHandling = NullValueHandling.Ignore;
//the import will vomit if this isn't "none"
serializer.TypeNameHandling = TypeNameHandling.None;
//you can change this if you want a more easily readable output file, but this makes for a smaller file size
serializer.Formatting = Newtonsoft.Json.Formatting.None;
//serialize and save
serializer.Serialize(writer, _savedSourceData);
}
}
LogMessage("INFO", "data serialization complete");
}
LogMessage("INFO", "job complete");
//stop logging
logger.Logger.Repository.Shutdown();
}
/// <summary>
/// loads the job from a file
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void loadJobButton_Click(object sender, EventArgs e)
{
OpenFileDialog openFileDialog1 = new OpenFileDialog();
openFileDialog1.Filter = "XML file|*.xml";
openFileDialog1.Title = "Open job file";
if (openFileDialog1.ShowDialog() == DialogResult.OK)
{
System.IO.StreamReader sr = new
System.IO.StreamReader(openFileDialog1.FileName);
string jobdata = (sr.ReadToEnd());
sr.Close();
XmlDocument xml = new XmlDocument();
try
{
xml.LoadXml(jobdata);
stepListBox.Items.Clear();
guidMappingGridView.Rows.Clear();
saveConnectionsCheckBox.Checked = false;
sourceTextBox.Text = string.Empty;
targetTextBox.Text = string.Empty;
XmlNodeList stepList = xml.GetElementsByTagName("Step");
foreach (XmlNode xn in stepList)
{
JobStep step = new JobStep();
step.StepName = xn.SelectSingleNode("Name").InnerText;
step.StepFetch = xn.SelectSingleNode("Fetch").InnerText;
step.UpdateOnly = Convert.ToBoolean(xn.Attributes["updateOnly"].Value);
stepListBox.Items.Add(step);
}
XmlNodeList configData = xml.GetElementsByTagName("JobConfig");
mapBuCheckBox.Checked = Convert.ToBoolean(configData[0].Attributes["mapBuGuid"].Value);
mapCurrencyCheckBox.Checked = Convert.ToBoolean(configData[0].Attributes["mapCurrencyGuid"].Value);
XmlNodeList mappingList = xml.GetElementsByTagName("GuidMapping");
foreach (XmlNode xn in mappingList)
{
string sourceId = xn.Attributes["source"].Value;
string targetId = xn.Attributes["target"].Value;
guidMappingGridView.Rows.Add(sourceId, targetId);
}
XmlNodeList connectionNodes = xml.GetElementsByTagName("ConnectionDetails");
if(connectionNodes.Count>0)
{
sourceTextBox.Text = connectionNodes[0].Attributes["source"].Value;
targetTextBox.Text = connectionNodes[0].Attributes["target"].Value;
saveConnectionsCheckBox.Checked = Convert.ToBoolean(connectionNodes[0].Attributes["save"].Value);
}
}
catch (Exception ex)
{
MessageBox.Show(string.Format("Could not parse job configuration data in {0}", openFileDialog1.FileName));
}
}
}
static void ParseConfig(string filepath)
{
StreamReader sr = new StreamReader(filepath);
string jobdata = (sr.ReadToEnd());
sr.Close();
XmlDocument xml = new XmlDocument();
try
{
xml.LoadXml(jobdata);
_jobSteps.Clear();
_guidMappings.Clear();
XmlNodeList stepList = xml.GetElementsByTagName("Step");
foreach (XmlNode xn in stepList)
{
JobStep step = new JobStep();
step.StepName = xn.SelectSingleNode("Name").InnerText;
step.StepFetch = xn.SelectSingleNode("Fetch").InnerText;
step.UpdateOnly = Convert.ToBoolean(xn.Attributes["updateOnly"].Value);
_jobSteps.Add(step);
}
XmlNodeList configData = xml.GetElementsByTagName("JobConfig");
_mapBaseBu = Convert.ToBoolean(configData[0].Attributes["mapBuGuid"].Value);
_mapBaseCurrency = Convert.ToBoolean(configData[0].Attributes["mapCurrencyGuid"].Value);
XmlNodeList mappingList = xml.GetElementsByTagName("GuidMapping");
foreach (XmlNode xn in mappingList)
{
Guid sourceGuid = new Guid(xn.Attributes["source"].Value);
Guid targetGuid = new Guid(xn.Attributes["target"].Value);
_guidMappings.Add(new GuidMapping { sourceId = sourceGuid, targetId = targetGuid });
}
XmlNodeList connectionNodes = xml.GetElementsByTagName("ConnectionDetails");
if (connectionNodes.Count > 0)
{
_sourceString = connectionNodes[0].Attributes["source"].Value;
_targetString = connectionNodes[0].Attributes["target"].Value;
//Console.WriteLine(connectionNodes[0].InnerText);
}
}
catch (Exception ex)
{
Console.WriteLine(string.Format("Could not parse job configuration data in {0} - exiting", filepath));
}
}
