public ShuffleBlocks MergeFiles(Dictionary <string, XmlDocument> datas)
{
log.StartSection("Start MergeFiles");
ShuffleBlocks blocks = new ShuffleBlocks();
foreach (var dataKeyValuePair in datas)
{
var blockNameItemIdCombinationString = dataKeyValuePair.Key;
var blockNameItemIdCombination = blockNameItemIdCombinationString.Split('\\');
var blockName = blockNameItemIdCombination[0];
var block = Deserialize(dataKeyValuePair.Value);
var blockEntityCollection = block.First().Value;
if (!blocks.ContainsKey(blockName))
{
blocks.Add(blockName, blockEntityCollection);
}
else
{
blocks[blockName].Add(blockEntityCollection.First());
}
}
log.EndSection();
return(blocks);
}
/// <summary>
/// Export entities from CRM to dictionary of blocks with entities
/// </summary>
/// <returns>Blocks with exported entities</returns>
public ShuffleBlocks ExportFromCRM()
{
log.StartSection("ExportFromCRM");
if (definition == null)
{
throw new ArgumentNullException("Definition", "Shuffle definition must be specified to export data");
}
var blocks = new ShuffleBlocks();
ExistingSolutionVersions = null;
if (ShuffleDefinition.Blocks.Items.Any(b => (b is DataBlock data && data.Export != null) || b is SolutionBlock sol && sol.Export != null))
{
stoponerror = ShuffleDefinition.StopOnError;
timeout = ShuffleDefinition.TimeoutSpecified ? ShuffleDefinition.Timeout : -1;
double savedtimeout = -1;
if (timeout > -1)
{
savedtimeout = SetTimeout();
}
var totalBlocks = ShuffleDefinition.Blocks.Items.Length;
var currentBlock = 0;
foreach (var block in ShuffleDefinition.Blocks.Items)
{
currentBlock++;
SendStatus(totalBlocks, currentBlock, -1, -1);
if (block is DataBlock datablock)
{
var cExported = ExportDataBlock(blocks, datablock);
var name = datablock.Name;
if (cExported != null)
{
if (blocks.ContainsKey(name))
{
SendLine($"Block already added: {name}");
}
else
{
blocks.Add(name, cExported);
}
}
}
else if (block is SolutionBlock solutionblock)
{
if (ExistingSolutionVersions == null)
{
GetCurrentVersions();
}
ExportSolutionBlock(solutionblock);
}
}
SendStatus(0, 0, 0, 0);
if (savedtimeout > -1)
{
ResetTimeout(savedtimeout);
}
}
log.EndSection();
return(blocks);
}
/// <summary>
///
/// </summary>
/// <param name="blocks"></param>
/// <param name="type"></param>
/// <param name="delimeter"></param>
/// <returns></returns>
public Dictionary <string, XmlDocument> SplitFiles(ShuffleBlocks blocks, SerializationType type, char delimeter)
{
log.StartSection("Start SplitFiles");
var dictionarySplitFiles = new Dictionary <string, XmlDocument>();
if (blocks.Count > 0)
{
foreach (var blockName in blocks.Keys)
{
var block = blocks[blockName];
foreach (var item in block)
{
//Build path to use later when writing to disk
string path = blockName;
path += "\\" + item.Id;
// Somehow create a single entity record shuffleBlock
var singleShuffleBlock = new ShuffleBlocks();
var entityCollection = new CintDynEntityCollection();
singleShuffleBlock.Add(blockName, entityCollection);
entityCollection.Add(item);
dictionarySplitFiles.Add(path, Serialize(singleShuffleBlock, type, delimeter));
}
}
}
log.EndSection();
return(dictionarySplitFiles);
}
// Start is called before the first frame update
void Start()
{
PopulateShapesList();
//Assign game objects to variables
shuffleBlocks = FindObjectOfType <ShuffleBlocks>();
pauseGame = FindObjectOfType <PauseGameStatus>();
shuffleBlocks.currentShuffleLoopDelay = 0;
}
/// <summary>
/// Serialize blocks with entities with given serialization type
/// </summary>
/// <param name="container"></param>
/// <param name="blocks"></param>
/// <param name="type"></param>
/// <param name="delimeter">Optional, only required for SerializationType: Text</param>
/// <returns></returns>
public XmlDocument Serialize(IExecutionContainer container, ShuffleBlocks blocks, SerializationType type, char delimeter)
{
container.StartSection("Serialize");
XmlDocument xml = null;
if (blocks.Count > 0)
{
SendLine(container, "Serializing {0} blocks with type {1}", blocks.Count, type);
xml = new XmlDocument();
XmlNode root = xml.CreateElement("ShuffleData");
xml.AppendChild(root);
XML.AppendAttribute(root, "Type", type.ToString());
XML.AppendAttribute(root, "ExportTime", DateTime.Now.ToString("s"));
switch (type)
{
case SerializationType.Full:
case SerializationType.Simple:
case SerializationType.SimpleWithValue:
case SerializationType.SimpleNoId:
case SerializationType.Explicit:
foreach (var block in blocks.Keys)
{
SendLine(container, $"Serializing {blocks[block].Count()} records in block {block}");
XmlNode xBlock = xml.CreateElement("Block");
root.AppendChild(xBlock);
XML.AppendAttribute(xBlock, "Name", block);
XML.AppendAttribute(xBlock, "Count", blocks[block].Count().ToString());
var xSerialized = blocks[block].Serialize(container, (SerializationStyle)type);
xBlock.AppendChild(xml.ImportNode(xSerialized.ChildNodes[0], true));
}
break;
case SerializationType.Text:
XML.AppendAttribute(root, "Delimeter", delimeter.ToString());
var text = new StringBuilder();
foreach (var block in blocks.Keys)
{
SendLine(container, $"Serializing {blocks[block].Count()} records in block {block}");
text.AppendLine("<<<" + block + ">>>");
var serializedblock = blocks[block].ToTextFile(container, delimeter);
text.Append(serializedblock);
}
XML.AddCDATANode(root, "Text", text.ToString());
break;
}
}
container.EndSection();
return(xml);
}
/// <summary>Import data in Data according to shuffle definition in Definition</summary>
/// <param name="Definition">Shuffle Definition</param>
/// <param name="Data">Exported data</param>
/// <param name="ShuffleEventHandler">Event handler processing messages from the import. May be null.</param>
/// <param name="container"></param>
/// <param name="defpath">Path to definition file, if not standard</param>
/// <param name="clearRemainingShuffleVars"></param>
/// <returns>Tuple with counters for: Created, Updated, Skipped and Failed records and a collection of entityreferences for the created/updated records</returns>
public static Tuple <int, int, int, int, int, EntityReferenceCollection> QuickImport(XmlDocument Definition, XmlDocument Data, EventHandler <ShuffleEventArgs> ShuffleEventHandler, CintContainer container, string defpath, bool clearRemainingShuffleVars)
{
container.Logger.StartSection("QuickImport");
Shuffler shuffle = new Shuffler(container);
if (ShuffleEventHandler != null)
{
shuffle.RaiseShuffleEvent += ShuffleEventHandler;
}
ShuffleHelper.VerifyShuffleVars(Definition, clearRemainingShuffleVars);
shuffle.Definition = Definition;
shuffle.definitionpath = defpath;
ShuffleBlocks blocks = shuffle.Deserialize(Data);
Tuple <int, int, int, int, int, EntityReferenceCollection> result = shuffle.ImportToCRM(blocks);
container.Logger.EndSection();
return(result);
}
/// <summary>Export data according to shuffle definition in Definition to format Type</summary>
/// <param name="Definition">Shuffle Definition</param>
/// <param name="Type">Type of target file</param>
/// <param name="Delimeter">Delimeter to use when exporting to Type: Text</param>
/// <param name="ShuffleEventHandler">Event handler processing messages from the export. May be null.</param>
/// <param name="container"></param>
/// <param name="defpath">Folder path for the shuffle definition file.</param>
/// <param name="clearRemainingShuffleVars"></param>
/// <returns>XmlDocument with exported data</returns>
public static XmlDocument QuickExport(XmlDocument Definition, SerializationType Type, char Delimeter, EventHandler <ShuffleEventArgs> ShuffleEventHandler, CintContainer container, string defpath, bool clearRemainingShuffleVars)
{
container.Logger.StartSection("QuickExport");
Shuffler shuffle = new Shuffler(container);
if (ShuffleEventHandler != null)
{
shuffle.RaiseShuffleEvent += ShuffleEventHandler;
}
ShuffleHelper.VerifyShuffleVars(Definition, clearRemainingShuffleVars);
shuffle.Definition = Definition;
shuffle.definitionpath = defpath;
ShuffleBlocks blocks = shuffle.ExportFromCRM();
XmlDocument result = shuffle.Serialize(blocks, Type, Delimeter);
container.Logger.EndSection();
return(result);
}
private static void AddRelationFilter(ShuffleBlocks blocks, DataBlockRelation relation, XmlNode xEntity, ILoggable log)
{
if (blocks != null && blocks.Count > 0)
{
var block = relation.Block;
var attribute = relation.Attribute;
var pkattribute = relation.PKAttribute;
var includenull = relation.IncludeNull;
var ids = new List <string>();
var parentcoll = blocks.ContainsKey(block) ? blocks[block] : null;
if (parentcoll != null && parentcoll.Count > 0)
{
foreach (var parent in parentcoll)
{
if (string.IsNullOrEmpty(pkattribute))
{
ids.Add(parent.Id.ToString());
}
else
{
ids.Add(parent.Property <EntityReference>(pkattribute, new EntityReference()).Id.ToString());
}
}
}
else
{
// Adding temp guid to indicate "no matches", as ConditionOperator.In will fail if no values are given
ids.Add(new Guid().ToString());
}
if (!includenull)
{
CintFetchXML.AppendFilter(xEntity, "and", attribute, "in", ids.ToArray());
}
else
{
var xFilter = CintFetchXML.AppendFilter(xEntity, "or");
CintFetchXML.AppendCondition(xFilter, attribute, "null");
CintFetchXML.AppendCondition(xFilter, attribute, "in", ids.ToArray());
}
log.Log("Adding relation condition for {0} in {1} values in {2}.{3}", attribute, ids.Count, block, pkattribute);
}
}
private static void AddRelationFilter(IExecutionContainer container, ShuffleBlocks blocks, DataBlockRelation relation, XmlNode xEntity)
{
if (blocks != null && blocks.Count > 0)
{
var block = relation.Block;
var attribute = relation.Attribute;
var pkattribute = relation.PKAttribute;
var includenull = relation.IncludeNull;
var ids = new List <string>();
var parentcoll = blocks.ContainsKey(block) ? blocks[block] : null;
if (parentcoll != null && parentcoll.Entities.Count > 0)
{
foreach (var parent in parentcoll.Entities)
{
if (string.IsNullOrEmpty(pkattribute))
{
ids.Add(parent.Id.ToString());
}
else
{
ids.Add(parent.GetAttribute(pkattribute, new EntityReference()).Id.ToString());
}
}
}
else
{
// Adding temp guid to indicate "no matches", as ConditionOperator.In will fail if no values are given
ids.Add(new Guid().ToString());
}
if (!includenull)
{
FetchXML.AppendFilter(xEntity, "and", attribute, "in", ids.ToArray());
}
else
{
var xFilter = FetchXML.AppendFilter(xEntity, "or");
FetchXML.AppendCondition(xFilter, attribute, "null");
FetchXML.AppendCondition(xFilter, attribute, "in", ids.ToArray());
}
container.Log($"Adding relation condition for {attribute} in {ids.Count} values in {block}.{pkattribute}");
}
}
/// <summary>
/// Deserialize xml/string to blocks with entities
/// </summary>
/// <param name="serialized"></param>
/// <returns>Optional, only required for SerializationType: Text</returns>
public ShuffleBlocks Deserialize(XmlDocument serialized)
{
log.StartSection("Deserialize");
ShuffleBlocks result = new ShuffleBlocks();
if (serialized != null)
{
XmlNode root = CintXML.FindChild(serialized, "ShuffleData");
string sertype = CintXML.GetAttribute(root, "Type");
SendLine("Deserialize from {0}", sertype);
if (sertype == SerializationType.Full.ToString() ||
sertype == SerializationType.Simple.ToString() ||
sertype == SerializationType.SimpleNoId.ToString() ||
sertype == SerializationType.SimpleWithValue.ToString() ||
sertype == SerializationType.Explicit.ToString())
{
foreach (XmlNode xBlock in root.ChildNodes)
{
if (xBlock.NodeType == XmlNodeType.Element && xBlock.Name == "Block" && xBlock.ChildNodes.Count == 1)
{
string name = CintXML.GetAttribute(xBlock, "Name");
XmlDocument xml = new XmlDocument();
xml.AppendChild(xml.ImportNode(xBlock.ChildNodes[0], true));
CintDynEntityCollection cEntities = new CintDynEntityCollection(xml, crmsvc, log);
SendLine("Block {0}: {1} records", name, cEntities.Count);
result.Add(name, cEntities);
}
}
}
else if (sertype == SerializationType.Text.ToString())
{
string strdelimeter = CintXML.GetAttribute(root, "Delimeter");
char delimeter = strdelimeter.Length == 1 ? strdelimeter[0] : '\t';
XmlNode xText = CintXML.FindChild(root, "Text");
StringReader reader = new StringReader(xText.InnerText);
int line = 0;
string name = "";
StringBuilder serializedblock = null;
string current = reader.ReadLine();
while (current != null)
{
log.Log("Line {0:000}: {1}", line, current);
if (current.StartsWith("<<<") && current.Contains(">>>"))
{
log.Log("Block start");
if (!string.IsNullOrWhiteSpace(name) && serializedblock != null)
{
CintDynEntityCollection cEntities = new CintDynEntityCollection(serializedblock.ToString(), delimeter, crmsvc, log);
result.Add(name, cEntities);
SendLine("Block {0}: {1} records", name, cEntities.Count);
}
name = current.Substring(3);
name = name.Substring(0, name.IndexOf(">>>", StringComparison.Ordinal));
serializedblock = new StringBuilder();
}
else
{
serializedblock.AppendLine(current);
}
current = reader.ReadLine();
line++;
}
if (!string.IsNullOrWhiteSpace(serializedblock.ToString()))
{
CintDynEntityCollection cEntities = new CintDynEntityCollection(serializedblock.ToString(), delimeter, crmsvc, log);
result.Add(name, cEntities);
SendLine("Block {0}: {1} records", name, cEntities.Count);
}
}
}
log.EndSection();
return(result);
}
private CintDynEntityCollection ExportDataBlock(ShuffleBlocks blocks, DataBlock block)
{
log.StartSection("ExportDataBlock");
log.Log("Block: {0}", block.Name);
CintDynEntityCollection cExportEntities = null;
if (block.Export != null)
{
#region Define attributes
var attributes = block.Export.Items.Where(i => i is DataBlockExportAttributes).FirstOrDefault() as DataBlockExportAttributes;
var allcolumns = false;
var lAttributes = new List <string>();
var lNullAttributes = new List <string>();
if (attributes != null)
{
foreach (var attribute in attributes.Attribute)
{
var attr = attribute.Name;
log.Log("Adding column: {0}", attr);
lAttributes.Add(attr.Replace("*", "%"));
if (attr.Contains("*"))
{
allcolumns = true;
log.Log("Found wildcard");
}
else
{
if (attribute.IncludeNull)
{
lNullAttributes.Add(attr);
}
}
}
}
else
{
allcolumns = true;
lAttributes.Add("*");
log.Log("Attributes not specified, retrieving all");
}
#endregion Define attributes
var fetchxml = block.Export.Items.Where(i => i is string).FirstOrDefault() as string;
if (!string.IsNullOrWhiteSpace(fetchxml))
{
log.StartSection("Export entity using FetchXML");
#if DEBUG
log.Log("FetchXML:\n{0}", fetchxml);
#endif
cExportEntities = CintDynEntity.RetrieveMultiple(crmsvc, new FetchExpression(fetchxml), log);
log.EndSection();
}
else if (!block.TypeSpecified || block.Type == EntityTypes.Entity)
{
#region QueryExpression Entity
log.StartSection("Export entity " + block.Entity);
var qExport = new QueryExpression(block.Entity);
if (block.Export.ActiveOnly)
{
CintQryExp.AppendConditionActive(qExport.Criteria);
}
if (block.Relation != null)
{
foreach (var relation in block.Relation)
{
AddRelationFilter(blocks, block.Entity, relation, qExport.Criteria, log);
}
}
foreach (var filter in block.Export.Items.Where(i => i is DataBlockExportFilter).Cast <DataBlockExportFilter>())
{
AddFilter(qExport, filter);
}
foreach (var sort in block.Export.Items.Where(i => i is DataBlockExportSort).Cast <DataBlockExportSort>())
{
qExport.AddOrder(sort.Attribute, sort.Type == SortTypes.Desc ? OrderType.Descending : OrderType.Ascending);
}
if (allcolumns)
{
qExport.ColumnSet = new ColumnSet(true);
}
else
{
foreach (var attr in lAttributes)
{
qExport.ColumnSet.AddColumn(attr);
}
}
#if DEBUG
log.Log("Converting to FetchXML");
try
{
var fetch = CintQryExp.ConvertToFetchXml(qExport, crmsvc);
log.Log("Exporting {0}:\n{1}", block.Entity, fetch);
}
catch (Exception ex)
{
log.Log("Conversion error:");
log.Log(ex);
}
#endif
cExportEntities = CintDynEntity.RetrieveMultiple(crmsvc, qExport, log);
if (allcolumns)
{
SelectAttributes(cExportEntities, lAttributes, lNullAttributes);
}
SendLine("Block {0} - {1} records", block.Name, cExportEntities.Count);
log.EndSection();
#endregion QueryExpression Entity
}
else if (block.Type == EntityTypes.Intersect)
{
#region FetchXML Intersect
log.StartSection("Export intersect " + block.Entity);
var xDoc = new XmlDocument();
var xEntity = CintFetchXML.Create(xDoc, block.Entity);
CintFetchXML.AddAttribute(xEntity, lAttributes.ToArray());
if (block.Relation != null)
{
foreach (var relation in block.Relation)
{
AddRelationFilter(blocks, relation, xEntity, log);
}
}
var fetch = xDoc.OuterXml;
fetch = fetch.Replace("<fetch ", "<fetch {0} {1} "); // Detta för att se till att CrmServiceProxy.RetrieveMultiple kan hantera paging
#if DEBUG
log.Log("Exporting intersect entity {0}\n{1}", block.Entity, fetch);
#endif
var qExport = new FetchExpression(fetch);
cExportEntities = CintDynEntity.RetrieveMultiple(crmsvc, qExport, log);
foreach (var cde in cExportEntities)
{
var newattributes = new List <KeyValuePair <string, object> >();
foreach (var attr in cde.Attributes)
{
if (attr.Value is Guid)
{
var attrname = attr.Key;
var relatedentity = attrname.Substring(0, attrname.Length - (attrname.EndsWith("idone") || attrname.EndsWith("idtwo") ? 5 : 2));
newattributes.Add(new KeyValuePair <string, object>(attrname, new EntityReference(relatedentity, (Guid)attr.Value)));
}
}
foreach (var newattr in newattributes)
{
if (!newattr.Key.Equals(cde.PrimaryIdAttribute))
{
cde.AddProperty(newattr.Key, newattr.Value);
}
}
}
log.EndSection();
#endregion FetchXML Intersect
}
log.Log("Returning {0} records", cExportEntities.Count);
}
log.EndSection();
return(cExportEntities);
}
private void AddRelationFilter(ShuffleBlocks blocks, string entityName, DataBlockRelation relation, FilterExpression filter, ILoggable log)
{
log.StartSection(MethodBase.GetCurrentMethod().Name);
if (blocks != null && blocks.Count > 0)
{
var block = relation.Block;
var attribute = relation.Attribute;
var pkattribute = relation.PKAttribute;
var includenull = relation.IncludeNull;
var type = GetAttributeType(attribute, entityName);
var cond = new ConditionExpression
{
AttributeName = attribute,
Operator = Microsoft.Xrm.Sdk.Query.ConditionOperator.In
};
var ids = new List <object>();
var parentcoll = blocks.ContainsKey(block) ? blocks[block] : null;
if (parentcoll != null && parentcoll.Count > 0)
{
foreach (var parent in parentcoll)
{
if (string.IsNullOrEmpty(pkattribute))
{
if (type == AttributeTypeCode.String)
{
ids.Add(parent.Id.ToString());
}
else
{
ids.Add(parent.Id);
}
}
else if (type == AttributeTypeCode.String)
{
ids.Add(parent.Property <EntityReference>(pkattribute, new EntityReference()).Id.ToString());
}
else
{
ids.Add(parent.Property <EntityReference>(pkattribute, new EntityReference()).Id);
}
}
}
else
{
// Adding temp guid to indicate "no matches", as ConditionOperator.In will fail if no values are given
ids.Add(new Guid());
}
cond.Values.AddRange(ids);
if (!includenull)
{
filter.AddCondition(cond);
}
else
{
var orfilter = new FilterExpression(LogicalOperator.Or);
orfilter.AddCondition(attribute, Microsoft.Xrm.Sdk.Query.ConditionOperator.Null);
orfilter.AddCondition(cond);
filter.AddFilter(orfilter);
}
log.Log("Adding relation condition for {0} in {1} values in {2}.{3}", attribute, ids.Count, block, pkattribute);
}
log.EndSection();
}
private EntityCollection ExportDataBlock(IExecutionContainer container, ShuffleBlocks blocks, DataBlock block)
{
container.StartSection("ExportDataBlock");
container.Log($"Block: {block.Name}");
EntityCollection cExportEntities = null;
if (block.Export != null)
{
#region Define attributes
var attributes = block.Export.Items.Where(i => i is DataBlockExportAttributes).FirstOrDefault() as DataBlockExportAttributes;
var allcolumns = false;
var lAttributes = new List <string>();
var lNullAttributes = new List <string>();
if (attributes != null)
{
foreach (var attribute in attributes.Attribute)
{
var attr = attribute.Name;
container.Log($"Adding column: {attr}");
lAttributes.Add(attr.Replace("*", "%"));
if (attr.Contains("*"))
{
allcolumns = true;
container.Log("Found wildcard");
}
else
{
if (attribute.IncludeNull)
{
lNullAttributes.Add(attr);
}
}
}
}
else
{
allcolumns = true;
lAttributes.Add("*");
container.Log("Attributes not specified, retrieving all");
}
#endregion Define attributes
var fetchxml = block.Export.Items.Where(i => i is string).FirstOrDefault() as string;
if (!string.IsNullOrWhiteSpace(fetchxml))
{
container.StartSection("Export entity using FetchXML");
#if DEBUG
container.Log($"FetchXML:\n{fetchxml}");
#endif
cExportEntities = container.RetrieveMultiple(new FetchExpression(fetchxml));
container.EndSection();
}
else if (!block.TypeSpecified || block.Type == EntityTypes.Entity)
{
#region QueryExpression Entity
container.StartSection($"Export entity {block.Entity}");
var qExport = new QueryExpression(block.Entity);
if (block.Export.ActiveOnly)
{
Query.AppendConditionActive(qExport.Criteria);
//CintQryExp.AppendConditionActive(qExport.Criteria);
}
if (block.Relation != null)
{
foreach (var relation in block.Relation)
{
AddRelationFilter(container, blocks, block.Entity, relation, qExport.Criteria);
}
}
foreach (var filter in block.Export.Items.Where(i => i is DataBlockExportFilter).Cast <DataBlockExportFilter>())
{
AddFilter(container, qExport, filter);
}
foreach (var sort in block.Export.Items.Where(i => i is DataBlockExportSort).Cast <DataBlockExportSort>())
{
qExport.AddOrder(sort.Attribute, sort.Type == SortTypes.Desc ? OrderType.Descending : OrderType.Ascending);
}
if (allcolumns)
{
qExport.ColumnSet = new ColumnSet(true);
}
else
{
foreach (var attr in lAttributes)
{
qExport.ColumnSet.AddColumn(attr);
}
}
#if DEBUG
container.Log("Converting to FetchXML");
try
{
var fetch = container.ConvertToFetchXml(qExport);
container.Log($"Exporting {block.Entity}:\n{fetch}");
}
catch (Exception ex)
{
container.Log("Conversion error:");
container.Log(ex);
}
#endif
cExportEntities = container.RetrieveMultiple(qExport);
if (allcolumns)
{
SelectAttributes(container, cExportEntities, lAttributes, lNullAttributes);
}
SendLine(container, $"Block {block.Name} - {cExportEntities.Count()} records");
container.EndSection();
#endregion QueryExpression Entity
}
else if (block.Type == EntityTypes.Intersect)
{
#region FetchXML Intersect
container.StartSection($"Export intersect {block.Entity}");
var xDoc = new XmlDocument();
var xEntity = FetchXML.Create(xDoc, block.Entity);
FetchXML.AddAttribute(xEntity, lAttributes.ToArray());
if (block.Relation != null)
{
foreach (var relation in block.Relation)
{
AddRelationFilter(container, blocks, relation, xEntity);
}
}
var fetch = xDoc.OuterXml;
//Imran: Removed because this is causing invalid Xml errors. Could not see the point of having these placeholders.
//fetch = fetch.Replace("<fetch ", "<fetch {0} {1} "); // Detta för att se till att CrmServiceProxy.RetrieveMultiple kan hantera paging
#if DEBUG
container.Log($"Exporting intersect entity {block.Entity}\n{fetch}");
#endif
var qExport = new FetchExpression(fetch);
cExportEntities = container.RetrieveMultiple(qExport);
foreach (var entity in cExportEntities.Entities)
{
var newattributes = new List <KeyValuePair <string, object> >();
foreach (var attr in entity.Attributes)
{
if (attr.Value is Guid guid)
{
var attrname = attr.Key;
var relatedentity = attrname.Substring(0, attrname.Length - (attrname.EndsWith("idone") || attrname.EndsWith("idtwo") ? 5 : 2));
if (!newattributes.Contains(new KeyValuePair <string, object>(attrname, new EntityReference(relatedentity, guid))))
{
container.Log($"Adding Attribute {attrname} - Related entity {relatedentity}");
newattributes.Add(new KeyValuePair <string, object>(attrname, new EntityReference(relatedentity, guid)));
#if DEBUG
container.Log($"{attrname} added");
#endif
}
else
{
#if DEBUG
container.Log($"{attrname} already exists.");
#endif
}
}
}
foreach (var newattr in newattributes)
{
#if DEBUG
container.Log($"Entity {entity.LogicalName} contains attribute {newattr.Key}: {entity.Attributes.Contains(newattr.Key)}");
#endif
if (!entity.Attributes.Contains(newattr.Key))
{
entity.Attributes.Add(newattr.Key, newattr.Value);
}
}
}
container.EndSection();
#endregion FetchXML Intersect
}
container.Log($"Returning {cExportEntities.Count()} records");
}
container.EndSection();
return(cExportEntities);
}
/// <summary>
/// Import entities to CRM from dictionary of blocks
/// </summary>
/// <param name="blocks">Blocks with entities to import</param>
/// <returns>Tuple with counters for: Created, Updated, Skipped and Failed records</returns>
public Tuple <int, int, int, int, int, EntityReferenceCollection> ImportToCRM(ShuffleBlocks blocks)
{
log.StartSection("ImportToCRM");
if (definition == null)
{
throw new ArgumentNullException("Definition", "Shuffle definition must be specified to import data");
}
var created = 0;
var updated = 0;
var skipped = 0;
var deleted = 0;
var failed = 0;
var references = new EntityReferenceCollection();
if (ShuffleDefinition.Blocks.Items.Any(b => (b is DataBlock data && data.Import != null) || b is SolutionBlock sol && sol.Import != null))
{
guidmap = new Dictionary <Guid, Guid>();
stoponerror = ShuffleDefinition.StopOnError;
timeout = ShuffleDefinition.TimeoutSpecified ? ShuffleDefinition.Timeout : -1;
double savedtimeout = -1;
if (timeout > -1)
{
savedtimeout = SetTimeout();
}
var totalBlocks = ShuffleDefinition.Blocks.Items.Length;
var currentBlock = 0;
foreach (var block in ShuffleDefinition.Blocks.Items)
{
currentBlock++;
SendStatus(totalBlocks, currentBlock, -1, -1);
if (block is DataBlock datablock)
{
var name = datablock.Name;
if (!blocks.ContainsKey(name))
{
blocks.Add(name, new CintDynEntityCollection());
}
var dataresult = ImportDataBlock(datablock, blocks[name]);
created += dataresult.Item1;
updated += dataresult.Item2;
skipped += dataresult.Item3;
deleted += dataresult.Item4;
failed += dataresult.Item5;
references.AddRange(dataresult.Item6);
}
else if (block is SolutionBlock solutionblock)
{
var solutionresult = ImportSolutionBlock(solutionblock);
switch (solutionresult)
{
case ItemImportResult.Created: created++; break;
case ItemImportResult.Updated: updated++; break;
case ItemImportResult.Skipped: skipped++; break;
case ItemImportResult.Failed: failed++; break;
}
}
}
SendStatus(0, 0, 0, 0);
if (savedtimeout > -1)
{
ResetTimeout(savedtimeout);
}
}
log.EndSection();
return(new Tuple <int, int, int, int, int, EntityReferenceCollection>(created, updated, skipped, deleted, failed, references));
}
/// <summary>
/// Deserialize xml/string to blocks with entities
/// </summary>
/// <param name="container"></param>
/// <param name="serialized"></param>
/// <returns>Optional, only required for SerializationType: Text</returns>
public ShuffleBlocks Deserialize(IExecutionContainer container, XmlDocument serialized)
{
container.StartSection("Deserialize");
var result = new ShuffleBlocks();
if (serialized != null)
{
var root = XML.FindChild(serialized, "ShuffleData");
var sertype = XML.GetAttribute(root, "Type");
SendLine(container, $"Deserialize from {sertype}");
if (sertype == SerializationType.Full.ToString() ||
sertype == SerializationType.Simple.ToString() ||
sertype == SerializationType.SimpleNoId.ToString() ||
sertype == SerializationType.SimpleWithValue.ToString() ||
sertype == SerializationType.Explicit.ToString())
{
foreach (XmlNode xBlock in root.ChildNodes)
{
if (xBlock.NodeType == XmlNodeType.Element && xBlock.Name == "Block" && xBlock.ChildNodes.Count == 1)
{
var name = XML.GetAttribute(xBlock, "Name");
var xml = new XmlDocument();
xml.AppendChild(xml.ImportNode(xBlock.ChildNodes[0], true));
var cEntities = container.CreateEntityCollection(xml);
SendLine(container, $"Block {name}: {cEntities.Count()} records");
result.Add(name, cEntities);
}
}
}
else if (sertype == SerializationType.Text.ToString())
{
var strdelimeter = XML.GetAttribute(root, "Delimeter");
var delimeter = strdelimeter.Length == 1 ? strdelimeter[0] : '\t';
var xText = XML.FindChild(root, "Text");
var reader = new StringReader(xText.InnerText);
var line = 0;
var name = "";
StringBuilder serializedblock = null;
var current = reader.ReadLine();
while (current != null)
{
container.Log("Line {0:000}: {1}", line, current);
if (current.StartsWith("<<<") && current.Contains(">>>"))
{
container.Log("Block start");
if (!string.IsNullOrWhiteSpace(name) && serializedblock != null)
{
var cEntities = container.CreateEntityCollection(serializedblock.ToString(), delimeter);
result.Add(name, cEntities);
SendLine(container, $"Block {name}: {cEntities.Count()} records");
}
name = current.Substring(3);
name = name.Substring(0, name.IndexOf(">>>", StringComparison.Ordinal));
serializedblock = new StringBuilder();
}
else
{
serializedblock.AppendLine(current);
}
current = reader.ReadLine();
line++;
}
if (!string.IsNullOrWhiteSpace(serializedblock.ToString()))
{
var cEntities = container.CreateEntityCollection(serializedblock.ToString(), delimeter);
result.Add(name, cEntities);
SendLine(container, $"Block {name}: {cEntities.Count()} records");
}
}
}
container.EndSection();
return(result);
}
/// <summary>
/// Export entities from CRM to dictionary of blocks with entities
/// </summary>
/// <returns>Blocks with exported entities</returns>
public ShuffleBlocks ExportFromCRM()
{
log.StartSection("ExportFromCRM");
if (definition == null)
{
throw new ArgumentNullException("Definition", "Shuffle definition must be specified to export data");
}
ShuffleBlocks blocks = new ShuffleBlocks();
ExistingSolutionVersions = null;
XmlNode xRoot = CintXML.FindChild(definition, "ShuffleDefinition");
XmlNode xBlocks = CintXML.FindChild(xRoot, "Blocks");
if (xBlocks != null)
{
stoponerror = CintXML.GetBoolAttribute(xRoot, "StopOnError", false);
timeout = CintXML.GetIntAttribute(xRoot, "Timeout", -1);
double savedtimeout = -1;
if (timeout > -1)
{
SendLine("Setting timeout: {0} minutes", timeout);
OrganizationServiceProxy orgsvcpxy = crmsvc.GetService <OrganizationServiceProxy>();
savedtimeout = orgsvcpxy.Timeout.TotalMinutes;
orgsvcpxy.Timeout = new TimeSpan(0, timeout, 0);
}
int totalBlocks = xBlocks.ChildNodes.Count;
int currentBlock = 0;
foreach (XmlNode xBlock in xBlocks.ChildNodes)
{
currentBlock++;
SendStatus(totalBlocks, currentBlock, -1, -1);
if (xBlock.NodeType == XmlNodeType.Element)
{
switch (xBlock.Name)
{
case "DataBlock":
CintDynEntityCollection cExported = ExportDataBlock(blocks, xBlock);
string name = CintXML.GetAttribute(xBlock, "Name");
if (cExported != null)
{
if (blocks.ContainsKey(name))
{
SendLine("Block already added: {0}", name);
}
else
{
blocks.Add(name, cExported);
}
}
break;
case "SolutionBlock":
if (ExistingSolutionVersions == null)
{
GetCurrentVersions();
}
ExportSolutionBlock(xBlock);
break;
}
}
}
SendStatus(0, 0, 0, 0);
if (savedtimeout > -1)
{
OrganizationServiceProxy orgsvcpxy = crmsvc.GetService <OrganizationServiceProxy>();
orgsvcpxy.Timeout = new TimeSpan(0, (int)savedtimeout, 0);
}
}
log.EndSection();
return(blocks);
}
/// <summary>
/// Import entities to CRM from dictionary of blocks
/// </summary>
/// <param name="blocks">Blocks with entities to import</param>
/// <returns>Tuple with counters for: Created, Updated, Skipped and Failed records</returns>
public Tuple <int, int, int, int, int, EntityReferenceCollection> ImportToCRM(ShuffleBlocks blocks)
{
log.StartSection("ImportToCRM");
if (definition == null)
{
throw new ArgumentNullException("Definition", "Shuffle definition must be specified to import data");
}
int created = 0;
int updated = 0;
int skipped = 0;
int deleted = 0;
int failed = 0;
EntityReferenceCollection references = new EntityReferenceCollection();
XmlNode xRoot = CintXML.FindChild(definition, "ShuffleDefinition");
XmlNode xBlocks = CintXML.FindChild(xRoot, "Blocks");
if (xBlocks != null)
{
guidmap = new Dictionary <Guid, Guid>();
stoponerror = CintXML.GetBoolAttribute(xRoot, "StopOnError", false);
timeout = CintXML.GetIntAttribute(xRoot, "Timeout", -1);
double savedtimeout = -1;
if (timeout > -1)
{
try
{
SendLine("Setting timeout: {0} minutes", timeout);
OrganizationServiceProxy orgsvcpxy = crmsvc.GetService <OrganizationServiceProxy>();
savedtimeout = orgsvcpxy.Timeout.TotalMinutes;
orgsvcpxy.Timeout = new TimeSpan(0, timeout, 0);
}
catch (InvalidPluginExecutionException)
{ // Couldn't cast to correct service type, for some reason...
savedtimeout = -1;
}
}
int totalBlocks = xBlocks.ChildNodes.Count;
int currentBlock = 0;
foreach (XmlNode xBlock in xBlocks.ChildNodes)
{
currentBlock++;
SendStatus(totalBlocks, currentBlock, -1, -1);
if (xBlock.NodeType == XmlNodeType.Element)
{
switch (xBlock.Name)
{
case "DataBlock":
string name = CintXML.GetAttribute(xBlock, "Name");
if (!blocks.ContainsKey(name))
{
blocks.Add(name, new CintDynEntityCollection());
}
Tuple <int, int, int, int, int, EntityReferenceCollection> dataresult = ImportDataBlock(xBlock, blocks[name]);
created += dataresult.Item1;
updated += dataresult.Item2;
skipped += dataresult.Item3;
deleted += dataresult.Item4;
failed += dataresult.Item5;
references.AddRange(dataresult.Item6);
break;
case "SolutionBlock":
var solutionresult = ImportSolutionBlock(xBlock);
switch (solutionresult)
{
case ItemImportResult.Created: created++; break;
case ItemImportResult.Updated: updated++; break;
case ItemImportResult.Skipped: skipped++; break;
case ItemImportResult.Failed: failed++; break;
}
break;
}
}
}
SendStatus(0, 0, 0, 0);
if (savedtimeout > -1)
{
OrganizationServiceProxy orgsvcpxy = crmsvc.GetService <OrganizationServiceProxy>();
orgsvcpxy.Timeout = new TimeSpan(0, (int)savedtimeout, 0);
}
}
log.EndSection();
return(new Tuple <int, int, int, int, int, EntityReferenceCollection>(created, updated, skipped, deleted, failed, references));
}
private CintDynEntityCollection ExportDataBlock(ShuffleBlocks blocks, XmlNode xBlock)
{
log.StartSection("ExportDataBlock");
string name = CintXML.GetAttribute(xBlock, "Name");
log.Log("Block: {0}", name);
CintDynEntityCollection cExportEntities = null;
if (xBlock.Name != "DataBlock")
{
throw new ArgumentOutOfRangeException("Type", xBlock.Name, "Invalid Block type");
}
string entity = CintXML.GetAttribute(xBlock, "Entity");
string type = CintXML.GetAttribute(xBlock, "Type");
XmlNode xExport = CintXML.FindChild(xBlock, "Export");
if (xExport != null)
{
if (string.IsNullOrEmpty(entity))
{
entity = CintXML.GetAttribute(xExport, "Entity");
}
#region Define attributes
XmlNode xAttributes = CintXML.FindChild(xExport, "Attributes");
bool allcolumns = false;
List <string> lAttributes = new List <string>();
List <string> lNullAttributes = new List <string>();
if (xAttributes != null)
{
foreach (XmlNode xAttr in xAttributes.ChildNodes)
{
if (xAttr.Name == "Attribute")
{
string attr = CintXML.GetAttribute(xAttr, "Name");
log.Log("Adding column: {0}", attr);
lAttributes.Add(attr.Replace("*", "%"));
if (attr.Contains("*"))
{
allcolumns = true;
log.Log("Found wildcard");
}
else
{
bool includenull = CintXML.GetBoolAttribute(xAttr, "IncludeNull", false);
if (includenull)
{
lNullAttributes.Add(attr);
}
}
}
}
}
else
{
allcolumns = true;
lAttributes.Add("*");
log.Log("Attributes not specified, retrieving all");
}
#endregion Define attributes
if (type == "Entity" || string.IsNullOrEmpty(type))
{
#region QueryExpression Entity
log.StartSection("Export entity " + entity);
QueryExpression qExport = new QueryExpression(entity);
if (CintXML.GetBoolAttribute(xExport, "ActiveOnly", true))
{
CintQryExp.AppendConditionActive(qExport.Criteria);
}
foreach (var xBlockChild in xBlock.ChildNodes.Cast <XmlNode>())
{
if (xBlockChild.Name == "Relation")
{
AddRelationFilter(blocks, xBlockChild, qExport.Criteria, log);
}
}
foreach (XmlNode xExpProp in xExport.ChildNodes)
{
if (xExport.NodeType == XmlNodeType.Element)
{
switch (xExpProp.Name)
{
case "#comment":
case "Attributes":
break;
case "Filter":
AddFilter(qExport, xExpProp);
break;
case "Sort":
qExport.AddOrder(
CintXML.GetAttribute(xExpProp, "Attribute"),
CintXML.GetAttribute(xExpProp, "Type") == "Desc" ? OrderType.Descending : OrderType.Ascending);
break;
default:
throw new ArgumentOutOfRangeException("Name", xExpProp.Name, "Invalid subitem to export block " + name);
}
}
}
if (allcolumns)
{
qExport.ColumnSet = new ColumnSet(true);
}
else
{
foreach (string attr in lAttributes)
{
qExport.ColumnSet.AddColumn(attr);
}
}
#if DEBUG
log.Log("Converting to FetchXML");
try
{
var fetch = CintQryExp.ConvertToFetchXml(qExport, crmsvc);
log.Log("Exporting {0}:\n{1}", entity, fetch);
}
catch (Exception ex)
{
log.Log("Conversion error:");
log.Log(ex);
}
#endif
cExportEntities = CintDynEntity.RetrieveMultiple(crmsvc, qExport, log);
if (allcolumns)
{
SelectAttributes(cExportEntities, lAttributes, lNullAttributes);
}
SendLine("Block {0} - {1} records", name, cExportEntities.Count);
log.EndSection();
#endregion QueryExpression Entity
}
else if (type == "Intersect")
{
#region FetchXML Intersect
log.StartSection("Export intersect " + entity);
XmlDocument xDoc = new XmlDocument();
XmlNode xEntity = CintFetchXML.Create(xDoc, entity);
CintFetchXML.AddAttribute(xEntity, lAttributes.ToArray());
foreach (var xBlockChild in xBlock.ChildNodes.Cast <XmlNode>())
{
if (xBlockChild.Name == "Relation")
{
AddRelationFilter(blocks, xBlockChild, xEntity, log);
}
}
var fetch = xDoc.OuterXml;
fetch = fetch.Replace("<fetch ", "<fetch {0} {1} "); // Detta för att se till att CrmServiceProxy.RetrieveMultiple kan hantera paging
#if DEBUG
log.Log("Exporting intersect entity {0}\n{1}", entity, fetch);
#endif
var qExport = new FetchExpression(fetch);
cExportEntities = CintDynEntity.RetrieveMultiple(crmsvc, qExport, log);
foreach (var cde in cExportEntities)
{
List <KeyValuePair <string, object> > newattributes = new List <KeyValuePair <string, object> >();
foreach (var attr in cde.Attributes)
{
if (attr.Value is Guid)
{
var attrname = attr.Key;
string relatedentity = attrname.Substring(0, attrname.Length - (attrname.EndsWith("idone") || attrname.EndsWith("idtwo") ? 5 : 2));
newattributes.Add(new KeyValuePair <string, object>(attrname, new EntityReference(relatedentity, (Guid)attr.Value)));
}
}
foreach (var newattr in newattributes)
{
if (!newattr.Key.Equals(cde.PrimaryIdAttribute))
{
cde.AddProperty(newattr.Key, newattr.Value);
}
}
}
log.EndSection();
#endregion FetchXML Intersect
}
log.Log("Returning {0} records", cExportEntities.Count);
}
log.EndSection();
return(cExportEntities);
}
