public override ISourceItem DeserializeItem(ISerializedItem serializedItem, bool ignoreMissingTemplateFields)
{
Assert.ArgumentNotNull(serializedItem, "serializedItem");
var typed = serializedItem as SitecoreSerializedItem;
if (typed == null) throw new ArgumentException("Serialized item must be a SitecoreSerializedItem", "serializedItem");
try
{
return new SitecoreSourceItem(_deserializer.PasteSyncItem(typed.InnerItem, ignoreMissingTemplateFields));
}
catch (ParentItemNotFoundException ex)
{
string error = "Cannot load item from path '{0}'. Probable reason: parent item with ID '{1}' not found.".FormatWith(serializedItem.ProviderId, ex.ParentID);
throw new DeserializationException(error, ex);
}
catch (ParentForMovedItemNotFoundException ex2)
{
string error = "Item from path '{0}' cannot be moved to appropriate location. Possible reason: parent item with ID '{1}' not found.".FormatWith(serializedItem.ProviderId, ex2.ParentID);
throw new DeserializationException(error, ex2);
}
}
public override ISourceItem DeserializeItem(ISerializedItem serializedItem, bool ignoreMissingTemplateFields)
{
Assert.ArgumentNotNull(serializedItem, "serializedItem");
var typed = serializedItem as SitecoreSerializedItem;
if (typed == null)
{
throw new ArgumentException("Serialized item must be a SitecoreSerializedItem", "serializedItem");
}
try
{
return(new SitecoreSourceItem(_deserializer.PasteSyncItem(typed.InnerItem, ignoreMissingTemplateFields)));
}
catch (ParentItemNotFoundException ex)
{
string error = "Cannot load item from path '{0}'. Probable reason: parent item with ID '{1}' not found.".FormatWith(serializedItem.ProviderId, ex.ParentID);
throw new DeserializationException(error, ex);
}
catch (ParentForMovedItemNotFoundException ex2)
{
string error = "Item from path '{0}' cannot be moved to appropriate location. Possible reason: parent item with ID '{1}' not found.".FormatWith(serializedItem.ProviderId, ex2.ParentID);
throw new DeserializationException(error, ex2);
}
}
protected virtual void UpdateSerializedItem(ISerializedItem serializedItem)
{
var typed = serializedItem as SitecoreSerializedItem;
if (typed == null)
{
throw new ArgumentException("Serialized item must be a SitecoreSerializedItem", "serializedItem");
}
// create any requisite parent folder(s) for the serialized item
var parentPath = Directory.GetParent(SerializationPathUtility.GetReferenceDirectoryPath(serializedItem));
if (parentPath != null && !parentPath.Exists)
{
Directory.CreateDirectory(parentPath.FullName);
}
// if the file already exists, delete it. Why? Because the FS is case-insensitive, if an item is renamed by case only it won't actually rename
// on the filesystem. Deleting it first makes sure this occurs.
if (File.Exists(serializedItem.ProviderId))
{
File.Delete(serializedItem.ProviderId);
}
using (var fileStream = File.Open(serializedItem.ProviderId, FileMode.Create, FileAccess.Write, FileShare.None))
{
using (var writer = new StreamWriter(fileStream))
{
typed.InnerItem.Serialize(writer);
}
}
}
public virtual ISourceItem Deserialize(ISerializedItem serializedItem, bool ignoreMissingTemplateFields)
{
if (ignoreMissingTemplateFields)
{
throw new NotSupportedException("The Sitecore serialization engine does not support ignoring missing fields.");
}
try
{
var options = new LoadOptions {
DisableEvents = true, ForceUpdate = true, UseNewID = false
};
return(new SitecoreSourceItem(ItemSynchronization.PasteSyncItem(((SitecoreSerializedItem)serializedItem).InnerItem, options, true)));
}
catch (ParentItemNotFoundException ex)
{
string error = "Cannot load item from path '{0}'. Probable reason: parent item with ID '{1}' not found.".FormatWith(serializedItem.ProviderId, ex.ParentID);
throw new DeserializationException(error, ex);
}
catch (ParentForMovedItemNotFoundException ex2)
{
string error = "Item from path '{0}' cannot be moved to appropriate location. Possible reason: parent item with ID '{1}' not found.".FormatWith(serializedItem.ProviderId, ex2.ParentID);
throw new DeserializationException(error, ex2);
}
}
public virtual ISourceItem DeserializeItem(ISerializedItem serializedItem, bool ignoreMissingTemplateFields)
{
Assert.ArgumentNotNull(serializedItem, "serializedItem");
var typed = serializedItem as SitecoreSerializedItem;
if (typed == null)
{
throw new ArgumentException("Serialized item must be a SitecoreSerializedItem", "serializedItem");
}
try
{
var options = new LoadOptions {
DisableEvents = true, ForceUpdate = true, UseNewID = false
};
return(new SitecoreSourceItem(ItemSynchronization.PasteSyncItem(typed.InnerItem, options, true)));
}
catch (ParentItemNotFoundException ex)
{
string error = "Cannot load item from path '{0}'. Probable reason: parent item with ID '{1}' not found.".FormatWith(serializedItem.ProviderId, ex.ParentID);
throw new DeserializationException(error, ex);
}
catch (ParentForMovedItemNotFoundException ex2)
{
string error = "Item from path '{0}' cannot be moved to appropriate location. Possible reason: parent item with ID '{1}' not found.".FormatWith(serializedItem.ProviderId, ex2.ParentID);
throw new DeserializationException(error, ex2);
}
}
protected virtual bool ShouldUpdateExisting(ISerializedItem serializedItem, ISourceItem existingItem, DeferredLogWriter <ISerializedAsMasterEvaluatorLogger> deferredUpdateLog)
{
Assert.ArgumentNotNull(serializedItem, "serializedItem");
Assert.ArgumentNotNull(existingItem, "existingItem");
if (existingItem.Id == RootId)
{
return(false); // we never want to update the Sitecore root item
}
// check if templates are different
if (IsTemplateMatch(existingItem, serializedItem, deferredUpdateLog))
{
return(true);
}
// check if names are different
if (IsNameMatch(existingItem, serializedItem, deferredUpdateLog))
{
return(true);
}
// check if source has version(s) that serialized does not
var orphanVersions = existingItem.Versions.Where(sourceItemVersion => serializedItem.GetVersion(sourceItemVersion.Language, sourceItemVersion.VersionNumber) == null).ToArray();
if (orphanVersions.Length > 0)
{
deferredUpdateLog.AddEntry(x => x.OrphanSourceVersion(existingItem, serializedItem, orphanVersions));
return(true); // source contained versions not present in the serialized version, which is a difference
}
// check if shared fields have any mismatching values
if (AnyFieldMatch(serializedItem.SharedFields, existingItem.SharedFields, existingItem, serializedItem, deferredUpdateLog))
{
return(true);
}
// see if the serialized versions have any mismatching values in the source data
return(serializedItem.Versions.Any(serializedItemVersion =>
{
var sourceItemVersion = existingItem.GetVersion(serializedItemVersion.Language, serializedItemVersion.VersionNumber);
// version exists in serialized item but does not in source version
if (sourceItemVersion == null)
{
deferredUpdateLog.AddEntry(x => x.NewSerializedVersionMatch(serializedItemVersion, serializedItem, existingItem));
return true;
}
// field values mismatch
var fieldMatch = AnyFieldMatch(serializedItemVersion.Fields, sourceItemVersion.Fields, existingItem, serializedItem, deferredUpdateLog, serializedItemVersion);
if (fieldMatch)
{
return true;
}
// if we get here everything matches to the best of our knowledge, so we return false (e.g. "do not update this item")
return false;
}));
}
protected virtual ISourceItem DoDeserialization(ISerializedItem serializedItem)
{
ISourceItem updatedItem = serializedItem.Deserialize(false);
Assert.IsNotNull(updatedItem, "Do not return null from DeserializeItem() - throw an exception if an error occurs.");
return(updatedItem);
}
public bool IsConsistent(ISerializedItem item)
{
ISerializedItem duplicateItem;
if(!_duplicateChecks.TryGetValue(CreateKey(item), out duplicateItem)) return true;
_logger.DuplicateFound(duplicateItem, item);
return false;
}
public static ItemVersion GetVersion(this ISerializedItem serializedItem, string language, int versionNumber)
{
if (language.Equals(Language.Invariant.Name))
{
language = LanguageManager.DefaultLanguage.Name;
}
return(serializedItem.Versions.FirstOrDefault(x => x.Language.Equals(language, StringComparison.OrdinalIgnoreCase) && x.VersionNumber == versionNumber));
}
public ISourceItem EvaluateNewSerializedItem(ISerializedItem newItem)
{
Assert.ArgumentNotNull(newItem, "newItem");
_logger.DeserializedNewItem(newItem);
var updatedItem = DoDeserialization(newItem);
return(updatedItem);
}
public virtual void IsVersionedFieldMatch(ISerializedItem serializedItem, ItemVersion version, string fieldName, string serializedValue, string sourceValue)
{
if (serializedValue.Length < MaxFieldLenthToDisplayValue && (sourceValue == null || sourceValue.Length < MaxFieldLenthToDisplayValue))
{
_logger.Debug("> Field {0} - {1}#{2}: Serialized {3}, Source {4}".FormatWith(fieldName, version.Language, version.VersionNumber, serializedValue, sourceValue));
}
else
{
_logger.Debug("> Field {0} - {1}#{2}: Value mismatch (values too long to display)".FormatWith(fieldName, version.Language, version.VersionNumber));
}
}
protected virtual bool IsNameMatch(ISourceItem existingItem, ISerializedItem serializedItem, DeferredLogWriter <ISerializedAsMasterEvaluatorLogger> deferredUpdateLog)
{
if (!serializedItem.Name.Equals(existingItem.Name))
{
deferredUpdateLog.AddEntry(x => x.IsNameMatch(serializedItem, existingItem));
return(true);
}
return(false);
}
public virtual void IsSharedFieldMatch(ISerializedItem serializedItem, string fieldName, string serializedValue, string sourceValue)
{
if (serializedValue.Length < MaxFieldLenthToDisplayValue && (sourceValue == null || sourceValue.Length < MaxFieldLenthToDisplayValue))
{
_logger.Debug("> Field {0} - Serialized {1}, Source {2}".FormatWith(fieldName, serializedValue, sourceValue));
}
else
{
_logger.Debug("> Field {0} - Value mismatch (values too long to display)".FormatWith(fieldName));
}
}
public virtual void IsVersionedFieldMatch(ISerializedItem serializedItem, ItemVersion version, string fieldName, string serializedValue, string sourceValue)
{
if (serializedValue.Length < MaxFieldLenthToDisplayValue && (sourceValue == null || sourceValue.Length < MaxFieldLenthToDisplayValue))
{
_logger.Debug("> Field {0} - {1}#{2}: Serialized {3}, Source {4}".FormatWith(fieldName, version.Language, version.VersionNumber, serializedValue, sourceValue));
}
else
{
_logger.Debug("> Field {0} - {1}#{2}: Value mismatch (values too long to display)".FormatWith(fieldName, version.Language, version.VersionNumber));
}
}
public virtual void IsSharedFieldMatch(ISerializedItem serializedItem, string fieldName, string serializedValue, string sourceValue)
{
if (serializedValue.Length < MaxFieldLenthToDisplayValue && (sourceValue == null || sourceValue.Length < MaxFieldLenthToDisplayValue))
{
_logger.Debug("> Field {0} - Serialized {1}, Source {2}".FormatWith(fieldName, serializedValue, sourceValue));
}
else
{
_logger.Debug("> Field {0} - Value mismatch (values too long to display)".FormatWith(fieldName));
}
}
public bool IsConsistent(ISerializedItem item)
{
ISerializedItem duplicateItem;
if (!_duplicateChecks.TryGetValue(CreateKey(item), out duplicateItem))
{
return(true);
}
_logger.DuplicateFound(duplicateItem, item);
return(false);
}
/// <summary>
/// Loads a specific item from disk
/// </summary>
protected virtual ItemLoadResult DoLoadItem(ISerializedItem serializedItem, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(serializedItem, "serializedItem");
if (consistencyChecker != null)
{
if (!consistencyChecker.IsConsistent(serializedItem))
{
throw new ConsistencyException("Consistency check failed - aborting loading.");
}
consistencyChecker.AddProcessedItem(serializedItem);
}
bool disableNewSerialization = UnicornDataProvider.DisableSerialization;
try
{
UnicornDataProvider.DisableSerialization = true;
_itemsProcessed++;
var included = Predicate.Includes(serializedItem);
if (!included.IsIncluded)
{
Logger.SkippedItemPresentInSerializationProvider(serializedItem, Predicate.GetType().Name, SerializationProvider.GetType().Name, included.Justification ?? string.Empty);
return(new ItemLoadResult(ItemLoadStatus.Skipped));
}
// detect if we should run an update for the item or if it's already up to date
var existingItem = SourceDataProvider.GetItemById(serializedItem.DatabaseName, serializedItem.Id);
ISourceItem updatedItem;
// note that the evaluator is responsible for actual action being taken here
// as well as logging what it does
if (existingItem == null)
{
updatedItem = Evaluator.EvaluateNewSerializedItem(serializedItem);
}
else
{
updatedItem = Evaluator.EvaluateUpdate(serializedItem, existingItem);
}
return(new ItemLoadResult(ItemLoadStatus.Success, updatedItem ?? existingItem));
}
finally
{
UnicornDataProvider.DisableSerialization = disableNewSerialization;
}
}
private void TestLoadTree(SerializationLoader loader, ISerializedItem root, IDeserializeFailureRetryer retryer = null, IConsistencyChecker consistencyChecker = null)
{
if (retryer == null)
{
retryer = new Mock <IDeserializeFailureRetryer>().Object;
}
if (consistencyChecker == null)
{
var checker = new Mock <IConsistencyChecker>();
checker.Setup(x => x.IsConsistent(It.IsAny <ISerializedItem>())).Returns(true);
consistencyChecker = checker.Object;
}
loader.LoadTree(root, retryer, consistencyChecker);
}
public bool IsStandardValuesItem(ISerializedItem item)
{
Assert.ArgumentNotNull(item, "item");
string[] array = item.ItemPath.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries);
if (array.Length > 0)
{
if (array.Any(s => s.Equals("templates", StringComparison.OrdinalIgnoreCase)))
{
return(array.Last().Equals("__Standard Values", StringComparison.OrdinalIgnoreCase));
}
}
return(false);
}
protected virtual bool IsTemplateMatch(ISourceItem existingItem, ISerializedItem serializedItem, DeferredLogWriter <ISerializedAsMasterEvaluatorLogger> deferredUpdateLog)
{
if (existingItem.TemplateId == (ID)null && serializedItem.TemplateId == (ID)null)
{
return(false);
}
bool match = !serializedItem.TemplateId.Equals(existingItem.TemplateId);
if (match)
{
deferredUpdateLog.AddEntry(x => x.IsTemplateMatch(serializedItem, existingItem));
}
return(match);
}
/// <summary>
/// Moves the descendant items of a serialized parent after it has been moved or renamed.
/// </summary>
/// <param name="oldReference">Reference to the original path pre-move/rename</param>
/// <param name="newItem">The newly renamed or moved parent item</param>
/// <param name="sourceItem">The source item representing the renamed/moved item. NOTE that the path of this item is incorrect a lot of the time so we ignore it.</param>
/// <param name="renaming">True for moving renamed children, false for moving moved children. For renames, the children already have correct new paths; for moves we have to recalculate it.</param>
/// <remarks>
/// This method basically gets all descendants of the source item that was moved/renamed, generates an appropriate new serialized item for it, and _if the new child item is in the predicate_ we
/// serialize it to its new location. Finally, we delete the old children directory if it existed.
///
/// Doing it this way allows handling crazy cases like moving trees of items between included and excluded locations - or even moves or renames causing SOME of the children to be ignored. Wild.
/// </remarks>
protected virtual void MoveDescendants(ISerializedReference oldReference, ISerializedItem newItem, ISourceItem sourceItem, bool renaming)
{
// remove the extension from the new item's provider ID
string newItemReferencePath = SerializationPathUtility.GetReferenceDirectoryPath(newItem);
// if the paths were the same, no moving occurs (this can happen when saving templates, which spuriously can report "renamed" when they are not actually any such thing)
if (oldReference.ProviderId.Equals(newItemReferencePath, StringComparison.Ordinal))
{
return;
}
// this is for renaming an item that differs only by case from the original. Because NTFS is case-insensitive the 'new parent' exists
// already, but it will use the old name. Not quite what we want. So we need to manually rename the folder.
if (oldReference.ProviderId.Equals(newItemReferencePath, StringComparison.OrdinalIgnoreCase) && Directory.Exists(oldReference.ProviderId))
{
Directory.Move(oldReference.ProviderId, oldReference.ProviderId + "_tempunicorn");
Directory.Move(oldReference.ProviderId + "_tempunicorn", newItemReferencePath);
}
var descendantItems = GetDescendants(sourceItem).Cast <SitecoreSourceItem>();
// Go through descendant source items and serialize all that are included by the predicate
foreach (var descendant in descendantItems)
{
var syncItem = ItemSynchronization.BuildSyncItem(descendant.InnerItem);
// the newPhysicalPath will point to the OLD physical path pre-move (but for renames we actually get the new path already).
// For moves, we re-root the path to point to the new parent item's base path to fix that before we write to disk
string newItemPath = (renaming) ? descendant.ItemPath : descendant.ItemPath.Replace(oldReference.ItemPath, newItem.ItemPath);
var newPhysicalPath = SerializationPathUtility.GetSerializedItemPath(_rootPath, syncItem.DatabaseName, newItemPath);
var newSerializedItem = new SitecoreSerializedItem(syncItem, newPhysicalPath, this);
if (!_predicate.Includes(newSerializedItem).IsIncluded)
{
continue; // if the moved child location is outside the predicate, do not re-serialize
}
UpdateSerializedItem(newSerializedItem);
}
// remove the old children folder if it exists - as long as the original name was not a case insensitive version of this item
if (Directory.Exists(oldReference.ProviderId) && !oldReference.ProviderId.Equals(newItemReferencePath, StringComparison.OrdinalIgnoreCase))
{
Directory.Delete(oldReference.ProviderId, true);
}
}
public void DeserializeAsset(AssetFile assetFile, StreamReader stream)
{
GameObject = new SerializedComponent();
GameObject.DeserializeAsset(assetFile, stream);
Enabled = stream.ReadBoolean();
stream.AlignTo();
var script = new SerializedComponent();
script.DeserializeAsset(assetFile, stream);
Name = stream.ReadString32();
Script = (SerializedMonoScript)assetFile.ReadObject(script);
if (Script != null)
{
Child = assetFile.ReadMonoScriptChild(Script);
}
}
public virtual void LoadAll(ISerializedItem[] rootItems, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker, Action<ISerializedItem> rootLoadedCallback = null)
{
Assert.ArgumentNotNull(rootItems, "rootItems");
Assert.IsTrue(rootItems.Length > 0, "No root items were passed!");
using (new EventDisabler())
{
foreach (var rootItem in rootItems)
{
LoadTree(rootItem, retryer, consistencyChecker);
if (rootLoadedCallback != null) rootLoadedCallback(rootItem);
}
}
retryer.RetryAll(SourceDataProvider, item => DoLoadItem(item, null), item => LoadTreeRecursive(item, retryer, null));
SourceDataProvider.DeserializationComplete(rootItems[0].DatabaseName);
}
public ISourceItem EvaluateUpdate(ISerializedItem serializedItem, ISourceItem existingItem)
{
Assert.ArgumentNotNull(serializedItem, "serializedItem");
Assert.ArgumentNotNull(existingItem, "existingItem");
var deferredUpdateLog = new DeferredLogWriter <ISerializedAsMasterEvaluatorLogger>();
if (ShouldUpdateExisting(serializedItem, existingItem, deferredUpdateLog))
{
_logger.SerializedUpdatedItem(serializedItem);
deferredUpdateLog.ExecuteDeferredActions(_logger);
var updatedItem = DoDeserialization(serializedItem);
return(updatedItem);
}
return(null);
}
public virtual ISourceItem Deserialize(ISerializedItem serializedItem, bool ignoreMissingTemplateFields)
{
if(ignoreMissingTemplateFields) throw new NotSupportedException("The Sitecore serialization engine does not support ignoring missing fields.");
try
{
var options = new LoadOptions { DisableEvents = true, ForceUpdate = true, UseNewID = false };
return new SitecoreSourceItem(ItemSynchronization.PasteSyncItem(((SitecoreSerializedItem)serializedItem).InnerItem, options, true));
}
catch (ParentItemNotFoundException ex)
{
string error = "Cannot load item from path '{0}'. Probable reason: parent item with ID '{1}' not found.".FormatWith(serializedItem.ProviderId, ex.ParentID);
throw new DeserializationException(error, ex);
}
catch (ParentForMovedItemNotFoundException ex2)
{
string error = "Item from path '{0}' cannot be moved to appropriate location. Possible reason: parent item with ID '{1}' not found.".FormatWith(serializedItem.ProviderId, ex2.ParentID);
throw new DeserializationException(error, ex2);
}
}
/// <summary>
/// Loads a tree from serialized items on disk.
/// </summary>
protected virtual void LoadTree(ISerializedItem rootItem, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(rootItem, "rootItem");
Assert.ArgumentNotNull(retryer, "retryer");
Assert.ArgumentNotNull(consistencyChecker, "consistencyChecker");
_itemsProcessed = 0;
var timer = new Stopwatch();
timer.Start();
Logger.BeginLoadingTree(rootItem);
// load the root item (LoadTreeRecursive only evaluates children)
DoLoadItem(rootItem, consistencyChecker);
// load children of the root
LoadTreeRecursive(rootItem, retryer, consistencyChecker);
Logger.EndLoadingTree(rootItem, _itemsProcessed, timer.ElapsedMilliseconds);
timer.Stop();
}
public virtual void MovedItemToNonIncludedLocation(string providerName, ISerializedItem existingItem)
{
_logger.Debug(string.Format("{0}: Moved item {1} was moved to a path that was not included in serialization, and the existing serialized item was deleted.", providerName, existingItem.ItemPath));
}
public void PushChangedItem(ISerializedItem serializedItem, ChangeType type)
{
_entries.Enqueue(new ChangeEntry(serializedItem, type));
}
/// <summary>
/// Loads a specific item from disk
/// </summary>
protected virtual ItemLoadResult DoLoadItem(ISerializedItem serializedItem, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(serializedItem, "serializedItem");
if (consistencyChecker != null)
{
if (!consistencyChecker.IsConsistent(serializedItem)) throw new ConsistencyException("Consistency check failed - aborting loading.");
consistencyChecker.AddProcessedItem(serializedItem);
}
bool disableNewSerialization = UnicornDataProvider.DisableSerialization;
try
{
UnicornDataProvider.DisableSerialization = true;
_itemsProcessed++;
var included = Predicate.Includes(serializedItem);
if (!included.IsIncluded)
{
Logger.SkippedItemPresentInSerializationProvider(serializedItem, Predicate.GetType().Name, SerializationProvider.GetType().Name, included.Justification ?? string.Empty);
return new ItemLoadResult(ItemLoadStatus.Skipped);
}
// detect if we should run an update for the item or if it's already up to date
var existingItem = SourceDataProvider.GetItemById(serializedItem.DatabaseName, serializedItem.Id);
ISourceItem updatedItem;
// note that the evaluator is responsible for actual action being taken here
// as well as logging what it does
if (existingItem == null)
updatedItem = Evaluator.EvaluateNewSerializedItem(serializedItem);
else
updatedItem = Evaluator.EvaluateUpdate(serializedItem, existingItem);
return new ItemLoadResult(ItemLoadStatus.Success, updatedItem ?? existingItem);
}
finally
{
UnicornDataProvider.DisableSerialization = disableNewSerialization;
}
}
protected virtual string CreateKey(ISerializedItem item)
{
return item.Id + item.DatabaseName;
}
/// <summary>
/// Loads a tree from serialized items on disk.
/// </summary>
protected internal virtual void LoadTree(ISerializedItem rootItem, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(rootItem, "rootItem");
Assert.ArgumentNotNull(retryer, "retryer");
Assert.ArgumentNotNull(consistencyChecker, "consistencyChecker");
_itemsProcessed = 0;
var timer = new Stopwatch();
timer.Start();
Logger.BeginLoadingTree(rootItem);
// load the root item (LoadTreeRecursive only evaluates children)
DoLoadItem(rootItem, consistencyChecker);
// load children of the root
LoadTreeRecursive(rootItem, retryer, consistencyChecker);
Logger.EndLoadingTree(rootItem, _itemsProcessed, timer.ElapsedMilliseconds);
timer.Stop();
}
protected virtual string CreateKey(ISerializedItem item)
{
return(item.Id + item.DatabaseName);
}
private string GetErrorValue(SaveArgs args)
{
var results = new Dictionary <Item, IList <FieldDesynchronization> >();
try
{
foreach (var item in args.Items)
{
Item existingItem = Client.ContentDatabase.GetItem(item.ID, item.Language, item.Version);
Assert.IsNotNull(existingItem, "Existing item {0} did not exist! This should never occur.", item.ID);
var existingSitecoreItem = new SitecoreSourceItem(existingItem);
foreach (var configuration in _configurations)
{
// ignore conflicts on items that Unicorn is not managing
if (!configuration.Resolve <IPredicate>().Includes(existingSitecoreItem).IsIncluded)
{
continue;
}
ISerializedReference serializedReference = configuration.Resolve <ISerializationProvider>().GetReference(existingSitecoreItem);
if (serializedReference == null)
{
continue;
}
// not having an existing serialized version means no possibility of conflict here
ISerializedItem serializedItem = serializedReference.GetItem();
if (serializedItem == null)
{
continue;
}
var fieldPredicate = configuration.Resolve <IFieldPredicate>();
var fieldIssues = GetFieldSyncStatus(existingSitecoreItem, serializedItem, fieldPredicate);
if (fieldIssues.Count == 0)
{
continue;
}
results.Add(existingItem, fieldIssues);
}
}
// no problems
if (results.Count == 0)
{
return(null);
}
var sb = new StringBuilder();
sb.Append("CRITICAL MESSAGE FROM UNICORN:\n");
sb.Append("You need to run a Unicorn sync. The following fields did not match the serialized version:\n");
foreach (var item in results)
{
if (results.Count > 1)
{
sb.AppendFormat("\n{0}: {1}", item.Key.DisplayName, string.Join(", ", item.Value.Select(x => x.FieldName)));
}
else
{
sb.AppendFormat("\n{0}", string.Join(", ", item.Value.Select(x => x.FieldName)));
}
}
sb.Append("\n\nDo you want to overwrite anyway?\nTHIS MAY CAUSE LOST WORK.");
return(sb.ToString());
}
catch (Exception ex)
{
Log.Error("Exception occurred while performing serialization conflict check!", ex, this);
return("Exception occurred: " + ex.Message); // this will cause a few retries
}
}
/// <summary>
/// Moves the descendant items of a serialized parent after it has been moved or renamed.
/// </summary>
/// <param name="oldReference">Reference to the original path pre-move/rename</param>
/// <param name="newItem">The newly renamed or moved parent item</param>
/// <param name="sourceItem">The source item representing the renamed/moved item. NOTE that the path of this item is incorrect a lot of the time so we ignore it.</param>
/// <remarks>
/// This method basically gets all descendants of the source item that was moved/renamed, generates an appropriate new serialized item for it, and _if the new child item is in the predicate_ we
/// serialize it to its new location. Finally, we delete the old children directory if it existed.
///
/// Doing it this way allows handling crazy cases like moving trees of items between included and excluded locations - or even moves or renames causing SOME of the children to be ignored. Wild.
/// </remarks>
protected virtual void MoveDescendants(ISerializedReference oldReference, ISerializedItem newItem, ISourceItem sourceItem)
{
// remove the extension from the new item's provider ID
string newItemReferencePath = SerializationPathUtility.GetReferenceDirectoryPath(newItem);
// if the paths were the same, no moving occurs (this can happen when saving templates, which spuriously can report "renamed" when they are not actually any such thing)
if (oldReference.ProviderId.Equals(newItemReferencePath, StringComparison.Ordinal)) return;
// this is for renaming an item that differs only by case from the original. Because NTFS is case-insensitive the 'new parent' exists
// already, but it will use the old name. Not quite what we want. So we need to manually rename the folder.
if (oldReference.ProviderId.Equals(newItemReferencePath, StringComparison.OrdinalIgnoreCase) && Directory.Exists(oldReference.ProviderId))
{
Directory.Move(oldReference.ProviderId, oldReference.ProviderId + "_tempunicorn");
Directory.Move(oldReference.ProviderId + "_tempunicorn", newItemReferencePath);
}
var descendantItems = GetDescendants(sourceItem).Cast<SitecoreSourceItem>();
// Go through descendant source items and serialize all that are included by the predicate
foreach (var descendant in descendantItems)
{
var syncItem = ItemSynchronization.BuildSyncItem(descendant.InnerItem);
// the newPhysicalPath will point to the OLD physical path pre-move/rename.
// We re-root the path to point to the new parent item's base path to fix that before we write to disk
var newPhysicalPath = SerializationPathUtility.GetSerializedItemPath(_rootPath, syncItem.DatabaseName, syncItem.ItemPath);
var newSerializedItem = new SitecoreSerializedItem(syncItem, newPhysicalPath, this);
if (!_predicate.Includes(newSerializedItem).IsIncluded) continue; // if the moved child location is outside the predicate, do not re-serialize
UpdateSerializedItem(newSerializedItem);
}
// remove the old children folder if it exists - as long as the original name was not a case insensitive version of this item
if (Directory.Exists(oldReference.ProviderId) && !oldReference.ProviderId.Equals(newItemReferencePath, StringComparison.OrdinalIgnoreCase))
{
Directory.Delete(oldReference.ProviderId, true);
}
}
public virtual void DeserializedNewItem(ISerializedItem serializedItem)
{
_logger.Info("[A] {0}".FormatWith(serializedItem.DisplayIdentifier));
}
public virtual void SerializedUpdatedItem(ISerializedItem serializedItem)
{
_logger.Info("[U] {0}".FormatWith(serializedItem.DisplayIdentifier));
_pipelineDataCollector.PushChangedItem(serializedItem, ChangeType.Modified);
}
public virtual void OrphanSourceVersion(ISourceItem existingItem, ISerializedItem serializedItem, ItemVersion[] orphanSourceVersions)
{
_logger.Debug("> Orphaned version{0} {1} (source)".FormatWith(orphanSourceVersions.Length > 1 ? "s" : string.Empty, string.Join(", ", orphanSourceVersions.Select(x => x.Language + "#" + x.VersionNumber))));
}
public virtual void NewSerializedVersionMatch(ItemVersion newSerializedVersion, ISerializedItem serializedItem, ISourceItem existingItem)
{
_logger.Debug("> New version {0}#{1} (serialized)".FormatWith(newSerializedVersion.Language, newSerializedVersion.VersionNumber));
}
protected virtual void UpdateSerializedItem(ISerializedItem serializedItem)
{
var typed = serializedItem as SitecoreSerializedItem;
if (typed == null) throw new ArgumentException("Serialized item must be a SitecoreSerializedItem", "serializedItem");
// create any requisite parent folder(s) for the serialized item
var parentPath = Directory.GetParent(SerializationPathUtility.GetReferenceDirectoryPath(serializedItem));
if (parentPath != null && !parentPath.Exists)
Directory.CreateDirectory(parentPath.FullName);
// if the file already exists, delete it. Why? Because the FS is case-insensitive, if an item is renamed by case only it won't actually rename
// on the filesystem. Deleting it first makes sure this occurs.
if (File.Exists(serializedItem.ProviderId)) File.Delete(serializedItem.ProviderId);
using (var fileStream = File.Open(serializedItem.ProviderId, FileMode.Create, FileAccess.Write, FileShare.None))
{
using (var writer = new StreamWriter(fileStream))
{
typed.InnerItem.Serialize(writer);
}
}
}
public bool IsStandardValuesItem(ISerializedItem item)
{
Assert.ArgumentNotNull(item, "item");
string[] array = item.ItemPath.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries);
if (array.Length > 0)
{
if (array.Any(s => s.Equals("templates", StringComparison.OrdinalIgnoreCase)))
{
return array.Last().Equals("__Standard Values", StringComparison.OrdinalIgnoreCase);
}
}
return false;
}
private void TestLoadTree(SerializationLoader loader, ISerializedItem root, IDeserializeFailureRetryer retryer = null, IConsistencyChecker consistencyChecker = null)
{
if (retryer == null) retryer = new Mock<IDeserializeFailureRetryer>().Object;
if (consistencyChecker == null)
{
var checker = new Mock<IConsistencyChecker>();
checker.Setup(x => x.IsConsistent(It.IsAny<ISerializedItem>())).Returns(true);
consistencyChecker = checker.Object;
}
loader.LoadTree(root, retryer, consistencyChecker);
}
private IList<FieldDesynchronization> GetFieldSyncStatus(SitecoreSourceItem item, ISerializedItem serializedItem, IFieldPredicate fieldPredicate)
{
var desyncs = new List<FieldDesynchronization>();
var serializedVersion = serializedItem.Versions.FirstOrDefault(x => x.VersionNumber == item.InnerItem.Version.Number && x.Language == item.InnerItem.Language.Name);
if (serializedVersion == null)
{
desyncs.Add(new FieldDesynchronization("Version"));
return desyncs;
}
item.InnerItem.Fields.ReadAll();
foreach (Field field in item.InnerItem.Fields)
{
if (field.ID == FieldIDs.Revision ||
field.ID == FieldIDs.Updated ||
field.ID == FieldIDs.Created ||
field.ID == FieldIDs.CreatedBy ||
field.ID == FieldIDs.UpdatedBy ||
field.Type.Equals("attachment", StringComparison.OrdinalIgnoreCase) ||
!fieldPredicate.Includes(field.ID).IsIncluded) continue;
// we're doing a data comparison here - revision, created (by), updated (by) don't matter
// skipping these fields allows us to ignore spurious saves the template builder makes to unchanged items being conflicts
// find the field in the serialized item in either versioned or shared fields
string serializedField;
if(!serializedVersion.Fields.TryGetValue(field.ID.ToString(), out serializedField))
serializedItem.SharedFields.TryGetValue(field.ID.ToString(), out serializedField);
// we ignore if the field doesn't exist in the serialized item. This is because if you added a field to a template,
// that does not immediately re-serialize all items based on that template so it's likely innocuous - we're not overwriting anything.
if (serializedField == null) continue;
if (!serializedField.Equals(field.Value, StringComparison.Ordinal))
{
desyncs.Add(new FieldDesynchronization(field.Name));
}
}
return desyncs;
}
public void AddProcessedItem(ISerializedItem item)
{
_duplicateChecks.Add(CreateKey(item), item);
}
public virtual ISourceItem DeserializeItem(ISerializedItem serializedItem, bool ignoreMissingTemplateFields)
{
Assert.ArgumentNotNull(serializedItem, "serializedItem");
var typed = serializedItem as SitecoreSerializedItem;
if (typed == null) throw new ArgumentException("Serialized item must be a SitecoreSerializedItem", "serializedItem");
try
{
var options = new LoadOptions { DisableEvents = true, ForceUpdate = true, UseNewID = false };
return new SitecoreSourceItem(ItemSynchronization.PasteSyncItem(typed.InnerItem, options, true));
}
catch (ParentItemNotFoundException ex)
{
string error = "Cannot load item from path '{0}'. Probable reason: parent item with ID '{1}' not found.".FormatWith(serializedItem.ProviderId, ex.ParentID);
throw new DeserializationException(error, ex);
}
catch (ParentForMovedItemNotFoundException ex2)
{
string error = "Item from path '{0}' cannot be moved to appropriate location. Possible reason: parent item with ID '{1}' not found.".FormatWith(serializedItem.ProviderId, ex2.ParentID);
throw new DeserializationException(error, ex2);
}
}
public virtual void DuplicateFound(ISerializedItem existingItem, ISerializedItem duplicateItem)
{
_logger.Error("Duplicate serialized item IDs were detected ({0}) - this usually indicates corruption in the serialization provider data.<br>Item 1: {1}<br> Item 1 ProviderId: {2}<br>Item 2: {3}<br>Item 2 ProviderId: {4}".FormatWith(existingItem.Id, existingItem.DisplayIdentifier, existingItem.ProviderId, duplicateItem.DisplayIdentifier, duplicateItem.ProviderId));
}
public void PushChangedItem(ISerializedItem serializedItem, ChangeType type)
{
_entries.Enqueue(new ChangeEntry(serializedItem, type));
}
public void AddProcessedItem(ISerializedItem item)
{
_duplicateChecks.Add(CreateKey(item), item);
}
public virtual void IsNameMatch(ISerializedItem serializedItem, ISourceItem existingItem)
{
_logger.Debug("> Name: Serialized \"{0}\", Source \"{1}\"".FormatWith(serializedItem.Name, existingItem.Name));
}
/// <summary>
/// Loads a preset from serialized items on disk.
/// </summary>
public virtual void LoadTree(ISerializedItem rootItem, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(rootItem, "rootItem");
Assert.ArgumentNotNull(retryer, "retryer");
Assert.ArgumentNotNull(consistencyChecker, "consistencyChecker");
_itemsProcessed = 0;
var timer = new Stopwatch();
timer.Start();
Logger.BeginLoadingTree(rootItem);
using (new EventDisabler())
{
// load the root item (LoadTreeRecursive only evaluates children)
DoLoadItem(rootItem, consistencyChecker);
// load children of the root
LoadTreeRecursive(rootItem, retryer, consistencyChecker);
retryer.RetryAll(SourceDataProvider, item => DoLoadItem(item, null), item => LoadTreeRecursive(item, retryer, null));
}
timer.Stop();
SourceDataProvider.DeserializationComplete(rootItem.DatabaseName);
Logger.EndLoadingTree(rootItem, _itemsProcessed, timer.ElapsedMilliseconds);
}
public virtual void SerializedUpdatedItem(ISerializedItem serializedItem)
{
_logger.Info("[U] {0}".FormatWith(serializedItem.DisplayIdentifier));
}
/// <summary>
/// Loads a preset from serialized items on disk.
/// </summary>
public virtual void LoadTree(ISerializedItem rootItem, IConfiguration dependencyConfiguration)
{
LoadTree(rootItem, dependencyConfiguration.Resolve<IDeserializeFailureRetryer>(), dependencyConfiguration.Resolve<IConsistencyChecker>());
}
public ChangeEntry(ISerializedItem item, ChangeType type)
{
SerializedItem = item;
ChangeType = type;
}
public virtual void DeletedItem(string providerName, ISerializedItem existingItem)
{
_logger.Info(string.Format("{0}: Serialized item {1} was deleted due to the source item being deleted.", providerName, existingItem.ProviderId));
}
public virtual void SerializedUpdatedItem(ISerializedItem serializedItem)
{
_logger.Info("[U] {0}".FormatWith(serializedItem.DisplayIdentifier));
_pipelineDataCollector.PushChangedItem(serializedItem, ChangeType.Modified);
}
public virtual void NewSerializedVersionMatch(ItemVersion newSerializedVersion, ISerializedItem serializedItem, ISourceItem existingItem)
{
_logger.Debug("> New version {0}#{1} (serialized)".FormatWith(newSerializedVersion.Language, newSerializedVersion.VersionNumber));
}
public virtual void IsTemplateMatch(ISerializedItem serializedItem, ISourceItem existingItem)
{
_logger.Debug("> Template: Serialized \"{0}\", Source \"{1}\"".FormatWith(serializedItem.TemplateName, existingItem.TemplateName));
}
/// <summary>
/// Pastes SyncItem into the database.
///
/// </summary>
/// <param name="syncItem">The sync item.</param>
/// <param name="ignoreMissingTemplateFields">Whether to ignore fields in the serialized item that do not exist on the Sitecore template</param>
/// <returns>
/// The pasted item.
/// </returns>
/// <exception cref="T:Sitecore.Data.Serialization.Exceptions.ParentItemNotFoundException"><c>ParentItemNotFoundException</c>.</exception><exception cref="T:System.Exception"><c>Exception</c>.</exception><exception cref="T:Sitecore.Data.Serialization.Exceptions.ParentForMovedItemNotFoundException"><c>ParentForMovedItemNotFoundException</c>.</exception>
public override ISourceItem Deserialize(ISerializedItem serializedItem, bool ignoreMissingTemplateFields)
{
Assert.ArgumentNotNull(serializedItem, "serializedItem");
var typed = serializedItem as SitecoreSerializedItem;
if (typed == null) throw new ArgumentException("Serialized item must be a SitecoreSerializedItem", "serializedItem");
var syncItem = typed.InnerItem;
Database database = Factory.GetDatabase(syncItem.DatabaseName);
Item destinationParentItem = database.GetItem(syncItem.ParentID);
ID itemId = ID.Parse(syncItem.ID);
Item targetItem = database.GetItem(itemId);
bool newItemWasCreated = false;
// the target item did not yet exist, so we need to start by creating it
if (targetItem == null)
{
targetItem = CreateTargetItem(syncItem, destinationParentItem);
_logger.CreatedNewItem(targetItem);
newItemWasCreated = true;
}
else
{
// check if the parent of the serialized item does not exist
// which, since the target item is known to exist, means that
// the serialized item was moved but its new parent item does
// not exist to paste it under
if (destinationParentItem == null)
{
throw new ParentForMovedItemNotFoundException
{
ParentID = syncItem.ParentID,
Item = targetItem
};
}
// if the parent IDs mismatch that means we need to move the existing
// target item to its new parent from the serialized item
if (destinationParentItem.ID != targetItem.ParentID)
{
var oldParent = targetItem.Parent;
targetItem.MoveTo(destinationParentItem);
_logger.MovedItemToNewParent(destinationParentItem, oldParent, targetItem);
}
}
try
{
ChangeTemplateIfNeeded(syncItem, targetItem);
RenameIfNeeded(syncItem, targetItem);
ResetTemplateEngineIfItemIsTemplate(targetItem);
using (new EditContext(targetItem))
{
targetItem.RuntimeSettings.ReadOnlyStatistics = true;
targetItem.RuntimeSettings.SaveAll = true;
foreach (Field field in targetItem.Fields)
{
if (field.Shared && syncItem.SharedFields.All(x => x.FieldID != field.ID.ToString()))
{
_logger.ResetFieldThatDidNotExistInSerialized(field);
field.Reset();
}
}
foreach (SyncField field in syncItem.SharedFields)
PasteSyncField(targetItem, field, ignoreMissingTemplateFields, newItemWasCreated);
}
ClearCaches(database, itemId);
targetItem.Reload();
ResetTemplateEngineIfItemIsTemplate(targetItem);
Hashtable versionTable = CommonUtils.CreateCIHashtable();
// this version table allows us to detect and remove orphaned versions that are not in the
// serialized version, but are in the database version
foreach (Item version in targetItem.Versions.GetVersions(true))
versionTable[version.Uri] = null;
foreach (SyncVersion syncVersion in syncItem.Versions)
{
var version = PasteSyncVersion(targetItem, syncVersion, ignoreMissingTemplateFields, newItemWasCreated);
if (versionTable.ContainsKey(version.Uri))
versionTable.Remove(version.Uri);
}
foreach (ItemUri uri in versionTable.Keys)
{
var versionToRemove = Database.GetItem(uri);
_logger.RemovingOrphanedVersion(versionToRemove);
versionToRemove.Versions.RemoveVersion();
}
ClearCaches(targetItem.Database, targetItem.ID);
return new SitecoreSourceItem(targetItem);
}
catch (ParentForMovedItemNotFoundException)
{
throw;
}
catch (ParentItemNotFoundException)
{
throw;
}
#if SITECORE_7
catch (FieldIsMissingFromTemplateException)
{
throw;
}
#endif
catch (Exception ex)
{
if (newItemWasCreated)
{
targetItem.Delete();
ClearCaches(database, itemId);
}
throw new Exception("Failed to paste item: " + syncItem.ItemPath, ex);
}
}
public virtual void OrphanSourceVersion(ISourceItem existingItem, ISerializedItem serializedItem, ItemVersion[] orphanSourceVersions)
{
_logger.Debug("> Orphaned version{0} {1} (source)".FormatWith(orphanSourceVersions.Length > 1 ? "s" : string.Empty, string.Join(", ", orphanSourceVersions.Select(x => x.Language + "#" + x.VersionNumber))));
}
public virtual void DuplicateFound(ISerializedItem existingItem, ISerializedItem duplicateItem)
{
_logger.Error("Duplicate serialized item IDs were detected ({0}) - this usually indicates corruption in the serialization provider data.<br>Item 1: {1}<br> Item 1 ProviderId: {2}<br>Item 2: {3}<br>Item 2 ProviderId: {4}".FormatWith(existingItem.Id, existingItem.DisplayIdentifier, existingItem.ProviderId, duplicateItem.DisplayIdentifier, duplicateItem.ProviderId));
}
private IList <FieldDesynchronization> GetFieldSyncStatus(SitecoreSourceItem item, ISerializedItem serializedItem, IFieldPredicate fieldPredicate)
{
var desyncs = new List <FieldDesynchronization>();
var serializedVersion = serializedItem.Versions.FirstOrDefault(x => x.VersionNumber == item.InnerItem.Version.Number && x.Language == item.InnerItem.Language.Name);
if (serializedVersion == null)
{
desyncs.Add(new FieldDesynchronization("Version"));
return(desyncs);
}
item.InnerItem.Fields.ReadAll();
foreach (Field field in item.InnerItem.Fields)
{
if (field.ID == FieldIDs.Revision ||
field.ID == FieldIDs.Updated ||
field.ID == FieldIDs.Created ||
field.ID == FieldIDs.CreatedBy ||
field.ID == FieldIDs.UpdatedBy ||
field.Type.Equals("attachment", StringComparison.OrdinalIgnoreCase) ||
!fieldPredicate.Includes(field.ID).IsIncluded)
{
continue;
}
// we're doing a data comparison here - revision, created (by), updated (by) don't matter
// skipping these fields allows us to ignore spurious saves the template builder makes to unchanged items being conflicts
// find the field in the serialized item in either versioned or shared fields
string serializedField;
if (!serializedVersion.Fields.TryGetValue(field.ID.ToString(), out serializedField))
{
serializedItem.SharedFields.TryGetValue(field.ID.ToString(), out serializedField);
}
// we ignore if the field doesn't exist in the serialized item. This is because if you added a field to a template,
// that does not immediately re-serialize all items based on that template so it's likely innocuous - we're not overwriting anything.
if (serializedField == null)
{
continue;
}
if (!serializedField.Equals(field.Value, StringComparison.Ordinal))
{
desyncs.Add(new FieldDesynchronization(field.Name));
}
}
return(desyncs);
}
