public void AddTreeRetry(ISerializedReference reference, Exception exception)
{
Assert.ArgumentNotNull(reference, "reference");
Assert.ArgumentNotNull(exception, "exception");
_treeFailures.Add(new Failure(reference, exception));
}
public virtual ISerializedItem[] GetChildItems(ISerializedReference parent)
{
Assert.ArgumentNotNull(parent, "parent");
var path = SerializationPathUtility.GetReferenceDirectoryPath(parent);
var shortPath = SerializationPathUtility.GetShortSerializedReferencePath(_rootPath, parent);
var fileNames = new List <string>();
bool longPathExists = Directory.Exists(path);
bool shortPathExists = Directory.Exists(shortPath);
if (!longPathExists && !shortPathExists)
{
return(new ISerializedItem[0]);
}
if (longPathExists)
{
fileNames.AddRange(Directory.GetFiles(path, "*" + PathUtils.Extension));
}
if (shortPathExists)
{
fileNames.AddRange(Directory.GetFiles(shortPath, "*" + PathUtils.Extension));
}
return(fileNames.Select(ReadItemFromDisk).ToArray());
}
/// <summary>
/// Gets the parent physical reference path of a serialized reference, respecting short paths
/// </summary>
public static string GetReferenceParentPath(string rootPath, ISerializedReference reference)
{
var itemRootPath = reference.ItemPath.TrimEnd('/');
var parentItemPath = itemRootPath.Substring(0, itemRootPath.LastIndexOf('/'));
return GetSerializedReferencePath(rootPath, reference.DatabaseName, parentItemPath);
}
/// <summary>
/// Recursive method that loads a given tree and retries failures already present if any
/// </summary>
protected virtual void LoadTreeRecursive(ISerializedReference root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(root, "root");
Assert.ArgumentNotNull(retryer, "retryer");
var included = Predicate.Includes(root);
if (!included.IsIncluded)
{
Logger.SkippedItemPresentInSerializationProvider(root, Predicate.GetType().Name, SerializationProvider.GetType().Name, included.Justification ?? string.Empty);
return;
}
try
{
// load the current level
LoadOneLevel(root, retryer, consistencyChecker);
// check if we have child paths to recurse down
var children = root.GetChildReferences(false);
if (children.Length > 0)
{
// make sure if a "templates" item exists in the current set, it goes first
if (children.Length > 1)
{
int templateIndex = Array.FindIndex(children, x => x.ItemPath.EndsWith("templates", StringComparison.OrdinalIgnoreCase));
if (templateIndex > 0)
{
var zero = children[0];
children[0] = children[templateIndex];
children[templateIndex] = zero;
}
}
// load each child path recursively
foreach (var child in children)
{
LoadTreeRecursive(child, retryer, consistencyChecker);
}
// pull out any standard values failures for immediate retrying
retryer.RetryStandardValuesFailures(item => DoLoadItem(item, null));
} // children.length > 0
}
catch (ConsistencyException)
{
throw;
}
catch (Exception ex)
{
retryer.AddTreeRetry(root, ex);
}
}
protected virtual void DeleteItemRecursive(ISerializedReference reference)
{
foreach (var child in reference.GetChildReferences(false))
{
DeleteItemRecursive(child);
}
// kill the serialized file
var fileItem = reference.GetItem();
if (fileItem != null && File.Exists(fileItem.ProviderId))
{
File.Delete(fileItem.ProviderId);
}
// remove any serialized children
var directory = SerializationPathUtility.GetReferenceDirectoryPath(reference);
if (Directory.Exists(directory))
{
Directory.Delete(directory, true);
}
// clean up any hashpaths for this item
var shortDirectory = SerializationPathUtility.GetShortSerializedReferencePath(_rootPath, reference);
if (Directory.Exists(shortDirectory))
{
Directory.Delete(shortDirectory, true);
}
// clean up empty parent folder(s)
var parentDirectory = Directory.GetParent(directory);
if (!parentDirectory.Exists)
{
return;
}
do
{
if (parentDirectory.GetFileSystemInfos().Length > 0)
{
break;
}
parentDirectory.Delete(true);
parentDirectory = parentDirectory.Parent;
} while (parentDirectory != null && parentDirectory.Exists);
}
/// <summary>
/// Moves the descendant items of a serialized parent after it has been moved or renamed.
/// </summary>
/// <param name="oldReference">Reference to the original path pre-move/rename</param>
/// <param name="newItem">The newly renamed or moved parent item</param>
/// <param name="sourceItem">The source item representing the renamed/moved item. NOTE that the path of this item is incorrect a lot of the time so we ignore it.</param>
/// <param name="renaming">True for moving renamed children, false for moving moved children. For renames, the children already have correct new paths; for moves we have to recalculate it.</param>
/// <remarks>
/// This method basically gets all descendants of the source item that was moved/renamed, generates an appropriate new serialized item for it, and _if the new child item is in the predicate_ we
/// serialize it to its new location. Finally, we delete the old children directory if it existed.
///
/// Doing it this way allows handling crazy cases like moving trees of items between included and excluded locations - or even moves or renames causing SOME of the children to be ignored. Wild.
/// </remarks>
protected virtual void MoveDescendants(ISerializedReference oldReference, ISerializedItem newItem, ISourceItem sourceItem, bool renaming)
{
// remove the extension from the new item's provider ID
string newItemReferencePath = SerializationPathUtility.GetReferenceDirectoryPath(newItem);
// if the paths were the same, no moving occurs (this can happen when saving templates, which spuriously can report "renamed" when they are not actually any such thing)
if (oldReference.ProviderId.Equals(newItemReferencePath, StringComparison.Ordinal))
{
return;
}
// this is for renaming an item that differs only by case from the original. Because NTFS is case-insensitive the 'new parent' exists
// already, but it will use the old name. Not quite what we want. So we need to manually rename the folder.
if (oldReference.ProviderId.Equals(newItemReferencePath, StringComparison.OrdinalIgnoreCase) && Directory.Exists(oldReference.ProviderId))
{
Directory.Move(oldReference.ProviderId, oldReference.ProviderId + "_tempunicorn");
Directory.Move(oldReference.ProviderId + "_tempunicorn", newItemReferencePath);
}
var descendantItems = GetDescendants(sourceItem).Cast <SitecoreSourceItem>();
// Go through descendant source items and serialize all that are included by the predicate
foreach (var descendant in descendantItems)
{
var syncItem = ItemSynchronization.BuildSyncItem(descendant.InnerItem);
// the newPhysicalPath will point to the OLD physical path pre-move (but for renames we actually get the new path already).
// For moves, we re-root the path to point to the new parent item's base path to fix that before we write to disk
string newItemPath = (renaming) ? descendant.ItemPath : descendant.ItemPath.Replace(oldReference.ItemPath, newItem.ItemPath);
var newPhysicalPath = SerializationPathUtility.GetSerializedItemPath(_rootPath, syncItem.DatabaseName, newItemPath);
var newSerializedItem = new SitecoreSerializedItem(syncItem, newPhysicalPath, this);
if (!_predicate.Includes(newSerializedItem).IsIncluded)
{
continue; // if the moved child location is outside the predicate, do not re-serialize
}
UpdateSerializedItem(newSerializedItem);
}
// remove the old children folder if it exists - as long as the original name was not a case insensitive version of this item
if (Directory.Exists(oldReference.ProviderId) && !oldReference.ProviderId.Equals(newItemReferencePath, StringComparison.OrdinalIgnoreCase))
{
Directory.Delete(oldReference.ProviderId, true);
}
}
/// <summary>
/// Checks if a preset includes a given serialized item
/// </summary>
protected PredicateResult Includes(IncludeEntry entry, ISerializedReference item)
{
// check for db match
if (item.DatabaseName != entry.Database)
{
return(new PredicateResult(false));
}
// check for path match
if (!item.ItemPath.StartsWith(entry.Path, StringComparison.OrdinalIgnoreCase))
{
return(new PredicateResult(false));
}
// check excludes
return(ExcludeMatches(entry, item));
}
public virtual ISerializedItem GetItem(ISerializedReference reference)
{
Assert.ArgumentNotNull(reference, "reference");
var path = SerializationPathUtility.GetReferenceItemPath(reference);
if (File.Exists(path))
{
return(ReadItemFromDisk(path));
}
var shortPath = SerializationPathUtility.GetShortSerializedItemPath(_rootPath, reference);
if (File.Exists(shortPath))
{
return(ReadItemFromDisk(shortPath));
}
return(null);
}
public PredicateResult Includes(ISerializedReference item)
{
var result = new PredicateResult(true);
PredicateResult priorityResult = null;
foreach (var entry in _preset)
{
result = Includes(entry, item);
if (result.IsIncluded)
{
return(result); // it's definitely included if anything includes it
}
if (!string.IsNullOrEmpty(result.Justification))
{
priorityResult = result; // a justification means this is probably a more 'important' fail than others
}
}
return(priorityResult ?? result); // return the last failure
}
protected virtual PredicateResult ExcludeMatches(IncludeEntry entry, ISerializedReference reference)
{
PredicateResult result = ExcludeMatchesPath(entry.Exclude, reference.ItemPath);
if (!result.IsIncluded)
{
return(result);
}
// many times the ISerializedReference may also have an item ref in it (e.g. be a serialized item)
// in this case we can check additional criteria
var item = reference as ISerializedItem;
if (item == null)
{
return(result);
}
result = ExcludeMatchesTemplateId(entry.Exclude, ID.Parse(item.TemplateId));
if (!result.IsIncluded)
{
return(result);
}
result = ExcludeMatchesTemplate(entry.Exclude, item.TemplateName);
if (!result.IsIncluded)
{
return(result);
}
result = ExcludeMatchesId(entry.Exclude, ID.Parse(item.Id));
return(result);
}
public virtual void SkippedItemMissingInSerializationProvider(ISerializedReference item, string serializationProviderName)
{
_logger.Warn("[S] {0}. Unable to get a serialized item for the path. <br />This usually indicates an orphaned serialized item tree in {1} which should be removed. <br />Less commonly, it could also indicate a sparsely serialized tree which is not supported, or a serialized item that is named differently than its metadata.".FormatWith(item.DisplayIdentifier, serializationProviderName));
}
public override void EndLoadingTree(ISerializedReference rootSerializedItem, int itemsProcessed, long elapsedMilliseconds)
{
_logger.Info("Unicorn completed loading {0}".FormatWith(rootSerializedItem.DisplayIdentifier));
_logger.Debug("Items processed: {0}, Elapsed time: {1}ms ({2:N2}ms/item)".FormatWith(itemsProcessed, elapsedMilliseconds, (double)elapsedMilliseconds / (double)itemsProcessed));
}
public virtual void EndLoadingTree(ISerializedReference rootSerializedItem, int itemsProcessed, long elapsedMilliseconds)
{
}
public PredicateResult Includes(ISerializedReference item)
{
var result = new PredicateResult(true);
PredicateResult priorityResult = null;
foreach (var entry in _preset)
{
result = Includes(entry, item);
if (result.IsIncluded) return result; // it's definitely included if anything includes it
if (!string.IsNullOrEmpty(result.Justification)) priorityResult = result; // a justification means this is probably a more 'important' fail than others
}
return priorityResult ?? result; // return the last failure
}
public Failure(ISerializedReference reference, Exception reason)
{
Reference = reference;
Reason = reason;
}
public virtual void SkippedItemPresentInSerializationProvider(ISerializedReference root, string predicateName, string serializationProviderName, string justification)
{
_logger.Warn("[S] {0} by {1} but it was in {2}. {3}<br />This usually indicates an extraneous excluded serialized item is present in the {3}, which should be removed.".FormatWith(root.DisplayIdentifier, predicateName, serializationProviderName, justification));
}
public override void BeginLoadingTree(ISerializedReference rootSerializedItem)
{
_logger.Info("Unicorn loading {0}".FormatWith(rootSerializedItem.DisplayIdentifier, rootSerializedItem.ProviderId));
_logger.Debug("Provider root ID: {0}".FormatWith(rootSerializedItem.ProviderId));
}
protected virtual PredicateResult ExcludeMatches(IncludeEntry entry, ISerializedReference reference)
{
PredicateResult result = ExcludeMatchesPath(entry.Exclude, reference.ItemPath);
if (!result.IsIncluded) return result;
// many times the ISerializedReference may also have an item ref in it (e.g. be a serialized item)
// in this case we can check additional criteria
var item = reference as ISerializedItem;
if (item == null) return result;
result = ExcludeMatchesTemplateId(entry.Exclude, ID.Parse(item.TemplateId));
if (!result.IsIncluded) return result;
result = ExcludeMatchesTemplate(entry.Exclude, item.TemplateName);
if (!result.IsIncluded) return result;
result = ExcludeMatchesId(entry.Exclude, ID.Parse(item.Id));
return result;
}
/// <summary>
/// A serialized reference might refer to a directory OR a serialized item file directly. This method makes sure we've got the directory version, if it refers to a file.
/// </summary>
public static string GetReferenceDirectoryPath(ISerializedReference reference)
{
return PathUtils.StripPath(reference.ProviderId);
}
public virtual ISerializedItem GetItem(ISerializedReference reference)
{
Assert.ArgumentNotNull(reference, "reference");
var path = SerializationPathUtility.GetReferenceItemPath(reference);
if (File.Exists(path)) return ReadItemFromDisk(path);
var shortPath = SerializationPathUtility.GetShortSerializedItemPath(_rootPath, reference);
if (File.Exists(shortPath)) return ReadItemFromDisk(shortPath);
return null;
}
protected virtual void DeleteItemRecursive(ISerializedReference reference)
{
foreach (var child in reference.GetChildReferences(false))
{
DeleteItemRecursive(child);
}
// kill the serialized file
var fileItem = reference.GetItem();
if (fileItem != null && File.Exists(fileItem.ProviderId)) File.Delete(fileItem.ProviderId);
// remove any serialized children
var directory = SerializationPathUtility.GetReferenceDirectoryPath(reference);
if (Directory.Exists(directory)) Directory.Delete(directory, true);
// clean up any hashpaths for this item
var shortDirectory = SerializationPathUtility.GetShortSerializedReferencePath(_rootPath, reference);
if (Directory.Exists(shortDirectory)) Directory.Delete(shortDirectory, true);
// clean up empty parent folder(s)
var parentDirectory = Directory.GetParent(directory);
if (!parentDirectory.Exists) return;
do
{
if (parentDirectory.GetFileSystemInfos().Length > 0) break;
parentDirectory.Delete(true);
parentDirectory = parentDirectory.Parent;
} while (parentDirectory != null && parentDirectory.Exists);
}
public virtual ISerializedReference[] GetChildReferences(ISerializedReference parent, bool recursive)
{
Assert.ArgumentNotNull(parent, "parent");
var longPath = SerializationPathUtility.GetReferenceDirectoryPath(parent);
var shortPath = SerializationPathUtility.GetShortSerializedReferencePath(_rootPath, parent);
Func<string, string[]> parseDirectory = path =>
{
if (!Directory.Exists(path)) return new string[0];
var resultSet = new HashSet<string>();
string[] files = Directory.GetFiles(path, "*" + PathUtils.Extension);
foreach (var file in files)
resultSet.Add(file);
string[] directories = SerializationPathUtility.GetDirectories(path, this);
// add directories that aren't already ref'd indirectly by a file
foreach (var directory in directories)
{
if (CommonUtils.IsDirectoryHidden(directory)) continue;
if (!resultSet.Contains(directory + PathUtils.Extension))
resultSet.Add(directory);
}
string[] resultArray = resultSet.ToArray();
// make sure if a "templates" item exists in the current set, it goes first
if (resultArray.Length > 1)
{
for (int i = 1; i < resultArray.Length; i++)
{
if ("templates".Equals(Path.GetFileName(resultArray[i]), StringComparison.OrdinalIgnoreCase))
{
string text = resultArray[0];
resultArray[0] = resultArray[i];
resultArray[i] = text;
}
}
}
return resultArray;
};
var results = Enumerable.Concat(parseDirectory(longPath), parseDirectory(shortPath));
List<ISerializedReference> referenceResults = results.Select(x => (ISerializedReference)new SitecoreSerializedReference(x, this)).ToList();
if (recursive)
{
var localReferenceResults = referenceResults.ToArray();
foreach (var child in localReferenceResults)
{
referenceResults.AddRange(GetChildReferences(child, true));
}
}
return referenceResults.ToArray();
}
public virtual ISerializedItem[] GetChildItems(ISerializedReference parent)
{
Assert.ArgumentNotNull(parent, "parent");
var path = SerializationPathUtility.GetReferenceDirectoryPath(parent);
var shortPath = SerializationPathUtility.GetShortSerializedReferencePath(_rootPath, parent);
var fileNames = new List<string>();
bool longPathExists = Directory.Exists(path);
bool shortPathExists = Directory.Exists(shortPath);
if (!longPathExists && !shortPathExists) return new ISerializedItem[0];
if (longPathExists) fileNames.AddRange(Directory.GetFiles(path, "*" + PathUtils.Extension));
if (shortPathExists) fileNames.AddRange(Directory.GetFiles(shortPath, "*" + PathUtils.Extension));
return fileNames.Select(ReadItemFromDisk).ToArray();
}
public override void EndLoadingTree(ISerializedReference rootSerializedItem, int itemsProcessed, long elapsedMilliseconds)
{
_logger.Info("Unicorn completed loading {0}".FormatWith(rootSerializedItem.DisplayIdentifier));
_logger.Debug("Items processed: {0}, Elapsed time: {1}ms ({2:N2}ms/item)".FormatWith(itemsProcessed, elapsedMilliseconds, (double)elapsedMilliseconds / (double)itemsProcessed));
}
public virtual void SkippedItemPresentInSerializationProvider(ISerializedReference root, string predicateName, string serializationProviderName, string justification)
{
_logger.Warn("[S] {0} by {1} but it was in {2}. {3}<br />This usually indicates an extraneous excluded serialized item is present in the {3}, which should be removed.".FormatWith(root.DisplayIdentifier, predicateName, serializationProviderName, justification));
}
public virtual void BeginLoadingTree(ISerializedReference rootSerializedItem)
{
}
private string GetErrorValue(SaveArgs args)
{
var results = new Dictionary <Item, IList <FieldDesynchronization> >();
try
{
foreach (var item in args.Items)
{
Item existingItem = Client.ContentDatabase.GetItem(item.ID, item.Language, item.Version);
Assert.IsNotNull(existingItem, "Existing item {0} did not exist! This should never occur.", item.ID);
var existingSitecoreItem = new SitecoreSourceItem(existingItem);
foreach (var configuration in _configurations)
{
// ignore conflicts on items that Unicorn is not managing
if (!configuration.Resolve <IPredicate>().Includes(existingSitecoreItem).IsIncluded)
{
continue;
}
ISerializedReference serializedReference = configuration.Resolve <ISerializationProvider>().GetReference(existingSitecoreItem);
if (serializedReference == null)
{
continue;
}
// not having an existing serialized version means no possibility of conflict here
ISerializedItem serializedItem = serializedReference.GetItem();
if (serializedItem == null)
{
continue;
}
var fieldPredicate = configuration.Resolve <IFieldPredicate>();
var fieldIssues = GetFieldSyncStatus(existingSitecoreItem, serializedItem, fieldPredicate);
if (fieldIssues.Count == 0)
{
continue;
}
results.Add(existingItem, fieldIssues);
}
}
// no problems
if (results.Count == 0)
{
return(null);
}
var sb = new StringBuilder();
sb.Append("CRITICAL MESSAGE FROM UNICORN:\n");
sb.Append("You need to run a Unicorn sync. The following fields did not match the serialized version:\n");
foreach (var item in results)
{
if (results.Count > 1)
{
sb.AppendFormat("\n{0}: {1}", item.Key.DisplayName, string.Join(", ", item.Value.Select(x => x.FieldName)));
}
else
{
sb.AppendFormat("\n{0}", string.Join(", ", item.Value.Select(x => x.FieldName)));
}
}
sb.Append("\n\nDo you want to overwrite anyway?\nTHIS MAY CAUSE LOST WORK.");
return(sb.ToString());
}
catch (Exception ex)
{
Log.Error("Exception occurred while performing serialization conflict check!", ex, this);
return("Exception occurred: " + ex.Message); // this will cause a few retries
}
}
/// <summary>
/// Recursive method that loads a given tree and retries failures already present if any
/// </summary>
protected virtual void LoadTreeRecursive(ISerializedReference root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(root, "root");
Assert.ArgumentNotNull(retryer, "retryer");
var included = Predicate.Includes(root);
if (!included.IsIncluded)
{
Logger.SkippedItemPresentInSerializationProvider(root, Predicate.GetType().Name, SerializationProvider.GetType().Name, included.Justification ?? string.Empty);
return;
}
try
{
// load the current level
LoadOneLevel(root, retryer, consistencyChecker);
// check if we have child paths to recurse down
var children = root.GetChildReferences(false);
if (children.Length > 0)
{
// make sure if a "templates" item exists in the current set, it goes first
if (children.Length > 1)
{
int templateIndex = Array.FindIndex(children, x => x.ItemPath.EndsWith("templates", StringComparison.OrdinalIgnoreCase));
if (templateIndex > 0)
{
var zero = children[0];
children[0] = children[templateIndex];
children[templateIndex] = zero;
}
}
// load each child path recursively
foreach (var child in children)
{
LoadTreeRecursive(child, retryer, consistencyChecker);
}
// pull out any standard values failures for immediate retrying
retryer.RetryStandardValuesFailures(item => DoLoadItem(item, null));
} // children.length > 0
}
catch (ConsistencyException)
{
throw;
}
catch (Exception ex)
{
retryer.AddTreeRetry(root, ex);
}
}
/// <summary>
/// Moves the descendant items of a serialized parent after it has been moved or renamed.
/// </summary>
/// <param name="oldReference">Reference to the original path pre-move/rename</param>
/// <param name="newItem">The newly renamed or moved parent item</param>
/// <param name="sourceItem">The source item representing the renamed/moved item. NOTE that the path of this item is incorrect a lot of the time so we ignore it.</param>
/// <remarks>
/// This method basically gets all descendants of the source item that was moved/renamed, generates an appropriate new serialized item for it, and _if the new child item is in the predicate_ we
/// serialize it to its new location. Finally, we delete the old children directory if it existed.
///
/// Doing it this way allows handling crazy cases like moving trees of items between included and excluded locations - or even moves or renames causing SOME of the children to be ignored. Wild.
/// </remarks>
protected virtual void MoveDescendants(ISerializedReference oldReference, ISerializedItem newItem, ISourceItem sourceItem)
{
// remove the extension from the new item's provider ID
string newItemReferencePath = SerializationPathUtility.GetReferenceDirectoryPath(newItem);
// if the paths were the same, no moving occurs (this can happen when saving templates, which spuriously can report "renamed" when they are not actually any such thing)
if (oldReference.ProviderId.Equals(newItemReferencePath, StringComparison.Ordinal)) return;
// this is for renaming an item that differs only by case from the original. Because NTFS is case-insensitive the 'new parent' exists
// already, but it will use the old name. Not quite what we want. So we need to manually rename the folder.
if (oldReference.ProviderId.Equals(newItemReferencePath, StringComparison.OrdinalIgnoreCase) && Directory.Exists(oldReference.ProviderId))
{
Directory.Move(oldReference.ProviderId, oldReference.ProviderId + "_tempunicorn");
Directory.Move(oldReference.ProviderId + "_tempunicorn", newItemReferencePath);
}
var descendantItems = GetDescendants(sourceItem).Cast<SitecoreSourceItem>();
// Go through descendant source items and serialize all that are included by the predicate
foreach (var descendant in descendantItems)
{
var syncItem = ItemSynchronization.BuildSyncItem(descendant.InnerItem);
// the newPhysicalPath will point to the OLD physical path pre-move/rename.
// We re-root the path to point to the new parent item's base path to fix that before we write to disk
var newPhysicalPath = SerializationPathUtility.GetSerializedItemPath(_rootPath, syncItem.DatabaseName, syncItem.ItemPath);
var newSerializedItem = new SitecoreSerializedItem(syncItem, newPhysicalPath, this);
if (!_predicate.Includes(newSerializedItem).IsIncluded) continue; // if the moved child location is outside the predicate, do not re-serialize
UpdateSerializedItem(newSerializedItem);
}
// remove the old children folder if it exists - as long as the original name was not a case insensitive version of this item
if (Directory.Exists(oldReference.ProviderId) && !oldReference.ProviderId.Equals(newItemReferencePath, StringComparison.OrdinalIgnoreCase))
{
Directory.Delete(oldReference.ProviderId, true);
}
}
/// <summary>
/// Checks if a preset includes a given serialized item
/// </summary>
protected PredicateResult Includes(IncludeEntry entry, ISerializedReference item)
{
// check for db match
if (item.DatabaseName != entry.Database) return new PredicateResult(false);
// check for path match
if (!item.ItemPath.StartsWith(entry.Path, StringComparison.OrdinalIgnoreCase)) return new PredicateResult(false);
// check excludes
return ExcludeMatches(entry, item);
}
/// <summary>
/// Gets the shortened version of a reference path. The short path is utilized when the actual path becomes longer than the OS allows paths to be, and is based on a hash.
/// This method will return the short path regardless of if it exists, and will return it even for path lengths that do not require using the shortened version.
/// </summary>
public static string GetShortSerializedReferencePath(string rootDirectory, ISerializedReference reference)
{
return GetShortSerializedReferencePath(rootDirectory, reference.DatabaseName, reference.ItemPath);
}
public virtual void DeleteSerializedItem(ISerializedReference item)
{
DeleteItemRecursive(item);
CleanupObsoleteShortens();
}
/// <summary>
/// A serialized reference might refer to a directory OR a serialized item file directly. This method makes sure we've got the item version, if it refers to a directory.
/// </summary>
public static string GetReferenceItemPath(ISerializedReference reference)
{
return reference.ProviderId.EndsWith(PathUtils.Extension, StringComparison.OrdinalIgnoreCase) ? reference.ProviderId : reference.ProviderId + PathUtils.Extension;
}
public virtual void EndLoadingTree(ISerializedReference rootSerializedItem, int itemsProcessed, long elapsedMilliseconds)
{
}
/// <summary>
/// Loads a set of children from a serialized path
/// </summary>
protected virtual void LoadOneLevel(ISerializedReference root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(root, "root");
Assert.ArgumentNotNull(retryer, "retryer");
var orphanCandidates = new Dictionary<ID, ISourceItem>();
// grab the root item's full metadata
var rootSerializedItem = root.GetItem();
if (rootSerializedItem == null)
{
Logger.SkippedItemMissingInSerializationProvider(root, SerializationProvider.GetType().Name);
return;
}
// get the corresponding item from Sitecore
ISourceItem rootItem = SourceDataProvider.GetItemById(rootSerializedItem.DatabaseName, rootSerializedItem.Id);
// we add all of the root item's direct children to the "maybe orphan" list (we'll remove them as we find matching serialized children)
if (rootItem != null)
{
var rootChildren = rootItem.Children;
foreach (ISourceItem child in rootChildren)
{
// if the preset includes the child add it to the orphan-candidate list (if we don't deserialize it below, it will be marked orphan)
var included = Predicate.Includes(child);
if (included.IsIncluded)
orphanCandidates[child.Id] = child;
else
{
Logger.SkippedItem(child, Predicate.GetType().Name, included.Justification ?? string.Empty);
}
}
}
// check for direct children of the target path
var children = rootSerializedItem.GetChildItems();
foreach (var child in children)
{
try
{
if (child.IsStandardValuesItem)
{
orphanCandidates.Remove(child.Id); // avoid marking standard values items orphans
retryer.AddItemRetry(child, new StandardValuesException(child.ItemPath));
}
else
{
// load a child item
var loadedItem = DoLoadItem(child, consistencyChecker);
if (loadedItem.Item != null)
{
orphanCandidates.Remove(loadedItem.Item.Id);
// check if we have any child serialized items under this loaded child item (existing children) -
// if we do not, we can orphan any children of the loaded item as well
var loadedItemsChildren = child.GetChildReferences(false);
if (loadedItemsChildren.Length == 0) // no children were serialized on disk
{
var loadedChildren = loadedItem.Item.Children;
foreach (ISourceItem loadedChild in loadedChildren)
{
orphanCandidates.Add(loadedChild.Id, loadedChild);
}
}
}
else if (loadedItem.Status == ItemLoadStatus.Skipped) // if the item got skipped we'll prevent it from being deleted
orphanCandidates.Remove(child.Id);
}
}
catch (ConsistencyException)
{
throw;
}
catch (Exception ex)
{
// if a problem occurs we attempt to retry later
retryer.AddItemRetry(child, ex);
// don't treat errors as cause to delete an item
orphanCandidates.Remove(child.Id);
}
}
// if we're forcing an update (ie deleting stuff not on disk) we send the items that we found that weren't on disk off to get evaluated as orphans
if (orphanCandidates.Count > 0)
{
bool disableNewSerialization = UnicornDataProvider.DisableSerialization;
try
{
UnicornDataProvider.DisableSerialization = true;
Evaluator.EvaluateOrphans(orphanCandidates.Values.ToArray());
}
finally
{
UnicornDataProvider.DisableSerialization = disableNewSerialization;
}
}
}
public virtual void DeleteSerializedItem(ISerializedReference item)
{
DeleteItemRecursive(item);
CleanupObsoleteShortens();
}
public virtual ISerializedReference[] GetChildReferences(ISerializedReference parent, bool recursive)
{
Assert.ArgumentNotNull(parent, "parent");
var longPath = SerializationPathUtility.GetReferenceDirectoryPath(parent);
var shortPath = SerializationPathUtility.GetShortSerializedReferencePath(_rootPath, parent);
Func <string, string[]> parseDirectory = path =>
{
if (!Directory.Exists(path))
{
return(new string[0]);
}
var resultSet = new HashSet <string>();
try
{
string[] files = Directory.GetFiles(path, "*" + PathUtils.Extension);
foreach (var file in files)
{
resultSet.Add(file);
}
string[] directories = SerializationPathUtility.GetDirectories(path, this);
// add directories that aren't already ref'd indirectly by a file
foreach (var directory in directories)
{
if (CommonUtils.IsDirectoryHidden(directory))
{
continue;
}
if (!resultSet.Contains(directory + PathUtils.Extension))
{
resultSet.Add(directory);
}
}
string[] resultArray = resultSet.ToArray();
// make sure if a "templates" item exists in the current set, it goes first
if (resultArray.Length > 1)
{
for (int i = 1; i < resultArray.Length; i++)
{
if ("templates".Equals(Path.GetFileName(resultArray[i]), StringComparison.OrdinalIgnoreCase))
{
string text = resultArray[0];
resultArray[0] = resultArray[i];
resultArray[i] = text;
}
}
}
return(resultArray);
}
catch (DirectoryNotFoundException)
{
// it seems like occasionally, even though we use Directory.Exists() to make sure the parent dir exists, that when we actually call Directory.GetFiles()
// it throws an error that the directory does not exist during recursive deletes. If the directory does not exist, then we can safely assume no children are present.
return(new string[0]);
}
};
var results = Enumerable.Concat(parseDirectory(longPath), parseDirectory(shortPath));
List <ISerializedReference> referenceResults = results.Select(x => (ISerializedReference) new SitecoreSerializedReference(x, this)).ToList();
if (recursive)
{
var localReferenceResults = referenceResults.ToArray();
foreach (var child in localReferenceResults)
{
referenceResults.AddRange(GetChildReferences(child, true));
}
}
return(referenceResults.ToArray());
}
/// <summary>
/// A serialized reference might refer to a directory OR a serialized item file directly. This method makes sure we've got the item version, if it refers to a directory.
/// </summary>
public static string GetReferenceItemPath(ISerializedReference reference)
{
return(reference.ProviderId.EndsWith(PathUtils.Extension, StringComparison.OrdinalIgnoreCase) ? reference.ProviderId : reference.ProviderId + PathUtils.Extension);
}
/// <summary>
/// Loads a set of children from a serialized path
/// </summary>
protected virtual void LoadOneLevel(ISerializedReference root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(root, "root");
Assert.ArgumentNotNull(retryer, "retryer");
var orphanCandidates = new Dictionary <ID, ISourceItem>();
// grab the root item's full metadata
var rootSerializedItem = root.GetItem();
if (rootSerializedItem == null)
{
Logger.SkippedItemMissingInSerializationProvider(root, SerializationProvider.GetType().Name);
return;
}
// get the corresponding item from Sitecore
ISourceItem rootItem = SourceDataProvider.GetItemById(rootSerializedItem.DatabaseName, rootSerializedItem.Id);
// we add all of the root item's direct children to the "maybe orphan" list (we'll remove them as we find matching serialized children)
if (rootItem != null)
{
var rootChildren = rootItem.Children;
foreach (ISourceItem child in rootChildren)
{
// if the preset includes the child add it to the orphan-candidate list (if we don't deserialize it below, it will be marked orphan)
var included = Predicate.Includes(child);
if (included.IsIncluded)
{
orphanCandidates[child.Id] = child;
}
else
{
Logger.SkippedItem(child, Predicate.GetType().Name, included.Justification ?? string.Empty);
}
}
}
// check for direct children of the target path
var children = rootSerializedItem.GetChildItems();
foreach (var child in children)
{
try
{
if (child.IsStandardValuesItem)
{
orphanCandidates.Remove(child.Id); // avoid marking standard values items orphans
retryer.AddItemRetry(child, new StandardValuesException(child.ItemPath));
}
else
{
// load a child item
var loadedItem = DoLoadItem(child, consistencyChecker);
if (loadedItem.Item != null)
{
orphanCandidates.Remove(loadedItem.Item.Id);
// check if we have any child serialized items under this loaded child item (existing children) -
// if we do not, we can orphan any included children of the loaded item as well
var loadedItemsChildren = child.GetChildReferences(false);
if (loadedItemsChildren.Length == 0) // no children were serialized on disk
{
var loadedChildren = loadedItem.Item.Children;
foreach (ISourceItem loadedChild in loadedChildren)
{
if (Predicate.Includes(loadedChild).IsIncluded)
{
orphanCandidates.Add(loadedChild.Id, loadedChild);
}
}
}
}
else if (loadedItem.Status == ItemLoadStatus.Skipped) // if the item got skipped we'll prevent it from being deleted
{
orphanCandidates.Remove(child.Id);
}
}
}
catch (ConsistencyException)
{
throw;
}
catch (Exception ex)
{
// if a problem occurs we attempt to retry later
retryer.AddItemRetry(child, ex);
// don't treat errors as cause to delete an item
orphanCandidates.Remove(child.Id);
}
}
// if we're forcing an update (ie deleting stuff not on disk) we send the items that we found that weren't on disk off to get evaluated as orphans
if (orphanCandidates.Count > 0)
{
bool disableNewSerialization = UnicornDataProvider.DisableSerialization;
try
{
UnicornDataProvider.DisableSerialization = true;
Evaluator.EvaluateOrphans(orphanCandidates.Values.ToArray());
}
finally
{
UnicornDataProvider.DisableSerialization = disableNewSerialization;
}
}
}
public virtual void SkippedItemMissingInSerializationProvider(ISerializedReference item, string serializationProviderName)
{
_logger.Warn("[S] {0}. Unable to get a serialized item for the path. <br />This usually indicates an orphaned serialized item tree in {1} which should be removed. <br />Less commonly, it could also indicate a sparsely serialized tree which is not supported, or a serialized item that is named differently than its metadata.".FormatWith(item.DisplayIdentifier, serializationProviderName));
}
/// <summary>
/// Gets the shortened version of a reference path. The short path is utilized when the actual path becomes longer than the OS allows paths to be, and is based on a hash.
/// This method will return the short path regardless of if it exists, and will return it even for path lengths that do not require using the shortened version.
/// </summary>
protected virtual string GetShortSerializedReferencePath(string rootDirectory, ISerializedReference reference)
{
return(GetShortSerializedReferencePath(rootDirectory, reference.DatabaseName, reference.ItemPath));
}
/// <summary>
/// A serialized reference might refer to a directory OR a serialized item file directly. This method makes sure we've got the directory version, if it refers to a file.
/// </summary>
protected virtual string GetReferenceDirectoryPath(ISerializedReference reference)
{
return(PathUtils.StripPath(reference.ProviderId));
}
/// <summary>
/// A serialized reference might refer to a directory OR a serialized item file directly. This method makes sure we've got the directory version, if it refers to a file.
/// </summary>
public static string GetReferenceDirectoryPath(ISerializedReference reference)
{
return(PathUtils.StripPath(reference.ProviderId));
}
public virtual void BeginLoadingTree(ISerializedReference rootSerializedItem)
{
}
/// <summary>
/// Gets the shortened version of a reference path. The short path is utilized when the actual path becomes longer than the OS allows paths to be, and is based on a hash.
/// This method will return the short path regardless of if it exists, and will return it even for path lengths that do not require using the shortened version.
/// </summary>
public static string GetShortSerializedReferencePath(string rootDirectory, ISerializedReference reference)
{
return(GetShortSerializedReferencePath(rootDirectory, reference.DatabaseName, reference.ItemPath));
}
public override void BeginLoadingTree(ISerializedReference rootSerializedItem)
{
_logger.Info("Unicorn loading {0}".FormatWith(rootSerializedItem.DisplayIdentifier, rootSerializedItem.ProviderId));
_logger.Debug("Provider root ID: {0}".FormatWith(rootSerializedItem.ProviderId));
}
