public async Task DownloadAsync(BuildConfig config, ISourceItem item, string filePath)
{
using (var source = File.OpenRead(item.Url)) {
using (var destination = File.OpenWrite(filePath))
{
await source.CopyToAsync(destination);
}
}
}
/// <summary>
/// Gets the physical path to the directory that contains children of the item path/database name
/// </summary>
public static string GetSerializedReferencePath(string rootDirectory, ISourceItem sourceItem)
{
var sitecoreSourceItem = sourceItem as SitecoreSourceItem;
Assert.IsNotNull(sitecoreSourceItem, "Source item must be a SitecoreSourceItem.");
// ReSharper disable PossibleNullReferenceException
return PathUtils.GetDirectoryPath(new ItemReference(sitecoreSourceItem.InnerItem).ToString(), rootDirectory);
// ReSharper restore PossibleNullReferenceException
}
public Task DownloadAsync(BuildConfig config, ISourceItem item, string filePath)
{
return Task.Run(()=> {
var local = item as LocalRepositoryFile;
var file = local==null ? (item as TFS2015FileItem) : local.Source as TFS2015FileItem;
if (file == null) {
throw new InvalidOperationException("Input=" + (item == null ? "null" : item.ToString()));
}
file.Item.DownloadFile(filePath);
});
}
/// <summary>
/// Gets the physical path to the .item file that defines the source item. Returns the path regardless of if the item file exists.
/// </summary>
public static string GetSerializedItemPath(string rootDirectory, ISourceItem sourceItem)
{
return GetSerializedReferencePath(rootDirectory, sourceItem) + PathUtils.Extension;
}
public virtual void DeletedItem(ISourceItem deletedItem)
{
_logger.Warn("[D] {0} because it did not exist in the serialization provider.".FormatWith(deletedItem.DisplayIdentifier));
_pipelineDataCollector.PushChangedItem(deletedItem, ChangeType.Deleted);
}
public virtual void NewSerializedVersionMatch(ItemVersion newSerializedVersion, ISerializedItem serializedItem, ISourceItem existingItem)
{
_logger.Debug("> New version {0}#{1} (serialized)".FormatWith(newSerializedVersion.Language, newSerializedVersion.VersionNumber));
}
protected IList<ISourceItem> GetDescendants(ISourceItem sourceItem)
{
var descendants = sourceItem.Children.ToList();
foreach (var child in descendants.ToArray())
{
descendants.AddRange(GetDescendants(child));
}
return descendants;
}
public ChangeEntry(ISourceItem item, ChangeType type)
{
SourceItem = item;
ChangeType = type;
}
/// <summary>
/// Loads a set of children from a serialized path
/// </summary>
protected virtual void LoadOneLevel(ISerializedReference root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(root, "root");
Assert.ArgumentNotNull(retryer, "retryer");
var orphanCandidates = new Dictionary <ID, ISourceItem>();
// grab the root item's full metadata
var rootSerializedItem = root.GetItem();
if (rootSerializedItem == null)
{
Logger.SkippedItemMissingInSerializationProvider(root, SerializationProvider.GetType().Name);
return;
}
// get the corresponding item from Sitecore
ISourceItem rootItem = SourceDataProvider.GetItemById(rootSerializedItem.DatabaseName, rootSerializedItem.Id);
// we add all of the root item's direct children to the "maybe orphan" list (we'll remove them as we find matching serialized children)
if (rootItem != null)
{
var rootChildren = rootItem.Children;
foreach (ISourceItem child in rootChildren)
{
// if the preset includes the child add it to the orphan-candidate list (if we don't deserialize it below, it will be marked orphan)
var included = Predicate.Includes(child);
if (included.IsIncluded)
{
orphanCandidates[child.Id] = child;
}
else
{
Logger.SkippedItem(child, Predicate.GetType().Name, included.Justification ?? string.Empty);
}
}
}
// check for direct children of the target path
var children = rootSerializedItem.GetChildItems();
foreach (var child in children)
{
try
{
if (child.IsStandardValuesItem)
{
orphanCandidates.Remove(child.Id); // avoid marking standard values items orphans
retryer.AddItemRetry(child, new StandardValuesException(child.ItemPath));
}
else
{
// load a child item
var loadedItem = DoLoadItem(child, consistencyChecker);
if (loadedItem.Item != null)
{
orphanCandidates.Remove(loadedItem.Item.Id);
// check if we have any child serialized items under this loaded child item (existing children) -
// if we do not, we can orphan any included children of the loaded item as well
var loadedItemsChildren = child.GetChildReferences(false);
if (loadedItemsChildren.Length == 0) // no children were serialized on disk
{
var loadedChildren = loadedItem.Item.Children;
foreach (ISourceItem loadedChild in loadedChildren)
{
if (Predicate.Includes(loadedChild).IsIncluded)
{
orphanCandidates.Add(loadedChild.Id, loadedChild);
}
}
}
}
else if (loadedItem.Status == ItemLoadStatus.Skipped) // if the item got skipped we'll prevent it from being deleted
{
orphanCandidates.Remove(child.Id);
}
}
}
catch (ConsistencyException)
{
throw;
}
catch (Exception ex)
{
// if a problem occurs we attempt to retry later
retryer.AddItemRetry(child, ex);
// don't treat errors as cause to delete an item
orphanCandidates.Remove(child.Id);
}
}
// if we're forcing an update (ie deleting stuff not on disk) we send the items that we found that weren't on disk off to get evaluated as orphans
if (orphanCandidates.Count > 0)
{
bool disableNewSerialization = UnicornDataProvider.DisableSerialization;
try
{
UnicornDataProvider.DisableSerialization = true;
Evaluator.EvaluateOrphans(orphanCandidates.Values.ToArray());
}
finally
{
UnicornDataProvider.DisableSerialization = disableNewSerialization;
}
}
}
public virtual void RenameSerializedItem(ISourceItem renamedItem, string oldName)
{
if (renamedItem == null || oldName == null) return;
var typed = renamedItem as SitecoreSourceItem;
if (typed == null) throw new ArgumentException("Renamed item must be a SitecoreSourceItem", "renamedItem");
// write the serialized item under its new name
var updatedItem = SerializeItem(renamedItem);
// find the children directory path of the previous item name, if it exists, and move them to the new child path
var renamedParentSerializationDirectory = SerializationPathUtility.GetReferenceParentPath(_rootPath, updatedItem);
var oldSerializedChildrenReference = new SitecoreSerializedReference(renamedParentSerializationDirectory + Path.DirectorySeparatorChar + oldName, this);
var shortOldSerializedChildrenPath = SerializationPathUtility.GetShortSerializedReferencePath(_rootPath, oldSerializedChildrenReference);
var shortOldSerializedChildrenReference = new SitecoreSerializedReference(shortOldSerializedChildrenPath, this);
if (Directory.Exists(oldSerializedChildrenReference.ProviderId))
MoveDescendants(oldSerializedChildrenReference, updatedItem, renamedItem);
if (Directory.Exists(shortOldSerializedChildrenPath))
MoveDescendants(shortOldSerializedChildrenReference, updatedItem, renamedItem);
// delete the original serialized item from pre-rename (unless the names only differ by case, in which case we'd delete the item entirely because NTFS is case insensitive!)
if (!renamedItem.Name.Equals(oldName, StringComparison.OrdinalIgnoreCase))
{
// note that we don't have to worry about short paths here because DeleteSerializedItem() knows how to find them
DeleteSerializedItem(oldSerializedChildrenReference);
}
}
public virtual void SkippedItem(ISourceItem skippedItem, string predicateName, string justification)
{
}
public ItemLoadResult(ItemLoadStatus status, ISourceItem item)
{
Item = item;
Status = status;
}
public virtual void OrphanSourceVersion(ISourceItem existingItem, ISerializedItem serializedItem, ItemVersion[] orphanSourceVersions)
{
_logger.Debug("> Orphaned version{0} {1} (source)".FormatWith(orphanSourceVersions.Length > 1 ? "s" : string.Empty, string.Join(", ", orphanSourceVersions.Select(x => x.Language + "#" + x.VersionNumber))));
}
Task <Uri> ITargetPageUriResolver.ResolveAsync(ISourceItem sourceItem, PnPContext targetContext, CancellationToken token)
{
throw new NotImplementedException();
}
public virtual void NewSerializedVersionMatch(ItemVersion newSerializedVersion, ISerializedItem serializedItem, ISourceItem existingItem)
{
_logger.Debug("> New version {0}#{1} (serialized)".FormatWith(newSerializedVersion.Language, newSerializedVersion.VersionNumber));
}
public virtual void IsNameMatch(ISerializedItem serializedItem, ISourceItem existingItem)
{
_logger.Debug("> Name: Serialized \"{0}\", Source \"{1}\"".FormatWith(serializedItem.Name, existingItem.Name));
}
public virtual void DeletedItem(ISourceItem deletedItem)
{
_logger.Warn("[D] {0} because it did not exist in the serialization provider.".FormatWith(deletedItem.DisplayIdentifier));
}
public override void SkippedItem(ISourceItem skippedItem, string predicateName, string justification)
{
_logger.Debug("[S] {0} (and children) by {1}: {2}".FormatWith(skippedItem.DisplayIdentifier, predicateName, justification));
}
/// <summary>
/// Allocates an object providing error information relating to a portion of a source document.
/// </summary>
/// <param name="sourceItem">The source item to which the error applies.</param>
/// <param name="error">An enumeration code that corresponds to this error. The enumeration identifier is used as the message key and the enumeration value is used as the code.</param>
/// <param name="messageArguments">Zero or more strings that are to be substituted for "{i}" sequences in the message string return by GetMessage.</param>
public AstErrorMessage(ISourceItem sourceItem, Error error, params string[] messageArguments)
: base(sourceItem.SourceLocation, (long)error, error.ToString(), messageArguments)
{
}
public virtual void SavedItem(string providerName, ISourceItem sourceItem, string triggerReason)
{
_logger.Info(string.Format("{0}: Serialized {1} ({2}) to disk ({3}).", providerName, sourceItem.ItemPath, sourceItem.Id, triggerReason));
}
/// <summary>
/// Allocates an object providing error information relating to a portion of a source document.
/// </summary>
/// <param name="sourceItem">The source item to which the error applies.</param>
/// <param name="error">An enumeration code that corresponds to this error. The enumeration identifier is used as the message key and the enumeration value is used as the code.</param>
/// <param name="relatedLocations">Zero ore more locations that are related to this error.</param>
/// <param name="messageArguments">Zero or more strings that are to be substituted for "{i}" sequences in the message string return by GetMessage.</param>
public AstErrorMessage(ISourceItem sourceItem, Error error, IEnumerable <ILocation> relatedLocations, params string[] messageArguments)
: base(sourceItem.SourceLocation, (long)error, error.ToString(), relatedLocations, messageArguments)
{
}
public virtual void MoveSerializedItem(ISourceItem sourceItem, ISourceItem newParentItem)
{
Assert.ArgumentNotNull(sourceItem, "sourceItem");
Assert.ArgumentNotNull(newParentItem, "newParentItem");
var sitecoreSource = sourceItem as SitecoreSourceItem;
var sitecoreParent = newParentItem as SitecoreSourceItem;
if (sitecoreParent == null) throw new ArgumentException("newParentItem must be a SitecoreSourceItem", "newParentItem");
if (sitecoreSource == null) throw new ArgumentException("sourceItem must be a SitecoreSourceItem", "sourceItem");
var oldRootDirectory = new SitecoreSerializedReference(SerializationPathUtility.GetSerializedReferencePath(_rootPath, sourceItem), this);
var oldRootItemPath = new SitecoreSerializedReference(SerializationPathUtility.GetReferenceItemPath(oldRootDirectory), this);
var newRootItemPath = SerializationPathUtility.GetSerializedReferencePath(_rootPath, newParentItem) + Path.DirectorySeparatorChar + sourceItem.Name + PathUtils.Extension;
var syncItem = ItemSynchronization.BuildSyncItem(sitecoreSource.InnerItem);
// update the path and parent IDs to the new location
syncItem.ParentID = newParentItem.Id.ToString();
syncItem.ItemPath = string.Concat(newParentItem.ItemPath, "/", syncItem.Name);
var serializedNewItem = new SitecoreSerializedItem(syncItem, newRootItemPath, this);
// write the moved sync item to its new destination
UpdateSerializedItem(serializedNewItem);
// move any children to the new destination (and fix their paths)
MoveDescendants(oldRootDirectory, serializedNewItem, sourceItem);
// remove the serialized item in the old location
DeleteSerializedItem(oldRootItemPath);
}
protected virtual PredicateResult ExcludeMatches(IncludeEntry entry, ISourceItem item)
{
PredicateResult result = ExcludeMatchesTemplate(entry.Exclude, item.TemplateName);
if (!result.IsIncluded) return result;
result = ExcludeMatchesTemplateId(entry.Exclude, item.TemplateId);
if (!result.IsIncluded) return result;
result = ExcludeMatchesPath(entry.Exclude, item.ItemPath);
if (!result.IsIncluded) return result;
result = ExcludeMatchesId(entry.Exclude, item.Id);
return result;
}
public virtual ISerializedItem SerializeItem(ISourceItem item)
{
Assert.ArgumentNotNull(item, "item");
var sitecoreSourceItem = item as SitecoreSourceItem;
var sitecoreItem = sitecoreSourceItem != null ? sitecoreSourceItem.InnerItem : Factory.GetDatabase(item.DatabaseName).GetItem(item.Id);
Assert.IsNotNull(sitecoreItem, "Item to serialize did not exist!");
var serializedPath = SerializationPathUtility.GetSerializedItemPath(_rootPath, item);
var serializedItem = new SitecoreSerializedItem(ItemSynchronization.BuildSyncItem(sitecoreItem), serializedPath, this);
UpdateSerializedItem(serializedItem);
return serializedItem;
}
/// <summary>
/// Checks if a preset includes a given item
/// </summary>
protected PredicateResult Includes(IncludeEntry entry, ISourceItem item)
{
// check for db match
if (item.DatabaseName != entry.Database) return new PredicateResult(false);
// check for path match
if (!item.ItemPath.StartsWith(entry.Path, StringComparison.OrdinalIgnoreCase)) return new PredicateResult(false);
// check excludes
return ExcludeMatches(entry, item);
}
/// <summary>
/// Moves the descendant items of a serialized parent after it has been moved or renamed.
/// </summary>
/// <param name="oldReference">Reference to the original path pre-move/rename</param>
/// <param name="newItem">The newly renamed or moved parent item</param>
/// <param name="sourceItem">The source item representing the renamed/moved item. NOTE that the path of this item is incorrect a lot of the time so we ignore it.</param>
/// <remarks>
/// This method basically gets all descendants of the source item that was moved/renamed, generates an appropriate new serialized item for it, and _if the new child item is in the predicate_ we
/// serialize it to its new location. Finally, we delete the old children directory if it existed.
///
/// Doing it this way allows handling crazy cases like moving trees of items between included and excluded locations - or even moves or renames causing SOME of the children to be ignored. Wild.
/// </remarks>
protected virtual void MoveDescendants(ISerializedReference oldReference, ISerializedItem newItem, ISourceItem sourceItem)
{
// remove the extension from the new item's provider ID
string newItemReferencePath = SerializationPathUtility.GetReferenceDirectoryPath(newItem);
// if the paths were the same, no moving occurs (this can happen when saving templates, which spuriously can report "renamed" when they are not actually any such thing)
if (oldReference.ProviderId.Equals(newItemReferencePath, StringComparison.Ordinal)) return;
// this is for renaming an item that differs only by case from the original. Because NTFS is case-insensitive the 'new parent' exists
// already, but it will use the old name. Not quite what we want. So we need to manually rename the folder.
if (oldReference.ProviderId.Equals(newItemReferencePath, StringComparison.OrdinalIgnoreCase) && Directory.Exists(oldReference.ProviderId))
{
Directory.Move(oldReference.ProviderId, oldReference.ProviderId + "_tempunicorn");
Directory.Move(oldReference.ProviderId + "_tempunicorn", newItemReferencePath);
}
var descendantItems = GetDescendants(sourceItem).Cast<SitecoreSourceItem>();
// Go through descendant source items and serialize all that are included by the predicate
foreach (var descendant in descendantItems)
{
var syncItem = ItemSynchronization.BuildSyncItem(descendant.InnerItem);
// the newPhysicalPath will point to the OLD physical path pre-move/rename.
// We re-root the path to point to the new parent item's base path to fix that before we write to disk
var newPhysicalPath = SerializationPathUtility.GetSerializedItemPath(_rootPath, syncItem.DatabaseName, syncItem.ItemPath);
var newSerializedItem = new SitecoreSerializedItem(syncItem, newPhysicalPath, this);
if (!_predicate.Includes(newSerializedItem).IsIncluded) continue; // if the moved child location is outside the predicate, do not re-serialize
UpdateSerializedItem(newSerializedItem);
}
// remove the old children folder if it exists - as long as the original name was not a case insensitive version of this item
if (Directory.Exists(oldReference.ProviderId) && !oldReference.ProviderId.Equals(newItemReferencePath, StringComparison.OrdinalIgnoreCase))
{
Directory.Delete(oldReference.ProviderId, true);
}
}
public PredicateResult Includes(ISourceItem item)
{
// no entries = include everything
if (_preset.FirstOrDefault() == null) return new PredicateResult(true);
var result = new PredicateResult(true);
PredicateResult priorityResult = null;
foreach (var entry in _preset)
{
result = Includes(entry, item);
if (result.IsIncluded) return result; // it's definitely included if anything includes it
if (!string.IsNullOrEmpty(result.Justification)) priorityResult = result; // a justification means this is probably a more 'important' fail than others
}
return priorityResult ?? result; // return the last failure
}
public virtual void SkippedItem(ISourceItem skippedItem, string predicateName, string justification)
{
}
private void Serialize(ISourceItem root, IPredicate predicate, ISerializationProvider serializationProvider, ILogger logger)
{
var predicateResult = predicate.Includes(root);
if (predicateResult.IsIncluded)
{
serializationProvider.SerializeItem(root);
foreach (var child in root.Children)
{
Serialize(child, predicate, serializationProvider, logger);
}
}
else
{
logger.Warn("[S] {0} because {1}".FormatWith(root.DisplayIdentifier, predicateResult.Justification));
}
}
public virtual void IsTemplateMatch(ISerializedItem serializedItem, ISourceItem existingItem)
{
_logger.Debug("> Template: Serialized \"{0}\", Source \"{1}\"".FormatWith(serializedItem.TemplateName, existingItem.TemplateName));
}
private ErrorForOutParameterReporter(LanguageSpecificCompilationHelper helper, ISourceItem defaultSourceItemForErrorReporting)
{
this.helper = helper;
this.defaultSourceItemForErrorReporting = defaultSourceItemForErrorReporting;
}
public virtual void OrphanSourceVersion(ISourceItem existingItem, ISerializedItem serializedItem, ItemVersion[] orphanSourceVersions)
{
_logger.Debug("> Orphaned version{0} {1} (source)".FormatWith(orphanSourceVersions.Length > 1 ? "s" : string.Empty, string.Join(", ", orphanSourceVersions.Select(x => x.Language + "#" + x.VersionNumber))));
}
public virtual void MovedItem(string providerName, ISourceItem sourceItem, ISourceItem destinationItem)
{
_logger.Info(string.Format("{0}: Moved serialized item {1} ({2}) to {3}", providerName, sourceItem.ItemPath, sourceItem.Id, destinationItem.ItemPath));
}
public void PushChangedItem(ISourceItem sourceItem, ChangeType type)
{
_entries.Enqueue(new ChangeEntry(sourceItem, type));
}
protected virtual bool AnyFieldMatch(FieldDictionary sourceFields, FieldDictionary targetFields, ISourceItem existingItem, ISerializedItem serializedItem, DeferredLogWriter <ISerializedAsMasterEvaluatorLogger> deferredUpdateLog, ItemVersion version = null)
{
if (sourceFields == null)
{
return(false);
}
return(sourceFields.Any(x =>
{
if (!_fieldPredicate.Includes(x.Key).IsIncluded)
{
return false;
}
if (!existingItem.IsFieldComparable(x.Key))
{
return false;
}
bool isMatch = IsFieldMatch(x.Value, targetFields, x.Key);
if (isMatch)
{
deferredUpdateLog.AddEntry(logger =>
{
string sourceFieldValue;
targetFields.TryGetValue(x.Key, out sourceFieldValue);
if (version == null)
{
logger.IsSharedFieldMatch(serializedItem, x.Key, x.Value, sourceFieldValue);
}
else
{
logger.IsVersionedFieldMatch(serializedItem, version, x.Key, x.Value, sourceFieldValue);
}
});
}
return isMatch;
}));
}
public ItemLoadResult(ItemLoadStatus status, ISourceItem item)
{
Item = item;
Status = status;
}
/// <summary>
/// Allocates an object providing error information relating to a portion of a source document.
/// </summary>
/// <param name="sourceItem">The source item to which the error applies.</param>
/// <param name="error">An enumeration code that corresponds to this error. The enumeration identifier is used as the message key and the enumeration value is used as the code.</param>
/// <param name="messageArguments">Zero or more strings that are to be substituted for "{i}" sequences in the message string return by GetMessage.</param>
public AstErrorMessage(ISourceItem sourceItem, Error error, params string[] messageArguments)
: base(sourceItem.SourceLocation, (long)error, error.ToString(), messageArguments) {
}
public virtual void CopiedItem(string providerName, Func<ISourceItem> sourceItem, ISourceItem copiedItem)
{
}
/// <summary>
/// Allocates an object providing error information relating to a portion of a source document.
/// </summary>
/// <param name="sourceItem">The source item to which the error applies.</param>
/// <param name="error">An enumeration code that corresponds to this error. The enumeration identifier is used as the message key and the enumeration value is used as the code.</param>
/// <param name="relatedLocations">Zero ore more locations that are related to this error.</param>
/// <param name="messageArguments">Zero or more strings that are to be substituted for "{i}" sequences in the message string return by GetMessage.</param>
public AstErrorMessage(ISourceItem sourceItem, Error error, IEnumerable<ILocation> relatedLocations, params string[] messageArguments)
: base(sourceItem.SourceLocation, (long)error, error.ToString(), relatedLocations, messageArguments) {
}
public virtual void RenamedItem(string providerName, ISourceItem sourceItem, string oldName)
{
_logger.Info(string.Format("{0}: Renamed serialized item to {1} from {2}", providerName, sourceItem.ItemPath, oldName));
}
public override void SkippedItem(ISourceItem skippedItem, string predicateName, string justification)
{
_logger.Debug("[S] {0} (and children) by {1}: {2}".FormatWith(skippedItem.DisplayIdentifier, predicateName, justification));
}
public virtual ISerializedReference GetReference(ISourceItem sourceItem)
{
Assert.ArgumentNotNull(sourceItem, "sourceItem");
var physicalPath = SerializationPathUtility.GetSerializedReferencePath(_rootPath, sourceItem);
if (!Directory.Exists(physicalPath))
{
physicalPath = SerializationPathUtility.GetSerializedItemPath(_rootPath, sourceItem);
if (!File.Exists(physicalPath))
return null;
}
return new SitecoreSerializedReference(physicalPath, this);
}
public ChangeEntry(ISourceItem item, ChangeType type)
{
SourceItem = item;
ChangeType = type;
}