/*
* FINDING CHILD PATHS, GIVEN A VIRTUAL PATH AND ID
*
* 1. Determine the physical path of the item to get children for (use 'finding file paths, given a virtual path'), compare matches against ID
* 2. Determine if Path.GetFileNameWithoutExtension(filename) exists as a directory in the physical path
* 3. If it does, get all children of that directory that are both files and have the expected serialized item extension from the formatter
* 4. Read the parent item file and get the item ID. Check in the SFS root for a folder named that ID (a loopback with children of the item whose names were too long to fit)
* - If the folder exists, add all children of that directory that are both files and have the expected serialized item extension from the formatter
* 5. Note: unlike searching by path, this guarantees ONLY children of the correct item if multiple same named items are present
*/
protected virtual string[] GetChildPaths(IItemMetadata item)
{
Assert.ArgumentNotNull(item, "item");
IItemMetadata serializedItem = GetItemForGlobalPath(item.Path, item.Id);
if (serializedItem == null)
{
throw new InvalidOperationException("Item {0} does not exist on disk.".FormatWith(item.Path));
}
IEnumerable <FileData> children = Enumerable.Empty <FileData>();
var childrenPath = Path.ChangeExtension(serializedItem.SerializedItemId, null);
if (Directory.Exists(childrenPath))
{
children = FastDirectoryEnumerator.GetFiles(childrenPath, "*" + _formatter.FileExtension, SearchOption.TopDirectoryOnly);
}
var shortPath = Path.Combine(_physicalRootPath, item.Id.ToString());
if (Directory.Exists(shortPath))
{
children = children.Concat(FastDirectoryEnumerator.GetFiles(shortPath, "*" + _formatter.FileExtension, SearchOption.TopDirectoryOnly));
}
return(children.Select(result => result.Path).ToArray());
}
protected virtual void AddToMetadataCache(IItemMetadata metadata, string path = null)
{
var cachedValue = new WrittenItemMetadata(metadata.Id, metadata.ParentId, metadata.TemplateId, metadata.Path, path ?? metadata.SerializedItemId);
_idCache[metadata.Id] = cachedValue;
_metadataCache.AddOrUpdate(cachedValue.SerializedItemId, cachedValue);
}
/*
* REMOVING ITEMS, GIVEN A VIRTUAL PATH AND ID
*
* 1. Determine the physical path of the item to get children for (use 'finding file paths, given a virtual path'), compare matches against ID
* 2. Use 'finding child paths, given a physical path' recursively to find all descendant items including through loopbacks
* 3. Starting at the deepest paths, begin deleting item files and - if present - children subfolders, until all are gone
*/
public virtual bool Remove(IItemMetadata item)
{
Assert.ArgumentNotNull(item, "item");
using (new SfsDuplicateIdCheckingDisabler())
{
IItemMetadata itemToRemove = GetItemForGlobalPath(item.Path, item.Id);
if (itemToRemove == null)
{
return(false);
}
var descendants = GetDescendants(item, true)
.Concat(new[] { itemToRemove })
.OrderByDescending(desc => desc.Path)
.ToArray();
foreach (var descendant in descendants)
{
RemoveWithoutChildren(descendant);
}
}
return(true);
}
private string[] GetChildPaths(IItemMetadata item)
{
Assert.ArgumentNotNull(item, nameof(item));
IItemMetadata serializedItem = this.GetItemForGlobalPath(item.Path, item.Id);
if (serializedItem == null)
{
throw new InvalidOperationException($"Item {item.Path} does not exist on disk.");
}
IEnumerable <string> childPaths = Enumerable.Empty <string>();
string childrenPath = Path.ChangeExtension(serializedItem.SerializedItemId, null);
if (this._azureManager.DirectoryExists(childrenPath))
{
childPaths = this._azureManager.EnumerateFiles(
childrenPath,
this._formatter.FileExtension,
SearchOption.TopDirectoryOnly);
}
string guidBasedPath = Path.Combine(this._physicalRootPath, item.Id.ToString());
if (this._azureManager.DirectoryExists(guidBasedPath))
{
childPaths = childPaths.Concat(this._azureManager.EnumerateFiles(
guidBasedPath,
this._formatter.FileExtension,
SearchOption.TopDirectoryOnly));
}
string[] result = childPaths.ToArray();
return(result);
}
protected virtual IList <IItemMetadata> FilterDescendantsAndSelf(IItemData root, Func <IItemMetadata, bool> predicate)
{
Assert.ArgumentNotNull(root, nameof(root));
var descendants = new List <IItemMetadata>();
var childQueue = new Queue <IItemMetadata>();
childQueue.Enqueue(root);
while (childQueue.Count > 0)
{
IItemMetadata parent = childQueue.Dequeue();
if (predicate(parent))
{
descendants.Add(parent);
}
SerializationBlobStorageTree tree = this.GetTreeForPath(parent.Path, root.DatabaseName);
if (tree == null)
{
continue;
}
IItemMetadata[] children = tree.GetChildrenMetadata(parent).ToArray();
foreach (IItemMetadata item in children)
{
childQueue.Enqueue(item);
}
}
return(descendants);
}
protected virtual IItemMetadata GetParentSerializedItem(IItemMetadata item)
{
Assert.ArgumentNotNull(item, "item");
var localPath = ConvertGlobalVirtualPathToTreeVirtualPath(item.Path);
// Start by using "finding file paths, given a virtual path" on the parent path of the item
var parentVirtualPath = localPath.Substring(0, localPath.LastIndexOf('/'));
if (parentVirtualPath == string.Empty)
{
return(null);
}
var parentPhysicalPaths = GetPhysicalFilePathsForVirtualPath(parentVirtualPath);
if (parentPhysicalPaths.Length == 0)
{
return(null);
}
// If multiple parent path matches exist, narrow them by the parent ID of the item - if no matches, throw
if (parentPhysicalPaths.Length > 1)
{
// find the expected parent's physical path
var parentItem = parentPhysicalPaths.Select(ReadItemMetadata).FirstOrDefault(parentCandiate => parentCandiate.Id == item.ParentId);
return(parentItem);
}
return(ReadItemMetadata(parentPhysicalPaths[0]));
}
/// <summary>
/// Removes an item but does not process any descendant items. You probably almost never want to use this, in favor of Remove() instead.
/// This method is here for when a specific item needs to be removed, without messing with children. This occurs for example when
/// you move an item which has loopback pathed children, who may preserve the same source and destination location.
/// </summary>
public virtual void RemoveWithoutChildren(IItemMetadata descendant)
{
lock (FileUtil.GetFileLock(descendant.SerializedItemId))
{
BeforeFilesystemDelete(descendant.SerializedItemId);
try
{
ActionRetryer.Perform(() =>
{
_treeWatcher.PushKnownUpdate(descendant.SerializedItemId, TreeWatcher.TreeWatcherChangeType.Delete);
File.Delete(descendant.SerializedItemId);
});
}
catch (Exception exception)
{
throw new SfsDeleteException("Error deleting SFS item " + descendant.SerializedItemId, exception);
}
AfterFilesystemDelete(descendant.SerializedItemId);
var childrenDirectory = Path.ChangeExtension(descendant.SerializedItemId, null);
if (Directory.Exists(childrenDirectory))
{
BeforeFilesystemDelete(childrenDirectory);
try
{
ActionRetryer.Perform(() =>
{
_treeWatcher.PushKnownUpdate(childrenDirectory, TreeWatcher.TreeWatcherChangeType.Delete);
Directory.Delete(childrenDirectory, true);
});
}
catch (Exception exception)
{
throw new SfsDeleteException("Error deleting SFS directory " + childrenDirectory, exception);
}
AfterFilesystemDelete(childrenDirectory);
}
var shortChildrenDirectory = new DirectoryInfo(Path.Combine(_physicalRootPath, descendant.Id.ToString()));
if (shortChildrenDirectory.Exists && !shortChildrenDirectory.EnumerateFiles().Any())
{
BeforeFilesystemDelete(shortChildrenDirectory.FullName);
try
{
ActionRetryer.Perform(() =>
{
_treeWatcher.PushKnownUpdate(shortChildrenDirectory.FullName, TreeWatcher.TreeWatcherChangeType.Delete);
Directory.Delete(shortChildrenDirectory.FullName);
});
}
catch (Exception exception)
{
throw new SfsDeleteException("Error deleting SFS directory " + shortChildrenDirectory, exception);
}
AfterFilesystemDelete(shortChildrenDirectory.FullName);
}
}
}
public IEnumerable <IItemMetadata> GetChildrenMetadata(IItemMetadata parentItem)
{
Assert.ArgumentNotNull(parentItem, nameof(parentItem));
return(this.GetChildPaths(parentItem)
.AsParallel().WithDegreeOfParallelism(Utils.Settings.DegreeOfParallelism)
.Select(this.ReadItemMetadata));
}
private void AddToMetadataCache(IItemMetadata metadata, string path = null)
{
var writtenItemMetadata = new WrittenItemMetadata(metadata.Id, metadata.ParentId, metadata.TemplateId,
metadata.Path, path ?? metadata.SerializedItemId);
this._idCache[metadata.Id] = writtenItemMetadata;
this._metadataCache.AddOrUpdate(writtenItemMetadata.SerializedItemId, writtenItemMetadata);
}
protected virtual void RemoveItemFromCaches(IItemMetadata metadata, string databaseName)
{
try
{
if (databaseName != Database.Name)
{
return;
}
// this is a bit heavy handed, sure.
// but the caches get interdependent stuff - like caching child IDs
// that make it difficult to cleanly remove a single item ID from all cases in the cache
// either way, this should be a relatively rare occurrence (from runtime changes on disk)
// and we're preserving prefetch, etc. Seems pretty zippy overall.
CacheManager.ClearAllCaches();
if (metadata == null)
{
return;
}
if (metadata.TemplateId == TemplateIDs.Template.Guid ||
metadata.TemplateId == TemplateIDs.TemplateField.Guid ||
(metadata.Path != null && metadata.Path.EndsWith("__Standard Values", StringComparison.OrdinalIgnoreCase)))
{
Database.Engines.TemplateEngine.Reset();
}
if (_syncConfiguration != null && (_syncConfiguration.UpdateLinkDatabase || _syncConfiguration.UpdateSearchIndex))
{
var item = GetSourceItemFromId(new ID(metadata.Id), true);
if (item == null)
{
return;
}
if (_syncConfiguration.UpdateLinkDatabase)
{
Globals.LinkDatabase.UpdateReferences(item);
}
if (_syncConfiguration.UpdateSearchIndex)
{
foreach (var index in ContentSearchManager.Indexes)
{
ReflectionUtil.CallMethod(typeof(IndexCustodian), "UpdateItem", true, true, true, new object[] { index, new SitecoreItemUniqueId(item.Uri) });
//IndexCustodian.UpdateItem(index, new SitecoreItemUniqueId(item.Uri));
}
}
}
}
catch (Exception ex)
{
// we catch this because this method runs on a background thread. If an unhandled exception occurs there, the app pool terminates and that's Naughty(tm).
Log.Error($"[Unicorn] Exception occurred while processing a background item cache removal on {metadata?.Path ?? "unknown item"}", ex, this);
}
}
private IItemMetadata GetItemMetaData()
{
if (_itemMetaData != null)
{
return(_itemMetaData);
}
using (var stream = File.OpenRead(ItemPath))
{
return(_itemMetaData = _formatter.ReadSerializedItemMetadata(stream, ItemPath));
}
}
/// <summary>
/// Gets an item from the tree by ID.
/// Note: the first call to this method ensures that the tree is configured for high speed reading by ID,
/// which fills the metadata cache with all instances from disk. Reading by ID is not recommended for very large datasets,
/// but is quite good with smaller datasets of 1000-2000 items.
/// </summary>
public virtual IItemData GetItemById(Guid id)
{
EnsureConfiguredForFastReads();
IItemMetadata cached = GetFromMetadataCache(id);
if (cached == null)
{
return(null);
}
return(ReadItem(cached.SerializedItemId));
}
/// <summary>
/// Gets the item data.
/// </summary>
/// <returns></returns>
protected virtual IItemData GetItemData()
{
if (_itemData != null)
{
return(_itemData);
}
using (var stream = File.OpenRead(ItemPath))
{
_itemData = _formatter.ReadSerializedItem(stream, ItemPath);
}
_itemMetaData = _itemData;
return(_itemData);
}
public IItemData GetItemById(Guid id)
{
this.EnsureConfiguredForFastReads();
IItemMetadata fromMetadataCache = this.GetFromMetadataCache(id);
if (fromMetadataCache == null)
{
return(null);
}
return(this.ReadItem(fromMetadataCache.SerializedItemId));
}
protected virtual void RemoveItemFromCaches(IItemMetadata metadata, string databaseName)
{
if (databaseName != Database.Name)
{
return;
}
// this is a bit heavy handed, sure.
// but the caches get interdependent stuff - like caching child IDs
// that make it difficult to cleanly remove a single item ID from all cases in the cache
// either way, this should be a relatively rare occurrence (from runtime changes on disk)
// and we're preserving prefetch, etc. Seems pretty zippy overall.
Database.Caches.DataCache.Clear();
Database.Caches.ItemCache.Clear();
Database.Caches.ItemPathsCache.Clear();
Database.Caches.PathCache.Clear();
if (metadata == null)
{
return;
}
if (metadata.TemplateId == TemplateIDs.Template.Guid || metadata.TemplateId == TemplateIDs.TemplateField.Guid || metadata.Path.EndsWith("__Standard Values", StringComparison.OrdinalIgnoreCase))
{
Database.Engines.TemplateEngine.Reset();
}
if (_syncConfiguration.UpdateLinkDatabase || _syncConfiguration.UpdateSearchIndex)
{
var item = GetItemFromId(new ID(metadata.Id), true);
if (item == null)
{
return;
}
if (_syncConfiguration.UpdateLinkDatabase)
{
Globals.LinkDatabase.UpdateReferences(item);
}
if (_syncConfiguration.UpdateSearchIndex)
{
foreach (var index in ContentSearchManager.Indexes)
{
IndexCustodian.UpdateItem(index, new SitecoreItemUniqueId(item.Uri));
}
}
}
}
public virtual IList <IItemMetadata> GetDescendants(IItemMetadata root, bool ignoreReadErrors)
{
Assert.ArgumentNotNull(root, "root");
var descendants = new List <IItemMetadata>();
var childQueue = new Queue <IItemMetadata>();
childQueue.Enqueue(root);
while (childQueue.Count > 0)
{
var parent = childQueue.Dequeue();
// add current item to descendant results
if (parent.Id != root.Id)
{
descendants.Add(parent);
}
var children = GetChildPaths(parent);
foreach (var physicalPath in children)
{
try
{
var child = ReadItemMetadata(physicalPath);
if (child != null)
{
childQueue.Enqueue(child);
}
}
catch (Exception)
{
if (ignoreReadErrors)
{
continue;
}
throw;
}
}
}
return(descendants);
}
protected virtual void RemoveItemFromCaches(IItemMetadata metadata, string databaseName)
{
if (databaseName != Database.Name)
{
return;
}
// this is a bit heavy handed, sure.
// but the caches get interdependent stuff - like caching child IDs
// that make it difficult to cleanly remove a single item ID from all cases in the cache
// either way, this should be a relatively rare occurrence (from runtime changes on disk)
// and we're preserving prefetch, etc. Seems pretty zippy overall.
Database.Caches.DataCache.Clear();
Database.Caches.ItemCache.Clear();
Database.Caches.ItemPathsCache.Clear();
Database.Caches.PathCache.Clear();
}
private IItemMetadata GetItemForGlobalPath(string globalPath, Guid expectedItemId)
{
Assert.ArgumentNotNullOrEmpty(globalPath, nameof(globalPath));
string localPath = this.ConvertGlobalVirtualPathToTreeVirtualPath(globalPath);
IItemMetadata cached = this.GetFromMetadataCache(expectedItemId);
if (cached != null && globalPath.Equals(cached.Path, StringComparison.OrdinalIgnoreCase))
{
return(cached);
}
IItemMetadata result = this.GetPhysicalFilePathsForVirtualPath(localPath)
.Select(this.ReadItemMetadata)
.FirstOrDefault(candidateItem => candidateItem != null && candidateItem.Id == expectedItemId);
if (result == null)
{
return(null);
}
// in a specific circumstance we want to ignore dupe items with the same IDs:
// when we move or rename an item we delete the old items after we wrote the newly moved/renamed items
// this means that the tree temporarily has known dupes. We need to be able to ignore those
// when we're deleting the old items to make the tree sane again.
if (Switcher <bool, SfsDuplicateIdCheckingDisabler> .CurrentValue)
{
return(result);
}
IItemMetadata temp = this.GetFromMetadataCache(expectedItemId);
if (temp != null && temp.SerializedItemId != result.SerializedItemId)
{
throw new InvalidOperationException("The item with ID {0} has duplicate item files serialized ({1}, {2}). Please remove the incorrect one and try again.".FormatWith(result.Id, temp.SerializedItemId, result.SerializedItemId));
}
// note: we only actually add to cache if we checked for dupe IDs. This is to avoid cache poisoning.
this.AddToMetadataCache(result);
return(result);
}
private IItemData ReadItem(string filePath)
{
Assert.ArgumentNotNullOrEmpty(filePath, nameof(filePath));
IItemData item = this._dataCache.GetValue(filePath, fileInfoPath =>
{
try
{
using (Stream stream = this._azureManager.GetFileStream(fileInfoPath))
{
IItemData itemData;
if (!this._useBigFilesLazyLoad || stream.Length < Utils.Settings.LazyAzureItemThreshold)
{
itemData = this._formatter.ReadSerializedItem(stream, fileInfoPath);
}
else
{
IItemMetadata itemMetadata = this._formatter.ReadSerializedItemMetadata(stream, fileInfoPath);
itemData = new AzureLazyItemData(itemMetadata, fileInfoPath, this._azureManager, this._formatter);
}
itemData.DatabaseName = this.DatabaseName;
this.AddToMetadataCache(itemData);
Log.Info($"[Rainbow] [AzureBlob] reading {fileInfoPath} data / metadata(with lazy loading). Stopped at position {stream?.Position}", this);
return(itemData);
}
}
catch (Exception ex)
{
throw new SfsReadException($"[Rainbow] [AzureBlob] Error while reading SFS item {filePath}", ex);
}
});
if (this._dataCache.Enabled && item != null)
{
return(new FsCachedItem(item, () => this.GetChildren(item)));
}
return(item);
}
private IList <IItemMetadata> GetDescendants(
IItemMetadata root,
bool ignoreReadErrors)
{
Assert.ArgumentNotNull(root, nameof(root));
IList <IItemMetadata> itemMetadataList = new List <IItemMetadata>();
Queue <IItemMetadata> itemMetadataQueue = new Queue <IItemMetadata>();
itemMetadataQueue.Enqueue(root);
while (itemMetadataQueue.Count > 0)
{
IItemMetadata processingItemMetadata = itemMetadataQueue.Dequeue();
if (processingItemMetadata.Id != root.Id)
{
itemMetadataList.Add(processingItemMetadata);
}
foreach (string childPath in this.GetChildPaths(processingItemMetadata))
{
try
{
IItemMetadata childItemMetadata = this.ReadItemMetadata(childPath);
if (childItemMetadata != null)
{
itemMetadataQueue.Enqueue(childItemMetadata);
}
}
catch (Exception)
{
if (!ignoreReadErrors)
{
throw;
}
}
}
}
return(itemMetadataList);
}
private IItemMetadata ReadItemMetadata(string filePath)
{
Assert.ArgumentNotNullOrEmpty(filePath, nameof(filePath));
return(this._metadataCache.GetValue(filePath, fileInfoPath =>
{
try
{
using (Stream stream = this._azureManager.GetFileStream(fileInfoPath))
{
IItemMetadata itemMetadata = this._formatter.ReadSerializedItemMetadata(stream, fileInfoPath);
this._idCache[itemMetadata.Id] = itemMetadata;
Log.Info($"[Rainbow] [AzureBlob] reading {fileInfoPath} metadata. Stopped at position {stream?.Position}", this);
return itemMetadata;
}
}
catch (Exception ex)
{
throw new SfsReadException($"[Rainbow] [AzureBlob] Error while reading SFS metadata {filePath}", ex);
}
}));
}
/// <summary>
/// Gets the item data.
/// </summary>
/// <returns></returns>
protected virtual IItemData GetItemData()
{
if (_itemData != null)
{
return(_itemData);
}
using (var stream = File.OpenRead(ItemPath))
{
try
{
_itemData = _formatter.ReadSerializedItem(stream, ItemPath);
}
catch (Exception)
{
Console.WriteLine("Unable to deserialize item: " + ItemPath);
return(null);
}
}
_itemMetaData = _itemData;
return(_itemData);
}
public AzureLazyItemData(
IItemMetadata itemMetadata,
string filePath,
IAzureManager azureManager,
ISerializationFormatter serializationFormatter)
{
Assert.ArgumentNotNull(itemMetadata, nameof(itemMetadata));
Assert.ArgumentNotNull(azureManager, nameof(azureManager));
Assert.ArgumentNotNull(filePath, nameof(filePath));
Assert.ArgumentNotNull(serializationFormatter, nameof(serializationFormatter));
this.itemMetadata = itemMetadata;
this.azureManager = azureManager;
this.filePath = filePath;
this.serializationFormatter = serializationFormatter;
// ToDo: fix it - to do the lazy read, the context is needed to be preserved.
if (serializationFormatter is YamlSerializationFormatter yamlSerializationFormatter)
{
this.parentDataStore = yamlSerializationFormatter.ParentDataStore;
}
this.itemDataLazy = new Lazy <IItemData>(this.LoadItemData, LazyThreadSafetyMode.ExecutionAndPublication);
}
/// <summary>
/// Gets the item data.
/// </summary>
/// <returns></returns>
protected virtual IItemData GetItemData()
{
if (_itemData != null)
return _itemData;
using (var stream = File.OpenRead(ItemPath))
{
_itemData = _formatter.ReadSerializedItem(stream, ItemPath);
}
_itemMetaData = _itemData;
return _itemData;
}
private IItemMetadata GetItemMetaData()
{
if (_itemMetaData != null)
return _itemMetaData;
using (var stream = File.OpenRead(ItemPath))
{
return _itemMetaData = _formatter.ReadSerializedItemMetadata(stream, ItemPath);
}
}
protected virtual void RemoveItemFromCaches(IItemMetadata metadata, string databaseName)
{
if (databaseName != Database.Name) return;
// this is a bit heavy handed, sure.
// but the caches get interdependent stuff - like caching child IDs
// that make it difficult to cleanly remove a single item ID from all cases in the cache
// either way, this should be a relatively rare occurrence (from runtime changes on disk)
// and we're preserving prefetch, etc. Seems pretty zippy overall.
Database.Caches.DataCache.Clear();
Database.Caches.ItemCache.Clear();
Database.Caches.ItemPathsCache.Clear();
Database.Caches.PathCache.Clear();
if (metadata == null) return;
if(metadata.TemplateId == TemplateIDs.Template.Guid || metadata.TemplateId == TemplateIDs.TemplateField.Guid || metadata.Path.EndsWith("__Standard Values", StringComparison.OrdinalIgnoreCase))
Database.Engines.TemplateEngine.Reset();
if (_syncConfiguration.UpdateLinkDatabase || _syncConfiguration.UpdateSearchIndex)
{
var item = GetItemFromId(new ID(metadata.Id), true);
if (item == null) return;
if (_syncConfiguration.UpdateLinkDatabase)
{
Globals.LinkDatabase.UpdateReferences(item);
}
if (_syncConfiguration.UpdateSearchIndex)
{
foreach (var index in ContentSearchManager.Indexes)
{
IndexCustodian.UpdateItem(index, new SitecoreItemUniqueId(item.Uri));
}
}
}
}
public virtual IEnumerable <IItemMetadata> GetChildrenMetadata(IItemMetadata parentItem)
{
Assert.ArgumentNotNull(parentItem, "parentItem");
return(GetChildPaths(parentItem).Select(ReadItemMetadata));
}
protected virtual void RemoveItemFromCaches(IItemMetadata metadata, string databaseName)
{
if (databaseName != Database.Name) return;
// this is a bit heavy handed, sure.
// but the caches get interdependent stuff - like caching child IDs
// that make it difficult to cleanly remove a single item ID from all cases in the cache
// either way, this should be a relatively rare occurrence (from runtime changes on disk)
// and we're preserving prefetch, etc. Seems pretty zippy overall.
Database.Caches.DataCache.Clear();
Database.Caches.ItemCache.Clear();
Database.Caches.ItemPathsCache.Clear();
Database.Caches.PathCache.Clear();
}
/*
FINDING CHILD PATHS, GIVEN A VIRTUAL PATH AND ID
1. Determine the physical path of the item to get children for (use 'finding file paths, given a virtual path'), compare matches against ID
2. Determine if Path.GetFileNameWithoutExtension(filename) exists as a directory in the physical path
3. If it does, get all children of that directory that are both files and have the expected serialized item extension from the formatter
4. Read the parent item file and get the item ID. Check in the SFS root for a folder named that ID (a loopback with children of the item whose names were too long to fit)
- If the folder exists, add all children of that directory that are both files and have the expected serialized item extension from the formatter
5. Note: unlike searching by path, this guarantees ONLY children of the correct item if multiple same named items are present
*/
protected virtual string[] GetChildPaths(IItemMetadata item)
{
Assert.ArgumentNotNull(item, "item");
IItemMetadata serializedItem = GetItemForGlobalPath(item.Path, item.Id);
if (serializedItem == null)
throw new InvalidOperationException("Item {0} does not exist on disk.".FormatWith(item.Path));
IEnumerable<FileData> children = Enumerable.Empty<FileData>();
var childrenPath = Path.ChangeExtension(serializedItem.SerializedItemId, null);
if (Directory.Exists(childrenPath))
{
children = FastDirectoryEnumerator.GetFiles(childrenPath, "*" + _formatter.FileExtension, SearchOption.TopDirectoryOnly);
}
var shortPath = Path.Combine(PhysicalRootPath, item.Id.ToString());
if (Directory.Exists(shortPath))
children = children.Concat(FastDirectoryEnumerator.GetFiles(shortPath, "*" + _formatter.FileExtension, SearchOption.TopDirectoryOnly));
return children.Select(result => result.Path).ToArray();
}
public virtual IEnumerable<IItemMetadata> GetChildrenMetadata(IItemMetadata parentItem)
{
Assert.ArgumentNotNull(parentItem, "parentItem");
return GetChildPaths(parentItem).Select(ReadItemMetadata);
}
public virtual IList<IItemMetadata> GetDescendants(IItemMetadata root, bool ignoreReadErrors)
{
Assert.ArgumentNotNull(root, "root");
var descendants = new List<IItemMetadata>();
var childQueue = new Queue<IItemMetadata>();
childQueue.Enqueue(root);
while (childQueue.Count > 0)
{
var parent = childQueue.Dequeue();
// add current item to descendant results
if (parent.Id != root.Id)
{
descendants.Add(parent);
}
var children = GetChildPaths(parent);
foreach (var physicalPath in children)
{
try
{
var child = ReadItemMetadata(physicalPath);
if (child != null) childQueue.Enqueue(child);
}
catch (Exception)
{
if (ignoreReadErrors) continue;
throw;
}
}
}
return descendants;
}
/*
REMOVING ITEMS, GIVEN A VIRTUAL PATH AND ID
1. Determine the physical path of the item to get children for (use 'finding file paths, given a virtual path'), compare matches against ID
2. Use 'finding child paths, given a physical path' recursively to find all descendant items including through loopbacks
3. Starting at the deepest paths, begin deleting item files and - if present - children subfolders, until all are gone
*/
public virtual bool Remove(IItemMetadata item)
{
Assert.ArgumentNotNull(item, "item");
using (new SfsDuplicateIdCheckingDisabler())
{
IItemMetadata itemToRemove = GetItemForGlobalPath(item.Path, item.Id);
if (itemToRemove == null) return false;
var descendants = GetDescendants(item, true)
.Concat(new[] { itemToRemove })
.OrderByDescending(desc => desc.Path)
.ToArray();
foreach (var descendant in descendants)
{
RemoveWithoutChildren(descendant);
}
}
return true;
}
protected virtual void AddToMetadataCache(IItemMetadata metadata, string path = null)
{
var cachedValue = new WrittenItemMetadata(metadata.Id, metadata.ParentId, metadata.TemplateId, metadata.Path, path ?? metadata.SerializedItemId);
_idCache[metadata.Id] = cachedValue;
_metadataCache.AddOrUpdate(cachedValue.SerializedItemId, cachedValue);
}
/// <summary>
/// Removes an item but does not process any descendant items. You probably almost never want to use this, in favor of Remove() instead.
/// This method is here for when a specific item needs to be removed, without messing with children. This occurs for example when
/// you move an item which has loopback pathed children, who may preserve the same source and destination location.
/// </summary>
public virtual void RemoveWithoutChildren(IItemMetadata descendant)
{
lock (FileUtil.GetFileLock(descendant.SerializedItemId))
{
BeforeFilesystemDelete(descendant.SerializedItemId);
try
{
ActionRetryer.Perform(() =>
{
lock (FileUtil.GetFileLock(descendant.SerializedItemId))
{
_treeWatcher.PushKnownUpdate(descendant.SerializedItemId);
File.Delete(descendant.SerializedItemId);
}
});
}
catch (Exception exception)
{
throw new SfsDeleteException("Error deleting SFS item " + descendant.SerializedItemId, exception);
}
AfterFilesystemDelete(descendant.SerializedItemId);
var childrenDirectory = Path.ChangeExtension(descendant.SerializedItemId, null);
if (Directory.Exists(childrenDirectory))
{
BeforeFilesystemDelete(childrenDirectory);
try
{
ActionRetryer.Perform(() =>
{
_treeWatcher.PushKnownUpdate(childrenDirectory);
Directory.Delete(childrenDirectory, true);
});
}
catch (Exception exception)
{
throw new SfsDeleteException("Error deleting SFS directory " + childrenDirectory, exception);
}
AfterFilesystemDelete(childrenDirectory);
}
var shortChildrenDirectory = new DirectoryInfo(Path.Combine(_physicalRootPath, descendant.Id.ToString()));
if (shortChildrenDirectory.Exists && !shortChildrenDirectory.EnumerateFiles().Any())
{
BeforeFilesystemDelete(shortChildrenDirectory.FullName);
try
{
ActionRetryer.Perform(() =>
{
_treeWatcher.PushKnownUpdate(shortChildrenDirectory.FullName);
Directory.Delete(shortChildrenDirectory.FullName);
});
}
catch (Exception exception)
{
throw new SfsDeleteException("Error deleting SFS directory " + shortChildrenDirectory, exception);
}
AfterFilesystemDelete(shortChildrenDirectory.FullName);
}
}
}
protected virtual IItemMetadata GetParentSerializedItem(IItemMetadata item)
{
Assert.ArgumentNotNull(item, "item");
var localPath = ConvertGlobalVirtualPathToTreeVirtualPath(item.Path);
// Start by using "finding file paths, given a virtual path" on the parent path of the item
var parentVirtualPath = localPath.Substring(0, localPath.LastIndexOf('/'));
if (parentVirtualPath == string.Empty) return null;
var parentPhysicalPaths = GetPhysicalFilePathsForVirtualPath(parentVirtualPath);
if (parentPhysicalPaths.Length == 0) return null;
// If multiple parent path matches exist, narrow them by the parent ID of the item - if no matches, throw
if (parentPhysicalPaths.Length > 1)
{
// find the expected parent's physical path
var parentItem = parentPhysicalPaths.Select(ReadItemMetadata).FirstOrDefault(parentCandiate => parentCandiate.Id == item.ParentId);
if (parentItem == null) return null;
return parentItem;
}
return ReadItemMetadata(parentPhysicalPaths[0]);
}
/// <summary>
/// Relies on TFS PendEdit to initiate the removal on the file system
/// </summary>
/// <param name="item">Item to be removed</param>
/// <returns></returns>
public override bool Remove(IItemData item)
{
Assert.ArgumentNotNull(item, "item");
using (new SfsDuplicateIdCheckingDisabler())
{
IItemMetadata itemToRemove = GetItemForGlobalPath(item.Path, item.Id);
if (itemToRemove == null)
{
return(false);
}
var descendants = GetDescendants(item, true).Concat(new[] { itemToRemove }).OrderByDescending(desc => desc.Path).ToArray();
foreach (var descendant in descendants)
{
lock (FileUtil.GetFileLock(descendant.SerializedItemId))
{
BeforeFilesystemDelete(descendant.SerializedItemId);
if (!_sourceControlManager.FileExistsInSourceControl(descendant.SerializedItemId))
{
try
{
File.Delete(descendant.SerializedItemId);
}
catch (Exception exception)
{
throw new SfsDeleteException("Error deleting SFS item " + descendant.SerializedItemId, exception);
}
}
AfterFilesystemDelete(descendant.SerializedItemId);
var childrenDirectory = Path.ChangeExtension(descendant.SerializedItemId, null);
if (Directory.Exists(childrenDirectory))
{
BeforeFilesystemDelete(childrenDirectory);
if (!_sourceControlManager.FileExistsInSourceControl(childrenDirectory))
{
try
{
Directory.Delete(childrenDirectory, true);
}
catch (Exception exception)
{
throw new SfsDeleteException("Error deleting SFS directory " + childrenDirectory, exception);
}
}
AfterFilesystemDelete(childrenDirectory);
}
var shortChildrenDirectory = Path.Combine(PhysicalRootPath, descendant.Id.ToString());
if (Directory.Exists(shortChildrenDirectory))
{
BeforeFilesystemDelete(shortChildrenDirectory);
if (!_sourceControlManager.FileExistsInSourceControl(childrenDirectory))
{
try
{
Directory.Delete(shortChildrenDirectory);
}
catch (Exception exception)
{
throw new SfsDeleteException("Error deleting SFS directory " + shortChildrenDirectory, exception);
}
}
AfterFilesystemDelete(shortChildrenDirectory);
}
}
}
}
return(true);
}
