public virtual void LoadAll(IItemData[] rootItemsData, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker, Action<IItemData> rootLoadedCallback = null)
{
Assert.ArgumentNotNull(rootItemsData, "rootItems");
Assert.IsTrue(rootItemsData.Length > 0, "No root items were passed!");
CacheManager.ClearAllCaches(); // BOOM! This clears all caches before we begin;
// because for a TpSync configuration we could have TpSync items in the data cache which 'taint' the item comparisons and result in missed updates
bool disableNewSerialization = UnicornDataProvider.DisableSerialization;
try
{
UnicornDataProvider.DisableSerialization = true;
using (new EventDisabler())
{
foreach (var rootItem in rootItemsData)
{
LoadTree(rootItem, retryer, consistencyChecker);
if (rootLoadedCallback != null) rootLoadedCallback(rootItem);
}
retryer.RetryAll(SourceDataStore, item => DoLoadItem(item, null), item => LoadTreeInternal(item, retryer, null));
}
}
finally
{
UnicornDataProvider.DisableSerialization = disableNewSerialization;
}
}
/// <summary>
/// Loads all items in the configured predicate
/// </summary>
public virtual void LoadAll(IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(retryer, "retryer");
var roots = PredicateRootPathResolver.GetRootSerializedItems();
LoadAll(roots, retryer, consistencyChecker);
}
public TaskService(DbManager dbManager, ILogger <TaskService> logger, IConsistencyChecker consistencyChecker, IJalonService jalonService)
{
_DbManager = dbManager;
__logger = logger;
_ConsistencyChecker = consistencyChecker;
_JalonService = jalonService;
}
/// <summary>
/// Loads all items in the configured predicate
/// </summary>
public virtual void LoadAll(IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(retryer, "retryer");
var roots = PredicateRootPathResolver.GetRootSerializedItems();
LoadAll(roots, retryer, consistencyChecker);
}
/// <summary>
/// Loads a specific item from disk
/// </summary>
protected virtual ItemLoadResult DoLoadItem(ISerializedItem serializedItem, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(serializedItem, "serializedItem");
if (consistencyChecker != null)
{
if (!consistencyChecker.IsConsistent(serializedItem))
{
throw new ConsistencyException("Consistency check failed - aborting loading.");
}
consistencyChecker.AddProcessedItem(serializedItem);
}
bool disableNewSerialization = UnicornDataProvider.DisableSerialization;
try
{
UnicornDataProvider.DisableSerialization = true;
_itemsProcessed++;
var included = Predicate.Includes(serializedItem);
if (!included.IsIncluded)
{
Logger.SkippedItemPresentInSerializationProvider(serializedItem, Predicate.GetType().Name, SerializationProvider.GetType().Name, included.Justification ?? string.Empty);
return(new ItemLoadResult(ItemLoadStatus.Skipped));
}
// detect if we should run an update for the item or if it's already up to date
var existingItem = SourceDataProvider.GetItemById(serializedItem.DatabaseName, serializedItem.Id);
ISourceItem updatedItem;
// note that the evaluator is responsible for actual action being taken here
// as well as logging what it does
if (existingItem == null)
{
updatedItem = Evaluator.EvaluateNewSerializedItem(serializedItem);
}
else
{
updatedItem = Evaluator.EvaluateUpdate(serializedItem, existingItem);
}
return(new ItemLoadResult(ItemLoadStatus.Success, updatedItem ?? existingItem));
}
finally
{
UnicornDataProvider.DisableSerialization = disableNewSerialization;
}
}
public virtual void LoadAll(IItemData[] rootItemsData, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker, Action<IItemData> rootLoadedCallback = null)
{
Assert.ArgumentNotNull(rootItemsData, "rootItems");
Assert.IsTrue(rootItemsData.Length > 0, "No root items were passed!");
using (new EventDisabler())
{
foreach (var rootItem in rootItemsData)
{
LoadTree(rootItem, retryer, consistencyChecker);
if (rootLoadedCallback != null) rootLoadedCallback(rootItem);
}
}
retryer.RetryAll(SourceDataStore, item => DoLoadItem(item, null), item => LoadTreeRecursive(item, retryer, null));
}
public MusicFileCop(IFileSystemLoader fileSystemLoader, IConfigurationLoader configurationLoader,
IMetadataLoader metadataLoader, IConsistencyChecker consistencyChecker,
IDefaultConfigurationNode defaultConfiguration, IConfigurationWriter configWriter, ITextOutputWriter outputWriter, IRuleSet ruleSet)
{
if (fileSystemLoader == null)
{
throw new ArgumentNullException(nameof(fileSystemLoader));
}
if (configurationLoader == null)
{
throw new ArgumentNullException(nameof(configurationLoader));
}
if (metadataLoader == null)
{
throw new ArgumentNullException(nameof(metadataLoader));
}
if (consistencyChecker == null)
{
throw new ArgumentNullException(nameof(consistencyChecker));
}
if (defaultConfiguration == null)
{
throw new ArgumentNullException(nameof(defaultConfiguration));
}
if (configWriter == null)
{
throw new ArgumentNullException(nameof(configWriter));
}
if (outputWriter == null)
{
throw new ArgumentNullException(nameof(outputWriter));
}
if (ruleSet == null)
{
throw new ArgumentNullException(nameof(ruleSet));
}
this.m_FileSystemLoader = fileSystemLoader;
this.m_ConfigLoader = configurationLoader;
this.m_MetadataLoader = metadataLoader;
this.m_ConsistencyChecker = consistencyChecker;
this.m_DefaultConfiguration = defaultConfiguration;
m_ConfigWriter = configWriter;
m_OutputWriter = outputWriter;
m_RuleSet = ruleSet;
}
public virtual void LoadAll(ISerializedItem[] rootItems, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker, Action<ISerializedItem> rootLoadedCallback = null)
{
Assert.ArgumentNotNull(rootItems, "rootItems");
Assert.IsTrue(rootItems.Length > 0, "No root items were passed!");
using (new EventDisabler())
{
foreach (var rootItem in rootItems)
{
LoadTree(rootItem, retryer, consistencyChecker);
if (rootLoadedCallback != null) rootLoadedCallback(rootItem);
}
}
retryer.RetryAll(SourceDataProvider, item => DoLoadItem(item, null), item => LoadTreeRecursive(item, retryer, null));
SourceDataProvider.DeserializationComplete(rootItems[0].DatabaseName);
}
/// <summary>
/// Loads a specific item from disk
/// </summary>
protected virtual ItemLoadResult DoLoadItem(IItemData serializedItemData, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(serializedItemData, "serializedItem");
if (consistencyChecker != null)
{
if (!consistencyChecker.IsConsistent(serializedItemData))
{
throw new ConsistencyException("Consistency check failed - aborting loading.");
}
consistencyChecker.AddProcessedItem(serializedItemData);
}
_itemsProcessed++;
var included = Predicate.Includes(serializedItemData);
if (!included.IsIncluded)
{
if (!ReactorContext.IsActive)
{
// we skip this when Dilithium is active because it's entirely probable that another config, containing ignored children, may also be in the cache - so we cannot guarantee this log message being accurate.
Logger.SkippedItemPresentInSerializationProvider(serializedItemData, Predicate.FriendlyName, TargetDataStore.FriendlyName, included.Justification ?? string.Empty);
}
return(new ItemLoadResult(ItemLoadStatus.Skipped));
}
// detect if we should run an update for the item or if it's already up to date
var existingItem = SourceDataStore.GetByPathAndId(serializedItemData.Path, serializedItemData.Id, serializedItemData.DatabaseName);
// note that the evaluator is responsible for actual action being taken here
// as well as logging what it does
if (existingItem == null)
{
existingItem = Evaluator.EvaluateNewSerializedItem(serializedItemData);
}
else
{
Evaluator.EvaluateUpdate(existingItem, serializedItemData);
}
return(new ItemLoadResult(ItemLoadStatus.Success, existingItem));
}
/// <summary>
/// Loads a specific item from disk
/// </summary>
protected virtual ItemLoadResult DoLoadItem(IItemData serializedItemData, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(serializedItemData, "serializedItem");
if (consistencyChecker != null)
{
if (!consistencyChecker.IsConsistent(serializedItemData))
{
throw new ConsistencyException("Consistency check failed - aborting loading.");
}
consistencyChecker.AddProcessedItem(serializedItemData);
}
_itemsProcessed++;
var included = Predicate.Includes(serializedItemData);
if (!included.IsIncluded)
{
Logger.SkippedItemPresentInSerializationProvider(serializedItemData, Predicate.FriendlyName, TargetDataStore.FriendlyName, included.Justification ?? string.Empty);
return(new ItemLoadResult(ItemLoadStatus.Skipped));
}
// detect if we should run an update for the item or if it's already up to date
var existingItem = SourceDataStore.GetByPathAndId(serializedItemData.Path, serializedItemData.Id, serializedItemData.DatabaseName);
// note that the evaluator is responsible for actual action being taken here
// as well as logging what it does
if (existingItem == null)
{
existingItem = Evaluator.EvaluateNewSerializedItem(serializedItemData);
}
else
{
Evaluator.EvaluateUpdate(existingItem, serializedItemData);
}
return(new ItemLoadResult(ItemLoadStatus.Success, existingItem));
}
/// <summary>
/// Loads a set of children from a serialized path
/// </summary>
protected virtual void LoadOneLevel(IItemData rootSerializedItemData, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(rootSerializedItemData, "root");
Assert.ArgumentNotNull(retryer, "retryer");
var orphanCandidates = new Dictionary <Guid, IItemData>();
// get the corresponding item from Sitecore
IItemData rootSourceItemData = SourceDataStore.GetByPathAndId(rootSerializedItemData.Path, rootSerializedItemData.Id, rootSerializedItemData.DatabaseName);
// we add all of the root item's direct children to the "maybe orphan" list (we'll remove them as we find matching serialized children)
if (rootSourceItemData != null)
{
var rootSourceChildren = SourceDataStore.GetChildren(rootSourceItemData);
foreach (IItemData child in rootSourceChildren)
{
// if the preset includes the child add it to the orphan-candidate list (if we don't deserialize it below, it will be marked orphan)
var included = Predicate.Includes(child);
if (included.IsIncluded)
{
orphanCandidates[child.Id] = child;
}
else
{
Logger.SkippedItem(child, Predicate.FriendlyName, included.Justification ?? string.Empty);
}
}
}
// check for direct children of the target path
var serializedChildren = TargetDataStore.GetChildren(rootSerializedItemData);
foreach (var serializedChild in serializedChildren)
{
try
{
if (serializedChild.IsStandardValuesItem())
{
orphanCandidates.Remove(serializedChild.Id); // avoid marking standard values items orphans
retryer.AddItemRetry(serializedChild, new StandardValuesException(serializedChild.Path));
}
else
{
// load a child item
var loadedSourceItem = DoLoadItem(serializedChild, consistencyChecker);
if (loadedSourceItem.ItemData != null)
{
orphanCandidates.Remove(loadedSourceItem.ItemData.Id);
// check if we have any child serialized items under this loaded child item (existing children) -
// if we do not, we can orphan any included children of the loaded item as well
var loadedItemSerializedChildren = TargetDataStore.GetChildren(serializedChild);
if (!loadedItemSerializedChildren.Any()) // no children were serialized on disk
{
var loadedSourceChildren = SourceDataStore.GetChildren(loadedSourceItem.ItemData);
foreach (IItemData loadedSourceChild in loadedSourceChildren)
{
// place any included source children on the orphan list for deletion, as no serialized children existed
if (Predicate.Includes(loadedSourceChild).IsIncluded)
{
orphanCandidates.Add(loadedSourceChild.Id, loadedSourceChild);
}
}
}
}
else if (loadedSourceItem.Status == ItemLoadStatus.Skipped) // if the item got skipped we'll prevent it from being deleted
{
orphanCandidates.Remove(serializedChild.Id);
}
}
}
catch (ConsistencyException)
{
throw;
}
catch (Exception ex)
{
// if a problem occurs we attempt to retry later
retryer.AddItemRetry(serializedChild, ex);
// don't treat errors as cause to delete an item
orphanCandidates.Remove(serializedChild.Id);
}
}
// if we're forcing an update (ie deleting stuff not on disk) we send the items that we found that weren't on disk off to get evaluated as orphans
if (orphanCandidates.Count > 0)
{
bool disableNewSerialization = UnicornDataProvider.DisableSerialization;
try
{
UnicornDataProvider.DisableSerialization = true;
Evaluator.EvaluateOrphans(orphanCandidates.Values.ToArray());
}
finally
{
UnicornDataProvider.DisableSerialization = disableNewSerialization;
}
}
}
/// <summary>
/// Recursive method that loads a given tree and retries failures already present if any
/// </summary>
protected virtual void LoadTreeRecursive(IItemData root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(root, "root");
Assert.ArgumentNotNull(retryer, "retryer");
var included = Predicate.Includes(root);
if (!included.IsIncluded)
{
Logger.SkippedItemPresentInSerializationProvider(root, Predicate.FriendlyName, TargetDataStore.GetType().Name, included.Justification ?? string.Empty);
return;
}
try
{
// load the current level
LoadOneLevel(root, retryer, consistencyChecker);
// check if we have child paths to recurse down
var children = TargetDataStore.GetChildren(root).ToArray();
if (children.Length > 0)
{
// make sure if a "templates" item exists in the current set, it goes first
if (children.Length > 1)
{
int templateIndex = Array.FindIndex(children, x => x.Path.EndsWith("templates", StringComparison.OrdinalIgnoreCase));
if (templateIndex > 0)
{
var zero = children[0];
children[0] = children[templateIndex];
children[templateIndex] = zero;
}
}
// load each child path recursively
foreach (var child in children)
{
LoadTreeRecursive(child, retryer, consistencyChecker);
}
// pull out any standard values failures for immediate retrying
retryer.RetryStandardValuesFailures(item => DoLoadItem(item, null));
} // children.length > 0
}
catch (ConsistencyException)
{
throw;
}
catch (Exception ex)
{
retryer.AddTreeRetry(root, ex);
}
}
public virtual void LoadAll(IItemData[] rootItemsData, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker, Action <IItemData> rootLoadedCallback = null)
{
Assert.ArgumentNotNull(rootItemsData, "rootItems");
Assert.IsTrue(rootItemsData.Length > 0, "No root items were passed!");
if (DataProviderConfiguration.EnableTransparentSync)
{
CacheManager.ClearAllCaches();
// BOOM! This clears all caches before we begin;
// because for a TpSync configuration we could have TpSync items in the data cache which 'taint' the item comparisons and result in missed updates
}
bool disableNewSerialization = UnicornDataProvider.DisableSerialization;
try
{
UnicornDataProvider.DisableSerialization = true;
using (new EventDisabler())
{
foreach (var rootItem in rootItemsData)
{
LoadTree(rootItem, retryer, consistencyChecker);
rootLoadedCallback?.Invoke(rootItem);
}
retryer.RetryAll(SourceDataStore, item => DoLoadItem(item, null), item => LoadTreeInternal(item, retryer, null));
}
}
finally
{
UnicornDataProvider.DisableSerialization = disableNewSerialization;
}
}
public JalonService(DbManager dbManager, ILogger <ProjectService> logger, IConsistencyChecker consistencyChecker)
{
_DbManager = dbManager;
__logger = logger;
_ConsistencyChecker = consistencyChecker;
}
/// <summary>
/// Recursive method that loads a given tree and retries failures already present if any
/// </summary>
protected virtual void LoadTreeInternal(IItemData root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(root, "root");
Assert.ArgumentNotNull(retryer, "retryer");
var included = Predicate.Includes(root);
if (!included.IsIncluded)
{
Logger.SkippedItemPresentInSerializationProvider(root, Predicate.FriendlyName, TargetDataStore.GetType().Name, included.Justification ?? string.Empty);
return;
}
// we throw items into this queue, and let a thread pool pick up anything available to process in parallel. only the children of queued items are processed, not the item itself
ConcurrentQueue <IItemData> processQueue = new ConcurrentQueue <IItemData>();
// exceptions thrown on background threads are left in here
ConcurrentQueue <Exception> errors = new ConcurrentQueue <Exception>();
// we keep track of how many threads are actively processing something so we know when to end the threads
// (e.g. a thread could have nothing in the queue right now, but that's because a different thread is about
// to add 8 things to the queue - so it shouldn't quit till all is done)
int activeThreads = 0;
// put the root in the queue
processQueue.Enqueue(root);
Thread[] pool = Enumerable.Range(0, ThreadCount).Select(i => new Thread(() =>
{
Process:
Interlocked.Increment(ref activeThreads);
using (new UnicornOperationContext()) // disablers only work on the current thread. So we need to disable on all worker threads
{
IItemData parentItem;
while (processQueue.TryDequeue(out parentItem) && errors.Count == 0)
{
try
{
// load the current level
LoadOneLevel(parentItem, retryer, consistencyChecker);
// check if we have child paths to process down
var children = TargetDataStore.GetChildren(parentItem).ToArray();
if (children.Length > 0)
{
// load each child path
foreach (var child in children)
{
processQueue.Enqueue(child);
}
} // children.length > 0
}
catch (ConsistencyException cex)
{
errors.Enqueue(cex);
break;
}
catch (Exception ex)
{
retryer.AddTreeRetry(root, ex);
}
} // end while
}
// if we get here, the queue was empty. let's make ourselves inactive.
Interlocked.Decrement(ref activeThreads);
// if some other thread in our pool was doing stuff, sleep for a sec to see if we can pick up their work
if (activeThreads > 0)
{
Thread.Sleep(10);
goto Process; // OH MY GOD :)
}
})).ToArray();
// start the thread pool
foreach (var thread in pool)
{
thread.Start();
}
// ...and then wait for all the threads to finish
foreach (var thread in pool)
{
thread.Join();
}
if (errors.Count > 0)
{
throw new AggregateException(errors);
}
}
/// <summary>
/// Loads a preset from serialized items on disk.
/// </summary>
public virtual void LoadTree(ISerializedItem rootItem, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(rootItem, "rootItem");
Assert.ArgumentNotNull(retryer, "retryer");
Assert.ArgumentNotNull(consistencyChecker, "consistencyChecker");
_itemsProcessed = 0;
var timer = new Stopwatch();
timer.Start();
Logger.BeginLoadingTree(rootItem);
using (new EventDisabler())
{
// load the root item (LoadTreeRecursive only evaluates children)
DoLoadItem(rootItem, consistencyChecker);
// load children of the root
LoadTreeRecursive(rootItem, retryer, consistencyChecker);
retryer.RetryAll(SourceDataProvider, item => DoLoadItem(item, null), item => LoadTreeRecursive(item, retryer, null));
}
timer.Stop();
SourceDataProvider.DeserializationComplete(rootItem.DatabaseName);
Logger.EndLoadingTree(rootItem, _itemsProcessed, timer.ElapsedMilliseconds);
}
/// <summary>
/// Loads a tree from serialized items on disk.
/// </summary>
protected internal virtual void LoadTree(IItemData rootItemData, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(rootItemData, "rootItem");
Assert.ArgumentNotNull(retryer, "retryer");
Assert.ArgumentNotNull(consistencyChecker, "consistencyChecker");
_itemsProcessed = 0;
var timer = new Stopwatch();
timer.Start();
Logger.BeginLoadingTree(rootItemData);
// load the root item (LoadTreeRecursive only evaluates children)
DoLoadItem(rootItemData, consistencyChecker);
// load children of the root
LoadTreeRecursive(rootItemData, retryer, consistencyChecker);
Logger.EndLoadingTree(rootItemData, _itemsProcessed, timer.ElapsedMilliseconds);
timer.Stop();
}
/// <summary>
/// Recursive method that loads a given tree and retries failures already present if any
/// </summary>
protected virtual void LoadTreeInternal(IItemData root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(root, "root");
Assert.ArgumentNotNull(retryer, "retryer");
var included = Predicate.Includes(root);
if (!included.IsIncluded)
{
Logger.SkippedItemPresentInSerializationProvider(root, Predicate.FriendlyName, TargetDataStore.GetType().Name, included.Justification ?? string.Empty);
return;
}
// we throw items into this queue, and let a thread pool pick up anything available to process in parallel. only the children of queued items are processed, not the item itself
ConcurrentQueue<IItemData> processQueue = new ConcurrentQueue<IItemData>();
// exceptions thrown on background threads are left in here
ConcurrentQueue<Exception> errors = new ConcurrentQueue<Exception>();
// we keep track of how many threads are actively processing something so we know when to end the threads
// (e.g. a thread could have nothing in the queue right now, but that's because a different thread is about
// to add 8 things to the queue - so it shouldn't quit till all is done)
int activeThreads = 0;
// put the root in the queue
processQueue.Enqueue(root);
if(SyncConfiguration.MaxConcurrency < 1) throw new InvalidOperationException("Max concurrency is set to zero. Please set it to one or more threads.");
Thread[] pool = Enumerable.Range(0, SyncConfiguration.MaxConcurrency).Select(i => new Thread(() =>
{
Process:
Interlocked.Increment(ref activeThreads);
using (new UnicornOperationContext()) // disablers only work on the current thread. So we need to disable on all worker threads
{
IItemData parentItem;
while (processQueue.TryDequeue(out parentItem) && errors.Count == 0)
{
try
{
// load the current level
LoadOneLevel(parentItem, retryer, consistencyChecker);
// check if we have child paths to process down
var children = TargetDataStore.GetChildren(parentItem).ToArray();
if (children.Length > 0)
{
// load each child path
foreach (var child in children)
{
processQueue.Enqueue(child);
}
} // children.length > 0
}
catch (ConsistencyException cex)
{
errors.Enqueue(cex);
break;
}
catch (Exception ex)
{
retryer.AddTreeRetry(root, ex);
}
} // end while
}
// if we get here, the queue was empty. let's make ourselves inactive.
Interlocked.Decrement(ref activeThreads);
// if some other thread in our pool was doing stuff, sleep for a sec to see if we can pick up their work
if (activeThreads > 0)
{
Thread.Sleep(10);
goto Process; // OH MY GOD :)
}
})).ToArray();
// start the thread pool
foreach (var thread in pool) thread.Start();
// ...and then wait for all the threads to finish
foreach (var thread in pool) thread.Join();
if (errors.Count > 0) throw new AggregateException(errors);
}
/// <summary>
/// Loads a set of children from a serialized path
/// </summary>
protected virtual void LoadOneLevel(IItemData rootSerializedItemData, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(rootSerializedItemData, "root");
Assert.ArgumentNotNull(retryer, "retryer");
var orphanCandidates = new Dictionary<Guid, IItemData>();
// get the corresponding item from Sitecore
IItemData rootSourceItemData = SourceDataStore.GetByPathAndId(rootSerializedItemData.Path, rootSerializedItemData.Id, rootSerializedItemData.DatabaseName);
// we add all of the root item's direct children to the "maybe orphan" list (we'll remove them as we find matching serialized children)
if (rootSourceItemData != null)
{
var rootSourceChildren = SourceDataStore.GetChildren(rootSourceItemData);
foreach (IItemData child in rootSourceChildren)
{
// if the preset includes the child add it to the orphan-candidate list (if we don't deserialize it below, it will be marked orphan)
var included = Predicate.Includes(child);
if (included.IsIncluded)
orphanCandidates[child.Id] = child;
else
{
Logger.SkippedItem(child, Predicate.FriendlyName, included.Justification ?? string.Empty);
}
}
}
// check for direct children of the target path
var serializedChildren = TargetDataStore.GetChildren(rootSerializedItemData);
foreach (var serializedChild in serializedChildren)
{
try
{
// Because the load order is breadth-first, standard values will be loaded PRIOR TO THEIR TEMPLATE FIELDS
// So if we find a standard values item, we throw it straight onto the retry list, which will make it load last
// after everything else, ensuring its fields all exist first. (This is also how Sitecore serialization does it...)
if (serializedChild.Path.EndsWith("__Standard Values", StringComparison.OrdinalIgnoreCase))
{
retryer.AddItemRetry(serializedChild, new Exception("Pushing standard values item to the end of loading."));
orphanCandidates.Remove(serializedChild.Id);
continue;
}
// load a child item
var loadedSourceItem = DoLoadItem(serializedChild, consistencyChecker);
if (loadedSourceItem.ItemData != null)
{
orphanCandidates.Remove(loadedSourceItem.ItemData.Id);
// check if we have any child serialized items under this loaded child item (existing children) -
// if we do not, we can orphan any included children of the loaded item as well
var loadedItemSerializedChildren = TargetDataStore.GetChildren(serializedChild);
if (!loadedItemSerializedChildren.Any()) // no children were serialized on disk
{
var loadedSourceChildren = SourceDataStore.GetChildren(loadedSourceItem.ItemData);
foreach (IItemData loadedSourceChild in loadedSourceChildren)
{
// place any included source children on the orphan list for deletion, as no serialized children existed
if (Predicate.Includes(loadedSourceChild).IsIncluded)
orphanCandidates.Add(loadedSourceChild.Id, loadedSourceChild);
}
}
}
else if (loadedSourceItem.Status == ItemLoadStatus.Skipped) // if the item got skipped we'll prevent it from being deleted
orphanCandidates.Remove(serializedChild.Id);
}
catch (ConsistencyException)
{
throw;
}
catch (Exception ex)
{
// if a problem occurs we attempt to retry later
retryer.AddItemRetry(serializedChild, ex);
// don't treat errors as cause to delete an item
orphanCandidates.Remove(serializedChild.Id);
}
}
// if we're forcing an update (ie deleting stuff not on disk) we send the items that we found that weren't on disk off to get evaluated as orphans
if (orphanCandidates.Count > 0)
{
Evaluator.EvaluateOrphans(orphanCandidates.Values.ToArray());
}
}
/// <summary>
/// Loads a tree from serialized items on disk.
/// </summary>
protected internal virtual void LoadTree(IItemData rootItemData, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(rootItemData, "rootItem");
Assert.ArgumentNotNull(retryer, "retryer");
Assert.ArgumentNotNull(consistencyChecker, "consistencyChecker");
_itemsProcessed = 0;
var timer = new Stopwatch();
timer.Start();
Logger.BeginLoadingTree(rootItemData);
// LoadTreeInternal does not load the root item passed to it (only the children thereof)
// so we have to seed the load by loading the root item
using (new UnicornOperationContext())
{
try
{
DoLoadItem(rootItemData, consistencyChecker);
}
catch (Exception exception)
{
retryer.AddItemRetry(rootItemData, exception);
}
}
// load children of the root
LoadTreeInternal(rootItemData, retryer, consistencyChecker);
Logger.EndLoadingTree(rootItemData, _itemsProcessed, timer.ElapsedMilliseconds);
timer.Stop();
}
/// <summary>
/// Recursive method that loads a given tree and retries failures already present if any
/// </summary>
protected virtual void LoadTreeInternal(IItemData root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(root, "root");
Assert.ArgumentNotNull(retryer, "retryer");
var included = Predicate.Includes(root);
if (!included.IsIncluded)
{
Logger.SkippedItemPresentInSerializationProvider(root, Predicate.FriendlyName, TargetDataStore.GetType().Name, included.Justification ?? string.Empty);
return;
}
// we throw items into this queue, and let a thread pool pick up anything available to process in parallel. only the children of queued items are processed, not the item itself
ConcurrentQueue<IItemData> processQueue = new ConcurrentQueue<IItemData>();
// we keep track of how many threads are actively processing something so we know when to end the threads
// (e.g. a thread could have nothing in the queue right now, but that's because a different thread is about
// to add 8 things to the queue - so it shouldn't quit till all is done)
int activeThreads = 0;
// put the root in the queue
processQueue.Enqueue(root);
Thread[] pool = Enumerable.Range(0, ThreadCount).Select(i => new Thread(() =>
{
Process:
Interlocked.Increment(ref activeThreads);
IItemData parentItem;
while (processQueue.TryDequeue(out parentItem))
{
try
{
// load the current level
LoadOneLevel(parentItem, retryer, consistencyChecker);
// check if we have child paths to process down
var children = TargetDataStore.GetChildren(parentItem).ToArray();
if (children.Length > 0)
{
// make sure if a "templates" item exists in the current set, it goes first
if (children.Length > 1)
{
int templateIndex = Array.FindIndex(children, x => x.Path.EndsWith("templates", StringComparison.OrdinalIgnoreCase));
if (templateIndex > 0)
{
var zero = children[0];
children[0] = children[templateIndex];
children[templateIndex] = zero;
}
}
// load each child path
foreach (var child in children)
{
processQueue.Enqueue(child);
}
// pull out any standard values failures for immediate retrying
retryer.RetryStandardValuesFailures(item => DoLoadItem(item, null));
} // children.length > 0
}
catch (ConsistencyException)
{
throw;
}
catch (Exception ex)
{
retryer.AddTreeRetry(root, ex);
}
}
// if we get here, the queue was empty. let's make ourselves inactive.
Interlocked.Decrement(ref activeThreads);
// if some other thread in our pool was doing stuff, sleep for a sec to see if we can pick up their work
if (activeThreads > 0)
{
Thread.Sleep(10);
goto Process; // OH MY GOD :)
}
})).ToArray();
// start the thread pool
foreach (var thread in pool) thread.Start();
// ...and then wait for all the threads to finish
foreach (var thread in pool) thread.Join();
}
private void TestLoadTree(SerializationLoader loader, IItemData root, IDeserializeFailureRetryer retryer = null, IConsistencyChecker consistencyChecker = null)
{
if (retryer == null)
{
retryer = Substitute.For <IDeserializeFailureRetryer>();
}
if (consistencyChecker == null)
{
var checker = Substitute.For <IConsistencyChecker>();
checker.IsConsistent(Arg.Any <IItemData>()).Returns(true);
consistencyChecker = checker;
}
loader.LoadTree(root, retryer, consistencyChecker);
}
private void TestLoadTree(SerializationLoader loader, ISerializedItem root, IDeserializeFailureRetryer retryer = null, IConsistencyChecker consistencyChecker = null)
{
if (retryer == null)
{
retryer = new Mock <IDeserializeFailureRetryer>().Object;
}
if (consistencyChecker == null)
{
var checker = new Mock <IConsistencyChecker>();
checker.Setup(x => x.IsConsistent(It.IsAny <ISerializedItem>())).Returns(true);
consistencyChecker = checker.Object;
}
loader.LoadTree(root, retryer, consistencyChecker);
}
public ProjectService(DbManager dbManager, ILogger <ProjectService> logger, IConsistencyChecker consistencyChecker, IJalonService jalonService)
{
_Context = dbManager;
__logger = logger;
_JalonService = jalonService;
}
/// <summary>
/// Loads a tree from serialized items on disk.
/// </summary>
protected internal virtual void LoadTree(IItemData rootItemData, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(rootItemData, "rootItem");
Assert.ArgumentNotNull(retryer, "retryer");
Assert.ArgumentNotNull(consistencyChecker, "consistencyChecker");
_itemsProcessed = 0;
var timer = new Stopwatch();
timer.Start();
Logger.BeginLoadingTree(rootItemData);
// load the root item (LoadTreeRecursive only evaluates children)
DoLoadItem(rootItemData, consistencyChecker);
// load children of the root
LoadTreeRecursive(rootItemData, retryer, consistencyChecker);
Logger.EndLoadingTree(rootItemData, _itemsProcessed, timer.ElapsedMilliseconds);
timer.Stop();
}
public virtual void LoadAll(IItemData[] rootItemsData, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker, Action <IItemData> rootLoadedCallback = null)
{
Assert.ArgumentNotNull(rootItemsData, "rootItems");
Assert.IsTrue(rootItemsData.Length > 0, "No root items were passed!");
using (new EventDisabler())
{
foreach (var rootItem in rootItemsData)
{
LoadTree(rootItem, retryer, consistencyChecker);
if (rootLoadedCallback != null)
{
rootLoadedCallback(rootItem);
}
}
}
retryer.RetryAll(SourceDataStore, item => DoLoadItem(item, null), item => LoadTreeRecursive(item, retryer, null));
}
/// <summary>
/// Loads a set of children from a serialized path
/// </summary>
protected virtual void LoadOneLevel(IItemData rootSerializedItemData, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(rootSerializedItemData, "root");
Assert.ArgumentNotNull(retryer, "retryer");
var orphanCandidates = new Dictionary<Guid, IItemData>();
// get the corresponding item from Sitecore
IItemData rootSourceItemData = SourceDataStore.GetByPathAndId(rootSerializedItemData.Path, rootSerializedItemData.Id, rootSerializedItemData.DatabaseName);
// we add all of the root item's direct children to the "maybe orphan" list (we'll remove them as we find matching serialized children)
if (rootSourceItemData != null)
{
var rootSourceChildren = SourceDataStore.GetChildren(rootSourceItemData);
foreach (IItemData child in rootSourceChildren)
{
// if the preset includes the child add it to the orphan-candidate list (if we don't deserialize it below, it will be marked orphan)
var included = Predicate.Includes(child);
if (included.IsIncluded)
orphanCandidates[child.Id] = child;
else
{
Logger.SkippedItem(child, Predicate.FriendlyName, included.Justification ?? string.Empty);
}
}
}
// check for direct children of the target path
var serializedChildren = TargetDataStore.GetChildren(rootSerializedItemData);
foreach (var serializedChild in serializedChildren)
{
try
{
if (serializedChild.IsStandardValuesItem())
{
orphanCandidates.Remove(serializedChild.Id); // avoid marking standard values items orphans
retryer.AddItemRetry(serializedChild, new StandardValuesException(serializedChild.Path));
}
else
{
// load a child item
var loadedSourceItem = DoLoadItem(serializedChild, consistencyChecker);
if (loadedSourceItem.ItemData != null)
{
orphanCandidates.Remove(loadedSourceItem.ItemData.Id);
// check if we have any child serialized items under this loaded child item (existing children) -
// if we do not, we can orphan any included children of the loaded item as well
var loadedItemSerializedChildren = TargetDataStore.GetChildren(serializedChild);
if (!loadedItemSerializedChildren.Any()) // no children were serialized on disk
{
var loadedSourceChildren = SourceDataStore.GetChildren(loadedSourceItem.ItemData);
foreach (IItemData loadedSourceChild in loadedSourceChildren)
{
// place any included source children on the orphan list for deletion, as no serialized children existed
if(Predicate.Includes(loadedSourceChild).IsIncluded)
orphanCandidates.Add(loadedSourceChild.Id, loadedSourceChild);
}
}
}
else if (loadedSourceItem.Status == ItemLoadStatus.Skipped) // if the item got skipped we'll prevent it from being deleted
orphanCandidates.Remove(serializedChild.Id);
}
}
catch (ConsistencyException)
{
throw;
}
catch (Exception ex)
{
// if a problem occurs we attempt to retry later
retryer.AddItemRetry(serializedChild, ex);
// don't treat errors as cause to delete an item
orphanCandidates.Remove(serializedChild.Id);
}
}
// if we're forcing an update (ie deleting stuff not on disk) we send the items that we found that weren't on disk off to get evaluated as orphans
if (orphanCandidates.Count > 0)
{
bool disableNewSerialization = UnicornDataProvider.DisableSerialization;
try
{
UnicornDataProvider.DisableSerialization = true;
Evaluator.EvaluateOrphans(orphanCandidates.Values.ToArray());
}
finally
{
UnicornDataProvider.DisableSerialization = disableNewSerialization;
}
}
}
private void TestLoadTree(SerializationLoader loader, IItemData root, IDeserializeFailureRetryer retryer = null, IConsistencyChecker consistencyChecker = null)
{
if (retryer == null) retryer = Substitute.For<IDeserializeFailureRetryer>();
if (consistencyChecker == null)
{
var checker = Substitute.For<IConsistencyChecker>();
checker.IsConsistent(Arg.Any<IItemData>()).Returns(true);
consistencyChecker = checker;
}
loader.LoadTree(root, retryer, consistencyChecker);
}
public virtual void LoadAll(ISerializedItem[] rootItems, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker, Action <ISerializedItem> rootLoadedCallback = null)
{
Assert.ArgumentNotNull(rootItems, "rootItems");
Assert.IsTrue(rootItems.Length > 0, "No root items were passed!");
using (new EventDisabler())
{
foreach (var rootItem in rootItems)
{
LoadTree(rootItem, retryer, consistencyChecker);
if (rootLoadedCallback != null)
{
rootLoadedCallback(rootItem);
}
}
}
retryer.RetryAll(SourceDataProvider, item => DoLoadItem(item, null), item => LoadTreeRecursive(item, retryer, null));
SourceDataProvider.DeserializationComplete(rootItems[0].DatabaseName);
}
/// <summary>
/// Loads a set of children from a serialized path
/// </summary>
protected virtual void LoadOneLevel(ISerializedReference root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(root, "root");
Assert.ArgumentNotNull(retryer, "retryer");
var orphanCandidates = new Dictionary<ID, ISourceItem>();
// grab the root item's full metadata
var rootSerializedItem = root.GetItem();
if (rootSerializedItem == null)
{
Logger.SkippedItemMissingInSerializationProvider(root, SerializationProvider.GetType().Name);
return;
}
// get the corresponding item from Sitecore
ISourceItem rootItem = SourceDataProvider.GetItemById(rootSerializedItem.DatabaseName, rootSerializedItem.Id);
// we add all of the root item's direct children to the "maybe orphan" list (we'll remove them as we find matching serialized children)
if (rootItem != null)
{
var rootChildren = rootItem.Children;
foreach (ISourceItem child in rootChildren)
{
// if the preset includes the child add it to the orphan-candidate list (if we don't deserialize it below, it will be marked orphan)
var included = Predicate.Includes(child);
if (included.IsIncluded)
orphanCandidates[child.Id] = child;
else
{
Logger.SkippedItem(child, Predicate.GetType().Name, included.Justification ?? string.Empty);
}
}
}
// check for direct children of the target path
var children = rootSerializedItem.GetChildItems();
foreach (var child in children)
{
try
{
if (child.IsStandardValuesItem)
{
orphanCandidates.Remove(child.Id); // avoid marking standard values items orphans
retryer.AddItemRetry(child, new StandardValuesException(child.ItemPath));
}
else
{
// load a child item
var loadedItem = DoLoadItem(child, consistencyChecker);
if (loadedItem.Item != null)
{
orphanCandidates.Remove(loadedItem.Item.Id);
// check if we have any child serialized items under this loaded child item (existing children) -
// if we do not, we can orphan any children of the loaded item as well
var loadedItemsChildren = child.GetChildReferences(false);
if (loadedItemsChildren.Length == 0) // no children were serialized on disk
{
var loadedChildren = loadedItem.Item.Children;
foreach (ISourceItem loadedChild in loadedChildren)
{
orphanCandidates.Add(loadedChild.Id, loadedChild);
}
}
}
else if (loadedItem.Status == ItemLoadStatus.Skipped) // if the item got skipped we'll prevent it from being deleted
orphanCandidates.Remove(child.Id);
}
}
catch (ConsistencyException)
{
throw;
}
catch (Exception ex)
{
// if a problem occurs we attempt to retry later
retryer.AddItemRetry(child, ex);
// don't treat errors as cause to delete an item
orphanCandidates.Remove(child.Id);
}
}
// if we're forcing an update (ie deleting stuff not on disk) we send the items that we found that weren't on disk off to get evaluated as orphans
if (orphanCandidates.Count > 0)
{
bool disableNewSerialization = UnicornDataProvider.DisableSerialization;
try
{
UnicornDataProvider.DisableSerialization = true;
Evaluator.EvaluateOrphans(orphanCandidates.Values.ToArray());
}
finally
{
UnicornDataProvider.DisableSerialization = disableNewSerialization;
}
}
}
private void TestLoadTree(SerializationLoader loader, ISerializedItem root, IDeserializeFailureRetryer retryer = null, IConsistencyChecker consistencyChecker = null)
{
if (retryer == null) retryer = new Mock<IDeserializeFailureRetryer>().Object;
if (consistencyChecker == null)
{
var checker = new Mock<IConsistencyChecker>();
checker.Setup(x => x.IsConsistent(It.IsAny<ISerializedItem>())).Returns(true);
consistencyChecker = checker.Object;
}
loader.LoadTree(root, retryer, consistencyChecker);
}
/// <summary>
/// Loads a specific item from disk
/// </summary>
protected virtual ItemLoadResult DoLoadItem(IItemData serializedItemData, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(serializedItemData, "serializedItem");
if (consistencyChecker != null)
{
if (!consistencyChecker.IsConsistent(serializedItemData)) throw new ConsistencyException("Consistency check failed - aborting loading.");
consistencyChecker.AddProcessedItem(serializedItemData);
}
bool disableNewSerialization = UnicornDataProvider.DisableSerialization;
try
{
UnicornDataProvider.DisableSerialization = true;
_itemsProcessed++;
var included = Predicate.Includes(serializedItemData);
if (!included.IsIncluded)
{
Logger.SkippedItemPresentInSerializationProvider(serializedItemData, Predicate.FriendlyName, TargetDataStore.FriendlyName, included.Justification ?? string.Empty);
return new ItemLoadResult(ItemLoadStatus.Skipped);
}
// detect if we should run an update for the item or if it's already up to date
var existingItem = SourceDataStore.GetByPathAndId(serializedItemData.Path, serializedItemData.Id, serializedItemData.DatabaseName);
// note that the evaluator is responsible for actual action being taken here
// as well as logging what it does
if (existingItem == null)
existingItem = Evaluator.EvaluateNewSerializedItem(serializedItemData);
else
Evaluator.EvaluateUpdate(existingItem, serializedItemData);
return new ItemLoadResult(ItemLoadStatus.Success, existingItem);
}
finally
{
UnicornDataProvider.DisableSerialization = disableNewSerialization;
}
}
public ExigenceService(DbManager dbManager, ILogger <ExigenceService> logger, IConsistencyChecker consistencyChecker)
{
_DbManager = dbManager;
__logger = logger;
_ConsistencyChecker = consistencyChecker;
}
/// <summary>
/// Recursive method that loads a given tree and retries failures already present if any
/// </summary>
protected virtual void LoadTreeRecursive(IItemData root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(root, "root");
Assert.ArgumentNotNull(retryer, "retryer");
var included = Predicate.Includes(root);
if (!included.IsIncluded)
{
Logger.SkippedItemPresentInSerializationProvider(root, Predicate.FriendlyName, TargetDataStore.GetType().Name, included.Justification ?? string.Empty);
return;
}
try
{
// load the current level
LoadOneLevel(root, retryer, consistencyChecker);
// check if we have child paths to recurse down
var children = TargetDataStore.GetChildren(root).ToArray();
if (children.Length > 0)
{
// make sure if a "templates" item exists in the current set, it goes first
if (children.Length > 1)
{
int templateIndex = Array.FindIndex(children, x => x.Path.EndsWith("templates", StringComparison.OrdinalIgnoreCase));
if (templateIndex > 0)
{
var zero = children[0];
children[0] = children[templateIndex];
children[templateIndex] = zero;
}
}
// load each child path recursively
foreach (var child in children)
{
LoadTreeRecursive(child, retryer, consistencyChecker);
}
// pull out any standard values failures for immediate retrying
retryer.RetryStandardValuesFailures(item => DoLoadItem(item, null));
} // children.length > 0
}
catch (ConsistencyException)
{
throw;
}
catch (Exception ex)
{
retryer.AddTreeRetry(root, ex);
}
}
/// <summary>
/// Loads a set of children from a serialized path
/// </summary>
protected virtual void LoadOneLevel(IItemData rootSerializedItemData, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(rootSerializedItemData, "root");
Assert.ArgumentNotNull(retryer, "retryer");
var orphanCandidates = new Dictionary <Guid, IItemData>();
// get the corresponding item from Sitecore
IItemData rootSourceItemData = SourceDataStore.GetByPathAndId(rootSerializedItemData.Path, rootSerializedItemData.Id, rootSerializedItemData.DatabaseName);
// we add all of the root item's direct children to the "maybe orphan" list (we'll remove them as we find matching serialized children)
if (rootSourceItemData != null)
{
var rootSourceChildren = SourceDataStore.GetChildren(rootSourceItemData);
foreach (IItemData child in rootSourceChildren)
{
// if the preset includes the child add it to the orphan-candidate list (if we don't deserialize it below, it will be marked orphan)
var included = Predicate.Includes(child);
if (included.IsIncluded)
{
orphanCandidates[child.Id] = child;
}
else
{
Logger.SkippedItem(child, Predicate.FriendlyName, included.Justification ?? string.Empty);
}
}
}
// check for direct children of the target path
var serializedChildren = TargetDataStore.GetChildren(rootSerializedItemData);
foreach (var serializedChild in serializedChildren)
{
try
{
// Because the load order is breadth-first, standard values will be loaded PRIOR TO THEIR TEMPLATE FIELDS
// So if we find a standard values item, we throw it straight onto the retry list, which will make it load last
// after everything else, ensuring its fields all exist first. (This is also how Sitecore serialization does it...)
if (serializedChild.Path.EndsWith("__Standard Values", StringComparison.OrdinalIgnoreCase))
{
retryer.AddItemRetry(serializedChild, new Exception("Pushing standard values item to the end of loading."));
orphanCandidates.Remove(serializedChild.Id);
continue;
}
// load a child item
var loadedSourceItem = DoLoadItem(serializedChild, consistencyChecker);
if (loadedSourceItem.ItemData != null)
{
orphanCandidates.Remove(loadedSourceItem.ItemData.Id);
// check if we have any child serialized items under this loaded child item (existing children) -
// if we do not, we can orphan any included children of the loaded item as well
var loadedItemSerializedChildren = TargetDataStore.GetChildren(serializedChild);
if (!loadedItemSerializedChildren.Any()) // no children were serialized on disk
{
var loadedSourceChildren = SourceDataStore.GetChildren(loadedSourceItem.ItemData);
foreach (IItemData loadedSourceChild in loadedSourceChildren)
{
// place any included source children on the orphan list for deletion, as no serialized children existed
if (Predicate.Includes(loadedSourceChild).IsIncluded)
{
orphanCandidates.Add(loadedSourceChild.Id, loadedSourceChild);
}
}
}
}
else if (loadedSourceItem.Status == ItemLoadStatus.Skipped) // if the item got skipped we'll prevent it from being deleted
{
orphanCandidates.Remove(serializedChild.Id);
}
}
catch (ConsistencyException)
{
throw;
}
catch (Exception ex)
{
// if a problem occurs we attempt to retry later
retryer.AddItemRetry(serializedChild, ex);
// don't treat errors as cause to delete an item
orphanCandidates.Remove(serializedChild.Id);
}
}
// if we're forcing an update (ie deleting stuff not on disk) we send the items that we found that weren't on disk off to get evaluated as orphans
if (orphanCandidates.Count > 0)
{
Evaluator.EvaluateOrphans(orphanCandidates.Values.ToArray());
}
}
/// <summary>
/// Loads a tree from serialized items on disk.
/// </summary>
protected internal virtual void LoadTree(IItemData rootItemData, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(rootItemData, "rootItem");
Assert.ArgumentNotNull(retryer, "retryer");
Assert.ArgumentNotNull(consistencyChecker, "consistencyChecker");
_itemsProcessed = 0;
var timer = new Stopwatch();
timer.Start();
Logger.BeginLoadingTree(rootItemData);
// LoadTreeInternal does not load the root item passed to it (only the children thereof)
// so we have to seed the load by loading the root item
using (new UnicornOperationContext())
{
try
{
DoLoadItem(rootItemData, consistencyChecker);
}
catch (Exception exception)
{
retryer.AddItemRetry(rootItemData, exception);
}
}
// load children of the root
LoadTreeInternal(rootItemData, retryer, consistencyChecker);
Logger.EndLoadingTree(rootItemData, _itemsProcessed, timer.ElapsedMilliseconds);
timer.Stop();
}
/// <summary>
/// Loads a set of children from a serialized path
/// </summary>
protected virtual void LoadOneLevel(ISerializedReference root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(root, "root");
Assert.ArgumentNotNull(retryer, "retryer");
var orphanCandidates = new Dictionary <ID, ISourceItem>();
// grab the root item's full metadata
var rootSerializedItem = root.GetItem();
if (rootSerializedItem == null)
{
Logger.SkippedItemMissingInSerializationProvider(root, SerializationProvider.GetType().Name);
return;
}
// get the corresponding item from Sitecore
ISourceItem rootItem = SourceDataProvider.GetItemById(rootSerializedItem.DatabaseName, rootSerializedItem.Id);
// we add all of the root item's direct children to the "maybe orphan" list (we'll remove them as we find matching serialized children)
if (rootItem != null)
{
var rootChildren = rootItem.Children;
foreach (ISourceItem child in rootChildren)
{
// if the preset includes the child add it to the orphan-candidate list (if we don't deserialize it below, it will be marked orphan)
var included = Predicate.Includes(child);
if (included.IsIncluded)
{
orphanCandidates[child.Id] = child;
}
else
{
Logger.SkippedItem(child, Predicate.GetType().Name, included.Justification ?? string.Empty);
}
}
}
// check for direct children of the target path
var children = rootSerializedItem.GetChildItems();
foreach (var child in children)
{
try
{
if (child.IsStandardValuesItem)
{
orphanCandidates.Remove(child.Id); // avoid marking standard values items orphans
retryer.AddItemRetry(child, new StandardValuesException(child.ItemPath));
}
else
{
// load a child item
var loadedItem = DoLoadItem(child, consistencyChecker);
if (loadedItem.Item != null)
{
orphanCandidates.Remove(loadedItem.Item.Id);
// check if we have any child serialized items under this loaded child item (existing children) -
// if we do not, we can orphan any included children of the loaded item as well
var loadedItemsChildren = child.GetChildReferences(false);
if (loadedItemsChildren.Length == 0) // no children were serialized on disk
{
var loadedChildren = loadedItem.Item.Children;
foreach (ISourceItem loadedChild in loadedChildren)
{
if (Predicate.Includes(loadedChild).IsIncluded)
{
orphanCandidates.Add(loadedChild.Id, loadedChild);
}
}
}
}
else if (loadedItem.Status == ItemLoadStatus.Skipped) // if the item got skipped we'll prevent it from being deleted
{
orphanCandidates.Remove(child.Id);
}
}
}
catch (ConsistencyException)
{
throw;
}
catch (Exception ex)
{
// if a problem occurs we attempt to retry later
retryer.AddItemRetry(child, ex);
// don't treat errors as cause to delete an item
orphanCandidates.Remove(child.Id);
}
}
// if we're forcing an update (ie deleting stuff not on disk) we send the items that we found that weren't on disk off to get evaluated as orphans
if (orphanCandidates.Count > 0)
{
bool disableNewSerialization = UnicornDataProvider.DisableSerialization;
try
{
UnicornDataProvider.DisableSerialization = true;
Evaluator.EvaluateOrphans(orphanCandidates.Values.ToArray());
}
finally
{
UnicornDataProvider.DisableSerialization = disableNewSerialization;
}
}
}
/// <summary>
/// Recursive method that loads a given tree and retries failures already present if any
/// </summary>
protected virtual void LoadTreeInternal(IItemData root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(root, "root");
Assert.ArgumentNotNull(retryer, "retryer");
var included = Predicate.Includes(root);
if (!included.IsIncluded)
{
if (!ReactorContext.IsActive)
{
// we skip this when Dilithium is active because it's entirely probable that another config, containing ignored children, may also be in the cache - so we cannot guarantee this log message being accurate.
Logger.SkippedItemPresentInSerializationProvider(root, Predicate.FriendlyName, TargetDataStore.GetType().Name, included.Justification ?? string.Empty);
}
return;
}
var processQueue = new Queue <IItemData>();
// put the root in the queue
processQueue.Enqueue(root);
using (new UnicornOperationContext()) // disablers only work on the current thread. So we need to disable on all worker threads
{
IItemData parentItem;
while (processQueue.Count > 0)
{
parentItem = processQueue.Dequeue();
try
{
// load the current level
LoadOneLevel(parentItem, retryer, consistencyChecker);
// check if we have child paths to process down
var children = TargetDataStore.GetChildren(parentItem).ToArray();
if (children.Length > 0)
{
// load each child path
foreach (var child in children)
{
processQueue.Enqueue(child);
}
} // children.length > 0
}
catch (ConsistencyException)
{
throw;
}
catch (Exception ex)
{
retryer.AddTreeRetry(root, ex);
}
} // end while
}
}
public virtual void LoadAll(IItemData[] rootItemsData, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker, Action<IItemData> rootLoadedCallback = null)
{
Assert.ArgumentNotNull(rootItemsData, "rootItems");
Assert.IsTrue(rootItemsData.Length > 0, "No root items were passed!");
// load the root item (LoadTreeInternal only evaluates children)
bool disableNewSerialization = UnicornDataProvider.DisableSerialization;
try
{
UnicornDataProvider.DisableSerialization = true;
using (new EventDisabler())
{
foreach (var rootItem in rootItemsData)
{
LoadTree(rootItem, retryer, consistencyChecker);
if (rootLoadedCallback != null) rootLoadedCallback(rootItem);
}
retryer.RetryAll(SourceDataStore, item => DoLoadItem(item, null), item => LoadTreeInternal(item, retryer, null));
}
}
finally
{
UnicornDataProvider.DisableSerialization = disableNewSerialization;
}
}
