/// <summary>
/// Recursive method that loads a given tree and retries failures already present if any
/// </summary>
protected virtual void LoadTreeInternal(IItemData root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(root, "root");
Assert.ArgumentNotNull(retryer, "retryer");
var included = Predicate.Includes(root);
if (!included.IsIncluded)
{
if (!ReactorContext.IsActive)
{
// we skip this when Dilithium is active because it's entirely probable that another config, containing ignored children, may also be in the cache - so we cannot guarantee this log message being accurate.
Logger.SkippedItemPresentInSerializationProvider(root, Predicate.FriendlyName, TargetDataStore.GetType().Name, included.Justification ?? string.Empty);
}
return;
}
var processQueue = new Queue <IItemData>();
// put the root in the queue
processQueue.Enqueue(root);
using (new UnicornOperationContext()) // disablers only work on the current thread. So we need to disable on all worker threads
{
IItemData parentItem;
while (processQueue.Count > 0)
{
parentItem = processQueue.Dequeue();
try
{
// load the current level
LoadOneLevel(parentItem, retryer, consistencyChecker);
// check if we have child paths to process down
var children = TargetDataStore.GetChildren(parentItem).ToArray();
if (children.Length > 0)
{
// load each child path
foreach (var child in children)
{
processQueue.Enqueue(child);
}
} // children.length > 0
}
catch (ConsistencyException)
{
throw;
}
catch (Exception ex)
{
retryer.AddTreeRetry(root, ex);
}
} // end while
}
}
/// <summary>
/// Recursive method that loads a given tree and retries failures already present if any
/// </summary>
protected virtual void LoadTreeRecursive(IItemData root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(root, "root");
Assert.ArgumentNotNull(retryer, "retryer");
var included = Predicate.Includes(root);
if (!included.IsIncluded)
{
Logger.SkippedItemPresentInSerializationProvider(root, Predicate.FriendlyName, TargetDataStore.GetType().Name, included.Justification ?? string.Empty);
return;
}
try
{
// load the current level
LoadOneLevel(root, retryer, consistencyChecker);
// check if we have child paths to recurse down
var children = TargetDataStore.GetChildren(root).ToArray();
if (children.Length > 0)
{
// make sure if a "templates" item exists in the current set, it goes first
if (children.Length > 1)
{
int templateIndex = Array.FindIndex(children, x => x.Path.EndsWith("templates", StringComparison.OrdinalIgnoreCase));
if (templateIndex > 0)
{
var zero = children[0];
children[0] = children[templateIndex];
children[templateIndex] = zero;
}
}
// load each child path recursively
foreach (var child in children)
{
LoadTreeRecursive(child, retryer, consistencyChecker);
}
// pull out any standard values failures for immediate retrying
retryer.RetryStandardValuesFailures(item => DoLoadItem(item, null));
} // children.length > 0
}
catch (ConsistencyException)
{
throw;
}
catch (Exception ex)
{
retryer.AddTreeRetry(root, ex);
}
}
/// <summary>
/// Recursive method that loads a given tree and retries failures already present if any
/// </summary>
protected virtual void LoadTreeRecursive(IItemData root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(root, "root");
Assert.ArgumentNotNull(retryer, "retryer");
var included = Predicate.Includes(root);
if (!included.IsIncluded)
{
Logger.SkippedItemPresentInSerializationProvider(root, Predicate.FriendlyName, TargetDataStore.GetType().Name, included.Justification ?? string.Empty);
return;
}
try
{
// load the current level
LoadOneLevel(root, retryer, consistencyChecker);
// check if we have child paths to recurse down
var children = TargetDataStore.GetChildren(root).ToArray();
if (children.Length > 0)
{
// make sure if a "templates" item exists in the current set, it goes first
if (children.Length > 1)
{
int templateIndex = Array.FindIndex(children, x => x.Path.EndsWith("templates", StringComparison.OrdinalIgnoreCase));
if (templateIndex > 0)
{
var zero = children[0];
children[0] = children[templateIndex];
children[templateIndex] = zero;
}
}
// load each child path recursively
foreach (var child in children)
{
LoadTreeRecursive(child, retryer, consistencyChecker);
}
// pull out any standard values failures for immediate retrying
retryer.RetryStandardValuesFailures(item => DoLoadItem(item, null));
} // children.length > 0
}
catch (ConsistencyException)
{
throw;
}
catch (Exception ex)
{
retryer.AddTreeRetry(root, ex);
}
}
/// <summary>
/// Recursive method that loads a given tree and retries failures already present if any
/// </summary>
protected virtual void LoadTreeInternal(IItemData root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(root, "root");
Assert.ArgumentNotNull(retryer, "retryer");
var included = Predicate.Includes(root);
if (!included.IsIncluded)
{
Logger.SkippedItemPresentInSerializationProvider(root, Predicate.FriendlyName, TargetDataStore.GetType().Name, included.Justification ?? string.Empty);
return;
}
// we throw items into this queue, and let a thread pool pick up anything available to process in parallel. only the children of queued items are processed, not the item itself
ConcurrentQueue<IItemData> processQueue = new ConcurrentQueue<IItemData>();
// exceptions thrown on background threads are left in here
ConcurrentQueue<Exception> errors = new ConcurrentQueue<Exception>();
// we keep track of how many threads are actively processing something so we know when to end the threads
// (e.g. a thread could have nothing in the queue right now, but that's because a different thread is about
// to add 8 things to the queue - so it shouldn't quit till all is done)
int activeThreads = 0;
// put the root in the queue
processQueue.Enqueue(root);
if(SyncConfiguration.MaxConcurrency < 1) throw new InvalidOperationException("Max concurrency is set to zero. Please set it to one or more threads.");
Thread[] pool = Enumerable.Range(0, SyncConfiguration.MaxConcurrency).Select(i => new Thread(() =>
{
Process:
Interlocked.Increment(ref activeThreads);
using (new UnicornOperationContext()) // disablers only work on the current thread. So we need to disable on all worker threads
{
IItemData parentItem;
while (processQueue.TryDequeue(out parentItem) && errors.Count == 0)
{
try
{
// load the current level
LoadOneLevel(parentItem, retryer, consistencyChecker);
// check if we have child paths to process down
var children = TargetDataStore.GetChildren(parentItem).ToArray();
if (children.Length > 0)
{
// load each child path
foreach (var child in children)
{
processQueue.Enqueue(child);
}
} // children.length > 0
}
catch (ConsistencyException cex)
{
errors.Enqueue(cex);
break;
}
catch (Exception ex)
{
retryer.AddTreeRetry(root, ex);
}
} // end while
}
// if we get here, the queue was empty. let's make ourselves inactive.
Interlocked.Decrement(ref activeThreads);
// if some other thread in our pool was doing stuff, sleep for a sec to see if we can pick up their work
if (activeThreads > 0)
{
Thread.Sleep(10);
goto Process; // OH MY GOD :)
}
})).ToArray();
// start the thread pool
foreach (var thread in pool) thread.Start();
// ...and then wait for all the threads to finish
foreach (var thread in pool) thread.Join();
if (errors.Count > 0) throw new AggregateException(errors);
}
/// <summary>
/// Recursive method that loads a given tree and retries failures already present if any
/// </summary>
protected virtual void LoadTreeInternal(IItemData root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(root, "root");
Assert.ArgumentNotNull(retryer, "retryer");
var included = Predicate.Includes(root);
if (!included.IsIncluded)
{
Logger.SkippedItemPresentInSerializationProvider(root, Predicate.FriendlyName, TargetDataStore.GetType().Name, included.Justification ?? string.Empty);
return;
}
// we throw items into this queue, and let a thread pool pick up anything available to process in parallel. only the children of queued items are processed, not the item itself
ConcurrentQueue<IItemData> processQueue = new ConcurrentQueue<IItemData>();
// we keep track of how many threads are actively processing something so we know when to end the threads
// (e.g. a thread could have nothing in the queue right now, but that's because a different thread is about
// to add 8 things to the queue - so it shouldn't quit till all is done)
int activeThreads = 0;
// put the root in the queue
processQueue.Enqueue(root);
Thread[] pool = Enumerable.Range(0, ThreadCount).Select(i => new Thread(() =>
{
Process:
Interlocked.Increment(ref activeThreads);
IItemData parentItem;
while (processQueue.TryDequeue(out parentItem))
{
try
{
// load the current level
LoadOneLevel(parentItem, retryer, consistencyChecker);
// check if we have child paths to process down
var children = TargetDataStore.GetChildren(parentItem).ToArray();
if (children.Length > 0)
{
// make sure if a "templates" item exists in the current set, it goes first
if (children.Length > 1)
{
int templateIndex = Array.FindIndex(children, x => x.Path.EndsWith("templates", StringComparison.OrdinalIgnoreCase));
if (templateIndex > 0)
{
var zero = children[0];
children[0] = children[templateIndex];
children[templateIndex] = zero;
}
}
// load each child path
foreach (var child in children)
{
processQueue.Enqueue(child);
}
// pull out any standard values failures for immediate retrying
retryer.RetryStandardValuesFailures(item => DoLoadItem(item, null));
} // children.length > 0
}
catch (ConsistencyException)
{
throw;
}
catch (Exception ex)
{
retryer.AddTreeRetry(root, ex);
}
}
// if we get here, the queue was empty. let's make ourselves inactive.
Interlocked.Decrement(ref activeThreads);
// if some other thread in our pool was doing stuff, sleep for a sec to see if we can pick up their work
if (activeThreads > 0)
{
Thread.Sleep(10);
goto Process; // OH MY GOD :)
}
})).ToArray();
// start the thread pool
foreach (var thread in pool) thread.Start();
// ...and then wait for all the threads to finish
foreach (var thread in pool) thread.Join();
}
/// <summary>
/// Recursive method that loads a given tree and retries failures already present if any
/// </summary>
protected virtual void LoadTreeInternal(IItemData root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker)
{
Assert.ArgumentNotNull(root, "root");
Assert.ArgumentNotNull(retryer, "retryer");
var included = Predicate.Includes(root);
if (!included.IsIncluded)
{
Logger.SkippedItemPresentInSerializationProvider(root, Predicate.FriendlyName, TargetDataStore.GetType().Name, included.Justification ?? string.Empty);
return;
}
// we throw items into this queue, and let a thread pool pick up anything available to process in parallel. only the children of queued items are processed, not the item itself
ConcurrentQueue <IItemData> processQueue = new ConcurrentQueue <IItemData>();
// exceptions thrown on background threads are left in here
ConcurrentQueue <Exception> errors = new ConcurrentQueue <Exception>();
// we keep track of how many threads are actively processing something so we know when to end the threads
// (e.g. a thread could have nothing in the queue right now, but that's because a different thread is about
// to add 8 things to the queue - so it shouldn't quit till all is done)
int activeThreads = 0;
// put the root in the queue
processQueue.Enqueue(root);
Thread[] pool = Enumerable.Range(0, ThreadCount).Select(i => new Thread(() =>
{
Process:
Interlocked.Increment(ref activeThreads);
using (new UnicornOperationContext()) // disablers only work on the current thread. So we need to disable on all worker threads
{
IItemData parentItem;
while (processQueue.TryDequeue(out parentItem) && errors.Count == 0)
{
try
{
// load the current level
LoadOneLevel(parentItem, retryer, consistencyChecker);
// check if we have child paths to process down
var children = TargetDataStore.GetChildren(parentItem).ToArray();
if (children.Length > 0)
{
// load each child path
foreach (var child in children)
{
processQueue.Enqueue(child);
}
} // children.length > 0
}
catch (ConsistencyException cex)
{
errors.Enqueue(cex);
break;
}
catch (Exception ex)
{
retryer.AddTreeRetry(root, ex);
}
} // end while
}
// if we get here, the queue was empty. let's make ourselves inactive.
Interlocked.Decrement(ref activeThreads);
// if some other thread in our pool was doing stuff, sleep for a sec to see if we can pick up their work
if (activeThreads > 0)
{
Thread.Sleep(10);
goto Process; // OH MY GOD :)
}
})).ToArray();
// start the thread pool
foreach (var thread in pool)
{
thread.Start();
}
// ...and then wait for all the threads to finish
foreach (var thread in pool)
{
thread.Join();
}
if (errors.Count > 0)
{
throw new AggregateException(errors);
}
}