/// <summary> /// Recursive method that loads a given tree and retries failures already present if any /// </summary> protected virtual void LoadTreeInternal(IItemData root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker) { Assert.ArgumentNotNull(root, "root"); Assert.ArgumentNotNull(retryer, "retryer"); var included = Predicate.Includes(root); if (!included.IsIncluded) { if (!ReactorContext.IsActive) { // we skip this when Dilithium is active because it's entirely probable that another config, containing ignored children, may also be in the cache - so we cannot guarantee this log message being accurate. Logger.SkippedItemPresentInSerializationProvider(root, Predicate.FriendlyName, TargetDataStore.GetType().Name, included.Justification ?? string.Empty); } return; } var processQueue = new Queue <IItemData>(); // put the root in the queue processQueue.Enqueue(root); using (new UnicornOperationContext()) // disablers only work on the current thread. So we need to disable on all worker threads { IItemData parentItem; while (processQueue.Count > 0) { parentItem = processQueue.Dequeue(); try { // load the current level LoadOneLevel(parentItem, retryer, consistencyChecker); // check if we have child paths to process down var children = TargetDataStore.GetChildren(parentItem).ToArray(); if (children.Length > 0) { // load each child path foreach (var child in children) { processQueue.Enqueue(child); } } // children.length > 0 } catch (ConsistencyException) { throw; } catch (Exception ex) { retryer.AddTreeRetry(root, ex); } } // end while } }
/// <summary> /// Recursive method that loads a given tree and retries failures already present if any /// </summary> protected virtual void LoadTreeRecursive(IItemData root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker) { Assert.ArgumentNotNull(root, "root"); Assert.ArgumentNotNull(retryer, "retryer"); var included = Predicate.Includes(root); if (!included.IsIncluded) { Logger.SkippedItemPresentInSerializationProvider(root, Predicate.FriendlyName, TargetDataStore.GetType().Name, included.Justification ?? string.Empty); return; } try { // load the current level LoadOneLevel(root, retryer, consistencyChecker); // check if we have child paths to recurse down var children = TargetDataStore.GetChildren(root).ToArray(); if (children.Length > 0) { // make sure if a "templates" item exists in the current set, it goes first if (children.Length > 1) { int templateIndex = Array.FindIndex(children, x => x.Path.EndsWith("templates", StringComparison.OrdinalIgnoreCase)); if (templateIndex > 0) { var zero = children[0]; children[0] = children[templateIndex]; children[templateIndex] = zero; } } // load each child path recursively foreach (var child in children) { LoadTreeRecursive(child, retryer, consistencyChecker); } // pull out any standard values failures for immediate retrying retryer.RetryStandardValuesFailures(item => DoLoadItem(item, null)); } // children.length > 0 } catch (ConsistencyException) { throw; } catch (Exception ex) { retryer.AddTreeRetry(root, ex); } }
/// <summary> /// Recursive method that loads a given tree and retries failures already present if any /// </summary> protected virtual void LoadTreeRecursive(IItemData root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker) { Assert.ArgumentNotNull(root, "root"); Assert.ArgumentNotNull(retryer, "retryer"); var included = Predicate.Includes(root); if (!included.IsIncluded) { Logger.SkippedItemPresentInSerializationProvider(root, Predicate.FriendlyName, TargetDataStore.GetType().Name, included.Justification ?? string.Empty); return; } try { // load the current level LoadOneLevel(root, retryer, consistencyChecker); // check if we have child paths to recurse down var children = TargetDataStore.GetChildren(root).ToArray(); if (children.Length > 0) { // make sure if a "templates" item exists in the current set, it goes first if (children.Length > 1) { int templateIndex = Array.FindIndex(children, x => x.Path.EndsWith("templates", StringComparison.OrdinalIgnoreCase)); if (templateIndex > 0) { var zero = children[0]; children[0] = children[templateIndex]; children[templateIndex] = zero; } } // load each child path recursively foreach (var child in children) { LoadTreeRecursive(child, retryer, consistencyChecker); } // pull out any standard values failures for immediate retrying retryer.RetryStandardValuesFailures(item => DoLoadItem(item, null)); } // children.length > 0 } catch (ConsistencyException) { throw; } catch (Exception ex) { retryer.AddTreeRetry(root, ex); } }
/// <summary> /// Recursive method that loads a given tree and retries failures already present if any /// </summary> protected virtual void LoadTreeInternal(IItemData root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker) { Assert.ArgumentNotNull(root, "root"); Assert.ArgumentNotNull(retryer, "retryer"); var included = Predicate.Includes(root); if (!included.IsIncluded) { Logger.SkippedItemPresentInSerializationProvider(root, Predicate.FriendlyName, TargetDataStore.GetType().Name, included.Justification ?? string.Empty); return; } // we throw items into this queue, and let a thread pool pick up anything available to process in parallel. only the children of queued items are processed, not the item itself ConcurrentQueue<IItemData> processQueue = new ConcurrentQueue<IItemData>(); // exceptions thrown on background threads are left in here ConcurrentQueue<Exception> errors = new ConcurrentQueue<Exception>(); // we keep track of how many threads are actively processing something so we know when to end the threads // (e.g. a thread could have nothing in the queue right now, but that's because a different thread is about // to add 8 things to the queue - so it shouldn't quit till all is done) int activeThreads = 0; // put the root in the queue processQueue.Enqueue(root); if(SyncConfiguration.MaxConcurrency < 1) throw new InvalidOperationException("Max concurrency is set to zero. Please set it to one or more threads."); Thread[] pool = Enumerable.Range(0, SyncConfiguration.MaxConcurrency).Select(i => new Thread(() => { Process: Interlocked.Increment(ref activeThreads); using (new UnicornOperationContext()) // disablers only work on the current thread. So we need to disable on all worker threads { IItemData parentItem; while (processQueue.TryDequeue(out parentItem) && errors.Count == 0) { try { // load the current level LoadOneLevel(parentItem, retryer, consistencyChecker); // check if we have child paths to process down var children = TargetDataStore.GetChildren(parentItem).ToArray(); if (children.Length > 0) { // load each child path foreach (var child in children) { processQueue.Enqueue(child); } } // children.length > 0 } catch (ConsistencyException cex) { errors.Enqueue(cex); break; } catch (Exception ex) { retryer.AddTreeRetry(root, ex); } } // end while } // if we get here, the queue was empty. let's make ourselves inactive. Interlocked.Decrement(ref activeThreads); // if some other thread in our pool was doing stuff, sleep for a sec to see if we can pick up their work if (activeThreads > 0) { Thread.Sleep(10); goto Process; // OH MY GOD :) } })).ToArray(); // start the thread pool foreach (var thread in pool) thread.Start(); // ...and then wait for all the threads to finish foreach (var thread in pool) thread.Join(); if (errors.Count > 0) throw new AggregateException(errors); }
/// <summary> /// Recursive method that loads a given tree and retries failures already present if any /// </summary> protected virtual void LoadTreeInternal(IItemData root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker) { Assert.ArgumentNotNull(root, "root"); Assert.ArgumentNotNull(retryer, "retryer"); var included = Predicate.Includes(root); if (!included.IsIncluded) { Logger.SkippedItemPresentInSerializationProvider(root, Predicate.FriendlyName, TargetDataStore.GetType().Name, included.Justification ?? string.Empty); return; } // we throw items into this queue, and let a thread pool pick up anything available to process in parallel. only the children of queued items are processed, not the item itself ConcurrentQueue<IItemData> processQueue = new ConcurrentQueue<IItemData>(); // we keep track of how many threads are actively processing something so we know when to end the threads // (e.g. a thread could have nothing in the queue right now, but that's because a different thread is about // to add 8 things to the queue - so it shouldn't quit till all is done) int activeThreads = 0; // put the root in the queue processQueue.Enqueue(root); Thread[] pool = Enumerable.Range(0, ThreadCount).Select(i => new Thread(() => { Process: Interlocked.Increment(ref activeThreads); IItemData parentItem; while (processQueue.TryDequeue(out parentItem)) { try { // load the current level LoadOneLevel(parentItem, retryer, consistencyChecker); // check if we have child paths to process down var children = TargetDataStore.GetChildren(parentItem).ToArray(); if (children.Length > 0) { // make sure if a "templates" item exists in the current set, it goes first if (children.Length > 1) { int templateIndex = Array.FindIndex(children, x => x.Path.EndsWith("templates", StringComparison.OrdinalIgnoreCase)); if (templateIndex > 0) { var zero = children[0]; children[0] = children[templateIndex]; children[templateIndex] = zero; } } // load each child path foreach (var child in children) { processQueue.Enqueue(child); } // pull out any standard values failures for immediate retrying retryer.RetryStandardValuesFailures(item => DoLoadItem(item, null)); } // children.length > 0 } catch (ConsistencyException) { throw; } catch (Exception ex) { retryer.AddTreeRetry(root, ex); } } // if we get here, the queue was empty. let's make ourselves inactive. Interlocked.Decrement(ref activeThreads); // if some other thread in our pool was doing stuff, sleep for a sec to see if we can pick up their work if (activeThreads > 0) { Thread.Sleep(10); goto Process; // OH MY GOD :) } })).ToArray(); // start the thread pool foreach (var thread in pool) thread.Start(); // ...and then wait for all the threads to finish foreach (var thread in pool) thread.Join(); }
/// <summary> /// Recursive method that loads a given tree and retries failures already present if any /// </summary> protected virtual void LoadTreeInternal(IItemData root, IDeserializeFailureRetryer retryer, IConsistencyChecker consistencyChecker) { Assert.ArgumentNotNull(root, "root"); Assert.ArgumentNotNull(retryer, "retryer"); var included = Predicate.Includes(root); if (!included.IsIncluded) { Logger.SkippedItemPresentInSerializationProvider(root, Predicate.FriendlyName, TargetDataStore.GetType().Name, included.Justification ?? string.Empty); return; } // we throw items into this queue, and let a thread pool pick up anything available to process in parallel. only the children of queued items are processed, not the item itself ConcurrentQueue <IItemData> processQueue = new ConcurrentQueue <IItemData>(); // exceptions thrown on background threads are left in here ConcurrentQueue <Exception> errors = new ConcurrentQueue <Exception>(); // we keep track of how many threads are actively processing something so we know when to end the threads // (e.g. a thread could have nothing in the queue right now, but that's because a different thread is about // to add 8 things to the queue - so it shouldn't quit till all is done) int activeThreads = 0; // put the root in the queue processQueue.Enqueue(root); Thread[] pool = Enumerable.Range(0, ThreadCount).Select(i => new Thread(() => { Process: Interlocked.Increment(ref activeThreads); using (new UnicornOperationContext()) // disablers only work on the current thread. So we need to disable on all worker threads { IItemData parentItem; while (processQueue.TryDequeue(out parentItem) && errors.Count == 0) { try { // load the current level LoadOneLevel(parentItem, retryer, consistencyChecker); // check if we have child paths to process down var children = TargetDataStore.GetChildren(parentItem).ToArray(); if (children.Length > 0) { // load each child path foreach (var child in children) { processQueue.Enqueue(child); } } // children.length > 0 } catch (ConsistencyException cex) { errors.Enqueue(cex); break; } catch (Exception ex) { retryer.AddTreeRetry(root, ex); } } // end while } // if we get here, the queue was empty. let's make ourselves inactive. Interlocked.Decrement(ref activeThreads); // if some other thread in our pool was doing stuff, sleep for a sec to see if we can pick up their work if (activeThreads > 0) { Thread.Sleep(10); goto Process; // OH MY GOD :) } })).ToArray(); // start the thread pool foreach (var thread in pool) { thread.Start(); } // ...and then wait for all the threads to finish foreach (var thread in pool) { thread.Join(); } if (errors.Count > 0) { throw new AggregateException(errors); } }