public static IQueueSource <U> Select <T, U>(this IQueueSource <T> queue, Func <T, Task <U> > func, int?capacity = null) { AsyncQueue <U> outQueue = new AsyncQueue <U>(capacity ?? 2); Func <Task> worker = async delegate { while (true) { Option <T> item = await queue.Dequeue(CancellationToken.None); if (!item.HasValue) { break; } U item2 = await func(item.Value); await outQueue.Enqueue(item2, CancellationToken.None); } outQueue.WriteEof(); }; Task.Run(worker); return(outQueue); }
public static Func <Task> ParallelForEach <T>(IQueueSource <T> source, ParallelWorker parallelWorker, Func <ForEachInfo <T>, Task> processAsync, Func <Task> onCloseAsync, ExceptionCollector ec) { Func <Task> t = async delegate() { IdleDetector idleDetector = new IdleDetector(); try { while (true) { var item = await source.Dequeue(ec.CancellationToken); if (!item.HasValue) { break; } T itemValue = item.Value; idleDetector.Enter(); await parallelWorker.StartWorkItem ( async workerId => { try { await processAsync(new ForEachInfo <T>(itemValue, 0, workerId, ec.CancellationToken)); } catch (Exception exc) { ec.Add(exc); } finally { idleDetector.Leave(); } }, ec.CancellationToken ); } } finally { await idleDetector.WaitForIdle(CancellationToken.None); if (onCloseAsync != null) { try { await onCloseAsync(); } catch (Exception exc) { ec.Add(exc); } } } }; return(t); }
public void FindResourcesByType_Execute_ExpectIQueueSource() { var expected = new IQueueSource[] { new RabbitMQSource() }; var mockResourceCatalog = new Mock <IResourceCatalog>(); mockResourceCatalog.Setup(o => o.FindByType(typeof(IQueueSource).FullName)).Returns(expected); //------------Setup for test------------------------- var service = new FindResourcesByType(new Lazy <IResourceCatalog>(() => mockResourceCatalog.Object)); IWorkspace workspace = null; var values = new Dictionary <string, StringBuilder>(); values.Add("Type", new StringBuilder(typeof(IQueueSource).FullName)); //------------Execute Test--------------------------- var result = service.Execute(values, workspace); //------------Assert Results------------------------- var serializer = new Dev2JsonSerializer(); var expectedString = serializer.Serialize(expected, Newtonsoft.Json.Formatting.None); Assert.AreEqual(expectedString, result.ToString()); }
public static IQueueSource <T> OrderedParallelWhere <T>(this IQueueSource <T> queue, ParallelWorker parallelWorker, Func <T, Task <bool> > predicate, int?capacity = null) { Func <Tuple <int, T>, Task <Tuple <int, T, bool> > > func2 = async pair => new Tuple <int, T, bool>(pair.Item1, pair.Item2, await predicate(pair.Item2)); return(queue.Indexed().ParallelSelect(parallelWorker, func2, capacity).Reorder(pair => pair.Item1, 0, capacity).Where(i => Task.FromResult(i.Item3), capacity) .SynchronousSelect(i => i.Item2)); }
public static IQueueSource <T> Where <T>(this IQueueSource <T> queue, Func <T, Task <bool> > predicate, int?capacity = null) { AsyncQueue <T> outQueue = new AsyncQueue <T>(capacity ?? 2); Func <Task> worker = async delegate { while (true) { Option <T> item = await queue.Dequeue(CancellationToken.None); if (!item.HasValue) { break; } if (await predicate(item.Value)) { await outQueue.Enqueue(item.Value, CancellationToken.None); } } outQueue.WriteEof(); }; Task _dummy = Task.Run(worker); return(outQueue); }
public GetCancellableResult(IQueueSource <U> queue, U value, Func <U, V> convertResult) { this.queue = queue; this.value = value; this.convertResult = convertResult; this.done = false; }
public static CancellableOperationStarter <V> StartableGet <U, V>(this IQueueSource <U> queue, Func <U, V> convertResult, V eofResult) { return(delegate(CancellationToken ctoken) { CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(ctoken); Func <Task <CancellableResult <V> > > taskSource = async delegate() { AcquireReadResult arr = await queue.AcquireReadAsync(1, cts.Token); if (arr is AcquireReadSucceeded <U> ) { AcquireReadSucceeded <U> succeededResult = (AcquireReadSucceeded <U>)arr; if (succeededResult.ItemCount == 1) { return new GetCancellableResult <U, V>(queue, succeededResult.Items[0], convertResult); } else { System.Diagnostics.Debug.Assert(succeededResult.ItemCount == 0); return new GetEofCancellableResult <U, V>(queue, eofResult); } } else if (arr is AcquireReadFaulted) { AcquireReadFaulted faultedResult = (AcquireReadFaulted)arr; throw faultedResult.Exception; } else { AcquireReadCancelled cancelledResult = (AcquireReadCancelled)arr; throw new OperationCanceledException(); } }; Task <CancellableResult <V> > task = null; try { task = Task.Run(taskSource); } catch (OperationCanceledException e) { task = Task.FromCanceled <CancellableResult <V> >(e.CancellationToken); } catch (Exception exc) { task = Task.FromException <CancellableResult <V> >(exc); } return new CancellableOperation <V>(task, cts); }); }
public static IQueueSource <T> ParallelWhere <T>(this IQueueSource <T> queue, ParallelWorker parallelWorker, Func <T, Task <bool> > predicate, int?capacity = null) { AsyncQueue <T> outQueue = new AsyncQueue <T>(capacity ?? 2); IdleDetector idleDetector = new IdleDetector(); Func <Task> workPoster = async delegate() { while (true) { Option <T> item = await queue.Dequeue(CancellationToken.None); if (!item.HasValue) { break; } T itemValue = item.Value; idleDetector.Enter(); Task doingWork = await parallelWorker.StartWorkItem ( async (int workerId) => { try { if (await predicate(itemValue)) { await outQueue.Enqueue(itemValue, CancellationToken.None); } } finally { idleDetector.Leave(); } }, CancellationToken.None ); } await idleDetector.WaitForIdle(CancellationToken.None); outQueue.WriteEof(); }; Task.Run(workPoster); return(outQueue); }
public static async Task <Option <T> > Dequeue <T>(this IQueueSource <T> queue, CancellationToken ctoken) { AcquireReadResult result = await queue.AcquireReadAsync(1, ctoken); return(result.Visit <Option <T> > ( new Func <AcquireReadSucceeded, Option <T> > ( succeeded => { if (succeeded.ItemCount == 0) { return new None <T>(); } else { System.Diagnostics.Debug.Assert(succeeded.ItemCount == 1); bool wasCancelled = false; if (ctoken.IsCancellationRequested) { wasCancelled = true; queue.ReleaseRead(0); } else { queue.ReleaseRead(1); } if (wasCancelled) { throw new OperationCanceledException(ctoken); } return new Some <T>(((AcquireReadSucceeded <T>)succeeded).Items[0]); } } ), new Func <AcquireReadCancelled, Option <T> > ( cancelled => { throw new OperationCanceledException(ctoken); } ), new Func <AcquireReadFaulted, Option <T> > ( faulted => { throw faulted.Exception; } ) )); }
public static IEnumerable <T> AsEnumerable <T>(this IQueueSource <T> queue) { BlockingCollection <T> bc = new BlockingCollection <T>(); Func <Task> feed = async delegate() { while (true) { Option <T> item = await queue.Dequeue(CancellationToken.None); if (item.HasValue) { bc.Add(item.Value); } else { break; } } bc.CompleteAdding(); }; Task _dummy = Task.Run(feed); while (true) { T item2 = default(T); bool success = false; try { item2 = bc.Take(); success = true; } catch (InvalidOperationException exc) { // BlockingCollection ran out of items System.Diagnostics.Debug.WriteLine(exc); } if (!success) { break; } yield return(item2); } }
public static Func <Task> ForEach <T>(IQueueSource <T> source, Func <ForEachInfo <T>, Task> processAsync, Func <Task> onCloseAsync, ExceptionCollector ec) { Func <Task> t = async delegate() { try { while (true) { var item = await source.Dequeue(ec.CancellationToken); if (!item.HasValue) { break; } try { await processAsync(new ForEachInfo <T>(item.Value, 0, 0, ec.CancellationToken)); } catch (Exception exc) { ec.Add(exc); break; } } } finally { if (onCloseAsync != null) { try { await onCloseAsync(); } catch (Exception exc) { ec.Add(exc); } } } }; return(t); }
public static IQueueSource <T> Reorder <T>(this IQueueSource <T> queue, Func <T, int> getOrder, int first, int?capacity = null) { AsyncQueue <T> outQueue = new AsyncQueue <T>(capacity ?? 2); Func <Task> worker = async delegate { int next = first; ImmutableDictionary <int, T> buffer = ImmutableDictionary <int, T> .Empty; while (true) { Option <T> item = await queue.Dequeue(CancellationToken.None); if (!item.HasValue) { break; } int order = getOrder(item.Value); if (order == next) { await outQueue.Enqueue(item.Value, CancellationToken.None); ++next; while (buffer.ContainsKey(next)) { await outQueue.Enqueue(buffer[next], CancellationToken.None); buffer = buffer.Remove(next); ++next; } } else { buffer = buffer.Add(order, item.Value); } } outQueue.WriteEof(); }; Task.Run(worker); return(outQueue); }
public IndexedSource(IQueueSource <T> parent) { this.parent = parent; }
public GetEofCancellableResult(IQueueSource <U> queue, V eofResult) { this.queue = queue; this.eofResult = eofResult; this.done = false; }
public static IQueueSource <U> OrderedParallelSelect <T, U>(this IQueueSource <T> queue, ParallelWorker parallelWorker, Func <T, Task <U> > func, int?capacity = null) { Func <Tuple <int, T>, Task <Tuple <int, U> > > func2 = async pair => new Tuple <int, U>(pair.Item1, await func(pair.Item2)); return(queue.Indexed().ParallelSelect(parallelWorker, func2, capacity).Reorder(pair => pair.Item1, 0, capacity).SynchronousSelect(pair => pair.Item2)); }
public static IQueueSource <U> SynchronousSelect <T, U>(this IQueueSource <T> queue, Func <T, U> func) { return(new SynchronousSelectedSource <T, U>(queue, func)); }
public SynchronousSelectedSource(IQueueSource <T> parent, Func <T, U> func) { this.parent = parent; this.func = func; }
public static IQueueSource <Tuple <int, T> > Indexed <T>(this IQueueSource <T> queue) { return(new IndexedSource <T>(queue)); }